diff options
| -rw-r--r-- | dex2oat/linker/image_writer.cc | 29 | ||||
| -rw-r--r-- | dex2oat/linker/image_writer.h | 3 | ||||
| -rw-r--r-- | libdexfile/dex/modifiers.h | 2 | ||||
| -rw-r--r-- | runtime/art_method.h | 9 | ||||
| -rw-r--r-- | runtime/class_linker.cc | 28 | ||||
| -rw-r--r-- | runtime/interpreter/mterp/riscv64/invoke.S | 33 | ||||
| -rw-r--r-- | runtime/nterp_helpers-inl.h | 57 | ||||
| -rw-r--r-- | runtime/nterp_helpers.h | 5 |
8 files changed, 124 insertions, 42 deletions
diff --git a/dex2oat/linker/image_writer.cc b/dex2oat/linker/image_writer.cc index c4954d3d29..6564fb8136 100644 --- a/dex2oat/linker/image_writer.cc +++ b/dex2oat/linker/image_writer.cc @@ -76,6 +76,7 @@ #include "mirror/object_array-inl.h" #include "mirror/string-inl.h" #include "mirror/var_handle.h" +#include "nterp_helpers-inl.h" #include "nterp_helpers.h" #include "oat.h" #include "oat_file.h" @@ -3468,6 +3469,7 @@ void ImageWriter::CopyAndFixupMethod(ArtMethod* orig, CopyAndFixupReference(copy->GetDeclaringClassAddressWithoutBarrier(), orig->GetDeclaringClassUnchecked<kWithoutReadBarrier>()); + ResetNterpFastPathFlags(copy, orig); // OatWriter replaces the code_ with an offset value. Here we re-adjust to a pointer relative to // oat_begin_ @@ -3761,5 +3763,32 @@ void ImageWriter::CopyAndFixupPointer(void* object, MemberOffset offset, ValueTy return CopyAndFixupPointer(object, offset, src_value, target_ptr_size_); } +void ImageWriter::ResetNterpFastPathFlags(ArtMethod* copy, ArtMethod* orig) { + DCHECK(copy != nullptr); + DCHECK(orig != nullptr); + if (orig->IsRuntimeMethod() || orig->IsProxyMethod()) { + return; // !IsRuntimeMethod() and !IsProxyMethod() for GetShortyView() + } + + // Clear old nterp fast path flags. + if (copy->HasNterpEntryPointFastPathFlag()) { + copy->ClearNterpEntryPointFastPathFlag(); // Flag has other uses, clear it conditionally. + } + copy->ClearNterpInvokeFastPathFlag(); + + // Check if nterp fast paths available on target ISA. + std::string_view shorty = orig->GetShortyView(); // Use orig, copy's class not yet ready. + uint32_t new_nterp_flags = + GetNterpFastPathFlags(shorty, copy->GetAccessFlags(), compiler_options_.GetInstructionSet()); + + // Set new nterp fast path flags, if approporiate. + if ((new_nterp_flags & kAccNterpEntryPointFastPathFlag) != 0) { + copy->SetNterpEntryPointFastPathFlag(); + } + if ((new_nterp_flags & kAccNterpInvokeFastPathFlag) != 0) { + copy->SetNterpInvokeFastPathFlag(); + } +} + } // namespace linker } // namespace art diff --git a/dex2oat/linker/image_writer.h b/dex2oat/linker/image_writer.h index cdaea8718a..86c0972d37 100644 --- a/dex2oat/linker/image_writer.h +++ b/dex2oat/linker/image_writer.h @@ -596,6 +596,9 @@ class ImageWriter final { void CopyAndFixupPointer(void* object, MemberOffset offset, ValueType src_value) REQUIRES_SHARED(Locks::mutator_lock_); + void ResetNterpFastPathFlags(ArtMethod* copy, ArtMethod* orig) + REQUIRES_SHARED(Locks::mutator_lock_); + ALWAYS_INLINE static bool IsStronglyInternedString(ObjPtr<mirror::String> str) REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/libdexfile/dex/modifiers.h b/libdexfile/dex/modifiers.h index a17545c52b..1307e0eae5 100644 --- a/libdexfile/dex/modifiers.h +++ b/libdexfile/dex/modifiers.h @@ -101,7 +101,7 @@ static constexpr uint32_t kAccSingleImplementation = 0x08000000; // method (ru // Whether nterp can take a fast path when entering this method (runtime; non-native) static constexpr uint32_t kAccNterpEntryPointFastPathFlag = 0x00100000; // Set by the class linker to mark that a method does not have floating points -// or longs in its shorty. +// or longs in its shorty. On RISC-V 64, a method that has only reference args. static constexpr uint32_t kAccNterpInvokeFastPathFlag = 0x00200000; // method (runtime) static constexpr uint32_t kAccPublicApi = 0x10000000; // field, method diff --git a/runtime/art_method.h b/runtime/art_method.h index d3108da1e5..e5384235fb 100644 --- a/runtime/art_method.h +++ b/runtime/art_method.h @@ -584,10 +584,19 @@ class ArtMethod final { AddAccessFlags(kAccNterpEntryPointFastPathFlag); } + void ClearNterpEntryPointFastPathFlag() REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK(!IsNative()); + ClearAccessFlags(kAccNterpEntryPointFastPathFlag); + } + void SetNterpInvokeFastPathFlag() REQUIRES_SHARED(Locks::mutator_lock_) { AddAccessFlags(kAccNterpInvokeFastPathFlag); } + void ClearNterpInvokeFastPathFlag() REQUIRES_SHARED(Locks::mutator_lock_) { + ClearAccessFlags(kAccNterpInvokeFastPathFlag); + } + // Returns whether the method is a string constructor. The method must not // be a class initializer. (Class initializers are called from a different // context where we do not need to check for string constructors.) diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index 003acaea34..838d6c1a80 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -39,7 +39,6 @@ #include "barrier.h" #include "base/arena_allocator.h" #include "base/arena_bit_vector.h" -#include "base/membarrier.h" #include "base/casts.h" #include "base/file_utils.h" #include "base/hash_map.h" @@ -47,6 +46,7 @@ #include "base/leb128.h" #include "base/logging.h" #include "base/mem_map_arena_pool.h" +#include "base/membarrier.h" #include "base/metrics/metrics.h" #include "base/mutex-inl.h" #include "base/os.h" @@ -135,6 +135,7 @@ #include "mirror/var_handle.h" #include "native/dalvik_system_DexFile.h" #include "nativehelper/scoped_local_ref.h" +#include "nterp_helpers-inl.h" #include "nterp_helpers.h" #include "oat.h" #include "oat_file-inl.h" @@ -3997,26 +3998,7 @@ void ClassLinker::LoadMethod(const DexFile& dex_file, } } - // Check for nterp invoke fast-path based on shorty. - bool all_parameters_are_reference = true; - bool all_parameters_are_reference_or_int = true; - for (size_t i = 1; i < shorty.length(); ++i) { - if (shorty[i] != 'L') { - all_parameters_are_reference = false; - if (shorty[i] == 'F' || shorty[i] == 'D' || shorty[i] == 'J') { - all_parameters_are_reference_or_int = false; - break; - } - } - } - if (all_parameters_are_reference_or_int && shorty[0] != 'F' && shorty[0] != 'D') { - // FIXME(riscv64): This optimization is currently disabled because riscv64 needs - // to distinguish between zero-extended references and sign-extended integers. - // We should enable this for references only and fix corresponding nterp fast-paths. - if (kRuntimeISA != InstructionSet::kRiscv64) { - access_flags |= kAccNterpInvokeFastPathFlag; - } - } + access_flags |= GetNterpFastPathFlags(shorty, access_flags, kRuntimeISA); if (UNLIKELY((access_flags & kAccNative) != 0u)) { // Check if the native method is annotated with @FastNative or @CriticalNative. @@ -4039,10 +4021,6 @@ void ClassLinker::LoadMethod(const DexFile& dex_file, DCHECK_EQ(method.GetCodeItemOffset(), 0u); dst->SetDataPtrSize(nullptr, image_pointer_size_); // Single implementation not set yet. } else { - // Check for nterp entry fast-path based on shorty. - if (all_parameters_are_reference) { - access_flags |= kAccNterpEntryPointFastPathFlag; - } const dex::ClassDef& class_def = dex_file.GetClassDef(klass->GetDexClassDefIndex()); if (annotations::MethodIsNeverCompile(dex_file, class_def, dex_method_idx)) { access_flags |= kAccCompileDontBother; diff --git a/runtime/interpreter/mterp/riscv64/invoke.S b/runtime/interpreter/mterp/riscv64/invoke.S index 569b750655..0bce65cc2f 100644 --- a/runtime/interpreter/mterp/riscv64/invoke.S +++ b/runtime/interpreter/mterp/riscv64/invoke.S @@ -15,9 +15,8 @@ // (0) If the next method's "quick code" is nterp, then set up a fresh nterp frame and perform a // vreg->vreg transfer. Jump to handler for the next method's first opcode. // - The following paths leave nterp. - -// (1) If the next method is guaranteed to avoid floats, doubles, and longs, then the managed ABI is -// very simple: just place all arguments in the native arg registers. We don't need to know the -// precise types or widths, just the order matters. Call the quick code. +// (1) If the next method is guaranteed to be only object refs, then the managed ABI is very simple: +// just place all arguments in the native arg registers using LWU. Call the quick code. // (2) If the next method has 0 or 1 argument, then the managed ABI is mildly overloaded by // pessimistically placing a singleton 32-bit arg in both a0 and fa0; we don't have to know if // the argument is an int or float. We might be able to avoid the shorty ... @@ -664,6 +663,7 @@ // Temporaries: z0, z1 %def try_simple_args(v_fedc="", z0="", z1="", arg_start="1", skip="", uniq=""): lwu $z0, ART_METHOD_ACCESS_FLAGS_OFFSET(a0) + // The meaning of nterp-invoke-fast-path-flag for RISC-V diverges from other ISAs. BRANCH_IF_BIT_CLEAR $z0, $z0, ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG_BIT, $skip srliw $z0, xINST, 12 // z0 := A @@ -679,22 +679,22 @@ // A = 5 srliw $z1, xINST, 8 // z1 := A|G andi $z1, $z1, 0xF // z1 := G - GET_VREG a5, $z1 + GET_VREG_OBJECT a5, $z1 .L${uniq}_simple_4: srliw $z1, $v_fedc, 12 // z1 := F - GET_VREG a4, $z1 + GET_VREG_OBJECT a4, $z1 .L${uniq}_simple_3: srliw $z1, $v_fedc, 8 // z1 := F|E andi $z1, $z1, 0xF // z1 := E - GET_VREG a3, $z1 + GET_VREG_OBJECT a3, $z1 .L${uniq}_simple_2: srliw $z1, $v_fedc, 4 // z1 := F|E|D andi $z1, $z1, 0xF // z1 := D - GET_VREG a2, $z1 + GET_VREG_OBJECT a2, $z1 .L${uniq}_simple_1: % if arg_start == "0": andi $z1, $v_fedc, 0xF // z1 := C - GET_VREG a1, $z1 + GET_VREG_OBJECT a1, $z1 // instance: a1 already set to "this" .L${uniq}_simple_done: @@ -702,6 +702,7 @@ // Range variant. %def try_simple_args_range(vC="", z0="", z1="", z2="", z3="", z4="", skip="", arg_start="1", uniq=""): lwu $z0, ART_METHOD_ACCESS_FLAGS_OFFSET(a0) + // The meaning of nterp-invoke-fast-path-flag for RISC-V diverges from other ISAs. BRANCH_IF_BIT_CLEAR $z0, $z0, ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG_BIT, $skip srliw $z0, xINST, 8 // z0 := AA @@ -734,7 +735,7 @@ // z4 := &OUT[CCCC + 7] bltz $z2, .L${uniq}_simple_loop_wide // if AA odd, branch to wide-copy - lw $z2, ($z3) + lwu $z2, ($z3) sw $z2, ($z4) addi $z3, $z3, 4 addi $z4, $z4, 4 @@ -750,20 +751,20 @@ // Bottom 7 slots of OUT array never written; first args are passed with a1-a7. .L${uniq}_simple_7: - lw a7, 6*4($z1) + lwu a7, 6*4($z1) .L${uniq}_simple_6: - lw a6, 5*4($z1) + lwu a6, 5*4($z1) .L${uniq}_simple_5: - lw a5, 4*4($z1) + lwu a5, 4*4($z1) .L${uniq}_simple_4: - lw a4, 3*4($z1) + lwu a4, 3*4($z1) .L${uniq}_simple_3: - lw a3, 2*4($z1) + lwu a3, 2*4($z1) .L${uniq}_simple_2: - lw a2, 1*4($z1) + lwu a2, 1*4($z1) .L${uniq}_simple_1: % if arg_start == "0": # static: - lw a1, 0*4($z1) + lwu a1, 0*4($z1) %#: .L${uniq}_simple_done: diff --git a/runtime/nterp_helpers-inl.h b/runtime/nterp_helpers-inl.h new file mode 100644 index 0000000000..84c4f25c18 --- /dev/null +++ b/runtime/nterp_helpers-inl.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_NTERP_HELPERS_INL_H_ +#define ART_RUNTIME_NTERP_HELPERS_INL_H_ + +#include "nterp_helpers.h" + +namespace art { + +ALWAYS_INLINE inline uint32_t GetNterpFastPathFlags(std::string_view shorty, + uint32_t access_flags, + InstructionSet isa) { + bool all_parameters_are_reference = true; + bool all_parameters_are_reference_or_int = true; + for (size_t i = 1; i < shorty.length(); ++i) { + if (shorty[i] != 'L') { + all_parameters_are_reference = false; + if (shorty[i] == 'F' || shorty[i] == 'D' || shorty[i] == 'J') { + all_parameters_are_reference_or_int = false; + break; + } + } + } + + // Check for nterp entry fast-path based on shorty. + uint32_t nterp_flags = 0u; + if ((access_flags & kAccNative) == 0u && all_parameters_are_reference) { + nterp_flags |= kAccNterpEntryPointFastPathFlag; + } + + // Check for nterp invoke fast-path based on shorty. + const bool no_float_return = shorty[0] != 'F' && shorty[0] != 'D'; + if (isa != InstructionSet::kRiscv64 && all_parameters_are_reference_or_int && no_float_return) { + nterp_flags |= kAccNterpInvokeFastPathFlag; + } else if (isa == InstructionSet::kRiscv64 && all_parameters_are_reference && no_float_return) { + nterp_flags |= kAccNterpInvokeFastPathFlag; + } + return nterp_flags; +} + +} // namespace art + +#endif // ART_RUNTIME_NTERP_HELPERS_INL_H_ diff --git a/runtime/nterp_helpers.h b/runtime/nterp_helpers.h index 236059b0ed..f79dcf94f3 100644 --- a/runtime/nterp_helpers.h +++ b/runtime/nterp_helpers.h @@ -75,6 +75,11 @@ uint32_t NterpGetVRegReference(ArtMethod** frame, uint16_t vreg) bool CanMethodUseNterp(ArtMethod* method, InstructionSet isa = kRuntimeISA) REQUIRES_SHARED(Locks::mutator_lock_); +/** + * Returns kAccNterpInvokeFastPathFlag and/or kAccNterpEntryPointFastPathFlag, if appropriate. + */ +uint32_t GetNterpFastPathFlags(std::string_view shorty, uint32_t access_flags, InstructionSet isa); + } // namespace art #endif // ART_RUNTIME_NTERP_HELPERS_H_ |