ART: Increase the number of potential instrinsics
The new limit is 256 intrinsics.
Adds additional sanity checks.
Avoids setting the kAccPreviouslyWarm bit for intrinics (defaults to
true).
Bug: 65872996
Test: art/test.py --host -j32
Change-Id: I33ea67c9b6b8500b3ceb8a085358f075f6fcbb82
diff --git a/compiler/optimizing/intrinsics.cc b/compiler/optimizing/intrinsics.cc
index 11725f4..daec634 100644
--- a/compiler/optimizing/intrinsics.cc
+++ b/compiler/optimizing/intrinsics.cc
@@ -30,6 +30,16 @@
namespace art {
+// Check that intrinsic enum values fit within space set aside in ArtMethod modifier flags.
+#define CHECK_INTRINSICS_ENUM_VALUES(Name, IsStatic, NeedsEnvironmentOrCache, SideEffects, Exceptions, ...) \
+ static_assert( \
+ static_cast<uint32_t>(Intrinsics::k ## Name) <= (kAccIntrinsicBits >> CTZ(kAccIntrinsicBits)), \
+ "Instrinsics enumeration space overflow: ");
+#include "intrinsics_list.h"
+ INTRINSICS_LIST(CHECK_INTRINSICS_ENUM_VALUES)
+#undef INTRINSICS_LIST
+#undef CHECK_INTRINSICS_ENUM_VALUES
+
// Function that returns whether an intrinsic is static/direct or virtual.
static inline InvokeType GetIntrinsicInvokeType(Intrinsics i) {
switch (i) {
@@ -109,6 +119,7 @@
// InvokeStaticOrDirect.
InvokeType intrinsic_type = GetIntrinsicInvokeType(intrinsic);
InvokeType invoke_type = invoke->GetInvokeType();
+
switch (intrinsic_type) {
case kStatic:
return (invoke_type == kStatic);
diff --git a/openjdkjvmti/ti_redefine.cc b/openjdkjvmti/ti_redefine.cc
index 1b4e910..5d9bf2c 100644
--- a/openjdkjvmti/ti_redefine.cc
+++ b/openjdkjvmti/ti_redefine.cc
@@ -1396,7 +1396,7 @@
linker->SetEntryPointsToInterpreter(&method);
method.SetCodeItemOffset(dex_file_->FindCodeItemOffset(class_def, dex_method_idx));
// Clear all the intrinsics related flags.
- method.ClearAccessFlags(art::kAccIntrinsic | (~art::kAccFlagsNotUsedByIntrinsic));
+ method.SetNotIntrinsic();
// Notify the jit that this method is redefined.
art::jit::Jit* jit = driver_->runtime_->GetJit();
if (jit != nullptr) {
diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h
index 7ff35ac..4181169 100644
--- a/runtime/art_method-inl.h
+++ b/runtime/art_method-inl.h
@@ -377,14 +377,14 @@
}
inline void ArtMethod::SetIntrinsic(uint32_t intrinsic) {
- DCHECK(IsUint<8>(intrinsic));
// Currently we only do intrinsics for static/final methods or methods of final
// classes. We don't set kHasSingleImplementation for those methods.
DCHECK(IsStatic() || IsFinal() || GetDeclaringClass()->IsFinal()) <<
"Potential conflict with kAccSingleImplementation";
- uint32_t new_value = (GetAccessFlags() & kAccFlagsNotUsedByIntrinsic) |
- kAccIntrinsic |
- (intrinsic << POPCOUNT(kAccFlagsNotUsedByIntrinsic));
+ static const int kAccFlagsShift = CTZ(kAccIntrinsicBits);
+ DCHECK_LE(intrinsic, kAccIntrinsicBits >> kAccFlagsShift);
+ uint32_t intrinsic_bits = intrinsic << kAccFlagsShift;
+ uint32_t new_value = (GetAccessFlags() & ~kAccIntrinsicBits) | kAccIntrinsic | intrinsic_bits;
if (kIsDebugBuild) {
uint32_t java_flags = (GetAccessFlags() & kAccJavaFlagsMask);
bool is_constructor = IsConstructor();
diff --git a/runtime/art_method.h b/runtime/art_method.h
index fbdc32d..caef81c 100644
--- a/runtime/art_method.h
+++ b/runtime/art_method.h
@@ -117,26 +117,6 @@
access_flags_.store(new_access_flags, std::memory_order_relaxed);
}
- // This setter guarantees atomicity.
- void AddAccessFlags(uint32_t flag) {
- uint32_t old_access_flags;
- uint32_t new_access_flags;
- do {
- old_access_flags = access_flags_.load(std::memory_order_relaxed);
- new_access_flags = old_access_flags | flag;
- } while (!access_flags_.compare_exchange_weak(old_access_flags, new_access_flags));
- }
-
- // This setter guarantees atomicity.
- void ClearAccessFlags(uint32_t flag) {
- uint32_t old_access_flags;
- uint32_t new_access_flags;
- do {
- old_access_flags = access_flags_.load(std::memory_order_relaxed);
- new_access_flags = old_access_flags & ~flag;
- } while (!access_flags_.compare_exchange_weak(old_access_flags, new_access_flags));
- }
-
static MemberOffset AccessFlagsOffset() {
return MemberOffset(OFFSETOF_MEMBER(ArtMethod, access_flags_));
}
@@ -196,12 +176,21 @@
ALWAYS_INLINE void SetIntrinsic(uint32_t intrinsic) REQUIRES_SHARED(Locks::mutator_lock_);
uint32_t GetIntrinsic() {
+ static const int kAccFlagsShift = CTZ(kAccIntrinsicBits);
+ static_assert(IsPowerOfTwo((kAccIntrinsicBits >> kAccFlagsShift) + 1),
+ "kAccIntrinsicBits are not continuous");
+ static_assert((kAccIntrinsic & kAccIntrinsicBits) == 0,
+ "kAccIntrinsic overlaps kAccIntrinsicBits");
DCHECK(IsIntrinsic());
- return (GetAccessFlags() >> POPCOUNT(kAccFlagsNotUsedByIntrinsic)) & kAccMaxIntrinsic;
+ return (GetAccessFlags() & kAccIntrinsicBits) >> kAccFlagsShift;
+ }
+
+ void SetNotIntrinsic() REQUIRES_SHARED(Locks::mutator_lock_) {
+ ClearAccessFlags(kAccIntrinsic | kAccIntrinsicBits);
}
bool IsCopied() {
- static_assert((kAccCopied & kAccFlagsNotUsedByIntrinsic) == kAccCopied,
+ static_assert((kAccCopied & (kAccIntrinsic | kAccIntrinsicBits)) == 0,
"kAccCopied conflicts with intrinsic modifier");
const bool copied = (GetAccessFlags() & kAccCopied) != 0;
// (IsMiranda() || IsDefaultConflicting()) implies copied
@@ -211,7 +200,7 @@
}
bool IsMiranda() {
- static_assert((kAccMiranda & kAccFlagsNotUsedByIntrinsic) == kAccMiranda,
+ static_assert((kAccMiranda & (kAccIntrinsic | kAccIntrinsicBits)) == 0,
"kAccMiranda conflicts with intrinsic modifier");
return (GetAccessFlags() & kAccMiranda) != 0;
}
@@ -245,7 +234,7 @@
// This is set by the class linker.
bool IsDefault() {
- static_assert((kAccDefault & kAccFlagsNotUsedByIntrinsic) == kAccDefault,
+ static_assert((kAccDefault & (kAccIntrinsic | kAccIntrinsicBits)) == 0,
"kAccDefault conflicts with intrinsic modifier");
return (GetAccessFlags() & kAccDefault) != 0;
}
@@ -290,6 +279,22 @@
AddAccessFlags(kAccSkipAccessChecks);
}
+ bool PreviouslyWarm() {
+ if (IsIntrinsic()) {
+ // kAccPreviouslyWarm overlaps with kAccIntrinsicBits.
+ return true;
+ }
+ return (GetAccessFlags() & kAccPreviouslyWarm) != 0;
+ }
+
+ void SetPreviouslyWarm() {
+ if (IsIntrinsic()) {
+ // kAccPreviouslyWarm overlaps with kAccIntrinsicBits.
+ return;
+ }
+ AddAccessFlags(kAccPreviouslyWarm);
+ }
+
// Should this method be run in the interpreter and count locks (e.g., failed structured-
// locking verification)?
bool MustCountLocks() {
@@ -299,6 +304,10 @@
return (GetAccessFlags() & kAccMustCountLocks) != 0;
}
+ void SetMustCountLocks() {
+ AddAccessFlags(kAccMustCountLocks);
+ }
+
// Checks to see if the method was annotated with @dalvik.annotation.optimization.FastNative
// -- Independent of kAccFastNative access flags.
bool IsAnnotatedWithFastNative();
@@ -782,6 +791,37 @@
template <ReadBarrierOption kReadBarrierOption> void GetAccessFlagsDCheck();
+ static inline bool IsValidIntrinsicUpdate(uint32_t modifier) {
+ return (((modifier & kAccIntrinsic) == kAccIntrinsic) &&
+ (((modifier & ~(kAccIntrinsic | kAccIntrinsicBits)) == 0)));
+ }
+
+ static inline bool OverlapsIntrinsicBits(uint32_t modifier) {
+ return (modifier & kAccIntrinsicBits) != 0;
+ }
+
+ // This setter guarantees atomicity.
+ void AddAccessFlags(uint32_t flag) {
+ DCHECK(!IsIntrinsic() || !OverlapsIntrinsicBits(flag) || IsValidIntrinsicUpdate(flag));
+ uint32_t old_access_flags;
+ uint32_t new_access_flags;
+ do {
+ old_access_flags = access_flags_.load(std::memory_order_relaxed);
+ new_access_flags = old_access_flags | flag;
+ } while (!access_flags_.compare_exchange_weak(old_access_flags, new_access_flags));
+ }
+
+ // This setter guarantees atomicity.
+ void ClearAccessFlags(uint32_t flag) {
+ DCHECK(!IsIntrinsic() || !OverlapsIntrinsicBits(flag) || IsValidIntrinsicUpdate(flag));
+ uint32_t old_access_flags;
+ uint32_t new_access_flags;
+ do {
+ old_access_flags = access_flags_.load(std::memory_order_relaxed);
+ new_access_flags = old_access_flags & ~flag;
+ } while (!access_flags_.compare_exchange_weak(old_access_flags, new_access_flags));
+ }
+
DISALLOW_COPY_AND_ASSIGN(ArtMethod); // Need to use CopyFrom to deal with 32 vs 64 bits.
};
diff --git a/runtime/image.cc b/runtime/image.cc
index 0236f47..4c6529b 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -26,7 +26,7 @@
namespace art {
const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '8', '\0' }; // Map boot image tables.
+const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '9', '\0' }; // 256 intrinsics
ImageHeader::ImageHeader(uint32_t image_begin,
uint32_t image_size,
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index 40a5212..ae08fe2 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -534,7 +534,7 @@
static void ClearMethodCounter(ArtMethod* method, bool was_warm) {
if (was_warm) {
- method->AddAccessFlags(kAccPreviouslyWarm);
+ method->SetPreviouslyWarm();
}
// We reset the counter to 1 so that the profile knows that the method was executed at least once.
// This is required for layout purposes.
diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc
index af6a45f..2bf8d8b 100644
--- a/runtime/jit/profile_saver.cc
+++ b/runtime/jit/profile_saver.cc
@@ -349,7 +349,7 @@
// Mark startup methods as hot if they have more than hot_method_sample_threshold
// samples. This means they will get compiled by the compiler driver.
if (method.GetProfilingInfo(kRuntimePointerSize) != nullptr ||
- (method.GetAccessFlags() & kAccPreviouslyWarm) != 0 ||
+ method.PreviouslyWarm() ||
counter >= hot_method_sample_threshold) {
hot_methods->AddReference(method.GetDexFile(), method.GetDexMethodIndex());
} else if (counter != 0) {
diff --git a/runtime/modifiers.h b/runtime/modifiers.h
index 68ab4a4..4b790a0 100644
--- a/runtime/modifiers.h
+++ b/runtime/modifiers.h
@@ -79,6 +79,11 @@
// virtual call.
static constexpr uint32_t kAccSingleImplementation = 0x08000000; // method (runtime)
+// Not currently used, except for intrinsic methods where these bits
+// are part of the intrinsic ordinal.
+static constexpr uint32_t kAccMayBeUnusedBits = 0x70000000;
+
+// Set by the compiler driver when compiling boot classes with instrinsic methods.
static constexpr uint32_t kAccIntrinsic = 0x80000000; // method (runtime)
// Special runtime-only flags.
@@ -89,8 +94,10 @@
// class/ancestor overrides finalize()
static constexpr uint32_t kAccClassIsFinalizable = 0x80000000;
-static constexpr uint32_t kAccFlagsNotUsedByIntrinsic = 0x00FFFFFF;
-static constexpr uint32_t kAccMaxIntrinsic = 0x7F;
+// Continuous sequence of bits used to hold the ordinal of an intrinsic method. Flags
+// which overlap are not valid when kAccIntrinsic is set.
+static constexpr uint32_t kAccIntrinsicBits = kAccMayBeUnusedBits | kAccSingleImplementation |
+ kAccMustCountLocks | kAccCompileDontBother | kAccDefaultConflict | kAccPreviouslyWarm;
// Valid (meaningful) bits for a field.
static constexpr uint32_t kAccValidFieldFlags = kAccPublic | kAccPrivate | kAccProtected |
diff --git a/runtime/verifier/method_verifier.cc b/runtime/verifier/method_verifier.cc
index cfdf20d..ee428ed 100644
--- a/runtime/verifier/method_verifier.cc
+++ b/runtime/verifier/method_verifier.cc
@@ -431,7 +431,7 @@
}
}
if ((verifier.encountered_failure_types_ & VerifyError::VERIFY_ERROR_LOCKING) != 0) {
- method->AddAccessFlags(kAccMustCountLocks);
+ method->SetMustCountLocks();
}
}
} else {