ART: Increase the number of potential instrinsics

The new limit is 256 intrinsics.

Adds additional sanity checks.

Avoids setting the kAccPreviouslyWarm bit for intrinics (defaults to
true).

Bug: 65872996
Test: art/test.py --host -j32
Change-Id: I33ea67c9b6b8500b3ceb8a085358f075f6fcbb82

9 files changed, 93 insertions, 35 deletions

diff --git a/compiler/optimizing/intrinsics.cc b/compiler/optimizing/intrinsics.cc
index 11725f43c3..daec634f28 100644
--- a/compiler/optimizing/intrinsics.cc
+++ b/compiler/optimizing/intrinsics.cc
@@ -30,6 +30,16 @@
 
 namespace art {
 
+// Check that intrinsic enum values fit within space set aside in ArtMethod modifier flags.
+#define CHECK_INTRINSICS_ENUM_VALUES(Name, IsStatic, NeedsEnvironmentOrCache, SideEffects, Exceptions, ...) \
+  static_assert( \
+      static_cast<uint32_t>(Intrinsics::k ## Name) <= (kAccIntrinsicBits >> CTZ(kAccIntrinsicBits)), \
+      "Instrinsics enumeration space overflow: ");
+#include "intrinsics_list.h"
+  INTRINSICS_LIST(CHECK_INTRINSICS_ENUM_VALUES)
+#undef INTRINSICS_LIST
+#undef CHECK_INTRINSICS_ENUM_VALUES
+
 // Function that returns whether an intrinsic is static/direct or virtual.
 static inline InvokeType GetIntrinsicInvokeType(Intrinsics i) {
   switch (i) {
@@ -109,6 +119,7 @@ static bool CheckInvokeType(Intrinsics intrinsic, HInvoke* invoke) {
   // InvokeStaticOrDirect.
   InvokeType intrinsic_type = GetIntrinsicInvokeType(intrinsic);
   InvokeType invoke_type = invoke->GetInvokeType();
+
   switch (intrinsic_type) {
     case kStatic:
       return (invoke_type == kStatic);
diff --git a/openjdkjvmti/ti_redefine.cc b/openjdkjvmti/ti_redefine.cc
index 1b4e910062..5d9bf2ce6e 100644
--- a/openjdkjvmti/ti_redefine.cc
+++ b/openjdkjvmti/ti_redefine.cc
@@ -1396,7 +1396,7 @@ void Redefiner::ClassRedefinition::UpdateMethods(art::ObjPtr<art::mirror::Class>
     linker->SetEntryPointsToInterpreter(&method);
     method.SetCodeItemOffset(dex_file_->FindCodeItemOffset(class_def, dex_method_idx));
     // Clear all the intrinsics related flags.
-    method.ClearAccessFlags(art::kAccIntrinsic | (~art::kAccFlagsNotUsedByIntrinsic));
+    method.SetNotIntrinsic();
     // Notify the jit that this method is redefined.
     art::jit::Jit* jit = driver_->runtime_->GetJit();
     if (jit != nullptr) {
diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h
index 7ff35acf74..4181169f5d 100644
--- a/runtime/art_method-inl.h
+++ b/runtime/art_method-inl.h
@@ -377,14 +377,14 @@ inline bool ArtMethod::HasSingleImplementation() {
 }
 
 inline void ArtMethod::SetIntrinsic(uint32_t intrinsic) {
-  DCHECK(IsUint<8>(intrinsic));
   // Currently we only do intrinsics for static/final methods or methods of final
   // classes. We don't set kHasSingleImplementation for those methods.
   DCHECK(IsStatic() || IsFinal() || GetDeclaringClass()->IsFinal()) <<
       "Potential conflict with kAccSingleImplementation";
-  uint32_t new_value = (GetAccessFlags() & kAccFlagsNotUsedByIntrinsic) |
-      kAccIntrinsic |
-      (intrinsic << POPCOUNT(kAccFlagsNotUsedByIntrinsic));
+  static const int kAccFlagsShift = CTZ(kAccIntrinsicBits);
+  DCHECK_LE(intrinsic, kAccIntrinsicBits >> kAccFlagsShift);
+  uint32_t intrinsic_bits = intrinsic << kAccFlagsShift;
+  uint32_t new_value = (GetAccessFlags() & ~kAccIntrinsicBits) | kAccIntrinsic | intrinsic_bits;
   if (kIsDebugBuild) {
     uint32_t java_flags = (GetAccessFlags() & kAccJavaFlagsMask);
     bool is_constructor = IsConstructor();
diff --git a/runtime/art_method.h b/runtime/art_method.h
index fbdc32d8ec..caef81c601 100644
--- a/runtime/art_method.h
+++ b/runtime/art_method.h
@@ -117,26 +117,6 @@ class ArtMethod FINAL {
     access_flags_.store(new_access_flags, std::memory_order_relaxed);
   }
 
-  // This setter guarantees atomicity.
-  void AddAccessFlags(uint32_t flag) {
-    uint32_t old_access_flags;
-    uint32_t new_access_flags;
-    do {
-      old_access_flags = access_flags_.load(std::memory_order_relaxed);
-      new_access_flags = old_access_flags | flag;
-    } while (!access_flags_.compare_exchange_weak(old_access_flags, new_access_flags));
-  }
-
-  // This setter guarantees atomicity.
-  void ClearAccessFlags(uint32_t flag) {
-    uint32_t old_access_flags;
-    uint32_t new_access_flags;
-    do {
-      old_access_flags = access_flags_.load(std::memory_order_relaxed);
-      new_access_flags = old_access_flags & ~flag;
-    } while (!access_flags_.compare_exchange_weak(old_access_flags, new_access_flags));
-  }
-
   static MemberOffset AccessFlagsOffset() {
     return MemberOffset(OFFSETOF_MEMBER(ArtMethod, access_flags_));
   }
@@ -196,12 +176,21 @@ class ArtMethod FINAL {
   ALWAYS_INLINE void SetIntrinsic(uint32_t intrinsic) REQUIRES_SHARED(Locks::mutator_lock_);
 
   uint32_t GetIntrinsic() {
+    static const int kAccFlagsShift = CTZ(kAccIntrinsicBits);
+    static_assert(IsPowerOfTwo((kAccIntrinsicBits >> kAccFlagsShift) + 1),
+                  "kAccIntrinsicBits are not continuous");
+    static_assert((kAccIntrinsic & kAccIntrinsicBits) == 0,
+                  "kAccIntrinsic overlaps kAccIntrinsicBits");
     DCHECK(IsIntrinsic());
-    return (GetAccessFlags() >> POPCOUNT(kAccFlagsNotUsedByIntrinsic)) & kAccMaxIntrinsic;
+    return (GetAccessFlags() & kAccIntrinsicBits) >> kAccFlagsShift;
+  }
+
+  void SetNotIntrinsic() REQUIRES_SHARED(Locks::mutator_lock_) {
+    ClearAccessFlags(kAccIntrinsic | kAccIntrinsicBits);
   }
 
   bool IsCopied() {
-    static_assert((kAccCopied & kAccFlagsNotUsedByIntrinsic) == kAccCopied,
+    static_assert((kAccCopied & (kAccIntrinsic | kAccIntrinsicBits)) == 0,
                   "kAccCopied conflicts with intrinsic modifier");
     const bool copied = (GetAccessFlags() & kAccCopied) != 0;
     // (IsMiranda() || IsDefaultConflicting()) implies copied
@@ -211,7 +200,7 @@ class ArtMethod FINAL {
   }
 
   bool IsMiranda() {
-    static_assert((kAccMiranda & kAccFlagsNotUsedByIntrinsic) == kAccMiranda,
+    static_assert((kAccMiranda & (kAccIntrinsic | kAccIntrinsicBits)) == 0,
                   "kAccMiranda conflicts with intrinsic modifier");
     return (GetAccessFlags() & kAccMiranda) != 0;
   }
@@ -245,7 +234,7 @@ class ArtMethod FINAL {
 
   // This is set by the class linker.
   bool IsDefault() {
-    static_assert((kAccDefault & kAccFlagsNotUsedByIntrinsic) == kAccDefault,
+    static_assert((kAccDefault & (kAccIntrinsic | kAccIntrinsicBits)) == 0,
                   "kAccDefault conflicts with intrinsic modifier");
     return (GetAccessFlags() & kAccDefault) != 0;
   }
@@ -290,6 +279,22 @@ class ArtMethod FINAL {
     AddAccessFlags(kAccSkipAccessChecks);
   }
 
+  bool PreviouslyWarm() {
+    if (IsIntrinsic()) {
+      // kAccPreviouslyWarm overlaps with kAccIntrinsicBits.
+      return true;
+    }
+    return (GetAccessFlags() & kAccPreviouslyWarm) != 0;
+  }
+
+  void SetPreviouslyWarm() {
+    if (IsIntrinsic()) {
+      // kAccPreviouslyWarm overlaps with kAccIntrinsicBits.
+      return;
+    }
+    AddAccessFlags(kAccPreviouslyWarm);
+  }
+
   // Should this method be run in the interpreter and count locks (e.g., failed structured-
   // locking verification)?
   bool MustCountLocks() {
@@ -299,6 +304,10 @@ class ArtMethod FINAL {
     return (GetAccessFlags() & kAccMustCountLocks) != 0;
   }
 
+  void SetMustCountLocks() {
+    AddAccessFlags(kAccMustCountLocks);
+  }
+
   // Checks to see if the method was annotated with @dalvik.annotation.optimization.FastNative
   // -- Independent of kAccFastNative access flags.
   bool IsAnnotatedWithFastNative();
@@ -782,6 +791,37 @@ class ArtMethod FINAL {
 
   template <ReadBarrierOption kReadBarrierOption> void GetAccessFlagsDCheck();
 
+  static inline bool IsValidIntrinsicUpdate(uint32_t modifier) {
+    return (((modifier & kAccIntrinsic) == kAccIntrinsic) &&
+            (((modifier & ~(kAccIntrinsic | kAccIntrinsicBits)) == 0)));
+  }
+
+  static inline bool OverlapsIntrinsicBits(uint32_t modifier) {
+    return (modifier & kAccIntrinsicBits) != 0;
+  }
+
+  // This setter guarantees atomicity.
+  void AddAccessFlags(uint32_t flag) {
+    DCHECK(!IsIntrinsic() || !OverlapsIntrinsicBits(flag) || IsValidIntrinsicUpdate(flag));
+    uint32_t old_access_flags;
+    uint32_t new_access_flags;
+    do {
+      old_access_flags = access_flags_.load(std::memory_order_relaxed);
+      new_access_flags = old_access_flags | flag;
+    } while (!access_flags_.compare_exchange_weak(old_access_flags, new_access_flags));
+  }
+
+  // This setter guarantees atomicity.
+  void ClearAccessFlags(uint32_t flag) {
+    DCHECK(!IsIntrinsic() || !OverlapsIntrinsicBits(flag) || IsValidIntrinsicUpdate(flag));
+    uint32_t old_access_flags;
+    uint32_t new_access_flags;
+    do {
+      old_access_flags = access_flags_.load(std::memory_order_relaxed);
+      new_access_flags = old_access_flags & ~flag;
+    } while (!access_flags_.compare_exchange_weak(old_access_flags, new_access_flags));
+  }
+
   DISALLOW_COPY_AND_ASSIGN(ArtMethod);  // Need to use CopyFrom to deal with 32 vs 64 bits.
 };
 
diff --git a/runtime/image.cc b/runtime/image.cc
index 0236f47fbe..4c6529b32c 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -26,7 +26,7 @@
 namespace art {
 
 const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '8', '\0' };  // Map boot image tables.
+const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '9', '\0' };  // 256 intrinsics
 
 ImageHeader::ImageHeader(uint32_t image_begin,
                          uint32_t image_size,
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index 40a5212629..ae08fe2651 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -534,7 +534,7 @@ void JitCodeCache::CopyInlineCacheInto(const InlineCache& ic,
 
 static void ClearMethodCounter(ArtMethod* method, bool was_warm) {
   if (was_warm) {
-    method->AddAccessFlags(kAccPreviouslyWarm);
+    method->SetPreviouslyWarm();
   }
   // We reset the counter to 1 so that the profile knows that the method was executed at least once.
   // This is required for layout purposes.
diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc
index af6a45f14a..2bf8d8b8f8 100644
--- a/runtime/jit/profile_saver.cc
+++ b/runtime/jit/profile_saver.cc
@@ -349,7 +349,7 @@ static void SampleClassesAndExecutedMethods(pthread_t profiler_pthread,
           // Mark startup methods as hot if they have more than hot_method_sample_threshold
           // samples. This means they will get compiled by the compiler driver.
           if (method.GetProfilingInfo(kRuntimePointerSize) != nullptr ||
-              (method.GetAccessFlags() & kAccPreviouslyWarm) != 0 ||
+              method.PreviouslyWarm() ||
               counter >= hot_method_sample_threshold) {
             hot_methods->AddReference(method.GetDexFile(), method.GetDexMethodIndex());
           } else if (counter != 0) {
diff --git a/runtime/modifiers.h b/runtime/modifiers.h
index 68ab4a4035..4b790a0f03 100644
--- a/runtime/modifiers.h
+++ b/runtime/modifiers.h
@@ -79,6 +79,11 @@ static constexpr uint32_t kAccMustCountLocks =        0x04000000;  // method (ru
 // virtual call.
 static constexpr uint32_t kAccSingleImplementation =  0x08000000;  // method (runtime)
 
+// Not currently used, except for intrinsic methods where these bits
+// are part of the intrinsic ordinal.
+static constexpr uint32_t kAccMayBeUnusedBits =       0x70000000;
+
+// Set by the compiler driver when compiling boot classes with instrinsic methods.
 static constexpr uint32_t kAccIntrinsic  =            0x80000000;  // method (runtime)
 
 // Special runtime-only flags.
@@ -89,8 +94,10 @@ static constexpr uint32_t kAccHasDefaultMethod          = 0x40000000;
 // class/ancestor overrides finalize()
 static constexpr uint32_t kAccClassIsFinalizable        = 0x80000000;
 
-static constexpr uint32_t kAccFlagsNotUsedByIntrinsic   = 0x00FFFFFF;
-static constexpr uint32_t kAccMaxIntrinsic              = 0x7F;
+// Continuous sequence of bits used to hold the ordinal of an intrinsic method. Flags
+// which overlap are not valid when kAccIntrinsic is set.
+static constexpr uint32_t kAccIntrinsicBits = kAccMayBeUnusedBits | kAccSingleImplementation |
+    kAccMustCountLocks | kAccCompileDontBother | kAccDefaultConflict | kAccPreviouslyWarm;
 
 // Valid (meaningful) bits for a field.
 static constexpr uint32_t kAccValidFieldFlags = kAccPublic | kAccPrivate | kAccProtected |
diff --git a/runtime/verifier/method_verifier.cc b/runtime/verifier/method_verifier.cc
index cfdf20d3b9..ee428edf28 100644
--- a/runtime/verifier/method_verifier.cc
+++ b/runtime/verifier/method_verifier.cc
@@ -431,7 +431,7 @@ MethodVerifier::FailureData MethodVerifier::VerifyMethod(Thread* self,
         }
       }
       if ((verifier.encountered_failure_types_ & VerifyError::VERIFY_ERROR_LOCKING) != 0) {
-        method->AddAccessFlags(kAccMustCountLocks);
+        method->SetMustCountLocks();
       }
     }
   } else {