Disable bitstring type check runtime hooks.

Introduce a build flag for the bitstring type check, put
runtime hooks behind the flag and set the flag to false.

Also add bitstring initialization for proxy classes, a test
and a benchmark for the type checks.

Test: m test-art-host-gtest
Test: testrunner.py --host --interpreter
Test: Repeat with kBitstringSubtypeCheckEnabled = true.
Bug: 73299705
Change-Id: Ibcd88a828c7addc0473d8c428818734f80226b19
diff --git a/benchmark/type-check/info.txt b/benchmark/type-check/info.txt
new file mode 100644
index 0000000..d14fb96
--- /dev/null
+++ b/benchmark/type-check/info.txt
@@ -0,0 +1 @@
+Benchmarks for repeating check-cast and instance-of instructions in a loop.
diff --git a/benchmark/type-check/src/TypeCheckBenchmark.java b/benchmark/type-check/src/TypeCheckBenchmark.java
new file mode 100644
index 0000000..96904d9
--- /dev/null
+++ b/benchmark/type-check/src/TypeCheckBenchmark.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TypeCheckBenchmark {
+    public void timeCheckCastLevel1ToLevel1(int count) {
+        Object[] arr = arr1;
+        for (int i = 0; i < count; ++i) {
+            Level1 l1 = (Level1) arr[i & 1023];
+        }
+    }
+
+    public void timeCheckCastLevel2ToLevel1(int count) {
+        Object[] arr = arr2;
+        for (int i = 0; i < count; ++i) {
+            Level1 l1 = (Level1) arr[i & 1023];
+        }
+    }
+
+    public void timeCheckCastLevel3ToLevel1(int count) {
+        Object[] arr = arr3;
+        for (int i = 0; i < count; ++i) {
+            Level1 l1 = (Level1) arr[i & 1023];
+        }
+    }
+
+    public void timeCheckCastLevel9ToLevel1(int count) {
+        Object[] arr = arr9;
+        for (int i = 0; i < count; ++i) {
+            Level1 l1 = (Level1) arr[i & 1023];
+        }
+    }
+
+    public void timeCheckCastLevel9ToLevel2(int count) {
+        Object[] arr = arr9;
+        for (int i = 0; i < count; ++i) {
+            Level2 l2 = (Level2) arr[i & 1023];
+        }
+    }
+
+    public void timeInstanceOfLevel1ToLevel1(int count) {
+        int sum = 0;
+        Object[] arr = arr1;
+        for (int i = 0; i < count; ++i) {
+            if (arr[i & 1023] instanceof Level1) {
+              ++sum;
+            }
+        }
+        result = sum;
+    }
+
+    public void timeInstanceOfLevel2ToLevel1(int count) {
+        int sum = 0;
+        Object[] arr = arr2;
+        for (int i = 0; i < count; ++i) {
+            if (arr[i & 1023] instanceof Level1) {
+              ++sum;
+            }
+        }
+        result = sum;
+    }
+
+    public void timeInstanceOfLevel3ToLevel1(int count) {
+        int sum = 0;
+        Object[] arr = arr3;
+        for (int i = 0; i < count; ++i) {
+            if (arr[i & 1023] instanceof Level1) {
+              ++sum;
+            }
+        }
+        result = sum;
+    }
+
+    public void timeInstanceOfLevel9ToLevel1(int count) {
+        int sum = 0;
+        Object[] arr = arr9;
+        for (int i = 0; i < count; ++i) {
+            if (arr[i & 1023] instanceof Level1) {
+              ++sum;
+            }
+        }
+        result = sum;
+    }
+
+    public void timeInstanceOfLevel9ToLevel2(int count) {
+        int sum = 0;
+        Object[] arr = arr9;
+        for (int i = 0; i < count; ++i) {
+            if (arr[i & 1023] instanceof Level2) {
+              ++sum;
+            }
+        }
+        result = sum;
+    }
+
+    public static Object[] createArray(int level) {
+        try {
+            Class<?>[] ls = {
+                    null,
+                    Level1.class,
+                    Level2.class,
+                    Level3.class,
+                    Level4.class,
+                    Level5.class,
+                    Level6.class,
+                    Level7.class,
+                    Level8.class,
+                    Level9.class,
+            };
+            Class<?> l = ls[level];
+            Object[] array = new Object[1024];
+            for (int i = 0; i < array.length; ++i) {
+                array[i] = l.newInstance();
+            }
+            return array;
+        } catch (Exception unexpected) {
+            throw new Error("Initialization failure!");
+        }
+    }
+    Object[] arr1 = createArray(1);
+    Object[] arr2 = createArray(2);
+    Object[] arr3 = createArray(3);
+    Object[] arr9 = createArray(9);
+    int result;
+}
+
+class Level1 { }
+class Level2 extends Level1 { }
+class Level3 extends Level2 { }
+class Level4 extends Level3 { }
+class Level5 extends Level4 { }
+class Level6 extends Level5 { }
+class Level7 extends Level6 { }
+class Level8 extends Level7 { }
+class Level9 extends Level8 { }
diff --git a/dex2oat/linker/image_writer.cc b/dex2oat/linker/image_writer.cc
index 73eaad4..42d2280 100644
--- a/dex2oat/linker/image_writer.cc
+++ b/dex2oat/linker/image_writer.cc
@@ -2364,7 +2364,7 @@
   FixupClassVisitor visitor(this, copy);
   ObjPtr<mirror::Object>(orig)->VisitReferences(visitor, visitor);
 
-  if (compile_app_image_) {
+  if (kBitstringSubtypeCheckEnabled && compile_app_image_) {
     // When we call SubtypeCheck::EnsureInitialize, it Assigns new bitstring
     // values to the parent of that class.
     //
diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc
index 6c9f569..82bb88a 100644
--- a/oatdump/oatdump.cc
+++ b/oatdump/oatdump.cc
@@ -2394,9 +2394,11 @@
     } else if (obj->IsClass()) {
       ObjPtr<mirror::Class> klass = obj->AsClass();
 
-      os << "SUBTYPE_CHECK_BITS: ";
-      SubtypeCheck<ObjPtr<mirror::Class>>::Dump(klass, os);
-      os << "\n";
+      if (kBitstringSubtypeCheckEnabled) {
+        os << "SUBTYPE_CHECK_BITS: ";
+        SubtypeCheck<ObjPtr<mirror::Class>>::Dump(klass, os);
+        os << "\n";
+      }
 
       if (klass->NumStaticFields() != 0) {
         os << "STATICS:\n";
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 800427d..c4b1bf8 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -457,7 +457,7 @@
                                                          VoidFunctor()));
 
   // Initialize the SubtypeCheck bitstring for java.lang.Object and java.lang.Class.
-  {
+  if (kBitstringSubtypeCheckEnabled) {
     // It might seem the lock here is unnecessary, however all the SubtypeCheck
     // functions are annotated to require locks all the way down.
     //
@@ -1856,7 +1856,7 @@
       visitor(root.Read());
     }
 
-    {
+    if (kBitstringSubtypeCheckEnabled) {
       // Every class in the app image has initially SubtypeCheckInfo in the
       // Uninitialized state.
       //
@@ -4484,6 +4484,14 @@
 
   Runtime::Current()->GetRuntimeCallbacks()->ClassPrepare(temp_klass, klass);
 
+  // SubtypeCheckInfo::Initialized must happen-before any new-instance for that type.
+  // See also ClassLinker::EnsureInitialized().
+  if (kBitstringSubtypeCheckEnabled) {
+    MutexLock subtype_check_lock(Thread::Current(), *Locks::subtype_check_lock_);
+    SubtypeCheck<ObjPtr<mirror::Class>>::EnsureInitialized(klass.Get());
+    // TODO: Avoid taking subtype_check_lock_ if SubtypeCheck for j.l.r.Proxy is already assigned.
+  }
+
   {
     // Lock on klass is released. Lock new class object.
     ObjectLock<mirror::Class> initialization_lock(self, klass);
@@ -5231,7 +5239,7 @@
   // can be used as a source for the IsSubClass check, and that all ancestors
   // of the class are Assigned (can be used as a target for IsSubClass check)
   // or Overflowed (can be used as a source for IsSubClass check).
-  {
+  if (kBitstringSubtypeCheckEnabled) {
     MutexLock subtype_check_lock(Thread::Current(), *Locks::subtype_check_lock_);
     SubtypeCheck<ObjPtr<mirror::Class>>::EnsureInitialized(c.Get());
     // TODO: Avoid taking subtype_check_lock_ if SubtypeCheck is already initialized.
diff --git a/runtime/image.cc b/runtime/image.cc
index 8e3615f..9940622 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -26,7 +26,7 @@
 namespace art {
 
 const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const uint8_t ImageHeader::kImageVersion[] = { '0', '5', '4', '\0' };  // Math.pow() intrinsic.
+const uint8_t ImageHeader::kImageVersion[] = { '0', '5', '5', '\0' };  // Bitstring type check off.
 
 ImageHeader::ImageHeader(uint32_t image_begin,
                          uint32_t image_size,
diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h
index 36388eb..86d538e 100644
--- a/runtime/mirror/class-inl.h
+++ b/runtime/mirror/class-inl.h
@@ -550,7 +550,7 @@
     current = current->GetSuperClass();
   } while (current != nullptr);
 
-  if (kIsDebugBuild) {
+  if (kIsDebugBuild && kBitstringSubtypeCheckEnabled) {
     ObjPtr<mirror::Class> dis(this);
 
     SubtypeCheckInfo::Result sc_result = SubtypeCheck<ObjPtr<Class>>::IsSubtypeOf(dis, klass);
diff --git a/runtime/mirror/class.cc b/runtime/mirror/class.cc
index 8a7defd..9246bae 100644
--- a/runtime/mirror/class.cc
+++ b/runtime/mirror/class.cc
@@ -156,9 +156,19 @@
     self->AssertPendingException();
   }
 
-  {
+  if (kBitstringSubtypeCheckEnabled) {
+    // FIXME: This looks broken with respect to aborted transactions.
     ObjPtr<mirror::Class> h_this_ptr = h_this.Get();
     SubtypeCheck<ObjPtr<mirror::Class>>::WriteStatus(h_this_ptr, new_status);
+  } else {
+    // The ClassStatus is always in the 4 most-significant bits of status_.
+    static_assert(sizeof(status_) == sizeof(uint32_t), "Size of status_ not equal to uint32");
+    uint32_t new_status_value = static_cast<uint32_t>(new_status) << (32 - kClassStatusBitSize);
+    if (Runtime::Current()->IsActiveTransaction()) {
+      h_this->SetField32Volatile<true>(StatusOffset(), new_status_value);
+    } else {
+      h_this->SetField32Volatile<false>(StatusOffset(), new_status_value);
+    }
   }
 
   // Setting the object size alloc fast path needs to be after the status write so that if the
diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h
index ced7c7c..b5deffb 100644
--- a/runtime/mirror/class.h
+++ b/runtime/mirror/class.h
@@ -81,7 +81,7 @@
   template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags>
   ClassStatus GetStatus() REQUIRES_SHARED(Locks::mutator_lock_) {
     // Avoid including "subtype_check_bits_and_status.h" to get the field.
-    // The ClassStatus is always in the 4 most-significant of status_.
+    // The ClassStatus is always in the 4 most-significant bits of status_.
     return enum_cast<ClassStatus>(
         static_cast<uint32_t>(GetField32Volatile<kVerifyFlags>(StatusOffset())) >> (32 - 4));
   }
diff --git a/runtime/subtype_check.h b/runtime/subtype_check.h
index 54d2f00..3b1d5f8 100644
--- a/runtime/subtype_check.h
+++ b/runtime/subtype_check.h
@@ -24,6 +24,9 @@
 #include "mirror/class.h"
 #include "runtime.h"
 
+// Build flag for the bitstring subtype check runtime hooks.
+constexpr bool kBitstringSubtypeCheckEnabled = false;
+
 /**
  * Any node in a tree can have its path (from the root to the node) represented as a string by
  * concatenating the path of the parent to that of the current node.
diff --git a/test/670-bitstring-type-check/build b/test/670-bitstring-type-check/build
new file mode 100644
index 0000000..38307f2
--- /dev/null
+++ b/test/670-bitstring-type-check/build
@@ -0,0 +1,216 @@
+#!/bin/bash
+#
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stop if something fails.
+set -e
+
+# Write out the source file.
+
+mkdir src
+cat >src/Main.java <<EOF
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+EOF
+
+for i in {0..8192}; do echo "class Level1Class$i { }" >>src/Main.java; done
+for i in {0..1024}; do echo "class Level2Class$i extends Level1Class0 { }" >>src/Main.java; done
+
+cat >>src/Main.java <<EOF
+class Level3Class0 extends Level2Class0 { }
+class Level4Class0 extends Level3Class0 { }
+class Level5Class0 extends Level4Class0 { }
+class Level6Class0 extends Level5Class0 { }
+class Level7Class0 extends Level6Class0 { }
+class Level8Class0 extends Level7Class0 { }
+class Level9Class0 extends Level8Class0 { }
+
+public class Main {
+  public static void main(String[] args) throws Exception {
+    // 8193 classes at level 1 make sure we shall have an overflow if there are 13 or
+    // less bits for the level 1 character. 1025 classes at level 2 similarly guarantees
+    // an overflow if the number of bits for level 2 character is 10 or less. To test
+    // type checks also for the depth overflow, we provide a hierarchy 9 levels deep.
+
+    // Make sure the bitstrings are initialized.
+    for (int i = 0; i <= 8192; ++i) {
+      Class.forName("Level1Class" + i).newInstance();
+    }
+    for (int i = 0; i <= 1024; ++i) {
+      Class.forName("Level2Class" + i).newInstance();
+    }
+
+    // Note: Using a different class for tests so that verification of Main.main() does
+    // not try to resolve classes used by the tests. This guarantees uninitialized type
+    // check bitstrings when we enter Main.main() and start initializing them above.
+    Helper.testInstanceOf();
+    Helper.testCheckCast();
+  }
+}
+
+class Helper {
+  public static void testInstanceOf() throws Exception {
+    for (int i = 1; i <= 9; ++i) {
+      Object o = createInstance("Level" + i + "Class0");
+      assertTrue(o instanceof Level1Class0);
+      if (o instanceof Level2Class0) {
+        assertFalse(i < 2);
+      } else {
+        assertTrue(i < 2);
+      }
+      if (o instanceof Level3Class0) {
+        assertFalse(i < 3);
+      } else {
+        assertTrue(i < 3);
+      }
+      if (o instanceof Level4Class0) {
+        assertFalse(i < 4);
+      } else {
+        assertTrue(i < 4);
+      }
+      if (o instanceof Level5Class0) {
+        assertFalse(i < 5);
+      } else {
+        assertTrue(i < 5);
+      }
+      if (o instanceof Level6Class0) {
+        assertFalse(i < 6);
+      } else {
+        assertTrue(i < 6);
+      }
+      if (o instanceof Level7Class0) {
+        assertFalse(i < 7);
+      } else {
+        assertTrue(i < 7);
+      }
+      if (o instanceof Level8Class0) {
+        assertFalse(i < 8);
+      } else {
+        assertTrue(i < 8);
+      }
+      if (o instanceof Level9Class0) {
+        assertFalse(i < 9);
+      } else {
+        assertTrue(i < 9);
+      }
+    }
+
+    assertTrue(createInstance("Level1Class8192") instanceof Level1Class8192);
+    assertFalse(createInstance("Level1Class8192") instanceof Level1Class0);
+    assertTrue(createInstance("Level2Class1024") instanceof Level2Class1024);
+    assertTrue(createInstance("Level2Class1024") instanceof Level1Class0);
+    assertFalse(createInstance("Level2Class1024") instanceof Level2Class0);
+  }
+
+  public static void testCheckCast() throws Exception {
+    for (int i = 1; i <= 9; ++i) {
+      Object o = createInstance("Level" + i + "Class0");
+      Level1Class0 l1c0 = (Level1Class0) o;
+      try {
+        Level2Class0 l2c0 = (Level2Class0) o;
+        assertFalse(i < 2);
+      } catch (ClassCastException cce) {
+        assertTrue(i < 2);
+      }
+      try {
+        Level3Class0 l3c0 = (Level3Class0) o;
+        assertFalse(i < 3);
+      } catch (ClassCastException cce) {
+        assertTrue(i < 3);
+      }
+      try {
+        Level4Class0 l4c0 = (Level4Class0) o;
+        assertFalse(i < 4);
+      } catch (ClassCastException cce) {
+        assertTrue(i < 4);
+      }
+      try {
+        Level5Class0 l5c0 = (Level5Class0) o;
+        assertFalse(i < 5);
+      } catch (ClassCastException cce) {
+        assertTrue(i < 5);
+      }
+      try {
+        Level6Class0 l6c0 = (Level6Class0) o;
+        assertFalse(i < 6);
+      } catch (ClassCastException cce) {
+        assertTrue(i < 6);
+      }
+      try {
+        Level7Class0 l7c0 = (Level7Class0) o;
+        assertFalse(i < 7);
+      } catch (ClassCastException cce) {
+        assertTrue(i < 7);
+      }
+      try {
+        Level8Class0 l8c0 = (Level8Class0) o;
+        assertFalse(i < 8);
+      } catch (ClassCastException cce) {
+        assertTrue(i < 8);
+      }
+      try {
+        Level9Class0 l9c0 = (Level9Class0) o;
+        assertFalse(i < 9);
+      } catch (ClassCastException cce) {
+        assertTrue(i < 9);
+      }
+    }
+
+    Level1Class8192 l1c8192 = (Level1Class8192) createInstance("Level1Class8192");
+    try {
+      Level1Class0 l1c0 = (Level1Class0) createInstance("Level1Class8192");
+      throw new AssertionError("Unexpected");
+    } catch (ClassCastException expected) {}
+    Level2Class1024 l2c1024 = (Level2Class1024) createInstance("Level2Class1024");
+    Level1Class0 l1c0 = (Level1Class0) createInstance("Level2Class1024");
+    try {
+      Level2Class0 l2c0 = (Level2Class0) createInstance("Level2Class1024");
+      throw new AssertionError("Unexpected");
+    } catch (ClassCastException expected) {}
+  }
+
+  public static Object createInstance(String className) throws Exception {
+    return Class.forName(className).newInstance();
+  }
+
+  public static void assertTrue(boolean value) throws Exception {
+    if (!value) {
+      throw new AssertionError();
+    }
+  }
+
+  public static void assertFalse(boolean value) throws Exception {
+    if (value) {
+      throw new AssertionError();
+    }
+  }
+}
+EOF
+
+./default-build "$@"
diff --git a/test/670-bitstring-type-check/expected.txt b/test/670-bitstring-type-check/expected.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/670-bitstring-type-check/expected.txt
diff --git a/test/670-bitstring-type-check/info.txt b/test/670-bitstring-type-check/info.txt
new file mode 100644
index 0000000..a34ba86
--- /dev/null
+++ b/test/670-bitstring-type-check/info.txt
@@ -0,0 +1 @@
+Tests for the bitstring type checks.