Merge changes If64a9720,I31c3b54c

* changes:
  Add fieldnull tool
  Implement IterateOverInstances JVMTI function

8 files changed, 457 insertions, 5 deletions

diff --git a/openjdkjvmti/OpenjdkJvmTi.cc b/openjdkjvmti/OpenjdkJvmTi.cc
index 59f61e2ee4..3213bbe91a 100644
--- a/openjdkjvmti/OpenjdkJvmTi.cc
+++ b/openjdkjvmti/OpenjdkJvmTi.cc
@@ -506,13 +506,15 @@ class JvmtiFunctions {
 
   static jvmtiError IterateOverInstancesOfClass(
       jvmtiEnv* env,
-      jclass klass ATTRIBUTE_UNUSED,
-      jvmtiHeapObjectFilter object_filter ATTRIBUTE_UNUSED,
-      jvmtiHeapObjectCallback heap_object_callback ATTRIBUTE_UNUSED,
-      const void* user_data ATTRIBUTE_UNUSED) {
+      jclass klass,
+      jvmtiHeapObjectFilter object_filter,
+      jvmtiHeapObjectCallback heap_object_callback,
+      const void* user_data) {
     ENSURE_VALID_ENV(env);
     ENSURE_HAS_CAP(env, can_tag_objects);
-    return ERR(NOT_IMPLEMENTED);
+    HeapUtil heap_util(ArtJvmTiEnv::AsArtJvmTiEnv(env)->object_tag_table.get());
+    return heap_util.IterateOverInstancesOfClass(
+        env, klass, object_filter, heap_object_callback, user_data);
   }
 
   static jvmtiError GetLocalObject(jvmtiEnv* env,
diff --git a/openjdkjvmti/ti_heap.cc b/openjdkjvmti/ti_heap.cc
index 85aa946356..6c79a602c3 100644
--- a/openjdkjvmti/ti_heap.cc
+++ b/openjdkjvmti/ti_heap.cc
@@ -653,6 +653,70 @@ void HeapUtil::Unregister() {
   art::Runtime::Current()->RemoveSystemWeakHolder(&gIndexCachingTable);
 }
 
+jvmtiError HeapUtil::IterateOverInstancesOfClass(jvmtiEnv* env,
+                                                 jclass klass,
+                                                 jvmtiHeapObjectFilter filter,
+                                                 jvmtiHeapObjectCallback cb,
+                                                 const void* user_data) {
+  if (cb == nullptr || klass == nullptr) {
+    return ERR(NULL_POINTER);
+  }
+
+  art::Thread* self = art::Thread::Current();
+  art::ScopedObjectAccess soa(self);      // Now we know we have the shared lock.
+  art::StackHandleScope<1> hs(self);
+
+  art::ObjPtr<art::mirror::Object> klass_ptr(soa.Decode<art::mirror::Class>(klass));
+  if (!klass_ptr->IsClass()) {
+    return ERR(INVALID_CLASS);
+  }
+  art::Handle<art::mirror::Class> filter_klass(hs.NewHandle(klass_ptr->AsClass()));
+  if (filter_klass->IsInterface()) {
+    // nothing is an 'instance' of an interface so just return without walking anything.
+    return OK;
+  }
+
+  ObjectTagTable* tag_table = ArtJvmTiEnv::AsArtJvmTiEnv(env)->object_tag_table.get();
+  bool stop_reports = false;
+  auto visitor = [&](art::mirror::Object* obj) REQUIRES_SHARED(art::Locks::mutator_lock_) {
+    // Early return, as we can't really stop visiting.
+    if (stop_reports) {
+      return;
+    }
+
+    art::ScopedAssertNoThreadSuspension no_suspension("IterateOverInstancesOfClass");
+
+    art::ObjPtr<art::mirror::Class> klass = obj->GetClass();
+
+    if (filter_klass != nullptr && !filter_klass->IsAssignableFrom(klass)) {
+      return;
+    }
+
+    jlong tag = 0;
+    tag_table->GetTag(obj, &tag);
+    if ((filter != JVMTI_HEAP_OBJECT_EITHER) &&
+        ((tag == 0 && filter == JVMTI_HEAP_OBJECT_TAGGED) ||
+         (tag != 0 && filter == JVMTI_HEAP_OBJECT_UNTAGGED))) {
+      return;
+    }
+
+    jlong class_tag = 0;
+    tag_table->GetTag(klass.Ptr(), &class_tag);
+
+    jlong saved_tag = tag;
+    jint ret = cb(class_tag, obj->SizeOf(), &tag, const_cast<void*>(user_data));
+
+    stop_reports = (ret == JVMTI_ITERATION_ABORT);
+
+    if (tag != saved_tag) {
+      tag_table->Set(obj, tag);
+    }
+  };
+  art::Runtime::Current()->GetHeap()->VisitObjects(visitor);
+
+  return OK;
+}
+
 template <typename T>
 static jvmtiError DoIterateThroughHeap(T fn,
                                        jvmtiEnv* env,
diff --git a/openjdkjvmti/ti_heap.h b/openjdkjvmti/ti_heap.h
index 62761b500c..382d80f576 100644
--- a/openjdkjvmti/ti_heap.h
+++ b/openjdkjvmti/ti_heap.h
@@ -30,6 +30,12 @@ class HeapUtil {
 
   jvmtiError GetLoadedClasses(jvmtiEnv* env, jint* class_count_ptr, jclass** classes_ptr);
 
+  jvmtiError IterateOverInstancesOfClass(jvmtiEnv* env,
+                                         jclass klass,
+                                         jvmtiHeapObjectFilter filter,
+                                         jvmtiHeapObjectCallback cb,
+                                         const void* user_data);
+
   jvmtiError IterateThroughHeap(jvmtiEnv* env,
                                 jint heap_filter,
                                 jclass klass,
diff --git a/test/906-iterate-heap/iterate_heap.cc b/test/906-iterate-heap/iterate_heap.cc
index 2a06a7b9d6..521f9a6c72 100644
--- a/test/906-iterate-heap/iterate_heap.cc
+++ b/test/906-iterate-heap/iterate_heap.cc
@@ -418,5 +418,21 @@ extern "C" JNIEXPORT jboolean JNICALL Java_art_Test906_checkInitialized(
   return (status & JVMTI_CLASS_STATUS_INITIALIZED) != 0;
 }
 
+extern "C" JNIEXPORT jint JNICALL Java_art_Test906_iterateOverInstancesCount(
+    JNIEnv* env, jclass, jclass target) {
+  jint cnt = 0;
+  auto count_func = [](jlong, jlong, jlong*, void* user_data) -> jvmtiIterationControl {
+    *reinterpret_cast<jint*>(user_data) += 1;
+    return JVMTI_ITERATION_CONTINUE;
+  };
+  JvmtiErrorToException(env,
+                        jvmti_env,
+                        jvmti_env->IterateOverInstancesOfClass(target,
+                                                               JVMTI_HEAP_OBJECT_EITHER,
+                                                               count_func,
+                                                               &cnt));
+  return cnt;
+}
+
 }  // namespace Test906IterateHeap
 }  // namespace art
diff --git a/test/906-iterate-heap/src/art/Test906.java b/test/906-iterate-heap/src/art/Test906.java
index be9663a6d4..190f36f5d4 100644
--- a/test/906-iterate-heap/src/art/Test906.java
+++ b/test/906-iterate-heap/src/art/Test906.java
@@ -18,6 +18,7 @@ package art;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Random;
 
 public class Test906 {
   public static void run() throws Exception {
@@ -69,6 +70,40 @@ public class Test906 {
     throw lastThrow;
   }
 
+  private static Object[] GenTs(Class<?> k) throws Exception {
+    Object[] ret = new Object[new Random().nextInt(100) + 10];
+    for (int i = 0; i < ret.length; i++) {
+      ret[i] = k.newInstance();
+    }
+    return ret;
+  }
+
+  private static void checkEq(int a, int b) {
+    if (a != b) {
+      Error e = new Error("Failed: Expected equal " + a + " and " + b);
+      System.out.println(e);
+      e.printStackTrace(System.out);
+    }
+  }
+
+  public static class Foo {}
+  public static class Bar extends Foo {}
+  public static class Baz extends Bar {}
+  public static class Alpha extends Bar {}
+  public static class MISSING extends Baz {}
+  private static void testIterateOverInstances() throws Exception {
+    Object[] foos = GenTs(Foo.class);
+    Object[] bars = GenTs(Bar.class);
+    Object[] bazs = GenTs(Baz.class);
+    Object[] alphas = GenTs(Alpha.class);
+    checkEq(0, iterateOverInstancesCount(MISSING.class));
+    checkEq(alphas.length, iterateOverInstancesCount(Alpha.class));
+    checkEq(bazs.length, iterateOverInstancesCount(Baz.class));
+    checkEq(bazs.length + alphas.length + bars.length, iterateOverInstancesCount(Bar.class));
+    checkEq(bazs.length + alphas.length + bars.length + foos.length,
+        iterateOverInstancesCount(Foo.class));
+  }
+
   public static void doTest() throws Exception {
     A a = new A();
     B b = new B();
@@ -86,6 +121,8 @@ public class Test906 {
 
     testHeapCount();
 
+    testIterateOverInstances();
+
     long classTags[] = new long[100];
     long sizes[] = new long[100];
     long tags[] = new long[100];
@@ -308,6 +345,8 @@ public class Test906 {
     return Main.getTag(o);
   }
 
+  private static native int iterateOverInstancesCount(Class<?> klass);
+
   private static native boolean checkInitialized(Class<?> klass);
   private static native int iterateThroughHeapCount(int heapFilter,
       Class<?> klassFilter, int stopAfter);
diff --git a/tools/field-null-percent/Android.bp b/tools/field-null-percent/Android.bp
new file mode 100644
index 0000000000..26bb1dc437
--- /dev/null
+++ b/tools/field-null-percent/Android.bp
@@ -0,0 +1,56 @@
+//
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Build variants {target,host} x {debug,ndebug} x {32,64}
+cc_defaults {
+    name: "fieldnull-defaults",
+    host_supported: true,
+    srcs: ["fieldnull.cc"],
+    defaults: ["art_defaults"],
+
+    // Note that this tool needs to be built for both 32-bit and 64-bit since it requires
+    // to be same ISA as what it is attached to.
+    compile_multilib: "both",
+
+    shared_libs: [
+        "libbase",
+    ],
+    header_libs: [
+        "libopenjdkjvmti_headers",
+    ],
+    multilib: {
+        lib32: {
+            suffix: "32",
+        },
+        lib64: {
+            suffix: "64",
+        },
+    },
+    symlink_preferred_arch: true,
+}
+
+art_cc_library {
+    name: "libfieldnull",
+    defaults: ["fieldnull-defaults"],
+}
+
+art_cc_library {
+    name: "libfieldnulld",
+    defaults: [
+        "art_debug_defaults",
+        "fieldnull-defaults",
+    ],
+}
diff --git a/tools/field-null-percent/README.md b/tools/field-null-percent/README.md
new file mode 100644
index 0000000000..d8bc65d9cc
--- /dev/null
+++ b/tools/field-null-percent/README.md
@@ -0,0 +1,51 @@
+# fieldnull
+
+fieldnull is a JVMTI agent designed for testing for a given field the number of
+instances with that field set to null. This can be useful for determining what
+fields should be moved into side structures in cases where memory use is
+important.
+
+# Usage
+### Build
+>    `make libfieldnull`
+
+The libraries will be built for 32-bit, 64-bit, host and target. Below examples
+assume you want to use the 64-bit version.
+
+### Command Line
+
+The agent is loaded using -agentpath like normal. It takes arguments in the
+following format:
+>     `Lname/of/class;.nameOfField:Ltype/of/field;[,...]`
+
+#### ART
+>    `art -Xplugin:$ANDROID_HOST_OUT/lib64/libopenjdkjvmti.so '-agentpath:libfieldnull.so=Lname/of/class;.nameOfField:Ltype/of/field;' -cp tmp/java/helloworld.dex -Xint helloworld`
+
+* `-Xplugin` and `-agentpath` need to be used, otherwise the agent will fail during init.
+* If using `libartd.so`, make sure to use the debug version of jvmti.
+
+>    `adb shell setenforce 0`
+>
+>    `adb push $ANDROID_PRODUCT_OUT/system/lib64/libfieldnull.so /data/local/tmp/`
+>
+>    `adb shell am start-activity --attach-agent '/data/local/tmp/libfieldnull.so=Ljava/lang/Class;.name:Ljava/lang/String;' some.debuggable.apps/.the.app.MainActivity`
+
+#### RI
+>    `java '-agentpath:libfieldnull.so=Lname/of/class;.nameOfField:Ltype/of/field;' -cp tmp/helloworld/classes helloworld`
+
+### Printing the Results
+All statistics gathered during the trace are printed automatically when the
+program normally exits. In the case of Android applications, they are always
+killed, so we need to manually print the results.
+
+>    `kill -SIGQUIT $(pid com.littleinc.orm_benchmark)`
+
+Will initiate a dump of the counts (to logcat).
+
+The dump will look something like this.
+
+> `dalvikvm32 I 08-30 14:51:20 84818 84818 fieldnull.cc:96] Dumping counts of null fields.`
+>
+> `dalvikvm32 I 08-30 14:51:20 84818 84818 fieldnull.cc:97] 	Field name	null count	total count`
+>
+> `dalvikvm32 I 08-30 14:51:20 84818 84818 fieldnull.cc:135] 	Ljava/lang/Class;.name:Ljava/lang/String;	5	2936`
diff --git a/tools/field-null-percent/fieldnull.cc b/tools/field-null-percent/fieldnull.cc
new file mode 100644
index 0000000000..86459d238b
--- /dev/null
+++ b/tools/field-null-percent/fieldnull.cc
@@ -0,0 +1,218 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <android-base/logging.h>
+
+#include <atomic>
+#include <iomanip>
+#include <iostream>
+#include <istream>
+#include <jni.h>
+#include <jvmti.h>
+#include <memory>
+#include <sstream>
+#include <string.h>
+#include <string>
+#include <vector>
+
+namespace fieldnull {
+
+#define CHECK_JVMTI(x) CHECK_EQ((x), JVMTI_ERROR_NONE)
+
+// Special art ti-version number. We will use this as a fallback if we cannot get a regular JVMTI
+// env.
+static constexpr jint kArtTiVersion = JVMTI_VERSION_1_2 | 0x40000000;
+
+static JavaVM* java_vm = nullptr;
+
+// Field is "Lclass/name/here;.field_name:Lfield/type/here;"
+static std::pair<jclass, jfieldID> SplitField(JNIEnv* env, const std::string& field_id) {
+  CHECK_EQ(field_id[0], 'L');
+  env->PushLocalFrame(1);
+  std::istringstream is(field_id);
+  std::string class_name;
+  std::string field_name;
+  std::string field_type;
+
+  std::getline(is, class_name, '.');
+  std::getline(is, field_name, ':');
+  std::getline(is, field_type, '\0');
+
+  jclass klass = reinterpret_cast<jclass>(
+      env->NewGlobalRef(env->FindClass(class_name.substr(1, class_name.size() - 2).c_str())));
+  jfieldID field = env->GetFieldID(klass, field_name.c_str(), field_type.c_str());
+  CHECK(klass != nullptr);
+  CHECK(field != nullptr);
+  LOG(INFO) << "listing field " << field_id;
+  env->PopLocalFrame(nullptr);
+  return std::make_pair(klass, field);
+}
+
+static std::vector<std::pair<jclass, jfieldID>> GetRequestedFields(JNIEnv* env,
+                                                                   const std::string& args) {
+  std::vector<std::pair<jclass, jfieldID>> res;
+  std::stringstream args_stream(args);
+  std::string item;
+  while (std::getline(args_stream, item, ',')) {
+    if (item == "") {
+      continue;
+    }
+    res.push_back(SplitField(env, item));
+  }
+  return res;
+}
+
+static jint SetupJvmtiEnv(JavaVM* vm, jvmtiEnv** jvmti) {
+  jint res = 0;
+  res = vm->GetEnv(reinterpret_cast<void**>(jvmti), JVMTI_VERSION_1_1);
+
+  if (res != JNI_OK || *jvmti == nullptr) {
+    LOG(ERROR) << "Unable to access JVMTI, error code " << res;
+    return vm->GetEnv(reinterpret_cast<void**>(jvmti), kArtTiVersion);
+  }
+  return res;
+}
+
+struct RequestList {
+  std::vector<std::pair<jclass, jfieldID>> fields_;
+};
+
+static void DataDumpRequestCb(jvmtiEnv* jvmti) {
+  JNIEnv* env = nullptr;
+  CHECK_EQ(java_vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6), JNI_OK);
+  LOG(INFO) << "Dumping counts of null fields.";
+  LOG(INFO) << "\t" << "Field name"
+            << "\t" << "null count"
+            << "\t" << "total count";
+  RequestList* list;
+  CHECK_JVMTI(jvmti->GetEnvironmentLocalStorage(reinterpret_cast<void**>(&list)));
+  for (std::pair<jclass, jfieldID>& p : list->fields_) {
+    jclass klass = p.first;
+    jfieldID field = p.second;
+    // Make sure all instances of the class are tagged with the klass ptr value. Since this is a
+    // global ref it's guaranteed to be unique.
+    CHECK_JVMTI(jvmti->IterateOverInstancesOfClass(
+        p.first,
+        // We need to do this to all objects every time since we might be looking for multiple
+        // fields in classes that are subtypes of each other.
+        JVMTI_HEAP_OBJECT_EITHER,
+        /* class_tag, size, tag_ptr, user_data*/
+        [](jlong, jlong, jlong* tag_ptr, void* klass) -> jvmtiIterationControl {
+          *tag_ptr = static_cast<jlong>(reinterpret_cast<intptr_t>(klass));
+          return JVMTI_ITERATION_CONTINUE;
+        },
+        klass));
+    jobject* obj_list;
+    jint obj_len;
+    jlong tag = static_cast<jlong>(reinterpret_cast<intptr_t>(klass));
+    CHECK_JVMTI(jvmti->GetObjectsWithTags(1, &tag, &obj_len, &obj_list, nullptr));
+
+    uint64_t null_cnt = 0;
+    for (jint i = 0; i < obj_len; i++) {
+      if (env->GetObjectField(obj_list[i], field) == nullptr) {
+        null_cnt++;
+      }
+    }
+
+    char* field_name;
+    char* field_sig;
+    char* class_name;
+    CHECK_JVMTI(jvmti->GetFieldName(klass, field, &field_name, &field_sig, nullptr));
+    CHECK_JVMTI(jvmti->GetClassSignature(klass, &class_name, nullptr));
+    LOG(INFO) << "\t" << class_name << "." << field_name << ":" << field_sig
+              << "\t" << null_cnt
+              << "\t" << obj_len;
+    CHECK_JVMTI(jvmti->Deallocate(reinterpret_cast<unsigned char*>(field_name)));
+    CHECK_JVMTI(jvmti->Deallocate(reinterpret_cast<unsigned char*>(field_sig)));
+    CHECK_JVMTI(jvmti->Deallocate(reinterpret_cast<unsigned char*>(class_name)));
+  }
+}
+
+static void VMDeathCb(jvmtiEnv* jvmti, JNIEnv* env ATTRIBUTE_UNUSED) {
+  DataDumpRequestCb(jvmti);
+  RequestList* list = nullptr;
+  CHECK_JVMTI(jvmti->GetEnvironmentLocalStorage(reinterpret_cast<void**>(&list)));
+  delete list;
+}
+
+static void CreateFieldList(jvmtiEnv* jvmti, JNIEnv* env, std::string args) {
+  RequestList* list = nullptr;
+  CHECK_JVMTI(jvmti->Allocate(sizeof(*list), reinterpret_cast<unsigned char**>(&list)));
+  new (list) RequestList { .fields_ = GetRequestedFields(env, args), };
+  CHECK_JVMTI(jvmti->SetEnvironmentLocalStorage(list));
+}
+
+static void VMInitCb(jvmtiEnv* jvmti, JNIEnv* env, jobject thr ATTRIBUTE_UNUSED) {
+  char* args = nullptr;
+  CHECK_JVMTI(jvmti->GetEnvironmentLocalStorage(reinterpret_cast<void**>(&args)));
+  CHECK_JVMTI(jvmti->SetEnvironmentLocalStorage(nullptr));
+  CreateFieldList(jvmti, env, args);
+  CHECK_JVMTI(jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VM_DEATH, nullptr));
+  CHECK_JVMTI(jvmti->SetEventNotificationMode(JVMTI_ENABLE,
+                                              JVMTI_EVENT_DATA_DUMP_REQUEST,
+                                              nullptr));
+  CHECK_JVMTI(jvmti->Deallocate(reinterpret_cast<unsigned char*>(args)));
+}
+
+static jint AgentStart(JavaVM* vm, char* options, bool is_onload) {
+  android::base::InitLogging(/* argv */nullptr);
+  java_vm = vm;
+  jvmtiEnv* jvmti = nullptr;
+  if (SetupJvmtiEnv(vm, &jvmti) != JNI_OK) {
+    LOG(ERROR) << "Could not get JVMTI env or ArtTiEnv!";
+    return JNI_ERR;
+  }
+  jvmtiCapabilities caps { .can_tag_objects = 1, };
+  CHECK_JVMTI(jvmti->AddCapabilities(&caps));
+  jvmtiEventCallbacks cb {
+    .VMInit = VMInitCb,
+    .DataDumpRequest = DataDumpRequestCb,
+    .VMDeath = VMDeathCb,
+  };
+  CHECK_JVMTI(jvmti->SetEventCallbacks(&cb, sizeof(cb)));
+  if (is_onload) {
+    unsigned char* ptr = nullptr;
+    CHECK_JVMTI(jvmti->Allocate(strlen(options) + 1, &ptr));
+    strcpy(reinterpret_cast<char*>(ptr), options);
+    CHECK_JVMTI(jvmti->SetEnvironmentLocalStorage(ptr));
+    CHECK_JVMTI(jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VM_INIT, nullptr));
+  } else {
+    JNIEnv* env = nullptr;
+    CHECK_EQ(vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6), JNI_OK);
+    CreateFieldList(jvmti, env, options);
+    CHECK_JVMTI(jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VM_DEATH, nullptr));
+    CHECK_JVMTI(jvmti->SetEventNotificationMode(JVMTI_ENABLE,
+                                                JVMTI_EVENT_DATA_DUMP_REQUEST,
+                                                nullptr));
+  }
+  return JNI_OK;
+}
+
+// Late attachment (e.g. 'am attach-agent').
+extern "C" JNIEXPORT jint JNICALL Agent_OnAttach(JavaVM *vm,
+                                                 char* options,
+                                                 void* reserved ATTRIBUTE_UNUSED) {
+  return AgentStart(vm, options, /*is_onload*/false);
+}
+
+// Early attachment
+extern "C" JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM* jvm,
+                                               char* options,
+                                               void* reserved ATTRIBUTE_UNUSED) {
+  return AgentStart(jvm, options, /*is_onload*/true);
+}
+
+}  // namespace fieldnull
+