Report any non-app signatures.

Before, veridex only reported signatures that were defined in the CSV file. This CL still preserves the same use case by default, but also allows reporting any other signatures as "invalid" if they are not defined in the --dex-files.

Test: manual
Change-Id: I10753d05bf9f418d2c8543be16cdfb335be9044c

8 files changed, 139 insertions, 62 deletions

diff --git a/libartbase/base/hiddenapi_flags.h b/libartbase/base/hiddenapi_flags.h
index 3f498bfec2..3b85e38cf8 100644
--- a/libartbase/base/hiddenapi_flags.h
+++ b/libartbase/base/hiddenapi_flags.h
@@ -295,6 +295,11 @@ class ApiList {
   void Dump(std::ostream& os) const {
     bool is_first = true;
 
+    if (IsEmpty()) {
+      os << "invalid";
+      return;
+    }
+
     if (GetValue() != Value::kInvalid) {
       os << kValueNames[GetIntValue()];
       is_first = false;
@@ -315,8 +320,12 @@ class ApiList {
     DCHECK_EQ(IsEmpty(), is_first);
   }
 
+  // Number of valid enum values in Value.
   static constexpr uint32_t kValueCount = helper::NumValues<Value>();
+  // Number of valid enum values in DomainApi.
   static constexpr uint32_t kDomainApiCount = helper::NumValues<DomainApi>();
+  // Total number of possible enum values, including invalid, in Value.
+  static constexpr uint32_t kValueSize = (1u << kValueBitSize) + 1;
 
   // Check min and max values are calculated correctly.
   static_assert(Value::kMin == helper::GetEnumAt<Value>(0));
diff --git a/tools/veridex/api_list_filter.h b/tools/veridex/api_list_filter.h
index 29929f671c..58065db2b2 100644
--- a/tools/veridex/api_list_filter.h
+++ b/tools/veridex/api_list_filter.h
@@ -28,7 +28,15 @@ class ApiListFilter {
  public:
   explicit ApiListFilter(const std::vector<std::string>& exclude_api_lists) {
     std::set<hiddenapi::ApiList> exclude_set;
+    bool include_invalid_list = true;
     for (const std::string& name : exclude_api_lists) {
+      if (name.empty()) {
+        continue;
+      }
+      if (name == "invalid") {
+        include_invalid_list = false;
+        continue;
+      }
       hiddenapi::ApiList list = hiddenapi::ApiList::FromName(name);
       if (!list.IsValid()) {
         LOG(ERROR) << "Unknown ApiList::Value " << name
@@ -37,6 +45,9 @@ class ApiListFilter {
       exclude_set.insert(list);
     }
 
+    if (include_invalid_list) {
+      lists_.push_back(hiddenapi::ApiList());
+    }
     for (size_t i = 0; i < hiddenapi::ApiList::kValueCount; ++i) {
       hiddenapi::ApiList list = hiddenapi::ApiList(i);
       if (exclude_set.find(list) == exclude_set.end()) {
diff --git a/tools/veridex/appcompat.sh b/tools/veridex/appcompat.sh
index f8e935ed93..ce90c06491 100755
--- a/tools/veridex/appcompat.sh
+++ b/tools/veridex/appcompat.sh
@@ -28,7 +28,7 @@ if [[ -e ${SCRIPT_DIR}/veridex && \
   exec ${SCRIPT_DIR}/veridex \
     --core-stubs=${SCRIPT_DIR}/system-stubs.zip:${SCRIPT_DIR}/org.apache.http.legacy-stubs.zip \
     --api-flags=${SCRIPT_DIR}/hiddenapi-flags.csv \
-    --exclude-api-lists=whitelist \
+    --exclude-api-lists=whitelist,invalid \
     $@
 fi
 
@@ -72,7 +72,7 @@ fi
 
 # If --exclude-api-lists is not passed directly, exclude whitelist APIs.
 if [[ "$@" != "*--exclude-api-lists=*" ]]; then
-  extra_flags="${extra_flags} --exclude-api-lists=whitelist"
+  extra_flags="${extra_flags} --exclude-api-lists=whitelist,invalid"
 fi
 
 ${ANDROID_HOST_OUT}/bin/veridex \
diff --git a/tools/veridex/hidden_api.cc b/tools/veridex/hidden_api.cc
index b21493ff8c..71ea56b56f 100644
--- a/tools/veridex/hidden_api.cc
+++ b/tools/veridex/hidden_api.cc
@@ -24,7 +24,8 @@
 
 namespace art {
 
-HiddenApi::HiddenApi(const char* filename, const ApiListFilter& api_list_filter) {
+HiddenApi::HiddenApi(const char* filename, const ApiListFilter& api_list_filter)
+    : api_list_filter_(api_list_filter) {
   CHECK(filename != nullptr);
 
   std::ifstream in(filename);
@@ -37,10 +38,6 @@ HiddenApi::HiddenApi(const char* filename, const ApiListFilter& api_list_filter)
     CHECK(success) << "Unknown ApiList flag: " << str;
     CHECK(membership.IsValid()) << "Invalid ApiList: " << membership;
 
-    if (!api_list_filter.Matches(membership)) {
-      continue;
-    }
-
     AddSignatureToApiList(signature, membership);
     size_t pos = signature.find("->");
     if (pos != std::string::npos) {
diff --git a/tools/veridex/hidden_api.h b/tools/veridex/hidden_api.h
index 3ac0125742..a8301743aa 100644
--- a/tools/veridex/hidden_api.h
+++ b/tools/veridex/hidden_api.h
@@ -30,6 +30,12 @@ namespace art {
 
 class DexFile;
 
+enum class SignatureSource {
+  UNKNOWN,
+  BOOT,
+  APP,
+};
+
 /**
  * Helper class for logging if a method/field is in a hidden API list.
  */
@@ -42,8 +48,33 @@ class HiddenApi {
     return (it == api_list_.end()) ? hiddenapi::ApiList() : it->second;
   }
 
-  bool IsInAnyList(const std::string& name) const {
-    return !GetApiList(name).IsEmpty();
+  bool ShouldReport(const std::string& signature) const {
+    return api_list_filter_.Matches(GetApiList(signature));
+  }
+
+  void AddSignatureSource(const std::string &signature, SignatureSource source) {
+    const auto type = GetApiClassName(signature);
+    auto it = source_.find(type);
+    if (it == source_.end() || it->second == SignatureSource::UNKNOWN) {
+      source_[type] = source;
+    } else if (it->second != source) {
+      LOG(WARNING) << type << "is present both in boot and in app.";
+      if (source == SignatureSource::BOOT) {
+        // Runtime resolves to boot type, so it takes precedence.
+        it->second = source;
+      }
+    } else {
+      // Already exists with the same source.
+    }
+  }
+
+  SignatureSource GetSignatureSource(const std::string& signature) const {
+    auto it = source_.find(GetApiClassName(signature));
+    return (it == source_.end()) ? SignatureSource::UNKNOWN : it->second;
+  }
+
+  bool IsInBoot(const std::string& signature) const {
+    return SignatureSource::BOOT == GetSignatureSource(signature);
   }
 
   static std::string GetApiMethodName(const DexFile& dex_file, uint32_t method_index);
@@ -63,16 +94,28 @@ class HiddenApi {
  private:
   void AddSignatureToApiList(const std::string& signature, hiddenapi::ApiList membership);
 
+  static std::string GetApiClassName(const std::string& signature) {
+    size_t pos = signature.find("->");
+    if (pos != std::string::npos) {
+      return signature.substr(0, pos);
+    }
+    return signature;
+  }
+
+  const ApiListFilter& api_list_filter_;
   std::map<std::string, hiddenapi::ApiList> api_list_;
+  std::map<std::string, SignatureSource> source_;
 };
 
 struct HiddenApiStats {
   uint32_t count = 0;
   uint32_t reflection_count = 0;
   uint32_t linking_count = 0;
-  uint32_t api_counts[hiddenapi::ApiList::kValueCount] = {};  // initialize all to zero
+  // Ensure enough space for kInvalid as well, and initialize all to zero
+  uint32_t api_counts[hiddenapi::ApiList::kValueSize] = {};
 };
 
+
 }  // namespace art
 
 #endif  // ART_TOOLS_VERIDEX_HIDDEN_API_H_
diff --git a/tools/veridex/hidden_api_finder.cc b/tools/veridex/hidden_api_finder.cc
index 3ab911e8d3..c6c1c073d8 100644
--- a/tools/veridex/hidden_api_finder.cc
+++ b/tools/veridex/hidden_api_finder.cc
@@ -32,23 +32,19 @@ namespace art {
 void HiddenApiFinder::CheckMethod(uint32_t method_id,
                                   VeridexResolver* resolver,
                                   MethodReference ref) {
-  // Note: we always query whether a method is in a list, as the app
+  // Note: we always query whether a method is in boot, as the app
   // might define blacklisted APIs (which won't be used at runtime).
-  std::string name = HiddenApi::GetApiMethodName(resolver->GetDexFile(), method_id);
-  if (hidden_api_.IsInAnyList(name)) {
-    method_locations_[name].push_back(ref);
-  }
+  const auto& name = HiddenApi::GetApiMethodName(resolver->GetDexFile(), method_id);
+  method_locations_[name].push_back(ref);
 }
 
 void HiddenApiFinder::CheckField(uint32_t field_id,
                                  VeridexResolver* resolver,
                                  MethodReference ref) {
-  // Note: we always query whether a field is in a list, as the app
+  // Note: we always query whether a field is in a boot, as the app
   // might define blacklisted APIs (which won't be used at runtime).
-  std::string name = HiddenApi::GetApiFieldName(resolver->GetDexFile(), field_id);
-  if (hidden_api_.IsInAnyList(name)) {
-    field_locations_[name].push_back(ref);
-  }
+  const auto& name = HiddenApi::GetApiFieldName(resolver->GetDexFile(), field_id);
+  field_locations_[name].push_back(ref);
 }
 
 void HiddenApiFinder::CollectAccesses(VeridexResolver* resolver,
@@ -58,9 +54,7 @@ void HiddenApiFinder::CollectAccesses(VeridexResolver* resolver,
   // types can lead to being used through reflection.
   for (uint32_t i = 0; i < dex_file.NumTypeIds(); ++i) {
     std::string name(dex_file.StringByTypeIdx(dex::TypeIndex(i)));
-    if (hidden_api_.IsInAnyList(name)) {
-      classes_.insert(name);
-    }
+    classes_.insert(name);
   }
   // Note: we collect strings constants only referenced in code items as the string table
   // contains other kind of strings (eg types).
@@ -71,7 +65,7 @@ void HiddenApiFinder::CollectAccesses(VeridexResolver* resolver,
           switch (inst->Opcode()) {
             case Instruction::CONST_STRING: {
               dex::StringIndex string_index(inst->VRegB_21c());
-              std::string name = std::string(dex_file.StringDataByIdx(string_index));
+              const auto& name = std::string(dex_file.StringDataByIdx(string_index));
               // Cheap filtering on the string literal. We know it cannot be a field/method/class
               // if it contains a space.
               if (name.find(' ') == std::string::npos) {
@@ -83,9 +77,9 @@ void HiddenApiFinder::CollectAccesses(VeridexResolver* resolver,
                 // private methods and fields in them.
                 // We don't add class names to the `strings_` set as we know method/field names
                 // don't have '.' or '/'. All hidden API class names have a '/'.
-                if (hidden_api_.IsInAnyList(str)) {
+                if (hidden_api_.IsInBoot(str)) {
                   classes_.insert(str);
-                } else if (hidden_api_.IsInAnyList(name)) {
+                } else if (hidden_api_.IsInBoot(name)) {
                   // Could be something passed to JNI.
                   classes_.insert(name);
                 } else {
@@ -178,30 +172,36 @@ void HiddenApiFinder::Run(const std::vector<std::unique_ptr<VeridexResolver>>& r
 void HiddenApiFinder::Dump(std::ostream& os,
                            HiddenApiStats* stats,
                            bool dump_reflection) {
-  stats->linking_count = method_locations_.size() + field_locations_.size();
-
   // Dump methods from hidden APIs linked against.
   for (const std::pair<const std::string,
                        std::vector<MethodReference>>& pair : method_locations_) {
-    hiddenapi::ApiList api_list = hidden_api_.GetApiList(pair.first);
-    CHECK(api_list.IsValid());
-    stats->api_counts[api_list.GetIntValue()]++;
-    os << "#" << ++stats->count << ": Linking " << api_list << " " << pair.first << " use(s):";
-    os << std::endl;
-    HiddenApiFinder::DumpReferences(os, pair.second);
-    os << std::endl;
+    const auto& name = pair.first;
+    if (hidden_api_.GetSignatureSource(name) != SignatureSource::APP &&
+        hidden_api_.ShouldReport(name)) {
+      stats->linking_count++;
+      hiddenapi::ApiList api_list = hidden_api_.GetApiList(pair.first);
+      stats->api_counts[api_list.GetIntValue()]++;
+      os << "#" << ++stats->count << ": Linking " << api_list << " " << pair.first << " use(s):";
+      os << std::endl;
+      HiddenApiFinder::DumpReferences(os, pair.second);
+      os << std::endl;
+    }
   }
 
   // Dump fields from hidden APIs linked against.
   for (const std::pair<const std::string,
                        std::vector<MethodReference>>& pair : field_locations_) {
-    hiddenapi::ApiList api_list = hidden_api_.GetApiList(pair.first);
-    CHECK(api_list.IsValid());
-    stats->api_counts[api_list.GetIntValue()]++;
-    os << "#" << ++stats->count << ": Linking " << api_list << " " << pair.first << " use(s):";
-    os << std::endl;
-    HiddenApiFinder::DumpReferences(os, pair.second);
-    os << std::endl;
+    const auto& name = pair.first;
+    if (hidden_api_.GetSignatureSource(name) != SignatureSource::APP &&
+        hidden_api_.ShouldReport(name)) {
+      stats->linking_count++;
+      hiddenapi::ApiList api_list = hidden_api_.GetApiList(pair.first);
+      stats->api_counts[api_list.GetIntValue()]++;
+      os << "#" << ++stats->count << ": Linking " << api_list << " " << pair.first << " use(s):";
+      os << std::endl;
+      HiddenApiFinder::DumpReferences(os, pair.second);
+      os << std::endl;
+    }
   }
 
   if (dump_reflection) {
@@ -209,8 +209,9 @@ void HiddenApiFinder::Dump(std::ostream& os,
     for (const std::string& cls : classes_) {
       for (const std::string& name : strings_) {
         std::string full_name = cls + "->" + name;
-        hiddenapi::ApiList api_list = hidden_api_.GetApiList(full_name);
-        if (api_list.IsValid()) {
+        if (hidden_api_.GetSignatureSource(full_name) != SignatureSource::APP &&
+            hidden_api_.ShouldReport(full_name)) {
+          hiddenapi::ApiList api_list = hidden_api_.GetApiList(full_name);
           stats->api_counts[api_list.GetIntValue()]++;
           stats->reflection_count++;
           os << "#" << ++stats->count << ": Reflection " << api_list << " " << full_name
diff --git a/tools/veridex/precise_hidden_api_finder.cc b/tools/veridex/precise_hidden_api_finder.cc
index 7aa86dc495..6f66a3368c 100644
--- a/tools/veridex/precise_hidden_api_finder.cc
+++ b/tools/veridex/precise_hidden_api_finder.cc
@@ -99,23 +99,24 @@ void PreciseHiddenApiFinder::Dump(std::ostream& os, HiddenApiStats* stats) {
       std::string cls(info.cls.ToString());
       std::string name(info.name.ToString());
       std::string full_name = cls + "->" + name;
-      if (hidden_api_.IsInAnyList(full_name)) {
-        named_uses[full_name].push_back(ref);
-      }
+      named_uses[full_name].push_back(ref);
     }
   }
 
   for (auto& it : named_uses) {
-    ++stats->reflection_count;
     const std::string& full_name = it.first;
-    hiddenapi::ApiList api_list = hidden_api_.GetApiList(full_name);
-    stats->api_counts[api_list.GetIntValue()]++;
-    os << "#" << ++stats->count << ": Reflection " << api_list << " " << full_name << " use(s):";
-    os << std::endl;
-    for (const MethodReference& ref : it.second) {
-      os << kPrefix << HiddenApi::GetApiMethodName(ref) << std::endl;
+    if (hidden_api_.GetSignatureSource(full_name) != SignatureSource::APP &&
+        hidden_api_.ShouldReport(full_name)) {
+      stats->reflection_count++;
+      hiddenapi::ApiList api_list = hidden_api_.GetApiList(full_name);
+      stats->api_counts[api_list.GetIntValue()]++;
+      os << "#" << ++stats->count << ": Reflection " << api_list << " " << full_name << " use(s):";
+      os << std::endl;
+      for (const MethodReference& ref : it.second) {
+        os << kPrefix << HiddenApi::GetApiMethodName(ref) << std::endl;
+      }
+      os << std::endl;
     }
-    os << std::endl;
   }
 }
 
diff --git a/tools/veridex/veridex.cc b/tools/veridex/veridex.cc
index 9079a1fcd7..ae1c33e1d2 100644
--- a/tools/veridex/veridex.cc
+++ b/tools/veridex/veridex.cc
@@ -174,6 +174,9 @@ class Veridex {
 
     // Resolve classes/methods/fields defined in each dex file.
 
+    ApiListFilter api_list_filter(options.exclude_api_lists);
+    HiddenApi hidden_api(options.flags_file, api_list_filter);
+
     // Cache of types we've seen, for quick class name lookups.
     TypeMap type_map;
     // Add internally defined primitives.
@@ -192,6 +195,9 @@ class Veridex {
 
     std::vector<std::unique_ptr<VeridexResolver>> boot_resolvers;
     Resolve(boot_dex_files, resolver_map, type_map, &boot_resolvers);
+    for (const auto &it : type_map) {
+        hidden_api.AddSignatureSource(it.first, SignatureSource::BOOT);
+    }
 
     if (options.precise) {
       // For precise mode we expect core-stubs to contain java.lang classes.
@@ -227,12 +233,15 @@ class Veridex {
 
     std::vector<std::unique_ptr<VeridexResolver>> app_resolvers;
     Resolve(app_dex_files, resolver_map, type_map, &app_resolvers);
+    for (const auto &it : type_map) {
+      if (!hidden_api.IsInBoot(it.first)) {
+        hidden_api.AddSignatureSource(it.first, SignatureSource::APP);
+      }
+    }
 
     ClassFilter app_class_filter(options.app_class_name_filter);
-    ApiListFilter api_list_filter(options.exclude_api_lists);
 
     // Find and log uses of hidden APIs.
-    HiddenApi hidden_api(options.flags_file, api_list_filter);
     HiddenApiStats stats;
 
     HiddenApiFinder api_finder(hidden_api);
@@ -259,15 +268,21 @@ class Veridex {
   static void DumpSummaryStats(std::ostream& os,
                                const HiddenApiStats& stats,
                                const ApiListFilter& api_list_filter) {
-    static const char* kPrefix = "       ";
     os << stats.count << " hidden API(s) used: "
        << stats.linking_count << " linked against, "
        << stats.reflection_count << " through reflection" << std::endl;
+    DumpApiListStats(os, stats, hiddenapi::ApiList(), api_list_filter);
     for (size_t i = 0; i < hiddenapi::ApiList::kValueCount; ++i) {
-      hiddenapi::ApiList api_list = hiddenapi::ApiList(i);
-      if (api_list_filter.Matches(api_list)) {
-        os << kPrefix << stats.api_counts[i] << " in " << api_list << std::endl;
-      }
+      DumpApiListStats(os, stats, hiddenapi::ApiList(i), api_list_filter);
+    }
+  }
+
+  static void DumpApiListStats(std::ostream& os,
+                               const HiddenApiStats& stats,
+                               const hiddenapi::ApiList& api_list,
+                               const ApiListFilter& api_list_filter) {
+    if (api_list_filter.Matches(api_list)) {
+      os << "\t" << stats.api_counts[api_list.GetIntValue()] << " in " << api_list << std::endl;
     }
   }