Support multiple boot image profile aggregations in profman
Update the logic to be able to aggregate and operate on multiple
boot image profiles.
To assist with this change the flatten profile data will store
annotations as a list instead of a set. This way we can keep
track of how popular a method is across multiple profile (the
source packages can now be duplicated, in which case it means
that the same method was used in 2 different profiles by the
same package)
Test: gtests
Bug: 152574358
Change-Id: Iecb6891d0ebbaecc56acde8d8aae08dbfd91f61f
diff --git a/libprofile/profile/profile_compilation_info.cc b/libprofile/profile/profile_compilation_info.cc
index 010949a..24419ef 100644
--- a/libprofile/profile/profile_compilation_info.cc
+++ b/libprofile/profile/profile_compilation_info.cc
@@ -2304,7 +2304,7 @@
std::unique_ptr<FlattenProfileData> result(new FlattenProfileData());
- auto createMetadataFn = []() { return FlattenProfileData::ItemMetadata(); };
+ auto create_metadata_fn = []() { return FlattenProfileData::ItemMetadata(); };
// Iterate through all the dex files, find the methods/classes associated with each of them,
// and add them to the flatten result.
@@ -2328,9 +2328,9 @@
// The method is in the profile, create metadata item for it and added to the result.
MethodReference ref(dex_file.get(), method_idx);
FlattenProfileData::ItemMetadata& metadata =
- result->method_metadata_.GetOrCreate(ref, createMetadataFn);
+ result->method_metadata_.GetOrCreate(ref, create_metadata_fn);
metadata.flags_ |= hotness.flags_;
- metadata.annotations_.insert(annotation);
+ metadata.annotations_.push_back(annotation);
// Update the max aggregation counter for methods.
// This is essentially a cache, to avoid traversing all the methods just to find out
// this value.
@@ -2343,8 +2343,8 @@
for (const dex::TypeIndex& type_index : dex_data->class_set) {
TypeReference ref(dex_file.get(), type_index);
FlattenProfileData::ItemMetadata& metadata =
- result->class_metadata_.GetOrCreate(ref, createMetadataFn);
- metadata.annotations_.insert(annotation);
+ result->class_metadata_.GetOrCreate(ref, create_metadata_fn);
+ metadata.annotations_.push_back(annotation);
// Update the max aggregation counter for classes.
result->max_aggregation_for_classes_ = std::max(
result->max_aggregation_for_classes_,
@@ -2356,4 +2356,40 @@
return result;
}
+void FlattenProfileData::MergeData(const FlattenProfileData& other) {
+ auto create_metadata_fn = []() { return FlattenProfileData::ItemMetadata(); };
+ for (const auto& it : other.method_metadata_) {
+ const MethodReference& otherRef = it.first;
+ const FlattenProfileData::ItemMetadata otherData = it.second;
+ const std::list<ProfileCompilationInfo::ProfileSampleAnnotation>& other_annotations =
+ otherData.GetAnnotations();
+
+ FlattenProfileData::ItemMetadata& metadata =
+ method_metadata_.GetOrCreate(otherRef, create_metadata_fn);
+ metadata.flags_ |= otherData.GetFlags();
+ metadata.annotations_.insert(
+ metadata.annotations_.end(), other_annotations.begin(), other_annotations.end());
+
+ max_aggregation_for_methods_ = std::max(
+ max_aggregation_for_methods_,
+ static_cast<uint32_t>(metadata.annotations_.size()));
+ }
+ for (const auto& it : other.class_metadata_) {
+ const TypeReference& otherRef = it.first;
+ const FlattenProfileData::ItemMetadata otherData = it.second;
+ const std::list<ProfileCompilationInfo::ProfileSampleAnnotation>& other_annotations =
+ otherData.GetAnnotations();
+
+ FlattenProfileData::ItemMetadata& metadata =
+ class_metadata_.GetOrCreate(otherRef, create_metadata_fn);
+ metadata.flags_ |= otherData.GetFlags();
+ metadata.annotations_.insert(
+ metadata.annotations_.end(), other_annotations.begin(), other_annotations.end());
+
+ max_aggregation_for_classes_ = std::max(
+ max_aggregation_for_classes_,
+ static_cast<uint32_t>(metadata.annotations_.size()));
+ }
+}
+
} // namespace art
diff --git a/libprofile/profile/profile_compilation_info.h b/libprofile/profile/profile_compilation_info.h
index f0dc9b4..0b0c423 100644
--- a/libprofile/profile/profile_compilation_info.h
+++ b/libprofile/profile/profile_compilation_info.h
@@ -989,7 +989,7 @@
return flags_;
}
- const std::set<ProfileCompilationInfo::ProfileSampleAnnotation>& GetAnnotations() const {
+ const std::list<ProfileCompilationInfo::ProfileSampleAnnotation>& GetAnnotations() const {
return annotations_;
}
@@ -1004,9 +1004,12 @@
private:
// will be 0 for classes and MethodHotness::Flags for methods.
uint16_t flags_;
- std::set<ProfileCompilationInfo::ProfileSampleAnnotation> annotations_;
+ // This is a list that may contain duplicates after a merge operation.
+ // It represents that a method was used multiple times across different devices.
+ std::list<ProfileCompilationInfo::ProfileSampleAnnotation> annotations_;
friend class ProfileCompilationInfo;
+ friend class FlattenProfileData;
};
FlattenProfileData();
@@ -1027,6 +1030,8 @@
return max_aggregation_for_classes_;
}
+ void MergeData(const FlattenProfileData& other);
+
private:
// Method data.
SafeMap<MethodReference, ItemMetadata> method_metadata_;
diff --git a/libprofile/profile/profile_compilation_info_test.cc b/libprofile/profile/profile_compilation_info_test.cc
index 81d0cc9..d6ae8a2 100644
--- a/libprofile/profile/profile_compilation_info_test.cc
+++ b/libprofile/profile/profile_compilation_info_test.cc
@@ -1708,7 +1708,14 @@
ASSERT_FALSE(info.IsForBootImage());
}
-// Verify we can merge samples with annotations.
+template<class T>
+static std::list<T> sort(const std::list<T>& list) {
+ std::list<T> copy(list);
+ copy.sort();
+ return copy;
+}
+
+// Verify we can extract profile data
TEST_F(ProfileCompilationInfoTest, ExtractProfileData) {
// Setup test data
ProfileCompilationInfo info;
@@ -1744,24 +1751,91 @@
ASSERT_EQ(methods.size(), 20u); // 10 methods in dex1, 10 in dex2
ASSERT_EQ(classes.size(), 10u); // 10 methods in dex1
- std::set<ProfileSampleAnnotation> expectedAnnotations1({psa1, psa2});
- std::set<ProfileSampleAnnotation> expectedAnnotations2({psa2});
+ std::list<ProfileSampleAnnotation> expectedAnnotations1({psa1, psa2});
+ std::list<ProfileSampleAnnotation> expectedAnnotations2({psa2});
for (uint16_t i = 0; i < 10; i++) {
// Check dex1 methods.
auto mIt1 = methods.find(MethodReference(dex1, i));
ASSERT_TRUE(mIt1 != methods.end());
ASSERT_EQ(mIt1->second.GetFlags(), Hotness::kFlagHot | Hotness::kFlagStartup);
- ASSERT_EQ(mIt1->second.GetAnnotations(), expectedAnnotations1);
+ ASSERT_EQ(sort(mIt1->second.GetAnnotations()), expectedAnnotations1);
// Check dex1 classes
auto cIt1 = classes.find(TypeReference(dex1, dex::TypeIndex(i)));
ASSERT_TRUE(cIt1 != classes.end());
ASSERT_EQ(cIt1->second.GetFlags(), 0);
- ASSERT_EQ(cIt1->second.GetAnnotations(), expectedAnnotations1);
+ ASSERT_EQ(sort(cIt1->second.GetAnnotations()), expectedAnnotations1);
// Check dex2 methods.
auto mIt2 = methods.find(MethodReference(dex2, i));
ASSERT_TRUE(mIt2 != methods.end());
ASSERT_EQ(mIt2->second.GetFlags(), Hotness::kFlagHot);
- ASSERT_EQ(mIt2->second.GetAnnotations(), expectedAnnotations2);
+ ASSERT_EQ(sort(mIt2->second.GetAnnotations()), expectedAnnotations2);
+ }
+
+ // Release the ownership as this is held by the test class;
+ for (std::unique_ptr<const DexFile>& dex : dex_files) {
+ UNUSED(dex.release());
+ }
+}
+
+// Verify we can merge 2 previously flatten data.
+TEST_F(ProfileCompilationInfoTest, MergeFlattenData) {
+ // Setup test data: two profiles with different content which will be used
+ // to extract FlattenProfileData, later to be merged.
+ ProfileCompilationInfo info1;
+ ProfileCompilationInfo info2;
+
+ ProfileSampleAnnotation psa1("test1");
+ ProfileSampleAnnotation psa2("test2");
+
+ for (uint16_t i = 0; i < 10; i++) {
+ // Add dex1 data with different annotations so that we can check the annotation count.
+ ASSERT_TRUE(AddMethod(&info1, dex1, /* method_idx= */ i, Hotness::kFlagHot, psa1));
+ ASSERT_TRUE(AddClass(&info2, dex1, dex::TypeIndex(i), psa1));
+ ASSERT_TRUE(AddMethod(&info1, dex1, /* method_idx= */ i, Hotness::kFlagStartup, psa2));
+ ASSERT_TRUE(AddClass(&info1, dex1, dex::TypeIndex(i), psa2));
+ ASSERT_TRUE(AddMethod(i % 2 == 0 ? &info1 : &info2, dex2,
+ /* method_idx= */ i,
+ Hotness::kFlagHot,
+ psa2));
+ }
+
+ std::vector<std::unique_ptr<const DexFile>> dex_files;
+ dex_files.push_back(std::unique_ptr<const DexFile>(dex1));
+ dex_files.push_back(std::unique_ptr<const DexFile>(dex2));
+
+ // Run the test: extract the data for dex1 and dex2 and then merge it into
+ std::unique_ptr<FlattenProfileData> flattenProfileData1 = info1.ExtractProfileData(dex_files);
+ std::unique_ptr<FlattenProfileData> flattenProfileData2 = info2.ExtractProfileData(dex_files);
+
+ flattenProfileData1->MergeData(*flattenProfileData2);
+ // Check the results
+ ASSERT_EQ(flattenProfileData1->GetMaxAggregationForMethods(), 2u);
+ ASSERT_EQ(flattenProfileData1->GetMaxAggregationForClasses(), 2u);
+
+ const SafeMap<MethodReference, ItemMetadata>& methods = flattenProfileData1->GetMethodData();
+ const SafeMap<TypeReference, ItemMetadata>& classes = flattenProfileData1->GetClassData();
+ ASSERT_EQ(methods.size(), 20u); // 10 methods in dex1, 10 in dex2
+ ASSERT_EQ(classes.size(), 10u); // 10 methods in dex1
+
+ std::list<ProfileSampleAnnotation> expectedAnnotations1({psa1, psa2});
+ std::list<ProfileSampleAnnotation> expectedAnnotations2({psa2});
+ for (uint16_t i = 0; i < 10; i++) {
+ // Check dex1 methods.
+ auto mIt1 = methods.find(MethodReference(dex1, i));
+ ASSERT_TRUE(mIt1 != methods.end());
+ ASSERT_EQ(mIt1->second.GetFlags(), Hotness::kFlagHot | Hotness::kFlagStartup);
+ ASSERT_EQ(sort(mIt1->second.GetAnnotations()), expectedAnnotations1);
+ // Check dex1 classes
+ auto cIt1 = classes.find(TypeReference(dex1, dex::TypeIndex(i)));
+ ASSERT_TRUE(cIt1 != classes.end());
+ ASSERT_EQ(cIt1->second.GetFlags(), 0);
+ ASSERT_EQ(sort(cIt1->second.GetAnnotations()).size(), expectedAnnotations1.size());
+ ASSERT_EQ(sort(cIt1->second.GetAnnotations()), expectedAnnotations1);
+ // Check dex2 methods.
+ auto mIt2 = methods.find(MethodReference(dex2, i));
+ ASSERT_TRUE(mIt2 != methods.end());
+ ASSERT_EQ(mIt2->second.GetFlags(), Hotness::kFlagHot);
+ ASSERT_EQ(sort(mIt2->second.GetAnnotations()), expectedAnnotations2);
}
// Release the ownership as this is held by the test class;
diff --git a/profman/boot_image_profile.cc b/profman/boot_image_profile.cc
index d4e7f2b..3f9665f 100644
--- a/profman/boot_image_profile.cc
+++ b/profman/boot_image_profile.cc
@@ -197,7 +197,7 @@
bool GenerateBootImageProfile(
const std::vector<std::unique_ptr<const DexFile>>& dex_files,
- const ProfileCompilationInfo& profile,
+ const std::vector<std::string>& profile_files,
const BootImageOptions& options,
const std::string& boot_profile_out_path,
const std::string& preloaded_classes_out_path) {
@@ -208,7 +208,16 @@
bool generate_preloaded_classes = !preloaded_classes_out_path.empty();
- std::unique_ptr<FlattenProfileData> flattenData = profile.ExtractProfileData(dex_files);
+ std::unique_ptr<FlattenProfileData> flattend_data(new FlattenProfileData());
+ for (const std::string& profile_file : profile_files) {
+ ProfileCompilationInfo profile;
+ if (!profile.Load(profile_file, /*clear_if_invalid=*/ false)) {
+ LOG(ERROR) << "Profile is not a valid: " << profile_file;
+ return false;
+ }
+ std::unique_ptr<FlattenProfileData> currentData = profile.ExtractProfileData(dex_files);
+ flattend_data->MergeData(*currentData);
+ }
// We want the output sorted by the method/class name.
// So we use an intermediate map for that.
@@ -218,8 +227,8 @@
SafeMap<std::string, FlattenProfileData::ItemMetadata> profile_classes;
SafeMap<std::string, FlattenProfileData::ItemMetadata> preloaded_classes;
- for (const auto& it : flattenData->GetMethodData()) {
- if (IncludeMethodInProfile(flattenData->GetMaxAggregationForMethods(), it.second, options)) {
+ for (const auto& it : flattend_data->GetMethodData()) {
+ if (IncludeMethodInProfile(flattend_data->GetMaxAggregationForMethods(), it.second, options)) {
FlattenProfileData::ItemMetadata metadata(it.second);
if (options.upgrade_startup_to_hot
&& ((metadata.GetFlags() & Hotness::Flag::kFlagStartup) != 0)) {
@@ -229,11 +238,11 @@
}
}
- for (const auto& it : flattenData->GetClassData()) {
+ for (const auto& it : flattend_data->GetClassData()) {
const TypeReference& type_ref = it.first;
const FlattenProfileData::ItemMetadata& metadata = it.second;
if (IncludeClassInProfile(type_ref,
- flattenData->GetMaxAggregationForClasses(),
+ flattend_data->GetMaxAggregationForClasses(),
metadata,
options)) {
profile_classes.Put(BootImageRepresentation(it.first), it.second);
@@ -241,7 +250,7 @@
std::string preloaded_class_representation = PreloadedClassesRepresentation(it.first);
if (generate_preloaded_classes && IncludeInPreloadedClasses(
preloaded_class_representation,
- flattenData->GetMaxAggregationForClasses(),
+ flattend_data->GetMaxAggregationForClasses(),
metadata,
options)) {
preloaded_classes.Put(preloaded_class_representation, it.second);
diff --git a/profman/boot_image_profile.h b/profman/boot_image_profile.h
index 88a2a4c..a3dd52c 100644
--- a/profman/boot_image_profile.h
+++ b/profman/boot_image_profile.h
@@ -75,7 +75,7 @@
// Returns true if the generation was successful, false otherwise.
bool GenerateBootImageProfile(
const std::vector<std::unique_ptr<const DexFile>>& dex_files,
- const ProfileCompilationInfo& profile,
+ const std::vector<std::string>& profile_files,
const BootImageOptions& options,
const std::string& boot_profile_out_path,
const std::string& preloaded_classes_out_path);
diff --git a/profman/profile_assistant_test.cc b/profman/profile_assistant_test.cc
index 4f42eb4..2f3f58d 100644
--- a/profman/profile_assistant_test.cc
+++ b/profman/profile_assistant_test.cc
@@ -860,22 +860,104 @@
ASSERT_EQ(ExecAndReturnCode(args, &error), 0) << error;
ASSERT_TRUE(out_profile.GetFile()->ResetOffset());
- // std::vector<std::string> args1({"cp", out_profile.GetFilename(), "~/profile-test"});
- // EXPECT_EQ(ExecAndReturnCode(args1, &error), 0) << error;
-
// Verify the boot profile contents.
std::string output_profile_contents;
ASSERT_TRUE(android::base::ReadFileToString(
out_profile.GetFilename(), &output_profile_contents));
ASSERT_EQ(output_profile_contents, expected_profile_content);
- // Verify the boot profile contents.
+ // Verify the preloaded classes content.
std::string output_preloaded_contents;
ASSERT_TRUE(android::base::ReadFileToString(
out_preloaded_classes.GetFilename(), &output_preloaded_contents));
ASSERT_EQ(output_preloaded_contents, expected_preloaded_content);
}
+TEST_F(ProfileAssistantTest, TestBootImageProfileWith2RawProfiles) {
+ const std::string core_dex = GetLibCoreDexFileNames()[0];
+
+ std::vector<ScratchFile> profiles;
+
+ const std::string kCommonClassUsedByDex1 = "Ljava/lang/CharSequence;";
+ const std::string kCommonClassUsedByDex1Dex2 = "Ljava/lang/Object;";
+ const std::string kUncommonClass = "Ljava/lang/Process;";
+ const std::string kCommonHotMethodUsedByDex1 =
+ "Ljava/lang/Comparable;->compareTo(Ljava/lang/Object;)I";
+ const std::string kCommonHotMethodUsedByDex1Dex2 = "Ljava/lang/Object;->hashCode()I";
+ const std::string kUncommonHotMethod = "Ljava/util/HashMap;-><init>()V";
+
+
+ // Thresholds for this test.
+ static const size_t kDirtyThreshold = 100;
+ static const size_t kCleanThreshold = 100;
+ static const size_t kMethodThreshold = 100;
+
+ // Create boot profile content, attributing the classes and methods to different dex files.
+ std::vector<std::string> input_data1 = {
+ "{dex1}" + kCommonClassUsedByDex1,
+ "{dex1}" + kCommonClassUsedByDex1Dex2,
+ "{dex1}" + kUncommonClass,
+ "{dex1}H" + kCommonHotMethodUsedByDex1Dex2,
+ "{dex1}" + kCommonHotMethodUsedByDex1,
+ };
+ std::vector<std::string> input_data2 = {
+ "{dex1}" + kCommonClassUsedByDex1,
+ "{dex2}" + kCommonClassUsedByDex1Dex2,
+ "{dex1}H" + kCommonHotMethodUsedByDex1,
+ "{dex2}" + kCommonHotMethodUsedByDex1Dex2,
+ "{dex1}" + kUncommonHotMethod,
+ };
+ std::string input_file_contents1 = JoinProfileLines(input_data1);
+ std::string input_file_contents2 = JoinProfileLines(input_data2);
+
+ // Expected data
+ std::vector<std::string> expected_data = {
+ kCommonClassUsedByDex1,
+ kCommonClassUsedByDex1Dex2,
+ "H" + kCommonHotMethodUsedByDex1,
+ "H" + kCommonHotMethodUsedByDex1Dex2
+ };
+ std::string expected_profile_content = JoinProfileLines(expected_data);
+
+ ScratchFile profile1;
+ ScratchFile profile2;
+ EXPECT_TRUE(CreateProfile(input_file_contents1, profile1.GetFilename(), core_dex));
+ EXPECT_TRUE(CreateProfile(input_file_contents2, profile2.GetFilename(), core_dex));
+
+ ProfileCompilationInfo boot_profile1;
+ ProfileCompilationInfo boot_profile2;
+ boot_profile1.Load(profile1.GetFilename(), /*for_boot_image*/ true);
+ boot_profile2.Load(profile2.GetFilename(), /*for_boot_image*/ true);
+
+ // Generate the boot profile.
+ ScratchFile out_profile;
+ ScratchFile out_preloaded_classes;
+ ASSERT_TRUE(out_profile.GetFile()->ResetOffset());
+ ASSERT_TRUE(out_preloaded_classes.GetFile()->ResetOffset());
+ std::vector<std::string> args;
+ args.push_back(GetProfmanCmd());
+ args.push_back("--generate-boot-image-profile");
+ args.push_back("--class-threshold=" + std::to_string(kDirtyThreshold));
+ args.push_back("--clean-class-threshold=" + std::to_string(kCleanThreshold));
+ args.push_back("--method-threshold=" + std::to_string(kMethodThreshold));
+ args.push_back("--profile-file=" + profile1.GetFilename());
+ args.push_back("--profile-file=" + profile2.GetFilename());
+ args.push_back("--out-profile-path=" + out_profile.GetFilename());
+ args.push_back("--out-preloaded-classes-path=" + out_preloaded_classes.GetFilename());
+ args.push_back("--apk=" + core_dex);
+ args.push_back("--dex-location=" + core_dex);
+
+ std::string error;
+ ASSERT_EQ(ExecAndReturnCode(args, &error), 0) << error;
+ ASSERT_TRUE(out_profile.GetFile()->ResetOffset());
+
+ // Verify the boot profile contents.
+ std::string output_profile_contents;
+ ASSERT_TRUE(android::base::ReadFileToString(
+ out_profile.GetFilename(), &output_profile_contents));
+ ASSERT_EQ(output_profile_contents, expected_profile_content);
+}
+
TEST_F(ProfileAssistantTest, TestProfileCreationOneNotMatched) {
// Class names put here need to be in sorted order.
std::vector<std::string> class_names = {
diff --git a/profman/profman.cc b/profman/profman.cc
index eea83cc..a011cd0 100644
--- a/profman/profman.cc
+++ b/profman/profman.cc
@@ -1345,8 +1345,8 @@
// Create and store a ProfileCompilationInfo for the boot image.
int CreateBootImageProfile() {
// Open the input profile file.
- if (profile_files_.size() != 1) {
- LOG(ERROR) << "A single --profile-file must be specified.";
+ if (profile_files_.size() < 1) {
+ LOG(ERROR) << "At least one --profile-file must be specified.";
return -1;
}
// Open the dex files.
@@ -1357,14 +1357,8 @@
return -2;
}
- ProfileCompilationInfo profile;
- if (!profile.Load(profile_files_[0], /*clear_if_invalid=*/ false)) {
- LOG(ERROR) << "Reference profile is not a valid profile.";
- return -3;
- }
-
if (!GenerateBootImageProfile(dex_files,
- profile,
+ profile_files_,
boot_image_options_,
boot_profile_out_path_,
preloaded_classes_out_path_)) {