Revert "Compress profile files"
This reverts commit 4f0a15a85eaa0a7665f703d435a9b61059eed536.
Reason for revert: art-test gtest-heap-poisoning is broken with this change.
Change-Id: I736374766b216122cae4e7e97451951cda1551f9
diff --git a/dexlayout/dexlayout_test.cc b/dexlayout/dexlayout_test.cc
index d2aef27..877ea92 100644
--- a/dexlayout/dexlayout_test.cc
+++ b/dexlayout/dexlayout_test.cc
@@ -41,7 +41,7 @@
"AAAAdQEAAAAQAAABAAAAjAEAAA==";
static const char kDexFileLayoutInputProfile[] =
- "cHJvADAwNgAAAAAAAAgAAAB4AQMAAAAAAQ==";
+ "cHJvADAwNQABCwABAAAAAAD1KW3+Y2xhc3Nlcy5kZXgBAA==";
// Dex file with catch handler unreferenced by try blocks.
// Constructed by building a dex file with try/catch blocks and hex editing.
diff --git a/runtime/jit/profile_compilation_info.cc b/runtime/jit/profile_compilation_info.cc
index f163a59..0acce1e 100644
--- a/runtime/jit/profile_compilation_info.cc
+++ b/runtime/jit/profile_compilation_info.cc
@@ -18,18 +18,11 @@
#include "errno.h"
#include <limits.h>
-#include <string>
#include <vector>
#include <stdlib.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <sys/uio.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <zlib.h>
-#include <base/time_utils.h>
#include "base/mutex.h"
#include "base/scoped_flock.h"
@@ -40,13 +33,13 @@
#include "os.h"
#include "safe_map.h"
#include "utils.h"
-#include "android-base/file.h"
namespace art {
const uint8_t ProfileCompilationInfo::kProfileMagic[] = { 'p', 'r', 'o', '\0' };
-// Last profile version: Compress profile data.
-const uint8_t ProfileCompilationInfo::kProfileVersion[] = { '0', '0', '6', '\0' };
+// Last profile version: fix profman merges. Update profile version to force
+// regeneration of possibly faulty profiles.
+const uint8_t ProfileCompilationInfo::kProfileVersion[] = { '0', '0', '5', '\0' };
static constexpr uint16_t kMaxDexFileKeyLength = PATH_MAX;
@@ -216,12 +209,12 @@
/**
* Serialization format:
- * magic,version,number_of_dex_files,uncompressed_size_of_zipped_data,compressed_data_size,
- * zipped[dex_location1,number_of_classes1,methods_region_size,dex_location_checksum1, \
+ * magic,version,number_of_dex_files
+ * dex_location1,number_of_classes1,methods_region_size,dex_location_checksum1, \
* method_encoding_11,method_encoding_12...,class_id1,class_id2...
* dex_location2,number_of_classes2,methods_region_size,dex_location_checksum2, \
* method_encoding_21,method_encoding_22...,,class_id1,class_id2...
- * .....]
+ * .....
* The method_encoding is:
* method_id,number_of_inline_caches,inline_cache1,inline_cache2...
* The inline_cache is:
@@ -235,53 +228,28 @@
* When present, there will be no class ids following.
**/
bool ProfileCompilationInfo::Save(int fd) {
- uint64_t start = NanoTime();
ScopedTrace trace(__PRETTY_FUNCTION__);
DCHECK_GE(fd, 0);
+ // Cache at most 50KB before writing.
+ static constexpr size_t kMaxSizeToKeepBeforeWriting = 50 * KB;
// Use a vector wrapper to avoid keeping track of offsets when we add elements.
std::vector<uint8_t> buffer;
- if (!WriteBuffer(fd, kProfileMagic, sizeof(kProfileMagic))) {
- return false;
- }
- if (!WriteBuffer(fd, kProfileVersion, sizeof(kProfileVersion))) {
- return false;
- }
+ WriteBuffer(fd, kProfileMagic, sizeof(kProfileMagic));
+ WriteBuffer(fd, kProfileVersion, sizeof(kProfileVersion));
DCHECK_LE(info_.size(), std::numeric_limits<uint8_t>::max());
AddUintToBuffer(&buffer, static_cast<uint8_t>(info_.size()));
- uint32_t required_capacity = 0;
- for (const DexFileData* dex_data_ptr : info_) {
- const DexFileData& dex_data = *dex_data_ptr;
- uint32_t methods_region_size = GetMethodsRegionSize(dex_data);
- required_capacity += kLineHeaderSize +
- dex_data.profile_key.size() +
- sizeof(uint16_t) * dex_data.class_set.size() +
- methods_region_size;
- }
- if (required_capacity > kProfileSizeErrorThresholdInBytes) {
- LOG(ERROR) << "Profile data size exceeds "
- << std::to_string(kProfileSizeErrorThresholdInBytes)
- << " bytes. Profile will not be written to disk.";
- return false;
- }
- if (required_capacity > kProfileSizeWarningThresholdInBytes) {
- LOG(WARNING) << "Profile data size exceeds "
- << std::to_string(kProfileSizeWarningThresholdInBytes);
- }
- AddUintToBuffer(&buffer, required_capacity);
- if (!WriteBuffer(fd, buffer.data(), buffer.size())) {
- return false;
- }
- // Make sure that the buffer has enough capacity to avoid repeated resizings
- // while we add data.
- buffer.reserve(required_capacity);
- buffer.clear();
-
// Dex files must be written in the order of their profile index. This
// avoids writing the index in the output file and simplifies the parsing logic.
for (const DexFileData* dex_data_ptr : info_) {
const DexFileData& dex_data = *dex_data_ptr;
+ if (buffer.size() > kMaxSizeToKeepBeforeWriting) {
+ if (!WriteBuffer(fd, buffer.data(), buffer.size())) {
+ return false;
+ }
+ buffer.clear();
+ }
// Note that we allow dex files without any methods or classes, so that
// inline caches can refer valid dex files.
@@ -291,8 +259,16 @@
return false;
}
+ // Make sure that the buffer has enough capacity to avoid repeated resizings
+ // while we add data.
uint32_t methods_region_size = GetMethodsRegionSize(dex_data);
+ size_t required_capacity = buffer.size() +
+ kLineHeaderSize +
+ dex_data.profile_key.size() +
+ sizeof(uint16_t) * dex_data.class_set.size() +
+ methods_region_size;
+ buffer.reserve(required_capacity);
DCHECK_LE(dex_data.profile_key.size(), std::numeric_limits<uint16_t>::max());
DCHECK_LE(dex_data.class_set.size(), std::numeric_limits<uint16_t>::max());
AddUintToBuffer(&buffer, static_cast<uint16_t>(dex_data.profile_key.size()));
@@ -309,29 +285,12 @@
for (const auto& class_id : dex_data.class_set) {
AddUintToBuffer(&buffer, class_id.index_);
}
+
+ DCHECK_LE(required_capacity, buffer.size())
+ << "Failed to add the expected number of bytes in the buffer";
}
- uint32_t output_size = 0;
- std::unique_ptr<uint8_t[]> compressed_buffer = DeflateBuffer(buffer.data(),
- required_capacity,
- &output_size);
-
- buffer.clear();
- AddUintToBuffer(&buffer, output_size);
-
- if (!WriteBuffer(fd, buffer.data(), buffer.size())) {
- return false;
- }
- if (!WriteBuffer(fd, compressed_buffer.get(), output_size)) {
- return false;
- }
- uint64_t total_time = NanoTime() - start;
- VLOG(profiler) << "Compressed from "
- << std::to_string(required_capacity)
- << " to "
- << std::to_string(output_size);
- VLOG(profiler) << "Time to save profile: " << std::to_string(total_time);
- return true;
+ return WriteBuffer(fd, buffer.data(), buffer.size());
}
void ProfileCompilationInfo::AddInlineCacheToBuffer(std::vector<uint8_t>* buffer,
@@ -625,14 +584,7 @@
uint8_t number_of_dex_files,
const ProfileLineHeader& line_header,
/*out*/std::string* error) {
- uint32_t unread_bytes_before_operation = buffer.CountUnreadBytes();
- if (unread_bytes_before_operation < line_header.method_region_size_bytes) {
- *error += "Profile EOF reached prematurely for ReadMethod";
- return kProfileLoadBadData;
- }
- size_t expected_unread_bytes_after_operation = buffer.CountUnreadBytes()
- - line_header.method_region_size_bytes;
- while (buffer.CountUnreadBytes() > expected_unread_bytes_after_operation) {
+ while (buffer.HasMoreData()) {
DexFileData* const data = GetOrAddDexFileData(line_header.dex_location, line_header.checksum);
uint16_t method_index;
READ_UINT(uint16_t, buffer, method_index, error);
@@ -642,24 +594,15 @@
return false;
}
}
- uint32_t total_bytes_read = unread_bytes_before_operation - buffer.CountUnreadBytes();
- if (total_bytes_read != line_header.method_region_size_bytes) {
- *error += "Profile data inconsistent for ReadMethods";
- return false;
- }
+
return true;
}
bool ProfileCompilationInfo::ReadClasses(SafeBuffer& buffer,
+ uint16_t classes_to_read,
const ProfileLineHeader& line_header,
/*out*/std::string* error) {
- size_t unread_bytes_before_op = buffer.CountUnreadBytes();
- if (unread_bytes_before_op < line_header.class_set_size) {
- *error += "Profile EOF reached prematurely for ReadClasses";
- return kProfileLoadBadData;
- }
-
- for (uint16_t i = 0; i < line_header.class_set_size; i++) {
+ for (uint16_t i = 0; i < classes_to_read; i++) {
uint16_t type_index;
READ_UINT(uint16_t, buffer, type_index, error);
if (!AddClassIndex(line_header.dex_location,
@@ -668,12 +611,6 @@
return false;
}
}
- size_t total_bytes_read = unread_bytes_before_op - buffer.CountUnreadBytes();
- uint32_t expected_bytes_read = line_header.class_set_size * sizeof(uint16_t);
- if (total_bytes_read != expected_bytes_read) {
- *error += "Profile data inconsistent for ReadClasses";
- return false;
- }
return true;
}
@@ -713,11 +650,15 @@
return false;
}
+bool ProfileCompilationInfo::SafeBuffer::HasMoreData() {
+ return ptr_current_ < ptr_end_;
+}
+
ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::SafeBuffer::FillFromFd(
int fd,
const std::string& source,
/*out*/std::string* error) {
- size_t byte_count = (ptr_end_ - ptr_current_) * sizeof(*ptr_current_);
+ size_t byte_count = ptr_end_ - ptr_current_;
uint8_t* buffer = ptr_current_;
while (byte_count > 0) {
int bytes_read = TEMP_FAILURE_RETRY(read(fd, buffer, byte_count));
@@ -734,31 +675,15 @@
return kProfileLoadSuccess;
}
-size_t ProfileCompilationInfo::SafeBuffer::CountUnreadBytes() {
- return (ptr_end_ - ptr_current_) * sizeof(*ptr_current_);
-}
-
-const uint8_t* ProfileCompilationInfo::SafeBuffer::GetCurrentPtr() {
- return ptr_current_;
-}
-
-void ProfileCompilationInfo::SafeBuffer::Advance(size_t data_size) {
- ptr_current_ += data_size;
-}
-
ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::ReadProfileHeader(
int fd,
/*out*/uint8_t* number_of_dex_files,
- /*out*/uint32_t* uncompressed_data_size,
- /*out*/uint32_t* compressed_data_size,
/*out*/std::string* error) {
// Read magic and version
const size_t kMagicVersionSize =
sizeof(kProfileMagic) +
sizeof(kProfileVersion) +
- sizeof(uint8_t) + // number of dex files
- sizeof(uint32_t) + // size of uncompressed profile data
- sizeof(uint32_t); // size of compressed profile data
+ sizeof(uint8_t); // number of dex files
SafeBuffer safe_buffer(kMagicVersionSize);
@@ -779,14 +704,6 @@
*error = "Cannot read the number of dex files";
return kProfileLoadBadData;
}
- if (!safe_buffer.ReadUintAndAdvance<uint32_t>(uncompressed_data_size)) {
- *error = "Cannot read the size of uncompressed data";
- return kProfileLoadBadData;
- }
- if (!safe_buffer.ReadUintAndAdvance<uint32_t>(compressed_data_size)) {
- *error = "Cannot read the size of compressed data";
- return kProfileLoadBadData;
- }
return kProfileLoadSuccess;
}
@@ -802,16 +719,17 @@
}
ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::ReadProfileLineHeader(
- SafeBuffer& buffer,
- /*out*/ProfileLineHeader* line_header,
- /*out*/std::string* error) {
- if (buffer.CountUnreadBytes() < kLineHeaderSize) {
- *error += "Profile EOF reached prematurely for ReadProfileLineHeader";
- return kProfileLoadBadData;
+ int fd,
+ /*out*/ProfileLineHeader* line_header,
+ /*out*/std::string* error) {
+ SafeBuffer header_buffer(kLineHeaderSize);
+ ProfileLoadSatus status = header_buffer.FillFromFd(fd, "ReadProfileLineHeader", error);
+ if (status != kProfileLoadSuccess) {
+ return status;
}
uint16_t dex_location_size;
- if (!ReadProfileLineHeaderElements(buffer, &dex_location_size, line_header, error)) {
+ if (!ReadProfileLineHeaderElements(header_buffer, &dex_location_size, line_header, error)) {
return kProfileLoadBadData;
}
@@ -821,19 +739,18 @@
return kProfileLoadBadData;
}
- if (buffer.CountUnreadBytes() < dex_location_size) {
- *error += "Profile EOF reached prematurely for ReadProfileHeaderDexLocation";
- return kProfileLoadBadData;
+ SafeBuffer location_buffer(dex_location_size);
+ status = location_buffer.FillFromFd(fd, "ReadProfileHeaderDexLocation", error);
+ if (status != kProfileLoadSuccess) {
+ return status;
}
- const uint8_t* base_ptr = buffer.GetCurrentPtr();
line_header->dex_location.assign(
- reinterpret_cast<const char*>(base_ptr), dex_location_size);
- buffer.Advance(dex_location_size);
+ reinterpret_cast<char*>(location_buffer.Get()), dex_location_size);
return kProfileLoadSuccess;
}
ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::ReadProfileLine(
- SafeBuffer& buffer,
+ int fd,
uint8_t number_of_dex_files,
const ProfileLineHeader& line_header,
/*out*/std::string* error) {
@@ -843,13 +760,29 @@
return kProfileLoadBadData;
}
- if (!ReadMethods(buffer, number_of_dex_files, line_header, error)) {
- return kProfileLoadBadData;
+ {
+ SafeBuffer buffer(line_header.method_region_size_bytes);
+ ProfileLoadSatus status = buffer.FillFromFd(fd, "ReadProfileLineMethods", error);
+ if (status != kProfileLoadSuccess) {
+ return status;
+ }
+
+ if (!ReadMethods(buffer, number_of_dex_files, line_header, error)) {
+ return kProfileLoadBadData;
+ }
}
- if (!ReadClasses(buffer, line_header, error)) {
- return kProfileLoadBadData;
+ {
+ SafeBuffer buffer(sizeof(uint16_t) * line_header.class_set_size);
+ ProfileLoadSatus status = buffer.FillFromFd(fd, "ReadProfileLineClasses", error);
+ if (status != kProfileLoadSuccess) {
+ return status;
+ }
+ if (!ReadClasses(buffer, line_header.class_set_size, line_header, error)) {
+ return kProfileLoadBadData;
+ }
}
+
return kProfileLoadSuccess;
}
@@ -888,135 +821,39 @@
}
// Read profile header: magic + version + number_of_dex_files.
uint8_t number_of_dex_files;
- uint32_t uncompressed_data_size;
- uint32_t compressed_data_size;
- ProfileLoadSatus status = ReadProfileHeader(fd,
- &number_of_dex_files,
- &uncompressed_data_size,
- &compressed_data_size,
- error);
-
- if (uncompressed_data_size > kProfileSizeErrorThresholdInBytes) {
- LOG(ERROR) << "Profile data size exceeds "
- << std::to_string(kProfileSizeErrorThresholdInBytes)
- << " bytes";
- return kProfileLoadBadData;
- }
- if (uncompressed_data_size > kProfileSizeWarningThresholdInBytes) {
- LOG(WARNING) << "Profile data size exceeds "
- << std::to_string(kProfileSizeWarningThresholdInBytes)
- << " bytes";
- }
-
+ ProfileLoadSatus status = ReadProfileHeader(fd, &number_of_dex_files, error);
if (status != kProfileLoadSuccess) {
return status;
}
- std::unique_ptr<uint8_t[]> compressed_data(new uint8_t[compressed_data_size]);
- bool bytes_read_success =
- android::base::ReadFully(fd, compressed_data.get(), compressed_data_size);
-
- if (testEOF(fd) != 0) {
- *error += "Unexpected data in the profile file.";
- return kProfileLoadBadData;
- }
-
- if (!bytes_read_success) {
- *error += "Unable to read compressed profile data";
- return kProfileLoadBadData;
- }
-
- SafeBuffer uncompressed_data(uncompressed_data_size);
-
- int ret = InflateBuffer(compressed_data.get(),
- compressed_data_size,
- uncompressed_data_size,
- uncompressed_data.Get());
-
- if (ret != Z_STREAM_END) {
- *error += "Error reading uncompressed profile data";
- return kProfileLoadBadData;
- }
-
for (uint8_t k = 0; k < number_of_dex_files; k++) {
ProfileLineHeader line_header;
// First, read the line header to get the amount of data we need to read.
- status = ReadProfileLineHeader(uncompressed_data, &line_header, error);
+ status = ReadProfileLineHeader(fd, &line_header, error);
if (status != kProfileLoadSuccess) {
return status;
}
// Now read the actual profile line.
- status = ReadProfileLine(uncompressed_data, number_of_dex_files, line_header, error);
+ status = ReadProfileLine(fd, number_of_dex_files, line_header, error);
if (status != kProfileLoadSuccess) {
return status;
}
}
// Check that we read everything and that profiles don't contain junk data.
- if (uncompressed_data.CountUnreadBytes() > 0) {
+ int result = testEOF(fd);
+ if (result == 0) {
+ return kProfileLoadSuccess;
+ } else if (result < 0) {
+ return kProfileLoadIOError;
+ } else {
*error = "Unexpected content in the profile file";
return kProfileLoadBadData;
- } else {
- return kProfileLoadSuccess;
}
}
-std::unique_ptr<uint8_t[]> ProfileCompilationInfo::DeflateBuffer(const uint8_t* in_buffer,
- uint32_t in_size,
- uint32_t* compressed_data_size) {
- z_stream strm;
- strm.zalloc = Z_NULL;
- strm.zfree = Z_NULL;
- strm.opaque = Z_NULL;
- int ret = deflateInit(&strm, 1);
- if (ret != Z_OK) {
- return nullptr;
- }
-
- uint32_t out_size = deflateBound(&strm, in_size);
-
- std::unique_ptr<uint8_t[]> compressed_buffer(new uint8_t[out_size]);
- strm.avail_in = in_size;
- strm.next_in = const_cast<uint8_t*>(in_buffer);
- strm.avail_out = out_size;
- strm.next_out = &compressed_buffer[0];
- ret = deflate(&strm, Z_FINISH);
- if (ret == Z_STREAM_ERROR) {
- return nullptr;
- }
- *compressed_data_size = out_size - strm.avail_out;
- deflateEnd(&strm);
- return compressed_buffer;
-}
-
-int ProfileCompilationInfo::InflateBuffer(const uint8_t* in_buffer,
- uint32_t in_size,
- uint32_t expected_uncompressed_data_size,
- uint8_t* out_buffer) {
- z_stream strm;
-
- /* allocate inflate state */
- strm.zalloc = Z_NULL;
- strm.zfree = Z_NULL;
- strm.opaque = Z_NULL;
- strm.avail_in = in_size;
- strm.next_in = const_cast<uint8_t*>(in_buffer);
- strm.avail_out = expected_uncompressed_data_size;
- strm.next_out = out_buffer;
-
- int ret;
- inflateInit(&strm);
- ret = inflate(&strm, Z_NO_FLUSH);
-
- if (strm.avail_in != 0 || strm.avail_out != 0) {
- return Z_DATA_ERROR;
- }
- inflateEnd(&strm);
- return ret;
-}
-
bool ProfileCompilationInfo::MergeWith(const ProfileCompilationInfo& other) {
// First verify that all checksums match. This will avoid adding garbage to
// the current profile info.
diff --git a/runtime/jit/profile_compilation_info.h b/runtime/jit/profile_compilation_info.h
index 9e47cc1..f68ed5d 100644
--- a/runtime/jit/profile_compilation_info.h
+++ b/runtime/jit/profile_compilation_info.h
@@ -284,9 +284,6 @@
kProfileLoadSuccess
};
- const uint32_t kProfileSizeWarningThresholdInBytes = 500000U;
- const uint32_t kProfileSizeErrorThresholdInBytes = 1000000U;
-
// Internal representation of the profile information belonging to a dex file.
// Note that we could do without profile_key (the key used to encode the dex
// file in the profile) and profile_index (the index of the dex file in the
@@ -356,21 +353,6 @@
// Checks if the profile is empty.
bool IsEmpty() const;
- // Inflate the input buffer (in_buffer) of size in_size. It returns a buffer of
- // compressed data for the input buffer of "compressed_data_size" size.
- std::unique_ptr<uint8_t[]> DeflateBuffer(const uint8_t* in_buffer,
- uint32_t in_size,
- /*out*/uint32_t* compressed_data_size);
-
- // Inflate the input buffer(in_buffer) of size in_size. out_size is the expected output
- // size of the buffer. It puts the output in out_buffer. It returns Z_STREAM_END on
- // success. On error, it returns Z_STREAM_ERROR if the compressed data is inconsistent
- // and Z_DATA_ERROR if the stream ended prematurely or the stream has extra data.
- int InflateBuffer(const uint8_t* in_buffer,
- uint32_t in_size,
- uint32_t out_size,
- /*out*/uint8_t* out_buffer);
-
// Parsing functionality.
// The information present in the header of each profile line.
@@ -394,10 +376,6 @@
const std::string& source,
/*out*/std::string* error);
- ProfileLoadSatus FillFromBuffer(uint8_t* buffer_ptr,
- const std::string& source,
- /*out*/std::string* error);
-
// Reads an uint value (high bits to low bits) and advances the current pointer
// with the number of bits read.
template <typename T> bool ReadUintAndAdvance(/*out*/ T* value);
@@ -406,22 +384,16 @@
// equal it advances the current pointer by data_size.
bool CompareAndAdvance(const uint8_t* data, size_t data_size);
- // Advances current pointer by data_size.
- void Advance(size_t data_size);
-
- // Returns the count of unread bytes.
- size_t CountUnreadBytes();
-
- // Returns the current pointer.
- const uint8_t* GetCurrentPtr();
+ // Returns true if the buffer has more data to read.
+ bool HasMoreData();
// Get the underlying raw buffer.
uint8_t* Get() { return storage_.get(); }
private:
std::unique_ptr<uint8_t[]> storage_;
- uint8_t* ptr_end_;
uint8_t* ptr_current_;
+ uint8_t* ptr_end_;
};
// Entry point for profile loding functionality.
@@ -431,12 +403,10 @@
// lines into number_of_dex_files.
ProfileLoadSatus ReadProfileHeader(int fd,
/*out*/uint8_t* number_of_dex_files,
- /*out*/uint32_t* size_uncompressed_data,
- /*out*/uint32_t* size_compressed_data,
/*out*/std::string* error);
// Read the header of a profile line from the given fd.
- ProfileLoadSatus ReadProfileLineHeader(SafeBuffer& buffer,
+ ProfileLoadSatus ReadProfileLineHeader(int fd,
/*out*/ProfileLineHeader* line_header,
/*out*/std::string* error);
@@ -447,13 +417,14 @@
/*out*/std::string* error);
// Read a single profile line from the given fd.
- ProfileLoadSatus ReadProfileLine(SafeBuffer& buffer,
+ ProfileLoadSatus ReadProfileLine(int fd,
uint8_t number_of_dex_files,
const ProfileLineHeader& line_header,
/*out*/std::string* error);
// Read all the classes from the buffer into the profile `info_` structure.
bool ReadClasses(SafeBuffer& buffer,
+ uint16_t classes_to_read,
const ProfileLineHeader& line_header,
/*out*/std::string* error);