diff options
| -rw-r--r-- | compiler/linker/arm/relative_patcher_arm_base.cc | 20 | ||||
| -rw-r--r-- | compiler/linker/arm/relative_patcher_thumb2_test.cc | 24 | ||||
| -rw-r--r-- | compiler/linker/arm64/relative_patcher_arm64.cc | 2 | ||||
| -rw-r--r-- | compiler/linker/arm64/relative_patcher_arm64_test.cc | 50 | ||||
| -rw-r--r-- | compiler/linker/relative_patcher_test.h | 22 | ||||
| -rw-r--r-- | compiler/oat_writer.cc | 22 |
6 files changed, 77 insertions, 63 deletions
diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc index d4dd978c5c..2471f798be 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.cc +++ b/compiler/linker/arm/relative_patcher_arm_base.cc @@ -31,10 +31,6 @@ uint32_t ArmBaseRelativePatcher::ReserveSpace(uint32_t offset, } uint32_t ArmBaseRelativePatcher::ReserveSpaceEnd(uint32_t offset) { - // NOTE: The final thunk can be reserved from InitCodeMethodVisitor::EndClass() while it - // may be written early by WriteCodeMethodVisitor::VisitMethod() for a deduplicated chunk - // of code. To avoid any alignment discrepancies for the final chunk, we always align the - // offset after reserving of writing any chunk. uint32_t aligned_offset = CompiledMethod::AlignCode(offset, instruction_set_); bool needs_thunk = ReserveSpaceProcessPatches(aligned_offset, MethodReference(nullptr, 0u), @@ -46,7 +42,7 @@ uint32_t ArmBaseRelativePatcher::ReserveSpaceEnd(uint32_t offset) { unprocessed_patches_.clear(); thunk_locations_.push_back(aligned_offset); - offset = CompiledMethod::AlignCode(aligned_offset + thunk_code_.size(), instruction_set_); + offset = aligned_offset + thunk_code_.size(); } return offset; } @@ -65,13 +61,7 @@ uint32_t ArmBaseRelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) if (UNLIKELY(!WriteRelCallThunk(out, ArrayRef<const uint8_t>(thunk_code_)))) { return 0u; } - uint32_t thunk_end_offset = aligned_offset + thunk_code_.size(); - // Align after writing chunk, see the ReserveSpace() above. - offset = CompiledMethod::AlignCode(thunk_end_offset, instruction_set_); - aligned_code_delta = offset - thunk_end_offset; - if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { - return 0u; - } + offset = aligned_offset + thunk_code_.size(); } return offset; } @@ -92,7 +82,7 @@ uint32_t ArmBaseRelativePatcher::ReserveSpaceInternal(uint32_t offset, MethodReference method_ref, uint32_t max_extra_space) { uint32_t quick_code_size = compiled_method->GetQuickCode().size(); - uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); + uint32_t quick_code_offset = compiled_method->AlignCode(offset + sizeof(OatQuickMethodHeader)); uint32_t next_aligned_offset = compiled_method->AlignCode(quick_code_offset + quick_code_size); // Adjust for extra space required by the subclass. next_aligned_offset = compiled_method->AlignCode(next_aligned_offset + max_extra_space); @@ -106,9 +96,9 @@ uint32_t ArmBaseRelativePatcher::ReserveSpaceInternal(uint32_t offset, if (needs_thunk) { // A single thunk will cover all pending patches. unprocessed_patches_.clear(); - uint32_t thunk_location = compiled_method->AlignCode(offset); + uint32_t thunk_location = CompiledMethod::AlignCode(offset, instruction_set_); thunk_locations_.push_back(thunk_location); - offset = CompiledMethod::AlignCode(thunk_location + thunk_code_.size(), instruction_set_); + offset = thunk_location + thunk_code_.size(); } } for (const LinkerPatch& patch : compiled_method->GetPatches()) { diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/compiler/linker/arm/relative_patcher_thumb2_test.cc index a8078e3049..eace3d4326 100644 --- a/compiler/linker/arm/relative_patcher_thumb2_test.cc +++ b/compiler/linker/arm/relative_patcher_thumb2_test.cc @@ -48,18 +48,18 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { const ArrayRef<const LinkerPatch>& method3_patches, uint32_t distance_without_thunks) { CHECK_EQ(distance_without_thunks % kArmAlignment, 0u); - const uint32_t method1_offset = - CompiledCode::AlignCode(kTrampolineSize, kThumb2) + sizeof(OatQuickMethodHeader); + uint32_t method1_offset = + kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader); AddCompiledMethod(MethodRef(1u), method1_code, method1_patches); // We want to put the method3 at a very precise offset. const uint32_t method3_offset = method1_offset + distance_without_thunks; - CHECK_ALIGNED(method3_offset - sizeof(OatQuickMethodHeader), kArmAlignment); + CHECK_ALIGNED(method3_offset, kArmAlignment); // Calculate size of method2 so that we put method3 at the correct place. + const uint32_t method1_end = method1_offset + method1_code.size(); const uint32_t method2_offset = - CompiledCode::AlignCode(method1_offset + method1_code.size(), kThumb2) + - sizeof(OatQuickMethodHeader); + method1_end + CodeAlignmentSize(method1_end) + sizeof(OatQuickMethodHeader); const uint32_t method2_size = (method3_offset - sizeof(OatQuickMethodHeader) - method2_offset); std::vector<uint8_t> method2_raw_code(method2_size); ArrayRef<const uint8_t> method2_code(method2_raw_code); @@ -78,8 +78,11 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { if (result3.second == method3_offset + 1 /* thumb mode */) { return false; // No thunk. } else { - uint32_t aligned_thunk_size = CompiledCode::AlignCode(ThunkSize(), kThumb2); - CHECK_EQ(result3.second, method3_offset + aligned_thunk_size + 1 /* thumb mode */); + uint32_t thunk_end = + CompiledCode::AlignCode(method3_offset - sizeof(OatQuickMethodHeader), kThumb2) + + ThunkSize(); + uint32_t header_offset = thunk_end + CodeAlignmentSize(thunk_end); + CHECK_EQ(result3.second, header_offset + sizeof(OatQuickMethodHeader) + 1 /* thumb mode */); return true; // Thunk present. } } @@ -352,9 +355,12 @@ TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarAfter) { uint32_t method1_offset = GetMethodOffset(1u); uint32_t method3_offset = GetMethodOffset(3u); + ASSERT_TRUE(IsAligned<kArmAlignment>(method3_offset)); uint32_t method3_header_offset = method3_offset - sizeof(OatQuickMethodHeader); - ASSERT_TRUE(IsAligned<kArmAlignment>(method3_header_offset)); - uint32_t thunk_offset = method3_header_offset - CompiledCode::AlignCode(ThunkSize(), kThumb2); + uint32_t thunk_offset = + RoundDown(method3_header_offset - ThunkSize(), GetInstructionSetAlignment(kThumb2)); + DCHECK_EQ(thunk_offset + ThunkSize() + CodeAlignmentSize(thunk_offset + ThunkSize()), + method3_header_offset); ASSERT_TRUE(IsAligned<kArmAlignment>(thunk_offset)); uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1 + 4u /* PC adjustment */); ASSERT_EQ(diff & 1u, 0u); diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc index fdd14be4c4..4c8788e30d 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -83,7 +83,7 @@ uint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, // Now that we have the actual offset where the code will be placed, locate the ADRP insns // that actually require the thunk. - uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); + uint32_t quick_code_offset = compiled_method->AlignCode(offset + sizeof(OatQuickMethodHeader)); ArrayRef<const uint8_t> code = compiled_method->GetQuickCode(); uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); DCHECK(compiled_method != nullptr); diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/compiler/linker/arm64/relative_patcher_arm64_test.cc index 09729fdf96..573de736c4 100644 --- a/compiler/linker/arm64/relative_patcher_arm64_test.cc +++ b/compiler/linker/arm64/relative_patcher_arm64_test.cc @@ -67,36 +67,39 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { const ArrayRef<const LinkerPatch>& last_method_patches, uint32_t distance_without_thunks) { CHECK_EQ(distance_without_thunks % kArm64Alignment, 0u); - const uint32_t method1_offset = - CompiledCode::AlignCode(kTrampolineSize, kArm64) + sizeof(OatQuickMethodHeader); + uint32_t method1_offset = + kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader); AddCompiledMethod(MethodRef(1u), method1_code, method1_patches); - const uint32_t gap_start = - CompiledCode::AlignCode(method1_offset + method1_code.size(), kArm64); + const uint32_t gap_start = method1_offset + method1_code.size(); // We want to put the method3 at a very precise offset. const uint32_t last_method_offset = method1_offset + distance_without_thunks; + CHECK_ALIGNED(last_method_offset, kArm64Alignment); const uint32_t gap_end = last_method_offset - sizeof(OatQuickMethodHeader); - CHECK_ALIGNED(gap_end, kArm64Alignment); - // Fill the gap with intermediate methods in chunks of 2MiB and the last in [2MiB, 4MiB). + // Fill the gap with intermediate methods in chunks of 2MiB and the first in [2MiB, 4MiB). // (This allows deduplicating the small chunks to avoid using 256MiB of memory for +-128MiB - // offsets by this test.) + // offsets by this test. Making the first chunk bigger makes it easy to give all intermediate + // methods the same alignment of the end, so the thunk insertion adds a predictable size as + // long as it's after the first chunk.) uint32_t method_idx = 2u; constexpr uint32_t kSmallChunkSize = 2 * MB; std::vector<uint8_t> gap_code; - size_t gap_size = gap_end - gap_start; - for (; gap_size >= 2u * kSmallChunkSize; gap_size -= kSmallChunkSize) { - uint32_t chunk_code_size = kSmallChunkSize - sizeof(OatQuickMethodHeader); + uint32_t gap_size = gap_end - gap_start; + uint32_t num_small_chunks = std::max(gap_size / kSmallChunkSize, 1u) - 1u; + uint32_t chunk_start = gap_start; + uint32_t chunk_size = gap_size - num_small_chunks * kSmallChunkSize; + for (uint32_t i = 0; i <= num_small_chunks; ++i) { // num_small_chunks+1 iterations. + uint32_t chunk_code_size = + chunk_size - CodeAlignmentSize(chunk_start) - sizeof(OatQuickMethodHeader); gap_code.resize(chunk_code_size, 0u); AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(gap_code), ArrayRef<const LinkerPatch>()); method_idx += 1u; + chunk_start += chunk_size; + chunk_size = kSmallChunkSize; // For all but the first chunk. + DCHECK_EQ(CodeAlignmentSize(gap_end), CodeAlignmentSize(chunk_start)); } - uint32_t chunk_code_size = gap_size - sizeof(OatQuickMethodHeader); - gap_code.resize(chunk_code_size, 0u); - AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(gap_code), - ArrayRef<const LinkerPatch>()); - method_idx += 1u; // Add the last method and link AddCompiledMethod(MethodRef(method_idx), last_method_code, last_method_patches); @@ -109,8 +112,9 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { // There may be a thunk before method2. if (last_result.second != last_method_offset) { // Thunk present. Check that there's only one. - uint32_t aligned_thunk_size = CompiledCode::AlignCode(ThunkSize(), kArm64); - CHECK_EQ(last_result.second, last_method_offset + aligned_thunk_size); + uint32_t thunk_end = CompiledCode::AlignCode(gap_end, kArm64) + ThunkSize(); + uint32_t header_offset = thunk_end + CodeAlignmentSize(thunk_end); + CHECK_EQ(last_result.second, header_offset + sizeof(OatQuickMethodHeader)); } return method_idx; } @@ -341,7 +345,7 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { uint32_t dex_cache_arrays_begin, uint32_t element_offset) { uint32_t method1_offset = - CompiledCode::AlignCode(kTrampolineSize, kArm64) + sizeof(OatQuickMethodHeader); + kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader); ASSERT_LT(method1_offset, adrp_offset); CHECK_ALIGNED(adrp_offset, 4u); uint32_t num_nops = (adrp_offset - method1_offset) / 4u; @@ -391,7 +395,7 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { bool has_thunk, uint32_t string_offset) { uint32_t method1_offset = - CompiledCode::AlignCode(kTrampolineSize, kArm64) + sizeof(OatQuickMethodHeader); + kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader); ASSERT_LT(method1_offset, adrp_offset); CHECK_ALIGNED(adrp_offset, 4u); uint32_t num_nops = (adrp_offset - method1_offset) / 4u; @@ -614,10 +618,12 @@ TEST_F(Arm64RelativePatcherTestDefault, CallOtherJustTooFarAfter) { uint32_t method1_offset = GetMethodOffset(1u); uint32_t last_method_offset = GetMethodOffset(last_method_idx); + ASSERT_TRUE(IsAligned<kArm64Alignment>(last_method_offset)); uint32_t last_method_header_offset = last_method_offset - sizeof(OatQuickMethodHeader); - ASSERT_TRUE(IsAligned<kArm64Alignment>(last_method_header_offset)); - uint32_t thunk_offset = last_method_header_offset - CompiledCode::AlignCode(ThunkSize(), kArm64); - ASSERT_TRUE(IsAligned<kArm64Alignment>(thunk_offset)); + uint32_t thunk_offset = + RoundDown(last_method_header_offset - ThunkSize(), GetInstructionSetAlignment(kArm64)); + DCHECK_EQ(thunk_offset + ThunkSize() + CodeAlignmentSize(thunk_offset + ThunkSize()), + last_method_header_offset); uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1); CHECK_ALIGNED(diff, 4u); ASSERT_LT(diff, 128 * MB); diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h index ec69107d8f..d21f33e46f 100644 --- a/compiler/linker/relative_patcher_test.h +++ b/compiler/linker/relative_patcher_test.h @@ -98,6 +98,14 @@ class RelativePatcherTest : public testing::Test { patches)); } + uint32_t CodeAlignmentSize(uint32_t header_offset_to_align) { + // We want to align the code rather than the preheader. + uint32_t unaligned_code_offset = header_offset_to_align + sizeof(OatQuickMethodHeader); + uint32_t aligned_code_offset = + CompiledMethod::AlignCode(unaligned_code_offset, instruction_set_); + return aligned_code_offset - unaligned_code_offset; + } + void Link() { // Reserve space. static_assert(kTrampolineOffset == 0u, "Unexpected trampoline offset."); @@ -106,9 +114,8 @@ class RelativePatcherTest : public testing::Test { for (auto& compiled_method : compiled_methods_) { offset = patcher_->ReserveSpace(offset, compiled_method.get(), compiled_method_refs_[idx]); - uint32_t aligned_offset = compiled_method->AlignCode(offset); - uint32_t aligned_code_delta = aligned_offset - offset; - offset += aligned_code_delta; + uint32_t alignment_size = CodeAlignmentSize(offset); + offset += alignment_size; offset += sizeof(OatQuickMethodHeader); uint32_t quick_code_offset = offset + compiled_method->CodeDelta(); @@ -136,11 +143,10 @@ class RelativePatcherTest : public testing::Test { for (auto& compiled_method : compiled_methods_) { offset = patcher_->WriteThunks(&out_, offset); - uint32_t aligned_offset = compiled_method->AlignCode(offset); - uint32_t aligned_code_delta = aligned_offset - offset; - CHECK_LE(aligned_code_delta, sizeof(kPadding)); - out_.WriteFully(kPadding, aligned_code_delta); - offset += aligned_code_delta; + uint32_t alignment_size = CodeAlignmentSize(offset); + CHECK_LE(alignment_size, sizeof(kPadding)); + out_.WriteFully(kPadding, alignment_size); + offset += alignment_size; out_.WriteFully(dummy_header, sizeof(OatQuickMethodHeader)); offset += sizeof(OatQuickMethodHeader); diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index f20c715f58..8273b15667 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -87,6 +87,13 @@ class ChecksumUpdatingOutputStream : public OutputStream { OatHeader* const oat_header_; }; +inline uint32_t CodeAlignmentSize(uint32_t header_offset, const CompiledMethod& compiled_method) { + // We want to align the code rather than the preheader. + uint32_t unaligned_code_offset = header_offset + sizeof(OatQuickMethodHeader); + uint32_t aligned_code_offset = compiled_method.AlignCode(unaligned_code_offset); + return aligned_code_offset - unaligned_code_offset; +} + } // anonymous namespace // Defines the location of the raw dex file to write. @@ -817,8 +824,8 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { uint32_t thumb_offset) { offset_ = writer_->relative_patcher_->ReserveSpace( offset_, compiled_method, MethodReference(dex_file_, it.GetMemberIndex())); - offset_ = compiled_method->AlignCode(offset_); - DCHECK_ALIGNED_PARAM(offset_, + offset_ += CodeAlignmentSize(offset_, *compiled_method); + DCHECK_ALIGNED_PARAM(offset_ + sizeof(OatQuickMethodHeader), GetInstructionSetAlignment(compiled_method->GetInstructionSet())); return offset_ + sizeof(OatQuickMethodHeader) + thumb_offset; } @@ -1011,17 +1018,16 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { ReportWriteFailure("relative call thunk", it); return false; } - uint32_t aligned_offset = compiled_method->AlignCode(offset_); - uint32_t aligned_code_delta = aligned_offset - offset_; - if (aligned_code_delta != 0) { - if (!writer_->WriteCodeAlignment(out, aligned_code_delta)) { + uint32_t alignment_size = CodeAlignmentSize(offset_, *compiled_method); + if (alignment_size != 0) { + if (!writer_->WriteCodeAlignment(out, alignment_size)) { ReportWriteFailure("code alignment padding", it); return false; } - offset_ += aligned_code_delta; + offset_ += alignment_size; DCHECK_OFFSET_(); } - DCHECK_ALIGNED_PARAM(offset_, + DCHECK_ALIGNED_PARAM(offset_ + sizeof(OatQuickMethodHeader), GetInstructionSetAlignment(compiled_method->GetInstructionSet())); DCHECK_EQ(method_offsets.code_offset_, offset_ + sizeof(OatQuickMethodHeader) + compiled_method->CodeDelta()) |