David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2022 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #ifndef ART_RUNTIME_BASE_ATOMIC_PAIR_H_ |
| 18 | #define ART_RUNTIME_BASE_ATOMIC_PAIR_H_ |
| 19 | |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 20 | #include <android-base/logging.h> |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 21 | |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 22 | #include <atomic> |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 23 | #include <type_traits> |
| 24 | |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 25 | #include "base/macros.h" |
| 26 | |
Dmitrii Ishcheikin | cb5edd3 | 2024-01-04 21:00:39 +0000 | [diff] [blame] | 27 | namespace art HIDDEN { |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 28 | |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 29 | // Implement 16-byte atomic pair using the seq-lock synchronization algorithm. |
| 30 | // This is currently only used for DexCache. |
| 31 | // |
| 32 | // This uses top 4-bytes of the key as version counter and lock bit, |
| 33 | // which means the stored pair key can not use those bytes. |
| 34 | // |
| 35 | // This allows us to read the cache without exclusive access to the cache line. |
| 36 | // |
| 37 | // The 8-byte atomic pair uses the normal single-instruction implementation. |
| 38 | // |
| 39 | static constexpr uint64_t kSeqMask = (0xFFFFFFFFull << 32); |
| 40 | static constexpr uint64_t kSeqLock = (0x80000000ull << 32); |
| 41 | static constexpr uint64_t kSeqIncr = (0x00000001ull << 32); |
| 42 | |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 43 | // std::pair<> is not trivially copyable and as such it is unsuitable for atomic operations. |
| 44 | template <typename IntType> |
| 45 | struct PACKED(2 * sizeof(IntType)) AtomicPair { |
| 46 | static_assert(std::is_integral_v<IntType>); |
| 47 | |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 48 | AtomicPair(IntType f, IntType s) : key(f), val(s) {} |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 49 | |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 50 | IntType key; |
| 51 | IntType val; |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 52 | }; |
| 53 | |
| 54 | template <typename IntType> |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 55 | ALWAYS_INLINE static inline AtomicPair<IntType> AtomicPairLoadAcquire(AtomicPair<IntType>* pair) { |
| 56 | static_assert(std::is_trivially_copyable<AtomicPair<IntType>>::value); |
| 57 | auto* target = reinterpret_cast<std::atomic<AtomicPair<IntType>>*>(pair); |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 58 | return target->load(std::memory_order_acquire); |
| 59 | } |
| 60 | |
| 61 | template <typename IntType> |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 62 | ALWAYS_INLINE static inline void AtomicPairStoreRelease(AtomicPair<IntType>* pair, |
Ulya Trafimovich | 4184f23 | 2023-01-26 12:25:22 +0000 | [diff] [blame] | 63 | AtomicPair<IntType> value) { |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 64 | static_assert(std::is_trivially_copyable<AtomicPair<IntType>>::value); |
| 65 | auto* target = reinterpret_cast<std::atomic<AtomicPair<IntType>>*>(pair); |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 66 | target->store(value, std::memory_order_release); |
| 67 | } |
| 68 | |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 69 | ALWAYS_INLINE static inline AtomicPair<uint64_t> AtomicPairLoadAcquire(AtomicPair<uint64_t>* pair) { |
| 70 | auto* key_ptr = reinterpret_cast<std::atomic_uint64_t*>(&pair->key); |
| 71 | auto* val_ptr = reinterpret_cast<std::atomic_uint64_t*>(&pair->val); |
| 72 | while (true) { |
| 73 | uint64_t key0 = key_ptr->load(std::memory_order_acquire); |
| 74 | uint64_t val = val_ptr->load(std::memory_order_acquire); |
| 75 | uint64_t key1 = key_ptr->load(std::memory_order_relaxed); |
| 76 | uint64_t key = key0 & ~kSeqMask; |
| 77 | if (LIKELY((key0 & kSeqLock) == 0 && key0 == key1)) { |
| 78 | return {key, val}; |
| 79 | } |
| 80 | } |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 81 | } |
| 82 | |
David Srbecky | 80cb419 | 2023-05-16 17:07:47 +0100 | [diff] [blame] | 83 | ALWAYS_INLINE static inline void AtomicPairStoreRelease(AtomicPair<uint64_t>* pair, |
| 84 | AtomicPair<uint64_t> value) { |
| 85 | DCHECK((value.key & kSeqMask) == 0) << "Key=0x" << std::hex << value.key; |
| 86 | auto* key_ptr = reinterpret_cast<std::atomic_uint64_t*>(&pair->key); |
| 87 | auto* val_ptr = reinterpret_cast<std::atomic_uint64_t*>(&pair->val); |
| 88 | uint64_t key = key_ptr->load(std::memory_order_relaxed); |
| 89 | do { |
| 90 | key &= ~kSeqLock; // Ensure that the CAS below fails if the lock bit is already set. |
| 91 | } while (!key_ptr->compare_exchange_weak(key, key | kSeqLock)); |
| 92 | key = (((key & kSeqMask) + kSeqIncr) & ~kSeqLock) | (value.key & ~kSeqMask); |
| 93 | val_ptr->store(value.val, std::memory_order_release); |
| 94 | key_ptr->store(key, std::memory_order_release); |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 95 | } |
David Srbecky | ae148fe | 2022-02-04 15:55:56 +0000 | [diff] [blame] | 96 | |
| 97 | } // namespace art |
| 98 | |
| 99 | #endif // ART_RUNTIME_BASE_ATOMIC_PAIR_H_ |