| /* |
| * Copyright 2019 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| // TODO(b/129481165): remove the #pragma below and fix conversion issues |
| #pragma clang diagnostic push |
| #pragma clang diagnostic ignored "-Wconversion" |
| #pragma clang diagnostic ignored "-Wextra" |
| |
| //#define LOG_NDEBUG 0 |
| #define ATRACE_TAG ATRACE_TAG_GRAPHICS |
| #undef LOG_TAG |
| #define LOG_TAG "RegionSamplingThread" |
| |
| #include "RegionSamplingThread.h" |
| |
| #include <compositionengine/Display.h> |
| #include <compositionengine/impl/OutputCompositionState.h> |
| #include <cutils/properties.h> |
| #include <ftl/future.h> |
| #include <gui/IRegionSamplingListener.h> |
| #include <gui/SyncScreenCaptureListener.h> |
| #include <ui/DisplayStatInfo.h> |
| #include <utils/Trace.h> |
| |
| #include <string> |
| |
| #include "DisplayDevice.h" |
| #include "DisplayRenderArea.h" |
| #include "Layer.h" |
| #include "Scheduler/VsyncController.h" |
| #include "SurfaceFlinger.h" |
| |
| namespace android { |
| using namespace std::chrono_literals; |
| |
| template <typename T> |
| struct SpHash { |
| size_t operator()(const sp<T>& p) const { return std::hash<T*>()(p.get()); } |
| }; |
| |
| constexpr auto lumaSamplingStepTag = "LumaSamplingStep"; |
| enum class samplingStep { |
| noWorkNeeded, |
| idleTimerWaiting, |
| waitForQuietFrame, |
| waitForSamplePhase, |
| sample |
| }; |
| |
| constexpr auto defaultRegionSamplingWorkDuration = 3ms; |
| constexpr auto defaultRegionSamplingPeriod = 100ms; |
| constexpr auto defaultRegionSamplingTimerTimeout = 100ms; |
| constexpr auto maxRegionSamplingDelay = 100ms; |
| // TODO: (b/127403193) duration to string conversion could probably be constexpr |
| template <typename Rep, typename Per> |
| inline std::string toNsString(std::chrono::duration<Rep, Per> t) { |
| return std::to_string(std::chrono::duration_cast<std::chrono::nanoseconds>(t).count()); |
| } |
| |
| RegionSamplingThread::EnvironmentTimingTunables::EnvironmentTimingTunables() { |
| char value[PROPERTY_VALUE_MAX] = {}; |
| |
| property_get("debug.sf.region_sampling_duration_ns", value, |
| toNsString(defaultRegionSamplingWorkDuration).c_str()); |
| int const samplingDurationNsRaw = atoi(value); |
| |
| property_get("debug.sf.region_sampling_period_ns", value, |
| toNsString(defaultRegionSamplingPeriod).c_str()); |
| int const samplingPeriodNsRaw = atoi(value); |
| |
| property_get("debug.sf.region_sampling_timer_timeout_ns", value, |
| toNsString(defaultRegionSamplingTimerTimeout).c_str()); |
| int const samplingTimerTimeoutNsRaw = atoi(value); |
| |
| if ((samplingPeriodNsRaw < 0) || (samplingTimerTimeoutNsRaw < 0)) { |
| ALOGW("User-specified sampling tuning options nonsensical. Using defaults"); |
| mSamplingDuration = defaultRegionSamplingWorkDuration; |
| mSamplingPeriod = defaultRegionSamplingPeriod; |
| mSamplingTimerTimeout = defaultRegionSamplingTimerTimeout; |
| } else { |
| mSamplingDuration = std::chrono::nanoseconds(samplingDurationNsRaw); |
| mSamplingPeriod = std::chrono::nanoseconds(samplingPeriodNsRaw); |
| mSamplingTimerTimeout = std::chrono::nanoseconds(samplingTimerTimeoutNsRaw); |
| } |
| } |
| |
| RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger, const TimingTunables& tunables) |
| : mFlinger(flinger), |
| mTunables(tunables), |
| mIdleTimer( |
| "RegSampIdle", |
| std::chrono::duration_cast<std::chrono::milliseconds>( |
| mTunables.mSamplingTimerTimeout), |
| [] {}, [this] { checkForStaleLuma(); }), |
| mLastSampleTime(0ns) { |
| mThread = std::thread([this]() { threadMain(); }); |
| pthread_setname_np(mThread.native_handle(), "RegionSampling"); |
| mIdleTimer.start(); |
| } |
| |
| RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger) |
| : RegionSamplingThread(flinger, |
| TimingTunables{defaultRegionSamplingWorkDuration, |
| defaultRegionSamplingPeriod, |
| defaultRegionSamplingTimerTimeout}) {} |
| |
| RegionSamplingThread::~RegionSamplingThread() { |
| mIdleTimer.stop(); |
| |
| { |
| std::lock_guard lock(mThreadControlMutex); |
| mRunning = false; |
| mCondition.notify_one(); |
| } |
| |
| if (mThread.joinable()) { |
| mThread.join(); |
| } |
| } |
| |
| void RegionSamplingThread::addListener(const Rect& samplingArea, const wp<Layer>& stopLayer, |
| const sp<IRegionSamplingListener>& listener) { |
| sp<IBinder> asBinder = IInterface::asBinder(listener); |
| asBinder->linkToDeath(this); |
| std::lock_guard lock(mSamplingMutex); |
| mDescriptors.emplace(wp<IBinder>(asBinder), Descriptor{samplingArea, stopLayer, listener}); |
| } |
| |
| void RegionSamplingThread::removeListener(const sp<IRegionSamplingListener>& listener) { |
| std::lock_guard lock(mSamplingMutex); |
| mDescriptors.erase(wp<IBinder>(IInterface::asBinder(listener))); |
| } |
| |
| void RegionSamplingThread::checkForStaleLuma() { |
| std::lock_guard lock(mThreadControlMutex); |
| |
| if (mSampleRequestTime.has_value()) { |
| ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForSamplePhase)); |
| mSampleRequestTime.reset(); |
| mFlinger.scheduleRegionSamplingThread(); |
| } |
| } |
| |
| void RegionSamplingThread::onCompositionComplete( |
| std::optional<std::chrono::steady_clock::time_point> samplingDeadline) { |
| doSample(samplingDeadline); |
| } |
| |
| void RegionSamplingThread::doSample( |
| std::optional<std::chrono::steady_clock::time_point> samplingDeadline) { |
| std::lock_guard lock(mThreadControlMutex); |
| const auto now = std::chrono::steady_clock::now(); |
| if (mLastSampleTime + mTunables.mSamplingPeriod > now) { |
| // content changed, but we sampled not too long ago, so we need to sample some time in the |
| // future. |
| ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::idleTimerWaiting)); |
| mSampleRequestTime = now; |
| return; |
| } |
| if (!mSampleRequestTime.has_value() || now - *mSampleRequestTime < maxRegionSamplingDelay) { |
| // If there is relatively little time left for surfaceflinger |
| // until the next vsync deadline, defer this sampling work |
| // to a later frame, when hopefully there will be more time. |
| if (samplingDeadline.has_value() && now + mTunables.mSamplingDuration > *samplingDeadline) { |
| ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForQuietFrame)); |
| mSampleRequestTime = mSampleRequestTime.value_or(now); |
| return; |
| } |
| } |
| |
| ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::sample)); |
| |
| mSampleRequestTime.reset(); |
| mLastSampleTime = now; |
| |
| mIdleTimer.reset(); |
| |
| mSampleRequested = true; |
| mCondition.notify_one(); |
| } |
| |
| void RegionSamplingThread::binderDied(const wp<IBinder>& who) { |
| std::lock_guard lock(mSamplingMutex); |
| mDescriptors.erase(who); |
| } |
| |
| float sampleArea(const uint32_t* data, int32_t width, int32_t height, int32_t stride, |
| uint32_t orientation, const Rect& sample_area) { |
| if (!sample_area.isValid() || (sample_area.getWidth() > width) || |
| (sample_area.getHeight() > height)) { |
| ALOGE("invalid sampling region requested"); |
| return 0.0f; |
| } |
| |
| // (b/133849373) ROT_90 screencap images produced upside down |
| auto area = sample_area; |
| if (orientation & ui::Transform::ROT_90) { |
| area.top = height - area.top; |
| area.bottom = height - area.bottom; |
| std::swap(area.top, area.bottom); |
| |
| area.left = width - area.left; |
| area.right = width - area.right; |
| std::swap(area.left, area.right); |
| } |
| |
| const uint32_t pixelCount = (area.bottom - area.top) * (area.right - area.left); |
| uint32_t accumulatedLuma = 0; |
| |
| // Calculates luma with approximation of Rec. 709 primaries |
| for (int32_t row = area.top; row < area.bottom; ++row) { |
| const uint32_t* rowBase = data + row * stride; |
| for (int32_t column = area.left; column < area.right; ++column) { |
| uint32_t pixel = rowBase[column]; |
| const uint32_t r = pixel & 0xFF; |
| const uint32_t g = (pixel >> 8) & 0xFF; |
| const uint32_t b = (pixel >> 16) & 0xFF; |
| const uint32_t luma = (r * 7 + b * 2 + g * 23) >> 5; |
| accumulatedLuma += luma; |
| } |
| } |
| |
| return accumulatedLuma / (255.0f * pixelCount); |
| } |
| |
| std::vector<float> RegionSamplingThread::sampleBuffer( |
| const sp<GraphicBuffer>& buffer, const Point& leftTop, |
| const std::vector<RegionSamplingThread::Descriptor>& descriptors, uint32_t orientation) { |
| void* data_raw = nullptr; |
| buffer->lock(GRALLOC_USAGE_SW_READ_OFTEN, &data_raw); |
| std::shared_ptr<uint32_t> data(reinterpret_cast<uint32_t*>(data_raw), |
| [&buffer](auto) { buffer->unlock(); }); |
| if (!data) return {}; |
| |
| const int32_t width = buffer->getWidth(); |
| const int32_t height = buffer->getHeight(); |
| const int32_t stride = buffer->getStride(); |
| std::vector<float> lumas(descriptors.size()); |
| std::transform(descriptors.begin(), descriptors.end(), lumas.begin(), |
| [&](auto const& descriptor) { |
| return sampleArea(data.get(), width, height, stride, orientation, |
| descriptor.area - leftTop); |
| }); |
| return lumas; |
| } |
| |
| void RegionSamplingThread::captureSample() { |
| ATRACE_CALL(); |
| std::lock_guard lock(mSamplingMutex); |
| |
| if (mDescriptors.empty()) { |
| return; |
| } |
| |
| wp<const DisplayDevice> displayWeak; |
| |
| ui::LayerStack layerStack; |
| ui::Transform::RotationFlags orientation; |
| ui::Size displaySize; |
| |
| { |
| // TODO(b/159112860): Don't keep sp<DisplayDevice> outside of SF main thread |
| const sp<const DisplayDevice> display = mFlinger.getDefaultDisplayDevice(); |
| displayWeak = display; |
| layerStack = display->getLayerStack(); |
| orientation = ui::Transform::toRotationFlags(display->getOrientation()); |
| displaySize = display->getSize(); |
| } |
| |
| std::vector<RegionSamplingThread::Descriptor> descriptors; |
| Region sampleRegion; |
| for (const auto& [listener, descriptor] : mDescriptors) { |
| sampleRegion.orSelf(descriptor.area); |
| descriptors.emplace_back(descriptor); |
| } |
| |
| const Rect sampledBounds = sampleRegion.bounds(); |
| constexpr bool kUseIdentityTransform = false; |
| |
| SurfaceFlinger::RenderAreaFuture renderAreaFuture = ftl::defer([=] { |
| return DisplayRenderArea::create(displayWeak, sampledBounds, sampledBounds.getSize(), |
| ui::Dataspace::V0_SRGB, kUseIdentityTransform); |
| }); |
| |
| std::unordered_set<sp<IRegionSamplingListener>, SpHash<IRegionSamplingListener>> listeners; |
| |
| auto traverseLayers = [&](const LayerVector::Visitor& visitor) { |
| bool stopLayerFound = false; |
| auto filterVisitor = [&](Layer* layer) { |
| // We don't want to capture any layers beyond the stop layer |
| if (stopLayerFound) return; |
| |
| // Likewise if we just found a stop layer, set the flag and abort |
| for (const auto& [area, stopLayer, listener] : descriptors) { |
| if (layer == stopLayer.promote().get()) { |
| stopLayerFound = true; |
| return; |
| } |
| } |
| |
| // Compute the layer's position on the screen |
| const Rect bounds = Rect(layer->getBounds()); |
| const ui::Transform transform = layer->getTransform(); |
| constexpr bool roundOutwards = true; |
| Rect transformed = transform.transform(bounds, roundOutwards); |
| |
| // If this layer doesn't intersect with the larger sampledBounds, skip capturing it |
| Rect ignore; |
| if (!transformed.intersect(sampledBounds, &ignore)) return; |
| |
| // If the layer doesn't intersect a sampling area, skip capturing it |
| bool intersectsAnyArea = false; |
| for (const auto& [area, stopLayer, listener] : descriptors) { |
| if (transformed.intersect(area, &ignore)) { |
| intersectsAnyArea = true; |
| listeners.insert(listener); |
| } |
| } |
| if (!intersectsAnyArea) return; |
| |
| ALOGV("Traversing [%s] [%d, %d, %d, %d]", layer->getDebugName(), bounds.left, |
| bounds.top, bounds.right, bounds.bottom); |
| visitor(layer); |
| }; |
| mFlinger.traverseLayersInLayerStack(layerStack, CaptureArgs::UNSET_UID, filterVisitor); |
| }; |
| |
| std::shared_ptr<renderengine::ExternalTexture> buffer = nullptr; |
| if (mCachedBuffer && mCachedBuffer->getBuffer()->getWidth() == sampledBounds.getWidth() && |
| mCachedBuffer->getBuffer()->getHeight() == sampledBounds.getHeight()) { |
| buffer = mCachedBuffer; |
| } else { |
| const uint32_t usage = |
| GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_HW_RENDER | GRALLOC_USAGE_HW_TEXTURE; |
| sp<GraphicBuffer> graphicBuffer = |
| new GraphicBuffer(sampledBounds.getWidth(), sampledBounds.getHeight(), |
| PIXEL_FORMAT_RGBA_8888, 1, usage, "RegionSamplingThread"); |
| const status_t bufferStatus = graphicBuffer->initCheck(); |
| LOG_ALWAYS_FATAL_IF(bufferStatus != OK, "captureSample: Buffer failed to allocate: %d", |
| bufferStatus); |
| buffer = std::make_shared< |
| renderengine::ExternalTexture>(graphicBuffer, mFlinger.getRenderEngine(), |
| renderengine::ExternalTexture::Usage::WRITEABLE); |
| } |
| |
| const sp<SyncScreenCaptureListener> captureListener = new SyncScreenCaptureListener(); |
| mFlinger.captureScreenCommon(std::move(renderAreaFuture), traverseLayers, buffer, |
| true /* regionSampling */, false /* grayscale */, captureListener); |
| ScreenCaptureResults captureResults = captureListener->waitForResults(); |
| |
| std::vector<Descriptor> activeDescriptors; |
| for (const auto& descriptor : descriptors) { |
| if (listeners.count(descriptor.listener) != 0) { |
| activeDescriptors.emplace_back(descriptor); |
| } |
| } |
| |
| ALOGV("Sampling %zu descriptors", activeDescriptors.size()); |
| std::vector<float> lumas = sampleBuffer(buffer->getBuffer(), sampledBounds.leftTop(), |
| activeDescriptors, orientation); |
| if (lumas.size() != activeDescriptors.size()) { |
| ALOGW("collected %zu median luma values for %zu descriptors", lumas.size(), |
| activeDescriptors.size()); |
| return; |
| } |
| |
| for (size_t d = 0; d < activeDescriptors.size(); ++d) { |
| activeDescriptors[d].listener->onSampleCollected(lumas[d]); |
| } |
| |
| mCachedBuffer = buffer; |
| ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::noWorkNeeded)); |
| } |
| |
| // NO_THREAD_SAFETY_ANALYSIS is because std::unique_lock presently lacks thread safety annotations. |
| void RegionSamplingThread::threadMain() NO_THREAD_SAFETY_ANALYSIS { |
| std::unique_lock<std::mutex> lock(mThreadControlMutex); |
| while (mRunning) { |
| if (mSampleRequested) { |
| mSampleRequested = false; |
| lock.unlock(); |
| captureSample(); |
| lock.lock(); |
| } |
| mCondition.wait(lock, [this]() REQUIRES(mThreadControlMutex) { |
| return mSampleRequested || !mRunning; |
| }); |
| } |
| } |
| |
| } // namespace android |
| |
| // TODO(b/129481165): remove the #pragma below and fix conversion issues |
| #pragma clang diagnostic pop // ignored "-Wconversion -Wextra" |