Merge "Guardrail for dimension in condition in duration tracker." into pi-dev

4 files changed, 87 insertions, 15 deletions

diff --git a/cmds/statsd/src/guardrail/StatsdStats.cpp b/cmds/statsd/src/guardrail/StatsdStats.cpp
index ef637df68ed3..22ff9428c83a 100644
--- a/cmds/statsd/src/guardrail/StatsdStats.cpp
+++ b/cmds/statsd/src/guardrail/StatsdStats.cpp
@@ -76,6 +76,7 @@ const int FIELD_ID_CONFIG_STATS_MATCHER_STATS = 13;
 const int FIELD_ID_CONFIG_STATS_CONDITION_STATS = 14;
 const int FIELD_ID_CONFIG_STATS_METRIC_STATS = 15;
 const int FIELD_ID_CONFIG_STATS_ALERT_STATS = 16;
+const int FIELD_ID_CONFIG_STATS_METRIC_DIMENSION_IN_CONDITION_STATS = 17;
 
 const int FIELD_ID_MATCHER_STATS_ID = 1;
 const int FIELD_ID_MATCHER_STATS_COUNT = 2;
@@ -255,6 +256,20 @@ void StatsdStats::noteMetricDimensionSize(const ConfigKey& key, const int64_t& i
     }
 }
 
+void StatsdStats::noteMetricDimensionInConditionSize(
+        const ConfigKey& key, const int64_t& id, int size) {
+    lock_guard<std::mutex> lock(mLock);
+    // if name doesn't exist before, it will create the key with count 0.
+    auto statsIt = mConfigStats.find(key);
+    if (statsIt == mConfigStats.end()) {
+        return;
+    }
+    auto& metricsDimensionMap = statsIt->second->metric_dimension_in_condition_stats;
+    if (size > metricsDimensionMap[id]) {
+        metricsDimensionMap[id] = size;
+    }
+}
+
 void StatsdStats::noteMatcherMatched(const ConfigKey& key, const int64_t& id) {
     lock_guard<std::mutex> lock(mLock);
 
@@ -339,6 +354,7 @@ void StatsdStats::resetInternalLocked() {
         config.second->matcher_stats.clear();
         config.second->condition_stats.clear();
         config.second->metric_stats.clear();
+        config.second->metric_dimension_in_condition_stats.clear();
         config.second->alert_stats.clear();
     }
 }
@@ -504,6 +520,13 @@ void addConfigStatsToProto(const ConfigStats& configStats, ProtoOutputStream* pr
         proto->write(FIELD_TYPE_INT32 | FIELD_ID_METRIC_STATS_COUNT, pair.second);
         proto->end(tmpToken);
     }
+    for (const auto& pair : configStats.metric_dimension_in_condition_stats) {
+        uint64_t tmpToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
+                                         FIELD_ID_CONFIG_STATS_METRIC_DIMENSION_IN_CONDITION_STATS);
+        proto->write(FIELD_TYPE_INT64 | FIELD_ID_METRIC_STATS_ID, (long long)pair.first);
+        proto->write(FIELD_TYPE_INT32 | FIELD_ID_METRIC_STATS_COUNT, pair.second);
+        proto->end(tmpToken);
+    }
 
     for (const auto& pair : configStats.alert_stats) {
         uint64_t tmpToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
diff --git a/cmds/statsd/src/guardrail/StatsdStats.h b/cmds/statsd/src/guardrail/StatsdStats.h
index 7f8755b8c1e0..bd395c4c232b 100644
--- a/cmds/statsd/src/guardrail/StatsdStats.h
+++ b/cmds/statsd/src/guardrail/StatsdStats.h
@@ -57,6 +57,12 @@ struct ConfigStats {
     // it means some data has been dropped. The map size is capped by kMaxConfigCount.
     std::map<const int64_t, int> metric_stats;
 
+    // Stores the max number of output tuple of dimensions in condition across dimensions in what
+    // when it's bigger than kDimensionKeySizeSoftLimit. When you see the number is
+    // kDimensionKeySizeHardLimit +1, it means some data has been dropped. The map size is capped by
+    // kMaxConfigCount.
+    std::map<const int64_t, int> metric_dimension_in_condition_stats;
+
     // Stores the number of times an anomaly detection alert has been declared.
     // The map size is capped by kMaxConfigCount.
     std::map<const int64_t, int> alert_stats;
@@ -183,6 +189,19 @@ public:
      */
     void noteMetricDimensionSize(const ConfigKey& key, const int64_t& id, int size);
 
+
+    /**
+     * Report the max size of output tuple of dimension in condition across dimensions in what.
+     *
+     * Note: only report when the metric has an output dimension in condition, and the max tuple
+     * count > kDimensionKeySizeSoftLimit.
+     *
+     * [key]: The config key that this metric belongs to.
+     * [id]: The id of the metric.
+     * [size]: The output tuple size.
+     */
+    void noteMetricDimensionInConditionSize(const ConfigKey& key, const int64_t& id, int size);
+
     /**
      * Report a matcher has been matched.
      *
diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.cpp b/cmds/statsd/src/metrics/DurationMetricProducer.cpp
index c28bb8800e20..f02f30756b2d 100644
--- a/cmds/statsd/src/metrics/DurationMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/DurationMetricProducer.cpp
@@ -308,11 +308,14 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked_opt2(bool conditio
                     if (mMetric2ConditionLinks.size() == 0 ||
                         trueDim.contains(linkedConditionDimensionKey)) {
                         if (!whatIt.second.empty()) {
+                            auto newEventKey = MetricDimensionKey(whatIt.first, trueDim);
+                            if (hitGuardRailLocked(newEventKey)) {
+                                continue;
+                            }
                             unique_ptr<DurationTracker> newTracker =
                                 whatIt.second.begin()->second->clone(eventTime);
                             if (newTracker != nullptr) {
-                                newTracker->setEventKey(
-                                    MetricDimensionKey(whatIt.first, trueDim));
+                                newTracker->setEventKey(newEventKey);
                                 newTracker->onConditionChanged(true, eventTime);
                                 whatIt.second[trueDim] = std::move(newTracker);
                             }
@@ -370,11 +373,14 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondit
         for (const auto& conditionDimension : conditionDimensionsKeySet) {
             for (auto& whatIt : mCurrentSlicedDurationTrackerMap) {
                 if (!whatIt.second.empty()) {
+                    auto newEventKey = MetricDimensionKey(whatIt.first, conditionDimension);
+                    if (hitGuardRailLocked(newEventKey)) {
+                        continue;
+                    }
                     unique_ptr<DurationTracker> newTracker =
                         whatIt.second.begin()->second->clone(eventTime);
                     if (newTracker != nullptr) {
-                        newTracker->setEventKey(MetricDimensionKey(
-                                whatIt.first, conditionDimension));
+                        newTracker->setEventKey(MetricDimensionKey(newEventKey));
                         newTracker->onSlicedConditionMayChange(overallCondition, eventTime);
                         whatIt.second[conditionDimension] = std::move(newTracker);
                     }
@@ -397,10 +403,13 @@ void DurationMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondit
             for (const auto& conditionDimension : conditionDimensionsKeys) {
                 if (!whatIt.second.empty() &&
                     whatIt.second.find(conditionDimension) == whatIt.second.end()) {
+                    auto newEventKey = MetricDimensionKey(whatIt.first, conditionDimension);
+                    if (hitGuardRailLocked(newEventKey)) {
+                        continue;
+                    }
                     auto newTracker = whatIt.second.begin()->second->clone(eventTime);
                     if (newTracker != nullptr) {
-                        newTracker->setEventKey(
-                            MetricDimensionKey(whatIt.first, conditionDimension));
+                        newTracker->setEventKey(newEventKey);
                         newTracker->onSlicedConditionMayChange(overallCondition, eventTime);
                         whatIt.second[conditionDimension] = std::move(newTracker);
                     }
@@ -552,15 +561,35 @@ void DurationMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
 }
 
 bool DurationMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
-    // 1. Report the tuple count if the tuple count > soft limit
-    if (mCurrentSlicedDurationTrackerMap.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {
-        size_t newTupleCount = mCurrentSlicedDurationTrackerMap.size() + 1;
-        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount);
-        // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
-        if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {
-            ALOGE("DurationMetric %lld dropping data for dimension key %s",
-                (long long)mMetricId, newKey.toString().c_str());
-            return true;
+    auto whatIt = mCurrentSlicedDurationTrackerMap.find(newKey.getDimensionKeyInWhat());
+    if (whatIt != mCurrentSlicedDurationTrackerMap.end()) {
+        auto condIt = whatIt->second.find(newKey.getDimensionKeyInCondition());
+        if (condIt != whatIt->second.end()) {
+            return false;
+        }
+        if (whatIt->second.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {
+            size_t newTupleCount = whatIt->second.size() + 1;
+            StatsdStats::getInstance().noteMetricDimensionInConditionSize(
+                    mConfigKey, mMetricId, newTupleCount);
+            // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
+            if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {
+                ALOGE("DurationMetric %lld dropping data for condition dimension key %s",
+                    (long long)mMetricId, newKey.getDimensionKeyInCondition().toString().c_str());
+                return true;
+            }
+        }
+    } else {
+        // 1. Report the tuple count if the tuple count > soft limit
+        if (mCurrentSlicedDurationTrackerMap.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {
+            size_t newTupleCount = mCurrentSlicedDurationTrackerMap.size() + 1;
+            StatsdStats::getInstance().noteMetricDimensionSize(
+                    mConfigKey, mMetricId, newTupleCount);
+            // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
+            if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {
+                ALOGE("DurationMetric %lld dropping data for what dimension key %s",
+                    (long long)mMetricId, newKey.getDimensionKeyInWhat().toString().c_str());
+                return true;
+            }
         }
     }
     return false;
diff --git a/cmds/statsd/src/stats_log.proto b/cmds/statsd/src/stats_log.proto
index dd3b37c8c2d7..a25df3fc4c09 100644
--- a/cmds/statsd/src/stats_log.proto
+++ b/cmds/statsd/src/stats_log.proto
@@ -241,6 +241,7 @@ message StatsdStatsReport {
         repeated ConditionStats condition_stats = 14;
         repeated MetricStats metric_stats = 15;
         repeated AlertStats alert_stats = 16;
+        repeated MetricStats metric_dimension_in_condition_stats = 17;
     }
 
     repeated ConfigStats config_stats = 3;