diff options
| author | 2017-11-21 18:07:17 -0800 | |
|---|---|---|
| committer | 2017-11-27 10:52:54 -0800 | |
| commit | b356151e63140085cb96fa16804ee18b3862a4fc (patch) | |
| tree | ecb892f399742cc0d7190b7700379b1a120bcb1e | |
| parent | 16fd2e9a9a0a064491196d02957c7e94eaa870a7 (diff) | |
Add StatsdStats and guardrail.
+ StatsdStats is the global class that tracks the stats about statsd.
+ Added guardrail for classes that have a map which could potentially grow
unboundedly with the number of logs.
TODO: add unit tests & CTS for StatsdStats, and guardrail
add stats for pulled atoms.
Test: statsd_test
Change-Id: I0ea562de4dd3f6162f7923a9c193420b482c1d51
41 files changed, 1093 insertions, 197 deletions
diff --git a/cmds/statsd/Android.mk b/cmds/statsd/Android.mk index 4d975fcf9c64..1f15c5e97e0d 100644 --- a/cmds/statsd/Android.mk +++ b/cmds/statsd/Android.mk @@ -57,7 +57,8 @@ statsd_common_src := \ src/StatsLogProcessor.cpp \ src/StatsService.cpp \ src/stats_util.cpp \ - src/guardrail/MemoryLeakTrackUtil.cpp + src/guardrail/MemoryLeakTrackUtil.cpp \ + src/guardrail/StatsdStats.cpp statsd_common_c_includes := \ $(LOCAL_PATH)/src \ @@ -167,7 +168,8 @@ LOCAL_SRC_FILES := \ tests/metrics/MaxDurationTracker_test.cpp \ tests/metrics/CountMetricProducer_test.cpp \ tests/metrics/EventMetricProducer_test.cpp \ - tests/metrics/ValueMetricProducer_test.cpp + tests/metrics/ValueMetricProducer_test.cpp \ + tests/guardrail/StatsdStats_test.cpp LOCAL_STATIC_LIBRARIES := \ libgmock diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp index b5cb20c274ea..2690c7e43d79 100644 --- a/cmds/statsd/src/StatsLogProcessor.cpp +++ b/cmds/statsd/src/StatsLogProcessor.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#define DEBUG true // STOPSHIP if true #include "Log.h" #include "statslog.h" @@ -21,6 +22,7 @@ #include <dirent.h> #include "StatsLogProcessor.h" #include "android-base/stringprintf.h" +#include "guardrail/StatsdStats.h" #include "metrics/CountMetricProducer.h" #include "stats_util.h" #include "storage/StorageManager.h" @@ -82,6 +84,7 @@ void StatsLogProcessor::onAnomalyAlarmFired( // TODO: what if statsd service restarts? How do we know what logs are already processed before? void StatsLogProcessor::OnLogEvent(const LogEvent& msg) { + StatsdStats::getInstance().noteAtomLogged(msg.GetTagId(), msg.GetTimestampNs() / NS_PER_SEC); // pass the event to metrics managers. for (auto& pair : mMetricsManagers) { pair.second->onLogEvent(msg); @@ -106,23 +109,26 @@ void StatsLogProcessor::OnLogEvent(const LogEvent& msg) { } void StatsLogProcessor::OnConfigUpdated(const ConfigKey& key, const StatsdConfig& config) { + ALOGD("Updated configuration for key %s", key.ToString().c_str()); + unique_ptr<MetricsManager> newMetricsManager = std::make_unique<MetricsManager>(key, config); + auto it = mMetricsManagers.find(key); if (it != mMetricsManagers.end()) { it->second->finish(); + } else if (mMetricsManagers.size() > StatsdStats::kMaxConfigCount) { + ALOGE("Can't accept more configs!"); + return; } - ALOGD("Updated configuration for key %s", key.ToString().c_str()); - - unique_ptr<MetricsManager> newMetricsManager = std::make_unique<MetricsManager>(config); if (newMetricsManager->isConfigValid()) { mUidMap->OnConfigUpdated(key); newMetricsManager->setAnomalyMonitor(mAnomalyMonitor); mMetricsManagers[key] = std::move(newMetricsManager); // Why doesn't this work? mMetricsManagers.insert({key, std::move(newMetricsManager)}); - ALOGD("StatsdConfig valid"); + VLOG("StatsdConfig valid"); } else { // If there is any error in the config, don't use it. - ALOGD("StatsdConfig NOT valid"); + ALOGE("StatsdConfig NOT valid"); } } @@ -204,6 +210,7 @@ void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { mMetricsManagers.erase(it); mUidMap->OnConfigRemoved(key); } + StatsdStats::getInstance().noteConfigRemoved(key); std::lock_guard<std::mutex> lock(mBroadcastTimesMutex); mLastBroadcastTimes.erase(key); @@ -223,12 +230,14 @@ void StatsLogProcessor::flushIfNecessary(uint64_t timestampNs, } } mLastBroadcastTimes[key] = timestampNs; - ALOGD("StatsD requesting broadcast for %s", key.ToString().c_str()); + VLOG("StatsD requesting broadcast for %s", key.ToString().c_str()); mSendBroadcast(key); + StatsdStats::getInstance().noteBroadcastSent(key); } else if (totalBytes > kMaxSerializedBytes) { // Too late. We need to start clearing data. // We ignore the return value so we force each metric producer to clear its contents. metricsManager->onDumpReport(); - ALOGD("StatsD had to toss out metrics for %s", key.ToString().c_str()); + StatsdStats::getInstance().noteDataDrop(key); + VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str()); } } diff --git a/cmds/statsd/src/StatsService.cpp b/cmds/statsd/src/StatsService.cpp index ec2650e72c08..8b64f0db5728 100644 --- a/cmds/statsd/src/StatsService.cpp +++ b/cmds/statsd/src/StatsService.cpp @@ -22,6 +22,7 @@ #include "config/ConfigKey.h" #include "config/ConfigManager.h" #include "guardrail/MemoryLeakTrackUtil.h" +#include "guardrail/StatsdStats.h" #include "storage/DropboxReader.h" #include "storage/StorageManager.h" @@ -222,7 +223,7 @@ status_t StatsService::command(FILE* in, FILE* out, FILE* err, Vector<String8>& } if (!args[0].compare(String8("print-stats"))) { - return cmd_print_stats(out); + return cmd_print_stats(out, args); } if (!args[0].compare(String8("clear-config"))) { @@ -305,8 +306,9 @@ void StatsService::print_cmd_help(FILE* out) { fprintf(out, " NAME The name of the configuration\n"); fprintf(out, "\n"); fprintf(out, "\n"); - fprintf(out, "usage: adb shell cmd stats print-stats\n"); + fprintf(out, "usage: adb shell cmd stats print-stats [reset]\n"); fprintf(out, " Prints some basic stats.\n"); + fprintf(out, " reset: 1 to reset the statsd stats. default=0.\n"); } status_t StatsService::cmd_trigger_broadcast(FILE* out, Vector<String8>& args) { @@ -474,12 +476,20 @@ status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String } } -status_t StatsService::cmd_print_stats(FILE* out) { +status_t StatsService::cmd_print_stats(FILE* out, const Vector<String8>& args) { vector<ConfigKey> configs = mConfigManager->GetAllConfigKeys(); for (const ConfigKey& key : configs) { fprintf(out, "Config %s uses %zu bytes\n", key.ToString().c_str(), mProcessor->GetMetricsSize(key)); } + fprintf(out, "Detailed statsd stats in logcat..."); + StatsdStats& statsdStats = StatsdStats::getInstance(); + bool reset = false; + if (args.size() > 1) { + reset = strtol(args[1].string(), NULL, 10); + } + vector<int8_t> output; + statsdStats.dumpStats(&output, reset); return NO_ERROR; } diff --git a/cmds/statsd/src/StatsService.h b/cmds/statsd/src/StatsService.h index 6b5c156969e2..a32595a4a7f4 100644 --- a/cmds/statsd/src/StatsService.h +++ b/cmds/statsd/src/StatsService.h @@ -134,7 +134,7 @@ private: /** * Prints some basic stats to std out. */ - status_t cmd_print_stats(FILE* out); + status_t cmd_print_stats(FILE* out, const Vector<String8>& args); /** * Print the event log. diff --git a/cmds/statsd/src/condition/SimpleConditionTracker.cpp b/cmds/statsd/src/condition/SimpleConditionTracker.cpp index d8099b34f39a..50cd130e9f39 100644 --- a/cmds/statsd/src/condition/SimpleConditionTracker.cpp +++ b/cmds/statsd/src/condition/SimpleConditionTracker.cpp @@ -18,6 +18,7 @@ #include "Log.h" #include "SimpleConditionTracker.h" +#include "guardrail/StatsdStats.h" #include <log/logprint.h> @@ -32,9 +33,10 @@ using std::unordered_map; using std::vector; SimpleConditionTracker::SimpleConditionTracker( - const string& name, const int index, const SimpleCondition& simpleCondition, + const ConfigKey& key, const string& name, const int index, + const SimpleCondition& simpleCondition, const unordered_map<string, int>& trackerNameIndexMap) - : ConditionTracker(name, index) { + : ConditionTracker(name, index), mConfigKey(key) { VLOG("creating SimpleConditionTracker %s", mName.c_str()); mCountNesting = simpleCondition.count_nesting(); @@ -126,6 +128,24 @@ void SimpleConditionTracker::handleStopAll(std::vector<ConditionState>& conditio conditionCache[mIndex] = ConditionState::kFalse; } +bool SimpleConditionTracker::hitGuardRail(const HashableDimensionKey& newKey) { + if (!mSliced || mSlicedConditionState.find(newKey) != mSlicedConditionState.end()) { + // if the condition is not sliced or the key is not new, we are good! + return false; + } + // 1. Report the tuple count if the tuple count > soft limit + if (mSlicedConditionState.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + size_t newTupleCount = mSlicedConditionState.size() + 1; + StatsdStats::getInstance().noteConditionDimensionSize(mConfigKey, mName, newTupleCount); + // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. + if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + ALOGE("Condition %s dropping data for dimension key %s", mName.c_str(), newKey.c_str()); + return true; + } + } + return false; +} + void SimpleConditionTracker::handleConditionEvent(const HashableDimensionKey& outputKey, bool matchStart, std::vector<ConditionState>& conditionCache, @@ -133,6 +153,12 @@ void SimpleConditionTracker::handleConditionEvent(const HashableDimensionKey& ou bool changed = false; auto outputIt = mSlicedConditionState.find(outputKey); ConditionState newCondition; + if (hitGuardRail(outputKey)) { + conditionChangedCache[mIndex] = false; + // Tells the caller it's evaluated. + conditionCache[mIndex] = ConditionState::kUnknown; + return; + } if (outputIt == mSlicedConditionState.end()) { // We get a new output key. newCondition = matchStart ? ConditionState::kTrue : ConditionState::kFalse; diff --git a/cmds/statsd/src/condition/SimpleConditionTracker.h b/cmds/statsd/src/condition/SimpleConditionTracker.h index c287d8f34e8f..d21afd1d8083 100644 --- a/cmds/statsd/src/condition/SimpleConditionTracker.h +++ b/cmds/statsd/src/condition/SimpleConditionTracker.h @@ -19,6 +19,7 @@ #include <gtest/gtest_prod.h> #include "ConditionTracker.h" +#include "config/ConfigKey.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" #include "stats_util.h" @@ -28,7 +29,7 @@ namespace statsd { class SimpleConditionTracker : public virtual ConditionTracker { public: - SimpleConditionTracker(const std::string& name, const int index, + SimpleConditionTracker(const ConfigKey& key, const std::string& name, const int index, const SimpleCondition& simpleCondition, const std::unordered_map<std::string, int>& trackerNameIndexMap); @@ -50,6 +51,7 @@ public: std::vector<ConditionState>& conditionCache) const override; private: + const ConfigKey mConfigKey; // The index of the LogEventMatcher which defines the start. int mStartLogMatcherIndex; @@ -75,6 +77,8 @@ private: std::vector<ConditionState>& conditionCache, std::vector<bool>& changedCache); + bool hitGuardRail(const HashableDimensionKey& newKey); + FRIEND_TEST(SimpleConditionTrackerTest, TestSlicedCondition); FRIEND_TEST(SimpleConditionTrackerTest, TestSlicedWithNoOutputDim); FRIEND_TEST(SimpleConditionTrackerTest, TestStopAll); diff --git a/cmds/statsd/src/guardrail/StatsdStats.cpp b/cmds/statsd/src/guardrail/StatsdStats.cpp new file mode 100644 index 000000000000..815e03f6d3f1 --- /dev/null +++ b/cmds/statsd/src/guardrail/StatsdStats.cpp @@ -0,0 +1,342 @@ +/* + * Copyright 2017, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define DEBUG true // STOPSHIP if true +#include "Log.h" + +#include "StatsdStats.h" + +#include <android/util/ProtoOutputStream.h> +#include "statslog.h" + +namespace android { +namespace os { +namespace statsd { + +using android::util::FIELD_COUNT_REPEATED; +using android::util::FIELD_TYPE_BOOL; +using android::util::FIELD_TYPE_FLOAT; +using android::util::FIELD_TYPE_INT32; +using android::util::FIELD_TYPE_INT64; +using android::util::FIELD_TYPE_MESSAGE; +using android::util::FIELD_TYPE_STRING; +using android::util::ProtoOutputStream; +using std::lock_guard; +using std::map; +using std::string; +using std::vector; + +const int FIELD_ID_BEGIN_TIME = 1; +const int FIELD_ID_END_TIME = 2; +const int FIELD_ID_CONFIG_STATS = 3; +const int FIELD_ID_MATCHER_STATS = 4; +const int FIELD_ID_CONDITION_STATS = 5; +const int FIELD_ID_METRIC_STATS = 6; +const int FIELD_ID_ATOM_STATS = 7; + +const int FIELD_ID_MATCHER_STATS_NAME = 1; +const int FIELD_ID_MATCHER_STATS_COUNT = 2; + +const int FIELD_ID_CONDITION_STATS_NAME = 1; +const int FIELD_ID_CONDITION_STATS_COUNT = 2; + +const int FIELD_ID_METRIC_STATS_NAME = 1; +const int FIELD_ID_METRIC_STATS_COUNT = 2; + +const int FIELD_ID_ATOM_STATS_TAG = 1; +const int FIELD_ID_ATOM_STATS_COUNT = 2; + +// TODO: add stats for pulled atoms. +StatsdStats::StatsdStats() { + mPushedAtomStats.resize(android::util::kMaxPushedAtomId + 1); + mStartTime = time(nullptr); +} + +StatsdStats& StatsdStats::getInstance() { + static StatsdStats statsInstance; + return statsInstance; +} + +void StatsdStats::noteConfigReceived(const ConfigKey& key, int metricsCount, int conditionsCount, + int matchersCount, int alertsCount, bool isValid) { + lock_guard<std::mutex> lock(mLock); + int32_t nowTimeSec = time(nullptr); + + // If there is an existing config for the same key, icebox the old config. + noteConfigRemovedInternalLocked(key); + + StatsdStatsReport_ConfigStats configStats; + configStats.set_uid(key.GetUid()); + configStats.set_name(key.GetName()); + configStats.set_creation_time_sec(nowTimeSec); + configStats.set_metric_count(metricsCount); + configStats.set_condition_count(conditionsCount); + configStats.set_matcher_count(matchersCount); + configStats.set_alert_count(alertsCount); + configStats.set_is_valid(isValid); + + if (isValid) { + mConfigStats[key] = configStats; + } else { + configStats.set_deletion_time_sec(nowTimeSec); + mIceBox.push_back(configStats); + } +} + +void StatsdStats::noteConfigRemovedInternalLocked(const ConfigKey& key) { + auto it = mConfigStats.find(key); + if (it != mConfigStats.end()) { + int32_t nowTimeSec = time(nullptr); + it->second.set_deletion_time_sec(nowTimeSec); + // Add condition stats, metrics stats, matcher stats + addSubStatsToConfig(key, it->second); + // Remove them after they are added to the config stats. + mMatcherStats.erase(key); + mMetricsStats.erase(key); + mConditionStats.erase(key); + mIceBox.push_back(it->second); + } +} + +void StatsdStats::noteConfigRemoved(const ConfigKey& key) { + lock_guard<std::mutex> lock(mLock); + noteConfigRemovedInternalLocked(key); +} + +void StatsdStats::noteBroadcastSent(const ConfigKey& key) { + lock_guard<std::mutex> lock(mLock); + auto it = mConfigStats.find(key); + if (it == mConfigStats.end()) { + ALOGE("Config key %s not found!", key.ToString().c_str()); + return; + } + + it->second.add_broadcast_sent_time_sec(time(nullptr)); +} + +void StatsdStats::noteDataDrop(const ConfigKey& key) { + lock_guard<std::mutex> lock(mLock); + auto it = mConfigStats.find(key); + if (it == mConfigStats.end()) { + ALOGE("Config key %s not found!", key.ToString().c_str()); + return; + } + + it->second.add_data_drop_time_sec(time(nullptr)); +} + +void StatsdStats::noteConditionDimensionSize(const ConfigKey& key, const string& name, int size) { + lock_guard<std::mutex> lock(mLock); + // if name doesn't exist before, it will create the key with count 0. + auto& conditionSizeMap = mConditionStats[key]; + if (size > conditionSizeMap[name]) { + conditionSizeMap[name] = size; + } +} + +void StatsdStats::noteMetricDimensionSize(const ConfigKey& key, const string& name, int size) { + lock_guard<std::mutex> lock(mLock); + // if name doesn't exist before, it will create the key with count 0. + auto& metricsDimensionMap = mMetricsStats[key]; + if (size > metricsDimensionMap[name]) { + metricsDimensionMap[name] = size; + } +} + +void StatsdStats::noteMatcherMatched(const ConfigKey& key, const string& name) { + lock_guard<std::mutex> lock(mLock); + auto& matcherStats = mMatcherStats[key]; + matcherStats[name]++; +} + +void StatsdStats::noteAtomLogged(int atomId, int32_t timeSec) { + lock_guard<std::mutex> lock(mLock); + + if (timeSec < mStartTime) { + return; + } + + if (atomId > android::util::kMaxPushedAtomId) { + ALOGW("not interested in atom %d", atomId); + return; + } + + mPushedAtomStats[atomId]++; +} + +void StatsdStats::reset() { + lock_guard<std::mutex> lock(mLock); + resetInternalLocked(); +} + +void StatsdStats::resetInternalLocked() { + // Reset the historical data, but keep the active ConfigStats + mStartTime = time(nullptr); + mIceBox.clear(); + mConditionStats.clear(); + mMetricsStats.clear(); + std::fill(mPushedAtomStats.begin(), mPushedAtomStats.end(), 0); + mMatcherStats.clear(); +} + +void StatsdStats::addSubStatsToConfig(const ConfigKey& key, + StatsdStatsReport_ConfigStats& configStats) { + // Add matcher stats + if (mMatcherStats.find(key) != mMatcherStats.end()) { + const auto& matcherStats = mMatcherStats[key]; + for (const auto& stats : matcherStats) { + auto output = configStats.add_matcher_stats(); + output->set_name(stats.first); + output->set_matched_times(stats.second); + VLOG("matcher %s matched %d times", stats.first.c_str(), stats.second); + } + } + // Add condition stats + if (mConditionStats.find(key) != mConditionStats.end()) { + const auto& conditionStats = mConditionStats[key]; + for (const auto& stats : conditionStats) { + auto output = configStats.add_condition_stats(); + output->set_name(stats.first); + output->set_max_tuple_counts(stats.second); + VLOG("condition %s max output tuple size %d", stats.first.c_str(), stats.second); + } + } + // Add metrics stats + if (mMetricsStats.find(key) != mMetricsStats.end()) { + const auto& conditionStats = mMetricsStats[key]; + for (const auto& stats : conditionStats) { + auto output = configStats.add_metric_stats(); + output->set_name(stats.first); + output->set_max_tuple_counts(stats.second); + VLOG("metrics %s max output tuple size %d", stats.first.c_str(), stats.second); + } + } +} + +void StatsdStats::dumpStats(std::vector<int8_t>* output, bool reset) { + lock_guard<std::mutex> lock(mLock); + + if (DEBUG) { + time_t t = time(nullptr); + struct tm* tm = localtime(&t); + char timeBuffer[80]; + strftime(timeBuffer, sizeof(timeBuffer), "%Y-%m-%d %I:%M%p", tm); + VLOG("=================StatsdStats dump begins===================="); + VLOG("Stats collection start second: %s", timeBuffer); + } + ProtoOutputStream proto; + proto.write(FIELD_TYPE_INT32 | FIELD_ID_BEGIN_TIME, mStartTime); + proto.write(FIELD_TYPE_INT32 | FIELD_ID_END_TIME, (int32_t)time(nullptr)); + + VLOG("%lu Config in icebox: ", (unsigned long)mIceBox.size()); + for (const auto& configStats : mIceBox) { + const int numBytes = configStats.ByteSize(); + vector<char> buffer(numBytes); + configStats.SerializeToArray(&buffer[0], numBytes); + proto.write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_CONFIG_STATS, &buffer[0], + buffer.size()); + + // surround the whole block with DEBUG, so that compiler can strip out the code + // in production. + if (DEBUG) { + VLOG("*****ICEBOX*****"); + VLOG("Config {%d-%s}: creation=%d, deletion=%d, #metric=%d, #condition=%d, " + "#matcher=%d, #alert=%d, #valid=%d", + configStats.uid(), configStats.name().c_str(), configStats.creation_time_sec(), + configStats.deletion_time_sec(), configStats.metric_count(), + configStats.condition_count(), configStats.matcher_count(), + configStats.alert_count(), configStats.is_valid()); + + for (const auto& broadcastTime : configStats.broadcast_sent_time_sec()) { + VLOG("\tbroadcast time: %d", broadcastTime); + } + + for (const auto& dataDropTime : configStats.data_drop_time_sec()) { + VLOG("\tdata drop time: %d", dataDropTime); + } + } + } + + for (auto& pair : mConfigStats) { + auto& configStats = pair.second; + if (DEBUG) { + VLOG("********Active Configs***********"); + VLOG("Config {%d-%s}: creation=%d, deletion=%d, #metric=%d, #condition=%d, " + "#matcher=%d, #alert=%d, #valid=%d", + configStats.uid(), configStats.name().c_str(), configStats.creation_time_sec(), + configStats.deletion_time_sec(), configStats.metric_count(), + configStats.condition_count(), configStats.matcher_count(), + configStats.alert_count(), configStats.is_valid()); + for (const auto& broadcastTime : configStats.broadcast_sent_time_sec()) { + VLOG("\tbroadcast time: %d", broadcastTime); + } + + for (const auto& dataDropTime : configStats.data_drop_time_sec()) { + VLOG("\tdata drop time: %d", dataDropTime); + } + } + + addSubStatsToConfig(pair.first, configStats); + + const int numBytes = configStats.ByteSize(); + vector<char> buffer(numBytes); + configStats.SerializeToArray(&buffer[0], numBytes); + proto.write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_CONFIG_STATS, &buffer[0], + buffer.size()); + // reset the sub stats, the source of truth is in the individual map + // they will be repopulated when dumpStats() is called again. + configStats.clear_matcher_stats(); + configStats.clear_condition_stats(); + configStats.clear_metric_stats(); + } + + VLOG("********Atom stats***********"); + const size_t atomCounts = mPushedAtomStats.size(); + for (size_t i = 2; i < atomCounts; i++) { + if (mPushedAtomStats[i] > 0) { + long long token = + proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_ATOM_STATS | FIELD_COUNT_REPEATED); + proto.write(FIELD_TYPE_INT32 | FIELD_ID_ATOM_STATS_TAG, (int32_t)i); + proto.write(FIELD_TYPE_INT32 | FIELD_ID_ATOM_STATS_COUNT, mPushedAtomStats[i]); + proto.end(token); + + VLOG("Atom %lu->%d\n", (unsigned long)i, mPushedAtomStats[i]); + } + } + + output->clear(); + size_t bufferSize = proto.size(); + output->resize(bufferSize); + + size_t pos = 0; + auto it = proto.data(); + while (it.readBuffer() != NULL) { + size_t toRead = it.currentToRead(); + std::memcpy(&((*output)[pos]), it.readBuffer(), toRead); + pos += toRead; + it.rp()->move(toRead); + } + + if (reset) { + resetInternalLocked(); + } + + VLOG("reset=%d, returned proto size %lu", reset, (unsigned long)bufferSize); + VLOG("=================StatsdStats dump ends===================="); +} + +} // namespace statsd +} // namespace os +} // namespace android
\ No newline at end of file diff --git a/cmds/statsd/src/guardrail/StatsdStats.h b/cmds/statsd/src/guardrail/StatsdStats.h new file mode 100644 index 000000000000..73ce2798cd6c --- /dev/null +++ b/cmds/statsd/src/guardrail/StatsdStats.h @@ -0,0 +1,160 @@ +/* + * Copyright 2017, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "config/ConfigKey.h" +#include "frameworks/base/cmds/statsd/src/stats_log.pb.h" + +#include <mutex> +#include <string> +#include <vector> + +namespace android { +namespace os { +namespace statsd { + +// Keeps track of stats of statsd. +// Single instance shared across the process. All methods are thread safe. +class StatsdStats { +public: + static StatsdStats& getInstance(); + ~StatsdStats(){}; + + // TODO: set different limit if the device is low ram. + const static int kDimensionKeySizeSoftLimit = 300; + const static int kDimensionKeySizeHardLimit = 500; + + const static int kMaxConfigCount = 10; + const static int kMaxConditionCountPerConfig = 200; + const static int kMaxMetricCountPerConfig = 300; + const static int kMaxMatcherCountPerConfig = 500; + + /** + * Report a new config has been received and report the static stats about the config. + * + * The static stats include: the count of metrics, conditions, matchers, and alerts. + * If the config is not valid, this config stats will be put into icebox immediately. + */ + void noteConfigReceived(const ConfigKey& key, int metricsCount, int conditionsCount, + int matchersCount, int alertCount, bool isValid); + /** + * Report a config has been removed. + */ + void noteConfigRemoved(const ConfigKey& key); + + /** + * Report a broadcast has been sent to a config owner to collect the data. + */ + void noteBroadcastSent(const ConfigKey& key); + + /** + * Report a config's metrics data has been dropped. + */ + void noteDataDrop(const ConfigKey& key); + + /** + * Report the size of output tuple of a condition. + * + * Note: only report when the condition has an output dimension, and the tuple + * count > kDimensionKeySizeSoftLimit. + * + * [key]: The config key that this condition belongs to. + * [name]: The name of the condition. + * [size]: The output tuple size. + */ + void noteConditionDimensionSize(const ConfigKey& key, const std::string& name, int size); + + /** + * Report the size of output tuple of a metric. + * + * Note: only report when the metric has an output dimension, and the tuple + * count > kDimensionKeySizeSoftLimit. + * + * [key]: The config key that this metric belongs to. + * [name]: The name of the metric. + * [size]: The output tuple size. + */ + void noteMetricDimensionSize(const ConfigKey& key, const std::string& name, int size); + + /** + * Report a matcher has been matched. + * + * [key]: The config key that this matcher belongs to. + * [name]: The name of the matcher. + */ + void noteMatcherMatched(const ConfigKey& key, const std::string& name); + + /** + * Report an atom event has been logged. + */ + void noteAtomLogged(int atomId, int32_t timeSec); + + /** + * Reset the historical stats. Including all stats in icebox, and the tracked stats about + * metrics, matchers, and atoms. The active configs will be kept and StatsdStats will continue + * to collect stats after reset() has been called. + */ + void reset(); + + /** + * Output the stats in protobuf binary format to [buffer]. + * + * [reset]: whether to clear the historical stats after the call. + */ + void dumpStats(std::vector<int8_t>* buffer, bool reset); + +private: + StatsdStats(); + + mutable std::mutex mLock; + + int32_t mStartTime; + + // The stats about the configs that are still in use. + std::map<const ConfigKey, StatsdStatsReport_ConfigStats> mConfigStats; + + // Stores the stats for the configs that are no longer in use. + std::vector<const StatsdStatsReport_ConfigStats> mIceBox; + + // Stores the number of output tuple of condition trackers when it's bigger than + // kDimensionKeySizeSoftLimit. When you see the number is kDimensionKeySizeHardLimit +1, + // it means some data has been dropped. + std::map<const ConfigKey, std::map<const std::string, int>> mConditionStats; + + // Stores the number of output tuple of metric producers when it's bigger than + // kDimensionKeySizeSoftLimit. When you see the number is kDimensionKeySizeHardLimit +1, + // it means some data has been dropped. + std::map<const ConfigKey, std::map<const std::string, int>> mMetricsStats; + + // Stores the number of times a pushed atom is logged. + // The size of the vector is the largest pushed atom id in atoms.proto + 1. Atoms + // out of that range will be dropped (it's either pulled atoms or test atoms). + // This is a vector, not a map because it will be accessed A LOT -- for each stats log. + std::vector<int> mPushedAtomStats; + + // Stores how many times a matcher have been matched. + std::map<const ConfigKey, std::map<const std::string, int>> mMatcherStats; + + void noteConfigRemovedInternalLocked(const ConfigKey& key); + + void resetInternalLocked(); + + void addSubStatsToConfig(const ConfigKey& key, StatsdStatsReport_ConfigStats& configStats); +}; + +} // namespace statsd +} // namespace os +} // namespace android
\ No newline at end of file diff --git a/cmds/statsd/src/matchers/LogMatchingTracker.h b/cmds/statsd/src/matchers/LogMatchingTracker.h index ffbf2488ad2c..fea3e9b367ef 100644 --- a/cmds/statsd/src/matchers/LogMatchingTracker.h +++ b/cmds/statsd/src/matchers/LogMatchingTracker.h @@ -69,6 +69,10 @@ public: return mTagIds; } + const std::string& getName() const { + return mName; + } + protected: // Name of this matching. We don't really need the name, but it makes log message easy to debug. const std::string mName; diff --git a/cmds/statsd/src/matchers/SimpleLogMatchingTracker.cpp b/cmds/statsd/src/matchers/SimpleLogMatchingTracker.cpp index b2c88a0d3013..ad37b010d1fb 100644 --- a/cmds/statsd/src/matchers/SimpleLogMatchingTracker.cpp +++ b/cmds/statsd/src/matchers/SimpleLogMatchingTracker.cpp @@ -19,8 +19,6 @@ #include "SimpleLogMatchingTracker.h" -#include <log/logprint.h> - namespace android { namespace os { namespace statsd { diff --git a/cmds/statsd/src/metrics/CountMetricProducer.cpp b/cmds/statsd/src/metrics/CountMetricProducer.cpp index 149b9c16fd88..ce60eb965d62 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.cpp +++ b/cmds/statsd/src/metrics/CountMetricProducer.cpp @@ -18,6 +18,7 @@ #include "Log.h" #include "CountMetricProducer.h" +#include "guardrail/StatsdStats.h" #include "stats_util.h" #include <limits.h> @@ -63,10 +64,11 @@ const int FIELD_ID_COUNT = 3; // TODO: add back AnomalyTracker. -CountMetricProducer::CountMetricProducer(const CountMetric& metric, const int conditionIndex, +CountMetricProducer::CountMetricProducer(const ConfigKey& key, const CountMetric& metric, + const int conditionIndex, const sp<ConditionWizard>& wizard, const uint64_t startTimeNs) - : MetricProducer(startTimeNs, conditionIndex, wizard), mMetric(metric) { + : MetricProducer(key, startTimeNs, conditionIndex, wizard), mMetric(metric) { // TODO: evaluate initial conditions. and set mConditionMet. if (metric.has_bucket() && metric.bucket().has_bucket_size_millis()) { mBucketSizeNs = metric.bucket().bucket_size_millis() * 1000 * 1000; @@ -180,6 +182,26 @@ void CountMetricProducer::onConditionChanged(const bool conditionMet, const uint mCondition = conditionMet; } +bool CountMetricProducer::hitGuardRail(const HashableDimensionKey& newKey) { + if (mCurrentSlicedCounter->find(newKey) != mCurrentSlicedCounter->end()) { + return false; + } + // ===========GuardRail============== + // 1. Report the tuple count if the tuple count > soft limit + if (mCurrentSlicedCounter->size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + size_t newTupleCount = mCurrentSlicedCounter->size() + 1; + StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(), + newTupleCount); + // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. + if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + ALOGE("CountMetric %s dropping data for dimension key %s", mMetric.name().c_str(), + newKey.c_str()); + return true; + } + } + + return false; +} void CountMetricProducer::onMatchedLogEventInternal( const size_t matcherIndex, const HashableDimensionKey& eventKey, const map<string, HashableDimensionKey>& conditionKey, bool condition, @@ -195,6 +217,11 @@ void CountMetricProducer::onMatchedLogEventInternal( auto it = mCurrentSlicedCounter->find(eventKey); if (it == mCurrentSlicedCounter->end()) { + // ===========GuardRail============== + if (hitGuardRail(eventKey)) { + return; + } + // create a counter for the new key (*mCurrentSlicedCounter)[eventKey] = 1; } else { diff --git a/cmds/statsd/src/metrics/CountMetricProducer.h b/cmds/statsd/src/metrics/CountMetricProducer.h index 293e5b971fa5..f78a199de103 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.h +++ b/cmds/statsd/src/metrics/CountMetricProducer.h @@ -42,8 +42,9 @@ struct CountBucket { class CountMetricProducer : public MetricProducer { public: // TODO: Pass in the start time from MetricsManager, it should be consistent for all metrics. - CountMetricProducer(const CountMetric& countMetric, const int conditionIndex, - const sp<ConditionWizard>& wizard, const uint64_t startTimeNs); + CountMetricProducer(const ConfigKey& key, const CountMetric& countMetric, + const int conditionIndex, const sp<ConditionWizard>& wizard, + const uint64_t startTimeNs); virtual ~CountMetricProducer(); @@ -84,6 +85,8 @@ private: static const size_t kBucketSize = sizeof(CountBucket{}); + bool hitGuardRail(const HashableDimensionKey& newKey); + FRIEND_TEST(CountMetricProducerTest, TestNonDimensionalEvents); FRIEND_TEST(CountMetricProducerTest, TestEventsWithNonSlicedCondition); FRIEND_TEST(CountMetricProducerTest, TestEventsWithSlicedCondition); diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.cpp b/cmds/statsd/src/metrics/DurationMetricProducer.cpp index c8e6ccec07cd..a0374c0ba67c 100644 --- a/cmds/statsd/src/metrics/DurationMetricProducer.cpp +++ b/cmds/statsd/src/metrics/DurationMetricProducer.cpp @@ -18,6 +18,7 @@ #include "Log.h" #include "DurationMetricProducer.h" +#include "guardrail/StatsdStats.h" #include "stats_util.h" #include <limits.h> @@ -60,14 +61,14 @@ const int FIELD_ID_START_BUCKET_NANOS = 1; const int FIELD_ID_END_BUCKET_NANOS = 2; const int FIELD_ID_DURATION = 3; -DurationMetricProducer::DurationMetricProducer(const DurationMetric& metric, +DurationMetricProducer::DurationMetricProducer(const ConfigKey& key, const DurationMetric& metric, const int conditionIndex, const size_t startIndex, const size_t stopIndex, const size_t stopAllIndex, const bool nesting, const sp<ConditionWizard>& wizard, const vector<KeyMatcher>& internalDimension, const uint64_t startTimeNs) - : MetricProducer(startTimeNs, conditionIndex, wizard), + : MetricProducer(key, startTimeNs, conditionIndex, wizard), mMetric(metric), mStartIndex(startIndex), mStopIndex(stopIndex), @@ -113,13 +114,13 @@ unique_ptr<DurationTracker> DurationMetricProducer::createDurationTracker( const HashableDimensionKey& eventKey, vector<DurationBucket>& bucket) { switch (mMetric.aggregation_type()) { case DurationMetric_AggregationType_SUM: - return make_unique<OringDurationTracker>(eventKey, mWizard, mConditionTrackerIndex, - mNested, mCurrentBucketStartTimeNs, - mBucketSizeNs, mAnomalyTrackers, bucket); + return make_unique<OringDurationTracker>( + mConfigKey, mMetric.name(), eventKey, mWizard, mConditionTrackerIndex, mNested, + mCurrentBucketStartTimeNs, mBucketSizeNs, mAnomalyTrackers, bucket); case DurationMetric_AggregationType_MAX_SPARSE: - return make_unique<MaxDurationTracker>(eventKey, mWizard, mConditionTrackerIndex, - mNested, mCurrentBucketStartTimeNs, - mBucketSizeNs, mAnomalyTrackers, bucket); + return make_unique<MaxDurationTracker>( + mConfigKey, mMetric.name(), eventKey, mWizard, mConditionTrackerIndex, mNested, + mCurrentBucketStartTimeNs, mBucketSizeNs, mAnomalyTrackers, bucket); } } @@ -238,6 +239,26 @@ void DurationMetricProducer::flushIfNeeded(uint64_t eventTime) { mCurrentBucketNum += numBucketsForward; } +bool DurationMetricProducer::hitGuardRail(const HashableDimensionKey& newKey) { + // the key is not new, we are good. + if (mCurrentSlicedDuration.find(newKey) != mCurrentSlicedDuration.end()) { + return false; + } + // 1. Report the tuple count if the tuple count > soft limit + if (mCurrentSlicedDuration.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + size_t newTupleCount = mCurrentSlicedDuration.size() + 1; + StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(), + newTupleCount); + // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. + if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + ALOGE("DurationMetric %s dropping data for dimension key %s", mMetric.name().c_str(), + newKey.c_str()); + return true; + } + } + return false; +} + void DurationMetricProducer::onMatchedLogEventInternal( const size_t matcherIndex, const HashableDimensionKey& eventKey, const map<string, HashableDimensionKey>& conditionKeys, bool condition, @@ -254,6 +275,9 @@ void DurationMetricProducer::onMatchedLogEventInternal( HashableDimensionKey atomKey = getHashableKey(getDimensionKey(event, mInternalDimension)); if (mCurrentSlicedDuration.find(eventKey) == mCurrentSlicedDuration.end()) { + if (hitGuardRail(eventKey)) { + return; + } mCurrentSlicedDuration[eventKey] = createDurationTracker(eventKey, mPastBuckets[eventKey]); } diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.h b/cmds/statsd/src/metrics/DurationMetricProducer.h index ebd5e8d36412..5b5373ec9aeb 100644 --- a/cmds/statsd/src/metrics/DurationMetricProducer.h +++ b/cmds/statsd/src/metrics/DurationMetricProducer.h @@ -37,9 +37,9 @@ namespace statsd { class DurationMetricProducer : public MetricProducer { public: - DurationMetricProducer(const DurationMetric& durationMetric, const int conditionIndex, - const size_t startIndex, const size_t stopIndex, - const size_t stopAllIndex, const bool nesting, + DurationMetricProducer(const ConfigKey& key, const DurationMetric& durationMetric, + const int conditionIndex, const size_t startIndex, + const size_t stopIndex, const size_t stopAllIndex, const bool nesting, const sp<ConditionWizard>& wizard, const vector<KeyMatcher>& internalDimension, const uint64_t startTimeNs); @@ -98,6 +98,7 @@ private: std::unique_ptr<DurationTracker> createDurationTracker(const HashableDimensionKey& eventKey, std::vector<DurationBucket>& bucket); + bool hitGuardRail(const HashableDimensionKey& newKey); static const size_t kBucketSize = sizeof(DurationBucket{}); }; diff --git a/cmds/statsd/src/metrics/EventMetricProducer.cpp b/cmds/statsd/src/metrics/EventMetricProducer.cpp index 567b4c77bf7b..95a18f7241f0 100644 --- a/cmds/statsd/src/metrics/EventMetricProducer.cpp +++ b/cmds/statsd/src/metrics/EventMetricProducer.cpp @@ -51,10 +51,11 @@ const int FIELD_ID_DATA = 1; const int FIELD_ID_TIMESTAMP_NANOS = 1; const int FIELD_ID_ATOMS = 2; -EventMetricProducer::EventMetricProducer(const EventMetric& metric, const int conditionIndex, +EventMetricProducer::EventMetricProducer(const ConfigKey& key, const EventMetric& metric, + const int conditionIndex, const sp<ConditionWizard>& wizard, const uint64_t startTimeNs) - : MetricProducer(startTimeNs, conditionIndex, wizard), mMetric(metric) { + : MetricProducer(key, startTimeNs, conditionIndex, wizard), mMetric(metric) { if (metric.links().size() > 0) { mConditionLinks.insert(mConditionLinks.begin(), metric.links().begin(), metric.links().end()); diff --git a/cmds/statsd/src/metrics/EventMetricProducer.h b/cmds/statsd/src/metrics/EventMetricProducer.h index 5afcebd29e58..33a951035dca 100644 --- a/cmds/statsd/src/metrics/EventMetricProducer.h +++ b/cmds/statsd/src/metrics/EventMetricProducer.h @@ -34,8 +34,9 @@ namespace statsd { class EventMetricProducer : public MetricProducer { public: // TODO: Pass in the start time from MetricsManager, it should be consistent for all metrics. - EventMetricProducer(const EventMetric& eventMetric, const int conditionIndex, - const sp<ConditionWizard>& wizard, const uint64_t startTimeNs); + EventMetricProducer(const ConfigKey& key, const EventMetric& eventMetric, + const int conditionIndex, const sp<ConditionWizard>& wizard, + const uint64_t startTimeNs); virtual ~EventMetricProducer(); diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp index be030d856aee..1791654ba7cc 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.cpp +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.cpp @@ -18,6 +18,7 @@ #include "Log.h" #include "GaugeMetricProducer.h" +#include "guardrail/StatsdStats.h" #include "stats_util.h" #include <cutils/log.h> @@ -62,10 +63,11 @@ const int FIELD_ID_START_BUCKET_NANOS = 1; const int FIELD_ID_END_BUCKET_NANOS = 2; const int FIELD_ID_GAUGE = 3; -GaugeMetricProducer::GaugeMetricProducer(const GaugeMetric& metric, const int conditionIndex, +GaugeMetricProducer::GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& metric, + const int conditionIndex, const sp<ConditionWizard>& wizard, const int pullTagId, const int64_t startTimeNs) - : MetricProducer(startTimeNs, conditionIndex, wizard), + : MetricProducer(key, startTimeNs, conditionIndex, wizard), mMetric(metric), mPullTagId(pullTagId) { if (metric.has_bucket() && metric.bucket().has_bucket_size_millis()) { @@ -225,6 +227,26 @@ void GaugeMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEven } } +bool GaugeMetricProducer::hitGuardRail(const HashableDimensionKey& newKey) { + if (mCurrentSlicedBucket->find(newKey) != mCurrentSlicedBucket->end()) { + return false; + } + // 1. Report the tuple count if the tuple count > soft limit + if (mCurrentSlicedBucket->size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + size_t newTupleCount = mCurrentSlicedBucket->size() + 1; + StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(), + newTupleCount); + // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. + if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + ALOGE("GaugeMetric %s dropping data for dimension key %s", mMetric.name().c_str(), + newKey.c_str()); + return true; + } + } + + return false; +} + void GaugeMetricProducer::onMatchedLogEventInternal( const size_t matcherIndex, const HashableDimensionKey& eventKey, const map<string, HashableDimensionKey>& conditionKey, bool condition, @@ -244,12 +266,15 @@ void GaugeMetricProducer::onMatchedLogEventInternal( flushIfNeeded(eventTimeNs); } - // For gauge metric, we just simply use the first guage in the given bucket. + // For gauge metric, we just simply use the first gauge in the given bucket. if (!mCurrentSlicedBucket->empty()) { return; } const long gauge = getGauge(event); if (gauge >= 0) { + if (hitGuardRail(eventKey)) { + return; + } (*mCurrentSlicedBucket)[eventKey] = gauge; } for (auto& tracker : mAnomalyTrackers) { diff --git a/cmds/statsd/src/metrics/GaugeMetricProducer.h b/cmds/statsd/src/metrics/GaugeMetricProducer.h index 4b7654b4ca6b..f344303179e8 100644 --- a/cmds/statsd/src/metrics/GaugeMetricProducer.h +++ b/cmds/statsd/src/metrics/GaugeMetricProducer.h @@ -47,9 +47,9 @@ class GaugeMetricProducer : public virtual MetricProducer, public virtual PullDa public: // TODO: Pass in the start time from MetricsManager, it should be consistent // for all metrics. - GaugeMetricProducer(const GaugeMetric& countMetric, const int conditionIndex, - const sp<ConditionWizard>& wizard, const int pullTagId, - const int64_t startTimeNs); + GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& countMetric, + const int conditionIndex, const sp<ConditionWizard>& wizard, + const int pullTagId, const int64_t startTimeNs); virtual ~GaugeMetricProducer(); @@ -100,6 +100,8 @@ private: int64_t getGauge(const LogEvent& event); + bool hitGuardRail(const HashableDimensionKey& newKey); + static const size_t kBucketSize = sizeof(GaugeBucket{}); FRIEND_TEST(GaugeMetricProducerTest, TestWithCondition); diff --git a/cmds/statsd/src/metrics/MetricProducer.h b/cmds/statsd/src/metrics/MetricProducer.h index ddccf9a1970b..b22ff6f3348c 100644 --- a/cmds/statsd/src/metrics/MetricProducer.h +++ b/cmds/statsd/src/metrics/MetricProducer.h @@ -19,6 +19,7 @@ #include "anomaly/AnomalyTracker.h" #include "condition/ConditionWizard.h" +#include "config/ConfigKey.h" #include "matchers/matcher_util.h" #include "packages/PackageInfoListener.h" @@ -35,9 +36,10 @@ namespace statsd { // be a no-op. class MetricProducer : public virtual PackageInfoListener { public: - MetricProducer(const int64_t startTimeNs, const int conditionIndex, + MetricProducer(const ConfigKey& key, const int64_t startTimeNs, const int conditionIndex, const sp<ConditionWizard>& wizard) - : mStartTimeNs(startTimeNs), + : mConfigKey(key), + mStartTimeNs(startTimeNs), mCurrentBucketStartTimeNs(startTimeNs), mCurrentBucketNum(0), mCondition(conditionIndex >= 0 ? false : true), @@ -83,6 +85,8 @@ public: } protected: + const ConfigKey mConfigKey; + const uint64_t mStartTimeNs; uint64_t mCurrentBucketStartTimeNs; diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp index 5916b040889c..c8669519119a 100644 --- a/cmds/statsd/src/metrics/MetricsManager.cpp +++ b/cmds/statsd/src/metrics/MetricsManager.cpp @@ -20,6 +20,7 @@ #include "CountMetricProducer.h" #include "condition/CombinationConditionTracker.h" #include "condition/SimpleConditionTracker.h" +#include "guardrail/StatsdStats.h" #include "matchers/CombinationLogMatchingTracker.h" #include "matchers/SimpleLogMatchingTracker.h" #include "metrics_manager_util.h" @@ -36,10 +37,24 @@ namespace android { namespace os { namespace statsd { -MetricsManager::MetricsManager(const StatsdConfig& config) { - mConfigValid = initStatsdConfig(config, mTagIds, mAllLogEntryMatchers, mAllConditionTrackers, - mAllMetricProducers, mAllAnomalyTrackers, mConditionToMetricMap, - mTrackerToMetricMap, mTrackerToConditionMap); +MetricsManager::MetricsManager(const ConfigKey& key, const StatsdConfig& config) : mConfigKey(key) { + mConfigValid = + initStatsdConfig(key, config, mTagIds, mAllLogEntryMatchers, mAllConditionTrackers, + mAllMetricProducers, mAllAnomalyTrackers, mConditionToMetricMap, + mTrackerToMetricMap, mTrackerToConditionMap); + + // TODO: add alert size. + // no matter whether this config is valid, log it in the stats. + StatsdStats::getInstance().noteConfigReceived(key, mAllMetricProducers.size(), + mAllConditionTrackers.size(), + mAllLogEntryMatchers.size(), 0, mConfigValid); + // Guardrail. Reject the config if it's too big. + if (mAllMetricProducers.size() > StatsdStats::kMaxMetricCountPerConfig || + mAllConditionTrackers.size() > StatsdStats::kMaxConditionCountPerConfig || + mAllLogEntryMatchers.size() > StatsdStats::kMaxMatcherCountPerConfig) { + ALOGE("This config is too big! Reject!"); + mConfigValid = false; + } } MetricsManager::~MetricsManager() { @@ -137,6 +152,8 @@ void MetricsManager::onLogEvent(const LogEvent& event) { // For matched LogEntryMatchers, tell relevant metrics that a matched event has come. for (size_t i = 0; i < mAllLogEntryMatchers.size(); i++) { if (matcherCache[i] == MatchingState::kMatched) { + StatsdStats::getInstance().noteMatcherMatched(mConfigKey, + mAllLogEntryMatchers[i]->getName()); auto pair = mTrackerToMetricMap.find(i); if (pair != mTrackerToMetricMap.end()) { auto& metricList = pair->second; diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h index a6054e353aa2..517831d1b09d 100644 --- a/cmds/statsd/src/metrics/MetricsManager.h +++ b/cmds/statsd/src/metrics/MetricsManager.h @@ -19,6 +19,7 @@ #include "anomaly/AnomalyMonitor.h" #include "anomaly/AnomalyTracker.h" #include "condition/ConditionTracker.h" +#include "config/ConfigKey.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" #include "logd/LogEvent.h" #include "matchers/LogMatchingTracker.h" @@ -33,7 +34,7 @@ namespace statsd { // A MetricsManager is responsible for managing metrics from one single config source. class MetricsManager { public: - MetricsManager(const StatsdConfig& config); + MetricsManager(const ConfigKey& configKey, const StatsdConfig& config); ~MetricsManager(); @@ -57,6 +58,8 @@ public: size_t byteSize(); private: + const ConfigKey mConfigKey; + // All event tags that are interesting to my metrics. std::set<int> mTagIds; diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.cpp b/cmds/statsd/src/metrics/ValueMetricProducer.cpp index cc02c69d4405..66c8419de662 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.cpp +++ b/cmds/statsd/src/metrics/ValueMetricProducer.cpp @@ -18,6 +18,7 @@ #include "Log.h" #include "ValueMetricProducer.h" +#include "guardrail/StatsdStats.h" #include <cutils/log.h> #include <limits.h> @@ -67,11 +68,12 @@ const int FIELD_ID_VALUE = 3; static const uint64_t kDefaultBucketSizeMillis = 60 * 60 * 1000L; // ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently -ValueMetricProducer::ValueMetricProducer(const ValueMetric& metric, const int conditionIndex, +ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric, + const int conditionIndex, const sp<ConditionWizard>& wizard, const int pullTagId, const uint64_t startTimeNs, shared_ptr<StatsPullerManager> statsPullerManager) - : MetricProducer(startTimeNs, conditionIndex, wizard), + : MetricProducer(key, startTimeNs, conditionIndex, wizard), mMetric(metric), mStatsPullerManager(statsPullerManager), mPullTagId(pullTagId) { @@ -103,10 +105,11 @@ ValueMetricProducer::ValueMetricProducer(const ValueMetric& metric, const int co } // for testing -ValueMetricProducer::ValueMetricProducer(const ValueMetric& metric, const int conditionIndex, +ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric, + const int conditionIndex, const sp<ConditionWizard>& wizard, const int pullTagId, const uint64_t startTimeNs) - : ValueMetricProducer(metric, conditionIndex, wizard, pullTagId, startTimeNs, + : ValueMetricProducer(key, metric, conditionIndex, wizard, pullTagId, startTimeNs, make_shared<StatsPullerManager>()) { } @@ -238,6 +241,27 @@ void ValueMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEven } } +bool ValueMetricProducer::hitGuardRail(const HashableDimensionKey& newKey) { + // ===========GuardRail============== + // 1. Report the tuple count if the tuple count > soft limit + if (mCurrentSlicedBucket.find(newKey) != mCurrentSlicedBucket.end()) { + return false; + } + if (mCurrentSlicedBucket.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + size_t newTupleCount = mCurrentSlicedBucket.size() + 1; + StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(), + newTupleCount); + // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. + if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + ALOGE("ValueMetric %s dropping data for dimension key %s", mMetric.name().c_str(), + newKey.c_str()); + return true; + } + } + + return false; +} + void ValueMetricProducer::onMatchedLogEventInternal( const size_t matcherIndex, const HashableDimensionKey& eventKey, const map<string, HashableDimensionKey>& conditionKey, bool condition, @@ -249,6 +273,9 @@ void ValueMetricProducer::onMatchedLogEventInternal( return; } + if (hitGuardRail(eventKey)) { + return; + } Interval& interval = mCurrentSlicedBucket[eventKey]; long value = get_value(event); diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.h b/cmds/statsd/src/metrics/ValueMetricProducer.h index 24c76f26dcaa..a024bd804d83 100644 --- a/cmds/statsd/src/metrics/ValueMetricProducer.h +++ b/cmds/statsd/src/metrics/ValueMetricProducer.h @@ -38,9 +38,9 @@ struct ValueBucket { class ValueMetricProducer : public virtual MetricProducer, public virtual PullDataReceiver { public: - ValueMetricProducer(const ValueMetric& valueMetric, const int conditionIndex, - const sp<ConditionWizard>& wizard, const int pullTagId, - const uint64_t startTimeNs); + ValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric, + const int conditionIndex, const sp<ConditionWizard>& wizard, + const int pullTagId, const uint64_t startTimeNs); virtual ~ValueMetricProducer(); @@ -77,9 +77,9 @@ private: std::shared_ptr<StatsPullerManager> mStatsPullerManager; // for testing - ValueMetricProducer(const ValueMetric& valueMetric, const int conditionIndex, - const sp<ConditionWizard>& wizard, const int pullTagId, - const uint64_t startTimeNs, + ValueMetricProducer(const ConfigKey& key, const ValueMetric& valueMetric, + const int conditionIndex, const sp<ConditionWizard>& wizard, + const int pullTagId, const uint64_t startTimeNs, std::shared_ptr<StatsPullerManager> statsPullerManager); Mutex mLock; @@ -104,6 +104,8 @@ private: long get_value(const LogEvent& event); + bool hitGuardRail(const HashableDimensionKey& newKey); + static const size_t kBucketSize = sizeof(ValueBucket{}); FRIEND_TEST(ValueMetricProducerTest, TestNonDimensionalEvents); diff --git a/cmds/statsd/src/metrics/duration_helper/DurationTracker.h b/cmds/statsd/src/metrics/duration_helper/DurationTracker.h index 7ce7f02e06bb..834f7f5b0b86 100644 --- a/cmds/statsd/src/metrics/duration_helper/DurationTracker.h +++ b/cmds/statsd/src/metrics/duration_helper/DurationTracker.h @@ -19,6 +19,7 @@ #include "anomaly/AnomalyTracker.h" #include "condition/ConditionWizard.h" +#include "config/ConfigKey.h" #include "stats_util.h" namespace android { @@ -59,11 +60,14 @@ struct DurationBucket { class DurationTracker { public: - DurationTracker(const HashableDimensionKey& eventKey, sp<ConditionWizard> wizard, - int conditionIndex, bool nesting, uint64_t currentBucketStartNs, - uint64_t bucketSizeNs, const std::vector<sp<AnomalyTracker>>& anomalyTrackers, + DurationTracker(const ConfigKey& key, const string& name, const HashableDimensionKey& eventKey, + sp<ConditionWizard> wizard, int conditionIndex, bool nesting, + uint64_t currentBucketStartNs, uint64_t bucketSizeNs, + const std::vector<sp<AnomalyTracker>>& anomalyTrackers, std::vector<DurationBucket>& bucket) - : mEventKey(eventKey), + : mConfigKey(key), + mName(name), + mEventKey(eventKey), mWizard(wizard), mConditionTrackerIndex(conditionIndex), mBucketSizeNs(bucketSizeNs), @@ -138,6 +142,10 @@ protected: } } } + // A reference to the DurationMetricProducer's config key. + const ConfigKey& mConfigKey; + + const std::string mName; HashableDimensionKey mEventKey; diff --git a/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.cpp b/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.cpp index e4b3693fb24d..4b346dd25050 100644 --- a/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.cpp +++ b/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.cpp @@ -18,23 +18,50 @@ #include "Log.h" #include "MaxDurationTracker.h" +#include "guardrail/StatsdStats.h" namespace android { namespace os { namespace statsd { -MaxDurationTracker::MaxDurationTracker(const HashableDimensionKey& eventKey, +MaxDurationTracker::MaxDurationTracker(const ConfigKey& key, const string& name, + const HashableDimensionKey& eventKey, sp<ConditionWizard> wizard, int conditionIndex, bool nesting, uint64_t currentBucketStartNs, uint64_t bucketSizeNs, const std::vector<sp<AnomalyTracker>>& anomalyTrackers, std::vector<DurationBucket>& bucket) - : DurationTracker(eventKey, wizard, conditionIndex, nesting, currentBucketStartNs, bucketSizeNs, - anomalyTrackers, bucket) { + : DurationTracker(key, name, eventKey, wizard, conditionIndex, nesting, currentBucketStartNs, + bucketSizeNs, anomalyTrackers, bucket) { +} + +bool MaxDurationTracker::hitGuardRail(const HashableDimensionKey& newKey) { + // ===========GuardRail============== + if (mInfos.find(newKey) != mInfos.end()) { + // if the key existed, we are good! + return false; + } + // 1. Report the tuple count if the tuple count > soft limit + if (mInfos.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + size_t newTupleCount = mInfos.size() + 1; + StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mName + mEventKey, + newTupleCount); + // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. + if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + ALOGE("MaxDurTracker %s dropping data for dimension key %s", mName.c_str(), + newKey.c_str()); + return true; + } + } + return false; } void MaxDurationTracker::noteStart(const HashableDimensionKey& key, bool condition, const uint64_t eventTime, const ConditionKey& conditionKey) { // this will construct a new DurationInfo if this key didn't exist. + if (hitGuardRail(key)) { + return; + } + DurationInfo& duration = mInfos[key]; duration.conditionKeys = conditionKey; VLOG("MaxDuration: key %s start condition %d", key.c_str(), condition); diff --git a/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.h b/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.h index 0f79ffed4ec2..e0d1466c6fc4 100644 --- a/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.h +++ b/cmds/statsd/src/metrics/duration_helper/MaxDurationTracker.h @@ -28,7 +28,8 @@ namespace statsd { // they stop or bucket expires. class MaxDurationTracker : public DurationTracker { public: - MaxDurationTracker(const HashableDimensionKey& eventKey, sp<ConditionWizard> wizard, + MaxDurationTracker(const ConfigKey& key, const string& name, + const HashableDimensionKey& eventKey, sp<ConditionWizard> wizard, int conditionIndex, bool nesting, uint64_t currentBucketStartNs, uint64_t bucketSizeNs, const std::vector<sp<AnomalyTracker>>& anomalyTrackers, @@ -53,6 +54,9 @@ private: void noteConditionChanged(const HashableDimensionKey& key, bool conditionMet, const uint64_t timestamp); + // return true if we should not allow newKey to be tracked because we are above the threshold + bool hitGuardRail(const HashableDimensionKey& newKey); + FRIEND_TEST(MaxDurationTrackerTest, TestSimpleMaxDuration); FRIEND_TEST(MaxDurationTrackerTest, TestCrossBucketBoundary); FRIEND_TEST(MaxDurationTrackerTest, TestMaxDurationWithCondition); diff --git a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp index 76f8514178f8..22c33d60169e 100644 --- a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp +++ b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.cpp @@ -16,6 +16,7 @@ #define DEBUG true #include "Log.h" #include "OringDurationTracker.h" +#include "guardrail/StatsdStats.h" namespace android { namespace os { @@ -23,21 +24,45 @@ namespace statsd { using std::pair; -OringDurationTracker::OringDurationTracker(const HashableDimensionKey& eventKey, +OringDurationTracker::OringDurationTracker(const ConfigKey& key, const string& name, + const HashableDimensionKey& eventKey, sp<ConditionWizard> wizard, int conditionIndex, bool nesting, uint64_t currentBucketStartNs, uint64_t bucketSizeNs, const std::vector<sp<AnomalyTracker>>& anomalyTrackers, std::vector<DurationBucket>& bucket) - : DurationTracker(eventKey, wizard, conditionIndex, nesting, currentBucketStartNs, bucketSizeNs, - anomalyTrackers, bucket), + : DurationTracker(key, name, eventKey, wizard, conditionIndex, nesting, currentBucketStartNs, + bucketSizeNs, anomalyTrackers, bucket), mStarted(), mPaused() { mLastStartTime = 0; } +bool OringDurationTracker::hitGuardRail(const HashableDimensionKey& newKey) { + // ===========GuardRail============== + // 1. Report the tuple count if the tuple count > soft limit + if (mConditionKeyMap.find(newKey) != mConditionKeyMap.end()) { + return false; + } + if (mConditionKeyMap.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) { + size_t newTupleCount = mConditionKeyMap.size() + 1; + StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mName + mEventKey, + newTupleCount); + // 2. Don't add more tuples, we are above the allowed threshold. Drop the data. + if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) { + ALOGE("OringDurTracker %s dropping data for dimension key %s", mName.c_str(), + newKey.c_str()); + return true; + } + } + return false; +} + void OringDurationTracker::noteStart(const HashableDimensionKey& key, bool condition, const uint64_t eventTime, const ConditionKey& conditionKey) { + if (hitGuardRail(key)) { + return; + } if (condition) { if (mStarted.size() == 0) { mLastStartTime = eventTime; diff --git a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.h b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.h index ef32fdb510d1..a8404a961dd2 100644 --- a/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.h +++ b/cmds/statsd/src/metrics/duration_helper/OringDurationTracker.h @@ -27,7 +27,8 @@ namespace statsd { // Tracks the "Or'd" duration -- if 2 durations are overlapping, they won't be double counted. class OringDurationTracker : public DurationTracker { public: - OringDurationTracker(const HashableDimensionKey& eventKey, sp<ConditionWizard> wizard, + OringDurationTracker(const ConfigKey& key, const string& name, + const HashableDimensionKey& eventKey, sp<ConditionWizard> wizard, int conditionIndex, bool nesting, uint64_t currentBucketStartNs, uint64_t bucketSizeNs, const std::vector<sp<AnomalyTracker>>& anomalyTrackers, @@ -58,6 +59,9 @@ private: int64_t mLastStartTime; std::map<HashableDimensionKey, ConditionKey> mConditionKeyMap; + // return true if we should not allow newKey to be tracked because we are above the threshold + bool hitGuardRail(const HashableDimensionKey& newKey); + FRIEND_TEST(OringDurationTrackerTest, TestDurationOverlap); FRIEND_TEST(OringDurationTrackerTest, TestCrossBucketBoundary); FRIEND_TEST(OringDurationTrackerTest, TestDurationConditionChange); diff --git a/cmds/statsd/src/metrics/metrics_manager_util.cpp b/cmds/statsd/src/metrics/metrics_manager_util.cpp index 466026350ddd..1b9efc51c7b7 100644 --- a/cmds/statsd/src/metrics/metrics_manager_util.cpp +++ b/cmds/statsd/src/metrics/metrics_manager_util.cpp @@ -134,7 +134,8 @@ bool initLogTrackers(const StatsdConfig& config, unordered_map<string, int>& log return true; } -bool initConditions(const StatsdConfig& config, const unordered_map<string, int>& logTrackerMap, +bool initConditions(const ConfigKey& key, const StatsdConfig& config, + const unordered_map<string, int>& logTrackerMap, unordered_map<string, int>& conditionTrackerMap, vector<sp<ConditionTracker>>& allConditionTrackers, unordered_map<int, std::vector<int>>& trackerToConditionMap) { @@ -149,7 +150,7 @@ bool initConditions(const StatsdConfig& config, const unordered_map<string, int> switch (condition.contents_case()) { case Condition::ContentsCase::kSimpleCondition: { allConditionTrackers.push_back(new SimpleConditionTracker( - condition.name(), index, condition.simple_condition(), logTrackerMap)); + key, condition.name(), index, condition.simple_condition(), logTrackerMap)); break; } case Condition::ContentsCase::kCombination: { @@ -184,7 +185,8 @@ bool initConditions(const StatsdConfig& config, const unordered_map<string, int> return true; } -bool initMetrics(const StatsdConfig& config, const unordered_map<string, int>& logTrackerMap, +bool initMetrics(const ConfigKey& key, const StatsdConfig& config, + const unordered_map<string, int>& logTrackerMap, const unordered_map<string, int>& conditionTrackerMap, const vector<sp<LogMatchingTracker>>& allLogEntryMatchers, vector<sp<ConditionTracker>>& allConditionTrackers, @@ -230,7 +232,7 @@ bool initMetrics(const StatsdConfig& config, const unordered_map<string, int>& l } sp<MetricProducer> countProducer = - new CountMetricProducer(metric, conditionIndex, wizard, startTimeNs); + new CountMetricProducer(key, metric, conditionIndex, wizard, startTimeNs); allMetricProducers.push_back(countProducer); } @@ -298,8 +300,8 @@ bool initMetrics(const StatsdConfig& config, const unordered_map<string, int>& l } sp<MetricProducer> durationMetric = new DurationMetricProducer( - metric, conditionIndex, trackerIndices[0], trackerIndices[1], trackerIndices[2], - nesting, wizard, internalDimension, startTimeNs); + key, metric, conditionIndex, trackerIndices[0], trackerIndices[1], + trackerIndices[2], nesting, wizard, internalDimension, startTimeNs); allMetricProducers.push_back(durationMetric); } @@ -332,7 +334,7 @@ bool initMetrics(const StatsdConfig& config, const unordered_map<string, int>& l } sp<MetricProducer> eventMetric = - new EventMetricProducer(metric, conditionIndex, wizard, startTimeNs); + new EventMetricProducer(key, metric, conditionIndex, wizard, startTimeNs); allMetricProducers.push_back(eventMetric); } @@ -378,8 +380,8 @@ bool initMetrics(const StatsdConfig& config, const unordered_map<string, int>& l } } - sp<MetricProducer> valueProducer = - new ValueMetricProducer(metric, conditionIndex, wizard, pullTagId, startTimeNs); + sp<MetricProducer> valueProducer = new ValueMetricProducer(key, metric, conditionIndex, + wizard, pullTagId, startTimeNs); allMetricProducers.push_back(valueProducer); } @@ -424,8 +426,8 @@ bool initMetrics(const StatsdConfig& config, const unordered_map<string, int>& l } } - sp<MetricProducer> gaugeProducer = - new GaugeMetricProducer(metric, conditionIndex, wizard, pullTagId, startTimeNs); + sp<MetricProducer> gaugeProducer = new GaugeMetricProducer(key, metric, conditionIndex, + wizard, pullTagId, startTimeNs); allMetricProducers.push_back(gaugeProducer); } return true; @@ -451,7 +453,7 @@ bool initAlerts(const StatsdConfig& config, const unordered_map<string, int>& me return true; } -bool initStatsdConfig(const StatsdConfig& config, set<int>& allTagIds, +bool initStatsdConfig(const ConfigKey& key, const StatsdConfig& config, set<int>& allTagIds, vector<sp<LogMatchingTracker>>& allLogEntryMatchers, vector<sp<ConditionTracker>>& allConditionTrackers, vector<sp<MetricProducer>>& allMetricProducers, @@ -469,13 +471,13 @@ bool initStatsdConfig(const StatsdConfig& config, set<int>& allTagIds, } ALOGD("initLogMatchingTrackers succeed..."); - if (!initConditions(config, logTrackerMap, conditionTrackerMap, allConditionTrackers, + if (!initConditions(key, config, logTrackerMap, conditionTrackerMap, allConditionTrackers, trackerToConditionMap)) { ALOGE("initConditionTrackers failed"); return false; } - if (!initMetrics(config, logTrackerMap, conditionTrackerMap, allLogEntryMatchers, + if (!initMetrics(key, config, logTrackerMap, conditionTrackerMap, allLogEntryMatchers, allConditionTrackers, allMetricProducers, conditionToMetricMap, trackerToMetricMap, metricProducerMap)) { ALOGE("initMetricProducers failed"); diff --git a/cmds/statsd/src/metrics/metrics_manager_util.h b/cmds/statsd/src/metrics/metrics_manager_util.h index 7d7e0c36951f..e7cbd533b5b4 100644 --- a/cmds/statsd/src/metrics/metrics_manager_util.h +++ b/cmds/statsd/src/metrics/metrics_manager_util.h @@ -36,6 +36,7 @@ namespace statsd { // Initialize the LogMatchingTrackers. // input: +// [key]: the config key that this config belongs to // [config]: the input StatsdConfig // output: // [logTrackerMap]: this map should contain matcher name to index mapping @@ -48,6 +49,7 @@ bool initLogTrackers(const StatsdConfig& config, // Initialize ConditionTrackers // input: +// [key]: the config key that this config belongs to // [config]: the input config // [logTrackerMap]: LogMatchingTracker name to index mapping from previous step. // output: @@ -55,7 +57,7 @@ bool initLogTrackers(const StatsdConfig& config, // [allConditionTrackers]: stores the sp to all the ConditionTrackers // [trackerToConditionMap]: contain the mapping from index of // log tracker to condition trackers that use the log tracker -bool initConditions(const StatsdConfig& config, +bool initConditions(const ConfigKey& key, const StatsdConfig& config, const std::unordered_map<std::string, int>& logTrackerMap, std::unordered_map<std::string, int>& conditionTrackerMap, std::vector<sp<ConditionTracker>>& allConditionTrackers, @@ -64,6 +66,7 @@ bool initConditions(const StatsdConfig& config, // Initialize MetricProducers. // input: +// [key]: the config key that this config belongs to // [config]: the input config // [logTrackerMap]: LogMatchingTracker name to index mapping from previous step. // [conditionTrackerMap]: condition name to index mapping @@ -73,7 +76,8 @@ bool initConditions(const StatsdConfig& config, // the list of MetricProducer index // [trackerToMetricMap]: contains the mapping from log tracker to MetricProducer index. bool initMetrics( - const StatsdConfig& config, const std::unordered_map<std::string, int>& logTrackerMap, + const ConfigKey& key, const StatsdConfig& config, + const std::unordered_map<std::string, int>& logTrackerMap, const std::unordered_map<std::string, int>& conditionTrackerMap, const std::unordered_map<int, std::vector<EventConditionLink>>& eventConditionLinks, const vector<sp<LogMatchingTracker>>& allLogEntryMatchers, @@ -84,7 +88,7 @@ bool initMetrics( // Initialize MetricsManager from StatsdConfig. // Parameters are the members of MetricsManager. See MetricsManager for declaration. -bool initStatsdConfig(const StatsdConfig& config, std::set<int>& allTagIds, +bool initStatsdConfig(const ConfigKey& key, const StatsdConfig& config, std::set<int>& allTagIds, std::vector<sp<LogMatchingTracker>>& allLogEntryMatchers, std::vector<sp<ConditionTracker>>& allConditionTrackers, std::vector<sp<MetricProducer>>& allMetricProducers, diff --git a/cmds/statsd/src/stats_log.proto b/cmds/statsd/src/stats_log.proto index 4783cd893ae5..81f8eb619d35 100644 --- a/cmds/statsd/src/stats_log.proto +++ b/cmds/statsd/src/stats_log.proto @@ -172,3 +172,51 @@ message ConfigMetricsReportList { repeated ConfigMetricsReport reports = 2; } + +message StatsdStatsReport { + optional int32 stats_begin_time_sec = 1; + + optional int32 stats_end_time_sec = 2; + + message MatcherStats { + optional string name = 1; + optional int32 matched_times = 2; + } + + message ConditionStats { + optional string name = 1; + optional int32 max_tuple_counts = 2; + } + + message MetricStats { + optional string name = 1; + optional int32 max_tuple_counts = 2; + } + + message ConfigStats { + optional int32 uid = 1; + optional string name = 2; + optional int32 creation_time_sec = 3; + optional int32 deletion_time_sec = 4; + optional int32 metric_count = 5; + optional int32 condition_count = 6; + optional int32 matcher_count = 7; + optional int32 alert_count = 8; + optional bool is_valid = 9; + + repeated int32 broadcast_sent_time_sec = 10; + repeated int32 data_drop_time_sec = 11; + repeated MatcherStats matcher_stats = 12; + repeated ConditionStats condition_stats = 13; + repeated MetricStats metric_stats = 14; + } + + repeated ConfigStats config_stats = 3; + + message AtomStats { + optional int32 tag = 1; + optional int32 count = 2; + } + + repeated AtomStats atom_stats = 7; +}
\ No newline at end of file diff --git a/cmds/statsd/tests/ConfigManager_test.cpp b/cmds/statsd/tests/ConfigManager_test.cpp index 618aef6d0847..696fddf003e1 100644 --- a/cmds/statsd/tests/ConfigManager_test.cpp +++ b/cmds/statsd/tests/ConfigManager_test.cpp @@ -62,7 +62,8 @@ MATCHER_P(StatsdConfigEq, name, "") { } TEST(ConfigManagerTest, TestFakeConfig) { - auto metricsManager = std::make_unique<MetricsManager>(build_fake_config()); + auto metricsManager = + std::make_unique<MetricsManager>(ConfigKey(0, "test"), build_fake_config()); EXPECT_TRUE(metricsManager->isConfigValid()); } diff --git a/cmds/statsd/tests/MetricsManager_test.cpp b/cmds/statsd/tests/MetricsManager_test.cpp index 3dd4e70ff49c..2adec945b85a 100644 --- a/cmds/statsd/tests/MetricsManager_test.cpp +++ b/cmds/statsd/tests/MetricsManager_test.cpp @@ -40,6 +40,8 @@ using android::os::statsd::Condition; // TODO: ADD MORE TEST CASES. +const ConfigKey kConfigKey(0, "test"); + StatsdConfig buildGoodConfig() { StatsdConfig config; config.set_name("12345"); @@ -254,9 +256,9 @@ TEST(MetricsManagerTest, TestGoodConfig) { unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; - EXPECT_TRUE(initStatsdConfig(config, allTagIds, allLogEntryMatchers, allConditionTrackers, - allMetricProducers, allAnomalyTrackers, conditionToMetricMap, - trackerToMetricMap, trackerToConditionMap)); + EXPECT_TRUE(initStatsdConfig(kConfigKey, config, allTagIds, allLogEntryMatchers, + allConditionTrackers, allMetricProducers, allAnomalyTrackers, + conditionToMetricMap, trackerToMetricMap, trackerToConditionMap)); EXPECT_EQ(1u, allMetricProducers.size()); EXPECT_EQ(1u, allAnomalyTrackers.size()); } @@ -272,9 +274,9 @@ TEST(MetricsManagerTest, TestDimensionMetricsWithMultiTags) { unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; - EXPECT_FALSE(initStatsdConfig(config, allTagIds, allLogEntryMatchers, allConditionTrackers, - allMetricProducers, allAnomalyTrackers, conditionToMetricMap, - trackerToMetricMap, trackerToConditionMap)); + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, allTagIds, allLogEntryMatchers, + allConditionTrackers, allMetricProducers, allAnomalyTrackers, + conditionToMetricMap, trackerToMetricMap, trackerToConditionMap)); } TEST(MetricsManagerTest, TestCircleLogMatcherDependency) { @@ -288,9 +290,9 @@ TEST(MetricsManagerTest, TestCircleLogMatcherDependency) { unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; - EXPECT_FALSE(initStatsdConfig(config, allTagIds, allLogEntryMatchers, allConditionTrackers, - allMetricProducers, allAnomalyTrackers, conditionToMetricMap, - trackerToMetricMap, trackerToConditionMap)); + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, allTagIds, allLogEntryMatchers, + allConditionTrackers, allMetricProducers, allAnomalyTrackers, + conditionToMetricMap, trackerToMetricMap, trackerToConditionMap)); } TEST(MetricsManagerTest, TestMissingMatchers) { @@ -303,9 +305,9 @@ TEST(MetricsManagerTest, TestMissingMatchers) { unordered_map<int, std::vector<int>> conditionToMetricMap; unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; - EXPECT_FALSE(initStatsdConfig(config, allTagIds, allLogEntryMatchers, allConditionTrackers, - allMetricProducers, allAnomalyTrackers, conditionToMetricMap, - trackerToMetricMap, trackerToConditionMap)); + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, allTagIds, allLogEntryMatchers, + allConditionTrackers, allMetricProducers, allAnomalyTrackers, + conditionToMetricMap, trackerToMetricMap, trackerToConditionMap)); } TEST(MetricsManagerTest, TestCircleConditionDependency) { @@ -319,9 +321,9 @@ TEST(MetricsManagerTest, TestCircleConditionDependency) { unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; - EXPECT_FALSE(initStatsdConfig(config, allTagIds, allLogEntryMatchers, allConditionTrackers, - allMetricProducers, allAnomalyTrackers, conditionToMetricMap, - trackerToMetricMap, trackerToConditionMap)); + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, allTagIds, allLogEntryMatchers, + allConditionTrackers, allMetricProducers, allAnomalyTrackers, + conditionToMetricMap, trackerToMetricMap, trackerToConditionMap)); } TEST(MetricsManagerTest, testAlertWithUnknownMetric) { @@ -335,9 +337,9 @@ TEST(MetricsManagerTest, testAlertWithUnknownMetric) { unordered_map<int, std::vector<int>> trackerToMetricMap; unordered_map<int, std::vector<int>> trackerToConditionMap; - EXPECT_FALSE(initStatsdConfig(config, allTagIds, allLogEntryMatchers, allConditionTrackers, - allMetricProducers, allAnomalyTrackers, conditionToMetricMap, - trackerToMetricMap, trackerToConditionMap)); + EXPECT_FALSE(initStatsdConfig(kConfigKey, config, allTagIds, allLogEntryMatchers, + allConditionTrackers, allMetricProducers, allAnomalyTrackers, + conditionToMetricMap, trackerToMetricMap, trackerToConditionMap)); } #else diff --git a/cmds/statsd/tests/condition/SimpleConditionTracker_test.cpp b/cmds/statsd/tests/condition/SimpleConditionTracker_test.cpp index 11fb011fd74e..92b4ffcdbc43 100644 --- a/cmds/statsd/tests/condition/SimpleConditionTracker_test.cpp +++ b/cmds/statsd/tests/condition/SimpleConditionTracker_test.cpp @@ -28,6 +28,8 @@ namespace android { namespace os { namespace statsd { +const ConfigKey kConfigKey(0, "test"); + SimpleCondition getWakeLockHeldCondition(bool countNesting, bool defaultFalse, bool outputSlicedUid) { SimpleCondition simpleCondition; @@ -76,8 +78,8 @@ TEST(SimpleConditionTrackerTest, TestNonSlicedCondition) { trackerNameIndexMap["SCREEN_TURNED_ON"] = 0; trackerNameIndexMap["SCREEN_TURNED_OFF"] = 1; - SimpleConditionTracker conditionTracker("SCREEN_IS_ON", 0 /*tracker index*/, simpleCondition, - trackerNameIndexMap); + SimpleConditionTracker conditionTracker(kConfigKey, "SCREEN_IS_ON", 0 /*tracker index*/, + simpleCondition, trackerNameIndexMap); LogEvent event(1 /*tagId*/, 0 /*timestamp*/); @@ -158,8 +160,9 @@ TEST(SimpleConditionTrackerTest, TestNonSlicedConditionNestCounting) { trackerNameIndexMap["SCREEN_TURNED_ON"] = 0; trackerNameIndexMap["SCREEN_TURNED_OFF"] = 1; - SimpleConditionTracker conditionTracker("SCREEN_IS_ON", 0 /*condition tracker index*/, - simpleCondition, trackerNameIndexMap); + SimpleConditionTracker conditionTracker(kConfigKey, "SCREEN_IS_ON", + 0 /*condition tracker index*/, simpleCondition, + trackerNameIndexMap); LogEvent event(1 /*tagId*/, 0 /*timestamp*/); @@ -227,8 +230,9 @@ TEST(SimpleConditionTrackerTest, TestSlicedCondition) { trackerNameIndexMap["WAKE_LOCK_RELEASE"] = 1; trackerNameIndexMap["RELEASE_ALL"] = 2; - SimpleConditionTracker conditionTracker(conditionName, 0 /*condition tracker index*/, - simpleCondition, trackerNameIndexMap); + SimpleConditionTracker conditionTracker(kConfigKey, conditionName, + 0 /*condition tracker index*/, simpleCondition, + trackerNameIndexMap); int uid = 111; LogEvent event(1 /*tagId*/, 0 /*timestamp*/); @@ -308,8 +312,9 @@ TEST(SimpleConditionTrackerTest, TestSlicedWithNoOutputDim) { trackerNameIndexMap["WAKE_LOCK_RELEASE"] = 1; trackerNameIndexMap["RELEASE_ALL"] = 2; - SimpleConditionTracker conditionTracker(conditionName, 0 /*condition tracker index*/, - simpleCondition, trackerNameIndexMap); + SimpleConditionTracker conditionTracker(kConfigKey, conditionName, + 0 /*condition tracker index*/, simpleCondition, + trackerNameIndexMap); int uid1 = 111; string uid1_wl1 = "wl1_1"; int uid2 = 222; @@ -392,8 +397,9 @@ TEST(SimpleConditionTrackerTest, TestStopAll) { trackerNameIndexMap["WAKE_LOCK_RELEASE"] = 1; trackerNameIndexMap["RELEASE_ALL"] = 2; - SimpleConditionTracker conditionTracker(conditionName, 0 /*condition tracker index*/, - simpleCondition, trackerNameIndexMap); + SimpleConditionTracker conditionTracker(kConfigKey, conditionName, + 0 /*condition tracker index*/, simpleCondition, + trackerNameIndexMap); int uid1 = 111; int uid2 = 222; diff --git a/cmds/statsd/tests/guardrail/StatsdStats_test.cpp b/cmds/statsd/tests/guardrail/StatsdStats_test.cpp new file mode 100644 index 000000000000..286f6bd75442 --- /dev/null +++ b/cmds/statsd/tests/guardrail/StatsdStats_test.cpp @@ -0,0 +1,50 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/guardrail/StatsdStats.h" + +#include <gtest/gtest.h> + +#ifdef __ANDROID__ + +namespace android { +namespace os { +namespace statsd { + +TEST(StatsdStatsTest, TestConfigAdd) { + // TODO: implement +} + +TEST(StatsdStatsTest, TestConfigRemove) { + // TODO: implement +} + +TEST(StatsdStatsTest, TestMatcherReport) { + // TODO: implement +} + +TEST(StatsdStatsTest, TestConditionReport) { + // TODO: implement +} + +TEST(StatsdStatsTest, TestAtomLog) { + // TODO: implement +} + +} // namespace statsd +} // namespace os +} // namespace android +#else +GTEST_LOG_(INFO) << "This test does nothing.\n"; +#endif diff --git a/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp b/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp index 35e08af68739..df743644c3d5 100644 --- a/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/CountMetricProducer_test.cpp @@ -32,6 +32,8 @@ namespace android { namespace os { namespace statsd { +const ConfigKey kConfigKey(0, "test"); + TEST(CountMetricProducerTest, TestNonDimensionalEvents) { int64_t bucketStartTimeNs = 10000000000; int64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; @@ -48,7 +50,7 @@ TEST(CountMetricProducerTest, TestNonDimensionalEvents) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - CountMetricProducer countProducer(metric, -1 /*-1 meaning no condition*/, wizard, + CountMetricProducer countProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, bucketStartTimeNs); // 2 events in bucket 1. @@ -106,7 +108,7 @@ TEST(CountMetricProducerTest, TestEventsWithNonSlicedCondition) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - CountMetricProducer countProducer(metric, 1, wizard, bucketStartTimeNs); + CountMetricProducer countProducer(kConfigKey, metric, 1, wizard, bucketStartTimeNs); countProducer.onConditionChanged(true, bucketStartTimeNs); countProducer.onMatchedLogEvent(1 /*matcher index*/, event1, false /*pulled*/); @@ -161,7 +163,7 @@ TEST(CountMetricProducerTest, TestEventsWithSlicedCondition) { EXPECT_CALL(*wizard, query(_, key2)).WillOnce(Return(ConditionState::kTrue)); - CountMetricProducer countProducer(metric, 1 /*condition tracker index*/, wizard, + CountMetricProducer countProducer(kConfigKey, metric, 1 /*condition tracker index*/, wizard, bucketStartTimeNs); countProducer.onMatchedLogEvent(1 /*log matcher index*/, event1, false); @@ -201,7 +203,7 @@ TEST(CountMetricProducerTest, TestAnomalyDetection) { metric.mutable_bucket()->set_bucket_size_millis(bucketSizeNs / 1000000); sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - CountMetricProducer countProducer(metric, -1 /*-1 meaning no condition*/, wizard, + CountMetricProducer countProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, bucketStartTimeNs); countProducer.addAnomalyTracker(anomalyTracker); diff --git a/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp b/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp index 18d177cb9173..724ad59eeab6 100644 --- a/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/EventMetricProducer_test.cpp @@ -32,6 +32,8 @@ namespace android { namespace os { namespace statsd { +const ConfigKey kConfigKey(0, "test"); + TEST(EventMetricProducerTest, TestNoCondition) { uint64_t bucketStartTimeNs = 10000000000; uint64_t eventStartTimeNs = bucketStartTimeNs + 1; @@ -45,7 +47,7 @@ TEST(EventMetricProducerTest, TestNoCondition) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - EventMetricProducer eventProducer(metric, -1 /*-1 meaning no condition*/, wizard, + EventMetricProducer eventProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, bucketStartTimeNs); eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1, false /*pulled*/); @@ -69,7 +71,7 @@ TEST(EventMetricProducerTest, TestEventsWithNonSlicedCondition) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - EventMetricProducer eventProducer(metric, 1, wizard, bucketStartTimeNs); + EventMetricProducer eventProducer(kConfigKey, metric, 1, wizard, bucketStartTimeNs); eventProducer.onConditionChanged(true /*condition*/, bucketStartTimeNs); eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1, false /*pulled*/); @@ -111,7 +113,7 @@ TEST(EventMetricProducerTest, TestEventsWithSlicedCondition) { EXPECT_CALL(*wizard, query(_, key2)).WillOnce(Return(ConditionState::kTrue)); - EventMetricProducer eventProducer(metric, 1, wizard, bucketStartTimeNs); + EventMetricProducer eventProducer(kConfigKey, metric, 1, wizard, bucketStartTimeNs); eventProducer.onMatchedLogEvent(1 /*matcher index*/, event1, false /*pulled*/); eventProducer.onMatchedLogEvent(1 /*matcher index*/, event2, false /*pulled*/); diff --git a/cmds/statsd/tests/metrics/MaxDurationTracker_test.cpp b/cmds/statsd/tests/metrics/MaxDurationTracker_test.cpp index d82ccfe6b43e..0763c944c556 100644 --- a/cmds/statsd/tests/metrics/MaxDurationTracker_test.cpp +++ b/cmds/statsd/tests/metrics/MaxDurationTracker_test.cpp @@ -36,6 +36,8 @@ namespace android { namespace os { namespace statsd { +const ConfigKey kConfigKey(0, "test"); + TEST(MaxDurationTrackerTest, TestSimpleMaxDuration) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); @@ -45,8 +47,8 @@ TEST(MaxDurationTrackerTest, TestSimpleMaxDuration) { uint64_t bucketStartTimeNs = 10000000000; uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; - MaxDurationTracker tracker("event", wizard, -1, false, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + MaxDurationTracker tracker(kConfigKey, "metric", "event", wizard, -1, false, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("1", true, bucketStartTimeNs, key1); // Event starts again. This would not change anything as it already starts. @@ -72,8 +74,8 @@ TEST(MaxDurationTrackerTest, TestStopAll) { uint64_t bucketStartTimeNs = 10000000000; uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; - MaxDurationTracker tracker("event", wizard, -1, false, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + MaxDurationTracker tracker(kConfigKey, "metric", "event", wizard, -1, false, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("1", true, bucketStartTimeNs + 1, key1); @@ -100,8 +102,8 @@ TEST(MaxDurationTrackerTest, TestCrossBucketBoundary) { uint64_t bucketStartTimeNs = 10000000000; uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; - MaxDurationTracker tracker("event", wizard, -1, false, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + MaxDurationTracker tracker(kConfigKey, "metric", "event", wizard, -1, false, bucketStartTimeNs, + bucketSizeNs, {}, buckets); // The event starts. tracker.noteStart("", true, bucketStartTimeNs + 1, key1); @@ -127,8 +129,8 @@ TEST(MaxDurationTrackerTest, TestCrossBucketBoundary_nested) { uint64_t bucketStartTimeNs = 10000000000; uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; - MaxDurationTracker tracker("event", wizard, -1, true, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + MaxDurationTracker tracker(kConfigKey, "metric", "event", wizard, -1, true, bucketStartTimeNs, + bucketSizeNs, {}, buckets); // 2 starts tracker.noteStart("", true, bucketStartTimeNs + 1, key1); @@ -168,8 +170,8 @@ TEST(MaxDurationTrackerTest, TestMaxDurationWithCondition) { uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; int64_t durationTimeNs = 2 * 1000; - MaxDurationTracker tracker("event", wizard, 1, false, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + MaxDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, false, bucketStartTimeNs, + bucketSizeNs, {}, buckets); EXPECT_TRUE(tracker.mAnomalyTrackers.empty()); tracker.noteStart("2:maps", true, eventStartTimeNs, key1); @@ -200,8 +202,8 @@ TEST(MaxDurationTrackerTest, TestAnomalyDetection) { uint64_t bucketSizeNs = 30 * NS_PER_SEC; sp<AnomalyTracker> anomalyTracker = new AnomalyTracker(alert, bucketSizeNs); - MaxDurationTracker tracker("event", wizard, -1, true, bucketStartTimeNs, bucketSizeNs, - {anomalyTracker}, buckets); + MaxDurationTracker tracker(kConfigKey, "metric", "event", wizard, -1, true, bucketStartTimeNs, + bucketSizeNs, {anomalyTracker}, buckets); tracker.noteStart("1", true, eventStartTimeNs, key1); tracker.noteStop("1", eventStartTimeNs + 10, false); diff --git a/cmds/statsd/tests/metrics/OringDurationTracker_test.cpp b/cmds/statsd/tests/metrics/OringDurationTracker_test.cpp index 54618b5fb588..63a0e2b3499f 100644 --- a/cmds/statsd/tests/metrics/OringDurationTracker_test.cpp +++ b/cmds/statsd/tests/metrics/OringDurationTracker_test.cpp @@ -34,6 +34,8 @@ namespace android { namespace os { namespace statsd { +const ConfigKey kConfigKey(0, "test"); + TEST(OringDurationTrackerTest, TestDurationOverlap) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); @@ -47,8 +49,8 @@ TEST(OringDurationTrackerTest, TestDurationOverlap) { uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; uint64_t durationTimeNs = 2 * 1000; - OringDurationTracker tracker("event", wizard, 1, false, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, false, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("2:maps", true, eventStartTimeNs, key1); EXPECT_EQ((long long)eventStartTimeNs, tracker.mLastStartTime); @@ -73,8 +75,8 @@ TEST(OringDurationTrackerTest, TestDurationNested) { uint64_t eventStartTimeNs = bucketStartTimeNs + 1; uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; - OringDurationTracker tracker("event", wizard, 1, true, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, true, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("2:maps", true, eventStartTimeNs, key1); tracker.noteStart("2:maps", true, eventStartTimeNs + 10, key1); // overlapping wl @@ -99,8 +101,8 @@ TEST(OringDurationTrackerTest, TestStopAll) { uint64_t eventStartTimeNs = bucketStartTimeNs + 1; uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; - OringDurationTracker tracker("event", wizard, 1, true, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, true, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("2:maps", true, eventStartTimeNs, key1); tracker.noteStart("3:maps", true, eventStartTimeNs + 10, key1); // overlapping wl @@ -125,8 +127,8 @@ TEST(OringDurationTrackerTest, TestCrossBucketBoundary) { uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; uint64_t durationTimeNs = 2 * 1000; - OringDurationTracker tracker("event", wizard, 1, true, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, true, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("2:maps", true, eventStartTimeNs, key1); EXPECT_EQ((long long)eventStartTimeNs, tracker.mLastStartTime); @@ -162,8 +164,8 @@ TEST(OringDurationTrackerTest, TestDurationConditionChange) { uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; uint64_t durationTimeNs = 2 * 1000; - OringDurationTracker tracker("event", wizard, 1, false, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, false, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("2:maps", true, eventStartTimeNs, key1); @@ -194,8 +196,8 @@ TEST(OringDurationTrackerTest, TestDurationConditionChange2) { uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; uint64_t durationTimeNs = 2 * 1000; - OringDurationTracker tracker("event", wizard, 1, false, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, false, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("2:maps", true, eventStartTimeNs, key1); // condition to false; record duration 5n @@ -225,8 +227,8 @@ TEST(OringDurationTrackerTest, TestDurationConditionChangeNested) { uint64_t eventStartTimeNs = bucketStartTimeNs + 1; uint64_t bucketSizeNs = 30 * 1000 * 1000 * 1000LL; - OringDurationTracker tracker("event", wizard, 1, true, bucketStartTimeNs, bucketSizeNs, {}, - buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, true, bucketStartTimeNs, + bucketSizeNs, {}, buckets); tracker.noteStart("2:maps", true, eventStartTimeNs, key1); tracker.noteStart("2:maps", true, eventStartTimeNs + 2, key1); @@ -259,8 +261,8 @@ TEST(OringDurationTrackerTest, TestPredictAnomalyTimestamp) { uint64_t bucketSizeNs = 30 * NS_PER_SEC; sp<AnomalyTracker> anomalyTracker = new AnomalyTracker(alert, bucketSizeNs); - OringDurationTracker tracker("event", wizard, 1, true, bucketStartTimeNs, bucketSizeNs, - {anomalyTracker}, buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, true, bucketStartTimeNs, + bucketSizeNs, {anomalyTracker}, buckets); // Nothing in the past bucket. tracker.noteStart("", true, eventStartTimeNs, key1); @@ -319,8 +321,8 @@ TEST(OringDurationTrackerTest, TestAnomalyDetection) { uint64_t bucketSizeNs = 30 * NS_PER_SEC; sp<AnomalyTracker> anomalyTracker = new AnomalyTracker(alert, bucketSizeNs); - OringDurationTracker tracker("event", wizard, 1, true /*nesting*/, bucketStartTimeNs, - bucketSizeNs, {anomalyTracker}, buckets); + OringDurationTracker tracker(kConfigKey, "metric", "event", wizard, 1, true /*nesting*/, + bucketStartTimeNs, bucketSizeNs, {anomalyTracker}, buckets); tracker.noteStart("", true, eventStartTimeNs, key1); tracker.noteStop("", eventStartTimeNs + 10, false); diff --git a/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp b/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp index 1ed3636f4ec4..9d784663de03 100644 --- a/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp +++ b/cmds/statsd/tests/metrics/ValueMetricProducer_test.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "metrics_test_helper.h" #include "src/metrics/ValueMetricProducer.h" +#include "metrics_test_helper.h" #include <gmock/gmock.h> #include <gtest/gtest.h> @@ -22,11 +22,11 @@ using namespace testing; using android::sp; +using std::make_shared; using std::set; +using std::shared_ptr; using std::unordered_map; using std::vector; -using std::shared_ptr; -using std::make_shared; #ifdef __ANDROID__ @@ -34,6 +34,7 @@ namespace android { namespace os { namespace statsd { +const ConfigKey kConfigKey(0, "test"); /* * Tests pulled atoms with no conditions */ @@ -42,7 +43,7 @@ TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; - int64_t bucket3StartTimeNs = bucketStartTimeNs + 2*bucketSizeNs; + int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; ValueMetric metric; metric.set_name("1"); @@ -54,12 +55,13 @@ TEST(ValueMetricProducerTest, TestNonDimensionalEvents) { sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); // TODO: pending refactor of StatsPullerManager // For now we still need this so that it doesn't do real pulling. - shared_ptr<MockStatsPullerManager> pullerManager = make_shared<StrictMock<MockStatsPullerManager>>(); + shared_ptr<MockStatsPullerManager> pullerManager = + make_shared<StrictMock<MockStatsPullerManager>>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillOnce(Return()); - ValueMetricProducer valueProducer(metric, -1 /*-1 meaning no condition*/, wizard,tagId, - bucketStartTimeNs, pullerManager); + ValueMetricProducer valueProducer(kConfigKey, metric, -1 /*-1 meaning no condition*/, wizard, + tagId, bucketStartTimeNs, pullerManager); vector<shared_ptr<LogEvent>> allData; allData.clear(); @@ -144,41 +146,43 @@ TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition) { int tagId = 1; sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = make_shared<StrictMock<MockStatsPullerManager>>(); + shared_ptr<MockStatsPullerManager> pullerManager = + make_shared<StrictMock<MockStatsPullerManager>>(); EXPECT_CALL(*pullerManager, RegisterReceiver(tagId, _, _)).WillOnce(Return()); EXPECT_CALL(*pullerManager, UnRegisterReceiver(tagId, _)).WillRepeatedly(Return()); - EXPECT_CALL(*pullerManager, Pull(tagId, _)).WillOnce(Invoke([] (int tagId, vector<std::shared_ptr<LogEvent>>* data) { - int64_t bucketStartTimeNs = 10000000000; - int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; - - int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; - int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; - data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); - event->write(1); - event->write(100); - event->init(); - data->push_back(event); - return true; - })) - .WillOnce(Invoke([] (int tagId, vector<std::shared_ptr<LogEvent>>* data) { - int64_t bucketStartTimeNs = 10000000000; - int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; - - int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; - int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; - data->clear(); - shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 10); - event->write(1); - event->write(120); - event->init(); - data->push_back(event); - return true; - })); - - ValueMetricProducer valueProducer(metric, 1, wizard,tagId, - bucketStartTimeNs, pullerManager); + EXPECT_CALL(*pullerManager, Pull(tagId, _)) + .WillOnce(Invoke([](int tagId, vector<std::shared_ptr<LogEvent>>* data) { + int64_t bucketStartTimeNs = 10000000000; + int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; + + int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; + int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); + event->write(1); + event->write(100); + event->init(); + data->push_back(event); + return true; + })) + .WillOnce(Invoke([](int tagId, vector<std::shared_ptr<LogEvent>>* data) { + int64_t bucketStartTimeNs = 10000000000; + int64_t bucketSizeNs = 60 * 1000 * 1000 * 1000LL; + + int64_t bucket2StartTimeNs = bucketStartTimeNs + bucketSizeNs; + int64_t bucket3StartTimeNs = bucketStartTimeNs + 2 * bucketSizeNs; + data->clear(); + shared_ptr<LogEvent> event = make_shared<LogEvent>(tagId, bucket2StartTimeNs + 10); + event->write(1); + event->write(120); + event->init(); + data->push_back(event); + return true; + })); + + ValueMetricProducer valueProducer(kConfigKey, metric, 1, wizard, tagId, bucketStartTimeNs, + pullerManager); valueProducer.onConditionChanged(true, bucketStartTimeNs + 10); @@ -241,10 +245,11 @@ TEST(ValueMetricProducerTest, TestPushedEventsWithoutCondition) { int tagId = 1; sp<MockConditionWizard> wizard = new NaggyMock<MockConditionWizard>(); - shared_ptr<MockStatsPullerManager> pullerManager = make_shared<StrictMock<MockStatsPullerManager>>(); + shared_ptr<MockStatsPullerManager> pullerManager = + make_shared<StrictMock<MockStatsPullerManager>>(); - ValueMetricProducer valueProducer(metric, -1, wizard,-1, - bucketStartTimeNs, pullerManager); + ValueMetricProducer valueProducer(kConfigKey, metric, -1, wizard, -1, bucketStartTimeNs, + pullerManager); shared_ptr<LogEvent> event1 = make_shared<LogEvent>(tagId, bucketStartTimeNs + 10); event1->write(1); diff --git a/tools/stats_log_api_gen/main.cpp b/tools/stats_log_api_gen/main.cpp index cca12947d26e..423d0285e96f 100644 --- a/tools/stats_log_api_gen/main.cpp +++ b/tools/stats_log_api_gen/main.cpp @@ -18,6 +18,10 @@ using namespace std; namespace android { namespace stats_log_api_gen { +const int PULL_ATOM_START_ID = 1000; + +int maxPushedAtomId = 2; + using android::os::statsd::Atom; // TODO: Support WorkSources @@ -195,12 +199,18 @@ write_stats_log_header(FILE* out, const Atoms& atoms) fprintf(out, " */\n"); char const* const comma = (i == atoms.decls.size() - 1) ? "" : ","; fprintf(out, " %s = %d%s\n", constant.c_str(), atom->code, comma); + if (atom->code < PULL_ATOM_START_ID && atom->code > maxPushedAtomId) { + maxPushedAtomId = atom->code; + } i++; } fprintf(out, "\n"); fprintf(out, "};\n"); fprintf(out, "\n"); + fprintf(out, "const static int kMaxPushedAtomId = %d;\n\n", + maxPushedAtomId); + // Print write methods fprintf(out, "//\n"); fprintf(out, "// Write methods\n"); |