diff options
35 files changed, 1628 insertions, 344 deletions
diff --git a/cmds/statsd/Android.mk b/cmds/statsd/Android.mk index 38dcc62d5f68..3c2f2d5d48ad 100644 --- a/cmds/statsd/Android.mk +++ b/cmds/statsd/Android.mk @@ -24,6 +24,7 @@ statsd_common_src := \ src/condition/CombinationConditionTracker.cpp \ src/condition/condition_util.cpp \ src/condition/SimpleConditionTracker.cpp \ + src/condition/ConditionWizard.cpp \ src/config/ConfigKey.cpp \ src/config/ConfigListener.cpp \ src/config/ConfigManager.cpp \ @@ -37,6 +38,7 @@ statsd_common_src := \ src/matchers/SimpleLogMatchingTracker.cpp \ src/metrics/CountAnomalyTracker.cpp \ src/metrics/CountMetricProducer.cpp \ + src/metrics/DurationMetricProducer.cpp \ src/metrics/MetricsManager.cpp \ src/metrics/metrics_manager_util.cpp \ src/packages/UidMap.cpp \ diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp index 3def13fdde4d..e7825cf75159 100644 --- a/cmds/statsd/src/StatsLogProcessor.cpp +++ b/cmds/statsd/src/StatsLogProcessor.cpp @@ -73,6 +73,16 @@ void StatsLogProcessor::OnConfigUpdated(const ConfigKey& key, const StatsdConfig } } +vector<StatsLogReport> StatsLogProcessor::onDumpReport(const ConfigKey& key) { + auto it = mMetricsManagers.find(key); + if (it == mMetricsManagers.end()) { + ALOGW("Config source %s does not exist", key.ToString().c_str()); + return vector<StatsLogReport>(); + } + + return it->second->onDumpReport(); +} + void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { auto it = mMetricsManagers.find(key); if (it != mMetricsManagers.end()) { diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h index dc604855b6b0..3cefd29fa026 100644 --- a/cmds/statsd/src/StatsLogProcessor.h +++ b/cmds/statsd/src/StatsLogProcessor.h @@ -33,7 +33,7 @@ namespace statsd { class StatsLogProcessor : public ConfigListener { public: - StatsLogProcessor(const sp<UidMap> &uidMap); + StatsLogProcessor(const sp<UidMap>& uidMap); virtual ~StatsLogProcessor(); virtual void OnLogEvent(const LogEvent& event); @@ -41,6 +41,9 @@ public: void OnConfigUpdated(const ConfigKey& key, const StatsdConfig& config); void OnConfigRemoved(const ConfigKey& key); + // TODO: Once we have the ProtoOutputStream in c++, we can just return byte array. + std::vector<StatsLogReport> onDumpReport(const ConfigKey& key); + private: // TODO: use EventMetrics to log the events. DropboxWriter m_dropbox_writer; diff --git a/cmds/statsd/src/StatsService.cpp b/cmds/statsd/src/StatsService.cpp index b72e04ed4ab6..87616d3287d6 100644 --- a/cmds/statsd/src/StatsService.cpp +++ b/cmds/statsd/src/StatsService.cpp @@ -64,7 +64,6 @@ void CompanionDeathRecipient::binderDied(const wp<IBinder>& who) { // ====================================================================== StatsService::StatsService(const sp<Looper>& handlerLooper) : mStatsPullerManager(), - mAnomalyMonitor(new AnomalyMonitor(2)) // TODO: Put this comment somewhere better { mUidMap = new UidMap(); @@ -89,7 +88,7 @@ void StatsService::init_system_properties() { void StatsService::init_build_type_callback(void* cookie, const char* /*name*/, const char* value, uint32_t serial) { - if (0 == strcmp("eng", value)) { + if (0 == strcmp("eng", value) || 0 == strcmp("userdebug", value)) { reinterpret_cast<StatsService*>(cookie)->mEngBuild = true; } } @@ -187,10 +186,13 @@ status_t StatsService::command(FILE* in, FILE* out, FILE* err, Vector<String8>& return cmd_print_stats_log(out, args); } - // adb shell cmd stats print-stats-log if (!args[0].compare(String8("print-uid-map"))) { return cmd_print_uid_map(out); } + + if (!args[0].compare(String8("dump-report"))) { + return cmd_dump_report(out, err, args); + } } print_cmd_help(out); @@ -248,7 +250,9 @@ status_t StatsService::cmd_config(FILE* in, FILE* out, FILE* err, Vector<String8 } } } else { - fprintf(err, "The config can only be set for other UIDs on eng builds.\n"); + fprintf(err, + "The config can only be set for other UIDs on eng or userdebug " + "builds.\n"); } } @@ -287,6 +291,54 @@ status_t StatsService::cmd_config(FILE* in, FILE* out, FILE* err, Vector<String8 return UNKNOWN_ERROR; } +status_t StatsService::cmd_dump_report(FILE* out, FILE* err, const Vector<String8>& args) { + if (mProcessor != nullptr) { + const int argCount = args.size(); + bool good = false; + int uid; + string name; + if (argCount == 2) { + // Automatically pick the UID + uid = IPCThreadState::self()->getCallingUid(); + // TODO: What if this isn't a binder call? Should we fail? + name.assign(args[2].c_str(), args[2].size()); + good = true; + } else if (argCount == 3) { + // If it's a userdebug or eng build, then the shell user can + // impersonate other uids. + if (mEngBuild) { + const char* s = args[1].c_str(); + if (*s != '\0') { + char* end = NULL; + uid = strtol(s, &end, 0); + if (*end == '\0') { + name.assign(args[2].c_str(), args[2].size()); + good = true; + } + } + } else { + fprintf(out, + "The metrics can only be dumped for other UIDs on eng or userdebug " + "builds.\n"); + } + } + if (good) { + mProcessor->onDumpReport(ConfigKey(uid, name)); + // TODO: print the returned StatsLogReport to file instead of printing to logcat. + fprintf(out, "Dump report for Config [%d,%s]\n", uid, name.c_str()); + fprintf(out, "See the StatsLogReport in logcat...\n"); + return android::OK; + } else { + // If arg parsing failed, print the help text and return an error. + print_cmd_help(out); + return UNKNOWN_ERROR; + } + } else { + fprintf(out, "Log processor does not exist...\n"); + return UNKNOWN_ERROR; + } +} + status_t StatsService::cmd_print_stats_log(FILE* out, const Vector<String8>& args) { long msec = 0; diff --git a/cmds/statsd/src/StatsService.h b/cmds/statsd/src/StatsService.h index 7d305e9ad15a..294aec868880 100644 --- a/cmds/statsd/src/StatsService.h +++ b/cmds/statsd/src/StatsService.h @@ -111,6 +111,11 @@ private: status_t cmd_print_stats_log(FILE* out, const Vector<String8>& args); /** + * Print the event log. + */ + status_t cmd_dump_report(FILE* out, FILE* err, const Vector<String8>& args); + + /** * Print the mapping of uids to package names. */ status_t cmd_print_uid_map(FILE* out); diff --git a/cmds/statsd/src/condition/CombinationConditionTracker.cpp b/cmds/statsd/src/condition/CombinationConditionTracker.cpp index 014747fe1833..f56c15a37086 100644 --- a/cmds/statsd/src/condition/CombinationConditionTracker.cpp +++ b/cmds/statsd/src/condition/CombinationConditionTracker.cpp @@ -25,6 +25,7 @@ namespace android { namespace os { namespace statsd { +using std::map; using std::string; using std::unique_ptr; using std::unordered_map; @@ -102,33 +103,62 @@ bool CombinationConditionTracker::init(const vector<Condition>& allConditionConf return true; } +void CombinationConditionTracker::isConditionMet( + const map<string, HashableDimensionKey>& conditionParameters, + const vector<sp<ConditionTracker>>& allConditions, vector<ConditionState>& conditionCache) { + for (const int childIndex : mChildren) { + if (conditionCache[childIndex] == ConditionState::kNotEvaluated) { + allConditions[childIndex]->isConditionMet(conditionParameters, allConditions, + conditionCache); + } + } + conditionCache[mIndex] = + evaluateCombinationCondition(mChildren, mLogicalOperation, conditionCache); +} + bool CombinationConditionTracker::evaluateCondition( const LogEvent& event, const std::vector<MatchingState>& eventMatcherValues, const std::vector<sp<ConditionTracker>>& mAllConditions, - std::vector<ConditionState>& conditionCache, std::vector<bool>& changedCache) { + std::vector<ConditionState>& nonSlicedConditionCache, + std::vector<bool>& nonSlicedChangedCache, vector<bool>& slicedConditionChanged) { // value is up to date. - if (conditionCache[mIndex] != ConditionState::kNotEvaluated) { + if (nonSlicedConditionCache[mIndex] != ConditionState::kNotEvaluated) { return false; } for (const int childIndex : mChildren) { - if (conditionCache[childIndex] == ConditionState::kNotEvaluated) { + if (nonSlicedConditionCache[childIndex] == ConditionState::kNotEvaluated) { const sp<ConditionTracker>& child = mAllConditions[childIndex]; - child->evaluateCondition(event, eventMatcherValues, mAllConditions, conditionCache, - changedCache); + child->evaluateCondition(event, eventMatcherValues, mAllConditions, + nonSlicedConditionCache, nonSlicedChangedCache, + slicedConditionChanged); } } ConditionState newCondition = - evaluateCombinationCondition(mChildren, mLogicalOperation, conditionCache); + evaluateCombinationCondition(mChildren, mLogicalOperation, nonSlicedConditionCache); + + bool nonSlicedChanged = (mNonSlicedConditionState != newCondition); + mNonSlicedConditionState = newCondition; - bool changed = (mConditionState != newCondition); - mConditionState = newCondition; + nonSlicedConditionCache[mIndex] = mNonSlicedConditionState; - conditionCache[mIndex] = mConditionState; + nonSlicedChangedCache[mIndex] = nonSlicedChanged; + + if (mSliced) { + for (const int childIndex : mChildren) { + // If any of the sliced condition in children condition changes, the combination + // condition may be changed too. + if (slicedConditionChanged[childIndex]) { + slicedConditionChanged[mIndex] = true; + break; + } + } + ALOGD("CombinationCondition %s sliced may changed? %d", mName.c_str(), + slicedConditionChanged[mIndex] == true); + } - changedCache[mIndex] = changed; - return changed; + return nonSlicedChanged; } } // namespace statsd diff --git a/cmds/statsd/src/condition/CombinationConditionTracker.h b/cmds/statsd/src/condition/CombinationConditionTracker.h index 5d2d77e80263..fc88a88f4d63 100644 --- a/cmds/statsd/src/condition/CombinationConditionTracker.h +++ b/cmds/statsd/src/condition/CombinationConditionTracker.h @@ -39,7 +39,14 @@ public: const std::vector<MatchingState>& eventMatcherValues, const std::vector<sp<ConditionTracker>>& mAllConditions, std::vector<ConditionState>& conditionCache, - std::vector<bool>& changedCache) override; + std::vector<bool>& changedCache, + std::vector<bool>& slicedConditionMayChanged) override; + + void isConditionMet(const std::map<std::string, HashableDimensionKey>& conditionParameters, + const std::vector<sp<ConditionTracker>>& allConditions, + std::vector<ConditionState>& conditionCache) override; + + void addDimensions(const std::vector<KeyMatcher>& keyMatchers) override{}; private: LogicalOperation mLogicalOperation; diff --git a/cmds/statsd/src/condition/ConditionTracker.h b/cmds/statsd/src/condition/ConditionTracker.h index 8cc7e2339887..055b4784f72c 100644 --- a/cmds/statsd/src/condition/ConditionTracker.h +++ b/cmds/statsd/src/condition/ConditionTracker.h @@ -38,8 +38,9 @@ public: : mName(name), mIndex(index), mInitialized(false), - mConditionState(ConditionState::kUnknown), - mTrackerIndex(){}; + mTrackerIndex(), + mNonSlicedConditionState(ConditionState::kUnknown), + mSliced(false){}; virtual ~ConditionTracker(){}; @@ -63,25 +64,47 @@ public: // event before ConditionTrackers, because ConditionTracker depends on // LogMatchingTrackers. // mAllConditions: the list of all ConditionTracker - // conditionCache: the cached results of the ConditionTrackers for this new event. - // changedCache: the bit map to record whether the condition has changed. + // conditionCache: the cached non-sliced condition of the ConditionTrackers for this new event. + // nonSlicedConditionChanged: the bit map to record whether non-sliced condition has changed. + // slicedConditionMayChanged: the bit map to record whether sliced condition may have changed. + // Because sliced condition needs parameters to determine the value. So the sliced + // condition is not pushed to metrics. We only inform the relevant metrics that the sliced + // condition may have changed, and metrics should pull the conditions that they are + // interested in. virtual bool evaluateCondition(const LogEvent& event, const std::vector<MatchingState>& eventMatcherValues, const std::vector<sp<ConditionTracker>>& mAllConditions, std::vector<ConditionState>& conditionCache, - std::vector<bool>& changedCache) = 0; + std::vector<bool>& nonSlicedConditionChanged, + std::vector<bool>& slicedConditionMayChanged) = 0; // Return the current condition state. virtual ConditionState isConditionMet() { - ALOGW("Condition %s value %d", mName.c_str(), mConditionState); - return mConditionState; + return mNonSlicedConditionState; }; + // Query the condition with parameters. + // [conditionParameters]: a map from condition name to the HashableDimensionKey to query the + // condition. + // [allConditions]: all condition trackers. This is needed because the condition evaluation is + // done recursively + // [conditionCache]: the cache holding the condition evaluation values. + virtual void isConditionMet( + const std::map<std::string, HashableDimensionKey>& conditionParameters, + const std::vector<sp<ConditionTracker>>& allConditions, + std::vector<ConditionState>& conditionCache) = 0; + // return the list of LogMatchingTracker index that this ConditionTracker uses. virtual const std::set<int>& getLogTrackerIndex() const { return mTrackerIndex; } + virtual void setSliced(bool sliced) { + mSliced = mSliced | sliced; + } + + virtual void addDimensions(const std::vector<KeyMatcher>& keyMatchers) = 0; + protected: // We don't really need the string name, but having a name here makes log messages // easy to debug. @@ -93,11 +116,12 @@ protected: // if it's properly initialized. bool mInitialized; - // current condition state. - ConditionState mConditionState; - // the list of LogMatchingTracker index that this ConditionTracker uses. std::set<int> mTrackerIndex; + + ConditionState mNonSlicedConditionState; + + bool mSliced; }; } // namespace statsd diff --git a/cmds/statsd/src/condition/ConditionWizard.cpp b/cmds/statsd/src/condition/ConditionWizard.cpp new file mode 100644 index 000000000000..411f7e510c3b --- /dev/null +++ b/cmds/statsd/src/condition/ConditionWizard.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "ConditionWizard.h" + +namespace android { +namespace os { +namespace statsd { + +using std::map; +using std::string; +using std::vector; + +ConditionState ConditionWizard::query(const int index, + const map<string, HashableDimensionKey>& parameters) { + vector<ConditionState> cache(mAllConditions.size(), ConditionState::kNotEvaluated); + + mAllConditions[index]->isConditionMet(parameters, mAllConditions, cache); + return cache[index]; +} + +} // namespace statsd +} // namespace os +} // namespace android
\ No newline at end of file diff --git a/cmds/statsd/src/condition/ConditionWizard.h b/cmds/statsd/src/condition/ConditionWizard.h new file mode 100644 index 000000000000..4889b64de2ed --- /dev/null +++ b/cmds/statsd/src/condition/ConditionWizard.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONDITION_WIZARD_H +#define CONDITION_WIZARD_H + +#include "ConditionTracker.h" +#include "condition_util.h" + +namespace android { +namespace os { +namespace statsd { + +// Held by MetricProducer, to query a condition state with input defined in EventConditionLink. +class ConditionWizard : public virtual android::RefBase { +public: + ConditionWizard(std::vector<sp<ConditionTracker>>& conditionTrackers) + : mAllConditions(conditionTrackers){}; + + // Query condition state, for a ConditionTracker at [conditionIndex], with [conditionParameters] + // [conditionParameters] mapping from condition name to the HashableDimensionKey to query the + // condition. + // The ConditionTracker at [conditionIndex] can be a CombinationConditionTracker. In this case, + // the conditionParameters contains the parameters for it's children SimpleConditionTrackers. + ConditionState query(const int conditionIndex, + const std::map<std::string, HashableDimensionKey>& conditionParameters); + +private: + std::vector<sp<ConditionTracker>>& mAllConditions; +}; + +} // namespace statsd +} // namespace os +} // namespace android +#endif // CONDITION_WIZARD_H diff --git a/cmds/statsd/src/condition/SimpleConditionTracker.cpp b/cmds/statsd/src/condition/SimpleConditionTracker.cpp index fa583bf6e6ec..bde3846ab66b 100644 --- a/cmds/statsd/src/condition/SimpleConditionTracker.cpp +++ b/cmds/statsd/src/condition/SimpleConditionTracker.cpp @@ -25,6 +25,7 @@ namespace android { namespace os { namespace statsd { +using std::map; using std::string; using std::unique_ptr; using std::unordered_map; @@ -64,7 +65,7 @@ SimpleConditionTracker::SimpleConditionTracker( if (simpleCondition.has_stop_all()) { auto pair = trackerNameIndexMap.find(simpleCondition.stop_all()); if (pair == trackerNameIndexMap.end()) { - ALOGW("Stop matcher %s not found in the config", simpleCondition.stop().c_str()); + ALOGW("Stop all matcher %s not found in the config", simpleCondition.stop().c_str()); return; } mStopAllLogMatcherIndex = pair->second; @@ -89,41 +90,136 @@ bool SimpleConditionTracker::init(const vector<Condition>& allConditionConfig, return mInitialized; } +void print(unordered_map<HashableDimensionKey, ConditionState>& conditions, const string& name) { + VLOG("%s DUMP:", name.c_str()); + + for (const auto& pair : conditions) { + VLOG("\t%s %d", pair.first.c_str(), pair.second); + } +} + +void SimpleConditionTracker::addDimensions(const std::vector<KeyMatcher>& keyMatchers) { + VLOG("Added dimensions size %lu", (unsigned long)keyMatchers.size()); + mDimensionsList.push_back(keyMatchers); + mSliced = true; +} + bool SimpleConditionTracker::evaluateCondition(const LogEvent& event, const vector<MatchingState>& eventMatcherValues, const vector<sp<ConditionTracker>>& mAllConditions, vector<ConditionState>& conditionCache, - vector<bool>& changedCache) { + vector<bool>& nonSlicedConditionChanged, + std::vector<bool>& slicedConditionChanged) { if (conditionCache[mIndex] != ConditionState::kNotEvaluated) { // it has been evaluated. - VLOG("Yes, already evaluated, %s %d", mName.c_str(), mConditionState); + VLOG("Yes, already evaluated, %s %d", mName.c_str(), mNonSlicedConditionState); return false; } // Ignore nesting, because we know we cannot trust ourselves on tracking nesting conditions. - ConditionState newCondition = mConditionState; + + ConditionState newCondition = mNonSlicedConditionState; + bool matched = false; // Note: The order to evaluate the following start, stop, stop_all matters. // The priority of overwrite is stop_all > stop > start. if (mStartLogMatcherIndex >= 0 && eventMatcherValues[mStartLogMatcherIndex] == MatchingState::kMatched) { + matched = true; newCondition = ConditionState::kTrue; } if (mStopLogMatcherIndex >= 0 && eventMatcherValues[mStopLogMatcherIndex] == MatchingState::kMatched) { + matched = true; newCondition = ConditionState::kFalse; } + bool stopAll = false; if (mStopAllLogMatcherIndex >= 0 && eventMatcherValues[mStopAllLogMatcherIndex] == MatchingState::kMatched) { + matched = true; newCondition = ConditionState::kFalse; + stopAll = true; + } + + if (matched == false) { + slicedConditionChanged[mIndex] = false; + nonSlicedConditionChanged[mIndex] = false; + conditionCache[mIndex] = mNonSlicedConditionState; + return false; + } + + bool nonSlicedChanged = mNonSlicedConditionState != newCondition; + + bool slicedChanged = false; + + if (stopAll) { + // TODO: handle stop all; all dimension should be cleared. } - bool changed = (mConditionState != newCondition); - mConditionState = newCondition; - conditionCache[mIndex] = mConditionState; - changedCache[mIndex] = changed; - return changed; + if (mDimensionsList.size() > 0) { + for (size_t i = 0; i < mDimensionsList.size(); i++) { + const auto& dim = mDimensionsList[i]; + vector<KeyValuePair> key = getDimensionKey(event, dim); + HashableDimensionKey hashableKey = getHashableKey(key); + if (mSlicedConditionState.find(hashableKey) == mSlicedConditionState.end() || + mSlicedConditionState[hashableKey] != newCondition) { + slicedChanged = true; + mSlicedConditionState[hashableKey] = newCondition; + } + VLOG("key: %s %d", hashableKey.c_str(), newCondition); + } + // dump all dimensions for debugging + if (DEBUG) { + print(mSlicedConditionState, mName); + } + } + + // even if this SimpleCondition is not sliced, it may be part of a sliced CombinationCondition + // if the nonSliced condition changed, it may affect the sliced condition in the parent node. + // so mark the slicedConditionChanged to be true. + // For example: APP_IN_BACKGROUND_OR_SCREEN_OFF + // APP_IN_BACKGROUND is sliced [App_A->True, App_B->False]. + // SCREEN_OFF is not sliced, and it changes from False -> True; + // We need to populate this change to parent condition. Because for App_B, + // the APP_IN_BACKGROUND_OR_SCREEN_OFF condition would change from False->True. + slicedConditionChanged[mIndex] = mSliced ? slicedChanged : nonSlicedChanged; + nonSlicedConditionChanged[mIndex] = nonSlicedChanged; + + VLOG("SimpleCondition %s nonSlicedChange? %d SlicedChanged? %d", mName.c_str(), + nonSlicedConditionChanged[mIndex] == true, slicedConditionChanged[mIndex] == true); + mNonSlicedConditionState = newCondition; + conditionCache[mIndex] = mNonSlicedConditionState; + + return nonSlicedConditionChanged[mIndex]; +} + +void SimpleConditionTracker::isConditionMet( + const map<string, HashableDimensionKey>& conditionParameters, + const vector<sp<ConditionTracker>>& allConditions, vector<ConditionState>& conditionCache) { + const auto pair = conditionParameters.find(mName); + if (pair == conditionParameters.end()) { + // the query does not need my sliced condition. just return the non sliced condition. + conditionCache[mIndex] = mNonSlicedConditionState; + VLOG("Condition %s return %d", mName.c_str(), mNonSlicedConditionState); + return; + } + + const HashableDimensionKey& key = pair->second; + VLOG("simpleCondition %s query key: %s", mName.c_str(), key.c_str()); + + if (mSlicedConditionState.find(key) == mSlicedConditionState.end()) { + // never seen this key before. the condition is unknown to us. + conditionCache[mIndex] = ConditionState::kUnknown; + } else { + conditionCache[mIndex] = mSlicedConditionState[key]; + } + + VLOG("Condition %s return %d", mName.c_str(), conditionCache[mIndex]); + + if (DEBUG) { + print(mSlicedConditionState, mName); + } } } // namespace statsd diff --git a/cmds/statsd/src/condition/SimpleConditionTracker.h b/cmds/statsd/src/condition/SimpleConditionTracker.h index 9dd06a15baf1..1f357f059aab 100644 --- a/cmds/statsd/src/condition/SimpleConditionTracker.h +++ b/cmds/statsd/src/condition/SimpleConditionTracker.h @@ -19,6 +19,7 @@ #include "ConditionTracker.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" +#include "stats_util.h" namespace android { namespace os { @@ -26,6 +27,8 @@ namespace statsd { class SimpleConditionTracker : public virtual ConditionTracker { public: + // dimensions is a vector of vector because for one single condition, different metrics may be + // interested in slicing in different ways. one vector<KeyMatcher> defines one type of slicing. SimpleConditionTracker(const std::string& name, const int index, const SimpleCondition& simpleCondition, const std::unordered_map<std::string, int>& trackerNameIndexMap); @@ -41,7 +44,14 @@ public: const std::vector<MatchingState>& eventMatcherValues, const std::vector<sp<ConditionTracker>>& mAllConditions, std::vector<ConditionState>& conditionCache, - std::vector<bool>& changedCache) override; + std::vector<bool>& changedCache, + std::vector<bool>& slicedChangedCache) override; + + void isConditionMet(const std::map<std::string, HashableDimensionKey>& conditionParameters, + const std::vector<sp<ConditionTracker>>& allConditions, + std::vector<ConditionState>& conditionCache) override; + + void addDimensions(const std::vector<KeyMatcher>& keyMatchers) override; private: // The index of the LogEventMatcher which defines the start. @@ -55,6 +65,13 @@ private: // The index of the LogEventMatcher which defines the stop all. int mStopAllLogMatcherIndex; + + // Different metrics may subscribe to different types of slicings. So it's a vector of vector. + std::vector<std::vector<KeyMatcher>> mDimensionsList; + + // Keep the map from the internal HashableDimensionKey to std::vector<KeyValuePair> + // that StatsLogReport wants. + std::unordered_map<HashableDimensionKey, ConditionState> mSlicedConditionState; }; } // namespace statsd diff --git a/cmds/statsd/src/condition/condition_util.cpp b/cmds/statsd/src/condition/condition_util.cpp index c7c8fcc9b361..40d41be381fd 100644 --- a/cmds/statsd/src/condition/condition_util.cpp +++ b/cmds/statsd/src/condition/condition_util.cpp @@ -23,6 +23,7 @@ #include <log/logprint.h> #include <utils/Errors.h> #include <unordered_map> +#include "../matchers/matcher_util.h" #include "ConditionTracker.h" #include "frameworks/base/cmds/statsd/src/stats_log.pb.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" @@ -90,6 +91,25 @@ ConditionState evaluateCombinationCondition(const std::vector<int>& children, return newCondition; } +HashableDimensionKey getDimensionKeyForCondition(const LogEvent& event, + const EventConditionLink& link) { + vector<KeyMatcher> eventKey; + eventKey.reserve(link.key_in_main().size()); + + for (const auto& key : link.key_in_main()) { + eventKey.push_back(key); + } + + vector<KeyValuePair> dimensionKey = getDimensionKey(event, eventKey); + + for (int i = 0; i < link.key_in_main_size(); i++) { + auto& kv = dimensionKey[i]; + kv.set_key(link.key_in_condition(i).key()); + } + + return getHashableKey(dimensionKey); +} + } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/condition/condition_util.h b/cmds/statsd/src/condition/condition_util.h index a4fcea38d6b5..47e245e4704b 100644 --- a/cmds/statsd/src/condition/condition_util.h +++ b/cmds/statsd/src/condition/condition_util.h @@ -18,6 +18,7 @@ #define CONDITION_UTIL_H #include <vector> +#include "../matchers/matcher_util.h" #include "frameworks/base/cmds/statsd/src/stats_log.pb.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" @@ -35,6 +36,9 @@ enum ConditionState { ConditionState evaluateCombinationCondition(const std::vector<int>& children, const LogicalOperation& operation, const std::vector<ConditionState>& conditionCache); + +HashableDimensionKey getDimensionKeyForCondition(const LogEvent& event, + const EventConditionLink& link); } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/config/ConfigManager.cpp b/cmds/statsd/src/config/ConfigManager.cpp index 2a4d6e22a2ed..038edd313081 100644 --- a/cmds/statsd/src/config/ConfigManager.cpp +++ b/cmds/statsd/src/config/ConfigManager.cpp @@ -118,130 +118,156 @@ static StatsdConfig build_fake_config() { StatsdConfig config; config.set_config_id(12345L); - // One count metric to count screen on + int WAKE_LOCK_TAG_ID = 11; + int WAKE_LOCK_UID_KEY_ID = 1; + int WAKE_LOCK_STATE_KEY = 2; + int WAKE_LOCK_ACQUIRE_VALUE = 1; + int WAKE_LOCK_RELEASE_VALUE = 0; + + int APP_USAGE_ID = 12345; + int APP_USAGE_UID_KEY_ID = 1; + int APP_USAGE_STATE_KEY = 2; + int APP_USAGE_FOREGROUND = 1; + int APP_USAGE_BACKGROUND = 0; + + int SCREEN_EVENT_TAG_ID = 2; + int SCREEN_EVENT_STATE_KEY = 1; + int SCREEN_EVENT_ON_VALUE = 2; + int SCREEN_EVENT_OFF_VALUE = 1; + + int UID_PROCESS_STATE_TAG_ID = 3; + int UID_PROCESS_STATE_UID_KEY = 1; + + // Count Screen ON events. CountMetric* metric = config.add_count_metric(); - metric->set_metric_id(20150717L); - metric->set_what("SCREEN_IS_ON"); + metric->set_metric_id(1); + metric->set_what("SCREEN_TURNED_ON"); metric->mutable_bucket()->set_bucket_size_millis(30 * 1000L); - // One count metric to count PHOTO_CHANGE_OR_CHROME_CRASH + // Count process state changes, slice by uid. metric = config.add_count_metric(); - metric->set_metric_id(20150718L); - metric->set_what("PHOTO_PROCESS_STATE_CHANGE"); - metric->mutable_bucket()->set_bucket_size_millis(60 * 1000L); - metric->set_condition("SCREEN_IS_ON"); + metric->set_metric_id(2); + metric->set_what("PROCESS_STATE_CHANGE"); + metric->mutable_bucket()->set_bucket_size_millis(30 * 1000L); + KeyMatcher* keyMatcher = metric->add_dimension(); + keyMatcher->set_key(UID_PROCESS_STATE_UID_KEY); - LogEntryMatcher* eventMatcher = config.add_log_entry_matcher(); - eventMatcher->set_name("SCREEN_IS_ON"); + // Count process state changes, slice by uid, while SCREEN_IS_OFF + metric = config.add_count_metric(); + metric->set_metric_id(3); + metric->set_what("PROCESS_STATE_CHANGE"); + metric->mutable_bucket()->set_bucket_size_millis(30 * 1000L); + keyMatcher = metric->add_dimension(); + keyMatcher->set_key(UID_PROCESS_STATE_UID_KEY); + metric->set_condition("SCREEN_IS_OFF"); + // Count wake lock, slice by uid, while SCREEN_IS_OFF and app in background + metric = config.add_count_metric(); + metric->set_metric_id(4); + metric->set_what("APP_GET_WL"); + metric->mutable_bucket()->set_bucket_size_millis(30 * 1000L); + keyMatcher = metric->add_dimension(); + keyMatcher->set_key(WAKE_LOCK_UID_KEY_ID); + metric->set_condition("APP_IS_BACKGROUND_AND_SCREEN_ON"); + EventConditionLink* link = metric->add_links(); + link->set_condition("APP_IS_BACKGROUND"); + link->add_key_in_main()->set_key(WAKE_LOCK_UID_KEY_ID); + link->add_key_in_condition()->set_key(APP_USAGE_UID_KEY_ID); + + // Duration of an app holding wl, while screen on and app in background + DurationMetric* durationMetric = config.add_duration_metric(); + durationMetric->set_metric_id(5); + durationMetric->set_start("APP_GET_WL"); + durationMetric->set_stop("APP_RELEASE_WL"); + durationMetric->mutable_bucket()->set_bucket_size_millis(30 * 1000L); + durationMetric->set_type(DurationMetric_AggregationType_DURATION_SUM); + keyMatcher = durationMetric->add_dimension(); + keyMatcher->set_key(WAKE_LOCK_UID_KEY_ID); + durationMetric->set_predicate("APP_IS_BACKGROUND_AND_SCREEN_ON"); + link = durationMetric->add_links(); + link->set_condition("APP_IS_BACKGROUND"); + link->add_key_in_main()->set_key(WAKE_LOCK_UID_KEY_ID); + link->add_key_in_condition()->set_key(APP_USAGE_UID_KEY_ID); + + // Event matchers............ + LogEntryMatcher* eventMatcher = config.add_log_entry_matcher(); + eventMatcher->set_name("SCREEN_TURNED_ON"); SimpleLogEntryMatcher* simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); - simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/); - simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()->set_key( - 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); - simpleLogEntryMatcher->mutable_key_value_matcher(0)->set_eq_int( - 2 /*SCREEN_STATE_CHANGE__DISPLAY_STATE__STATE_ON*/); + simpleLogEntryMatcher->set_tag(SCREEN_EVENT_TAG_ID); + KeyValueMatcher* keyValueMatcher = simpleLogEntryMatcher->add_key_value_matcher(); + keyValueMatcher->mutable_key_matcher()->set_key(SCREEN_EVENT_STATE_KEY); + keyValueMatcher->set_eq_int(SCREEN_EVENT_ON_VALUE); eventMatcher = config.add_log_entry_matcher(); - eventMatcher->set_name("SCREEN_IS_OFF"); + eventMatcher->set_name("SCREEN_TURNED_OFF"); + simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); + simpleLogEntryMatcher->set_tag(SCREEN_EVENT_TAG_ID); + keyValueMatcher = simpleLogEntryMatcher->add_key_value_matcher(); + keyValueMatcher->mutable_key_matcher()->set_key(SCREEN_EVENT_STATE_KEY); + keyValueMatcher->set_eq_int(SCREEN_EVENT_OFF_VALUE); + eventMatcher = config.add_log_entry_matcher(); + eventMatcher->set_name("PROCESS_STATE_CHANGE"); + simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); + simpleLogEntryMatcher->set_tag(UID_PROCESS_STATE_TAG_ID); + + eventMatcher = config.add_log_entry_matcher(); + eventMatcher->set_name("APP_GOES_BACKGROUND"); + simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); + simpleLogEntryMatcher->set_tag(APP_USAGE_ID); + keyValueMatcher = simpleLogEntryMatcher->add_key_value_matcher(); + keyValueMatcher->mutable_key_matcher()->set_key(APP_USAGE_STATE_KEY); + keyValueMatcher->set_eq_int(APP_USAGE_BACKGROUND); + + eventMatcher = config.add_log_entry_matcher(); + eventMatcher->set_name("APP_GOES_FOREGROUND"); simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); - simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/); - simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()->set_key( - 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); - simpleLogEntryMatcher->mutable_key_value_matcher(0)->set_eq_int( - 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE__STATE_OFF*/); - - LogEntryMatcher* procEventMatcher = config.add_log_entry_matcher(); - procEventMatcher->set_name("PHOTO_CRASH"); - - SimpleLogEntryMatcher* simpleLogMatcher2 = procEventMatcher->mutable_simple_log_entry_matcher(); - simpleLogMatcher2->add_tag(1112 /*PROCESS_STATE_CHANGE*/); - KeyValueMatcher* keyValueMatcher = simpleLogMatcher2->add_key_value_matcher(); - keyValueMatcher->mutable_key_matcher()->set_key(1002 /*pkg*/); - keyValueMatcher->set_eq_string( - "com.google.android.apps.photos" /*SCREEN_STATE_CHANGE__DISPLAY_STATE__STATE_ON*/); - - keyValueMatcher = simpleLogMatcher2->add_key_value_matcher(); - keyValueMatcher->mutable_key_matcher()->set_key(1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); - keyValueMatcher->set_eq_int(2); - - procEventMatcher = config.add_log_entry_matcher(); - procEventMatcher->set_name("PHOTO_START"); - - simpleLogMatcher2 = procEventMatcher->mutable_simple_log_entry_matcher(); - simpleLogMatcher2->add_tag(1112 /*PROCESS_STATE_CHANGE*/); - keyValueMatcher = simpleLogMatcher2->add_key_value_matcher(); - keyValueMatcher->mutable_key_matcher()->set_key(1002 /*pkg*/); - keyValueMatcher->set_eq_string( - "com.google.android.apps.photos" /*SCREEN_STATE_CHANGE__DISPLAY_STATE__STATE_ON*/); - - keyValueMatcher = simpleLogMatcher2->add_key_value_matcher(); - keyValueMatcher->mutable_key_matcher()->set_key(1 /*STATE*/); - keyValueMatcher->set_eq_int(1); - - procEventMatcher = config.add_log_entry_matcher(); - procEventMatcher->set_name("PHOTO_PROCESS_STATE_CHANGE"); - LogEntryMatcher_Combination* combinationMatcher = procEventMatcher->mutable_combination(); - combinationMatcher->set_operation(LogicalOperation::OR); - combinationMatcher->add_matcher("PHOTO_START"); - combinationMatcher->add_matcher("PHOTO_CRASH"); - - procEventMatcher = config.add_log_entry_matcher(); - procEventMatcher->set_name("CHROME_CRASH"); - - simpleLogMatcher2 = procEventMatcher->mutable_simple_log_entry_matcher(); - simpleLogMatcher2->add_tag(1112 /*PROCESS_STATE_CHANGE*/); - keyValueMatcher = simpleLogMatcher2->add_key_value_matcher(); - keyValueMatcher->mutable_key_matcher()->set_key(1002 /*pkg*/); - keyValueMatcher->set_eq_string( - "com.android.chrome" /*SCREEN_STATE_CHANGE__DISPLAY_STATE__STATE_ON*/); - - keyValueMatcher = simpleLogMatcher2->add_key_value_matcher(); - keyValueMatcher->mutable_key_matcher()->set_key(1 /*STATE*/); - keyValueMatcher->set_eq_int(2); - - procEventMatcher = config.add_log_entry_matcher(); - procEventMatcher->set_name("PHOTO_CHANGE_OR_CHROME_CRASH"); - combinationMatcher = procEventMatcher->mutable_combination(); - combinationMatcher->set_operation(LogicalOperation::OR); - combinationMatcher->add_matcher("PHOTO_PROCESS_STATE_CHANGE"); - combinationMatcher->add_matcher("CHROME_CRASH"); + simpleLogEntryMatcher->set_tag(APP_USAGE_ID); + keyValueMatcher = simpleLogEntryMatcher->add_key_value_matcher(); + keyValueMatcher->mutable_key_matcher()->set_key(APP_USAGE_STATE_KEY); + keyValueMatcher->set_eq_int(APP_USAGE_FOREGROUND); + eventMatcher = config.add_log_entry_matcher(); + eventMatcher->set_name("APP_GET_WL"); + simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); + simpleLogEntryMatcher->set_tag(WAKE_LOCK_TAG_ID); + keyValueMatcher = simpleLogEntryMatcher->add_key_value_matcher(); + keyValueMatcher->mutable_key_matcher()->set_key(WAKE_LOCK_STATE_KEY); + keyValueMatcher->set_eq_int(WAKE_LOCK_ACQUIRE_VALUE); + + eventMatcher = config.add_log_entry_matcher(); + eventMatcher->set_name("APP_RELEASE_WL"); + simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); + simpleLogEntryMatcher->set_tag(WAKE_LOCK_TAG_ID); + keyValueMatcher = simpleLogEntryMatcher->add_key_value_matcher(); + keyValueMatcher->mutable_key_matcher()->set_key(WAKE_LOCK_STATE_KEY); + keyValueMatcher->set_eq_int(WAKE_LOCK_RELEASE_VALUE); + + // Conditions............. Condition* condition = config.add_condition(); condition->set_name("SCREEN_IS_ON"); SimpleCondition* simpleCondition = condition->mutable_simple_condition(); - simpleCondition->set_start("SCREEN_IS_ON"); - simpleCondition->set_stop("SCREEN_IS_OFF"); - - condition = config.add_condition(); - condition->set_name("PHOTO_STARTED"); - - simpleCondition = condition->mutable_simple_condition(); - simpleCondition->set_start("PHOTO_START"); - simpleCondition->set_stop("PHOTO_CRASH"); + simpleCondition->set_start("SCREEN_TURNED_ON"); + simpleCondition->set_stop("SCREEN_TURNED_OFF"); condition = config.add_condition(); condition->set_name("SCREEN_IS_OFF"); - simpleCondition = condition->mutable_simple_condition(); - simpleCondition->set_start("SCREEN_IS_OFF"); - simpleCondition->set_stop("SCREEN_IS_ON"); + simpleCondition->set_start("SCREEN_TURNED_OFF"); + simpleCondition->set_stop("SCREEN_TURNED_ON"); condition = config.add_condition(); - condition->set_name("SCREEN_IS_EITHER_ON_OFF"); - - Condition_Combination* combination = condition->mutable_combination(); - combination->set_operation(LogicalOperation::OR); - combination->add_condition("SCREEN_IS_ON"); - combination->add_condition("SCREEN_IS_OFF"); + condition->set_name("APP_IS_BACKGROUND"); + simpleCondition = condition->mutable_simple_condition(); + simpleCondition->set_start("APP_GOES_BACKGROUND"); + simpleCondition->set_stop("APP_GOES_FOREGROUND"); condition = config.add_condition(); - condition->set_name("SCREEN_IS_NEITHER_ON_OFF"); - - combination = condition->mutable_combination(); - combination->set_operation(LogicalOperation::NOR); - combination->add_condition("SCREEN_IS_ON"); - combination->add_condition("SCREEN_IS_OFF"); + condition->set_name("APP_IS_BACKGROUND_AND_SCREEN_ON"); + Condition_Combination* combination_condition = condition->mutable_combination(); + combination_condition->set_operation(LogicalOperation::AND); + combination_condition->add_condition("APP_IS_BACKGROUND"); + combination_condition->add_condition("SCREEN_IS_ON"); return config; } diff --git a/cmds/statsd/src/logd/LogEvent.cpp b/cmds/statsd/src/logd/LogEvent.cpp index 9fa2baf653e3..032b4b86fe76 100644 --- a/cmds/statsd/src/logd/LogEvent.cpp +++ b/cmds/statsd/src/logd/LogEvent.cpp @@ -150,6 +150,29 @@ float LogEvent::GetFloat(size_t key, status_t* err) const { } } +KeyValuePair LogEvent::GetKeyValueProto(size_t key) const { + KeyValuePair pair; + pair.set_key(key); + // If the value is not valid, return the KeyValuePair without assigning the value. + // Caller can detect the error by checking the enum for "one of" proto type. + if (key < 1 || (key - 1) >= mElements.size()) { + return pair; + } + key--; + + const android_log_list_element& elem = mElements[key]; + if (elem.type == EVENT_TYPE_INT) { + pair.set_value_int(elem.data.int32); + } else if (elem.type == EVENT_TYPE_LONG) { + pair.set_value_int(elem.data.int64); + } else if (elem.type == EVENT_TYPE_STRING) { + pair.set_value_str(elem.data.string); + } else if (elem.type == EVENT_TYPE_FLOAT) { + pair.set_value_float(elem.data.float32); + } + return pair; +} + string LogEvent::ToString() const { ostringstream result; result << "{ " << mTimestampNs << " (" << mTagId << ")"; diff --git a/cmds/statsd/src/logd/LogEvent.h b/cmds/statsd/src/logd/LogEvent.h index b523201017ca..464afca1361f 100644 --- a/cmds/statsd/src/logd/LogEvent.h +++ b/cmds/statsd/src/logd/LogEvent.h @@ -80,6 +80,11 @@ public: */ void ToProto(EventMetricData* out) const; + /* + * Get a KeyValuePair proto object. + */ + KeyValuePair GetKeyValueProto(size_t key) const; + private: /** * Don't copy, it's slower. If we really need this we can add it but let's try to diff --git a/cmds/statsd/src/matchers/SimpleLogMatchingTracker.cpp b/cmds/statsd/src/matchers/SimpleLogMatchingTracker.cpp index 71078ea8aaba..b2c88a0d3013 100644 --- a/cmds/statsd/src/matchers/SimpleLogMatchingTracker.cpp +++ b/cmds/statsd/src/matchers/SimpleLogMatchingTracker.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#define DEBUG true // STOPSHIP if true +#define DEBUG false // STOPSHIP if true #include "Log.h" #include "SimpleLogMatchingTracker.h" @@ -34,10 +34,12 @@ using std::vector; SimpleLogMatchingTracker::SimpleLogMatchingTracker(const string& name, const int index, const SimpleLogEntryMatcher& matcher) : LogMatchingTracker(name, index), mMatcher(matcher) { - for (int i = 0; i < matcher.tag_size(); i++) { - mTagIds.insert(matcher.tag(i)); + if (!matcher.has_tag()) { + mInitialized = false; + } else { + mTagIds.insert(matcher.tag()); + mInitialized = true; } - mInitialized = true; } SimpleLogMatchingTracker::~SimpleLogMatchingTracker() { @@ -48,7 +50,7 @@ bool SimpleLogMatchingTracker::init(const vector<LogEntryMatcher>& allLogMatcher const unordered_map<string, int>& matcherMap, vector<bool>& stack) { // no need to do anything. - return true; + return mInitialized; } void SimpleLogMatchingTracker::onLogEvent(const LogEvent& event, diff --git a/cmds/statsd/src/matchers/matcher_util.cpp b/cmds/statsd/src/matchers/matcher_util.cpp index 1c1e3c76a6bb..6aa22118b320 100644 --- a/cmds/statsd/src/matchers/matcher_util.cpp +++ b/cmds/statsd/src/matchers/matcher_util.cpp @@ -30,9 +30,9 @@ #include <sstream> #include <unordered_map> +using std::ostringstream; using std::set; using std::string; -using std::ostringstream; using std::unordered_map; using std::vector; @@ -91,109 +91,103 @@ bool combinationMatch(const vector<int>& children, const LogicalOperation& opera bool matchesSimple(const SimpleLogEntryMatcher& simpleMatcher, const LogEvent& event) { const int tagId = event.GetTagId(); - /* - const unordered_map<int, long>& intMap = event.intMap; - const unordered_map<int, string>& strMap = event.strMap; - const unordered_map<int, float>& floatMap = event.floatMap; - const unordered_map<int, bool>& boolMap = event.boolMap; - */ - - for (int i = 0; i < simpleMatcher.tag_size(); i++) { - if (simpleMatcher.tag(i) != tagId) { - continue; - } - // TODO Is this right? Shouldn't this second loop be outside the outer loop? - // If I understand correctly, the event matches if one of the tags match, - // and ALL of the key-value matchers match. --joeo - - // now see if this event is interesting to us -- matches ALL the matchers - // defined in the metrics. - bool allMatched = true; - for (int j = 0; allMatched && j < simpleMatcher.key_value_matcher_size(); j++) { - auto cur = simpleMatcher.key_value_matcher(j); + if (simpleMatcher.tag() != tagId) { + return false; + } + // now see if this event is interesting to us -- matches ALL the matchers + // defined in the metrics. + bool allMatched = true; + for (int j = 0; allMatched && j < simpleMatcher.key_value_matcher_size(); j++) { + auto cur = simpleMatcher.key_value_matcher(j); - // TODO: Check if this key is a magic key (eg package name). - // TODO: Maybe make packages a different type in the config? - int key = cur.key_matcher().key(); + // TODO: Check if this key is a magic key (eg package name). + // TODO: Maybe make packages a different type in the config? + int key = cur.key_matcher().key(); - const KeyValueMatcher::ValueMatcherCase matcherCase = cur.value_matcher_case(); - if (matcherCase == KeyValueMatcher::ValueMatcherCase::kEqString) { - // String fields - status_t err = NO_ERROR; - const char* val = event.GetString(key, &err); - if (err == NO_ERROR && val != NULL) { - if (!(cur.eq_string() == val)) { + const KeyValueMatcher::ValueMatcherCase matcherCase = cur.value_matcher_case(); + if (matcherCase == KeyValueMatcher::ValueMatcherCase::kEqString) { + // String fields + status_t err = NO_ERROR; + const char* val = event.GetString(key, &err); + if (err == NO_ERROR && val != NULL) { + if (!(cur.eq_string() == val)) { + allMatched = false; + } + } + } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kEqInt || + matcherCase == KeyValueMatcher::ValueMatcherCase::kLtInt || + matcherCase == KeyValueMatcher::ValueMatcherCase::kGtInt || + matcherCase == KeyValueMatcher::ValueMatcherCase::kLteInt || + matcherCase == KeyValueMatcher::ValueMatcherCase::kGteInt) { + // Integer fields + status_t err = NO_ERROR; + int64_t val = event.GetLong(key, &err); + if (err == NO_ERROR) { + if (matcherCase == KeyValueMatcher::ValueMatcherCase::kEqInt) { + if (!(val == cur.eq_int())) { allMatched = false; } - } - } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kEqInt - || matcherCase == KeyValueMatcher::ValueMatcherCase::kLtInt - || matcherCase == KeyValueMatcher::ValueMatcherCase::kGtInt - || matcherCase == KeyValueMatcher::ValueMatcherCase::kLteInt - || matcherCase == KeyValueMatcher::ValueMatcherCase::kGteInt) { - // Integer fields - status_t err = NO_ERROR; - int64_t val = event.GetLong(key, &err); - if (err == NO_ERROR) { - if (matcherCase == KeyValueMatcher::ValueMatcherCase::kEqInt) { - if (!(val == cur.eq_int())) { - allMatched = false; - } - } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kLtInt) { - if (!(val < cur.lt_int())) { - allMatched = false; - } - } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kGtInt) { - if (!(val > cur.gt_int())) { - allMatched = false; - } - } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kLteInt) { - if (!(val <= cur.lte_int())) { - allMatched = false; - } - } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kGteInt) { - if (!(val >= cur.gte_int())) { - allMatched = false; - } + } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kLtInt) { + if (!(val < cur.lt_int())) { + allMatched = false; } - } - break; - } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kEqBool) { - // Boolean fields - status_t err = NO_ERROR; - bool val = event.GetBool(key, &err); - if (err == NO_ERROR) { - if (!(cur.eq_bool() == val)) { + } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kGtInt) { + if (!(val > cur.gt_int())) { + allMatched = false; + } + } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kLteInt) { + if (!(val <= cur.lte_int())) { allMatched = false; } + } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kGteInt) { + if (!(val >= cur.gte_int())) { + allMatched = false; + } + } + } + break; + } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kEqBool) { + // Boolean fields + status_t err = NO_ERROR; + bool val = event.GetBool(key, &err); + if (err == NO_ERROR) { + if (!(cur.eq_bool() == val)) { + allMatched = false; } - } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kLtFloat - || matcherCase == KeyValueMatcher::ValueMatcherCase::kGtFloat) { - // Float fields - status_t err = NO_ERROR; - bool val = event.GetFloat(key, &err); - if (err == NO_ERROR) { - if (matcherCase == KeyValueMatcher::ValueMatcherCase::kLtFloat) { - if (!(cur.lt_float() <= val)) { - allMatched = false; - } - } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kGtFloat) { - if (!(cur.gt_float() >= val)) { - allMatched = false; - } + } + } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kLtFloat || + matcherCase == KeyValueMatcher::ValueMatcherCase::kGtFloat) { + // Float fields + status_t err = NO_ERROR; + bool val = event.GetFloat(key, &err); + if (err == NO_ERROR) { + if (matcherCase == KeyValueMatcher::ValueMatcherCase::kLtFloat) { + if (!(cur.lt_float() <= val)) { + allMatched = false; + } + } else if (matcherCase == KeyValueMatcher::ValueMatcherCase::kGtFloat) { + if (!(cur.gt_float() >= val)) { + allMatched = false; } } - } else { - // If value matcher is not present, assume that we match. } + } else { + // If value matcher is not present, assume that we match. } + } + return allMatched; +} - if (allMatched) { - return true; - } +vector<KeyValuePair> getDimensionKey(const LogEvent& event, + const std::vector<KeyMatcher>& dimensions) { + vector<KeyValuePair> key; + key.reserve(dimensions.size()); + for (const KeyMatcher& dimension : dimensions) { + KeyValuePair k = event.GetKeyValueProto(dimension.key()); + key.push_back(k); } - return false; + return key; } } // namespace statsd diff --git a/cmds/statsd/src/matchers/matcher_util.h b/cmds/statsd/src/matchers/matcher_util.h index 3a5925c138c9..4ea6f0b4a3ed 100644 --- a/cmds/statsd/src/matchers/matcher_util.h +++ b/cmds/statsd/src/matchers/matcher_util.h @@ -27,6 +27,7 @@ #include <vector> #include "frameworks/base/cmds/statsd/src/stats_log.pb.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" +#include "stats_util.h" namespace android { namespace os { @@ -43,6 +44,9 @@ bool combinationMatch(const std::vector<int>& children, const LogicalOperation& bool matchesSimple(const SimpleLogEntryMatcher& simpleMatcher, const LogEvent& wrapper); +std::vector<KeyValuePair> getDimensionKey(const LogEvent& event, + const std::vector<KeyMatcher>& dimensions); + } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/metrics/CountMetricProducer.cpp b/cmds/statsd/src/metrics/CountMetricProducer.cpp index 7df62fb60244..1f07914175d2 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.cpp +++ b/cmds/statsd/src/metrics/CountMetricProducer.cpp @@ -17,35 +17,49 @@ #define DEBUG true // STOPSHIP if true #include "Log.h" -#include "CountMetricProducer.h" #include "CountAnomalyTracker.h" +#include "CountMetricProducer.h" +#include "stats_util.h" #include <cutils/log.h> #include <limits.h> #include <stdlib.h> +using std::map; +using std::string; using std::unordered_map; +using std::vector; namespace android { namespace os { namespace statsd { -CountMetricProducer::CountMetricProducer(const CountMetric& metric, const bool hasCondition) - : mMetric(metric), - mStartTime(time(nullptr)), - mCounter(0), - mCurrentBucketStartTime(mStartTime), +// TODO: add back AnomalyTracker. +CountMetricProducer::CountMetricProducer(const CountMetric& metric, const int conditionIndex, + const sp<ConditionWizard>& wizard) + // TODO: Pass in the start time from MetricsManager, instead of calling time() here. + : MetricProducer((time(nullptr) * NANO_SECONDS_IN_A_SECOND), conditionIndex, wizard), + mMetric(metric), // TODO: read mAnomalyTracker parameters from config file. - mAnomalyTracker(6, 10), - mCondition(hasCondition ? ConditionState::kUnknown : ConditionState::kTrue) { + mAnomalyTracker(6, 10) { // TODO: evaluate initial conditions. and set mConditionMet. if (metric.has_bucket() && metric.bucket().has_bucket_size_millis()) { - mBucketSize_sec = metric.bucket().bucket_size_millis() / 1000; + mBucketSizeNs = metric.bucket().bucket_size_millis() * 1000 * 1000; } else { - mBucketSize_sec = LONG_MAX; + mBucketSizeNs = LLONG_MAX; + } + + // TODO: use UidMap if uid->pkg_name is required + mDimension.insert(mDimension.begin(), metric.dimension().begin(), metric.dimension().end()); + + if (metric.links().size() > 0) { + mConditionLinks.insert(mConditionLinks.begin(), metric.links().begin(), + metric.links().end()); + mConditionSliced = true; } - VLOG("created. bucket size %lu start_time: %lu", mBucketSize_sec, mStartTime); + VLOG("metric %lld created. bucket size %lld start_time: %lld", metric.metric_id(), + (long long)mBucketSizeNs, (long long)mStartTimeNs); } CountMetricProducer::~CountMetricProducer() { @@ -57,54 +71,146 @@ void CountMetricProducer::finish() { // DropboxWriter. } -void CountMetricProducer::onDumpReport() { - VLOG("dump report now..."); +static void addSlicedCounterToReport(StatsLogReport_CountMetricDataWrapper& wrapper, + const vector<KeyValuePair>& key, + const vector<CountBucketInfo>& buckets) { + CountMetricData* data = wrapper.add_data(); + for (const auto& kv : key) { + data->add_dimension()->CopyFrom(kv); + } + for (const auto& bucket : buckets) { + data->add_bucket_info()->CopyFrom(bucket); + VLOG("\t bucket [%lld - %lld] count: %lld", bucket.start_bucket_nanos(), + bucket.end_bucket_nanos(), bucket.count()); + } +} + +void CountMetricProducer::onSlicedConditionMayChange() { + VLOG("Metric %lld onSlicedConditionMayChange", mMetric.metric_id()); +} + +StatsLogReport CountMetricProducer::onDumpReport() { + VLOG("metric %lld dump report now...", mMetric.metric_id()); + + StatsLogReport report; + report.set_metric_id(mMetric.metric_id()); + report.set_start_report_nanos(mStartTimeNs); + + // Dump current bucket if it's stale. + // If current bucket is still on-going, don't force dump current bucket. + // In finish(), We can force dump current bucket. + flushCounterIfNeeded(time(nullptr) * NANO_SECONDS_IN_A_SECOND); + report.set_end_report_nanos(mCurrentBucketStartTimeNs); + + StatsLogReport_CountMetricDataWrapper* wrapper = report.mutable_count_metrics(); + + for (const auto& pair : mPastBuckets) { + const HashableDimensionKey& hashableKey = pair.first; + auto it = mDimensionKeyMap.find(hashableKey); + if (it == mDimensionKeyMap.end()) { + ALOGE("Dimension key %s not found?!?! skip...", hashableKey.c_str()); + continue; + } + + VLOG(" dimension key %s", hashableKey.c_str()); + addSlicedCounterToReport(*wrapper, it->second, pair.second); + } + return report; + // TODO: Clear mPastBuckets, mDimensionKeyMap once the report is dumped. } void CountMetricProducer::onConditionChanged(const bool conditionMet) { - VLOG("onConditionChanged"); + VLOG("Metric %lld onConditionChanged", mMetric.metric_id()); mCondition = conditionMet; } -void CountMetricProducer::onMatchedLogEvent(const LogEvent& event) { - time_t eventTime = event.GetTimestampNs() / 1000000000; - +void CountMetricProducer::onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) { + uint64_t eventTimeNs = event.GetTimestampNs(); // this is old event, maybe statsd restarted? - if (eventTime < mStartTime) { + if (eventTimeNs < mStartTimeNs) { return; } - if (mCondition == ConditionState::kTrue) { - flushCounterIfNeeded(eventTime); - mCounter++; - mAnomalyTracker.checkAnomaly(mCounter); - VLOG("metric %lld count %d", mMetric.metric_id(), mCounter); + flushCounterIfNeeded(eventTimeNs); + + if (mConditionSliced) { + map<string, HashableDimensionKey> conditionKeys; + for (const auto& link : mConditionLinks) { + VLOG("Condition link key_in_main size %d", link.key_in_main_size()); + HashableDimensionKey conditionKey = getDimensionKeyForCondition(event, link); + conditionKeys[link.condition()] = conditionKey; + } + if (mWizard->query(mConditionTrackerIndex, conditionKeys) != ConditionState::kTrue) { + VLOG("metric %lld sliced condition not met", mMetric.metric_id()); + return; + } + } else { + if (!mCondition) { + VLOG("metric %lld condition not met", mMetric.metric_id()); + return; + } + } + + HashableDimensionKey hashableKey; + + if (mDimension.size() > 0) { + vector<KeyValuePair> key = getDimensionKey(event, mDimension); + hashableKey = getHashableKey(key); + // Add the HashableDimensionKey->vector<KeyValuePair> to the map, because StatsLogReport + // expects vector<KeyValuePair>. + if (mDimensionKeyMap.find(hashableKey) == mDimensionKeyMap.end()) { + mDimensionKeyMap[hashableKey] = key; + } + } else { + hashableKey = DEFAULT_DIMENSION_KEY; + } + + auto it = mCurrentSlicedCounter.find(hashableKey); + + if (it == mCurrentSlicedCounter.end()) { + // create a counter for the new key + mCurrentSlicedCounter[hashableKey] = 1; + + } else { + // increment the existing value + auto& count = it->second; + count++; } + + VLOG("metric %lld %s->%d", mMetric.metric_id(), hashableKey.c_str(), + mCurrentSlicedCounter[hashableKey]); } -// When a new matched event comes in, we check if it falls into the current -// bucket. And flush the counter to the StatsLogReport and adjust the bucket if -// needed. -void CountMetricProducer::flushCounterIfNeeded(const time_t& eventTime) { - if (mCurrentBucketStartTime + mBucketSize_sec > eventTime) { +// When a new matched event comes in, we check if event falls into the current +// bucket. If not, flush the old counter to past buckets and initialize the new bucket. +void CountMetricProducer::flushCounterIfNeeded(const uint64_t eventTimeNs) { + if (mCurrentBucketStartTimeNs + mBucketSizeNs > eventTimeNs) { return; } - // TODO: add a KeyValuePair to StatsLogReport. - ALOGD("%lld: dump counter %d", mMetric.metric_id(), mCounter); - // adjust the bucket start time - time_t numBucketsForward = (eventTime - mCurrentBucketStartTime) - / mBucketSize_sec; + int64_t numBucketsForward = (eventTimeNs - mCurrentBucketStartTimeNs) / mBucketSizeNs; + + CountBucketInfo info; + info.set_start_bucket_nanos(mCurrentBucketStartTimeNs); + info.set_end_bucket_nanos(mCurrentBucketStartTimeNs + mBucketSizeNs); - mCurrentBucketStartTime = mCurrentBucketStartTime + - (numBucketsForward) * mBucketSize_sec; + for (const auto& counter : mCurrentSlicedCounter) { + info.set_count(counter.second); + // it will auto create new vector of CountbucketInfo if the key is not found. + auto& bucketList = mPastBuckets[counter.first]; + bucketList.push_back(info); + + VLOG("metric %lld, dump key value: %s -> %d", mMetric.metric_id(), counter.first.c_str(), + counter.second); + } - // reset counter - mAnomalyTracker.addPastBucket(mCounter, numBucketsForward); - mCounter = 0; + // Reset counters + mCurrentSlicedCounter.clear(); - VLOG("%lld: new bucket start time: %lu", mMetric.metric_id(), mCurrentBucketStartTime); + mCurrentBucketStartTimeNs = mCurrentBucketStartTimeNs + numBucketsForward * mBucketSizeNs; + VLOG("metric %lld: new bucket start time: %lld", mMetric.metric_id(), + (long long)mCurrentBucketStartTimeNs); } } // namespace statsd diff --git a/cmds/statsd/src/metrics/CountMetricProducer.h b/cmds/statsd/src/metrics/CountMetricProducer.h index 94abd6211ac5..f0d60255af51 100644 --- a/cmds/statsd/src/metrics/CountMetricProducer.h +++ b/cmds/statsd/src/metrics/CountMetricProducer.h @@ -19,12 +19,13 @@ #include <unordered_map> -#include "CountAnomalyTracker.h" #include "../condition/ConditionTracker.h" #include "../matchers/matcher_util.h" +#include "CountAnomalyTracker.h" #include "MetricProducer.h" #include "frameworks/base/cmds/statsd/src/stats_log.pb.h" #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" +#include "stats_util.h" using namespace std; @@ -34,38 +35,37 @@ namespace statsd { class CountMetricProducer : public MetricProducer { public: - CountMetricProducer(const CountMetric& countMetric, const bool hasCondition); + // TODO: Pass in the start time from MetricsManager, it should be consistent for all metrics. + CountMetricProducer(const CountMetric& countMetric, const int conditionIndex, + const sp<ConditionWizard>& wizard); virtual ~CountMetricProducer(); - void onMatchedLogEvent(const LogEvent& event) override; + void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) override; void onConditionChanged(const bool conditionMet) override; void finish() override; - void onDumpReport() override; + StatsLogReport onDumpReport() override; + + void onSlicedConditionMayChange() override; // TODO: Implement this later. - virtual void notifyAppUpgrade(const string& apk, const int uid, const int version) override {}; + virtual void notifyAppUpgrade(const string& apk, const int uid, const int version) override{}; private: const CountMetric mMetric; - const time_t mStartTime; - // TODO: Add dimensions. - // Counter value for the current bucket. - int mCounter; - - time_t mCurrentBucketStartTime; - - long mBucketSize_sec; - CountAnomalyTracker mAnomalyTracker; - bool mCondition; + // Save the past buckets and we can clear when the StatsLogReport is dumped. + std::unordered_map<HashableDimensionKey, std::vector<CountBucketInfo>> mPastBuckets; + + // The current bucket. + std::unordered_map<HashableDimensionKey, int> mCurrentSlicedCounter; - void flushCounterIfNeeded(const time_t& newEventTime); + void flushCounterIfNeeded(const uint64_t newEventTime); }; } // namespace statsd diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.cpp b/cmds/statsd/src/metrics/DurationMetricProducer.cpp new file mode 100644 index 000000000000..aa597f421cb6 --- /dev/null +++ b/cmds/statsd/src/metrics/DurationMetricProducer.cpp @@ -0,0 +1,384 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define DEBUG true +#include "DurationMetricProducer.h" +#include "Log.h" +#include "stats_util.h" + +#include <cutils/log.h> +#include <limits.h> +#include <stdlib.h> + +using std::string; +using std::unordered_map; +using std::vector; + +namespace android { +namespace os { +namespace statsd { + +DurationMetricProducer::DurationMetricProducer(const DurationMetric& metric, + const int conditionIndex, const size_t startIndex, + const size_t stopIndex, const size_t stopAllIndex, + const sp<ConditionWizard>& wizard) + // TODO: Pass in the start time from MetricsManager, instead of calling time() here. + : MetricProducer(time(nullptr) * NANO_SECONDS_IN_A_SECOND, conditionIndex, wizard), + mMetric(metric), + mStartIndex(startIndex), + mStopIndex(stopIndex), + mStopAllIndex(stopAllIndex) { + // TODO: The following boiler plate code appears in all MetricProducers, but we can't abstract + // them in the base class, because the proto generated CountMetric, and DurationMetric are + // not related. Maybe we should add a template in the future?? + if (metric.has_bucket() && metric.bucket().has_bucket_size_millis()) { + mBucketSizeNs = metric.bucket().bucket_size_millis() * 1000000; + } else { + mBucketSizeNs = LLONG_MAX; + } + + // TODO: use UidMap if uid->pkg_name is required + mDimension.insert(mDimension.begin(), metric.dimension().begin(), metric.dimension().end()); + + if (metric.links().size() > 0) { + mConditionLinks.insert(mConditionLinks.begin(), metric.links().begin(), + metric.links().end()); + mConditionSliced = true; + } + + VLOG("metric %lld created. bucket size %lld start_time: %lld", metric.metric_id(), + (long long)mBucketSizeNs, (long long)mStartTimeNs); +} + +DurationMetricProducer::~DurationMetricProducer() { + VLOG("~DurationMetric() called"); +} + +void DurationMetricProducer::finish() { + // TODO: write the StatsLogReport to dropbox using + // DropboxWriter. +} + +void DurationMetricProducer::onSlicedConditionMayChange() { + VLOG("Metric %lld onSlicedConditionMayChange", mMetric.metric_id()); + // Now for each of the on-going event, check if the condition has changed for them. + for (auto& pair : mCurrentSlicedDuration) { + VLOG("Metric %lld current %s state: %d", mMetric.metric_id(), pair.first.c_str(), + pair.second.state); + if (pair.second.state == kStopped) { + continue; + } + bool conditionMet = mWizard->query(mConditionTrackerIndex, pair.second.conditionKeys) == + ConditionState::kTrue; + VLOG("key: %s, condition: %d", pair.first.c_str(), conditionMet); + noteConditionChanged(pair.first, conditionMet, time(nullptr) * 1000000000); + } +} + +void DurationMetricProducer::onConditionChanged(const bool conditionMet) { + VLOG("Metric %lld onConditionChanged", mMetric.metric_id()); + mCondition = conditionMet; + // TODO: need to populate the condition change time from the event which triggers the condition + // change, instead of using current time. + for (auto& pair : mCurrentSlicedDuration) { + noteConditionChanged(pair.first, conditionMet, time(nullptr) * 1000000000); + } +} + +static void addDurationBucketsToReport(StatsLogReport_DurationMetricDataWrapper& wrapper, + const vector<KeyValuePair>& key, + const vector<DurationBucketInfo>& buckets) { + DurationMetricData* data = wrapper.add_data(); + for (const auto& kv : key) { + data->add_dimension()->CopyFrom(kv); + } + for (const auto& bucket : buckets) { + data->add_bucket_info()->CopyFrom(bucket); + VLOG("\t bucket [%lld - %lld] count: %lld", bucket.start_bucket_nanos(), + bucket.end_bucket_nanos(), bucket.duration_nanos()); + } +} + +StatsLogReport DurationMetricProducer::onDumpReport() { + VLOG("metric %lld dump report now...", mMetric.metric_id()); + StatsLogReport report; + report.set_metric_id(mMetric.metric_id()); + report.set_start_report_nanos(mStartTimeNs); + // Dump current bucket if it's stale. + // If current bucket is still on-going, don't force dump current bucket. + // In finish(), We can force dump current bucket. + flushDurationIfNeeded(time(nullptr) * NANO_SECONDS_IN_A_SECOND); + report.set_end_report_nanos(mCurrentBucketStartTimeNs); + + StatsLogReport_DurationMetricDataWrapper* wrapper = report.mutable_duration_metrics(); + for (const auto& pair : mPastBuckets) { + const HashableDimensionKey& hashableKey = pair.first; + auto it = mDimensionKeyMap.find(hashableKey); + if (it == mDimensionKeyMap.end()) { + ALOGW("Dimension key %s not found?!?! skip...", hashableKey.c_str()); + continue; + } + VLOG(" dimension key %s", hashableKey.c_str()); + addDurationBucketsToReport(*wrapper, it->second, pair.second); + } + return report; +}; + +void DurationMetricProducer::onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) { + if (event.GetTimestampNs() < mStartTimeNs) { + return; + } + + flushDurationIfNeeded(event.GetTimestampNs()); + + if (matcherIndex == mStopAllIndex) { + noteStopAll(event.GetTimestampNs()); + return; + } + + HashableDimensionKey hashableKey; + if (mDimension.size() > 0) { + // hook up sliced counter with AnomalyMonitor. + vector<KeyValuePair> key = getDimensionKey(event, mDimension); + hashableKey = getHashableKey(key); + // Add the HashableDimensionKey->DimensionKey to the map, because StatsLogReport expects + // vector<KeyValuePair>. + if (mDimensionKeyMap.find(hashableKey) == mDimensionKeyMap.end()) { + mDimensionKeyMap[hashableKey] = key; + } + } else { + hashableKey = DEFAULT_DIMENSION_KEY; + } + + if (mCurrentSlicedDuration.find(hashableKey) == mCurrentSlicedDuration.end() && + mConditionSliced) { + // add the durationInfo for the current bucket. + auto& durationInfo = mCurrentSlicedDuration[hashableKey]; + auto& conditionKeys = durationInfo.conditionKeys; + // get and cache the keys for query condition. + for (const auto& link : mConditionLinks) { + HashableDimensionKey conditionKey = getDimensionKeyForCondition(event, link); + conditionKeys[link.condition()] = conditionKey; + } + } + + bool conditionMet; + if (mConditionSliced) { + const auto& conditionKeys = mCurrentSlicedDuration[hashableKey].conditionKeys; + conditionMet = + mWizard->query(mConditionTrackerIndex, conditionKeys) == ConditionState::kTrue; + } else { + conditionMet = mCondition; + } + + if (matcherIndex == mStartIndex) { + VLOG("Metric %lld Key: %s Start, Condition %d", mMetric.metric_id(), hashableKey.c_str(), + conditionMet); + noteStart(hashableKey, conditionMet, event.GetTimestampNs()); + } else if (matcherIndex == mStopIndex) { + VLOG("Metric %lld Key: %s Stop, Condition %d", mMetric.metric_id(), hashableKey.c_str(), + conditionMet); + noteStop(hashableKey, event.GetTimestampNs()); + } +} + +void DurationMetricProducer::noteConditionChanged(const HashableDimensionKey& key, + const bool conditionMet, + const uint64_t eventTime) { + flushDurationIfNeeded(eventTime); + + auto it = mCurrentSlicedDuration.find(key); + if (it == mCurrentSlicedDuration.end()) { + return; + } + + switch (it->second.state) { + case kStarted: + // if condition becomes false, kStarted -> kPaused. Record the current duration. + if (!conditionMet) { + it->second.state = DurationState::kPaused; + it->second.lastDuration = + updateDuration(it->second.lastDuration, + eventTime - it->second.lastStartTime, mMetric.type()); + VLOG("Metric %lld Key: %s Paused because condition is false ", mMetric.metric_id(), + key.c_str()); + } + break; + case kStopped: + // nothing to do if it's stopped. + break; + case kPaused: + // if condition becomes true, kPaused -> kStarted. and the start time is the condition + // change time. + if (conditionMet) { + it->second.state = DurationState::kStarted; + it->second.lastStartTime = eventTime; + VLOG("Metric %lld Key: %s Paused->Started", mMetric.metric_id(), key.c_str()); + } + break; + } +} + +void DurationMetricProducer::noteStart(const HashableDimensionKey& key, const bool conditionMet, + const uint64_t eventTime) { + // this will add an empty bucket for this key if it didn't exist before. + DurationInfo& duration = mCurrentSlicedDuration[key]; + + switch (duration.state) { + case kStarted: + // It's safe to do nothing here. even if condition is not true, it means we are about + // to receive the condition change event. + break; + case kPaused: + // Safe to do nothing here. kPaused is waiting for the condition change. + break; + case kStopped: + if (!conditionMet) { + // event started, but we need to wait for the condition to become true. + duration.state = DurationState::kPaused; + break; + } + duration.state = DurationState::kStarted; + duration.lastStartTime = eventTime; + break; + } +} + +void DurationMetricProducer::noteStop(const HashableDimensionKey& key, const uint64_t eventTime) { + if (mCurrentSlicedDuration.find(key) == mCurrentSlicedDuration.end()) { + // we didn't see a start event before. do nothing. + return; + } + DurationInfo& duration = mCurrentSlicedDuration[key]; + + switch (duration.state) { + case DurationState::kStopped: + // already stopped, do nothing. + break; + case DurationState::kStarted: { + duration.state = DurationState::kStopped; + int64_t durationTime = eventTime - duration.lastStartTime; + VLOG("Metric %lld, key %s, Stop %lld %lld %lld", mMetric.metric_id(), key.c_str(), + (long long)duration.lastStartTime, (long long)eventTime, (long long)durationTime); + duration.lastDuration = + updateDuration(duration.lastDuration, durationTime, mMetric.type()); + VLOG(" record duration: %lld ", (long long)duration.lastDuration); + break; + } + case DurationState::kPaused: { + duration.state = DurationState::kStopped; + break; + } + } +} + +int64_t DurationMetricProducer::updateDuration(const int64_t lastDuration, + const int64_t durationTime, + const DurationMetric_AggregationType type) { + int64_t result = lastDuration; + switch (type) { + case DurationMetric_AggregationType_DURATION_SUM: + result += durationTime; + break; + case DurationMetric_AggregationType_DURATION_MAX_SPARSE: + if (lastDuration < durationTime) { + result = durationTime; + } + break; + case DurationMetric_AggregationType_DURATION_MIN_SPARSE: + if (lastDuration > durationTime) { + result = durationTime; + } + break; + } + return result; +} + +void DurationMetricProducer::noteStopAll(const uint64_t eventTime) { + for (auto& duration : mCurrentSlicedDuration) { + noteStop(duration.first, eventTime); + } +} + +// When a new matched event comes in, we check if event falls into the current +// bucket. If not, flush the old counter to past buckets and initialize the current buckt. +void DurationMetricProducer::flushDurationIfNeeded(const uint64_t eventTime) { + if (mCurrentBucketStartTimeNs + mBucketSizeNs > eventTime) { + return; + } + + // adjust the bucket start time + int numBucketsForward = (eventTime - mCurrentBucketStartTimeNs) / mBucketSizeNs; + + DurationBucketInfo info; + uint64_t endTime = mCurrentBucketStartTimeNs + mBucketSizeNs; + info.set_start_bucket_nanos(mCurrentBucketStartTimeNs); + info.set_end_bucket_nanos(endTime); + + uint64_t oldBucketStartTimeNs = mCurrentBucketStartTimeNs; + mCurrentBucketStartTimeNs += (numBucketsForward)*mBucketSizeNs; + VLOG("Metric %lld: new bucket start time: %lld", mMetric.metric_id(), + (long long)mCurrentBucketStartTimeNs); + + for (auto it = mCurrentSlicedDuration.begin(); it != mCurrentSlicedDuration.end(); ++it) { + int64_t finalDuration = it->second.lastDuration; + if (it->second.state == kStarted) { + // the event is still on-going, duration needs to be updated. + int64_t durationTime = endTime - it->second.lastStartTime; + finalDuration = updateDuration(it->second.lastDuration, durationTime, mMetric.type()); + } + + VLOG(" final duration for last bucket: %lld", (long long)finalDuration); + + // Don't record empty bucket. + if (finalDuration != 0) { + info.set_duration_nanos(finalDuration); + // it will auto create new vector of CountbucketInfo if the key is not found. + auto& bucketList = mPastBuckets[it->first]; + bucketList.push_back(info); + } + + // if the event is still on-going, add the buckets between previous bucket and now. Because + // the event has been going on across all the buckets in between. + // |prev_bucket|...|..|...|now_bucket| + if (it->second.state == kStarted) { + for (int i = 1; i < numBucketsForward; i++) { + DurationBucketInfo info; + info.set_start_bucket_nanos(oldBucketStartTimeNs + mBucketSizeNs * i); + info.set_end_bucket_nanos(endTime + mBucketSizeNs * i); + info.set_duration_nanos(mBucketSizeNs); + auto& bucketList = mPastBuckets[it->first]; + bucketList.push_back(info); + VLOG(" add filling bucket with duration %lld", (long long)mBucketSizeNs); + } + } + + if (it->second.state == DurationState::kStopped) { + // No need to keep buckets for events that were stopped before. If the event starts + // again, we will add it back. + mCurrentSlicedDuration.erase(it); + } else { + // for kPaused, and kStarted event, we will keep the buckets, and reset the start time + // and duration. + it->second.lastStartTime = mCurrentBucketStartTimeNs; + it->second.lastDuration = 0; + } + } +} + +} // namespace statsd +} // namespace os +} // namespace android diff --git a/cmds/statsd/src/metrics/DurationMetricProducer.h b/cmds/statsd/src/metrics/DurationMetricProducer.h new file mode 100644 index 000000000000..44c325417bd7 --- /dev/null +++ b/cmds/statsd/src/metrics/DurationMetricProducer.h @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DURATION_METRIC_PRODUCER_H +#define DURATION_METRIC_PRODUCER_H + +#include <unordered_map> + +#include "../condition/ConditionTracker.h" +#include "../matchers/matcher_util.h" +#include "MetricProducer.h" +#include "frameworks/base/cmds/statsd/src/stats_log.pb.h" +#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" +#include "stats_util.h" + +using namespace std; + +namespace android { +namespace os { +namespace statsd { + +enum DurationState { + kStopped = 0, // The event is stopped. + kStarted = 1, // The event is on going. + kPaused = 2, // The event is started, but condition is false, clock is paused. When condition + // turns to true, kPaused will become kStarted. +}; + +// Hold duration information for current on-going bucket. +struct DurationInfo { + DurationState state; + // most recent start time. + int64_t lastStartTime; + // existing duration in current bucket. Eventually, the duration will be aggregated in + // the way specified in AggregateType (Sum, Max, or Min). + int64_t lastDuration; + // cache the HashableDimensionKeys we need to query the condition for this duration event. + std::map<string, HashableDimensionKey> conditionKeys; + + DurationInfo() : state(kStopped), lastStartTime(0), lastDuration(0){}; +}; + +class DurationMetricProducer : public MetricProducer { +public: + DurationMetricProducer(const DurationMetric& durationMetric, const int conditionIndex, + const size_t startIndex, const size_t stopIndex, + const size_t stopAllIndex, const sp<ConditionWizard>& wizard); + + virtual ~DurationMetricProducer(); + + void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) override; + + void onConditionChanged(const bool conditionMet) override; + + void finish() override; + + StatsLogReport onDumpReport() override; + + void onSlicedConditionMayChange() override; + + // TODO: Implement this later. + virtual void notifyAppUpgrade(const string& apk, const int uid, const int version) override{}; + +private: + const DurationMetric mMetric; + + // Index of the SimpleLogEntryMatcher which defines the start. + const size_t mStartIndex; + + // Index of the SimpleLogEntryMatcher which defines the stop. + const size_t mStopIndex; + + // Index of the SimpleLogEntryMatcher which defines the stop all for all dimensions. + const size_t mStopAllIndex; + + // Save the past buckets and we can clear when the StatsLogReport is dumped. + std::unordered_map<HashableDimensionKey, std::vector<DurationBucketInfo>> mPastBuckets; + + // The current bucket. + std::unordered_map<HashableDimensionKey, DurationInfo> mCurrentSlicedDuration; + + void flushDurationIfNeeded(const uint64_t newEventTime); + + void noteStart(const HashableDimensionKey& key, const bool conditionMet, + const uint64_t eventTime); + + void noteStop(const HashableDimensionKey& key, const uint64_t eventTime); + + void noteStopAll(const uint64_t eventTime); + + static int64_t updateDuration(const int64_t lastDuration, const int64_t durationTime, + const DurationMetric_AggregationType type); + + void noteConditionChanged(const HashableDimensionKey& key, const bool conditionMet, + const uint64_t eventTime); +}; + +} // namespace statsd +} // namespace os +} // namespace android + +#endif // DURATION_METRIC_PRODUCER_H diff --git a/cmds/statsd/src/metrics/MetricProducer.h b/cmds/statsd/src/metrics/MetricProducer.h index 589df84476c3..afaab648c09f 100644 --- a/cmds/statsd/src/metrics/MetricProducer.h +++ b/cmds/statsd/src/metrics/MetricProducer.h @@ -17,11 +17,13 @@ #ifndef METRIC_PRODUCER_H #define METRIC_PRODUCER_H +#include "condition/ConditionWizard.h" #include "matchers/matcher_util.h" #include "packages/PackageInfoListener.h" #include <log/logprint.h> #include <utils/RefBase.h> +#include "frameworks/base/cmds/statsd/src/stats_log.pb.h" namespace android { namespace os { @@ -33,18 +35,57 @@ namespace statsd { // be a no-op. class MetricProducer : public virtual PackageInfoListener { public: + MetricProducer(const int64_t startTimeNs, const int conditionIndex, + const sp<ConditionWizard>& wizard) + : mStartTimeNs(startTimeNs), + mCurrentBucketStartTimeNs(startTimeNs), + mCondition(conditionIndex >= 0 ? false : true), + mWizard(wizard), + mConditionTrackerIndex(conditionIndex) { + // reuse the same map for non-sliced metrics too. this way, we avoid too many if-else. + mDimensionKeyMap[DEFAULT_DIMENSION_KEY] = std::vector<KeyValuePair>(); + }; virtual ~MetricProducer(){}; // Consume the parsed stats log entry that already matched the "what" of the metric. - virtual void onMatchedLogEvent(const LogEvent& event) = 0; + virtual void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) = 0; virtual void onConditionChanged(const bool condition) = 0; + virtual void onSlicedConditionMayChange() = 0; + // This is called when the metric collecting is done, e.g., when there is a new configuration // coming. MetricProducer should do the clean up, and dump existing data to dropbox. virtual void finish() = 0; - virtual void onDumpReport() = 0; + virtual StatsLogReport onDumpReport() = 0; + + virtual bool isConditionSliced() const { + return mConditionSliced; + }; + +protected: + const uint64_t mStartTimeNs; + + uint64_t mCurrentBucketStartTimeNs; + + int64_t mBucketSizeNs; + + bool mCondition; + + bool mConditionSliced; + + sp<ConditionWizard> mWizard; + + int mConditionTrackerIndex; + + std::vector<KeyMatcher> mDimension; // The dimension defined in statsd_config + + // Keep the map from the internal HashableDimensionKey to std::vector<KeyValuePair> + // that StatsLogReport wants. + std::unordered_map<HashableDimensionKey, std::vector<KeyValuePair>> mDimensionKeyMap; + + std::vector<EventConditionLink> mConditionLinks; }; } // namespace statsd diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp index 5b4ca806e494..c19d46298de5 100644 --- a/cmds/statsd/src/metrics/MetricsManager.cpp +++ b/cmds/statsd/src/metrics/MetricsManager.cpp @@ -15,8 +15,6 @@ */ #define DEBUG true // STOPSHIP if true #include "Log.h" -#define VLOG(...) \ - if (DEBUG) ALOGD(__VA_ARGS__); #include "MetricsManager.h" #include <log/logprint.h> @@ -58,6 +56,17 @@ void MetricsManager::finish() { } } +vector<StatsLogReport> MetricsManager::onDumpReport() { + VLOG("=========================Metric Reports Start=========================="); + // one StatsLogReport per MetricProduer + vector<StatsLogReport> reportList; + for (auto& metric : mAllMetricProducers) { + reportList.push_back(metric->onDumpReport()); + } + VLOG("=========================Metric Reports End=========================="); + return reportList; +} + // Consume the stats log if it's interesting to this metric. void MetricsManager::onLogEvent(const LogEvent& event) { if (!mConfigValid) { @@ -71,6 +80,7 @@ void MetricsManager::onLogEvent(const LogEvent& event) { } // Since at least one of the metrics is interested in this event, we parse it now. + ALOGD("%s", event.ToString().c_str()); vector<MatchingState> matcherCache(mAllLogEntryMatchers.size(), MatchingState::kNotComputed); for (auto& matcher : mAllLogEntryMatchers) { @@ -93,20 +103,34 @@ void MetricsManager::onLogEvent(const LogEvent& event) { ConditionState::kNotEvaluated); // A bitmap to track if a condition has changed value. vector<bool> changedCache(mAllConditionTrackers.size(), false); + vector<bool> slicedChangedCache(mAllConditionTrackers.size(), false); for (size_t i = 0; i < mAllConditionTrackers.size(); i++) { if (conditionToBeEvaluated[i] == false) { continue; } - sp<ConditionTracker>& condition = mAllConditionTrackers[i]; condition->evaluateCondition(event, matcherCache, mAllConditionTrackers, conditionCache, - changedCache); - if (changedCache[i]) { - auto pair = mConditionToMetricMap.find(i); - if (pair != mConditionToMetricMap.end()) { - auto& metricList = pair->second; - for (auto metricIndex : metricList) { + changedCache, slicedChangedCache); + } + + for (size_t i = 0; i < mAllConditionTrackers.size(); i++) { + if (changedCache[i] == false && slicedChangedCache[i] == false) { + continue; + } + auto pair = mConditionToMetricMap.find(i); + if (pair != mConditionToMetricMap.end()) { + auto& metricList = pair->second; + for (auto metricIndex : metricList) { + // metric cares about non sliced condition, and it's changed. + // Push the new condition to it directly. + if (!mAllMetricProducers[metricIndex]->isConditionSliced() && changedCache[i]) { mAllMetricProducers[metricIndex]->onConditionChanged(conditionCache[i]); + // metric cares about sliced conditions, and it may have changed. Send + // notification, and the metric can query the sliced conditions that are + // interesting to it. + } else if (mAllMetricProducers[metricIndex]->isConditionSliced() && + slicedChangedCache[i]) { + mAllMetricProducers[metricIndex]->onSlicedConditionMayChange(); } } } @@ -119,7 +143,7 @@ void MetricsManager::onLogEvent(const LogEvent& event) { if (pair != mTrackerToMetricMap.end()) { auto& metricList = pair->second; for (const int metricIndex : metricList) { - mAllMetricProducers[metricIndex]->onMatchedLogEvent(event); + mAllMetricProducers[metricIndex]->onMatchedLogEvent(i, event); } } } diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h index 7aca0b5a6fef..56f57d3d3654 100644 --- a/cmds/statsd/src/metrics/MetricsManager.h +++ b/cmds/statsd/src/metrics/MetricsManager.h @@ -43,17 +43,19 @@ public: // Called when everything should wrap up. We are about to finish (e.g., new config comes). void finish(); + // Config source owner can call onDumpReport() to get all the metrics collected. + std::vector<StatsLogReport> onDumpReport(); + private: // All event tags that are interesting to my metrics. std::set<int> mTagIds; // We only store the sp of LogMatchingTracker, MetricProducer, and ConditionTracker in // MetricManager. There are relationship between them, and the relationship are denoted by index - // instead of poiters. The reasons for this are: (1) the relationship between them are + // instead of pointers. The reasons for this are: (1) the relationship between them are // complicated, store index instead of pointers reduce the risk of A holds B's sp, and B holds // A's sp. (2) When we evaluate matcher results, or condition results, we can quickly get the // related results from a cache using the index. - // TODO: using unique_ptr may be more appriopreate? // Hold all the log entry matchers from the config. std::vector<sp<LogMatchingTracker>> mAllLogEntryMatchers; diff --git a/cmds/statsd/src/metrics/metrics_manager_util.cpp b/cmds/statsd/src/metrics/metrics_manager_util.cpp index 6fdd228f4910..23071aa0899c 100644 --- a/cmds/statsd/src/metrics/metrics_manager_util.cpp +++ b/cmds/statsd/src/metrics/metrics_manager_util.cpp @@ -19,6 +19,7 @@ #include "../matchers/CombinationLogMatchingTracker.h" #include "../matchers/SimpleLogMatchingTracker.h" #include "CountMetricProducer.h" +#include "DurationMetricProducer.h" #include "stats_util.h" using std::set; @@ -30,11 +31,23 @@ namespace android { namespace os { namespace statsd { +int getTrackerIndex(const string& name, const unordered_map<string, int>& logTrackerMap) { + auto logTrackerIt = logTrackerMap.find(name); + if (logTrackerIt == logTrackerMap.end()) { + ALOGW("cannot find the LogEventMatcher %s in config", name.c_str()); + return MATCHER_NOT_FOUND; + } + return logTrackerIt->second; +} + bool initLogTrackers(const StatsdConfig& config, unordered_map<string, int>& logTrackerMap, vector<sp<LogMatchingTracker>>& allLogEntryMatchers, set<int>& allTagIds) { vector<LogEntryMatcher> matcherConfigs; + const int logEntryMatcherCount = config.log_entry_matcher_size(); + matcherConfigs.reserve(logEntryMatcherCount); + allLogEntryMatchers.reserve(logEntryMatcherCount); - for (int i = 0; i < config.log_entry_matcher_size(); i++) { + for (int i = 0; i < logEntryMatcherCount; i++) { const LogEntryMatcher& logMatcher = config.log_entry_matcher(i); int index = allLogEntryMatchers.size(); @@ -77,8 +90,11 @@ bool initConditions(const StatsdConfig& config, const unordered_map<string, int> vector<sp<ConditionTracker>>& allConditionTrackers, unordered_map<int, std::vector<int>>& trackerToConditionMap) { vector<Condition> conditionConfigs; + const int conditionTrackerCount = config.condition_size(); + conditionConfigs.reserve(conditionTrackerCount); + allConditionTrackers.reserve(conditionTrackerCount); - for (int i = 0; i < config.condition_size(); i++) { + for (int i = 0; i < conditionTrackerCount; i++) { const Condition& condition = config.condition(i); int index = allConditionTrackers.size(); switch (condition.contents_case()) { @@ -121,9 +137,14 @@ bool initConditions(const StatsdConfig& config, const unordered_map<string, int> bool initMetrics(const StatsdConfig& config, const unordered_map<string, int>& logTrackerMap, const unordered_map<string, int>& conditionTrackerMap, + vector<sp<ConditionTracker>>& allConditionTrackers, vector<sp<MetricProducer>>& allMetricProducers, unordered_map<int, std::vector<int>>& conditionToMetricMap, unordered_map<int, std::vector<int>>& trackerToMetricMap) { + sp<ConditionWizard> wizard = new ConditionWizard(allConditionTrackers); + const int allMetricsCount = config.count_metric_size() + config.duration_metric_size(); + allMetricProducers.reserve(allMetricsCount); + // Build MetricProducers for each metric defined in config. // (1) build CountMetricProducer for (int i = 0; i < config.count_metric_size(); i++) { @@ -133,36 +154,109 @@ bool initMetrics(const StatsdConfig& config, const unordered_map<string, int>& l return false; } + int metricIndex = allMetricProducers.size(); + auto logTrackerIt = logTrackerMap.find(metric.what()); if (logTrackerIt == logTrackerMap.end()) { ALOGW("cannot find the LogEntryMatcher %s in config", metric.what().c_str()); return false; } + int logTrackerIndex = logTrackerIt->second; + auto& metric_list = trackerToMetricMap[logTrackerIndex]; + metric_list.push_back(metricIndex); sp<MetricProducer> countProducer; - int metricIndex = allMetricProducers.size(); + if (metric.has_condition()) { auto condition_it = conditionTrackerMap.find(metric.condition()); if (condition_it == conditionTrackerMap.end()) { ALOGW("cannot find the Condition %s in the config", metric.condition().c_str()); return false; } - countProducer = new CountMetricProducer(metric, true /*has condition*/); + + for (const auto& link : metric.links()) { + auto it = conditionTrackerMap.find(link.condition()); + if (it == conditionTrackerMap.end()) { + ALOGW("cannot find the Condition %s in the config", link.condition().c_str()); + return false; + } + allConditionTrackers[condition_it->second]->setSliced(true); + allConditionTrackers[it->second]->setSliced(true); + allConditionTrackers[it->second]->addDimensions(vector<KeyMatcher>( + link.key_in_condition().begin(), link.key_in_condition().end())); + } + + countProducer = new CountMetricProducer(metric, condition_it->second, wizard); // will create new vector if not exist before. auto& metricList = conditionToMetricMap[condition_it->second]; metricList.push_back(metricIndex); } else { - countProducer = new CountMetricProducer(metric, false /*no condition*/); + countProducer = new CountMetricProducer(metric, -1 /*no condition*/, wizard); } - - int logTrackerIndex = logTrackerIt->second; - auto& metric_list = trackerToMetricMap[logTrackerIndex]; - metric_list.push_back(metricIndex); allMetricProducers.push_back(countProducer); } - // TODO: build other types of metrics too. + for (int i = 0; i < config.duration_metric_size(); i++) { + int metricIndex = allMetricProducers.size(); + const DurationMetric metric = config.duration_metric(i); + if (!metric.has_start()) { + ALOGW("cannot find start in DurationMetric %lld", metric.metric_id()); + return false; + } + + int trackerIndices[] = {-1, -1, -1}; + trackerIndices[0] = getTrackerIndex(metric.start(), logTrackerMap); + if (metric.has_stop()) { + trackerIndices[1] = getTrackerIndex(metric.stop(), logTrackerMap); + } + + if (metric.has_stop_all()) { + trackerIndices[2] = getTrackerIndex(metric.stop_all(), logTrackerMap); + } + + for (const int& index : trackerIndices) { + if (index == MATCHER_NOT_FOUND) { + return false; + } + if (index >= 0) { + auto& metric_list = trackerToMetricMap[index]; + metric_list.push_back(metricIndex); + } + } + + int conditionIndex = -1; + + if (metric.has_predicate()) { + auto condition_it = conditionTrackerMap.find(metric.predicate()); + if (condition_it == conditionTrackerMap.end()) { + ALOGW("cannot find the Condition %s in the config", metric.predicate().c_str()); + return false; + } + conditionIndex = condition_it->second; + + for (const auto& link : metric.links()) { + auto it = conditionTrackerMap.find(link.condition()); + if (it == conditionTrackerMap.end()) { + ALOGW("cannot find the Condition %s in the config", link.condition().c_str()); + return false; + } + allConditionTrackers[condition_it->second]->setSliced(true); + allConditionTrackers[it->second]->setSliced(true); + allConditionTrackers[it->second]->addDimensions(vector<KeyMatcher>( + link.key_in_condition().begin(), link.key_in_condition().end())); + } + + auto& metricList = conditionToMetricMap[conditionIndex]; + metricList.push_back(metricIndex); + } + + sp<MetricProducer> durationMetric = + new DurationMetricProducer(metric, conditionIndex, trackerIndices[0], + trackerIndices[1], trackerIndices[2], wizard); + + allMetricProducers.push_back(durationMetric); + } return true; } @@ -187,8 +281,8 @@ bool initStatsdConfig(const StatsdConfig& config, set<int>& allTagIds, return false; } - if (!initMetrics(config, logTrackerMap, conditionTrackerMap, allMetricProducers, - conditionToMetricMap, trackerToMetricMap)) { + if (!initMetrics(config, logTrackerMap, conditionTrackerMap, allConditionTrackers, + allMetricProducers, conditionToMetricMap, trackerToMetricMap)) { ALOGE("initMetricProducers failed"); return false; } diff --git a/cmds/statsd/src/metrics/metrics_manager_util.h b/cmds/statsd/src/metrics/metrics_manager_util.h index 5f1f295d450a..38149a6aecd6 100644 --- a/cmds/statsd/src/metrics/metrics_manager_util.h +++ b/cmds/statsd/src/metrics/metrics_manager_util.h @@ -59,7 +59,8 @@ bool initConditions(const StatsdConfig& config, const std::unordered_map<std::string, int>& logTrackerMap, std::unordered_map<std::string, int>& conditionTrackerMap, std::vector<sp<ConditionTracker>>& allConditionTrackers, - std::unordered_map<int, std::vector<int>>& trackerToConditionMap); + std::unordered_map<int, std::vector<int>>& trackerToConditionMap, + std::unordered_map<int, std::vector<EventConditionLink>>& eventConditionLinks); // Initialize MetricProducers. // input: @@ -71,12 +72,14 @@ bool initConditions(const StatsdConfig& config, // [conditionToMetricMap]: contains the mapping from condition tracker index to // the list of MetricProducer index // [trackerToMetricMap]: contains the mapping from log tracker to MetricProducer index. -bool initMetrics(const StatsdConfig& config, - const std::unordered_map<std::string, int>& logTrackerMap, - const std::unordered_map<std::string, int>& conditionTrackerMap, - std::vector<sp<MetricProducer>>& allMetricProducers, - std::unordered_map<int, std::vector<int>>& conditionToMetricMap, - std::unordered_map<int, std::vector<int>>& trackerToMetricMap); +bool initMetrics( + const StatsdConfig& config, const std::unordered_map<std::string, int>& logTrackerMap, + const std::unordered_map<std::string, int>& conditionTrackerMap, + const std::unordered_map<int, std::vector<EventConditionLink>>& eventConditionLinks, + vector<sp<ConditionTracker>>& allConditionTrackers, + std::vector<sp<MetricProducer>>& allMetricProducers, + std::unordered_map<int, std::vector<int>>& conditionToMetricMap, + std::unordered_map<int, std::vector<int>>& trackerToMetricMap); // Initialize MetricManager from StatsdConfig. // Parameters are the members of MetricsManager. See MetricsManager for declaration. @@ -88,6 +91,8 @@ bool initStatsdConfig(const StatsdConfig& config, std::set<int>& allTagIds, std::unordered_map<int, std::vector<int>>& trackerToMetricMap, std::unordered_map<int, std::vector<int>>& trackerToConditionMap); +int getTrackerIndex(const std::string& name, const std::unordered_map<string, int>& logTrackerMap); + } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/stats_log.proto b/cmds/statsd/src/stats_log.proto index 4ca06fa7a2fe..29cd94b21b1b 100644 --- a/cmds/statsd/src/stats_log.proto +++ b/cmds/statsd/src/stats_log.proto @@ -106,7 +106,7 @@ message StatsLogReport { repeated CountMetricData data = 1; } message DurationMetricDataWrapper { - repeated CountMetricData data = 1; + repeated DurationMetricData data = 1; } oneof data { EventMetricDataWrapper event_metrics = 4; diff --git a/cmds/statsd/src/stats_util.cpp b/cmds/statsd/src/stats_util.cpp index 5157adf74607..fcce2ff3b36e 100644 --- a/cmds/statsd/src/stats_util.cpp +++ b/cmds/statsd/src/stats_util.cpp @@ -14,8 +14,8 @@ * limitations under the License. */ -#include <log/log_event_list.h> #include "stats_util.h" +#include <log/log_event_list.h> namespace android { namespace os { @@ -128,6 +128,35 @@ EventMetricData parse(log_msg msg) { return eventMetricData; } +// There is no existing hash function for the dimension key ("repeated KeyValuePair"). +// Temporarily use a string concatenation as the hashable key. +// TODO: Find a better hash function for std::vector<KeyValuePair>. +HashableDimensionKey getHashableKey(std::vector<KeyValuePair> keys) { + std::string flattened; + for (const KeyValuePair& pair : keys) { + flattened += std::to_string(pair.key()); + flattened += ":"; + switch (pair.value_case()) { + case KeyValuePair::ValueCase::kValueStr: + flattened += pair.value_str(); + break; + case KeyValuePair::ValueCase::kValueInt: + flattened += std::to_string(pair.value_int()); + break; + case KeyValuePair::ValueCase::kValueBool: + flattened += std::to_string(pair.value_bool()); + break; + case KeyValuePair::ValueCase::kValueFloat: + flattened += std::to_string(pair.value_float()); + break; + default: + break; + } + flattened += "|"; + } + return flattened; +} + } // namespace statsd } // namespace os } // namespace android diff --git a/cmds/statsd/src/stats_util.h b/cmds/statsd/src/stats_util.h index 38174bfae080..575588b3ba72 100644 --- a/cmds/statsd/src/stats_util.h +++ b/cmds/statsd/src/stats_util.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef PARSE_UTIL_H -#define PARSE_UTIL_H +#ifndef STATS_UTIL_H +#define STATS_UTIL_H #include "logd/LogReader.h" #include "storage/DropboxWriter.h" @@ -26,12 +26,19 @@ namespace android { namespace os { namespace statsd { +#define DEFAULT_DIMENSION_KEY "" +#define MATCHER_NOT_FOUND -2 +#define NANO_SECONDS_IN_A_SECOND (1000 * 1000 * 1000) + +typedef std::string HashableDimensionKey; + EventMetricData parse(log_msg msg); int getTagId(log_msg msg); +std::string getHashableKey(std::vector<KeyValuePair> key); } // namespace statsd } // namespace os } // namespace android -#endif // PARSE_UTIL_H +#endif // STATS_UTIL_H diff --git a/cmds/statsd/src/statsd_config.proto b/cmds/statsd/src/statsd_config.proto index d7702cdd42ad..afb3f2b0db25 100644 --- a/cmds/statsd/src/statsd_config.proto +++ b/cmds/statsd/src/statsd_config.proto @@ -55,7 +55,7 @@ enum LogicalOperation { } message SimpleLogEntryMatcher { - repeated int32 tag = 1; + optional int32 tag = 1; repeated KeyValueMatcher key_value_matcher = 2; } @@ -103,12 +103,28 @@ message Bucket { optional int64 bucket_size_millis = 1; } +message Alert { + message IncidentdDetails { + optional string alert_name = 1; + repeated int32 incidentd_sections = 2; + } + optional IncidentdDetails incidentd_details = 1; + + optional int32 number_of_buckets = 3; + + optional int32 refractory_period_secs = 4; + + optional int64 trigger_if_gt = 5; +} + message EventMetric { optional int64 metric_id = 1; optional string what = 2; optional string condition = 3; + + repeated EventConditionLink links = 4; } message CountMetric { @@ -121,26 +137,73 @@ message CountMetric { repeated KeyMatcher dimension = 4; optional Bucket bucket = 5; + + repeated Alert alerts = 6; + + optional bool include_in_output = 7; + + repeated EventConditionLink links = 8; } message DurationMetric { optional int64 metric_id = 1; + optional string start = 2; + + optional string stop = 3; + + optional string stop_all = 4; + enum AggregationType { DURATION_SUM = 1; DURATION_MAX_SPARSE = 2; DURATION_MIN_SPARSE = 3; } - optional AggregationType type = 2; + optional AggregationType type = 5; - optional string predicate = 3; + optional string predicate = 6; - repeated KeyMatcher dimension = 4; + repeated KeyMatcher dimension = 7; - optional Bucket bucket = 5; + optional Bucket bucket = 8; + + repeated EventConditionLink links = 9; +} + +message ValueMetric { + optional int64 metric_id = 1; + + optional string what = 2; + + optional int32 value_field = 3; + + optional string condition = 4; + + repeated KeyMatcher dimension = 5; + + optional Bucket bucket = 6; + + enum Operation { + SUM_DIFF = 1; + MIN_DIFF = 2; + MAX_DIFF = 3; + SUM = 4; + MIN = 5; + MAX = 6; + FIRST = 7; + LAST = 8; + } + optional Operation operation = 7; } +message EventConditionLink { + optional string condition = 1; + + repeated KeyMatcher key_in_main = 2; + repeated KeyMatcher key_in_condition = 3; +}; + message StatsdConfig { optional int64 config_id = 1; @@ -148,7 +211,11 @@ message StatsdConfig { repeated CountMetric count_metric = 3; - repeated LogEntryMatcher log_entry_matcher = 4; + repeated ValueMetric value_metric = 4; + + repeated DurationMetric duration_metric = 5; + + repeated LogEntryMatcher log_entry_matcher = 6; - repeated Condition condition = 5; + repeated Condition condition = 7; } diff --git a/cmds/statsd/tests/LogEntryMatcher_test.cpp b/cmds/statsd/tests/LogEntryMatcher_test.cpp index 9d8ba92c563b..19403c0d0e87 100644 --- a/cmds/statsd/tests/LogEntryMatcher_test.cpp +++ b/cmds/statsd/tests/LogEntryMatcher_test.cpp @@ -40,7 +40,7 @@ TEST(LogEntryMatcherTest, TestSimpleMatcher) { // Set up the matcher LogEntryMatcher matcher; auto simpleMatcher = matcher.mutable_simple_log_entry_matcher(); - simpleMatcher->add_tag(TAG_ID); + simpleMatcher->set_tag(TAG_ID); // Set up the event android_log_event_list list(TAG_ID); @@ -57,7 +57,7 @@ TEST(LogEntryMatcherTest, TestBoolMatcher) { // Set up the matcher LogEntryMatcher matcher; auto simpleMatcher = matcher.mutable_simple_log_entry_matcher(); - simpleMatcher->add_tag(TAG_ID); + simpleMatcher->set_tag(TAG_ID); auto keyValue1 = simpleMatcher->add_key_value_matcher(); keyValue1->mutable_key_matcher()->set_key(FIELD_ID_1); auto keyValue2 = simpleMatcher->add_key_value_matcher(); @@ -94,7 +94,7 @@ TEST(LogEntryMatcherTest, TestStringMatcher) { // Set up the matcher LogEntryMatcher matcher; auto simpleMatcher = matcher.mutable_simple_log_entry_matcher(); - simpleMatcher->add_tag(TAG_ID); + simpleMatcher->set_tag(TAG_ID); auto keyValue = simpleMatcher->add_key_value_matcher(); keyValue->mutable_key_matcher()->set_key(FIELD_ID_1); keyValue->set_eq_string("some value"); @@ -115,7 +115,8 @@ TEST(LogEntryMatcherTest, TestIntComparisonMatcher) { // Set up the matcher LogEntryMatcher matcher; auto simpleMatcher = matcher.mutable_simple_log_entry_matcher(); - simpleMatcher->add_tag(TAG_ID); + + simpleMatcher->set_tag(TAG_ID); auto keyValue = simpleMatcher->add_key_value_matcher(); keyValue->mutable_key_matcher()->set_key(FIELD_ID_1); @@ -176,7 +177,8 @@ TEST(LogEntryMatcherTest, TestFloatComparisonMatcher) { // Set up the matcher LogEntryMatcher matcher; auto simpleMatcher = matcher.mutable_simple_log_entry_matcher(); - simpleMatcher->add_tag(TAG_ID); + simpleMatcher->set_tag(TAG_ID); + auto keyValue = simpleMatcher->add_key_value_matcher(); keyValue->mutable_key_matcher()->set_key(FIELD_ID_1); @@ -199,7 +201,7 @@ TEST(LogEntryMatcherTest, TestFloatComparisonMatcher) { // Helper for the composite matchers. void addSimpleMatcher(SimpleLogEntryMatcher* simpleMatcher, int tag, int key, int val) { - simpleMatcher->add_tag(tag); + simpleMatcher->set_tag(tag); auto keyValue = simpleMatcher->add_key_value_matcher(); keyValue->mutable_key_matcher()->set_key(key); keyValue->set_eq_int(val); diff --git a/cmds/statsd/tests/MetricsManager_test.cpp b/cmds/statsd/tests/MetricsManager_test.cpp index 32661dcc4752..b000e1388632 100644 --- a/cmds/statsd/tests/MetricsManager_test.cpp +++ b/cmds/statsd/tests/MetricsManager_test.cpp @@ -45,7 +45,7 @@ StatsdConfig buildGoodConfig() { eventMatcher->set_name("SCREEN_IS_ON"); SimpleLogEntryMatcher* simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); - simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/); + simpleLogEntryMatcher->set_tag(2 /*SCREEN_STATE_CHANGE*/); simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()->set_key( 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); simpleLogEntryMatcher->mutable_key_value_matcher(0)->set_eq_int( @@ -55,7 +55,7 @@ StatsdConfig buildGoodConfig() { eventMatcher->set_name("SCREEN_IS_OFF"); simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); - simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/); + simpleLogEntryMatcher->set_tag(2 /*SCREEN_STATE_CHANGE*/); simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()->set_key( 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); simpleLogEntryMatcher->mutable_key_value_matcher(0)->set_eq_int( @@ -80,7 +80,7 @@ StatsdConfig buildCircleMatchers() { eventMatcher->set_name("SCREEN_IS_ON"); SimpleLogEntryMatcher* simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); - simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/); + simpleLogEntryMatcher->set_tag(2 /*SCREEN_STATE_CHANGE*/); simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()->set_key( 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); simpleLogEntryMatcher->mutable_key_value_matcher(0)->set_eq_int( @@ -106,7 +106,7 @@ StatsdConfig buildMissingMatchers() { eventMatcher->set_name("SCREEN_IS_ON"); SimpleLogEntryMatcher* simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); - simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/); + simpleLogEntryMatcher->set_tag(2 /*SCREEN_STATE_CHANGE*/); simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()->set_key( 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); simpleLogEntryMatcher->mutable_key_value_matcher(0)->set_eq_int( @@ -132,7 +132,7 @@ StatsdConfig buildCircleConditions() { eventMatcher->set_name("SCREEN_IS_ON"); SimpleLogEntryMatcher* simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); - simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/); + simpleLogEntryMatcher->set_tag(2 /*SCREEN_STATE_CHANGE*/); simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()->set_key( 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); simpleLogEntryMatcher->mutable_key_value_matcher(0)->set_eq_int( @@ -142,7 +142,7 @@ StatsdConfig buildCircleConditions() { eventMatcher->set_name("SCREEN_IS_OFF"); simpleLogEntryMatcher = eventMatcher->mutable_simple_log_entry_matcher(); - simpleLogEntryMatcher->add_tag(2 /*SCREEN_STATE_CHANGE*/); + simpleLogEntryMatcher->set_tag(2 /*SCREEN_STATE_CHANGE*/); simpleLogEntryMatcher->add_key_value_matcher()->mutable_key_matcher()->set_key( 1 /*SCREEN_STATE_CHANGE__DISPLAY_STATE*/); simpleLogEntryMatcher->mutable_key_value_matcher(0)->set_eq_int( |