/* * Copyright (C) 2021 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.car.telemetry.publisher; import static com.android.car.telemetry.AtomsProto.Atom.ACTIVITY_FOREGROUND_STATE_CHANGED_FIELD_NUMBER; import static com.android.car.telemetry.AtomsProto.Atom.ANR_OCCURRED_FIELD_NUMBER; import static com.android.car.telemetry.AtomsProto.Atom.APP_CRASH_OCCURRED_FIELD_NUMBER; import static com.android.car.telemetry.AtomsProto.Atom.APP_START_MEMORY_STATE_CAPTURED_FIELD_NUMBER; import static com.android.car.telemetry.AtomsProto.Atom.PROCESS_CPU_TIME_FIELD_NUMBER; import static com.android.car.telemetry.AtomsProto.Atom.PROCESS_MEMORY_STATE_FIELD_NUMBER; import static com.android.car.telemetry.AtomsProto.Atom.WTF_OCCURRED_FIELD_NUMBER; import static java.nio.charset.StandardCharsets.UTF_16; import android.app.StatsManager.StatsUnavailableException; import android.os.Handler; import android.os.PersistableBundle; import android.os.Process; import android.util.AtomicFile; import android.util.LongSparseArray; import com.android.car.CarLog; import com.android.car.telemetry.AtomsProto.ProcessCpuTime; import com.android.car.telemetry.AtomsProto.ProcessMemoryState; import com.android.car.telemetry.StatsLogProto; import com.android.car.telemetry.StatsdConfigProto; import com.android.car.telemetry.StatsdConfigProto.StatsdConfig; import com.android.car.telemetry.TelemetryProto; import com.android.car.telemetry.TelemetryProto.Publisher.PublisherCase; import com.android.car.telemetry.databroker.DataSubscriber; import com.android.car.telemetry.publisher.statsconverters.ConfigMetricsReportListConverter; import com.android.car.telemetry.publisher.statsconverters.StatsConversionException; import com.android.internal.annotations.VisibleForTesting; import com.android.internal.util.Preconditions; import com.android.server.utils.Slogf; import com.google.protobuf.InvalidProtocolBufferException; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; /** * Publisher for {@link TelemetryProto.StatsPublisher}. * *
The publisher adds subscriber configurations in StatsD and they persist between reboots and
* CarTelemetryService restarts. Please use {@link #removeAllDataSubscribers} to clean-up these
* configs from StatsD store.
*/
public class StatsPublisher extends AbstractPublisher {
// These IDs are used in StatsdConfig and ConfigMetricsReport.
@VisibleForTesting
static final long APP_START_MEMORY_STATE_CAPTURED_ATOM_MATCHER_ID = 1;
@VisibleForTesting
static final long APP_START_MEMORY_STATE_CAPTURED_EVENT_METRIC_ID = 2;
@VisibleForTesting
static final long PROCESS_MEMORY_STATE_MATCHER_ID = 3;
@VisibleForTesting
static final long PROCESS_MEMORY_STATE_GAUGE_METRIC_ID = 4;
@VisibleForTesting
static final long ACTIVITY_FOREGROUND_STATE_CHANGED_ATOM_MATCHER_ID = 5;
@VisibleForTesting
static final long ACTIVITY_FOREGROUND_STATE_CHANGED_EVENT_METRIC_ID = 6;
@VisibleForTesting
static final long PROCESS_CPU_TIME_MATCHER_ID = 7;
@VisibleForTesting
static final long PROCESS_CPU_TIME_GAUGE_METRIC_ID = 8;
@VisibleForTesting
static final long APP_CRASH_OCCURRED_ATOM_MATCHER_ID = 9;
@VisibleForTesting
static final long APP_CRASH_OCCURRED_EVENT_METRIC_ID = 10;
@VisibleForTesting
static final long ANR_OCCURRED_ATOM_MATCHER_ID = 11;
@VisibleForTesting
static final long ANR_OCCURRED_EVENT_METRIC_ID = 12;
@VisibleForTesting
static final long WTF_OCCURRED_ATOM_MATCHER_ID = 13;
@VisibleForTesting
static final long WTF_OCCURRED_EVENT_METRIC_ID = 14;
// The file that contains stats config key and stats config version
@VisibleForTesting
static final String SAVED_STATS_CONFIGS_FILE = "stats_config_keys_versions";
// TODO(b/202115033): Flatten the load spike by pulling reports for each MetricsConfigs
// using separate periodical timers.
private static final Duration PULL_REPORTS_PERIOD = Duration.ofMinutes(10);
private static final String BUNDLE_CONFIG_KEY_PREFIX = "statsd-publisher-config-id-";
private static final String BUNDLE_CONFIG_VERSION_PREFIX = "statsd-publisher-config-version-";
/**
* Binder transaction size limit is 1MB for all binders per process, so for large script input
* file pipe will be used to transfer the data to script executor instead of binder call. This
* is the input size threshold above which piping is used.
*/
private static final int SCRIPT_INPUT_SIZE_THRESHOLD_BYTES = 20 * 1024; // 20 kb
@VisibleForTesting
static final StatsdConfigProto.FieldMatcher PROCESS_MEMORY_STATE_FIELDS_MATCHER =
StatsdConfigProto.FieldMatcher.newBuilder()
.setField(
PROCESS_MEMORY_STATE_FIELD_NUMBER)
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessMemoryState.OOM_ADJ_SCORE_FIELD_NUMBER))
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessMemoryState.PAGE_FAULT_FIELD_NUMBER))
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessMemoryState.PAGE_MAJOR_FAULT_FIELD_NUMBER))
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessMemoryState.RSS_IN_BYTES_FIELD_NUMBER))
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessMemoryState.CACHE_IN_BYTES_FIELD_NUMBER))
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessMemoryState.SWAP_IN_BYTES_FIELD_NUMBER))
.build();
@VisibleForTesting
static final StatsdConfigProto.FieldMatcher PROCESS_CPU_TIME_FIELDS_MATCHER =
StatsdConfigProto.FieldMatcher.newBuilder()
.setField(PROCESS_CPU_TIME_FIELD_NUMBER)
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessCpuTime.USER_TIME_MILLIS_FIELD_NUMBER))
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessCpuTime.SYSTEM_TIME_MILLIS_FIELD_NUMBER))
.build();
private final StatsManagerProxy mStatsManager;
private final AtomicFile mSavedStatsConfigsFile;
private final Handler mTelemetryHandler;
// True if the publisher is periodically pulling reports from StatsD.
private boolean mIsPullingReports = false;
/** Assign the method to {@link Runnable}, otherwise the handler fails to remove it. */
private final Runnable mPullReportsPeriodically = this::pullReportsPeriodically;
// LongSparseArray is memory optimized, but they can be bit slower for more
// than 100 items. We're expecting much less number of subscribers, so these data structures
// are ok.
// Maps config_key to the set of DataSubscriber.
private final LongSparseArray StatsD uses ConfigKey struct to uniquely identify StatsdConfigs. StatsD ConfigKey consists
* of two parts: client uid and config_key number. The StatsdConfig is added to StatsD from
* CarService - which has uid=1000. Currently there is no client under uid=1000 and there will
* not be config_key collision.
*/
private static long buildConfigKey(DataSubscriber subscriber) {
// Not to be confused with statsd metric, this one is a global CarTelemetry metric name.
String metricConfigName = subscriber.getMetricsConfig().getName();
String handlerFnName = subscriber.getSubscriber().getHandler();
return HashUtils.sha256(metricConfigName + "-" + handlerFnName);
}
/** Builds {@link StatsdConfig} proto for given subscriber. */
@VisibleForTesting
static StatsdConfig buildStatsdConfig(DataSubscriber subscriber, long configId) {
TelemetryProto.StatsPublisher.SystemMetric metric =
subscriber.getPublisherParam().getStats().getSystemMetric();
StatsdConfig.Builder builder = StatsdConfig.newBuilder()
// This id is not used in StatsD, but let's make it the same as config_key
// just in case.
.setId(configId)
.addAllowedLogSource("AID_SYSTEM");
switch (metric) {
case APP_START_MEMORY_STATE_CAPTURED:
return buildAppStartMemoryStateStatsdConfig(builder);
case PROCESS_MEMORY_STATE:
return buildProcessMemoryStateStatsdConfig(builder);
case ACTIVITY_FOREGROUND_STATE_CHANGED:
return buildActivityForegroundStateStatsdConfig(builder);
case PROCESS_CPU_TIME:
return buildProcessCpuTimeStatsdConfig(builder);
case APP_CRASH_OCCURRED:
return buildAppCrashOccurredStatsdConfig(builder);
case ANR_OCCURRED:
return buildAnrOccurredStatsdConfig(builder);
case WTF_OCCURRED:
return buildWtfOccurredStatsdConfig(builder);
default:
throw new IllegalArgumentException("Unsupported metric " + metric.name());
}
}
private static StatsdConfig buildAppStartMemoryStateStatsdConfig(StatsdConfig.Builder builder) {
return builder
.addAtomMatcher(StatsdConfigProto.AtomMatcher.newBuilder()
// The id must be unique within StatsdConfig/matchers
.setId(APP_START_MEMORY_STATE_CAPTURED_ATOM_MATCHER_ID)
.setSimpleAtomMatcher(StatsdConfigProto.SimpleAtomMatcher.newBuilder()
.setAtomId(APP_START_MEMORY_STATE_CAPTURED_FIELD_NUMBER)))
.addEventMetric(StatsdConfigProto.EventMetric.newBuilder()
// The id must be unique within StatsdConfig/metrics
.setId(APP_START_MEMORY_STATE_CAPTURED_EVENT_METRIC_ID)
.setWhat(APP_START_MEMORY_STATE_CAPTURED_ATOM_MATCHER_ID))
.build();
}
private static StatsdConfig buildProcessMemoryStateStatsdConfig(StatsdConfig.Builder builder) {
return builder
.addAtomMatcher(StatsdConfigProto.AtomMatcher.newBuilder()
// The id must be unique within StatsdConfig/matchers
.setId(PROCESS_MEMORY_STATE_MATCHER_ID)
.setSimpleAtomMatcher(StatsdConfigProto.SimpleAtomMatcher.newBuilder()
.setAtomId(PROCESS_MEMORY_STATE_FIELD_NUMBER)))
.addGaugeMetric(StatsdConfigProto.GaugeMetric.newBuilder()
// The id must be unique within StatsdConfig/metrics
.setId(PROCESS_MEMORY_STATE_GAUGE_METRIC_ID)
.setWhat(PROCESS_MEMORY_STATE_MATCHER_ID)
.setDimensionsInWhat(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(PROCESS_MEMORY_STATE_FIELD_NUMBER)
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessMemoryState.UID_FIELD_NUMBER))
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessMemoryState.PROCESS_NAME_FIELD_NUMBER))
)
.setGaugeFieldsFilter(StatsdConfigProto.FieldFilter.newBuilder()
.setFields(PROCESS_MEMORY_STATE_FIELDS_MATCHER)
) // setGaugeFieldsFilter
.setSamplingType(
StatsdConfigProto.GaugeMetric.SamplingType.RANDOM_ONE_SAMPLE)
.setBucket(StatsdConfigProto.TimeUnit.FIVE_MINUTES)
)
.addPullAtomPackages(StatsdConfigProto.PullAtomPackages.newBuilder()
.setAtomId(PROCESS_MEMORY_STATE_FIELD_NUMBER)
.addPackages("AID_SYSTEM"))
.build();
}
private static StatsdConfig buildActivityForegroundStateStatsdConfig(
StatsdConfig.Builder builder) {
return builder
.addAtomMatcher(StatsdConfigProto.AtomMatcher.newBuilder()
// The id must be unique within StatsdConfig/matchers
.setId(ACTIVITY_FOREGROUND_STATE_CHANGED_ATOM_MATCHER_ID)
.setSimpleAtomMatcher(StatsdConfigProto.SimpleAtomMatcher.newBuilder()
.setAtomId(ACTIVITY_FOREGROUND_STATE_CHANGED_FIELD_NUMBER)))
.addEventMetric(StatsdConfigProto.EventMetric.newBuilder()
// The id must be unique within StatsdConfig/metrics
.setId(ACTIVITY_FOREGROUND_STATE_CHANGED_EVENT_METRIC_ID)
.setWhat(ACTIVITY_FOREGROUND_STATE_CHANGED_ATOM_MATCHER_ID))
.build();
}
private static StatsdConfig buildProcessCpuTimeStatsdConfig(StatsdConfig.Builder builder) {
return builder
.addAtomMatcher(StatsdConfigProto.AtomMatcher.newBuilder()
// The id must be unique within StatsdConfig/matchers
.setId(PROCESS_CPU_TIME_MATCHER_ID)
.setSimpleAtomMatcher(StatsdConfigProto.SimpleAtomMatcher.newBuilder()
.setAtomId(PROCESS_CPU_TIME_FIELD_NUMBER)))
.addGaugeMetric(StatsdConfigProto.GaugeMetric.newBuilder()
// The id must be unique within StatsdConfig/metrics
.setId(PROCESS_CPU_TIME_GAUGE_METRIC_ID)
.setWhat(PROCESS_CPU_TIME_MATCHER_ID)
.setDimensionsInWhat(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(PROCESS_CPU_TIME_FIELD_NUMBER)
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessCpuTime.UID_FIELD_NUMBER))
.addChild(StatsdConfigProto.FieldMatcher.newBuilder()
.setField(ProcessCpuTime.PROCESS_NAME_FIELD_NUMBER))
)
.setGaugeFieldsFilter(StatsdConfigProto.FieldFilter.newBuilder()
.setFields(PROCESS_CPU_TIME_FIELDS_MATCHER)
) // setGaugeFieldsFilter
.setSamplingType(
StatsdConfigProto.GaugeMetric.SamplingType.RANDOM_ONE_SAMPLE)
.setBucket(StatsdConfigProto.TimeUnit.FIVE_MINUTES)
)
.addPullAtomPackages(StatsdConfigProto.PullAtomPackages.newBuilder()
.setAtomId(PROCESS_CPU_TIME_FIELD_NUMBER)
.addPackages("AID_SYSTEM"))
.build();
}
private static StatsdConfig buildAppCrashOccurredStatsdConfig(StatsdConfig.Builder builder) {
return builder
.addAtomMatcher(StatsdConfigProto.AtomMatcher.newBuilder()
// The id must be unique within StatsdConfig/matchers
.setId(APP_CRASH_OCCURRED_ATOM_MATCHER_ID)
.setSimpleAtomMatcher(StatsdConfigProto.SimpleAtomMatcher.newBuilder()
.setAtomId(APP_CRASH_OCCURRED_FIELD_NUMBER)))
.addEventMetric(StatsdConfigProto.EventMetric.newBuilder()
// The id must be unique within StatsdConfig/metrics
.setId(APP_CRASH_OCCURRED_EVENT_METRIC_ID)
.setWhat(APP_CRASH_OCCURRED_ATOM_MATCHER_ID))
.build();
}
private static StatsdConfig buildAnrOccurredStatsdConfig(StatsdConfig.Builder builder) {
return builder
.addAtomMatcher(StatsdConfigProto.AtomMatcher.newBuilder()
// The id must be unique within StatsdConfig/matchers
.setId(ANR_OCCURRED_ATOM_MATCHER_ID)
.setSimpleAtomMatcher(StatsdConfigProto.SimpleAtomMatcher.newBuilder()
.setAtomId(ANR_OCCURRED_FIELD_NUMBER)))
.addEventMetric(StatsdConfigProto.EventMetric.newBuilder()
// The id must be unique within StatsdConfig/metrics
.setId(ANR_OCCURRED_EVENT_METRIC_ID)
.setWhat(ANR_OCCURRED_ATOM_MATCHER_ID))
.build();
}
private static StatsdConfig buildWtfOccurredStatsdConfig(StatsdConfig.Builder builder) {
return builder
.addAtomMatcher(StatsdConfigProto.AtomMatcher.newBuilder()
// The id must be unique within StatsdConfig/matchers
.setId(WTF_OCCURRED_ATOM_MATCHER_ID)
.setSimpleAtomMatcher(StatsdConfigProto.SimpleAtomMatcher.newBuilder()
.setAtomId(WTF_OCCURRED_FIELD_NUMBER)))
.addEventMetric(StatsdConfigProto.EventMetric.newBuilder()
// The id must be unique within StatsdConfig/metrics
.setId(WTF_OCCURRED_EVENT_METRIC_ID)
.setWhat(WTF_OCCURRED_ATOM_MATCHER_ID))
.build();
}
}