Add scheduled job high water mark metric.

Add a metric to track the maximum number of jobs that are scheduled at a
time, in 30 minute increments.

Bug: 138239687
Bug: 279935506
Test: atest CtsJobSchedulerTestCases:ConnectivityConstraintTest
Test: atest CtsJobSchedulerTestCases:JobSchedulingTest
Test: atest CtsJobSchedulerTestCases:NotificationTest
Change-Id: If0c190bf8ebcca55ef1907020b37a0e1c72181ae

apex/jobscheduler/service/java/com/android/server/job/JobStore.java

@@ -49,8 +49,10 @@ import com.android.internal.annotations.GuardedBy;
 import com.android.internal.annotations.VisibleForTesting;
 import com.android.internal.util.ArrayUtils;
 import com.android.internal.util.BitUtils;
+import com.android.modules.expresslog.Histogram;
 import com.android.modules.utils.TypedXmlPullParser;
 import com.android.modules.utils.TypedXmlSerializer;
+import com.android.server.AppSchedulingModuleThread;
 import com.android.server.IoThread;
 import com.android.server.job.JobSchedulerInternal.JobStorePersistStats;
 import com.android.server.job.controllers.JobStatus;
@@ -94,6 +96,7 @@ public final class JobStore {
 
     /** Threshold to adjust how often we want to write to the db. */
     private static final long JOB_PERSIST_DELAY = 2000L;
+    private static final long SCHEDULED_JOB_HIGH_WATER_MARK_PERIOD_MS = 30 * 60_000L;
     @VisibleForTesting
     static final String JOB_FILE_SPLIT_PREFIX = "jobs_";
     private static final int ALL_UIDS = -1;
@@ -131,6 +134,30 @@ public final class JobStore {
 
     private JobStorePersistStats mPersistInfo = new JobStorePersistStats();
 
+    /**
+     * Separately updated value of the JobSet size to avoid recalculating it frequently for logging
+     * purposes. Continue to use {@link JobSet#size()} for the up-to-date and accurate value.
+     */
+    private int mCurrentJobSetSize = 0;
+    private int mScheduledJob30MinHighWaterMark = 0;
+    private static final Histogram sScheduledJob30MinHighWaterMarkLogger = new Histogram(
+            "job_scheduler.value_hist_scheduled_job_30_min_high_water_mark",
+            new Histogram.ScaledRangeOptions(15, 1, 99, 1.5f));
+    private final Runnable mScheduledJobHighWaterMarkLoggingRunnable = new Runnable() {
+        @Override
+        public void run() {
+            AppSchedulingModuleThread.getHandler().removeCallbacks(this);
+            synchronized (mLock) {
+                sScheduledJob30MinHighWaterMarkLogger.logSample(mScheduledJob30MinHighWaterMark);
+                mScheduledJob30MinHighWaterMark = mJobSet.size();
+            }
+            // The count doesn't need to be logged at exact times. Logging based on system uptime
+            // should be fine.
+            AppSchedulingModuleThread.getHandler()
+                    .postDelayed(this, SCHEDULED_JOB_HIGH_WATER_MARK_PERIOD_MS);
+        }
+    };
+
     /** Used by the {@link JobSchedulerService} to instantiate the JobStore. */
     static JobStore get(JobSchedulerService jobManagerService) {
         synchronized (sSingletonLock) {
@@ -183,6 +210,9 @@ public final class JobStore {
         mXmlTimestamp = mJobsFile.exists()
                 ? mJobsFile.getLastModifiedTime() : mJobFileDirectory.lastModified();
         mRtcGood = (sSystemClock.millis() > mXmlTimestamp);
+
+        AppSchedulingModuleThread.getHandler().postDelayed(
+                mScheduledJobHighWaterMarkLoggingRunnable, SCHEDULED_JOB_HIGH_WATER_MARK_PERIOD_MS);
     }
 
     private void init() {
@@ -252,7 +282,10 @@ public final class JobStore {
      * @param jobStatus Job to add.
      */
     public void add(JobStatus jobStatus) {
-        mJobSet.add(jobStatus);
+        if (mJobSet.add(jobStatus)) {
+            mCurrentJobSetSize++;
+            maybeUpdateHighWaterMark();
+        }
         if (jobStatus.isPersisted()) {
             mPendingJobWriteUids.put(jobStatus.getUid(), true);
             maybeWriteStatusToDiskAsync();
@@ -267,7 +300,10 @@ public final class JobStore {
      */
     @VisibleForTesting
     public void addForTesting(JobStatus jobStatus) {
-        mJobSet.add(jobStatus);
+        if (mJobSet.add(jobStatus)) {
+            mCurrentJobSetSize++;
+            maybeUpdateHighWaterMark();
+        }
         if (jobStatus.isPersisted()) {
             mPendingJobWriteUids.put(jobStatus.getUid(), true);
         }
@@ -303,6 +339,7 @@ public final class JobStore {
             }
             return false;
         }
+        mCurrentJobSetSize--;
         if (removeFromPersisted && jobStatus.isPersisted()) {
             mPendingJobWriteUids.put(jobStatus.getUid(), true);
             maybeWriteStatusToDiskAsync();
@@ -315,7 +352,9 @@ public final class JobStore {
      */
     @VisibleForTesting
     public void removeForTesting(JobStatus jobStatus) {
-        mJobSet.remove(jobStatus);
+        if (mJobSet.remove(jobStatus)) {
+            mCurrentJobSetSize--;
+        }
         if (jobStatus.isPersisted()) {
             mPendingJobWriteUids.put(jobStatus.getUid(), true);
         }
@@ -327,6 +366,7 @@ public final class JobStore {
      */
     public void removeJobsOfUnlistedUsers(int[] keepUserIds) {
         mJobSet.removeJobsOfUnlistedUsers(keepUserIds);
+        mCurrentJobSetSize = mJobSet.size();
     }
 
     /** Note a change in the specified JobStatus that necessitates writing job state to disk. */
@@ -342,6 +382,7 @@ public final class JobStore {
     public void clear() {
         mJobSet.clear();
         mPendingJobWriteUids.put(ALL_UIDS, true);
+        mCurrentJobSetSize = 0;
         maybeWriteStatusToDiskAsync();
     }
 
@@ -352,6 +393,7 @@ public final class JobStore {
     public void clearForTesting() {
         mJobSet.clear();
         mPendingJobWriteUids.put(ALL_UIDS, true);
+        mCurrentJobSetSize = 0;
     }
 
     void setUseSplitFiles(boolean useSplitFiles) {
@@ -442,6 +484,12 @@ public final class JobStore {
         mJobSet.forEachJobForSourceUid(sourceUid, functor);
     }
 
+    private void maybeUpdateHighWaterMark() {
+        if (mScheduledJob30MinHighWaterMark < mCurrentJobSetSize) {
+            mScheduledJob30MinHighWaterMark = mCurrentJobSetSize;
+        }
+    }
+
     /** Version of the db schema. */
     private static final int JOBS_FILE_VERSION = 1;
     /**
@@ -1125,6 +1173,12 @@ public final class JobStore {
             if (needFileMigration) {
                 migrateJobFilesAsync();
             }
+
+            // Log the count immediately after loading from boot.
+            mCurrentJobSetSize = numJobs;
+            mScheduledJob30MinHighWaterMark = mCurrentJobSetSize;
+            mScheduledJobHighWaterMarkLoggingRunnable.run();
+
             if (mCompletionLatch != null) {
                 mCompletionLatch.countDown();
             }