flutter · fluttergithubbot · Aug 12, 2020 · Jul 22, 2020 · Jul 29, 2020 · Jul 29, 2020
diff --git a/dev/benchmarks/macrobenchmarks/test/util.dart b/dev/benchmarks/macrobenchmarks/test/util.dart
@@ -13,13 +13,6 @@ import 'package:macrobenchmarks/common.dart';
 import 'package:e2e/e2e.dart';
 import 'package:macrobenchmarks/main.dart' as app;
 
-/// The maximum amount of time considered safe to spend for a frame's build
-/// phase. Anything past that is in the danger of missing the frame as 60FPS.
-///
-/// Changing this doesn't re-evaluate existing summary.
-Duration kBuildBudget = const Duration(milliseconds: 16);
-// TODO(CareF): Automatically calculate the refresh budget (#61958)
-
 typedef ControlCallback = Future<void> Function(WidgetController controller);
 
 void macroPerfTestE2E(
@@ -31,10 +24,6 @@ void macroPerfTestE2E(
   ControlCallback body,
   ControlCallback setup,
 }) {
-  assert(() {
-    debugPrint(kDebugWarning);
-    return true;
-  }());
   final WidgetsBinding _binding = E2EWidgetsFlutterBinding.ensureInitialized();
   assert(_binding is E2EWidgetsFlutterBinding);
   final E2EWidgetsFlutterBinding binding = _binding as E2EWidgetsFlutterBinding;
@@ -82,140 +71,28 @@ void macroPerfTestE2E(
   }, semanticsEnabled: false, timeout: Timeout(timeout));
 }
 
+bool _firstRun = true;
+
+// TODO(CareF): move this to e2e after FrameTimingSummarizer goes into stable
+// branch (#63537)
+/// watches the [FrameTiming] of `action` and report it to the e2e binding.
 Future<void> watchPerformance(
   E2EWidgetsFlutterBinding binding,
-  Future<void> action(),
-) async {
+  Future<void> action(), {
+  String reportKey = 'performance',
+}) async {
+  assert(() {
+    if (_firstRun) {
+      debugPrint(kDebugWarning);
+      _firstRun = false;
+    }
+    return true;
+  }());
   final List<FrameTiming> frameTimings = <FrameTiming>[];
   final TimingsCallback watcher = frameTimings.addAll;
   binding.addTimingsCallback(watcher);
   await action();
   binding.removeTimingsCallback(watcher);
-  // TODO(CareF): determine if it's running on firebase and report metric online
   final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings);
-  binding.reportData = <String, dynamic>{'performance': frameTimes.summary};
-}
-
-/// This class and summarizes a list of [FrameTiming] for the performance
-/// metrics.
-class FrameTimingSummarizer {
-  factory FrameTimingSummarizer(List<FrameTiming> data) {
-    assert(data != null);
-    assert(data.isNotEmpty);
-    final List<Duration> frameBuildTime = List<Duration>.unmodifiable(
-      data.map<Duration>((FrameTiming datum) => datum.buildDuration),
-    );
-    final List<Duration> frameBuildTimeSorted = List<Duration>.from(frameBuildTime)..sort();
-    final List<Duration> frameRasterizerTime = List<Duration>.unmodifiable(
-      data.map<Duration>((FrameTiming datum) => datum.rasterDuration),
-    );
-    final List<Duration> frameRasterizerTimeSorted = List<Duration>.from(frameRasterizerTime)..sort();
-    final Duration Function(Duration, Duration) add = (Duration a, Duration b) => a + b;
-    return FrameTimingSummarizer._(
-      frameBuildTime: frameBuildTime,
-      frameRasterizerTime: frameRasterizerTime,
-      // This avarage calculation is microsecond precision, which is fine
-      // because typical values of these times are milliseconds.
-      averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length,
-      p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90),
-      p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99),
-      worstFrameBuildTime: frameBuildTimeSorted.last,
-      missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget),
-      averageFrameRasterizerTime: frameRasterizerTime.reduce(add) ~/ data.length,
-      p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90),
-      p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90),
-      worstFrameRasterizerTime: frameRasterizerTimeSorted.last,
-      missedFrameRasterizerBudget: _countExceed(frameRasterizerTimeSorted, kBuildBudget),
-    );
-  }
-
-  const FrameTimingSummarizer._({
-    @required this.frameBuildTime,
-    @required this.frameRasterizerTime,
-    @required this.averageFrameBuildTime,
-    @required this.p90FrameBuildTime,
-    @required this.p99FrameBuildTime,
-    @required this.worstFrameBuildTime,
-    @required this.missedFrameBuildBudget,
-    @required this.averageFrameRasterizerTime,
-    @required this.p90FrameRasterizerTime,
-    @required this.p99FrameRasterizerTime,
-    @required this.worstFrameRasterizerTime,
-    @required this.missedFrameRasterizerBudget
-  });
-
-  /// List of frame build time in microseconds
-  final List<Duration> frameBuildTime;
-
-  /// List of frame rasterizer time in microseconds
-  final List<Duration> frameRasterizerTime;
-
-  /// The average value of [frameBuildTime] in milliseconds.
-  final Duration averageFrameBuildTime;
-
-  /// The 90-th percentile value of [frameBuildTime] in milliseconds
-  final Duration p90FrameBuildTime;
-
-  /// The 99-th percentile value of [frameBuildTime] in milliseconds
-  final Duration p99FrameBuildTime;
-
-  /// The largest value of [frameBuildTime] in milliseconds
-  final Duration worstFrameBuildTime;
-
-  /// Number of items in [frameBuildTime] that's greater than [kBuildBudget]
-  final int missedFrameBuildBudget;
-
-  /// The average value of [frameRasterizerTime] in milliseconds.
-  final Duration averageFrameRasterizerTime;
-
-  /// The 90-th percentile value of [frameRasterizerTime] in milliseconds.
-  final Duration p90FrameRasterizerTime;
-
-  /// The 99-th percentile value of [frameRasterizerTime] in milliseconds.
-  final Duration p99FrameRasterizerTime;
-
-  /// The largest value of [frameRasterizerTime] in milliseconds.
-  final Duration worstFrameRasterizerTime;
-
-  /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget]
-  final int missedFrameRasterizerBudget;
-
-  Map<String, dynamic> get summary => <String, dynamic>{
-    'average_frame_build_time_millis':
-        averageFrameBuildTime.inMicroseconds / 1E3,
-    '90th_percentile_frame_build_time_millis':
-        p90FrameBuildTime.inMicroseconds / 1E3,
-    '99th_percentile_frame_build_time_millis':
-        p99FrameBuildTime.inMicroseconds / 1E3,
-    'worst_frame_build_time_millis':
-        worstFrameBuildTime.inMicroseconds / 1E3,
-    'missed_frame_build_budget_count': missedFrameBuildBudget,
-    'average_frame_rasterizer_time_millis':
-        averageFrameRasterizerTime.inMicroseconds / 1E3,
-    '90th_percentile_frame_rasterizer_time_millis':
-        p90FrameRasterizerTime.inMicroseconds / 1E3,
-    '99th_percentile_frame_rasterizer_time_millis':
-        p99FrameRasterizerTime.inMicroseconds / 1E3,
-    'worst_frame_rasterizer_time_millis':
-        worstFrameRasterizerTime.inMicroseconds / 1E3,
-    'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget,
-    'frame_count': frameBuildTime.length,
-    'frame_build_times': frameBuildTime
-        .map<int>((Duration datum) => datum.inMicroseconds).toList(),
-    'frame_rasterizer_times': frameRasterizerTime
-        .map<int>((Duration datum) => datum.inMicroseconds).toList(),
-  };
-}
-
-// The following helper functions require data sorted
-
-// return the 100*p-th percentile of the data
-T _findPercentile<T>(List<T> data, double p) {
-  assert(p >= 0 && p <= 1);
-  return data[((data.length - 1) * p).round()];
-}
-
-// return the number of items in data that > threshold
-int _countExceed<T extends Comparable<T>>(List<T> data, T threshold) {
-  return data.length - data.indexWhere((T datum) => datum.compareTo(threshold) > 0);
+  binding.reportData = <String, dynamic>{reportKey: frameTimes.summary};
 }
diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart
@@ -0,0 +1,14 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/gallery.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.android;
+  await task(createGalleryTransitionE2ETest());
+}
diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart
@@ -15,11 +15,34 @@ TaskFunction createGalleryTransitionTest({ bool semanticsEnabled = false }) {
   return GalleryTransitionTest(semanticsEnabled: semanticsEnabled);
 }
 
+TaskFunction createGalleryTransitionE2ETest({ bool semanticsEnabled = false }) {
+  return GalleryTransitionTest(
+    semanticsEnabled: semanticsEnabled,
+    testFile: 'transitions_perf_e2e',
+    needFullTimeline: false,
+    timelineSummaryFile: 'e2e_perf_summary',
+    transitionDurationFile: null,
+    driverFile: 'transitions_perf_e2e_test',
+  );
+}
+
 class GalleryTransitionTest {
 
-  GalleryTransitionTest({ this.semanticsEnabled = false });
+  GalleryTransitionTest({
+    this.semanticsEnabled = false,
+    this.testFile = 'transitions_perf',
+    this.needFullTimeline = true,
+    this.timelineSummaryFile = 'transitions.timeline_summary',
+    this.transitionDurationFile = 'transition_durations.timeline',
+    this.driverFile,
+  });
 
   final bool semanticsEnabled;
+  final bool needFullTimeline;
+  final String testFile;
+  final String timelineSummaryFile;
+  final String transitionDurationFile;
+  final String driverFile;
 
   Future<TaskResult> call() async {
     final Device device = await devices.workingDevice;
@@ -31,41 +54,41 @@ class GalleryTransitionTest {
       await flutter('packages', options: <String>['get']);
 
       final String testDriver = semanticsEnabled
-          ? 'transitions_perf_with_semantics.dart'
-          : 'transitions_perf.dart';
+          ? '${testFile}_with_semantics.dart'
+          : '$testFile.dart';
 
       await flutter('drive', options: <String>[
         '--profile',
-        '--trace-startup',
+        if (needFullTimeline)
+          '--trace-startup',
         '-t',
         'test_driver/$testDriver',
+        if (driverFile != null)
+          ...<String>['--driver', 'test_driver/$driverFile.dart'],
         '-d',
         deviceId,
       ]);
     });
 
-    // Route paths contains slashes, which Firebase doesn't accept in keys, so we
-    // remove them.
-    final Map<String, dynamic> original = json.decode(
-      file('${galleryDirectory.path}/build/transition_durations.timeline.json').readAsStringSync(),
-    ) as Map<String, dynamic>;
-    final Map<String, List<int>> transitions = <String, List<int>>{};
-    for (final String key in original.keys) {
-      transitions[key.replaceAll('/', '')] = List<int>.from(original[key] as List<dynamic>);
-    }
-
     final Map<String, dynamic> summary = json.decode(
-      file('${galleryDirectory.path}/build/transitions.timeline_summary.json').readAsStringSync(),
+      file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(),
     ) as Map<String, dynamic>;
 
-    final Map<String, dynamic> data = <String, dynamic>{
-      'transitions': transitions,
-      'missed_transition_count': _countMissedTransitions(transitions),
-      ...summary,
-    };
+    if (transitionDurationFile != null) {
+      final Map<String, dynamic> original = json.decode(
+        file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(),
+      ) as Map<String, dynamic>;
+      final Map<String, List<int>> transitions = <String, List<int>>{};
+      for (final String key in original.keys) {
+        transitions[key] = List<int>.from(original[key] as List<dynamic>);
+      }
+      summary['transitions'] = transitions;
+      summary['missed_transition_count'] = _countMissedTransitions(transitions);
+    }
 
-    return TaskResult.success(data, benchmarkScoreKeys: <String>[
-      'missed_transition_count',
+    return TaskResult.success(summary, benchmarkScoreKeys: <String>[
+      if (transitionDurationFile != null)
+        'missed_transition_count',
       'average_frame_build_time_millis',
       'worst_frame_build_time_millis',
       '90th_percentile_frame_build_time_millis',

diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml
@@ -411,15 +411,6 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["linux/android"]
 
-  # flutter_gallery_instrumentation_test:
-  #   description: >
-  #     Same as flutter_gallery__transition_perf but uses Android instrumentation
-  #     framework, and therefore does not require a host computer to run. This
-  #     test can run on off-the-shelf infrastructures, such as Firebase Test Lab.
-  #   stage: devicelab
-  #   required_agent_capabilities: ["linux/android"]
-  #   flaky: true
-
   linux_chrome_dev_mode:
     description: >
       Run flutter web on the devicelab and hot restart.
@@ -763,6 +754,14 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["linux/android"]
 
+  flutter_gallery__transition_perf_e2e:
+    description: >
+      Measures the performance of screen transitions in Flutter Gallery on
+      Android with e2e.
+    stage: devicelab
+    required_agent_capabilities: ["linux/android"]
+    flaky: true
+
   flutter_gallery_sksl_warmup__transition_perf:
     description: >
       Measures the runtime performance of Flutter gallery transitions on Android

diff --git a/...d/app/src/androidTest/java/io/flutter/demo/gallery/FlutterGalleryInstrumentationTest.java b/...d/app/src/androidTest/java/io/flutter/demo/gallery/FlutterGalleryInstrumentationTest.java
diff --git a/...er_gallery/android/app/src/androidTest/java/io/flutter/demo/gallery/MainActivityTest.java b/...er_gallery/android/app/src/androidTest/java/io/flutter/demo/gallery/MainActivityTest.java
@@ -0,0 +1,16 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package io.flutter.demo.gallery;
+
+import androidx.test.rule.ActivityTestRule;
+import dev.flutter.plugins.e2e.FlutterTestRunner;
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+
+@RunWith(FlutterTestRunner.class)
+public class MainActivityTest {
+  @Rule
+  public ActivityTestRule<MainActivity> rule = new ActivityTestRule<>(MainActivity.class, true, false);
+}