| #!/usr/bin/env python |
| # Copyright 2017 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Runs several telemetry benchmarks. |
| |
| This script attempts to emulate the contract of gtest-style tests |
| invoked via recipes. The main contract is that the caller passes the |
| argument: |
| |
| --isolated-script-test-output=[FILENAME] |
| |
| json is written to that file in the format detailed here: |
| https://www.chromium.org/developers/the-json-test-results-format |
| |
| Optional argument: |
| |
| --isolated-script-test-filter=[TEST_NAMES] |
| |
| is a double-colon-separated ("::") list of test names, to run just that subset |
| of tests. This list is forwarded to the run_telemetry_benchmark_as_googletest |
| script. |
| |
| This script is intended to be the base command invoked by the isolate, |
| followed by a subsequent Python script. It could be generalized to |
| invoke an arbitrary executable. |
| |
| It currently runs several benchmarks. The benchmarks it will execute are |
| based on the shard it is running on and the sharding_map_path. |
| |
| If this is executed with a non-telemetry perf test, the flag --non-telemetry |
| has to be passed in to the script so the script knows it is running |
| an executable and not the run_benchmark command. |
| |
| The results of running the benchmark are put in separate directories per |
| benchmark. Two files will be present in each directory; perf_results.json, which |
| is the perf specific results (with unenforced format, could be histogram, |
| legacy, or chartjson), and test_results.json, which is a JSON test results |
| format file |
| (https://www.chromium.org/developers/the-json-test-results-format) |
| |
| This script was derived from run_telemetry_benchmark_as_googletest, and calls |
| into that script. |
| """ |
| |
| import argparse |
| import json |
| import os |
| import shutil |
| import sys |
| import tempfile |
| import traceback |
| |
| import common |
| |
| import run_telemetry_benchmark_as_googletest |
| import run_gtest_perf_test |
| |
| # Current whitelist of benchmarks outputting histograms |
| BENCHMARKS_TO_OUTPUT_HISTOGRAMS = [ |
| 'dummy_benchmark.histogram_benchmark_1', |
| 'blink_perf.bindings', |
| 'blink_perf.canvas', |
| 'blink_perf.css', |
| 'blink_perf.dom', |
| 'blink_perf.events', |
| 'blink_perf.image_decoder', |
| 'blink_perf.layout', |
| 'blink_perf.owp_storage', |
| 'blink_perf.paint', |
| 'blink_perf.parser', |
| 'blink_perf.shadow_dom', |
| 'blink_perf.svg', |
| 'memory.top_10_mobile', |
| 'system_health.common_desktop', |
| 'system_health.common_mobile', |
| 'system_health.memory_desktop', |
| 'system_health.memory_mobile', |
| 'system_health.webview_startup', |
| 'smoothness.gpu_rasterization.tough_filters_cases', |
| 'smoothness.gpu_rasterization.tough_path_rendering_cases', |
| 'smoothness.gpu_rasterization.tough_scrolling_cases', |
| 'smoothness.gpu_rasterization_and_decoding.image_decoding_cases', |
| 'smoothness.image_decoding_cases', |
| 'smoothness.key_desktop_move_cases', |
| 'smoothness.maps', |
| 'smoothness.oop_rasterization.top_25_smooth', |
| 'smoothness.top_25_smooth', |
| 'smoothness.tough_ad_cases', |
| 'smoothness.tough_animation_cases', |
| 'smoothness.tough_canvas_cases', |
| 'smoothness.tough_filters_cases', |
| 'smoothness.tough_image_decode_cases', |
| 'smoothness.tough_path_rendering_cases', |
| 'smoothness.tough_scrolling_cases', |
| 'smoothness.tough_texture_upload_cases', |
| 'smoothness.tough_webgl_ad_cases', |
| 'smoothness.tough_webgl_cases', |
| 'dromaeo', |
| 'jetstream', |
| 'kraken', |
| 'octane', |
| 'speedometer', |
| 'speedometer-future', |
| 'speedometer2', |
| 'speedometer2-future', |
| 'wasm', |
| 'battor.steady_state', |
| 'battor.trivial_pages', |
| 'rasterize_and_record_micro.partial_invalidation', |
| 'rasterize_and_record_micro.top_25', |
| 'scheduler.tough_scheduling_cases', |
| 'tab_switching.typical_25', |
| 'thread_times.key_hit_test_cases', |
| 'thread_times.key_idle_power_cases', |
| 'thread_times.key_mobile_sites_smooth', |
| 'thread_times.key_noop_cases', |
| 'thread_times.key_silk_cases', |
| 'thread_times.simple_mobile_sites', |
| 'thread_times.oop_rasterization.key_mobile', |
| 'thread_times.tough_compositor_cases', |
| 'thread_times.tough_scrolling_cases', |
| 'tracing.tracing_with_background_memory_infra', |
| 'tracing.tracing_with_debug_overhead', |
| ] |
| |
| def get_sharding_map_path(args): |
| return os.path.join( |
| os.path.dirname(__file__), '..', '..', 'tools', 'perf', 'core', |
| args.test_shard_map_filename) |
| |
| def write_results( |
| perf_test_name, perf_results, json_test_results, benchmark_log, |
| isolated_out_dir, encoded): |
| benchmark_path = os.path.join(isolated_out_dir, perf_test_name) |
| |
| os.makedirs(benchmark_path) |
| with open(os.path.join(benchmark_path, 'perf_results.json'), 'w') as f: |
| # non telemetry perf results are already json encoded |
| if encoded: |
| f.write(perf_results) |
| else: |
| json.dump(perf_results, f) |
| with open(os.path.join(benchmark_path, 'test_results.json'), 'w') as f: |
| json.dump(json_test_results, f) |
| |
| with open(os.path.join(benchmark_path, 'benchmark_log.txt'), 'w') as f: |
| f.write(benchmark_log) |
| |
| |
| def execute_benchmark(benchmark, isolated_out_dir, |
| args, rest_args, is_reference): |
| # While we are between chartjson and histogram set we need |
| # to determine which output format to look for or see if it was |
| # already passed in in which case that format applies to all benchmarks |
| # in this run. |
| is_histograms = append_output_format(benchmark, args, rest_args) |
| # Insert benchmark name as first argument to run_benchmark call |
| # which is the first argument in the rest_args. Also need to append |
| # output format and smoke test mode. |
| per_benchmark_args = (rest_args[:1] + [benchmark] + rest_args[1:]) |
| benchmark_name = benchmark |
| if is_reference: |
| # Need to parse out the browser to replace browser flag with |
| # reference build so we run it reference build as well |
| browser_index = 0 |
| for arg in per_benchmark_args: |
| if "browser" in arg: |
| break |
| browser_index = browser_index + 1 |
| per_benchmark_args[browser_index] = '--browser=reference' |
| # Now we need to add in the rest of the reference build args |
| per_benchmark_args.append('--max-failures=5') |
| per_benchmark_args.append('--output-trace-tag=_ref') |
| benchmark_name = benchmark + '.reference' |
| |
| # We don't care exactly what these are. In particular, the perf results |
| # could be any format (chartjson, legacy, histogram). We just pass these |
| # through, and expose these as results for this task. |
| rc, perf_results, json_test_results, benchmark_log = ( |
| run_telemetry_benchmark_as_googletest.run_benchmark( |
| args, per_benchmark_args, is_histograms)) |
| |
| write_results( |
| benchmark_name, perf_results, json_test_results, benchmark_log, |
| isolated_out_dir, False) |
| return rc |
| |
| |
| def append_output_format(benchmark, args, rest_args): |
| # We need to determine if the output format is already passed in |
| # or if we need to define it for this benchmark |
| perf_output_specified = False |
| is_histograms = False |
| if args.output_format: |
| for output_format in args.output_format: |
| if 'histograms' in output_format: |
| perf_output_specified = True |
| is_histograms = True |
| if 'chartjson' in output_format: |
| perf_output_specified = True |
| rest_args.append('--output-format=' + output_format) |
| # When crbug.com/744736 is resolved we no longer have to check |
| # the type of format per benchmark and can rely on it being passed |
| # in as an arg as all benchmarks will output the same format. |
| if not perf_output_specified: |
| if benchmark in BENCHMARKS_TO_OUTPUT_HISTOGRAMS: |
| rest_args.append('--output-format=histograms') |
| is_histograms = True |
| else: |
| rest_args.append('--output-format=chartjson') |
| return is_histograms |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| '--isolated-script-test-output', required=True) |
| # These two flags are passed in from the swarming recipe |
| # but will no longer be needed when we migrate to this new recipe. |
| # For now we need to recognize them so they don't get passed |
| # through to telemetry. |
| parser.add_argument( |
| '--isolated-script-test-chartjson-output', required=False) |
| parser.add_argument( |
| '--isolated-script-test-perf-output', required=False) |
| |
| parser.add_argument( |
| '--isolated-script-test-filter', type=str, required=False) |
| parser.add_argument('--xvfb', help='Start xvfb.', action='store_true') |
| parser.add_argument('--non-telemetry', |
| help='Type of perf test', type=bool, default=False) |
| parser.add_argument('--benchmarks', |
| help='Comma separated list of benchmark names' |
| ' to run in lieu of indexing into our benchmark bot maps', |
| required=False) |
| # Some executions may have a different sharding scheme and/or set of tests. |
| # These files must live in src/tools/perf/core/ |
| parser.add_argument('--test-shard-map-filename', type=str, required=False) |
| parser.add_argument('--output-format', action='append') |
| parser.add_argument('--run-ref-build', |
| help='Run test on reference browser', action='store_true') |
| |
| args, rest_args = parser.parse_known_args() |
| isolated_out_dir = os.path.dirname(args.isolated_script_test_output) |
| return_code = 0 |
| |
| if args.non_telemetry: |
| # For non telemetry tests the benchmark name is the name of the executable. |
| benchmark_name = rest_args[0] |
| return_code, charts, output_json = run_gtest_perf_test.execute_perf_test( |
| args, rest_args) |
| |
| write_results(benchmark_name, charts, output_json, |
| benchmark_log='Not available for C++ perf test', |
| isolated_out_dir=isolated_out_dir, encoded=True) |
| else: |
| # If the user has supplied a list of benchmark names, execute those instead |
| # of the entire suite of benchmarks. |
| if args.benchmarks: |
| benchmarks = args.benchmarks.split(',') |
| for benchmark in benchmarks: |
| return_code = (execute_benchmark( |
| benchmark, isolated_out_dir, args, rest_args, False) or return_code) |
| else: |
| # First determine what shard we are running on to know how to |
| # index into the bot map to get list of benchmarks to run. |
| total_shards = None |
| shard_index = None |
| |
| env = os.environ.copy() |
| if 'GTEST_TOTAL_SHARDS' in env: |
| total_shards = env['GTEST_TOTAL_SHARDS'] |
| if 'GTEST_SHARD_INDEX' in env: |
| shard_index = env['GTEST_SHARD_INDEX'] |
| |
| if not (total_shards or shard_index): |
| raise Exception('Shard indicators must be present for perf tests') |
| |
| sharding_map_path = get_sharding_map_path(args) |
| with open(sharding_map_path) as f: |
| sharding_map = json.load(f) |
| sharding = None |
| sharding = sharding_map[shard_index]['benchmarks'] |
| |
| for benchmark in sharding: |
| # Need to run the benchmark twice on browser and reference build |
| return_code = (execute_benchmark( |
| benchmark, isolated_out_dir, args, rest_args, False) or return_code) |
| # We ignore the return code of the reference build since we do not |
| # monitor it. |
| if args.run_ref_build: |
| execute_benchmark(benchmark, isolated_out_dir, args, rest_args, True) |
| |
| return return_code |
| |
| |
| # This is not really a "script test" so does not need to manually add |
| # any additional compile targets. |
| def main_compile_targets(args): |
| json.dump([], args.output) |
| |
| |
| if __name__ == '__main__': |
| # Conform minimally to the protocol defined by ScriptTest. |
| if 'compile_targets' in sys.argv: |
| funcs = { |
| 'run': None, |
| 'compile_targets': main_compile_targets, |
| } |
| sys.exit(common.run_script(sys.argv[1:], funcs)) |
| sys.exit(main()) |