Source code for wlauto.workloads.telemetry

#    Copyright 2015 ARM Limited
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=attribute-defined-outside-init
import os
import re
import csv
import shutil
import json
import urllib
import stat
from zipfile import is_zipfile, ZipFile

    import pandas as pd
except ImportError:
    pd = None

from wlauto import Workload, Parameter
from wlauto.exceptions import WorkloadError, ConfigError
from wlauto.utils.misc import check_output, get_null, get_meansd
from wlauto.utils.types import numeric

RESULT_REGEX = re.compile(r'RESULT ([^:]+): ([^=]+)\s*=\s*'  # preamble and test/metric name
                          r'(\[([^\]]+)\]|(\S+))'  # value
                          r'\s*(\S+)')  # units
TRACE_REGEX = re.compile(r'Trace saved as ([^\n]+)')

# Trace event that signifies rendition of a Frame
FRAME_EVENT = 'SwapBuffersLatency'


[docs]class Telemetry(Workload): name = 'telemetry' description = """ Executes Google's Telemetery benchmarking framework Url: From the web site: Telemetry is Chrome's performance testing framework. It allows you to perform arbitrary actions on a set of web pages and report metrics about it. The framework abstracts: - Launching a browser with arbitrary flags on any platform. - Opening a tab and navigating to the page under test. - Fetching data via the Inspector timeline and traces. - Using Web Page Replay to cache real-world websites so they don't change when used in benchmarks. Design Principles - Write one performance test that runs on all platforms - Windows, Mac, Linux, Chrome OS, and Android for both Chrome and ContentShell. - Runs on browser binaries, without a full Chromium checkout, and without having to build the browser yourself. - Use WebPageReplay to get repeatable test results. - Clean architecture for writing benchmarks that keeps measurements and use cases separate. - Run on non-Chrome browsers for comparative studies. This instrument runs telemetry via its ``run_benchmark`` script (which must be in PATH or specified using ``run_benchmark_path`` parameter) and parses metrics from the resulting output. **device setup** The device setup will depend on whether you're running a test image (in which case little or no setup should be necessary) """ supported_platforms = ['android', 'chromeos'] parameters = [ Parameter('run_benchmark_path', default=None, description=""" This is the path to run_benchmark script which runs a Telemetry benchmark. If not specified, WA will look for Telemetry in its dependencies; if not found there, Telemetry will be downloaded. """), Parameter('test', default='page_cycler.top_10_mobile', description=""" Specifies the telemetry test to run. """), Parameter('run_benchmark_params', default='', description=""" Additional paramters to be passed to ``run_benchmark``. """), Parameter('run_timeout', kind=int, default=900, description=""" Timeout for execution of the test. """), Parameter('extract_fps', kind=bool, default=False, description=""" if ``True``, FPS for the run will be computed from the trace (must be enabled). """), Parameter('target_config', kind=str, default=None, description=""" Manually specify target configuration for telemetry. This must contain --browser option plus any addition options Telemetry requires for a particular target (e.g. --device or --remote) """), ] def validate(self): ret = os.system('{} > {} 2>&1'.format(self.run_benchmark_path, get_null())) if ret > 255: pass # telemetry found and appears to be installed properly. elif ret == 127: raise WorkloadError('run_benchmark not found (did you specify correct run_benchmark_path?)') else: raise WorkloadError('Unexected error from run_benchmark: {}'.format(ret)) if self.extract_fps and 'trace' not in self.run_benchmark_params: raise ConfigError('"trace" profiler must be enabled in order to extract FPS for Telemetry') self._resolve_run_benchmark_path()
[docs] def setup(self, context): self.raw_output = None self.error_output = None self.command = self.build_command()
[docs] def run(self, context): self.logger.debug(self.command) self.raw_output, self.error_output = check_output(self.command, shell=True, timeout=self.run_timeout, ignore='all')
[docs] def update_result(self, context): # pylint: disable=too-many-locals if self.error_output: self.logger.error('run_benchmarks output contained errors:\n' + self.error_output) elif not self.raw_output: self.logger.warning('Did not get run_benchmark output.') return raw_outfile = os.path.join(context.output_directory, 'telemetry_raw.out') with open(raw_outfile, 'w') as wfh: wfh.write(self.raw_output) context.add_artifact('telemetry-raw', raw_outfile, kind='raw') results, artifacts = parse_telemetry_results(raw_outfile) csv_outfile = os.path.join(context.output_directory, 'telemetry.csv') with open(csv_outfile, 'wb') as wfh: writer = csv.writer(wfh) writer.writerow(['kind', 'url', 'iteration', 'value', 'units']) for result in results: writer.writerows(result.rows) for i, value in enumerate(result.values, 1): context.add_metric(result.kind, value, units=result.units, classifiers={'url': result.url, 'time': i}) context.add_artifact('telemetry', csv_outfile, kind='data') for idx, artifact in enumerate(artifacts): if is_zipfile(artifact): zf = ZipFile(artifact) for item in zf.infolist(): zf.extract(item, context.output_directory) zf.close() context.add_artifact('telemetry_trace_{}'.format(idx), path=item.filename, kind='data') else: # not a zip archive wa_path = os.path.join(context.output_directory, os.path.basename(artifact)) shutil.copy(artifact, wa_path) context.add_artifact('telemetry_artifact_{}'.format(idx), path=wa_path, kind='data') if self.extract_fps: self.logger.debug('Extracting FPS...') _extract_fps(context)
[docs] def build_command(self): device_opts = '' if self.target_config: device_opts = self.target_config else: if self.device.platform == 'chromeos': if '--remote' not in self.run_benchmark_params: device_opts += '--remote={} '.format( if '--browser' not in self.run_benchmark_params: device_opts += '--browser=cros-chrome ' elif self.device.platform == 'android': if '--device' not in self.run_benchmark_params and self.device.adb_name: device_opts += '--device={} '.format(self.device.adb_name) if '--browser' not in self.run_benchmark_params: device_opts += '--browser=android-webview-shell ' else: raise WorkloadError('Unless you\'re running Telemetry on a ChromeOS or Android device, ' 'you mast specify target_config option') return '{} {} {} {}'.format(self.run_benchmark_path, self.test, device_opts, self.run_benchmark_params)
def _resolve_run_benchmark_path(self): # pylint: disable=access-member-before-definition if self.run_benchmark_path: if not os.path.exists(self.run_benchmark_path): raise ConfigError('run_benchmark path "{}" does not exist'.format(self.run_benchmark_path)) else: self.run_benchmark_path = os.path.join(self.dependencies_directory, 'telemetry', 'run_benchmark') self.logger.debug('run_benchmark_path not specified using {}'.format(self.run_benchmark_path)) if not os.path.exists(self.run_benchmark_path): self.logger.debug('Telemetry not found locally; downloading...') local_archive = os.path.join(self.dependencies_directory, '') urllib.urlretrieve(TELEMETRY_ARCHIVE_URL, local_archive) zf = ZipFile(local_archive) zf.extractall(self.dependencies_directory) if not os.path.exists(self.run_benchmark_path): raise WorkloadError('Could not download and extract Telemetry') old_mode = os.stat(self.run_benchmark_path).st_mode os.chmod(self.run_benchmark_path, old_mode | stat.S_IXUSR)
def _extract_fps(context): trace_files = [a.path for a in context.iteration_artifacts if'telemetry_trace_')] for tf in trace_files: name = os.path.splitext(os.path.basename(tf))[0] fps_file = os.path.join(context.output_directory, name + '-fps.csv') with open(tf) as fh: data = json.load(fh) events = pd.Series([e['ts'] for e in data['traceEvents'] if FRAME_EVENT == e['name']]) fps = (1000000 / (events - events.shift(1))) fps.index = events df = fps.dropna().reset_index() df.columns = ['timestamp', 'fps'] with open(fps_file, 'w') as wfh: df.to_csv(wfh, index=False) context.add_artifact('{}_fps'.format(name), fps_file, kind='data') context.result.add_metric('{} FPS'.format(name), df.fps.mean(), units='fps') context.result.add_metric('{} FPS (std)'.format(name), df.fps.std(), units='fps', lower_is_better=True)
[docs]class TelemetryResult(object): @property def average(self): return get_meansd(self.values)[0] @property def std(self): return get_meansd(self.values)[1] @property def rows(self): for i, v in enumerate(self.values): yield [self.kind, self.url, i, v, self.units] def __init__(self, kind=None, url=None, values=None, units=None): self.kind = kind self.url = url self.values = values or [] self.units = units def __str__(self): return 'TR({kind},{url},{values},{units})'.format(**self.__dict__) __repr__ = __str__
[docs]def parse_telemetry_results(filepath): results = [] artifacts = [] with open(filepath) as fh: for line in fh: match = if match: result = TelemetryResult() result.kind = result.url = if result.values = map(numeric,',')) else: result.values = [numeric(] result.units = results.append(result) match = if match: artifacts.append( return results, artifacts
if __name__ == '__main__': import sys # pylint: disable=wrong-import-order,wrong-import-position from pprint import pprint # pylint: disable=wrong-import-order,wrong-import-position path = sys.argv[1] pprint(parse_telemetry_results(path))