Source code for wlauto.workloads.spec2000

#    Copyright 2014-2015 ARM Limited
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

#pylint: disable=E1101,W0201
import operator
import os
import re
import string
import tarfile
from collections import defaultdict

from wlauto import Workload, Parameter, Alias
from wlauto.exceptions import ConfigError, WorkloadError
from wlauto.common.resources import ExtensionAsset
from wlauto.utils.misc import get_cpu_mask
from wlauto.utils.types import boolean, list_or_string

[docs]class Spec2000(Workload): name = 'spec2000' description = """ SPEC2000 benchmarks measuring processor, memory and compiler. From the web site: SPEC CPU2000 is the next-generation industry-standardized CPU-intensive benchmark suite. SPEC designed CPU2000 to provide a comparative measure of compute intensive performance across the widest practical range of hardware. The implementation resulted in source code benchmarks developed from real user applications. These benchmarks measure the performance of the processor, memory and compiler on the tested system. .. note:: At the moment, this workload relies on pre-built SPEC binaries (included in an asset bundle). These binaries *must* be built according to rules outlined here:: in order for the results to be valid SPEC2000 results. .. note:: This workload does not attempt to generate results in an admissible SPEC format. No metadata is provided (though some, but not all, of the required metdata is collected by WA elsewhere). It is upto the user to post-process results to generated SPEC-admissible results file, if that is their intention. *base vs peak* SPEC2000 defines two build/test configuration: base and peak. Base is supposed to use basic configuration (e.g. default compiler flags) with no tuning, and peak is specifically optimized for a system. Since this workload uses externally-built binaries, there is no way for WA to be sure what configuration is used -- the user is expected to keep track of that. Be aware that base/peak also come with specific requirements for the way workloads are run (e.g. how many instances on multi-core systems):: These are not enforced by WA, so it is again up to the user to ensure that correct workload parameters are specfied inthe agenda, if they intend to collect "official" SPEC results. (Those interested in collecting official SPEC results should also note that setting runtime parameters would violate SPEC runs rules that state that no configuration must be done to the platform after boot). *bundle structure* This workload expects the actual benchmark binaries to be provided in a tarball "bundle" that has a very specific structure. At the top level of the tarball, there should be two directories: "fp" and "int" -- for each of the SPEC2000 categories. Under those, there is a sub-directory per benchmark. Each benchmark sub-directory contains three sub-sub-directorie: - "cpus" contains a subdirectory for each supported cpu (e.g. a15) with a single executable binary for that cpu, in addition to a "generic" subdirectory that has not been optimized for a specific cpu and should run on any ARM system. - "data" contains all additional files (input, configuration, etc) that the benchmark executable relies on. - "scripts" contains one or more one-liner shell scripts that invoke the benchmark binary with appropriate command line parameters. The name of the script must be in the format <benchmark name>[.<variant name>].sh, i.e. name of benchmark, optionally followed by variant name, followed by ".sh" extension. If there is more than one script, then all of them must have a variant; if there is only one script the it should not contain a variant. A typical bundle may look like this:: |- fp | |-- ammp | | |-- cpus | | | |-- generic | | | | |-- ammp | | | |-- a15 | | | | |-- ammp | | | |-- a7 | | | | |-- ammp | | |-- data | | | |-- | | |-- scripts | | | |-- | |-- applu . . . . . . . . . |- int . """ # TODO: This is a bit of a hack. Need to re-think summary metric indication # (also more than just summary/non-summary classification?) class _SPECSummaryMetrics(object): def __contains__(self, item): return item.endswith('_real') asset_file = 'spec2000-assets.tar.gz' aliases = [ Alias('spec2k'), ] summary_metrics = _SPECSummaryMetrics() parameters = [ Parameter('benchmarks', kind=list_or_string, description='Specifies the SPEC benchmarks to run.'), Parameter('mode', kind=str, allowed_values=['speed', 'rate'], default='speed', description='SPEC benchmarks can report either speed to execute or throughput/rate. ' 'In the latter case, several "threads" will be spawned.'), Parameter('number_of_threads', kind=int, default=None, description='Specify the number of "threads" to be used in \'rate\' mode. (Note: ' 'on big.LITTLE systems this is the number of threads, for *each cluster*). '), Parameter('force_extract_assets', kind=boolean, default=False, description='if set to ``True``, will extract assets from the bundle, even if they are ' 'already extracted. Note: this option implies ``force_push_assets``.'), Parameter('force_push_assets', kind=boolean, default=False, description='If set to ``True``, assets will be pushed to device even if they\'re already ' 'present.'), Parameter('timeout', kind=int, default=20 * 60, description='Timeout, in seconds, for the execution of single spec test.'), ] speed_run_template = 'cd {datadir}; time ({launch_command})' rate_run_template = 'cd {datadir}; time ({loop}; wait)' loop_template = 'for i in $({busybox} seq 1 {threads}); do {launch_command} 1>/dev/null 2>&1 & done' launch_template = '{busybox} taskset {cpumask} {command} 1>/dev/null 2>&1' timing_regex = re.compile(r'(?P<minutes>\d+)m(?P<seconds>[\d.]+)s\s+(?P<category>\w+)')
[docs] def init_resources(self, context): self._load_spec_benchmarks(context)
[docs] def setup(self, context): cpus = self.device.core_names if not cpus: raise WorkloadError('Device has not specified CPU cores configuration.') cpumap = defaultdict(list) for i, cpu in enumerate(cpus): cpumap[cpu.lower()].append(i) for benchspec in self.benchmarks: commandspecs = self._verify_and_deploy_benchmark(benchspec, cpumap) self._build_command(benchspec, commandspecs)
[docs] def run(self, context): for name, command in self.commands: self.timings[name] = self.device.execute(command, timeout=self.timeout)
[docs] def update_result(self, context): for benchmark, output in self.timings.iteritems(): matches = self.timing_regex.finditer(output) found = False for match in matches: category ='category') mins = float('minutes')) secs = float('seconds')) total = secs + 60 * mins context.result.add_metric('_'.join([benchmark, category]), total, 'seconds', lower_is_better=True) found = True if not found: self.logger.error('Could not get timings for {}'.format(benchmark))
def validate(self): if self.force_extract_assets: self.force_push_assets = True if self.benchmarks is None: # pylint: disable=access-member-before-definition self.benchmarks = ['all'] for benchname in self.benchmarks: if benchname == 'all': self.benchmarks = self.loaded_benchmarks.keys() break if benchname not in self.loaded_benchmarks: raise ConfigError('Unknown SPEC benchmark: {}'.format(benchname)) if self.mode == 'speed': if self.number_of_threads is not None: raise ConfigError('number_of_threads cannot be specified in speed mode.') else: raise ValueError('Unexpected SPEC2000 mode: {}'.format(self.mode)) # Should never get here self.commands = [] self.timings = {} def _load_spec_benchmarks(self, context): self.loaded_benchmarks = {} self.categories = set() if self.force_extract_assets or len(os.listdir(self.dependencies_directory)) < 2: bundle = context.resolver.get(ExtensionAsset(self, self.asset_file)) with, 'r:gz') as tf: tf.extractall(self.dependencies_directory) for entry in os.listdir(self.dependencies_directory): entrypath = os.path.join(self.dependencies_directory, entry) if os.path.isdir(entrypath): for bench in os.listdir(entrypath): self.categories.add(entry) benchpath = os.path.join(entrypath, bench) self._load_benchmark(benchpath, entry) def _load_benchmark(self, path, category): datafiles = [] cpus = [] for df in os.listdir(os.path.join(path, 'data')): datafiles.append(os.path.join(path, 'data', df)) for cpu in os.listdir(os.path.join(path, 'cpus')): cpus.append(cpu) commandsdir = os.path.join(path, 'commands') for command in os.listdir(commandsdir): bench = SpecBenchmark() = os.path.splitext(command)[0] bench.path = path bench.category = category bench.datafiles = datafiles bench.cpus = cpus with open(os.path.join(commandsdir, command)) as fh: bench.command_template = string.Template( self.loaded_benchmarks[] = bench def _verify_and_deploy_benchmark(self, benchspec, cpumap): # pylint: disable=R0914 """Verifies that the supplied benchmark spec is valid and deploys the required assets to the device (if necessary). Returns a list of command specs (one for each CPU cluster) that can then be used to construct the final command.""" bench = self.loaded_benchmarks[benchspec] basename = benchspec.split('.')[0] datadir = self.device.path.join(self.device.working_directory,, basename) if self.force_push_assets or not self.device.file_exists(datadir): self.device.execute('mkdir -p {}'.format(datadir)) for datafile in bench.datafiles: self.device.push_file(datafile, self.device.path.join(datadir, os.path.basename(datafile))) if self.mode == 'speed': cpus = [self._get_fastest_cpu().lower()] else: cpus = cpumap.keys() cmdspecs = [] for cpu in cpus: try: host_bin_file = bench.get_binary(cpu) except ValueError, e: try: msg = e.message msg += ' Attempting to use generic binary instead.' self.logger.debug(msg) host_bin_file = bench.get_binary('generic') cpu = 'generic' except ValueError, e: raise ConfigError(e.message) # re-raising as user error binname = os.path.basename(host_bin_file) binary = self.device.install(host_bin_file, with_name='.'.join([binname, cpu])) commandspec = CommandSpec() commandspec.command = bench.command_template.substitute({'binary': binary}) commandspec.datadir = datadir if cpu == 'generic': all_cpus = reduce(operator.add, cpumap.values()) commandspec.cpumask = get_cpu_mask(all_cpus) else: commandspec.cpumask = get_cpu_mask(cpumap[cpu]) cmdspecs.append(commandspec) return cmdspecs def _build_command(self, name, commandspecs): if self.mode == 'speed': if len(commandspecs) != 1: raise AssertionError('Must be exactly one command spec specified in speed mode.') spec = commandspecs[0] launch_command = self.launch_template.format(busybox=self.device.busybox, command=spec.command, cpumask=spec.cpumask) self.commands.append((name, self.speed_run_template.format(datadir=spec.datadir, launch_command=launch_command))) elif self.mode == 'rate': loops = [] for spec in commandspecs: launch_command = self.launch_template.format(busybox=self.device.busybox, command=spec.command, cpumask=spec.cpumask) loops.append(self.loop_template.format(busybox=self.device.busybox, launch_command=launch_command, threads=spec.threads)) self.commands.append((name, self.rate_run_template.format(datadir=spec.datadir, loop='; '.join(loops)))) else: raise ValueError('Unexpected SPEC2000 mode: {}'.format(self.mode)) # Should never get here def _get_fastest_cpu(self): cpu_types = set(self.device.core_names) if len(cpu_types) == 1: return cpu_types.pop() fastest_cpu = None fastest_freq = 0 for cpu_type in cpu_types: try: idx = self.device.get_core_online_cpu(cpu_type) freq = self.device.get_cpu_max_frequency(idx) if freq > fastest_freq: fastest_freq = freq fastest_cpu = cpu_type except ValueError: pass if not fastest_cpu: raise WorkloadError('No active CPUs found on device. Something is very wrong...') return fastest_cpu
[docs]class SpecBenchmark(object): def __init__(self): = None self.path = None self.category = None self.command_template = None self.cpus = [] self.datafiles = []
[docs] def get_binary(self, cpu): if cpu not in self.cpus: raise ValueError('CPU {} is not supported by {}.'.format(cpu, binpath = os.path.join(self.path, 'cpus', cpu,'.')[0]) if not os.path.isfile(binpath): raise ValueError('CPU {} is not supported by {}.'.format(cpu, return binpath
[docs]class CommandSpec(object): def __init__(self): self.cpumask = None self.datadir = None self.command = None self.threads = None