Source code for wlauto.result_processors.dvfs

#    Copyright 2013-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import csv
import re

from wlauto import ResultProcessor, settings, instrumentation
from wlauto.exceptions import ConfigError, ResultProcessorError


[docs]class DVFS(ResultProcessor): name = 'dvfs' description = """ Reports DVFS state residency data form ftrace power events. This generates a ``dvfs.csv`` in the top-level results directory that, for each workload iteration, reports the percentage of time each CPU core spent in each of the DVFS frequency states (P-states), as well as percentage of the time spent in idle, during the execution of the workload. .. note:: ``trace-cmd`` instrument *MUST* be enabled in the instrumentation, and at least ``'power*'`` events must be enabled. """ def __init__(self, **kwargs): super(DVFS, self).__init__(**kwargs) self.device = None self.infile = None self.outfile = None self.current_cluster = None self.currentstates_of_clusters = [] self.current_frequency_of_clusters = [] self.timestamp = [] self.state_time_map = {} # hold state at timestamp self.cpuid_time_map = {} # hold cpuid at timestamp self.cpu_freq_time_spent = {} self.cpuids_of_clusters = [] self.power_state = [0, 1, 2, 3] self.UNKNOWNSTATE = 4294967295 self.multiply_factor = None self.corename_of_clusters = [] self.numberofcores_in_cluster = [] self.minimum_frequency_cluster = [] self.idlestate_description = {} def validate(self): if not instrumentation.instrument_is_installed('trace-cmd'): raise ConfigError('"dvfs" works only if "trace_cmd" in enabled in instrumentation') def initialize(self, context): # pylint: disable=R0912 self.device = context.device if not self.device.has('cpuidle'): raise ConfigError('Device does not appear to have cpuidle capability; is the right module installed?') if not self.device.core_names: message = 'Device does not specify its core types (core_names/core_clusters not set in device_config).' raise ResultProcessorError(message) number_of_clusters = max(self.device.core_clusters) + 1 # In IKS devices, actual number of cores is double # from what we get from device.number_of_cores if self.device.scheduler == 'iks': self.multiply_factor = 2 elif self.device.scheduler == 'unknown': # Device doesn't specify its scheduler type. It could be IKS, in # which case reporeted values would be wrong, so error out. message = ('The Device doesn not specify it\'s scheduler type. If you are ' 'using a generic device interface, please make sure to set the ' '"scheduler" parameter in the device config.') raise ResultProcessorError(message) else: self.multiply_factor = 1 # separate out the cores in each cluster # It is list of list of cores in cluster listof_cores_clusters = [] for cluster in range(number_of_clusters): listof_cores_clusters.append([core for core in self.device.core_clusters if core == cluster]) # Extract minimum frequency of each cluster and # the idle power state with its descriptive name # total_cores = 0 current_cores = 0 for cluster, cores_list in enumerate(listof_cores_clusters): self.corename_of_clusters.append(self.device.core_names[total_cores]) if self.device.scheduler != 'iks': self.idlestate_description.update({s.id: s.desc for s in self.device.get_cpuidle_states(total_cores)}) else: self.idlestate_description.update({s.id: s.desc for s in self.device.get_cpuidle_states()}) total_cores += len(cores_list) self.numberofcores_in_cluster.append(len(cores_list)) for i in range(current_cores, total_cores): if i in self.device.online_cpus: self.minimum_frequency_cluster.append(int(self.device.get_cpu_min_frequency("cpu{}".format(i)))) break current_cores = total_cores length_frequency_cluster = len(self.minimum_frequency_cluster) if length_frequency_cluster != number_of_clusters: diff = number_of_clusters - length_frequency_cluster offline_value = -1 for i in range(diff): if self.device.scheduler != 'iks': self.minimum_frequency_cluster.append(offline_value) else: self.minimum_frequency_cluster.append(self.device.iks_switch_frequency)
[docs] def process_iteration_result(self, result, context): """ Parse the trace.txt for each iteration, calculate DVFS residency state/frequencies and dump the result in csv and flush the data for next iteration. """ self.infile = os.path.join(context.output_directory, 'trace.txt') if os.path.isfile(self.infile): self.logger.debug('Running result_processor "dvfs"') self.outfile = os.path.join(settings.output_directory, 'dvfs.csv') self.flush_parse_initialize() self.calculate() self.percentage() self.generate_csv(context) self.logger.debug('Completed result_processor "dvfs"') else: self.logger.debug('trace.txt not found.')
[docs] def flush_parse_initialize(self): """ Store state, cpu_id for each timestamp from trace.txt and flush all the values for next iterations. """ self.current_cluster = 0 self.current_frequency_of_clusters = [] self.timestamp = [] self.currentstates_of_clusters = [] self.state_time_map = {} self.cpuid_time_map = {} self.cpu_freq_time_spent = {} self.cpuids_of_clusters = [] self.parse() # Parse trace.txt generated from trace-cmd instrumentation # Initialize the states of each core of clusters and frequency of # each clusters with its minimum freq # cpu_id is assigned for each of clusters. # For IKS devices cpuid remains same in other clusters # and for other it will increment by 1 count = 0 for cluster, cores_number in enumerate(self.numberofcores_in_cluster): self.currentstates_of_clusters.append([-1 for dummy in range(cores_number)]) self.current_frequency_of_clusters.append(self.minimum_frequency_cluster[cluster]) if self.device.scheduler == 'iks': self.cpuids_of_clusters.append([j for j in range(cores_number)]) else: self.cpuids_of_clusters.append(range(count, count + cores_number)) count += cores_number # Initialize the time spent in each state/frequency for each core. for i in range(self.device.number_of_cores * self.multiply_factor): self.cpu_freq_time_spent["cpu{}".format(i)] = {} for j in self.unique_freq(): self.cpu_freq_time_spent["cpu{}".format(i)][j] = 0 # To determine offline -1 state is added offline_value = -1 self.cpu_freq_time_spent["cpu{}".format(i)][offline_value] = 0 if 0 not in self.unique_freq(): self.cpu_freq_time_spent["cpu{}".format(i)][0] = 0
[docs] def update_cluster_freq(self, state, cpu_id): """ Update the cluster frequency and current cluster""" # For IKS devices cluster changes only possible when # freq changes, for other it is determine by cpu_id. if self.device.scheduler != 'iks': self.current_cluster = self.get_cluster(cpu_id, state) if self.get_state_name(state) == "freqstate": self.current_cluster = self.get_cluster(cpu_id, state) self.current_frequency_of_clusters[self.current_cluster] = state
[docs] def get_cluster(self, cpu_id, state): # For IKS if current state is greater than switch # freq then it is in cluster2 else cluster1 # For other, Look the current cpu_id and check this id # belong to which cluster. if self.device.scheduler == 'iks': return 1 if state >= self.device.iks_switch_frequency else 0 else: for cluster, cpuids_list in enumerate(self.cpuids_of_clusters): if cpu_id in cpuids_list: return cluster
[docs] def get_cluster_freq(self): return self.current_frequency_of_clusters[self.current_cluster]
[docs] def update_state(self, state, cpu_id): # pylint: disable=R0912 """ Update state of each cores in every cluster. This is done for each timestamp. """ POWERDOWN = 2 offline_value = -1 # if state is in unknowstate, then change state of current cpu_id # with cluster freq of current cluster. # if state is in powerstate then change state with that power state. if self.get_state_name(state) in ["unknownstate", "powerstate"]: for i in range(len(self.cpuids_of_clusters[self.current_cluster])): if cpu_id == self.cpuids_of_clusters[self.current_cluster][i]: if self.get_state_name(state) == "unknownstate": self.currentstates_of_clusters[self.current_cluster][i] = self.current_frequency_of_clusters[self.current_cluster] elif self.get_state_name(state) == "powerstate": self.currentstates_of_clusters[self.current_cluster][i] = state # If state is in freqstate then update the state with current state. # For IKS, if all cores is in power down and current state is freqstate # then update the all the cores in current cluster to current state # and other state cluster changed to Power down. if self.get_state_name(state) == "freqstate": for i, j in enumerate(self.currentstates_of_clusters[self.current_cluster]): if j != offline_value: self.currentstates_of_clusters[self.current_cluster][i] = state if cpu_id == self.cpuids_of_clusters[self.current_cluster][i]: self.currentstates_of_clusters[self.current_cluster][i] = state if self.device.scheduler == 'iks': check = False # All core in cluster is power down for i in range(len(self.currentstates_of_clusters[self.current_cluster])): if self.currentstates_of_clusters[self.current_cluster][i] != POWERDOWN: check = True break if not check: for i in range(len(self.currentstates_of_clusters[self.current_cluster])): self.currentstates_of_clusters[self.current_cluster][i] = self.current_frequency_of_clusters[self.current_cluster] for cluster, state_list in enumerate(self.currentstates_of_clusters): if cluster != self.current_cluster: for j in range(len(state_list)): self.currentstates_of_clusters[i][j] = POWERDOWN
[docs] def unique_freq(self): """ Determine the unique Frequency and state""" unique_freq = [] for i in self.timestamp: if self.state_time_map[i] not in unique_freq and self.state_time_map[i] != self.UNKNOWNSTATE: unique_freq.append(self.state_time_map[i]) for i in self.minimum_frequency_cluster: if i not in unique_freq: unique_freq.append(i) return unique_freq
[docs] def parse(self): """ Parse the trace.txt :: store timestamp, state, cpu_id --------------------------------------------------------------------------------- |timestamp| |state| |cpu_id| <idle>-0 [001] 294.554380: cpu_idle: state=4294967295 cpu_id=1 <idle>-0 [001] 294.554454: power_start: type=1 state=0 cpu_id=1 <idle>-0 [001] 294.554458: cpu_idle: state=0 cpu_id=1 <idle>-0 [001] 294.554464: power_end: cpu_id=1 <idle>-0 [001] 294.554471: cpu_idle: state=4294967295 cpu_id=1 <idle>-0 [001] 294.554590: power_start: type=1 state=0 cpu_id=1 <idle>-0 [001] 294.554593: cpu_idle: state=0 cpu_id=1 <idle>-0 [001] 294.554636: power_end: cpu_id=1 <idle>-0 [001] 294.554639: cpu_idle: state=4294967295 cpu_id=1 <idle>-0 [001] 294.554669: power_start: type=1 state=0 cpu_id=1 """ pattern = re.compile(r'\s+(?P<time>\S+)\S+\s*(?P<desc>(cpu_idle:|cpu_frequency:))\s*state=(?P<state>\d+)\s*cpu_id=(?P<cpu_id>\d+)') start_trace = False stop_trace = False with open(self.infile, 'r') as f: for line in f: #Start collecting data from label "TRACE_MARKER_START" and #stop with label "TRACE_MARKER_STOP" if line.find("TRACE_MARKER_START") != -1: start_trace = True if line.find("TRACE_MARKER_STOP") != -1: stop_trace = True if start_trace and not stop_trace: match = pattern.search(line) if match: self.timestamp.append(float(match.group('time'))) self.state_time_map[float(match.group('time'))] = int(match.group('state')) self.cpuid_time_map[float(match.group('time'))] = int(match.group('cpu_id'))
[docs] def get_state_name(self, state): if state in self.power_state: return "powerstate" elif state == self.UNKNOWNSTATE: return "unknownstate" else: return "freqstate"
[docs] def populate(self, time1, time2): diff = time2 - time1 for cluster, states_list in enumerate(self.currentstates_of_clusters): for k, j in enumerate(states_list): if self.device.scheduler == 'iks' and cluster == 1: self.cpu_freq_time_spent["cpu{}".format(self.cpuids_of_clusters[cluster][k] + len(self.currentstates_of_clusters[0]))][j] += diff else: self.cpu_freq_time_spent["cpu{}".format(self.cpuids_of_clusters[cluster][k])][j] += diff
[docs] def calculate(self): for i in range(len(self.timestamp) - 1): self.update_cluster_freq(self.state_time_map[self.timestamp[i]], self.cpuid_time_map[self.timestamp[i]]) self.update_state(self.state_time_map[self.timestamp[i]], self.cpuid_time_map[self.timestamp[i]]) self.populate(self.timestamp[i], self.timestamp[i + 1])
[docs] def percentage(self): """Normalize the result with total execution time.""" temp = self.cpu_freq_time_spent.copy() for i in self.cpu_freq_time_spent: total = 0 for j in self.cpu_freq_time_spent[i]: total += self.cpu_freq_time_spent[i][j] for j in self.cpu_freq_time_spent[i]: if total != 0: temp[i][j] = self.cpu_freq_time_spent[i][j] * 100 / total else: temp[i][j] = 0 return temp
[docs] def generate_csv(self, context): # pylint: disable=R0912,R0914 """ generate the '''dvfs.csv''' with the state, frequency and cores """ temp = self.percentage() total_state = self.unique_freq() offline_value = -1 ghz_conversion = 1000000 mhz_conversion = 1000 with open(self.outfile, 'a+') as f: writer = csv.writer(f, delimiter=',') reader = csv.reader(f) # Create the header in the format below # workload name, iteration, state, A7 CPU0,A7 CPU1,A7 CPU2,A7 CPU3,A15 CPU4,A15 CPU5 if sum(1 for row in reader) == 0: header_row = ['workload', 'iteration', 'state'] count = 0 for cluster, states_list in enumerate(self.currentstates_of_clusters): for dummy_index in range(len(states_list)): header_row.append("{} CPU{}".format(self.corename_of_clusters[cluster], count)) count += 1 writer.writerow(header_row) if offline_value in total_state: total_state.remove(offline_value) # remove the offline state for i in sorted(total_state): temprow = [] temprow.extend([context.result.spec.label, context.result.iteration]) if "state{}".format(i) in self.idlestate_description: temprow.append(self.idlestate_description["state{}".format(i)]) else: state_value = float(i) if state_value / ghz_conversion >= 1: temprow.append("{} Ghz".format(state_value / ghz_conversion)) else: temprow.append("{} Mhz".format(state_value / mhz_conversion)) for j in range(self.device.number_of_cores * self.multiply_factor): temprow.append("{0:.3f}".format(temp["cpu{}".format(j)][i])) writer.writerow(temprow) check_off = True # Checking whether core is OFFLINE for i in range(self.device.number_of_cores * self.multiply_factor): temp_val = "{0:.3f}".format(temp["cpu{}".format(i)][offline_value]) if float(temp_val) > 1: check_off = False break if check_off is False: temprow = [] temprow.extend([context.result.spec.label, context.result.iteration]) temprow.append("OFFLINE") for i in range(self.device.number_of_cores * self.multiply_factor): temprow.append("{0:.3f}".format(temp["cpu{}".format(i)][offline_value])) writer.writerow(temprow)