#!/usr/bin/env python
"""Compare 2 perf runs.
To use, specify the local directories that contain
the run information::
$ ./perfcmp /results/2016-01-01-1111/ /results/2016-01-01-2222/
"""
import os
import json
import argparse
from colorama import Fore, Style
from tabulate import tabulate
class RunComparison(object):
MEMORY_FIELDS = ['average_memory', 'max_memory']
TIME_FIELDS = ['total_time']
# Fields that aren't memory or time fields, they require
# no special formatting.
OTHER_FIELDS = ['average_cpu']
def __init__(self, old_summary, new_summary):
self.old_summary = old_summary
self.new_summary = new_summary
def iter_field_names(self):
for field in self.TIME_FIELDS + self.MEMORY_FIELDS + self.OTHER_FIELDS:
yield field
def old(self, field):
value = self.old_summary[field]
return self._format(field, value)
def old_suffix(self, field):
value = self.old_summary[field]
return self._format_suffix(field, value)
def new_suffix(self, field):
value = self.new_summary[field]
return self._format_suffix(field, value)
def _format_suffix(self, field, value):
if field in self.TIME_FIELDS:
return 'sec'
elif field in self.OTHER_FIELDS:
return ''
else:
# The suffix depends on the actual value.
return self._human_readable_size(value)[1]
def old_stddev(self, field):
real_field = 'std_dev_%s' % field
return self.old(real_field)
def new(self, field):
value = self.new_summary[field]
return self._format(field, value)
def new_stddev(self, field):
real_field = 'std_dev_%s' % field
return self.new(real_field)
def _format(self, field, value):
if field.startswith('std_dev_'):
field = field[len('std_dev_'):]
if field in self.MEMORY_FIELDS:
return self._human_readable_size(value)[0]
elif field in self.TIME_FIELDS:
return '%-3.2f' % value
else:
return '%.2f' % value
def _human_readable_size(self, value):
hummanize_suffixes = ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB')
base = 1024
bytes_int = float(value)
if bytes_int == 1:
return '1 Byte'
elif bytes_int < base:
return '%d Bytes' % bytes_int
for i, suffix in enumerate(hummanize_suffixes):
unit = base ** (i+2)
if round((bytes_int / unit) * base) < base:
return ['%.2f' % (base * bytes_int / unit), suffix]
def diff_percent(self, field):
diff_percent = (
(self.new_summary[field] - self.old_summary[field]) /
float(self.old_summary[field])) * 100
return diff_percent
def compare_runs(old_dir, new_dir):
for dirname in os.listdir(old_dir):
old_run_dir = os.path.join(old_dir, dirname)
new_run_dir = os.path.join(new_dir, dirname)
if not os.path.isdir(old_run_dir):
continue
old_summary = get_summary(old_run_dir)
new_summary = get_summary(new_run_dir)
comp = RunComparison(old_summary, new_summary)
header = [Style.BRIGHT + dirname + Style.RESET_ALL,
Style.BRIGHT + 'old' + Style.RESET_ALL,
# Numeric suffix (MiB, GiB, sec).
'',
'std_dev',
Style.BRIGHT + 'new' + Style.RESET_ALL,
# Numeric suffix (MiB, GiB, sec).
'',
'std_dev',
Style.BRIGHT + 'delta' + Style.RESET_ALL]
rows = []
for field in comp.iter_field_names():
row = [field, comp.old(field), comp.old_suffix(field),
comp.old_stddev(field), comp.new(field),
comp.new_suffix(field), comp.new_stddev(field)]
diff_percent = comp.diff_percent(field)
diff_percent_str = '%.2f%%' % diff_percent
if diff_percent < 0:
diff_percent_str = (
Fore.GREEN + diff_percent_str + Style.RESET_ALL)
else:
diff_percent_str = (
Fore.RED + diff_percent_str + Style.RESET_ALL)
row.append(diff_percent_str)
rows.append(row)
print(tabulate(rows, headers=header, tablefmt='plain'))
print('')
def get_summary(benchmark_dir):
summary_json = os.path.join(benchmark_dir, 'summary.json')
with open(summary_json) as f:
return json.load(f)
def main():
parser = argparse.ArgumentParser(description='__doc__')
parser.add_argument('oldrunid', help='Path to old run idir')
parser.add_argument('newrunid', help='Local to new run dir')
args = parser.parse_args()
compare_runs(args.oldrunid, args.newrunid)
if __name__ == '__main__':
main()
