mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-10-28 10:45:06 +08:00
Make plotting scripts runnable with uv
This commit is contained in:
David Peter authored and David Peter committed
1 parent
4a9e066b41
commit
ce49529780
10 files changed
+526
-434
No files matched your search
@@ -257,7 +257,16 @@ The JSON output is useful if you want to analyze the benchmark results in more d
|
||||
[`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot
|
||||
of helpful Python programs to further analyze benchmark results and create helpful
|
||||
visualizations, like a histogram of runtimes or a whisker plot to compare
|
||||
multiple benchmarks:
|
||||
multiple benchmarks.
|
||||
|
||||
For example, these two commands create the following plots:
|
||||
|
||||
```bash
|
||||
uvx --from 'git+https://github.com/sharkdp/hyperfine' \
|
||||
plot_histogram results.json
|
||||
uvx --from 'git+https://github.com/sharkdp/hyperfine' \
|
||||
plot_whisker results.json
|
||||
```
|
||||
|
||||
|  |  |
|
||||
|---:|---:|
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
[project]
|
||||
name = "hyperfine-scripts"
|
||||
version = "0.1.0"
|
||||
description = "Plot and analyze hyperfine JSON benchmark results"
|
||||
requires-python = ">=3.10"
|
||||
license-files = ["LICENSE-APACHE", "LICENSE-MIT"]
|
||||
dependencies = [
|
||||
"matplotlib",
|
||||
"numpy",
|
||||
"pyqt6",
|
||||
"scipy",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
advanced_statistics = "advanced_statistics:main"
|
||||
plot_benchmark_comparison = "plot_benchmark_comparison:main"
|
||||
plot_histogram = "plot_histogram:main"
|
||||
plot_parametrized = "plot_parametrized:main"
|
||||
plot_progression = "plot_progression:main"
|
||||
plot_whisker = "plot_whisker:main"
|
||||
welch_ttest = "welch_ttest:main"
|
||||
|
||||
[tool.uv.build-backend]
|
||||
module-root = "scripts"
|
||||
module-name = ""
|
||||
namespace = true
|
||||
source-exclude = [
|
||||
"/scripts/README.md",
|
||||
"/scripts/ruff.toml",
|
||||
"/scripts/.*",
|
||||
"*.egg-info",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["uv_build>=0.12.23,<0.13"]
|
||||
build-backend = "uv_build"
|
||||
+23
-13
@@ -1,23 +1,33 @@
|
||||
This folder contains scripts that can be used in combination with hyperfine's `--export-json` option.
|
||||
|
||||
### Example:
|
||||
### Run directly from GitHub
|
||||
|
||||
Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/), then
|
||||
run one of the scripts:
|
||||
|
||||
```bash
|
||||
hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
|
||||
uv run plot_whisker.py sleep.json
|
||||
uvx --from 'git+https://github.com/sharkdp/hyperfine' \
|
||||
plot_whisker sleep.json
|
||||
```
|
||||
|
||||
All plotting and analysis scripts use the JSON export's `primary_metric` by default.
|
||||
Replace `plot_whisker` with any of the following commands.
|
||||
|
||||
| Command | Description |
|
||||
| --- | --- |
|
||||
| `plot_whisker` | Compare benchmarks with a box and whisker plot |
|
||||
| `plot_histogram` | Show the distribution of measurements |
|
||||
| `plot_parametrized` | Plot results from a parameter scan |
|
||||
| `plot_progression` | Show measurements in run order |
|
||||
| `plot_benchmark_comparison` | Compare commands across multiple JSON exports |
|
||||
| `advanced_statistics` | Print additional statistics and percentiles |
|
||||
| `welch_ttest` | Compare two benchmarks using Welch's t-test |
|
||||
|
||||
By default, the plotting and analysis scripts use the metric that was selected
|
||||
first in the actual hyperfine run (`--metrics` option).
|
||||
Use `--metric` to select another metric:
|
||||
|
||||
```bash
|
||||
uv run plot_histogram.py benchmark.json --metric instructions
|
||||
uvx --from 'git+https://github.com/sharkdp/hyperfine' \
|
||||
plot_histogram benchmark.json --metric instructions
|
||||
```
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the
|
||||
commands above from this directory.
|
||||
|
||||
The scripts declare their dependencies using [PEP 723](https://peps.python.org/pep-0723/)
|
||||
inline script metadata. `uv run` automatically installs these dependencies in an
|
||||
isolated environment before running the script.
|
||||
@@ -12,51 +12,55 @@ import numpy as np
|
||||
|
||||
from plot_utils import METRICS, add_metric_argument, load_results
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument(
|
||||
"--time-unit",
|
||||
help="Display unit for time metrics (default: second).",
|
||||
choices=["second", "millisecond"],
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
description = METRICS[metric]
|
||||
unit, scale = description.unit, description.scale
|
||||
if args.time_unit is not None:
|
||||
if not metric.startswith("time_"):
|
||||
parser.error("--time-unit can only be used with a time metric")
|
||||
if args.time_unit == "millisecond":
|
||||
unit, scale = "ms", 0.001
|
||||
|
||||
|
||||
def format_value(value):
|
||||
suffix = f" {unit}" if unit else ""
|
||||
return f"{value:.3f}{suffix}"
|
||||
|
||||
|
||||
print(f"Metric: {description.label}\n")
|
||||
for result in results:
|
||||
summary = result["summary"][metric]
|
||||
values = [m[metric]["value"] / scale for m in result["measurements"]]
|
||||
|
||||
p05, p25, p75, p95 = np.percentile(values, [5, 25, 75, 95])
|
||||
|
||||
iqr = p75 - p25
|
||||
|
||||
print(f"Command '{result.get('name', result['command'])}'")
|
||||
print(f" runs: {summary['count']:8d}")
|
||||
for statistic in ["mean", "stddev", "median", "min", "max"]:
|
||||
value = summary[statistic]
|
||||
formatted = "N/A" if value is None else format_value(value / scale)
|
||||
print(f" {statistic + ':':<8}{formatted:>8}")
|
||||
print()
|
||||
print(" percentiles:")
|
||||
print(f" P_05 .. P_95: {format_value(p05)} .. {format_value(p95)}")
|
||||
print(
|
||||
f" P_25 .. P_75: {format_value(p25)} .. {format_value(p75)}"
|
||||
f" (IQR = {format_value(iqr)})"
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument(
|
||||
"--time-unit",
|
||||
help="Display unit for time metrics (default: second).",
|
||||
choices=["second", "millisecond"],
|
||||
)
|
||||
print()
|
||||
args = parser.parse_args()
|
||||
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
description = METRICS[metric]
|
||||
unit, scale = description.unit, description.scale
|
||||
if args.time_unit is not None:
|
||||
if not metric.startswith("time_"):
|
||||
parser.error("--time-unit can only be used with a time metric")
|
||||
if args.time_unit == "millisecond":
|
||||
unit, scale = "ms", 0.001
|
||||
|
||||
def format_value(value):
|
||||
suffix = f" {unit}" if unit else ""
|
||||
return f"{value:.3f}{suffix}"
|
||||
|
||||
print(f"Metric: {description.label}\n")
|
||||
for result in results:
|
||||
summary = result["summary"][metric]
|
||||
values = [m[metric]["value"] / scale for m in result["measurements"]]
|
||||
|
||||
p05, p25, p75, p95 = np.percentile(values, [5, 25, 75, 95])
|
||||
|
||||
iqr = p75 - p25
|
||||
|
||||
print(f"Command '{result.get('name', result['command'])}'")
|
||||
print(f" runs: {summary['count']:8d}")
|
||||
for statistic in ["mean", "stddev", "median", "min", "max"]:
|
||||
value = summary[statistic]
|
||||
formatted = "N/A" if value is None else format_value(value / scale)
|
||||
print(f" {statistic + ':':<8}{formatted:>8}")
|
||||
print()
|
||||
print(" percentiles:")
|
||||
print(f" P_05 .. P_95: {format_value(p05)} .. {format_value(p95)}")
|
||||
print(
|
||||
f" P_25 .. P_75: {format_value(p25)} .. {format_value(p75)}"
|
||||
f" (IQR = {format_value(iqr)})"
|
||||
)
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -21,65 +21,71 @@ import numpy as np
|
||||
|
||||
from plot_utils import METRICS, add_metric_argument, load_results
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument(
|
||||
"files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
|
||||
)
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument(
|
||||
"--benchmark-names", nargs="+", help="Names of the benchmark groups"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename")
|
||||
|
||||
args = parser.parse_args()
|
||||
metric, datasets = load_results(parser, args.files, args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
|
||||
commands = None
|
||||
data = []
|
||||
inputs = []
|
||||
|
||||
if args.benchmark_names:
|
||||
assert len(args.files) == len(args.benchmark_names), (
|
||||
"Number of benchmark names must match the number of input files."
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument(
|
||||
"files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
|
||||
)
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument(
|
||||
"--benchmark-names", nargs="+", help="Names of the benchmark groups"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename")
|
||||
|
||||
args = parser.parse_args()
|
||||
metric, datasets = load_results(parser, args.files, args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
|
||||
commands = None
|
||||
data = []
|
||||
inputs = []
|
||||
|
||||
for i, (filename, results) in enumerate(zip(args.files, datasets)):
|
||||
benchmark_commands = [b.get("name", b["command"]) for b in results]
|
||||
if commands is None:
|
||||
commands = benchmark_commands
|
||||
else:
|
||||
assert commands == benchmark_commands, (
|
||||
f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
|
||||
)
|
||||
data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results])
|
||||
if args.benchmark_names:
|
||||
inputs.append(args.benchmark_names[i])
|
||||
assert len(args.files) == len(args.benchmark_names), (
|
||||
"Number of benchmark names must match the number of input files."
|
||||
)
|
||||
|
||||
for i, (filename, results) in enumerate(zip(args.files, datasets)):
|
||||
benchmark_commands = [b.get("name", b["command"]) for b in results]
|
||||
if commands is None:
|
||||
commands = benchmark_commands
|
||||
else:
|
||||
assert commands == benchmark_commands, (
|
||||
f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
|
||||
)
|
||||
data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results])
|
||||
if args.benchmark_names:
|
||||
inputs.append(args.benchmark_names[i])
|
||||
else:
|
||||
inputs.append(filename.stem)
|
||||
|
||||
data = np.transpose(data)
|
||||
x = np.arange(len(inputs)) # the label locations
|
||||
width = 0.25 # the width of the bars
|
||||
|
||||
fig, ax = plt.subplots(layout="constrained")
|
||||
fig.set_figheight(5)
|
||||
fig.set_figwidth(10)
|
||||
for i, command in enumerate(commands):
|
||||
offset = width * (i + 1)
|
||||
ax.bar(x + offset, data[i], width, label=command)
|
||||
|
||||
ax.set_xticks(x + 0.5, inputs)
|
||||
ax.grid(visible=True, axis="y")
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
plt.xlabel("Benchmark")
|
||||
plt.ylabel(metric_info.axis_label)
|
||||
plt.legend(title="Command")
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
inputs.append(filename.stem)
|
||||
plt.show()
|
||||
|
||||
data = np.transpose(data)
|
||||
x = np.arange(len(inputs)) # the label locations
|
||||
width = 0.25 # the width of the bars
|
||||
|
||||
fig, ax = plt.subplots(layout="constrained")
|
||||
fig.set_figheight(5)
|
||||
fig.set_figwidth(10)
|
||||
for i, command in enumerate(commands):
|
||||
offset = width * (i + 1)
|
||||
rects = ax.bar(x + offset, data[i], width, label=command)
|
||||
|
||||
ax.set_xticks(x + 0.5, inputs)
|
||||
ax.grid(visible=True, axis="y")
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
plt.xlabel("Benchmark")
|
||||
plt.ylabel(metric_info.axis_label)
|
||||
plt.legend(title="Command")
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+104
-97
@@ -17,108 +17,115 @@ import numpy as np
|
||||
|
||||
from plot_utils import METRICS, add_metric_argument, load_results
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
parser.add_argument("--title", help="Plot title")
|
||||
parser.add_argument(
|
||||
"--labels", help="Comma-separated list of entries for the plot legend"
|
||||
)
|
||||
parser.add_argument("--bins", help="Number of bins (default: auto)")
|
||||
parser.add_argument(
|
||||
"--legend-location",
|
||||
help="Location of the legend on plot (default: upper center)",
|
||||
choices=[
|
||||
"upper center",
|
||||
"lower center",
|
||||
"right",
|
||||
"left",
|
||||
"best",
|
||||
"upper left",
|
||||
"upper right",
|
||||
"lower left",
|
||||
"lower right",
|
||||
"center left",
|
||||
"center right",
|
||||
"center",
|
||||
],
|
||||
default="upper center",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--type", help="Type of histogram (*bar*, barstacked, step, stepfilled)"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
parser.add_argument(
|
||||
"--min",
|
||||
"--t-min",
|
||||
dest="value_min",
|
||||
type=float,
|
||||
help="Minimum metric value to display",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max",
|
||||
"--t-max",
|
||||
dest="value_max",
|
||||
type=float,
|
||||
help="Maximum metric value to display",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-count",
|
||||
help="Use a logarithmic y-axis for the event count",
|
||||
action="store_true",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
parser.add_argument("--title", help="Plot title")
|
||||
parser.add_argument(
|
||||
"--labels", help="Comma-separated list of entries for the plot legend"
|
||||
)
|
||||
parser.add_argument("--bins", help="Number of bins (default: auto)")
|
||||
parser.add_argument(
|
||||
"--legend-location",
|
||||
help="Location of the legend on plot (default: upper center)",
|
||||
choices=[
|
||||
"upper center",
|
||||
"lower center",
|
||||
"right",
|
||||
"left",
|
||||
"best",
|
||||
"upper left",
|
||||
"upper right",
|
||||
"lower left",
|
||||
"lower right",
|
||||
"center left",
|
||||
"center right",
|
||||
"center",
|
||||
],
|
||||
default="upper center",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--type", help="Type of histogram (*bar*, barstacked, step, stepfilled)"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
parser.add_argument(
|
||||
"--min",
|
||||
"--t-min",
|
||||
dest="value_min",
|
||||
type=float,
|
||||
help="Minimum metric value to display",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max",
|
||||
"--t-max",
|
||||
dest="value_max",
|
||||
type=float,
|
||||
help="Maximum metric value to display",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-count",
|
||||
help="Use a logarithmic y-axis for the event count",
|
||||
action="store_true",
|
||||
)
|
||||
|
||||
if args.labels:
|
||||
labels = args.labels.split(",")
|
||||
else:
|
||||
labels = [b.get("name", b["command"]) for b in results]
|
||||
all_values = [
|
||||
[m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results
|
||||
]
|
||||
args = parser.parse_args()
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
|
||||
value_min = (
|
||||
args.value_min
|
||||
if args.value_min is not None
|
||||
else np.min(list(map(np.min, all_values)))
|
||||
)
|
||||
value_max = (
|
||||
args.value_max
|
||||
if args.value_max is not None
|
||||
else np.max(list(map(np.max, all_values)))
|
||||
)
|
||||
if args.labels:
|
||||
labels = args.labels.split(",")
|
||||
else:
|
||||
labels = [b.get("name", b["command"]) for b in results]
|
||||
all_values = [
|
||||
[m[metric]["value"] / metric_info.scale for m in b["measurements"]]
|
||||
for b in results
|
||||
]
|
||||
|
||||
bins = int(args.bins) if args.bins else "auto"
|
||||
histtype = args.type if args.type else "bar"
|
||||
value_min = (
|
||||
args.value_min
|
||||
if args.value_min is not None
|
||||
else np.min(list(map(np.min, all_values)))
|
||||
)
|
||||
value_max = (
|
||||
args.value_max
|
||||
if args.value_max is not None
|
||||
else np.max(list(map(np.max, all_values)))
|
||||
)
|
||||
|
||||
plt.figure(figsize=(10, 5))
|
||||
plt.hist(
|
||||
all_values,
|
||||
label=labels,
|
||||
bins=bins,
|
||||
histtype=histtype,
|
||||
range=(value_min, value_max),
|
||||
)
|
||||
plt.legend(
|
||||
loc=args.legend_location,
|
||||
fancybox=True,
|
||||
shadow=True,
|
||||
prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]},
|
||||
)
|
||||
bins = int(args.bins) if args.bins else "auto"
|
||||
histtype = args.type if args.type else "bar"
|
||||
|
||||
plt.xlabel(metric_info.axis_label)
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
plt.figure(figsize=(10, 5))
|
||||
plt.hist(
|
||||
all_values,
|
||||
label=labels,
|
||||
bins=bins,
|
||||
histtype=histtype,
|
||||
range=(value_min, value_max),
|
||||
)
|
||||
plt.legend(
|
||||
loc=args.legend_location,
|
||||
fancybox=True,
|
||||
shadow=True,
|
||||
prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]},
|
||||
)
|
||||
|
||||
if args.log_count:
|
||||
plt.yscale("log")
|
||||
else:
|
||||
plt.ylim(0, None)
|
||||
plt.xlabel(metric_info.axis_label)
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output, dpi=600)
|
||||
else:
|
||||
plt.show()
|
||||
if args.log_count:
|
||||
plt.yscale("log")
|
||||
else:
|
||||
plt.ylim(0, None)
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output, dpi=600)
|
||||
else:
|
||||
plt.show()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -17,105 +17,107 @@ import matplotlib.pyplot as plt
|
||||
|
||||
from plot_utils import METRICS, add_metric_argument, load_results
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument("file", help="JSON file with benchmark results", nargs="+")
|
||||
parser.add_argument(
|
||||
"--parameter-name",
|
||||
metavar="name",
|
||||
type=str,
|
||||
help="Deprecated; parameter names are now inferred from benchmark files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-x", help="Use a logarithmic x (parameter) axis", action="store_true"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-y",
|
||||
"--log-time",
|
||||
dest="log_y",
|
||||
help="Use a logarithmic metric axis",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--titles", help="Comma-separated list of titles for the plot legend"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
|
||||
args = parser.parse_args()
|
||||
metric, datasets = load_results(parser, args.file, args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
if args.parameter_name is not None:
|
||||
sys.stderr.write(
|
||||
"warning: --parameter-name is deprecated; names are inferred from "
|
||||
"benchmark results\n"
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument("file", help="JSON file with benchmark results", nargs="+")
|
||||
parser.add_argument(
|
||||
"--parameter-name",
|
||||
metavar="name",
|
||||
type=str,
|
||||
help="Deprecated; parameter names are now inferred from benchmark files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-x", help="Use a logarithmic x (parameter) axis", action="store_true"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-y",
|
||||
"--log-time",
|
||||
dest="log_y",
|
||||
help="Use a logarithmic metric axis",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--titles", help="Comma-separated list of titles for the plot legend"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
|
||||
|
||||
def die(msg):
|
||||
sys.stderr.write(f"fatal: {msg}\n")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def extract_parameters(results):
|
||||
"""Return `(parameter_name: str, parameter_values: List[float])`."""
|
||||
if not results:
|
||||
die("no benchmark data to plot")
|
||||
(names, values) = zip(*(unique_parameter(b) for b in results))
|
||||
names = frozenset(names)
|
||||
if len(names) != 1:
|
||||
die(
|
||||
f"benchmarks must all have the same parameter name, but found: {sorted(names)}"
|
||||
args = parser.parse_args()
|
||||
metric, datasets = load_results(parser, args.file, args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
if args.parameter_name is not None:
|
||||
sys.stderr.write(
|
||||
"warning: --parameter-name is deprecated; names are inferred from "
|
||||
"benchmark results\n"
|
||||
)
|
||||
return (next(iter(names)), list(values))
|
||||
|
||||
def die(msg):
|
||||
sys.stderr.write(f"fatal: {msg}\n")
|
||||
sys.exit(1)
|
||||
|
||||
def extract_parameters(results):
|
||||
"""Return `(parameter_name: str, parameter_values: List[float])`."""
|
||||
if not results:
|
||||
die("no benchmark data to plot")
|
||||
(names, values) = zip(*(unique_parameter(b) for b in results))
|
||||
names = frozenset(names)
|
||||
if len(names) != 1:
|
||||
die(
|
||||
f"benchmarks must all have the same parameter name, but found: {sorted(names)}"
|
||||
)
|
||||
return (next(iter(names)), list(values))
|
||||
|
||||
def unique_parameter(benchmark):
|
||||
"""Return the unique parameter `(name: str, value: float)`, or die."""
|
||||
params_dict = benchmark.get("parameters", {})
|
||||
if not params_dict:
|
||||
die("benchmarks must have exactly one parameter, but found none")
|
||||
if len(params_dict) > 1:
|
||||
die(
|
||||
f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}"
|
||||
)
|
||||
[(name, value)] = params_dict.items()
|
||||
return (name, float(value["value"]))
|
||||
|
||||
parameter_name = None
|
||||
|
||||
for results in datasets:
|
||||
(this_parameter_name, parameter_values) = extract_parameters(results)
|
||||
if parameter_name is not None and this_parameter_name != parameter_name:
|
||||
die(
|
||||
f"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}"
|
||||
)
|
||||
parameter_name = this_parameter_name
|
||||
|
||||
summaries = [b["summary"][metric] for b in results]
|
||||
means = [s["mean"] / metric_info.scale for s in summaries]
|
||||
stddevs = [
|
||||
s["stddev"] / metric_info.scale if s["stddev"] is not None else float("nan")
|
||||
for s in summaries
|
||||
]
|
||||
|
||||
plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2)
|
||||
|
||||
plt.xlabel(parameter_name)
|
||||
plt.ylabel(metric_info.axis_label)
|
||||
|
||||
if args.log_y:
|
||||
plt.yscale("log")
|
||||
else:
|
||||
plt.ylim(0, None)
|
||||
|
||||
if args.log_x:
|
||||
plt.xscale("log")
|
||||
|
||||
if args.titles:
|
||||
plt.legend(args.titles.split(","))
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
|
||||
|
||||
def unique_parameter(benchmark):
|
||||
"""Return the unique parameter `(name: str, value: float)`, or die."""
|
||||
params_dict = benchmark.get("parameters", {})
|
||||
if not params_dict:
|
||||
die("benchmarks must have exactly one parameter, but found none")
|
||||
if len(params_dict) > 1:
|
||||
die(
|
||||
f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}"
|
||||
)
|
||||
[(name, value)] = params_dict.items()
|
||||
return (name, float(value["value"]))
|
||||
|
||||
|
||||
parameter_name = None
|
||||
|
||||
for results in datasets:
|
||||
(this_parameter_name, parameter_values) = extract_parameters(results)
|
||||
if parameter_name is not None and this_parameter_name != parameter_name:
|
||||
die(
|
||||
f"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}"
|
||||
)
|
||||
parameter_name = this_parameter_name
|
||||
|
||||
summaries = [b["summary"][metric] for b in results]
|
||||
means = [s["mean"] / metric_info.scale for s in summaries]
|
||||
stddevs = [
|
||||
s["stddev"] / metric_info.scale if s["stddev"] is not None else float("nan")
|
||||
for s in summaries
|
||||
]
|
||||
|
||||
plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2)
|
||||
|
||||
plt.xlabel(parameter_name)
|
||||
plt.ylabel(metric_info.axis_label)
|
||||
|
||||
if args.log_y:
|
||||
plt.yscale("log")
|
||||
else:
|
||||
plt.ylim(0, None)
|
||||
|
||||
if args.log_x:
|
||||
plt.xscale("log")
|
||||
|
||||
if args.titles:
|
||||
plt.legend(args.titles.split(","))
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+62
-57
@@ -29,62 +29,67 @@ def moving_average(values, num_runs):
|
||||
return np.convolve(values_padded, kernel, mode="valid")
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
parser.add_argument(
|
||||
"-w",
|
||||
"--moving-average-width",
|
||||
type=int,
|
||||
metavar="num_runs",
|
||||
help="Width of the moving-average window (default: N/5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-moving-average",
|
||||
action="store_true",
|
||||
help="Do not show moving average curve",
|
||||
)
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
parser.add_argument(
|
||||
"-w",
|
||||
"--moving-average-width",
|
||||
type=int,
|
||||
metavar="num_runs",
|
||||
help="Width of the moving-average window (default: N/5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-moving-average",
|
||||
action="store_true",
|
||||
help="Do not show moving average curve",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
|
||||
for result in results:
|
||||
values = [
|
||||
m[metric]["value"] / metric_info.scale for m in result["measurements"]
|
||||
]
|
||||
num = len(values)
|
||||
nums = range(num)
|
||||
|
||||
plt.scatter(x=nums, y=values, marker=".")
|
||||
plt.ylim([0, None])
|
||||
plt.xlim([-1, num])
|
||||
|
||||
if not args.no_moving_average:
|
||||
moving_average_width = (
|
||||
max(1, num // 5)
|
||||
if args.moving_average_width is None
|
||||
else args.moving_average_width
|
||||
)
|
||||
|
||||
average = moving_average(values, moving_average_width)
|
||||
plt.plot(nums, average, "-")
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
|
||||
legend = []
|
||||
for result in results:
|
||||
legend.append(result.get("name", result["command"]))
|
||||
if not args.no_moving_average:
|
||||
legend.append("moving average")
|
||||
plt.legend(legend)
|
||||
|
||||
plt.ylabel(metric_info.axis_label)
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
|
||||
for result in results:
|
||||
label = result.get("name", result["command"])
|
||||
values = [m[metric]["value"] / metric_info.scale for m in result["measurements"]]
|
||||
num = len(values)
|
||||
nums = range(num)
|
||||
|
||||
plt.scatter(x=nums, y=values, marker=".")
|
||||
plt.ylim([0, None])
|
||||
plt.xlim([-1, num])
|
||||
|
||||
if not args.no_moving_average:
|
||||
moving_average_width = (
|
||||
max(1, num // 5)
|
||||
if args.moving_average_width is None
|
||||
else args.moving_average_width
|
||||
)
|
||||
|
||||
average = moving_average(values, moving_average_width)
|
||||
plt.plot(nums, average, "-")
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
|
||||
legend = []
|
||||
for result in results:
|
||||
legend.append(result.get("name", result["command"]))
|
||||
if not args.no_moving_average:
|
||||
legend.append("moving average")
|
||||
plt.legend(legend)
|
||||
|
||||
plt.ylabel(metric_info.axis_label)
|
||||
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+47
-40
@@ -21,49 +21,56 @@ import matplotlib.pyplot as plt
|
||||
|
||||
from plot_utils import METRICS, add_metric_argument, load_results
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument("--sort-by", choices=["median"], help="Sort method")
|
||||
parser.add_argument(
|
||||
"--labels", help="Comma-separated list of entries for the plot legend"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
|
||||
args = parser.parse_args()
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
add_metric_argument(parser)
|
||||
parser.add_argument("file", help="JSON file with benchmark results")
|
||||
parser.add_argument("--title", help="Plot Title")
|
||||
parser.add_argument("--sort-by", choices=["median"], help="Sort method")
|
||||
parser.add_argument(
|
||||
"--labels", help="Comma-separated list of entries for the plot legend"
|
||||
)
|
||||
parser.add_argument("-o", "--output", help="Save image to the given filename.")
|
||||
|
||||
if args.labels:
|
||||
labels = args.labels.split(",")
|
||||
else:
|
||||
labels = [b.get("name", b["command"]) for b in results]
|
||||
values = [
|
||||
[m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results
|
||||
]
|
||||
args = parser.parse_args()
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
metric_info = METRICS[metric]
|
||||
|
||||
if args.sort_by == "median":
|
||||
medians = [b["summary"][metric]["median"] for b in results]
|
||||
indices = sorted(range(len(labels)), key=lambda k: medians[k])
|
||||
labels = [labels[i] for i in indices]
|
||||
values = [values[i] for i in indices]
|
||||
if args.labels:
|
||||
labels = args.labels.split(",")
|
||||
else:
|
||||
labels = [b.get("name", b["command"]) for b in results]
|
||||
values = [
|
||||
[m[metric]["value"] / metric_info.scale for m in b["measurements"]]
|
||||
for b in results
|
||||
]
|
||||
|
||||
plt.figure(figsize=(10, 6), constrained_layout=True)
|
||||
boxplot = plt.boxplot(values, vert=True, patch_artist=True)
|
||||
cmap = plt.get_cmap("rainbow")
|
||||
colors = [cmap(val / len(values)) for val in range(len(values))]
|
||||
if args.sort_by == "median":
|
||||
medians = [b["summary"][metric]["median"] for b in results]
|
||||
indices = sorted(range(len(labels)), key=lambda k: medians[k])
|
||||
labels = [labels[i] for i in indices]
|
||||
values = [values[i] for i in indices]
|
||||
|
||||
for patch, color in zip(boxplot["boxes"], colors):
|
||||
patch.set_facecolor(color)
|
||||
plt.figure(figsize=(10, 6), constrained_layout=True)
|
||||
boxplot = plt.boxplot(values, vert=True, patch_artist=True)
|
||||
cmap = plt.get_cmap("rainbow")
|
||||
colors = [cmap(val / len(values)) for val in range(len(values))]
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium")
|
||||
plt.ylabel(metric_info.axis_label)
|
||||
plt.ylim(0, None)
|
||||
plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45)
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
for patch, color in zip(boxplot["boxes"], colors):
|
||||
patch.set_facecolor(color)
|
||||
|
||||
if args.title:
|
||||
plt.title(args.title)
|
||||
plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium")
|
||||
plt.ylabel(metric_info.axis_label)
|
||||
plt.ylim(0, None)
|
||||
plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45)
|
||||
if args.output:
|
||||
plt.savefig(args.output)
|
||||
else:
|
||||
plt.show()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+35
-29
@@ -16,38 +16,44 @@ from scipy import stats
|
||||
|
||||
from plot_utils import METRICS, add_metric_argument, load_results
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("file", help="JSON file with two benchmark results")
|
||||
add_metric_argument(parser)
|
||||
args = parser.parse_args()
|
||||
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("file", help="JSON file with two benchmark results")
|
||||
add_metric_argument(parser)
|
||||
args = parser.parse_args()
|
||||
|
||||
if len(results) != 2:
|
||||
parser.error("the input file must contain exactly two benchmarks")
|
||||
metric, [results] = load_results(parser, [args.file], args.metric)
|
||||
|
||||
a, b = (x.get("name", x["command"]) for x in results[:2])
|
||||
X, Y = ([m[metric]["value"] for m in x["measurements"]] for x in results[:2])
|
||||
if min(len(X), len(Y)) < 2:
|
||||
parser.error("Welch's t-test requires at least two measurements per benchmark")
|
||||
if len(results) != 2:
|
||||
parser.error("the input file must contain exactly two benchmarks")
|
||||
|
||||
print(f"Metric: {METRICS[metric].label}")
|
||||
print(f"Command 1: {a}")
|
||||
print(f"Command 2: {b}\n")
|
||||
a, b = (x.get("name", x["command"]) for x in results[:2])
|
||||
X, Y = ([m[metric]["value"] for m in x["measurements"]] for x in results[:2])
|
||||
if min(len(X), len(Y)) < 2:
|
||||
parser.error("Welch's t-test requires at least two measurements per benchmark")
|
||||
|
||||
t, p = stats.ttest_ind(X, Y, equal_var=False)
|
||||
if not math.isfinite(p):
|
||||
parser.error("Welch's t-test is undefined for these measurements")
|
||||
th = 0.05
|
||||
dispose = p < th
|
||||
print(f"t = {t:.3}, p = {p:.3}")
|
||||
print()
|
||||
print(f"Metric: {METRICS[metric].label}")
|
||||
print(f"Command 1: {a}")
|
||||
print(f"Command 2: {b}\n")
|
||||
|
||||
if dispose:
|
||||
print(
|
||||
f"A statistically significant difference in mean value was detected (p < {th})."
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"No statistically significant difference in mean value was detected (p >= {th})."
|
||||
)
|
||||
t, p = stats.ttest_ind(X, Y, equal_var=False)
|
||||
if not math.isfinite(p):
|
||||
parser.error("Welch's t-test is undefined for these measurements")
|
||||
th = 0.05
|
||||
dispose = p < th
|
||||
print(f"t = {t:.3}, p = {p:.3}")
|
||||
print()
|
||||
|
||||
if dispose:
|
||||
print(
|
||||
f"A statistically significant difference in mean value was detected (p < {th})."
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"No statistically significant difference in mean value was detected (p >= {th})."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user