basic-benchmark: portable Fedora benchmark kit
Headless + headed suites with power/thermal logging and efficiency reporting. Includes PassMark, llama.cpp, app workloads (LibreOffice/GEGL/Inkscape/GIMP), a GOVERNOR selector, compare-report.py, and captured results.
This commit is contained in:
smill
commit
0cd63d0233
533 changed files with 43679 additions and 0 deletions
309
compare-report.py
Normal file
309
compare-report.py
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
#!/usr/bin/env python3
|
||||
"""basic-benchmark: compare benchmark results across devices and before/after.
|
||||
|
||||
python3 compare-report.py # every tag found in results/
|
||||
python3 compare-report.py --tags t14-1 server-1-headless
|
||||
python3 compare-report.py --label t14-1="ThinkPad T14" --label t14-1=...
|
||||
python3 compare-report.py --no-graphs # table only
|
||||
|
||||
Primary source is results/scores-<tag>.csv. For metrics that predate the score
|
||||
file (older runs), it falls back to the raw results:
|
||||
fio-<job>-<tag>.json openssl-aes/sha-<tag>.txt 7z-<tag>.txt
|
||||
sysbench-mem-<tag>.txt glmark2-<tag>.txt vkmark-<tag>.txt
|
||||
|
||||
Writes results/compare-scores.csv and one bar chart per benchmark to
|
||||
results/graphs/compare-<metric>.png, then prints the table.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
RES = os.path.join(HERE, "results")
|
||||
GRAPHS = os.path.join(RES, "graphs")
|
||||
|
||||
|
||||
def fnum(v):
|
||||
try:
|
||||
return float(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def read_text(path):
|
||||
try:
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
return fh.read()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def fmt(v):
|
||||
if v is None:
|
||||
return "-"
|
||||
if abs(v) >= 10000:
|
||||
return f"{v:,.0f}"
|
||||
if abs(v) >= 100:
|
||||
return f"{v:,.1f}"
|
||||
if abs(v) >= 1:
|
||||
return f"{v:.2f}"
|
||||
return f"{v:.3f}"
|
||||
|
||||
|
||||
# --- raw-file fallbacks -------------------------------------------------------
|
||||
def raw_7zip(tag):
|
||||
txt = read_text(os.path.join(RES, f"7z-{tag}.txt"))
|
||||
if not txt:
|
||||
return None
|
||||
for line in txt.splitlines():
|
||||
if line.startswith("Avr:"):
|
||||
parts = line.split()
|
||||
if len(parts) > 4:
|
||||
return fnum(parts[4])
|
||||
return None
|
||||
|
||||
|
||||
def _openssl(tag, kind):
|
||||
txt = read_text(os.path.join(RES, f"openssl-{kind}-{tag}.txt"))
|
||||
if not txt:
|
||||
return None
|
||||
val = None
|
||||
for line in txt.splitlines():
|
||||
parts = line.split()
|
||||
if parts and parts[-1].endswith("k"):
|
||||
v = fnum(parts[-1][:-1])
|
||||
if v is not None:
|
||||
val = v / 1e6 # kbytes/s -> GB/s
|
||||
return val
|
||||
|
||||
|
||||
def raw_openssl_aes(tag):
|
||||
return _openssl(tag, "aes")
|
||||
|
||||
|
||||
def raw_openssl_sha(tag):
|
||||
return _openssl(tag, "sha")
|
||||
|
||||
|
||||
def raw_sysbench_mem(tag):
|
||||
txt = read_text(os.path.join(RES, f"sysbench-mem-{tag}.txt"))
|
||||
if not txt:
|
||||
return None
|
||||
m = re.search(r"\(([\d.]+) MiB/sec\)", txt)
|
||||
return fnum(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _fio(tag, job, rw, field):
|
||||
try:
|
||||
with open(os.path.join(RES, f"fio-{job}-{tag}.json")) as fh:
|
||||
data = json.load(fh)
|
||||
return fnum(data["jobs"][0].get(rw, {}).get(field))
|
||||
except (OSError, ValueError, KeyError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def raw_fio_seqread(tag):
|
||||
v = _fio(tag, "seqread", "read", "bw_bytes")
|
||||
return v / 1e6 if v is not None else None
|
||||
|
||||
|
||||
def raw_fio_seqwrite(tag):
|
||||
v = _fio(tag, "seqwrite", "write", "bw_bytes")
|
||||
return v / 1e6 if v is not None else None
|
||||
|
||||
|
||||
def raw_fio_randread(tag):
|
||||
return _fio(tag, "randread", "read", "iops")
|
||||
|
||||
|
||||
def raw_fio_randwrite(tag):
|
||||
return _fio(tag, "randwrite", "write", "iops")
|
||||
|
||||
|
||||
def _gpu_score(tag, tool):
|
||||
txt = read_text(os.path.join(RES, f"{tool}-{tag}.txt"))
|
||||
if not txt:
|
||||
return None
|
||||
m = re.search(rf"{tool} Score:\s*([\d.]+)", txt)
|
||||
return fnum(m.group(1)) if m else None
|
||||
|
||||
|
||||
def raw_glmark2(tag):
|
||||
return _gpu_score(tag, "glmark2")
|
||||
|
||||
|
||||
def raw_vkmark(tag):
|
||||
return _gpu_score(tag, "vkmark")
|
||||
|
||||
|
||||
# key, title, unit, higher-is-better, raw fallback
|
||||
METRICS = [
|
||||
("PassMark-cpu", "PassMark CPU Mark", "mark", True, None),
|
||||
("PassMark-cpu-single", "PassMark CPU Single", "mark", True, None),
|
||||
("PassMark-mem", "PassMark Memory Mark", "mark", True, None),
|
||||
("7zip", "7-Zip compression (all)", "MIPS", True, raw_7zip),
|
||||
("7zip-1t", "7-Zip compression (1 thread)", "MIPS", True, None),
|
||||
("openssl-aes", "OpenSSL AES-256-GCM", "GB/s", True, raw_openssl_aes),
|
||||
("openssl-sha", "OpenSSL SHA-256", "GB/s", True, raw_openssl_sha),
|
||||
("sysbench-1t", "sysbench CPU (1 thread)", "events/s", True, None),
|
||||
("sysbench-nt", "sysbench CPU (all threads)", "events/s", True, None),
|
||||
("sysbench-mem", "sysbench memory", "MiB/s", True, raw_sysbench_mem),
|
||||
("llama-tg", "llama.cpp generation", "tok/s", True, None),
|
||||
("libreoffice", "LibreOffice documents->PDF", "docs/s", True, None),
|
||||
("gegl", "GEGL image operations", "ops/s", True, None),
|
||||
("inkscape", "Inkscape SVG->PNG", "img/s", True, None),
|
||||
("gimp", "GIMP batch operations", "ops/s", True, None),
|
||||
("fio-seqread", "fio sequential read", "MB/s", True, raw_fio_seqread),
|
||||
("fio-seqwrite", "fio sequential write", "MB/s", True, raw_fio_seqwrite),
|
||||
("fio-randread", "fio random read 4k", "IOPS", True, raw_fio_randread),
|
||||
("fio-randwrite", "fio random write 4k", "IOPS", True, raw_fio_randwrite),
|
||||
("glmark2", "glmark2", "score", True, raw_glmark2),
|
||||
("vkmark", "vkmark", "score", True, raw_vkmark),
|
||||
]
|
||||
|
||||
RAW_TAG_GLOBS = [
|
||||
("7z-*.txt", "7z-", ".txt"),
|
||||
("openssl-aes-*.txt", "openssl-aes-", ".txt"),
|
||||
("sysbench-mem-*.txt", "sysbench-mem-", ".txt"),
|
||||
("fio-seqread-*.json", "fio-seqread-", ".json"),
|
||||
("glmark2-*.txt", "glmark2-", ".txt"),
|
||||
("vkmark-*.txt", "vkmark-", ".txt"),
|
||||
]
|
||||
|
||||
|
||||
def discover_tags():
|
||||
tags = set()
|
||||
for path in glob.glob(os.path.join(RES, "scores-*.csv")):
|
||||
tags.add(os.path.basename(path)[len("scores-"):-len(".csv")])
|
||||
for path in glob.glob(os.path.join(RES, "env-*.txt")):
|
||||
tags.add(os.path.basename(path)[len("env-"):-len(".txt")])
|
||||
for pattern, prefix, suffix in RAW_TAG_GLOBS:
|
||||
for path in glob.glob(os.path.join(RES, pattern)):
|
||||
base = os.path.basename(path)
|
||||
if prefix == "7z-" and "-1t-" in base:
|
||||
continue # 7z-1t-<tag>.txt is a different metric
|
||||
tags.add(base[len(prefix):-len(suffix)])
|
||||
return {t for t in tags if t and "merged" not in t}
|
||||
|
||||
|
||||
def load_scores(tag):
|
||||
out = {}
|
||||
path = os.path.join(RES, f"scores-{tag}.csv")
|
||||
if os.path.exists(path):
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
parts = [p.strip() for p in line.strip().split(",")]
|
||||
if len(parts) >= 2 and fnum(parts[1]) is not None:
|
||||
out[parts[0]] = fnum(parts[1])
|
||||
return out
|
||||
|
||||
|
||||
def bar_chart(key, title, unit, tags, values, color):
|
||||
try:
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
return None
|
||||
present = [(t, v) for t, v in zip(tags, values) if v is not None]
|
||||
names = [p[0] for p in present]
|
||||
vals = [p[1] for p in present]
|
||||
colors = [color[p[0]] for p in present]
|
||||
fig, ax = plt.subplots(figsize=(max(6.0, 1.1 * len(names) + 2.0), 4.5))
|
||||
bars = ax.bar(names, vals, color=colors)
|
||||
for b, v in zip(bars, vals):
|
||||
ax.annotate(fmt(v), (b.get_x() + b.get_width() / 2, b.get_height()),
|
||||
ha="center", va="bottom", fontsize=8)
|
||||
ax.set_title(f"{title} ({unit})")
|
||||
ax.set_ylabel(unit)
|
||||
ax.tick_params(axis="x", labelrotation=30)
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
for lbl in ax.get_xticklabels():
|
||||
lbl.set_horizontalalignment("right")
|
||||
fig.tight_layout()
|
||||
os.makedirs(GRAPHS, exist_ok=True)
|
||||
out = os.path.join(GRAPHS, f"compare-{key}.png")
|
||||
fig.savefig(out, dpi=110)
|
||||
plt.close(fig)
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--tags", nargs="*", help="tags in display order (default: all found)")
|
||||
ap.add_argument("--label", action="append", default=[], metavar="TAG=NAME",
|
||||
help="friendly name for a tag (repeatable)")
|
||||
ap.add_argument("--no-graphs", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
labels = {}
|
||||
for spec in args.label:
|
||||
if "=" in spec:
|
||||
k, v = spec.split("=", 1)
|
||||
labels[k] = v
|
||||
|
||||
def label(t):
|
||||
return labels.get(t, t)
|
||||
|
||||
tags = args.tags or sorted(discover_tags())
|
||||
if not tags:
|
||||
raise SystemExit(f"no results found in {RES}")
|
||||
|
||||
scoremaps = {t: load_scores(t) for t in tags}
|
||||
|
||||
palette = ("#c0392b", "#2980b9", "#27ae60", "#8e44ad", "#d35400",
|
||||
"#16a085", "#2c3e50", "#c2185b", "#7f8c8d", "#f39c12")
|
||||
color = {t: palette[i % len(palette)] for i, t in enumerate(tags)}
|
||||
|
||||
rows = []
|
||||
for key, title, unit, higher, raw in METRICS:
|
||||
vals = []
|
||||
for t in tags:
|
||||
v = scoremaps[t].get(key)
|
||||
if v is None and raw is not None:
|
||||
v = raw(t)
|
||||
vals.append(v)
|
||||
if any(v is not None for v in vals):
|
||||
rows.append((key, title, unit, higher, vals))
|
||||
|
||||
if not rows:
|
||||
raise SystemExit("no benchmark values found")
|
||||
|
||||
name_w = max([len("benchmark")] + [len(r[1]) for r in rows])
|
||||
tag_w = [max(len(label(t)), 9) for t in tags]
|
||||
unit_w = max([len("unit")] + [len(r[2]) for r in rows])
|
||||
|
||||
header = f"{'benchmark':<{name_w}} {'unit':<{unit_w}} " + \
|
||||
" ".join(f"{label(t):>{tag_w[i]}}" for i, t in enumerate(tags))
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for key, title, unit, higher, vals in rows:
|
||||
cells = " ".join(f"{fmt(v):>{tag_w[i]}}" for i, v in enumerate(vals))
|
||||
print(f"{title:<{name_w}} {unit:<{unit_w}} {cells}")
|
||||
|
||||
csv_path = os.path.join(RES, "compare-scores.csv")
|
||||
with open(csv_path, "w", newline="") as fh:
|
||||
w = csv.writer(fh)
|
||||
w.writerow(["benchmark", "unit"] + [label(t) for t in tags])
|
||||
for key, title, unit, higher, vals in rows:
|
||||
w.writerow([title, unit] + ["" if v is None else f"{v:g}" for v in vals])
|
||||
print(f"\ntable: {csv_path}")
|
||||
|
||||
if args.no_graphs:
|
||||
return
|
||||
graphs = []
|
||||
for key, title, unit, higher, vals in rows:
|
||||
out = bar_chart(key, title, unit, tags, vals, color)
|
||||
if out:
|
||||
graphs.append(out)
|
||||
if graphs:
|
||||
print(f"graphs: {len(graphs)} -> {GRAPHS}/compare-*.png")
|
||||
else:
|
||||
print("graphs: skipped (matplotlib not installed)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in a new issue