basic-benchmark/run-benchmarks.sh

#!/usr/bin/env bash
#
# basic-benchmark / run-benchmarks.sh
# Run the SAME benchmark set before and after the CPU/platform upgrade.
#
# The suite is split in two parts so it can be driven from another device:
#
#   ./run-benchmarks.sh before headless    # no display needed -> run over SSH
#   ./run-benchmarks.sh before headed      # needs the KDE session -> run locally
#   ./run-benchmarks.sh before             # legacy: headless + headed (if display)
#
# The part is appended to the tag so the two parts never overwrite each other:
#
#   before headless -> results/{power,phases}-before-headless.csv
#   before headed   -> results/{power,phases}-before-headed.csv
#
# Compare:
#   python3 power-report.py --tags before-headless after-headless
#   python3 power-report.py --tags before-headless before-headed after-headless after-headed
#
# Primary cross-platform score: PassMark PerformanceTest (cpubenchmark.net).
# GPU tests use glmark2/vkmark.
#
# Env options:
#   INSTALL_DEPS=1         auto-install missing tools via sudo dnf (fresh Fedora)
#                          default is 'ask' (prompts); 0 disables
#   RUN_GPU=0              skip the headed GPU tests in 'all' mode
#   TOOLS=/path            where PassMark lives (default ./tools)
#   FIOFILE=/path          fio scratch file (must be on the target filesystem)
#   FIO_SIZE=1G            fio file size
#   FIO_RUNTIME=30         seconds per fio job
#   GPU_PCI=0000:03:00.0   pin GPU tests to this PCI device  (auto-detected)
#   GPU_VD=0x1002:0x73df   vendor:device hex for Vulkan selection (auto-detected)
#   LLAMA_MODEL=/path      GGUF for the LLM test (default: bundled model)
#   LLAMA_NGL=0            GPU layers to offload (0 = CPU/RAM, the default)
#   RUN_APPS=1             app workloads: LibreOffice, GEGL, Inkscape, GIMP
#   CORPUS=/path           where the sample inputs are cached (default ./tools/corpus)
#   CORPUS_URL=...         base URL the sample corpora are fetched from
#   GOVERNOR=powersave     CPU governor to pin (default performance); an explicit
#                          value is appended to the tag, e.g. after-powersave-headless
#   GLMARK2_DURATION=10    seconds for the glmark2 scene (headed part)
#   VKMARK_DURATION=10     seconds for the vkmark scene (headed part)
#   VKMARK_BENCH=texture   vkmark scene to run (e.g. texture, shading, vertex)
#
# Run the headless part with a TTY so sudo can prompt for power logging:
#   ssh -t <target> '~/basic-benchmark/run-benchmarks.sh before headless'

set -u

# User-space tools (e.g. a source-built tool) live here; llama.cpp ships prebuilt.
export PATH="$HOME/.local/bin:$PATH"
# PassMark is a curses app; it needs a real terminal type even when piped. 'dumb'
# and unset both fail ("cannot initialize terminal type"), so force xterm then.
case "${TERM:-}" in ''|dumb|unknown) export TERM=xterm ;; esac

usage() {
  printf 'usage: %s <tag> [headless|headed|all]\n' "$(basename "$0")" >&2
  printf '  e.g. %s before headless   # over SSH, no display needed\n' "$(basename "$0")" >&2
  printf '       %s before headed     # at the machine, in a graphical session\n' "$(basename "$0")" >&2
  printf '       %s before            # both parts (all)\n' "$(basename "$0")" >&2
  printf '       %s doctor            # check readiness without benchmarking\n' "$(basename "$0")" >&2
}

TAG_BASE="${1:-}"
PART="${2:-all}"
DOCTOR=0
[[ "$TAG_BASE" == "doctor" || "$TAG_BASE" == "--check" ]] && DOCTOR=1

if [[ "$DOCTOR" == "0" ]]; then
  if [[ -z "$TAG_BASE" || "$TAG_BASE" == "-h" || "$TAG_BASE" == "--help" ]]; then
    usage
    [[ "$TAG_BASE" == "-h" || "$TAG_BASE" == "--help" ]] && exit 0
    exit 1
  fi
  case "$PART" in
    headless|headed|all) ;;
    *) printf 'bad part: %s\n' "$PART" >&2; usage; exit 1 ;;
  esac
fi

DIR="$(cd "$(dirname "$0")" && pwd)"
RES="$DIR/results"
mkdir -p "$RES"
cd "$RES" || exit 1

# CPU governor to pin for the timed runs (the original is restored on exit).
# Set GOVERNOR=... to choose one; an explicit choice is appended to the tag so
# several profiles never overwrite each other:
#   GOVERNOR=powersave ./run-benchmarks.sh after headless   -> after-powersave-headless
#   GOVERNOR=performance ./run-benchmarks.sh after headless -> after-performance-headless
#   ./run-benchmarks.sh after headless                      -> after-headless (performance)
GOVERNOR_SET=0
[[ -n "${GOVERNOR:-}" ]] && GOVERNOR_SET=1
GOVERNOR="${GOVERNOR:-performance}"

TAG="$TAG_BASE"
[[ "$GOVERNOR_SET" == "1" ]] && TAG="$TAG-$GOVERNOR"
[[ "$PART" != "all" ]] && TAG="$TAG-$PART"

RUN_GPU="${RUN_GPU:-1}"
TOOLS="${TOOLS:-$DIR/tools}"
if [[ -z "${PT:-}" ]]; then
  for cand in "$TOOLS/pt/pt_linux_x64" \
              "$TOOLS/pt/PerformanceTest/PerformanceTest_Linux_x86-64"; do
    [[ -x "$cand" ]] && { PT="$cand"; break; }
  done
fi
PT="${PT:-$TOOLS/pt/pt_linux_x64}"
# Bundled LLM bench (self-contained under tools/llama/): token generation on CPU/RAM.
LLAMA_DIR="$TOOLS/llama"
LLAMA_BIN="$LLAMA_DIR/bin/llama-bench"
LLAMA_MODEL="${LLAMA_MODEL:-$LLAMA_DIR/models/MiniCPM5-2B-Q8_0.gguf}"
LLAMA_NGL="${LLAMA_NGL:-0}"     # 0 = CPU/RAM only (no GPU offload)
LLAMA_TG="${LLAMA_TG:-128}"     # generated tokens
LLAMA_REPS="${LLAMA_REPS:-2}"
# App workloads: stock distro applications run over a
# fixed sample corpus, fetched once and checksum-pinned, then cached under
# CORPUS and reused by every run so both sides process identical input bytes.
RUN_APPS="${RUN_APPS:-1}"
CORPUS="${CORPUS:-$TOOLS/corpus}"
CORPUS_URL="${CORPUS_URL:-http://phoronix-test-suite.com/benchmark-files}"  # sample-file CDN
SAMPLER="$DIR/powerlog.py"
REPORT="$DIR/power-report.py"
PHASES="$RES/phases-$TAG.csv"
FIOFILE="${FIOFILE:-$HOME/.cache/basic-benchmark-fio.bin}"
FIO_SIZE="${FIO_SIZE:-1G}"
FIO_RUNTIME="${FIO_RUNTIME:-30}"

section() { printf '\n\033[1m=== %s ===\033[0m\n' "$*"; }
have()    { command -v "$1" >/dev/null 2>&1; }
phase()   { printf '%s,%s\n' "$(date +%s)" "$1" >> "$PHASES"; }

# --- dependency provisioning (Fedora / dnf) -----------------------------------
# The kit targets a fresh Fedora 44 with internet. Map each benchmark tool to the
# Fedora package that provides it; on request, install what's missing. Tools that
# are not packaged (or already bundled) are listed separately.
PKG_FOR_TOOL_CMD="7z:7zip openssl:openssl fio:fio stress-ng:stress-ng \
sensors:lm_sensors turbostat:kernel-tools cpupower:cpupowerutils \
glmark2:glmark2 vkmark:vkmark glxinfo:mesa-demos vulkaninfo:vulkan-tools \
sysbench:sysbench"

missing_tools() {   # prints tool names whose command is absent
  local pair cmd
  for pair in $PKG_FOR_TOOL_CMD; do
    cmd="${pair%%:*}"
    have "$cmd" || printf '%s\n' "$cmd"
  done
}
pkgs_for_missing() {   # prints the packages needed for the missing commands
  local pkgs="" pair cmd
  for pair in $PKG_FOR_TOOL_CMD; do
    cmd="${pair%%:*}"; pkg="${pair#*:}"
    have "$cmd" || pkgs="$pkgs $pkg"
  done
  python3 -c 'import matplotlib' >/dev/null 2>&1 || pkgs="$pkgs python3-matplotlib"
  # PassMark needs the ncurses compat libs (rpm check, these aren't commands)
  if [ -x "$PT" ]; then
    rpm -q ncurses-libs >/dev/null 2>&1     || pkgs="$pkgs ncurses-libs"
    rpm -q ncurses-compat-libs >/dev/null 2>&1 || pkgs="$pkgs ncurses-compat-libs"
  fi
  printf '%s' "$pkgs"
}
install_deps() {   # INSTALL_DEPS=1 (or 'ask')
  local mode="${INSTALL_DEPS:-ask}" pkgs
  pkgs="$(pkgs_for_missing)"
  if [ -z "${pkgs// /}" ]; then printf 'all dnf-provided tools present\n'; return 0; fi
  printf 'missing (dnf):%s\n' "$pkgs"
  case "$mode" in
    0|no|off)   printf 'INSTALL_DEPS=%s -> not installing; some tests will skip\n' "$mode"; return 0 ;;
    ask)
      if [ -t 0 ]; then
        printf 'install now with sudo dnf? [y/N] '
        read -r ans; case "$ans" in [yY]*) ;; *) printf 'skipped\n'; return 0 ;; esac
      else
        printf 'non-interactive; set INSTALL_DEPS=1 to auto-install, else tests skip\n'; return 0
      fi ;;
  esac
  if ! have dnf; then printf 'dnf not found (non-Fedora?); install manually\n'; return 0; fi
  if ! sudo -n true 2>/dev/null; then
    printf 'need sudo for dnf; run in a terminal (ssh -t)\n'; return 0
  fi
  section "installing missing tools (dnf)"
  # shellcheck disable=SC2086
  sudo dnf install -y $pkgs || printf 'dnf install failed; tests may skip\n'
}

# Idle baseline: let the machine settle and log >=15 s of true idle so the
# report has a real idle window (the 'env' phase alone was only ~2 s).
run_idle() {
  local secs="${IDLE_SECONDS:-15}"
  section "IDLE baseline  (${secs}s, CPU: $CPU_MODEL)"
  phase "idle"
  # settle: give the governor/load from startup time to calm down
  sleep 3
  # keep the phase open for at least $secs of sampling
  sleep "$secs"
}

# Model string for titles (env capture, graphs, stdout headers).
cpu_model() {
  local m=""
  m="$(sed -n 's/^model name[[:space:]]*:[[:space:]]*//p' /proc/cpuinfo | head -1)"
  [ -z "$m" ] && m="$(lscpu 2>/dev/null | sed -n 's/^Model name:[[:space:]]*//p' | head -1)"
  [ -z "$m" ] && m="$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_driver 2>/dev/null)"
  printf '%s' "${m:-unknown CPU}"
}
CPU_MODEL="$(cpu_model)"

# Register one score (phase,value) into scores-<tag>.csv for the efficiency table.
reg_score() {   # $1 = phase name (must match a phase), $2 = numeric score
  [ -n "${2:-}" ] || return 0
  printf '%s,%s\n' "$1" "$2" >> "scores-$TAG.csv"
}
# 7-Zip "Avr:" row = "Avr: <speed> <usage> <R/U> <rating> | <decomp...>"; the
# compressing MIPS is the Rating, field 5 (the "Tot:" row's first field is usage %).
7z_mips()   { awk '/^Avr:/{print $5; exit}' "$1" 2>/dev/null; }
# sysbench prints "events per second: <N>".
sysbench_eps() { awk -F: '/events per second/{gsub(/ /,"",$2); print $2; exit}' "$1" 2>/dev/null; }
# sysbench memory prints "... MiB transferred (<N> MiB/sec)".
sysbench_mbs() { sed -n 's/.*(\([0-9.]*\) MiB\/sec).*/\1/p' "$1" 2>/dev/null | head -1; }
# openssl speed ends each algorithm with a "...k" row; take its last (largest) value.
openssl_gbs() { awk 'NF && $NF ~ /k$/ {gsub(/k$/,"",$NF); v=$NF} END{ if (v!="") printf "%.3f", v/1000000 }' "$1" 2>/dev/null; }
# fio JSON: read one number from the first job.
fio_json() { python3 -c 'import json,sys; j=json.load(open(sys.argv[1]))["jobs"][0]; print((j.get(sys.argv[2]) or {}).get(sys.argv[3]) or "")' "$1" "$2" "$3" 2>/dev/null; }
fio_reg() {   # $1=job $2=read|write $3=bw_bytes|iops
  local v; v="$(fio_json "fio-$1-$TAG.json" "$2" "$3")"
  [ -n "$v" ] || return 0
  if [ "$3" = "bw_bytes" ]; then
    reg_score "fio-$1" "$(awk -v v="$v" 'BEGIN{printf "%.1f", v/1000000}')"
  else
    reg_score "fio-$1" "$(awk -v v="$v" 'BEGIN{printf "%.0f", v}')"
  fi
}
# glmark2/vkmark print a final "<tool> Score: <N>".
gpu_score() { awk -v tool="$1" 'index($0, tool " Score:"){print $NF; exit}' "$2" 2>/dev/null; }

# PassMark's .txt is a curses screen capture (unreadable); the numbers live in
# a results_*.yml file. Write scores-<tag>.csv so power-report.py can compute
# score/W efficiency. Each PassMark suite is its own phase (PassMark-cpu /
# PassMark-mem) so the power window matched to a score is the suite that made it.
# The single-thread mark rides in the CPU suite and shares its window.
parse_passmark() {   # $1 = yml, $2 = phase name, $3 = suite ("cpu"|"mem")
  local yml="$1" ph="$2" suite="$3"
  [ -f "$yml" ] || return 0
  local out="scores-$TAG.csv"
  awk -v out="$out" -v ph="$ph" -v suite="$suite" '
    /SUMM_CPU:/        { cpu = $2 }
    /SUMM_ME:/         { mem = $2 }
    /CPU_SINGLETHREAD:/{ st  = $2 }
    END {
      if (suite == "cpu" && cpu != "") {
        print ph "," cpu > out
        if (st != "") print ph "-single," st >> out
      }
      if (suite == "mem" && mem != "") print ph "," mem >> out
    }' "$yml"
}

# --- app workloads: fixed corpora + timing ------------------------------------
# Each app benchmark runs the stock command-line tool over a fixed input set and
# reports a rate (items/s, higher is better) so the score/W table stays meaningful.
fetch_corpus() {   # $1=file $2=url $3=sha256  -> ensures $CORPUS/<file> is present
  local f="$CORPUS/$1" sha
  if [ -s "$f" ]; then
    sha="$(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)"
    [ "$sha" = "$3" ] && return 0
  fi
  mkdir -p "$CORPUS"
  if have curl; then
    curl -fsSL --retry 2 -o "$f.part" "$2" || { rm -f "$f.part"; return 1; }
  elif have wget; then
    wget -q -O "$f.part" "$2" || { rm -f "$f.part"; return 1; }
  else
    printf 'need curl or wget to fetch %s\n' "$1" >&2; return 1
  fi
  mv -f "$f.part" "$f"
  sha="$(sha256sum "$f" | cut -d' ' -f1)"
  if [ "$sha" != "$3" ]; then
    printf 'checksum mismatch for %s (got %s)\n' "$1" "$sha" >&2
    rm -f "$f"; return 1
  fi
  printf 'fetched %s -> %s\n' "$1" "$f"
}
extract_zip() {    # $1=zip $2=destdir
  mkdir -p "$2"
  if have unzip; then unzip -oq "$1" -d "$2"
  elif have 7z; then 7z x -y -o"$2" "$1" >/dev/null
  else printf 'need unzip or 7z to extract %s\n' "$1" >&2; return 1
  fi
}
setup_corpus() {   # $1=zip $2=url $3=sha256 $4=subdir -> extracted once under $CORPUS/$4
  local d="$CORPUS/$4"
  fetch_corpus "$1" "$2" "$3" || return 1
  if [ -z "$(ls -A "$d" 2>/dev/null)" ]; then
    extract_zip "$CORPUS/$1" "$d" || return 1
  fi
}
now()   { date +%s.%N; }
secs()  { awk -v a="$1" -v b="$2" 'BEGIN{printf "%.3f", b-a}'; }
per_s() { awk -v n="$1" -v s="$2" 'BEGIN{ if (s>0) printf "%.3f", n/s }'; }

# --- sudo: acquire once, then keep the timestamp alive for the whole run ------
SUDO=""
KEEPID=""
acquire_sudo() {
  if sudo -n true 2>/dev/null; then SUDO="sudo"; return 0; fi
  if [[ -t 0 || -t 1 ]]; then
    if sudo -v 2>/dev/null; then SUDO="sudo"; return 0; fi
  fi
  SUDO=""
}
start_sudo_keepalive() {
  [[ -n "$SUDO" ]] || return 0
  ( while :; do sudo -n true 2>/dev/null || exit 0; sleep 50; done ) &
  KEEPID=$!
}

# --- CPU governor: pin and restore -------------------------------------------
GOV_ORIG=""
pin_governor() {
  [[ -n "$SUDO" ]] || { printf 'governor not pinned (no sudo)\n'; return 0; }
  GOV_ORIG="$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || true)"
  section "pin CPU governor: ${GOVERNOR} (was ${GOV_ORIG:-unknown})"
  $SUDO cpupower frequency-set -g "${GOVERNOR}" >/dev/null 2>&1 || \
    for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
      printf '%s' "${GOVERNOR}" | $SUDO tee "$f" >/dev/null 2>&1
    done
}
restore_governor() {
  [[ -n "$SUDO" && -n "$GOV_ORIG" && "$GOV_ORIG" != "$GOVERNOR" ]] || return 0
  $SUDO cpupower frequency-set -g "$GOV_ORIG" >/dev/null 2>&1 || \
    for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
      printf '%s' "$GOV_ORIG" | $SUDO tee "$f" >/dev/null 2>&1
    done
  printf 'governor restored to %s\n' "$GOV_ORIG"
}

# --- discrete GPU detection & pinning (for the headed GL/Vulkan tests) --------
# Picks the GPU that owns no connected display connector (the headless dGPU on
# this box: the RX 6700 XT, while both monitors hang off the Cezanne iGPU).
detect_dgpu() {
  local c pci conn v d best_pci="" best_vd=""
  for c in /sys/class/drm/card[0-9]*; do
    [[ "$(basename "$c")" =~ ^card[0-9]+$ ]] || continue
    [[ -r "$c/device/vendor" && -r "$c/device/device" ]] || continue
    conn=0
    for s in "$c"/card*-*/status; do
      [[ -e "$s" ]] || continue
      [[ "$(cat "$s" 2>/dev/null)" == "connected" ]] && conn=$((conn + 1))
    done
    v="$(cat "$c/device/vendor" 2>/dev/null)"
    d="$(cat "$c/device/device" 2>/dev/null)"
    pci="$(basename "$(readlink -f "$c/device")")"
    if [[ "$conn" -eq 0 ]]; then
      best_pci="$pci"; best_vd="$v:$d"; break
    fi
    [[ -z "$best_pci" ]] && { best_pci="$pci"; best_vd="$v:$d"; }
  done
  GPU_PCI="${GPU_PCI:-$best_pci}"
  GPU_VD="${GPU_VD:-$best_vd}"
}
pin_gpu_env() {
  detect_dgpu
  if [[ -n "${GPU_PCI:-}" ]]; then export DRI_PRIME="pci-$GPU_PCI"; fi
  if [[ -n "${GPU_VD:-}" ]]; then
    local vd="${GPU_VD//0x/}"
    export MESA_VK_DEVICE_SELECT="$vd"
    export VK_LOADER_DEVICE_SELECT="$vd"
  fi
  printf 'GPU pinning: DRI_PRIME=%s MESA_VK_DEVICE_SELECT=%s\n' \
    "${DRI_PRIME:-<none>}" "${MESA_VK_DEVICE_SELECT:-<none>}"
}

# --- readiness check (no benchmarks run) -------------------------------------
doctor() {
  section "basic-benchmark doctor  ($(hostname))"
  printf 'kit:      %s\n' "$DIR"
  printf 'user:     %s\n' "$(id -un)"
  printf 'TERM:     %s\n' "${TERM:-<unset>}"
  printf 'display:  %s\n' "${WAYLAND_DISPLAY:-}${DISPLAY:-<none (headed part would refuse)>}"
  if sudo -n true 2>/dev/null; then printf 'sudo:     cached (ok)\n'
  else printf 'sudo:     will prompt - run the headless part as: ssh -t <target> ...\n'; fi
  detect_dgpu
  printf 'dGPU:     pci=%s vd=%s\n' "${GPU_PCI:-<none>}" "${GPU_VD:-<none>}"
  printf 'RAPL:     %s\n' "$(ls -d /sys/class/powercap/intel-rapl:* 2>/dev/null | tr '\n' ' ')"
  printf 'PassMark: %s\n' "$([ -x "$PT" ] && echo "$PT" || echo "MISSING ($PT)")"
  printf 'llama:    %s\n' "$([ -x "$LLAMA_BIN" ] && echo "$LLAMA_BIN" || echo "MISSING ($LLAMA_BIN)")"
  printf 'model:    %s\n' "$([ -f "$LLAMA_MODEL" ] && echo "$LLAMA_MODEL ($(du -h "$LLAMA_MODEL" 2>/dev/null | cut -f1))" || echo "MISSING ($LLAMA_MODEL)")"
  printf 'tools:\n'
  for t in 7z openssl sysbench fio stress-ng sensors turbostat cpupower \
           glmark2 vkmark glxinfo vulkaninfo nproc; do
    printf '  %-22s %s\n' "$t" "$(command -v "$t" || echo MISSING)"
  done
  printf 'apps:\n'
  for t in libreoffice inkscape gegl gimp; do
    printf '  %-22s %s\n' "$t" "$(command -v "$t" || echo MISSING)"
  done
  printf 'corpus:   %s\n' "$CORPUS"
  printf '          install apps: sudo dnf install -y libreoffice-writer libreoffice-calc inkscape gegl04-tools gimp\n'
  printf 'governor: %s (pinned for timed runs; set GOVERNOR=... to choose)\n' "$GOVERNOR"
  local need; need="$(pkgs_for_missing)"
  if [ -z "${need// /}" ]; then
    printf 'dnf:      all installable tools present\n'
  else
    printf 'dnf:      missing:%s\n' "$need"
    printf '          install with:  INSTALL_DEPS=1 sudo dnf install -y%s\n' "$need"
  fi
  printf 'headless part needs: sudo (power/governor/turbostat) + a GGUF model for the LLM test.\n'
  printf 'headed   part needs: a graphical session + glmark2/vkmark.\n'
}

# --- sampler + cleanup --------------------------------------------------------
SPID=""
start_sampler() {
  if [[ -n "$SUDO" ]] && have python3 && [[ -f "$SAMPLER" ]]; then
    section "start power/thermal sampler (powerlog.py)"
    $SUDO python3 "$SAMPLER" --tag "$TAG" --interval 1 &
    SPID=$!
    sleep 1
    printf 'sampler pid %s -> results/power-%s.csv\n' "$SPID" "$TAG"
  else
    printf 'power sampler skipped (needs sudo + python3; use "ssh -t" over SSH)\n'
  fi
}
stop_sampler() {
  [[ -n "$SPID" ]] || return 0
  kill -INT "$SPID" 2>/dev/null || $SUDO pkill -INT -f "powerlog.py --tag $TAG" 2>/dev/null
  wait "$SPID" 2>/dev/null
  SPID=""
}
CLEANED=0
cleanup() {
  [[ "$CLEANED" == "1" ]] && return 0
  CLEANED=1
  stop_sampler
  [[ -n "$KEEPID" ]] && { kill "$KEEPID" 2>/dev/null; wait "$KEEPID" 2>/dev/null; KEEPID=""; }
  restore_governor
  rm -f "$FIOFILE" 2>/dev/null
}
on_signal() { cleanup; exit 130; }
trap 'on_signal' INT TERM
trap 'cleanup' EXIT

# --- test groups --------------------------------------------------------------
run_env() {
  section "environment  (tag: $TAG)"
  phase "env"
  {
    printf 'CPU_MODEL=%s\n' "$CPU_MODEL"
    date
    uname -a
    lscpu
    free -h
    lsblk
    have stress-ng && stress-ng --version
    have glxinfo && glxinfo -B 2>/dev/null
    have vulkaninfo && vulkaninfo --summary 2>/dev/null | head -70
    printf 'DRI_PRIME=%s MESA_VK_DEVICE_SELECT=%s VK_LOADER_DEVICE_SELECT=%s\n' \
      "${DRI_PRIME:-}" "${MESA_VK_DEVICE_SELECT:-}" "${VK_LOADER_DEVICE_SELECT:-}"
    [ -x "$PT" ] && sha256sum "$PT"
    printf 'LLAMA_BIN=%s\nLLAMA_MODEL=%s\n' "$LLAMA_BIN" "$LLAMA_MODEL"
    printf 'MEM/CAPACITY: MemTotal=%s VRAM(dGPU)=%s\n' \
      "$(awk '/MemTotal/{printf "%.1f GiB", $2/1048576}' /proc/meminfo)" \
      "$(cat /sys/class/drm/card*/device/mem_info_vram_total 2>/dev/null | awk 'BEGIN{m=0}{if($1>m)m=$1}END{if(m)printf "%.1f GiB",m/1073741824; else print "?"}')"
    printf 'MODELS:\n'
    find "$HOME/models" -iname '*.gguf' -printf '%s\t%p\n' 2>/dev/null \
      | sort -n | awk '{printf "  %.2f GiB\t%s\n", $1/1073741824, $2}'
  } > "env-$TAG.txt" 2>&1
  printf 'wrote results/env-%s.txt\n' "$TAG"
}

# --- LLM inference (llama.cpp, bundled) ---------------------------------------
# Self-contained: the llama-bench runtime + a small model live under tools/llama/.
# Targets token GENERATION on CPU/RAM (ngl=0), which is memory-bound, so the tg
# number reflects how well this machine's RAM bandwidth serves an LLM.

run_llama() {
  if [ ! -x "$LLAMA_BIN" ]; then
    printf 'skip (missing %s)\n' "$LLAMA_BIN"; return 0
  fi
  if [ ! -f "$LLAMA_MODEL" ]; then
    printf 'skip (missing model %s)\n' "$LLAMA_MODEL"; return 0
  fi
  local size; size="$(du -h "$LLAMA_MODEL" 2>/dev/null | cut -f1)"
  printf 'bench:  %s\nmodel:  %s (%s)\nngl=%s (CPU/RAM) tg=%s reps=%s\n' \
    "$LLAMA_BIN" "$LLAMA_MODEL" "${size:-?}" "$LLAMA_NGL" "$LLAMA_TG" "$LLAMA_REPS"
  (
    cd "$LLAMA_DIR/bin" || exit 1
    LD_LIBRARY_PATH="$PWD" ./llama-bench \
      -m "$LLAMA_MODEL" -ngl "$LLAMA_NGL" -p 0 -n "$LLAMA_TG" -r "$LLAMA_REPS"
  ) 2>&1 | tee "llama-$TAG.txt"
  local tg
  tg="$(awk -F'|' '/ tg[0-9]/{gsub(/ /,"",$8); print $8; exit}' "llama-$TAG.txt" | sed 's/±.*//')"
  if [ -z "$tg" ]; then
    printf 'llama: no t/s parsed — check %s (missing libs?)\n' "llama-$TAG.txt"
    return 0
  fi
  reg_score "llama-tg" "$tg"
}

# --- app workloads (stock apps + fixed corpora) -------------------------------
# Phases and score names are the benchmark names: libreoffice gegl inkscape gimp.
# Each registers a rate (items/s) so higher stays better for the score/W table.

run_libreoffice() {
  section "APP: LibreOffice — documents to PDF"
  phase "libreoffice"
  if ! have libreoffice; then
    printf 'skip (libreoffice missing) — sudo dnf install -y libreoffice-writer libreoffice-calc\n'
    return 0
  fi
  setup_corpus lo-sample-documents-1.zip "$CORPUS_URL/lo-sample-documents-1.zip" \
    a1e92da67999d8ed91ad6330b47828207285c146a3a4ebd7b95ad39cb2d62dcd libreoffice \
    || { printf 'skip (corpus unavailable)\n'; return 0; }
  local d="$CORPUS/libreoffice" out="$CORPUS/libreoffice-out" log="libreoffice-$TAG.txt"
  local n t0 t1 el r
  n=$(find "$d/documents" -type f 2>/dev/null | wc -l)
  [ "$n" -gt 0 ] || { printf 'skip (no documents in corpus)\n'; return 0; }
  rm -rf "$out"; mkdir -p "$out"
  { libreoffice --version; printf 'convert %s documents -> PDF (headless, default filters)\n' "$n"; } > "$log" 2>&1
  t0=$(now)
  libreoffice --headless --convert-to pdf --outdir "$out" "$d"/documents/* >> "$log" 2>&1
  t1=$(now)
  el="$(secs "$t0" "$t1")"; r="$(per_s "$n" "$el")"
  cat "$log"
  printf 'libreoffice: %s docs in %ss = %s docs/s\n' "$n" "$el" "$r"
  reg_score "libreoffice" "$r"
}

run_gegl() {
  section "APP: GEGL — image operations"
  phase "gegl"
  if ! have gegl; then
    printf 'skip (gegl missing) — sudo dnf install -y gegl04-tools\n'
    return 0
  fi
  setup_corpus sample-photo-6000x4000-1.zip "$CORPUS_URL/sample-photo-6000x4000-1.zip" \
    cadcd434abcdc82a220e7022c749b6fa5f46bb7a32ec14423ca8aab067b0e989 gegl \
    || { printf 'skip (corpus unavailable)\n'; return 0; }
  local d="$CORPUS/gegl" img log="gegl-$TAG.txt" t0 t1 el r op
  img="$(find "$d" -maxdepth 1 -iname '*.jpg' 2>/dev/null | head -1)"
  [ -n "$img" ] || { printf 'skip (no JPEG in corpus)\n'; return 0; }
  local -a ops=( "rotate-on-center degrees=90" "scale-size x=400 y=400" "antialias" \
    "cartoon" "color-enhance" "crop x=100 y=100 width=1920 height=1080" \
    "wavelet-blur" "reflect" "tile-glass tile-width=20 tile-height=20" )
  { gegl --version 2>&1 | head -1; printf '%s ops x 1 image (%s)\n' "${#ops[@]}" "$(basename "$img")"; } > "$log"
  t0=$(now)
  for op in "${ops[@]}"; do
    printf -- '-- %s --\n' "$op" >> "$log"
    # shellcheck disable=SC2086
    gegl -i "$img" -o "$d/out.png" -- $op >> "$log" 2>&1
  done
  t1=$(now)
  el="$(secs "$t0" "$t1")"; r="$(per_s "${#ops[@]}" "$el")"
  cat "$log"
  printf 'gegl: %s ops in %ss = %s ops/s\n' "${#ops[@]}" "$el" "$r"
  reg_score "gegl" "$r"
}

run_inkscape() {
  section "APP: Inkscape — SVG to PNG"
  phase "inkscape"
  if ! have inkscape; then
    printf 'skip (inkscape missing) — sudo dnf install -y inkscape\n'
    return 0
  fi
  setup_corpus svg-test-files-1.zip "$CORPUS_URL/svg-test-files-1.zip" \
    c373998bebff0a88f0c4e6fec383eaa62690f544c6eb2b1ad7c55b1547e4db08 inkscape \
    || { printf 'skip (corpus unavailable)\n'; return 0; }
  local d="$CORPUS/inkscape" out="$CORPUS/inkscape-out" log="inkscape-$TAG.txt"
  local n t0 t1 el r f
  n=$(find "$d" -maxdepth 1 -name '*.svg' 2>/dev/null | wc -l)
  [ "$n" -gt 0 ] || { printf 'skip (no SVG in corpus)\n'; return 0; }
  rm -rf "$out"; mkdir -p "$out"
  { inkscape --version; printf 'export %s SVG files -> PNG\n' "$n"; } > "$log" 2>&1
  t0=$(now)
  for f in "$d"/*.svg; do
    inkscape --export-filename="$out/$(basename "${f%.svg}").png" "$f" >> "$log" 2>&1
  done
  t1=$(now)
  el="$(secs "$t0" "$t1")"; r="$(per_s "$n" "$el")"
  cat "$log"
  printf 'inkscape: %s SVGs in %ss = %s images/s\n' "$n" "$el" "$r"
  reg_score "inkscape" "$r"
}

# GIMP batch Script-Fu. GIMP 3 renamed/reshaped much of the PDB, so pick the
# script for the installed major version. GIMP 3 support is best-effort: verify
# the log if the op count or timing looks off.
gimp_script_v2() {
  cat > "$1" <<'SCM'
(define (batch-op op pattern)
  (let* ((filelist (cadr (file-glob pattern 1))))
    (while (not (null? filelist))
      (let* ((filename (car filelist))
             (image (car (gimp-file-load RUN-NONINTERACTIVE filename filename)))
             (drawable (car (gimp-image-get-active-layer image))))
        (cond ((string=? op "unsharp-mask")
               (plug-in-unsharp-mask RUN-NONINTERACTIVE image drawable 15.0 0.6 0))
              ((string=? op "resize")      (gimp-image-scale image 600 400))
              ((string=? op "rotate")      (gimp-image-rotate image 0))
              ((string=? op "auto-levels") (gimp-levels-stretch drawable)))
        (gimp-file-save RUN-NONINTERACTIVE image drawable filename filename)
        (gimp-image-delete image))
      (set! filelist (cdr filelist)))))
SCM
}
gimp_script_v3() {
  cat > "$1" <<'SCM'
(script-fu-use-v3)
(define (batch-op op pattern)
  (let* ((filelist (file-glob #:pattern pattern #:filename-encoding TRUE)))
    (while (not (null? filelist))
      (let* ((filename (car filelist))
             (image (gimp-file-load RUN-NONINTERACTIVE filename))
             (drawable (vector-ref (gimp-image-get-selected-drawables image) 0)))
        (cond ((string=? op "unsharp-mask")
               (plug-in-unsharp-mask RUN-NONINTERACTIVE image drawable 15.0 0.6 0))
              ((string=? op "resize")      (gimp-image-scale image 600 400))
              ((string=? op "rotate")      (gimp-image-rotate image 0))
              ((string=? op "auto-levels") (gimp-levels-stretch drawable)))
        (gimp-file-save RUN-NONINTERACTIVE image filename)
        (gimp-image-delete image))
      (set! filelist (cdr filelist)))))
SCM
}
run_gimp() {
  section "APP: GIMP — batch image operations"
  phase "gimp"
  if ! have gimp; then
    printf 'skip (gimp missing) — sudo dnf install -y gimp\n'
    return 0
  fi
  setup_corpus sample-photo-6000x4000-1.zip "$CORPUS_URL/sample-photo-6000x4000-1.zip" \
    cadcd434abcdc82a220e7022c749b6fa5f46bb7a32ec14423ca8aab067b0e989 gimp \
    || { printf 'skip (corpus unavailable)\n'; return 0; }
  local d="$CORPUS/gimp" src log="gimp-$TAG.txt" w scm gv t0 t1 el r op
  src="$(find "$d" -maxdepth 1 -iname '*.jpg' 2>/dev/null | head -1)"
  [ -n "$src" ] || { printf 'skip (no JPEG in corpus)\n'; return 0; }
  w="$CORPUS/gimp-work"; rm -rf "$w"; mkdir -p "$w"
  scm="$w/bench.scm"
  gv="$(gimp --version 2>&1 | grep -oE '[0-9]+' | head -1)"; gv="${gv:-3}"
  if [ "$gv" -ge 3 ] 2>/dev/null; then gimp_script_v3 "$scm"; else gimp_script_v2 "$scm"; fi
  { gimp --version 2>&1 | head -1; printf '4 ops x 1 image (v%s script)\n' "$gv"; } > "$log"
  t0=$(now)
  for op in unsharp-mask resize rotate auto-levels; do
    cp -f "$src" "$w/sample.JPG"
    printf -- '-- %s --\n' "$op" >> "$log"
    if [ "$gv" -ge 3 ] 2>/dev/null; then
      GIMP3_DIRECTORY="$w/cfg" gimp -i --batch-interpreter=plug-in-script-fu-eval \
        -b "(load \"$scm\")" -b "(batch-op \"$op\" \"$w/*.JPG\")" -b '(gimp-quit 0)' >> "$log" 2>&1
    else
      GIMP2_DIRECTORY="$w/cfg" gimp -i \
        -b "(load \"$scm\")" -b "(batch-op \"$op\" \"$w/*.JPG\")" -b '(gimp-quit 0)' >> "$log" 2>&1
    fi
  done
  t1=$(now)
  el="$(secs "$t0" "$t1")"; r="$(per_s 4 "$el")"
  cat "$log"
  printf 'gimp: 4 ops in %ss = %s ops/s\n' "$el" "$r"
  reg_score "gimp" "$r"
}

run_apps() {
  section "APP WORKLOADS (LibreOffice / GEGL / Inkscape / GIMP)"
  run_libreoffice
  run_gegl
  run_inkscape
  run_gimp
}

run_headless() {
  section "PASSMARK PERFORMANCETEST LINUX — CPU suite  (cpubenchmark.net)"
  phase "PassMark-cpu"
  if [ -x "$PT" ]; then
    "$PT" -r 1 -d 2 </dev/null | tee "passmark-cpu-$TAG.txt"
    for cand in results_cpu.yml "$TOOLS/pt/results_cpu.yml" \
                "$(dirname "$PT")/results_cpu.yml"; do
      [ -f "$cand" ] && { mv -f "$cand" "passmark-cpu-$TAG.yml"; break; }
    done
    if [ -s "passmark-cpu-$TAG.yml" ]; then
      parse_passmark "passmark-cpu-$TAG.yml" "PassMark-cpu" "cpu"
    else
      printf 'WARNING: PassMark CPU suite produced no results_cpu.yml — is %s a real\n' "$PT"
      printf '         binary? (a placeholder/stub here silently breaks the score)\n'
    fi
  else
    printf 'skip (not found: %s) — see README section 4\n' "$PT"
  fi

  section "PASSMARK PERFORMANCETEST LINUX — Memory suite"
  phase "PassMark-mem"
  if [ -x "$PT" ]; then
    "$PT" -r 2 -d 2 </dev/null | tee "passmark-mem-$TAG.txt"
    for cand in results_memory.yml "$TOOLS/pt/results_memory.yml" \
                "$(dirname "$PT")/results_memory.yml"; do
      [ -f "$cand" ] && { mv -f "$cand" "passmark-mem-$TAG.yml"; break; }
    done
    if [ -s "passmark-mem-$TAG.yml" ]; then
      parse_passmark "passmark-mem-$TAG.yml" "PassMark-mem" "mem"
    else
      printf 'WARNING: PassMark Memory suite produced no results_memory.yml (see above)\n'
    fi
  else
    printf 'skip (not found: %s) — see README section 4\n' "$PT"
  fi
  [ -s "scores-$TAG.csv" ] && printf 'wrote results/scores-%s.csv (PassMark)\n' "$TAG"

  section "CPU: 7zip"
  phase "7zip"
  if have 7z; then
    7z b -mmt"$(nproc)" | tee "7z-$TAG.txt"
    reg_score "7zip" "$(7z_mips "7z-$TAG.txt")"
  else
    printf 'skip (7z missing)\n'
  fi

  section "CPU: 7zip 1 thread (isolated single-core)"
  phase "7zip-1t"
  if have 7z; then
    7z b -mmt1 | tee "7z-1t-$TAG.txt"
    reg_score "7zip-1t" "$(7z_mips "7z-1t-$TAG.txt")"
  else
    printf 'skip (7z missing)\n'
  fi

  section "CPU: openssl aes-256-gcm / sha256"
  phase "openssl"
  if have openssl; then
    openssl speed -multi "$(nproc)" -evp aes-256-gcm 2>&1 | tail -20 | tee "openssl-aes-$TAG.txt"
    openssl speed -multi "$(nproc)" -evp sha256      2>&1 | tail -20 | tee "openssl-sha-$TAG.txt"
    reg_score "openssl-aes" "$(openssl_gbs "openssl-aes-$TAG.txt")"
    reg_score "openssl-sha" "$(openssl_gbs "openssl-sha-$TAG.txt")"
  fi

  section "CPU: sysbench 1 thread / all threads"
  phase "sysbench-1t"
  if have sysbench; then
    sysbench cpu --threads=1 run | tee "sysbench-1t-$TAG.txt"
    reg_score "sysbench-1t" "$(sysbench_eps "sysbench-1t-$TAG.txt")"
    phase "sysbench-nt"
    sysbench cpu --threads="$(nproc)" run | tee "sysbench-nt-$TAG.txt"
    reg_score "sysbench-nt" "$(sysbench_eps "sysbench-nt-$TAG.txt")"
    phase "sysbench-mem"
    sysbench memory --threads="$(nproc)" run | tee "sysbench-mem-$TAG.txt"
    reg_score "sysbench-mem" "$(sysbench_mbs "sysbench-mem-$TAG.txt")"
  else
    printf 'skip (sysbench missing) — sudo dnf install -y sysbench\n'
  fi

  section "LLM INFERENCE: llama.cpp  (estimate model capacity & tok/s)"
  phase "llama"
  run_llama

  if [[ "$RUN_APPS" == "1" ]]; then
    run_apps
  else
    section "APP WORKLOADS: skipped (RUN_APPS=0)"
  fi

  section "STORAGE: fio (same drive, $FIOFILE)"
  phase "fio"
  if have fio; then
    mkdir -p "$(dirname "$FIOFILE")"
    for job in "seqread read 1M 16" "seqwrite write 1M 16" "randread randread 4k 32" "randwrite randwrite 4k 32"; do
      set -- $job
      printf '\n-- fio %s --\n' "$1"
      phase "fio-$1"
      fio --name="$1" --rw="$2" --bs="$3" --numjobs=1 --iodepth="$4" \
          --size="$FIO_SIZE" --direct=1 --time_based --runtime="$FIO_RUNTIME" \
          --filename="$FIOFILE" --group_reporting \
          --output-format=json --output="fio-$1-$TAG.json"
      case "$1" in
        seqread)   fio_reg "$1" read  bw_bytes ;;
        seqwrite)  fio_reg "$1" write bw_bytes ;;
        randread)  fio_reg "$1" read  iops ;;
        randwrite) fio_reg "$1" write iops ;;
      esac
      rm -f "$FIOFILE"
    done
  else
    printf 'skip (fio missing)\n'
  fi

  section "THERMALS / POWER / SUSTAINED CLOCKS"
  phase "thermal-load"
  if have stress-ng; then
    stress-ng --cpu 0 --cpu-method matrixprod --timeout 300 --metrics-brief > "stress-ng-$TAG.txt" 2>&1 &
    LOADPID=$!
    sleep 120
    have sensors && sensors > "sensors-load-$TAG.txt"
    cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq >> "sensors-load-$TAG.txt" 2>/dev/null
    if have turbostat && [[ -n "$SUDO" ]]; then
      # this turbostat has no --timeout; bound it with coreutils timeout instead
      timeout 30 $SUDO turbostat --interval 5 --quiet \
          --show Core,CPU,Busy%,Bzy_MHz,PkgWatt,PkgTmp \
          | tee "turbostat-load-$TAG.txt"
    fi
    wait "$LOADPID" 2>/dev/null
  else
    printf 'skip (stress-ng missing)\n'
  fi

  section "DESKTOP / SYSTEM"
  phase "systemd"
  if have systemd-analyze; then
    systemd-analyze            | tee "systemd-analyze-$TAG.txt"
    systemd-analyze blame | head -20 >> "systemd-analyze-$TAG.txt"
  fi
}

run_headed() {
  section "GPU: glmark2 / vkmark (pinned to ${GPU_PCI:-default})"
  # One representative scene each by default (fast, comparable). Override the
  # scene/options with GLMARK2_BENCH / VKMARK_BENCH, or the seconds with
  # GLMARK2_DURATION / VKMARK_DURATION.
  phase "glmark2"
  if have glmark2; then
    glmark2 -b "terrain:duration=${GLMARK2_DURATION:-10}" 2>&1 | tail -25 | tee "glmark2-$TAG.txt"
    reg_score "glmark2" "$(gpu_score glmark2 "glmark2-$TAG.txt")"
  else
    printf 'skip (glmark2 missing) — sudo dnf install -y glmark2\n'
  fi
  phase "vkmark"
  if have vkmark; then
    vkmark -b "${VKMARK_BENCH:-texture}:duration=${VKMARK_DURATION:-10}" 2>&1 | tail -30 | tee "vkmark-$TAG.txt"
    reg_score "vkmark" "$(gpu_score vkmark "vkmark-$TAG.txt")"
  else
    printf 'skip (vkmark missing) — sudo dnf install -y vkmark\n'
  fi
  have glxinfo && glxinfo -B | tee "glxinfo-$TAG.txt"
  have vulkaninfo && vulkaninfo --summary 2>/dev/null | tee "vulkan-$TAG.txt" >/dev/null
}

require_display() {
  if [[ -z "${WAYLAND_DISPLAY:-}${DISPLAY:-}" ]]; then
    printf 'ERROR: no display (WAYLAND_DISPLAY/DISPLAY unset).\n' >&2
    printf 'The "headed" part must run inside the KDE session (e.g. Konsole),\n' >&2
    printf 'not over a plain SSH command. Use "ssh -X"/"ssh -t" only for "headless".\n' >&2
    exit 2
  fi
}

# --- main ---------------------------------------------------------------------
TITLE="$TAG  [$PART]"
printf '\033[1mbasic-benchmark — %s\033[0m\n' "$TITLE"

if [[ "$DOCTOR" == "1" ]]; then doctor; exit 0; fi

# Fail fast: the headed part is defined by having a display.
[[ "$PART" == "headed" ]] && require_display

acquire_sudo
if [[ -n "$SUDO" ]]; then start_sudo_keepalive; else
  printf 'WARNING: no sudo — power logging, governor and turbostat will be skipped\n'
  printf '         run over "ssh -t" (allocate a TTY) to let sudo prompt.\n'
fi

: > "$PHASES"
: > "scores-$TAG.csv"
printf '\033[1mCPU under test: %s\033[0m\n' "$CPU_MODEL"
printf '%s\n' "$CPU_MODEL" > "cpu-$TAG.txt"
install_deps
pin_gpu_env
pin_governor
start_sampler
run_env
run_idle

case "$PART" in
  headless) run_headless ;;
  headed)   run_headed ;;
  all)
    run_headless
    if [[ "$RUN_GPU" == "1" ]] && [[ -n "${WAYLAND_DISPLAY:-}${DISPLAY:-}" ]]; then
      run_headed
    else
      section "GPU: skipped (no desktop session or RUN_GPU=0)"
    fi
    ;;
esac

stop_sampler

section "POWER / THERMAL REPORT"
if have python3 && [[ -f "$REPORT" ]]; then
  python3 "$REPORT" --tags "$TAG" || printf 'report failed\n'
fi

section "DONE ($TAG)"
printf 'results in: %s\n' "$RES"
printf 'score file (optional, for efficiency): %s/scores-%s.csv\n' "$RES" "$TAG"
printf 'compare:  python3 power-report.py --tags before-headless after-headless\n'
printf 'remember to back up results/ before a reinstall.\n'