Skip to content

A treemap for finding and removing what fills your disk, for Omarchy. Rust + GPUI.

Rust
5.6 KB
Raw Blame
#!/usr/bin/env python3
"""Time whole scans at several fixed worker counts and print the PR table.

Each run is one `scan_workers` example process under `/usr/bin/time`, so CPU
time is the scan's own user + system seconds. Whole-machine busy time is
sampled every half second while it runs (macOS `host_statistics`, Linux
`/proc/stat`). Rounds rotate the order so drift in the machine or the cache
does not favour one count. The first scan is an untimed warm-up.

    scripts/bench-scan-workers.py /System/Volumes/Data --workers 18 8 4 2

Hold the machine quiet: a VM, an indexer or a build skews every column.
"""

import argparse
import ctypes
import fcntl
import json
import os
import pathlib
import platform
import re
import statistics
import subprocess
import sys
import time

REPO = pathlib.Path(__file__).resolve().parent.parent
EXAMPLE = REPO / "target/release/examples/scan_workers"


def cpu_ticks():
    """(busy, total) ticks for the whole machine since boot."""
    if platform.system() == "Darwin":
        lib = ctypes.CDLL("/usr/lib/libSystem.B.dylib")
        lib.mach_host_self.restype = ctypes.c_uint
        ticks = (ctypes.c_uint * 4)()
        count = ctypes.c_uint(4)
        # HOST_CPU_LOAD_INFO: user, system, idle, nice.
        if lib.host_statistics(lib.mach_host_self(), 3, ticks, ctypes.byref(count)):
            raise OSError("host_statistics failed")
        return sum(ticks) - ticks[2], sum(ticks)
    fields = [int(x) for x in open("/proc/stat").readline().split()[1:]]
    idle = fields[3] + fields[4]
    return sum(fields) - idle, sum(fields)


def percent(value):
    return "n/a" if value is None else f"{value:.0f}%"


def busy_percent(before, after):
    busy, total = (a - b for a, b in zip(after, before))
    return 100 * busy / total if total else None


def cpu_seconds(time_output):
    """User + system seconds from BSD or GNU `/usr/bin/time` output."""
    user = re.search(r"([\d.]+) user|User time \(seconds\): ([\d.]+)", time_output)
    system = re.search(r"([\d.]+) sys|System time \(seconds\): ([\d.]+)", time_output)
    return sum(float(next(g for g in m.groups() if g)) for m in (user, system))


def scan(root, workers):
    flag = "-l" if platform.system() == "Darwin" else "-v"
    before = cpu_ticks()
    started = time.monotonic()
    process = subprocess.Popen(
        ["/usr/bin/time", flag, str(EXAMPLE), root, str(workers)],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        # Its own group, so an interrupt never leaves a scan running.
        start_new_session=True,
    )
    try:
        out, err = process.communicate()
    except BaseException:
        os.killpg(process.pid, 9)
        raise
    if process.returncode:
        sys.exit(f"scan failed:\n{err}")
    run = json.loads(out.strip().splitlines()[-1])
    run["wall"] = time.monotonic() - started
    run["cpu_s"] = cpu_seconds(err)
    run["machine"] = busy_percent(before, cpu_ticks())
    return run


def main():
    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    parser.add_argument("root", help="directory or volume to scan")
    parser.add_argument("--workers", type=int, nargs="+", default=[18, 8, 4, 2])
    parser.add_argument("--rounds", type=int, default=3)
    parser.add_argument("--out", type=pathlib.Path, help="write every run as JSON here")
    parser.add_argument("--lock", help="flock this file for the whole block")
    args = parser.parse_args()

    subprocess.run(
        ["cargo", "build", "--release", "-p", "disktree-core", "--example", "scan_workers"],
        cwd=REPO,
        check=True,
    )
    lock = open(args.lock, "w") if args.lock else None
    if lock:
        fcntl.flock(lock, fcntl.LOCK_EX)
    cores = os.cpu_count() or 1
    base = max(args.workers)
    print(f"warm-up: {base} workers", file=sys.stderr)
    scan(args.root, base)
    runs = []
    for round_ in range(args.rounds):
        shift = round_ % len(args.workers)
        order = args.workers[shift:] + args.workers[:shift]
        if round_ % 2:
            order.reverse()
        for workers in order:
            run = scan(args.root, workers)
            run["round"] = round_ + 1
            runs.append(run)
            print(
                f"round {round_ + 1}, {workers} workers: {run['elapsed']:.1f} s, "
                f"{run['cpu_s']:.0f} CPU-s, machine {percent(run['machine'])}",
                file=sys.stderr,
            )
    if args.out:
        args.out.write_text(json.dumps(runs, indent=1))

    def median(workers, key):
        values = [r[key] for r in runs if r["workers"] == workers and r[key] is not None]
        return statistics.median(values) if values else None

    # The scan's own share of every core, which background load cannot move.
    for run in runs:
        run["share"] = 100 * run["cpu_s"] / (run["elapsed"] * cores)
    b_time, b_cpu, b_share = (median(base, k) for k in ("elapsed", "cpu_s", "share"))
    print(f"\nMedian of {args.rounds}; Δ against {base} workers; {cores} cores.\n")
    print("| Workers | Scan time | Δ time | CPU time | Δ CPU time | Machine used | Δ machine | Whole machine |")
    print("| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |")
    for workers in sorted(args.workers, reverse=True):
        t, c, s, m = (median(workers, k) for k in ("elapsed", "cpu_s", "share", "machine"))
        first = workers == base
        pct = lambda x, y: "baseline" if first else f"{(x - y) / y * 100:+.0f}%"
        pts = "baseline" if first else f"{s - b_share:+.0f} pts"
        print(
            f"| {workers} | {t:.0f} s | {pct(t, b_time)} | {c:,.0f} CPU-s | {pct(c, b_cpu)} "
            f"| {s:.0f}% | {pts} | {percent(m)} |"
        )


if __name__ == "__main__":
    main()