Skip to content

Worktrunk is a CLI for Git worktree management, designed for parallel AI agent workflows

bench: use BatchSize::PerIteration for cold-cache variants (#2728)

Maximilian Roosd7c8428e16c6
diff --git a/benches/CLAUDE.md b/benches/CLAUDE.md
index 47429f8c6..5cb067cc0 100644
--- a/benches/CLAUDE.md
+++ b/benches/CLAUDE.md
@@ -80,6 +80,18 @@ a cache hit. Invalidate via `criterion::Bencher::iter_batched` with
`wt_perf::invalidate_caches_auto` as the setup closure (see the cold-cache variants in
`benches/list.rs` and `benches/remove.rs` for the pattern).
+**Pass `BatchSize::PerIteration`, not `BatchSize::SmallInput`.** When the setup
+invalidates a cache that the routine repopulates, the batch size matters:
+`SmallInput` calls `setup()` once per batch up front, then times the routines
+back-to-back inside one timing window, so only iter 1 per batch is actually cold
+— iters 2-N hit a cache that the previous iter just populated. The reported
+"cold" median is a warm-biased average. `PerIteration` runs `setup → time(routine)`
+per iter, so every measured iter is genuinely cold. The setup is far cheaper than
+a `wt` subprocess, so per-iter `Instant::now` overhead doesn't dominate. When the
+fix landed across `list.rs` / `remove.rs` / `time_to_first_output.rs`, cold variance
+tightened (e.g. `first_output/remove` spread 2.4ms → 0.65ms) and the median rose
+to its true cold cost (e.g. `remove_e2e/first_output` 48ms → 86ms).
+
`invalidate_caches_auto` clears:
- `.git/index` (main and linked worktrees)
diff --git a/benches/list.rs b/benches/list.rs
index e8eadd106..750c39af1 100644
--- a/benches/list.rs
+++ b/benches/list.rs
@@ -83,12 +83,19 @@ fn run_benchmark(
};
if config.cold_cache {
+ // `BatchSize::PerIteration` (not `SmallInput`): under `SmallInput`,
+ // criterion calls `setup` for an entire batch up front and then runs
+ // the timed routines back-to-back — so only the first `wt` per batch
+ // is cold and the rest hit a freshly populated `.git/wt/cache/`,
+ // biasing "cold" warm. `PerIteration` invalidates immediately before
+ // every measured iteration; the setup is far cheaper than a `wt`
+ // subprocess, so per-iter `Instant::now` overhead doesn't dominate.
b.iter_batched(
|| invalidate_caches_auto(repo_path),
|_| {
cmd_factory().output().unwrap();
},
- criterion::BatchSize::SmallInput,
+ criterion::BatchSize::PerIteration,
);
} else {
b.iter(|| {
@@ -216,12 +223,14 @@ fn bench_real_repo(c: &mut Criterion) {
};
if cold {
+ // `PerIteration` so every measured run is actually
+ // cold — see `run_benchmark` above for the rationale.
b.iter_batched(
|| invalidate_caches_auto(&workspace_main),
|_| {
make_cmd().output().unwrap();
},
- criterion::BatchSize::SmallInput,
+ criterion::BatchSize::PerIteration,
);
} else {
b.iter(|| {
diff --git a/benches/remove.rs b/benches/remove.rs
index 0f4d322f4..0a19d910b 100644
--- a/benches/remove.rs
+++ b/benches/remove.rs
@@ -114,7 +114,10 @@ fn bench_remove_e2e(c: &mut Criterion) {
// Invalidates caches per iteration so the timing reflects first-invocation
// TTFO — `prepare_worktree_removal` writes to `.git/wt/cache/` via
// `compute_integration_lazy`, and reusing it across iterations would
- // measure warm-cache cost instead.
+ // measure warm-cache cost instead. `BatchSize::PerIteration` (not
+ // `SmallInput`) so the setup actually runs before every measured iter —
+ // under `SmallInput`, criterion calls `setup` once per batch and runs
+ // the timed routines back-to-back, leaving only iter 1 cold per batch.
group.bench_function("first_output", |b| {
b.iter_batched(
|| invalidate_caches_auto(&repo_no_hooks),
@@ -131,7 +134,7 @@ fn bench_remove_e2e(c: &mut Criterion) {
String::from_utf8_lossy(&output.stderr)
);
},
- criterion::BatchSize::SmallInput,
+ criterion::BatchSize::PerIteration,
);
});
diff --git a/benches/time_to_first_output.rs b/benches/time_to_first_output.rs
index 143887d49..daca32c07 100644
--- a/benches/time_to_first_output.rs
+++ b/benches/time_to_first_output.rs
@@ -43,7 +43,11 @@ fn bench_first_output(c: &mut Criterion) {
// on the first invocation. Without invalidation between iterations, iter 1
// is cold and iter 2+ read from cache — the reported timing would be warm
// cache, which doesn't reflect the first-invocation TTFO a user sees.
- // Invalidate via `iter_batched` so every iteration starts cold.
+ // Invalidate via `iter_batched` so every iteration starts cold; use
+ // `BatchSize::PerIteration` (not `SmallInput`) so setup runs immediately
+ // before every measured iter — under `SmallInput`, criterion would run
+ // setup once per batch and time the routines back-to-back, leaving only
+ // iter 1 actually cold per batch.
group.bench_function("remove", |b| {
b.iter_batched(
|| invalidate_caches_auto(&repo_path),
@@ -58,7 +62,7 @@ fn bench_first_output(c: &mut Criterion) {
String::from_utf8_lossy(&output.stderr)
);
},
- criterion::BatchSize::SmallInput,
+ criterion::BatchSize::PerIteration,
);
});