Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Read the hot-path I/O cost at its steady minimum across repeats (#12029)

Daniel Hane19760a65df8
diff --git a/studio/backend/tests/multi_account/test_perf_gate.py b/studio/backend/tests/multi_account/test_perf_gate.py
index 079ec888c94..f6e0c3c4745 100644
--- a/studio/backend/tests/multi_account/test_perf_gate.py
+++ b/studio/backend/tests/multi_account/test_perf_gate.py
@@ -66,3 +66,25 @@ def test_io_counter_distinguishes_existing_directory_attempts(tmp_path):
result = module.measure_cost(lambda: tmp_path.mkdir(exist_ok = True))
assert result["mkdir_calls"] == 1
assert result["directories_created"] == 0
+
+
+def test_steady_cost_drops_a_one_off_rebuild_but_keeps_a_persistent_one(tmp_path):
+ module = load_script("probe")
+ calls = []
+
+ def one_off_rebuild():
+ calls.append(None)
+ # A cache revalidation that happens to land inside the second measured call.
+ for _ in range(2 if len(calls) == 2 else 1):
+ with closing(sqlite3.connect(tmp_path / "one-off.db")) as conn:
+ conn.execute("SELECT 1")
+
+ assert module.steady_cost(one_off_rebuild)["connections"] == 1
+ assert len(calls) == 5
+
+ def persistent_increase():
+ for _ in range(2):
+ with closing(sqlite3.connect(tmp_path / "persistent.db")) as conn:
+ conn.execute("SELECT 1")
+
+ assert module.steady_cost(persistent_increase)["connections"] == 2
diff --git a/tests/studio/multi_account/perf/probe.py b/tests/studio/multi_account/perf/probe.py
index 4af47a9b019..7a5c902a2c0 100644
--- a/tests/studio/multi_account/perf/probe.py
+++ b/tests/studio/multi_account/perf/probe.py
@@ -58,6 +58,25 @@ def measure_cost(operation, warm = None) -> dict:
return dict(counters)
+def steady_cost(
+ operation,
+ warm = None,
+ repeats = 5,
+) -> dict:
+ """The per-counter minimum of `repeats` warm measurements of the same operation.
+
+ One warm reading is not steady state. The keyless-access settings cache holds its
+ answer for one second from when it was filled, not from its last hit, so now and then
+ the warm-up hits it just before it expires and the measured request refills it,
+ opening a second sqlite connection: about 3 readings in 1500 with eight probes running
+ at once, and a full backend run under xdist read authenticated_get at 2 connections
+ where 1 is pinned. Noise like that only ever adds work, so the minimum of each counter
+ is the cost with nothing refilling, while a cost that really grew grew in every repeat.
+ """
+ readings = [measure_cost(operation, warm = warm) for _ in range(repeats)]
+ return {key: min(reading[key] for reading in readings) for key in readings[0]}
+
+
def main() -> None:
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument("--backend", type = Path, required = True)
@@ -100,13 +119,13 @@ def main() -> None:
assert len(listing.json()["threads"]) == 100
if args.mode == "cost":
measurements = {
- "status": measure_cost(
+ "status": steady_cost(
lambda: get("/api/auth/status"), warm = lambda: get("/api/auth/status")
),
- "authenticated_get": measure_cost(
+ "authenticated_get": steady_cost(
lambda: get("/account-probe"), warm = lambda: get("/account-probe")
),
- "workspace_1000": measure_cost(lambda: [workspace_root() for _ in range(1000)]),
+ "workspace_1000": steady_cost(lambda: [workspace_root() for _ in range(1000)]),
}
else:
measurements = {}