Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: show context checkpoints apart from the KV cache in the memory estimate (#11581)

oobabooga6748f09e2886
diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.py
index 6595b5edf76..1e9c6c04520 100644
--- a/studio/backend/core/inference/memory_contract.py
+++ b/studio/backend/core/inference/memory_contract.py
@@ -131,6 +131,7 @@ def build_memory_estimate(
quant_file_bytes = quant,
resident_files_bytes = resident,
kv_bytes = int(getattr(breakdown, "kv_bytes", 0) or 0),
+ kv_checkpoint_bytes = int(getattr(breakdown, "kv_checkpoint_bytes", 0) or 0),
compute_bytes = (
int(getattr(breakdown, "compute_bytes", 0) or 0)
if isinstance(compute_bytes, _Unset)
@@ -185,6 +186,7 @@ def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:
# The aggregate meaning. See the module docstring.
"weights_bytes": estimate.resident_files_bytes,
"kv_bytes": estimate.kv_bytes,
+ "kv_checkpoint_bytes": estimate.kv_checkpoint_bytes,
"compute_bytes": estimate.compute_bytes,
"drafter_runtime_bytes": estimate.drafter_runtime_bytes,
"drafter_runtime_gpu_bytes": estimate.drafter_runtime_gpu_bytes,
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 56319b1daa5..47b988efa64 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -943,6 +943,11 @@ class EstimateMemoryResponse(BaseModel):
)
weights_bytes: int = Field(0, description = "Resident model files: weights, projector, drafter")
kv_bytes: int = Field(0, description = "KV cache at the requested context and slots")
+ kv_checkpoint_bytes: int = Field(
+ 0,
+ description = "Context checkpoints in host RAM: included in kv_bytes and total_bytes, "
+ "excluded from gpu_bytes.",
+ )
compute_bytes: int = Field(0, description = "Compute / graph buffers, flat plus context-linear")
drafter_runtime_bytes: int = Field(
0,
@@ -1047,6 +1052,9 @@ class MemoryEstimate(BaseModel):
)
kv_bytes: int = Field(0, description = "KV cache at the requested context and slots")
+ kv_checkpoint_bytes: int = Field(
+ 0, description = "The host-RAM part of kv_bytes: per-slot context checkpoints"
+ )
compute_bytes: int = Field(0, description = "Compute / graph buffers, flat plus context-linear")
drafter_runtime_bytes: int = Field(
0, description = "A separate drafter's own KV cache and rollback state"
diff --git a/studio/backend/tests/test_memory_contract.py b/studio/backend/tests/test_memory_contract.py
index 38dafd30996..74c850719e8 100644
--- a/studio/backend/tests/test_memory_contract.py
+++ b/studio/backend/tests/test_memory_contract.py
@@ -48,6 +48,7 @@ from core.inference.memory_contract import ( # noqa: E402
_BREAKDOWN = SimpleNamespace(
weights_bytes = 5_000_000_000, # resident: quant + projector + drafter
kv_bytes = 3_000_000_000,
+ kv_checkpoint_bytes = 600_000_000,
compute_bytes = 700_000_000,
drafter_runtime_bytes = 400_000_000,
drafter_runtime_gpu_bytes = 250_000_000,
@@ -205,6 +206,11 @@ class TestTheLegacyProjections:
out["weights_bytes"] == _BREAKDOWN.weights_bytes
), "the Load Model panel itemizes weights_bytes as every resident file"
+ def test_estimate_memory_carries_the_checkpoint_share(self, estimate):
+ out = project_estimate_memory_response(estimate)
+ assert out["kv_checkpoint_bytes"] == _BREAKDOWN.kv_checkpoint_bytes
+ assert out["kv_bytes"] == _BREAKDOWN.kv_bytes
+
def test_kv_cache_estimate_gets_the_quant_file(self, estimate):
out = project_kv_cache_estimate(estimate)
assert out["weights_bytes"] == _QUANT_FILE_BYTES, (
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index ade01122918..336c51678f8 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -1934,6 +1934,29 @@ class TestLaunchShapedPricing:
# But the GPU figure does not move: that is the whole finding.
assert many.gpu_bytes == none.gpu_bytes
+ def test_the_panel_route_reports_the_checkpoint_share(self, swa, monkeypatch):
+ config = SimpleNamespace(
+ identifier = "local/swa",
+ gguf_file = swa,
+ is_gguf = True,
+ gguf_mmproj_file = None,
+ gguf_mtp_file = None,
+ gguf_dspark_file = None,
+ gguf_dflash_file = None,
+ )
+ monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
+ monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 1.0)
+ priced = dict(model_path = swa, n_ctx = 131072, cache_type_kv = "f16")
+ none = _estimate(ctx_checkpoints = 0, **priced)
+ many = _estimate(ctx_checkpoints = 8, **priced)
+
+ assert none.kv_checkpoint_bytes == 0
+ assert many.kv_checkpoint_bytes > 0
+ assert many.kv_checkpoint_bytes == many.kv_bytes - none.kv_bytes
+ assert (many.total_bytes - many.gpu_bytes) - (
+ none.total_bytes - none.gpu_bytes
+ ) == many.kv_checkpoint_bytes
+
def test_one_card_is_priced_as_the_layer_load_it_launches(self, spec_config, swa):
# Tensor mode needs two usable GPUs. Below that load_model drops it, so the
# panel prices the layer load. A tensor device holds the single-device compute
diff --git a/studio/backend/tests/test_memory_estimate_contract_freeze.py b/studio/backend/tests/test_memory_estimate_contract_freeze.py
index bf5707afb90..714dc053343 100644
--- a/studio/backend/tests/test_memory_estimate_contract_freeze.py
+++ b/studio/backend/tests/test_memory_estimate_contract_freeze.py
@@ -225,6 +225,7 @@ class TestTheEstimateMemoryEnvelope:
"reason",
"weights_bytes",
"kv_bytes",
+ "kv_checkpoint_bytes",
"compute_bytes",
"drafter_runtime_bytes",
"drafter_runtime_gpu_bytes",
diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
index 36e41eafef4..588d89af955 100644
--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts
+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts
@@ -22,6 +22,8 @@ export interface MemoryEstimate {
weightsBytes: number;
/** KV cache at the priced context and slot count. Meaningless unless `kvEstimable`. */
kvBytes: number;
+ /** Host RAM checkpoints included in `kvBytes`, capped at `kvBytes`. */
+ kvCheckpointBytes: number;
/** Compute / graph buffers, flat plus the context-linear growth. */
computeBytes: number;
/** A separate drafter's own cache and rollback state, on top of its file. */
@@ -86,6 +88,7 @@ const UNAVAILABLE: MemoryEstimate = {
reason: "unsizable",
weightsBytes: 0,
kvBytes: 0,
+ kvCheckpointBytes: 0,
computeBytes: 0,
drafterRuntimeBytes: 0,
drafterRuntimeGpuBytes: 0,
@@ -109,6 +112,7 @@ interface ApiEstimateResponse {
reason: MemoryEstimateReason | null;
weights_bytes: number;
kv_bytes: number;
+ kv_checkpoint_bytes?: number;
compute_bytes: number;
drafter_runtime_bytes: number;
drafter_runtime_gpu_bytes: number;
@@ -195,6 +199,7 @@ const ESTIMATE_REASONS: readonly MemoryEstimateReason[] = [
function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {
const drafterRuntimeBytes = finiteBytes(body.drafter_runtime_bytes, 0);
+ const kvBytes = finiteBytes(body.kv_bytes, 0);
return {
available: flag(body.available, false),
// A reason the panel has no copy for is not a reason. An unknown string would reach the copy
@@ -203,7 +208,12 @@ function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {
? (body.reason as MemoryEstimateReason)
: null,
weightsBytes: finiteBytes(body.weights_bytes, 0),
- kvBytes: finiteBytes(body.kv_bytes, 0),
+ kvBytes,
+ // Default to zero for older backends; clamp to avoid a negative KV cache row.
+ kvCheckpointBytes: Math.min(
+ kvBytes,
+ finiteBytes(body.kv_checkpoint_bytes, 0),
+ ),
computeBytes: finiteBytes(body.compute_bytes, 0),
drafterRuntimeBytes,
// Absent on a backend predating the split: fall back to the whole term, which keeps the old
diff --git a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
index fb2806bc0de..b0185ae3c82 100644
--- a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
+++ b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx
@@ -299,11 +299,20 @@ export function MemoryEstimateRow({
<MemoryBreakdownLine
label="KV cache"
value={
- estimate.kvEstimable ? formatMemoryGb(estimate.kvBytes) : "unknown"
+ estimate.kvEstimable
+ ? formatMemoryGb(estimate.kvBytes - estimate.kvCheckpointBytes)
+ : "unknown"
}
note={estimate.kvEstimable ? kvNote : undefined}
muted={!estimate.kvEstimable}
/>
+ {estimate.kvEstimable && estimate.kvCheckpointBytes > 0 && (
+ <MemoryBreakdownLine
+ label="Context checkpoints"
+ value={formatMemoryGb(estimate.kvCheckpointBytes)}
+ note="host RAM, filled as conversations grow"
+ />
+ )}
<MemoryBreakdownLine
label="Compute buffers"
value={formatMemoryGb(estimate.computeBytes)}
diff --git a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
index 8027a5f0bb3..6cc19f16a26 100644
--- a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
+++ b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts
@@ -40,6 +40,7 @@ const props: Props = {
reason: null,
weightsBytes: 3.25 * GIB,
kvBytes: 1.5 * GIB,
+ kvCheckpointBytes: 0,
computeBytes: 0.75 * GIB,
drafterRuntimeBytes: 0,
drafterRuntimeGpuBytes: 0,
@@ -103,6 +104,26 @@ test("the disclosure controls an existing breakdown in both states", () => {
}
});
+test("context checkpoints get their own host RAM row instead of inflating the KV cache", () => {
+ // Qwen3.8-27B: 4 slots x 10 checkpoints x 149.625 MiB of recurrent state.
+ const checkpoints = 40 * 149.625 * 1024 ** 2;
+ const html = render({
+ expanded: true,
+ estimate: {
+ ...props.estimate!,
+ kvBytes: 16.584 * GIB + checkpoints,
+ kvCheckpointBytes: checkpoints,
+ },
+ });
+ assert.match(html, /KV cache[\s\S]*?16\.58 GiB/);
+ assert.doesNotMatch(html, /22\.43 GiB/);
+ assert.match(
+ html,
+ /Context checkpoints[\s\S]*?5\.84 GiB[\s\S]*?host RAM, filled as conversations grow/,
+ );
+ assert.doesNotMatch(render({ expanded: true }), /Context checkpoints/);
+});
+
test("a shared pool shows the total without a duplicate GPU figure", () => {
const html = render({ singleMemoryPool: true, isUnifiedMemory: true });
assert.match(html, />Unified<\/span>/);
diff --git a/studio/frontend/tests/memory-estimate-skew.test.ts b/studio/frontend/tests/memory-estimate-skew.test.ts
index f1c96970f55..e8aeb3c7a65 100644
--- a/studio/frontend/tests/memory-estimate-skew.test.ts
+++ b/studio/frontend/tests/memory-estimate-skew.test.ts
@@ -313,6 +313,21 @@ test("a field arriving as a string or a non-finite number does not become a figu
assert.equal(classifyMemoryFit(estimate.totalBytes, 24), "unknown");
});
+test("the checkpoint share defaults to none and never exceeds the cache", async () => {
+ answer(() => json(FULL));
+ assert.equal((await fetchMemoryEstimate(REQUEST)).kvCheckpointBytes, 0);
+ answer(() => json({ ...FULL, kv_checkpoint_bytes: 1024 ** 3 }));
+ assert.equal(
+ (await fetchMemoryEstimate(REQUEST)).kvCheckpointBytes,
+ 1024 ** 3,
+ );
+ answer(() => json({ ...FULL, kv_checkpoint_bytes: 5 * 1024 ** 3 }));
+ assert.equal(
+ (await fetchMemoryEstimate(REQUEST)).kvCheckpointBytes,
+ FULL.kv_bytes,
+ );
+});
+
test("a reason the panel has no copy for is dropped rather than rendered blank", async () => {
answer(() => json({ ...FULL, available: false, reason: "brand_new_reason" }));
assert.equal((await fetchMemoryEstimate(REQUEST)).reason, null);