Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: show context checkpoints apart from the KV cache in the memory estimate (#11581)
oobabooga6748f09e2886
diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.pyindex 6595b5edf76..1e9c6c04520 100644--- a/studio/backend/core/inference/memory_contract.py+++ b/studio/backend/core/inference/memory_contract.py@@ -131,6 +131,7 @@ def build_memory_estimate(quant_file_bytes = quant,resident_files_bytes = resident,kv_bytes = int(getattr(breakdown, "kv_bytes", 0) or 0),+ kv_checkpoint_bytes = int(getattr(breakdown, "kv_checkpoint_bytes", 0) or 0),compute_bytes = (int(getattr(breakdown, "compute_bytes", 0) or 0)if isinstance(compute_bytes, _Unset)@@ -185,6 +186,7 @@ def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:# The aggregate meaning. See the module docstring."weights_bytes": estimate.resident_files_bytes,"kv_bytes": estimate.kv_bytes,+ "kv_checkpoint_bytes": estimate.kv_checkpoint_bytes,"compute_bytes": estimate.compute_bytes,"drafter_runtime_bytes": estimate.drafter_runtime_bytes,"drafter_runtime_gpu_bytes": estimate.drafter_runtime_gpu_bytes,diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 56319b1daa5..47b988efa64 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -943,6 +943,11 @@ class EstimateMemoryResponse(BaseModel):)weights_bytes: int = Field(0, description = "Resident model files: weights, projector, drafter")kv_bytes: int = Field(0, description = "KV cache at the requested context and slots")+ kv_checkpoint_bytes: int = Field(+ 0,+ description = "Context checkpoints in host RAM: included in kv_bytes and total_bytes, "+ "excluded from gpu_bytes.",+ )compute_bytes: int = Field(0, description = "Compute / graph buffers, flat plus context-linear")drafter_runtime_bytes: int = Field(0,@@ -1047,6 +1052,9 @@ class MemoryEstimate(BaseModel):)kv_bytes: int = Field(0, description = "KV cache at the requested context and slots")+ kv_checkpoint_bytes: int = Field(+ 0, description = "The host-RAM part of kv_bytes: per-slot context checkpoints"+ )compute_bytes: int = Field(0, description = "Compute / graph buffers, flat plus context-linear")drafter_runtime_bytes: int = Field(0, description = "A separate drafter's own KV cache and rollback state"diff --git a/studio/backend/tests/test_memory_contract.py b/studio/backend/tests/test_memory_contract.pyindex 38dafd30996..74c850719e8 100644--- a/studio/backend/tests/test_memory_contract.py+++ b/studio/backend/tests/test_memory_contract.py@@ -48,6 +48,7 @@ from core.inference.memory_contract import ( # noqa: E402_BREAKDOWN = SimpleNamespace(weights_bytes = 5_000_000_000, # resident: quant + projector + drafterkv_bytes = 3_000_000_000,+ kv_checkpoint_bytes = 600_000_000,compute_bytes = 700_000_000,drafter_runtime_bytes = 400_000_000,drafter_runtime_gpu_bytes = 250_000_000,@@ -205,6 +206,11 @@ class TestTheLegacyProjections:out["weights_bytes"] == _BREAKDOWN.weights_bytes), "the Load Model panel itemizes weights_bytes as every resident file"+ def test_estimate_memory_carries_the_checkpoint_share(self, estimate):+ out = project_estimate_memory_response(estimate)+ assert out["kv_checkpoint_bytes"] == _BREAKDOWN.kv_checkpoint_bytes+ assert out["kv_bytes"] == _BREAKDOWN.kv_bytes+def test_kv_cache_estimate_gets_the_quant_file(self, estimate):out = project_kv_cache_estimate(estimate)assert out["weights_bytes"] == _QUANT_FILE_BYTES, (diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex ade01122918..336c51678f8 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -1934,6 +1934,29 @@ class TestLaunchShapedPricing:# But the GPU figure does not move: that is the whole finding.assert many.gpu_bytes == none.gpu_bytes+ def test_the_panel_route_reports_the_checkpoint_share(self, swa, monkeypatch):+ config = SimpleNamespace(+ identifier = "local/swa",+ gguf_file = swa,+ is_gguf = True,+ gguf_mmproj_file = None,+ gguf_mtp_file = None,+ gguf_dspark_file = None,+ gguf_dflash_file = None,+ )+ monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)+ monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 1.0)+ priced = dict(model_path = swa, n_ctx = 131072, cache_type_kv = "f16")+ none = _estimate(ctx_checkpoints = 0, **priced)+ many = _estimate(ctx_checkpoints = 8, **priced)++ assert none.kv_checkpoint_bytes == 0+ assert many.kv_checkpoint_bytes > 0+ assert many.kv_checkpoint_bytes == many.kv_bytes - none.kv_bytes+ assert (many.total_bytes - many.gpu_bytes) - (+ none.total_bytes - none.gpu_bytes+ ) == many.kv_checkpoint_bytes+def test_one_card_is_priced_as_the_layer_load_it_launches(self, spec_config, swa):# Tensor mode needs two usable GPUs. Below that load_model drops it, so the# panel prices the layer load. A tensor device holds the single-device computediff --git a/studio/backend/tests/test_memory_estimate_contract_freeze.py b/studio/backend/tests/test_memory_estimate_contract_freeze.pyindex bf5707afb90..714dc053343 100644--- a/studio/backend/tests/test_memory_estimate_contract_freeze.py+++ b/studio/backend/tests/test_memory_estimate_contract_freeze.py@@ -225,6 +225,7 @@ class TestTheEstimateMemoryEnvelope:"reason","weights_bytes","kv_bytes",+ "kv_checkpoint_bytes","compute_bytes","drafter_runtime_bytes","drafter_runtime_gpu_bytes",diff --git a/studio/frontend/src/features/model-picker/api/memory-estimate.ts b/studio/frontend/src/features/model-picker/api/memory-estimate.tsindex 36e41eafef4..588d89af955 100644--- a/studio/frontend/src/features/model-picker/api/memory-estimate.ts+++ b/studio/frontend/src/features/model-picker/api/memory-estimate.ts@@ -22,6 +22,8 @@ export interface MemoryEstimate {weightsBytes: number;/** KV cache at the priced context and slot count. Meaningless unless `kvEstimable`. */kvBytes: number;+ /** Host RAM checkpoints included in `kvBytes`, capped at `kvBytes`. */+ kvCheckpointBytes: number;/** Compute / graph buffers, flat plus the context-linear growth. */computeBytes: number;/** A separate drafter's own cache and rollback state, on top of its file. */@@ -86,6 +88,7 @@ const UNAVAILABLE: MemoryEstimate = {reason: "unsizable",weightsBytes: 0,kvBytes: 0,+ kvCheckpointBytes: 0,computeBytes: 0,drafterRuntimeBytes: 0,drafterRuntimeGpuBytes: 0,@@ -109,6 +112,7 @@ interface ApiEstimateResponse {reason: MemoryEstimateReason | null;weights_bytes: number;kv_bytes: number;+ kv_checkpoint_bytes?: number;compute_bytes: number;drafter_runtime_bytes: number;drafter_runtime_gpu_bytes: number;@@ -195,6 +199,7 @@ const ESTIMATE_REASONS: readonly MemoryEstimateReason[] = [function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {const drafterRuntimeBytes = finiteBytes(body.drafter_runtime_bytes, 0);+ const kvBytes = finiteBytes(body.kv_bytes, 0);return {available: flag(body.available, false),// A reason the panel has no copy for is not a reason. An unknown string would reach the copy@@ -203,7 +208,12 @@ function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {? (body.reason as MemoryEstimateReason): null,weightsBytes: finiteBytes(body.weights_bytes, 0),- kvBytes: finiteBytes(body.kv_bytes, 0),+ kvBytes,+ // Default to zero for older backends; clamp to avoid a negative KV cache row.+ kvCheckpointBytes: Math.min(+ kvBytes,+ finiteBytes(body.kv_checkpoint_bytes, 0),+ ),computeBytes: finiteBytes(body.compute_bytes, 0),drafterRuntimeBytes,// Absent on a backend predating the split: fall back to the whole term, which keeps the olddiff --git a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsxindex fb2806bc0de..b0185ae3c82 100644--- a/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx+++ b/studio/frontend/src/features/model-picker/components/memory-estimate-row.tsx@@ -299,11 +299,20 @@ export function MemoryEstimateRow({<MemoryBreakdownLinelabel="KV cache"value={- estimate.kvEstimable ? formatMemoryGb(estimate.kvBytes) : "unknown"+ estimate.kvEstimable+ ? formatMemoryGb(estimate.kvBytes - estimate.kvCheckpointBytes)+ : "unknown"}note={estimate.kvEstimable ? kvNote : undefined}muted={!estimate.kvEstimable}/>+ {estimate.kvEstimable && estimate.kvCheckpointBytes > 0 && (+ <MemoryBreakdownLine+ label="Context checkpoints"+ value={formatMemoryGb(estimate.kvCheckpointBytes)}+ note="host RAM, filled as conversations grow"+ />+ )}<MemoryBreakdownLinelabel="Compute buffers"value={formatMemoryGb(estimate.computeBytes)}diff --git a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts b/studio/frontend/tests/memory-estimate-narrow-panel.test.tsindex 8027a5f0bb3..6cc19f16a26 100644--- a/studio/frontend/tests/memory-estimate-narrow-panel.test.ts+++ b/studio/frontend/tests/memory-estimate-narrow-panel.test.ts@@ -40,6 +40,7 @@ const props: Props = {reason: null,weightsBytes: 3.25 * GIB,kvBytes: 1.5 * GIB,+ kvCheckpointBytes: 0,computeBytes: 0.75 * GIB,drafterRuntimeBytes: 0,drafterRuntimeGpuBytes: 0,@@ -103,6 +104,26 @@ test("the disclosure controls an existing breakdown in both states", () => {}});+test("context checkpoints get their own host RAM row instead of inflating the KV cache", () => {+ // Qwen3.8-27B: 4 slots x 10 checkpoints x 149.625 MiB of recurrent state.+ const checkpoints = 40 * 149.625 * 1024 ** 2;+ const html = render({+ expanded: true,+ estimate: {+ ...props.estimate!,+ kvBytes: 16.584 * GIB + checkpoints,+ kvCheckpointBytes: checkpoints,+ },+ });+ assert.match(html, /KV cache[\s\S]*?16\.58 GiB/);+ assert.doesNotMatch(html, /22\.43 GiB/);+ assert.match(+ html,+ /Context checkpoints[\s\S]*?5\.84 GiB[\s\S]*?host RAM, filled as conversations grow/,+ );+ assert.doesNotMatch(render({ expanded: true }), /Context checkpoints/);+});+test("a shared pool shows the total without a duplicate GPU figure", () => {const html = render({ singleMemoryPool: true, isUnifiedMemory: true });assert.match(html, />Unified<\/span>/);diff --git a/studio/frontend/tests/memory-estimate-skew.test.ts b/studio/frontend/tests/memory-estimate-skew.test.tsindex f1c96970f55..e8aeb3c7a65 100644--- a/studio/frontend/tests/memory-estimate-skew.test.ts+++ b/studio/frontend/tests/memory-estimate-skew.test.ts@@ -313,6 +313,21 @@ test("a field arriving as a string or a non-finite number does not become a figuassert.equal(classifyMemoryFit(estimate.totalBytes, 24), "unknown");});+test("the checkpoint share defaults to none and never exceeds the cache", async () => {+ answer(() => json(FULL));+ assert.equal((await fetchMemoryEstimate(REQUEST)).kvCheckpointBytes, 0);+ answer(() => json({ ...FULL, kv_checkpoint_bytes: 1024 ** 3 }));+ assert.equal(+ (await fetchMemoryEstimate(REQUEST)).kvCheckpointBytes,+ 1024 ** 3,+ );+ answer(() => json({ ...FULL, kv_checkpoint_bytes: 5 * 1024 ** 3 }));+ assert.equal(+ (await fetchMemoryEstimate(REQUEST)).kvCheckpointBytes,+ FULL.kv_bytes,+ );+});+test("a reason the panel has no copy for is dropped rather than rendered blank", async () => {answer(() => json({ ...FULL, available: false, reason: "brand_new_reason" }));assert.equal((await fetchMemoryEstimate(REQUEST)).reason, null);