Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
DeepSeek-V4.1 context scratch: charge the dense KQ scores when flash attention is off (#14838)
Daniel Han54e09f8a4aa5
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex c26e7312883..0441d23fdac 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -19653,10 +19653,20 @@ class LlamaCppBackend:# deepseek4 the rate has little headroom at 1M (~4%), so a layer split adds the# extra mask copies like every other arch.ub_scale = ub / self._DEFAULT_N_UBATCH- return int(- self._DSV41_CTX_COMPUTE_FLAT_BYTES- + (self._DSV41_CTX_COMPUTE_BYTES_PER_TOK * ub_scale + split_extra) * n_ctx- )+ per_tok = self._DSV41_CTX_COMPUTE_BYTES_PER_TOK * ub_scale + split_extra+ flat = self._DSV41_CTX_COMPUTE_FLAT_BYTES+ if not flash_attn:+ # Dense branch: f32 KQ scores [window + n_comp, ub, n_head]. Layer score sizes+ # differ, so the allocator fragments and keeps two copies live (one undercounts).+ n_head = getattr(self, "_n_heads", None) or max(1, n_embd // 64)+ score_rate = 2 * ub * n_head * 4+ window_cells = _pad_kv_cells(+ self._effective_sliding_window() * max(1, int(n_parallel)) + ub+ )+ # The measured rate holds one f16 KQ mask; here it is f32.+ per_tok += score_rate + ub * 2+ flat += score_rate * window_cells+ return int(flat + per_tok * n_ctx)if getattr(self, "_architecture", None) == "inkling":ub_scale = ub / self._DEFAULT_N_UBATCH# The fused banded path requires an f32/f16/bf16 KV cache. A quantizeddiff --git a/studio/backend/tests/test_compute_buffer.py b/studio/backend/tests/test_compute_buffer.pyindex bc11dbe51b5..bdb788a5a4e 100644--- a/studio/backend/tests/test_compute_buffer.py+++ b/studio/backend/tests/test_compute_buffer.py@@ -521,6 +521,20 @@ class TestContextBufferDSV41:split = b._compute_buffer_ctx_bytes(262144, 512, "f16", layer_split = True)assert split - one == (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * 512 * 2 * 262144+ def test_flash_attention_off_charges_dense_scores(self):+ b = _backend(embd = 5120, arch = "deepseek41")+ b._n_heads = 64+ b._sliding_window = 128+ b._context_length = 1048576+ n_ctx = 262144+ on = b._compute_buffer_ctx_bytes(n_ctx, 512, "f16")+ off = b._compute_buffer_ctx_bytes(n_ctx, 512, "f16", flash_attn = False)+ scores = 2 * 512 * 64 * 4+ # 768 = window 128 + ub 512, padded.+ assert off - on == (scores + 512 * 2) * n_ctx + scores * 768+ # Measured flash-off delta on a CPU reserve graph at these shapes.+ assert off - on >= 64945 * 1024**2+class TestContextBufferLayerSplit:"""The per-device f16 (KQ-mask) rate steps 4x once the model spans >1 device under