Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

DeepSeek-V4.1 context scratch: charge the dense KQ scores when flash attention is off (#14838)

Daniel Han54e09f8a4aa5
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index c26e7312883..0441d23fdac 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -19653,10 +19653,20 @@ class LlamaCppBackend:
# deepseek4 the rate has little headroom at 1M (~4%), so a layer split adds the
# extra mask copies like every other arch.
ub_scale = ub / self._DEFAULT_N_UBATCH
- return int(
- self._DSV41_CTX_COMPUTE_FLAT_BYTES
- + (self._DSV41_CTX_COMPUTE_BYTES_PER_TOK * ub_scale + split_extra) * n_ctx
- )
+ per_tok = self._DSV41_CTX_COMPUTE_BYTES_PER_TOK * ub_scale + split_extra
+ flat = self._DSV41_CTX_COMPUTE_FLAT_BYTES
+ if not flash_attn:
+ # Dense branch: f32 KQ scores [window + n_comp, ub, n_head]. Layer score sizes
+ # differ, so the allocator fragments and keeps two copies live (one undercounts).
+ n_head = getattr(self, "_n_heads", None) or max(1, n_embd // 64)
+ score_rate = 2 * ub * n_head * 4
+ window_cells = _pad_kv_cells(
+ self._effective_sliding_window() * max(1, int(n_parallel)) + ub
+ )
+ # The measured rate holds one f16 KQ mask; here it is f32.
+ per_tok += score_rate + ub * 2
+ flat += score_rate * window_cells
+ return int(flat + per_tok * n_ctx)
if getattr(self, "_architecture", None) == "inkling":
ub_scale = ub / self._DEFAULT_N_UBATCH
# The fused banded path requires an f32/f16/bf16 KV cache. A quantized
diff --git a/studio/backend/tests/test_compute_buffer.py b/studio/backend/tests/test_compute_buffer.py
index bc11dbe51b5..bdb788a5a4e 100644
--- a/studio/backend/tests/test_compute_buffer.py
+++ b/studio/backend/tests/test_compute_buffer.py
@@ -521,6 +521,20 @@ class TestContextBufferDSV41:
split = b._compute_buffer_ctx_bytes(262144, 512, "f16", layer_split = True)
assert split - one == (LlamaCppBackend._CTX_COMPUTE_SPLIT_MULT - 1) * 512 * 2 * 262144
+ def test_flash_attention_off_charges_dense_scores(self):
+ b = _backend(embd = 5120, arch = "deepseek41")
+ b._n_heads = 64
+ b._sliding_window = 128
+ b._context_length = 1048576
+ n_ctx = 262144
+ on = b._compute_buffer_ctx_bytes(n_ctx, 512, "f16")
+ off = b._compute_buffer_ctx_bytes(n_ctx, 512, "f16", flash_attn = False)
+ scores = 2 * 512 * 64 * 4
+ # 768 = window 128 + ub 512, padded.
+ assert off - on == (scores + 512 * 2) * n_ctx + scores * 768
+ # Measured flash-off delta on a CPU reserve graph at these shapes.
+ assert off - on >= 64945 * 1024**2
+
class TestContextBufferLayerSplit:
"""The per-device f16 (KQ-mask) rate steps 4x once the model spans >1 device under