Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fall back to a dynamic KV cache when the static one would not fit (#13075)

Daniel Han8435b96d8872
diff --git a/tests/test_static_cache_memory_fallback.py b/tests/test_static_cache_memory_fallback.py
new file mode 100644
index 00000000000..9720ebd720c
--- /dev/null
+++ b/tests/test_static_cache_memory_fallback.py
@@ -0,0 +1,415 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Static cache memory fallback for generate (unslothai/unsloth#2590).
+
+FastModel generate forces a static KV cache, sized for prompt + max_new_tokens before the first
+token. On a memory-tight or unified-memory device (Jetson) that allocation alone can OOM where a
+growing cache would not, so generate must fall back to a dynamic cache when it does not fit.
+"""
+
+import types
+
+import pytest
+import torch
+
+from unsloth.models import vision
+from unsloth.models.vision import _static_cache_bytes, _static_cache_does_not_fit
+
+GB = 1024**3
+
+
+def _llama_config(**overrides):
+ from transformers import LlamaConfig
+
+ kwargs = dict(
+ hidden_size = 256,
+ num_attention_heads = 8,
+ num_key_value_heads = 2,
+ num_hidden_layers = 4,
+ intermediate_size = 512,
+ vocab_size = 1000,
+ )
+ kwargs.update(overrides)
+ return LlamaConfig(**kwargs)
+
+
+def _model(
+ config,
+ dtype = torch.bfloat16,
+ device = "cuda",
+ device_map = None,
+):
+ from transformers import GenerationConfig
+ return types.SimpleNamespace(
+ config = config,
+ generation_config = GenerationConfig(),
+ dtype = dtype,
+ device = torch.device(device),
+ hf_device_map = device_map,
+ )
+
+
+def _real_static_cache_bytes(config, batch, max_cache_len, dtype):
+ """What transformers itself allocates, the reference the estimate must match."""
+ from transformers import StaticCache
+
+ cache = StaticCache(config = config, max_cache_len = max_cache_len)
+ head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
+ cache.early_initialization(
+ batch_size = batch,
+ num_heads = config.num_key_value_heads,
+ head_dim = head_dim,
+ dtype = dtype,
+ device = "cpu",
+ )
+ return sum(
+ t.numel() * t.element_size() for layer in cache.layers for t in (layer.keys, layer.values)
+ )
+
+
+def _require_early_initialization():
+ from transformers import StaticCache
+ if not hasattr(StaticCache, "early_initialization"):
+ pytest.skip(
+ reason = "the reference, StaticCache.early_initialization, is missing from this "
+ "transformers; the remaining tests still check the estimate",
+ )
+
+
[email protected]("batch", [1, 3])
+def test_estimate_matches_transformers_full_attention(batch):
+ _require_early_initialization()
+ config = _llama_config()
+ input_ids = torch.zeros(batch, 37, dtype = torch.long)
+ need = _static_cache_bytes(_model(config), input_ids, {"max_new_tokens": 91})
+ assert need == _real_static_cache_bytes(config, batch, 37 + 91 - 1, torch.bfloat16)
+
+
+def test_estimate_matches_transformers_sliding_window():
+ _require_early_initialization()
+ config = _llama_config(
+ sliding_window = 16,
+ layer_types = ["sliding_attention", "full_attention"] * 2,
+ )
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ need = _static_cache_bytes(_model(config), input_ids, {"max_new_tokens": 100})
+ assert need == _real_static_cache_bytes(config, 1, 109, torch.bfloat16)
+
+
+def test_estimate_matches_transformers_sliding_window_without_layer_types():
+ """Mistral style: a sliding_window and no layer_types makes every layer sliding."""
+ _require_early_initialization()
+ from transformers import MistralConfig
+
+ config = MistralConfig(
+ hidden_size = 256,
+ num_attention_heads = 8,
+ num_key_value_heads = 2,
+ num_hidden_layers = 4,
+ intermediate_size = 512,
+ vocab_size = 1000,
+ sliding_window = 16,
+ )
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ need = _static_cache_bytes(_model(config), input_ids, {"max_new_tokens": 100})
+ assert need == _real_static_cache_bytes(config, 1, 109, torch.bfloat16)
+
+
+def test_estimate_uses_mla_key_and_value_dims():
+ from transformers import DeepseekV3Config
+
+ config = DeepseekV3Config(
+ hidden_size = 256,
+ num_attention_heads = 4,
+ num_key_value_heads = 4,
+ num_hidden_layers = 3,
+ qk_nope_head_dim = 48,
+ qk_rope_head_dim = 16,
+ v_head_dim = 32,
+ vocab_size = 1000,
+ )
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ need = _static_cache_bytes(_model(config), input_ids, {"max_new_tokens": 90})
+ assert need == 1 * 4 * (48 + 16 + 32) * 2 * 99 * 3
+
+
+def test_estimate_default_length_adds_the_prompt():
+ """transformers turns a default max_length of 20 into 20 new tokens after the prompt."""
+ config = _llama_config()
+ model = _model(config)
+ model.generation_config.max_length = None
+ input_ids = torch.zeros(1, 500, dtype = torch.long)
+ assert _static_cache_bytes(model, input_ids, {}) == _static_cache_bytes(
+ model, input_ids, {"max_new_tokens": 20}
+ )
+
+
+def test_estimate_default_length_capped_at_context():
+ config = _llama_config(max_position_embeddings = 4096)
+ model = _model(config)
+ model.generation_config.max_length = 4096
+ input_ids = torch.zeros(1, 500, dtype = torch.long)
+ assert _static_cache_bytes(model, input_ids, {}) == _static_cache_bytes(
+ model, torch.zeros(1, 1, dtype = torch.long), {"max_new_tokens": 4095}
+ )
+
+
+def test_partial_caller_config_inherits_the_model_length():
+ """A caller GenerationConfig that only sets sampling still gets the model's max_length."""
+ from transformers import GenerationConfig
+
+ config = _llama_config(max_position_embeddings = 131072)
+ model = _model(config)
+ model.generation_config.max_length = 131072
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ caller = GenerationConfig(do_sample = True, temperature = 0.7)
+ caller.max_length = None
+ assert _static_cache_bytes(model, input_ids, {"generation_config": caller}) == (
+ _static_cache_bytes(model, torch.zeros(1, 1, dtype = torch.long), {"max_new_tokens": 131071})
+ )
+
+
+def test_explicit_max_length_is_the_total():
+ config = _llama_config()
+ model = _model(config)
+ input_ids = torch.zeros(1, 900, dtype = torch.long)
+ assert _static_cache_bytes(model, input_ids, {"max_length": 1000}) == _static_cache_bytes(
+ model, input_ids, {"max_new_tokens": 100}
+ )
+
+
+def test_legacy_static_cache_is_never_constructed(monkeypatch):
+ """Before transformers 4.56 StaticCache allocates in __init__: count every layer as full."""
+ import transformers
+
+ def explode(*args, **kwargs):
+ raise AssertionError("constructed a legacy StaticCache")
+
+ monkeypatch.setattr(vision, "_static_cache_preallocates", lambda: True)
+ monkeypatch.setattr(transformers, "StaticCache", explode)
+ config = _llama_config(sliding_window = 16, layer_types = ["sliding_attention"] * 4)
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ need = _static_cache_bytes(_model(config), input_ids, {"max_new_tokens": 90})
+ assert need == 1 * 2 * (32 + 32) * 2 * 99 * 4
+
+
+def test_estimate_uses_the_compile_decode_bucket(monkeypatch):
+ config = _llama_config()
+ input_ids = torch.zeros(1, 1, dtype = torch.long)
+ plain = _static_cache_bytes(_model(config), input_ids, {"max_new_tokens": 1026})
+ monkeypatch.setattr(vision, "_compiles_decode", lambda model: True)
+ bucketed = _static_cache_bytes(_model(config), input_ids, {"max_new_tokens": 1026})
+ assert bucketed * 1026 == plain * 2048
+
+
+def test_estimate_counts_beams_and_previous_length():
+ config = _llama_config()
+ model = _model(config)
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ one = _static_cache_bytes(model, input_ids, {"max_new_tokens": 90})
+ assert _static_cache_bytes(model, input_ids, {"max_new_tokens": 90, "num_beams": 4}) == 4 * one
+ # transformers reuses the largest length an earlier call allocated
+ model._previous_max_cache_length = 990
+ assert _static_cache_bytes(model, input_ids, {"max_new_tokens": 90}) == 10 * one
+
+
+def test_no_estimate_with_caller_cache_or_encoder_decoder():
+ config = _llama_config()
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ assert _static_cache_bytes(_model(config), input_ids, {"past_key_values": object()}) is None
+ config.is_encoder_decoder = True
+ assert _static_cache_bytes(_model(config), input_ids, {"max_new_tokens": 5}) is None
+
+
+def fake_cuda(monkeypatch):
+ state = {"free": 0, "reserved": 0, "allocated": 0, "total": 1024}
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (state["free"], 80 * GB))
+ # A tiny total keeps every test cache above the 1/16 of the card that skips the probe.
+ monkeypatch.setattr(
+ torch.cuda,
+ "get_device_properties",
+ lambda device = None: types.SimpleNamespace(total_memory = state["total"]),
+ )
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: state["reserved"])
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: state["allocated"])
+ monkeypatch.setattr(vision.logger, "warning_once", lambda *a, **k: None)
+ return state
+
+
+def _cache_bytes_for(tokens):
+ config = _llama_config()
+ return config, _static_cache_bytes(
+ _model(config), torch.zeros(1, 1, dtype = torch.long), {"max_new_tokens": tokens - 1}
+ )
+
+
+def test_small_cache_keeps_static(fake_cuda):
+ config, need = _cache_bytes_for(4096)
+ fake_cuda["free"] = 4 * need
+ ids = torch.zeros(1, 1, dtype = torch.long)
+ assert not _static_cache_does_not_fit(_model(config), ids, {"max_new_tokens": 4095})
+
+
+def test_cache_over_half_of_free_falls_back(fake_cuda):
+ config, need = _cache_bytes_for(4096)
+ fake_cuda["free"] = need
+ ids = torch.zeros(1, 1, dtype = torch.long)
+ assert _static_cache_does_not_fit(_model(config), ids, {"max_new_tokens": 4095})
+
+
+def test_allocator_cached_blocks_count_as_free(fake_cuda):
+ config, need = _cache_bytes_for(4096)
+ fake_cuda.update(free = need, reserved = 5 * need, allocated = 2 * need)
+ ids = torch.zeros(1, 1, dtype = torch.long)
+ assert not _static_cache_does_not_fit(_model(config), ids, {"max_new_tokens": 4095})
+
+
+ "model_kwargs",
+ [
+ {"device": "cpu"},
+ {"device_map": {"model.layers.0": 0, "model.layers.1": 1}},
+ ],
+)
+def test_unmeasured_placements_keep_static(fake_cuda, model_kwargs):
+ config, need = _cache_bytes_for(4096)
+ fake_cuda["free"] = 0
+ ids = torch.zeros(1, 1, dtype = torch.long)
+ assert not _static_cache_does_not_fit(
+ _model(config, **model_kwargs), ids, {"max_new_tokens": 4095}
+ )
+
+
+def test_xpu_device_is_measured(monkeypatch):
+ xpu = types.SimpleNamespace(
+ get_device_properties = lambda device = None: types.SimpleNamespace(total_memory = 1024),
+ mem_get_info = lambda device = None: (0, 80 * GB),
+ memory_reserved = lambda device = None: 0,
+ memory_allocated = lambda device = None: 0,
+ )
+ monkeypatch.setattr(torch, "xpu", xpu, raising = False)
+ monkeypatch.setattr(vision.logger, "warning_once", lambda *a, **k: None)
+ ids = torch.zeros(1, 1, dtype = torch.long)
+ model = _model(_llama_config(), device = "xpu")
+ assert _static_cache_does_not_fit(model, ids, {"max_new_tokens": 4095})
+
+
+def test_probe_failure_keeps_static(fake_cuda, monkeypatch):
+ def boom(device = None):
+ raise RuntimeError("no device")
+
+ monkeypatch.setattr(torch.cuda, "mem_get_info", boom)
+ config = _llama_config()
+ ids = torch.zeros(1, 1, dtype = torch.long)
+ assert not _static_cache_does_not_fit(_model(config), ids, {"max_new_tokens": 4095})
+
+
+def _tiny_generate(monkeypatch, free_bytes):
+ """Real generate through unsloth_base_fast_generate on a random tiny Llama, with the
+ device reporting `free_bytes` free. Returns the cache class generate ended with."""
+ if not torch.cuda.is_available():
+ pytest.skip(reason = "the guard only measures CUDA devices")
+ from transformers import GenerationConfig, LlamaForCausalLM
+
+ torch.manual_seed(0)
+ dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
+ model = LlamaForCausalLM(_llama_config()).to("cuda", dtype).eval()
+ model.generation_config = GenerationConfig(max_length = 4096, pad_token_id = 0)
+ model._old_generate = model.generate
+ model.generate = types.MethodType(vision.unsloth_base_fast_generate, model)
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (free_bytes, 80 * GB))
+ real_properties = torch.cuda.get_device_properties
+
+ class SmallCard:
+ def __init__(self, props):
+ self.props = props
+
+ total_memory = 1024
+
+ def __getattr__(self, name):
+ return getattr(self.props, name)
+
+ monkeypatch.setattr(
+ torch.cuda,
+ "get_device_properties",
+ lambda device = None: SmallCard(real_properties(device)),
+ )
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 0)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)
+ monkeypatch.setattr(vision.logger, "warning_once", lambda *a, **k: None)
+ ids = torch.ones(1, 5, dtype = torch.long, device = "cuda")
+ out = model.generate(
+ input_ids = ids,
+ attention_mask = torch.ones_like(ids),
+ max_new_tokens = 3,
+ do_sample = False,
+ return_dict_in_generate = True,
+ )
+ return type(out.past_key_values).__name__
+
+
+def test_generate_uses_dynamic_cache_when_static_does_not_fit(monkeypatch):
+ assert _tiny_generate(monkeypatch, free_bytes = 1024) == "DynamicCache"
+
+
+def test_generate_keeps_static_cache_when_it_fits(monkeypatch):
+ assert _tiny_generate(monkeypatch, free_bytes = 40 * GB) == "StaticCache"
+
+
+def test_upper_bound_is_never_below_the_exact_size():
+ config = _llama_config(
+ sliding_window = 16, layer_types = ["sliding_attention", "full_attention"] * 2
+ )
+ model = _model(config)
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ exact = _static_cache_bytes(model, input_ids, {"max_new_tokens": 500})
+ bound = _static_cache_bytes(model, input_ids, {"max_new_tokens": 500}, exact = False)
+ assert bound >= exact and bound == 4 * 509 * 2 * (32 + 32) * 2
+
+
+def test_sliding_model_that_fits_exactly_keeps_static(fake_cuda):
+ """The upper bound alone would fall back; the exact layer plan says it fits."""
+ config = _llama_config(
+ sliding_window = 16, layer_types = ["sliding_attention"] * 3 + ["full_attention"]
+ )
+ model = _model(config)
+ ids = torch.zeros(1, 1, dtype = torch.long)
+ exact = _static_cache_bytes(model, ids, {"max_new_tokens": 4095})
+ bound = _static_cache_bytes(model, ids, {"max_new_tokens": 4095}, exact = False)
+ fake_cuda["free"] = 2 * exact + 1
+ assert bound > fake_cuda["free"] // 2
+ assert not _static_cache_does_not_fit(model, ids, {"max_new_tokens": 4095})
+
+
+def test_caller_config_default_values_count_as_unset(monkeypatch):
+ """transformers 4.x leaves max_length=20 / num_beams=1 on a partial GenerationConfig."""
+ from transformers import GenerationConfig
+
+ four_x_defaults = types.SimpleNamespace(max_length = 20, num_beams = 1, num_return_sequences = 1)
+ monkeypatch.setattr(vision, "_default_generation_config", lambda: four_x_defaults)
+
+ config = _llama_config(max_position_embeddings = 131072)
+ model = _model(config)
+ model.generation_config.max_length = 131072
+ model.generation_config.num_beams = 2
+ input_ids = torch.zeros(1, 10, dtype = torch.long)
+ caller = GenerationConfig(do_sample = True, max_length = 20, num_beams = 1)
+ one_beam = _static_cache_bytes(
+ model, torch.zeros(1, 1, dtype = torch.long), {"max_new_tokens": 131071, "num_beams": 1}
+ )
+ assert _static_cache_bytes(model, input_ids, {"generation_config": caller}) == 2 * one_beam
+
+
+def test_small_cache_skips_the_driver_probe(fake_cuda, monkeypatch):
+ """A cache under 1/16 of the card never calls mem_get_info (a slow driver call)."""
+
+ def probe(device = None):
+ raise AssertionError("mem_get_info called")
+
+ config, need = _cache_bytes_for(4096)
+ fake_cuda["total"] = 64 * need
+ monkeypatch.setattr(torch.cuda, "mem_get_info", probe)
+ ids = torch.zeros(1, 1, dtype = torch.long)
+ assert not _static_cache_does_not_fit(_model(config), ids, {"max_new_tokens": 4095})
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 01d2af1b443..5301ac5d444 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1435,6 +1435,129 @@ def _dynamic_cache_choice(kwargs):
return requested
[email protected]_cache(maxsize = None)
+def _default_generation_config():
+ return GenerationConfig()
+
+
[email protected]_cache(maxsize = None)
+def _static_cache_preallocates():
+ from transformers import StaticCache
+ return "max_batch_size" in inspect.signature(StaticCache.__init__).parameters
+
+
+def _static_cache_bytes(
+ model,
+ input_ids,
+ kwargs,
+ exact = True,
+):
+ """Bytes the static KV cache transformers preallocates for this call, or None when it cannot
+ be sized. Worst case length (prompt + max_new_tokens), whatever length the reply turns out.
+ exact = False skips the layer plan and counts every layer full length: an upper bound."""
+ from transformers import StaticCache
+
+ config = model.config
+ if getattr(config, "is_encoder_decoder", False) or kwargs.get("past_key_values") is not None:
+ return None
+ text_config = config.get_text_config(decoder = True)
+ # transformers fills a caller config's unset fields from the model's before its defaults.
+ configs = (kwargs.get("generation_config"), getattr(model, "generation_config", None))
+
+ def option(name, default = None):
+ if kwargs.get(name) is not None:
+ return kwargs[name]
+ caller, own = configs
+ value = getattr(caller, name, None)
+ # transformers 4.x fills unset caller fields with the global default, not None.
+ if value is not None and value != getattr(_default_generation_config(), name, None):
+ return value
+ if getattr(own, name, None) is not None:
+ return getattr(own, name)
+ return default if value is None else value
+
+ prompt = input_ids.shape[1]
+ max_new_tokens = option("max_new_tokens")
+ if max_new_tokens is not None:
+ length = prompt + max_new_tokens
+ elif kwargs.get("max_length") is not None:
+ length = kwargs["max_length"]
+ else:
+ # Upper bound of _prepare_generated_length: a default max_length counts new tokens.
+ max_length = option("max_length", 20)
+ length = max_length + prompt
+ max_positions = getattr(text_config, "max_position_embeddings", None)
+ if type(max_positions) is int and max_length <= max_positions:
+ length = min(length, max_positions)
+ if type(length) is not int:
+ return None
+ # transformers sizes the cache max_length - 1: the last token is never cached.
+ length = max(length - 1, getattr(model, "_previous_max_cache_length", -1))
+ if _compiles_decode(model):
+ length = _decode_cache_bucket(length)
+ copies = max(option("num_beams", 1), option("num_return_sequences", 1))
+ if not exact or _static_cache_preallocates():
+ # Before transformers 4.56 constructing one allocates it, so count every layer as full.
+ tokens = text_config.num_hidden_layers * length
+ else:
+ # transformers' own layer plan: sliding / chunked windows, shared-KV and linear layers.
+ layers = StaticCache(config = text_config, max_cache_len = length).layers
+ tokens = sum(
+ layer.max_cache_len
+ for layer in layers
+ if type(getattr(layer, "max_cache_len", None)) is int
+ )
+ n_heads = text_config.num_attention_heads
+ kv_heads = getattr(text_config, "num_key_value_heads", None) or n_heads
+ head_dim = getattr(text_config, "head_dim", None) or text_config.hidden_size // n_heads
+ # MLA (DeepSeek) caches keys of nope + rope dims and values of v_head_dim.
+ nope = getattr(text_config, "qk_nope_head_dim", None)
+ rope = getattr(text_config, "qk_rope_head_dim", None)
+ key_dim = nope + rope if type(nope) is int and type(rope) is int else head_dim
+ value_dim = getattr(text_config, "v_head_dim", None) or head_dim
+ itemsize = torch.empty((), dtype = model.dtype).element_size()
+ return input_ids.shape[0] * copies * kv_heads * (key_dim + value_dim) * itemsize * tokens
+
+
+def _static_cache_does_not_fit(model, input_ids, kwargs):
+ """True when the preallocated static cache would take over half the memory this device has
+ left; the rest is for prefill activations and logits. Fails open: unknown keeps static."""
+ try:
+ device = model.device
+ backend = {"cuda": torch.cuda, "xpu": getattr(torch, "xpu", None)}.get(device.type)
+ if backend is None:
+ return False
+ device_map = getattr(model, "hf_device_map", None)
+ if isinstance(device_map, dict) and len(set(map(str, device_map.values()))) > 1:
+ return False
+ # The upper bound settles the common case without building the layer plan (~0.3 ms).
+ need = _static_cache_bytes(model, input_ids, kwargs, exact = False)
+ if need is None:
+ return False
+ # mem_get_info is a driver call (p90 7-10 ms on a busy host), so a cache under 1/16
+ # of the card never pays it.
+ if need * 16 <= backend.get_device_properties(device).total_memory:
+ return False
+ free = backend.mem_get_info(device)[0]
+ if need <= free // 2:
+ return False
+ need = _static_cache_bytes(model, input_ids, kwargs)
+ if need <= free // 2:
+ return False
+ # memory_stats is slow, so allocator-held free blocks are read only when needed.
+ free += backend.memory_reserved(device) - backend.memory_allocated(device)
+ except Exception:
+ return False
+ if need <= free // 2:
+ return False
+ logger.warning_once(
+ f"Unsloth: A static KV cache for this generate call needs {need / 1024**3:.1f} GB, "
+ f"but only {free / 1024**3:.1f} GB is free. Using a dynamic cache instead; "
+ "lower max_new_tokens to keep the static cache."
+ )
+ return True
+
+
def _uses_flash_attention_for_generation(config):
language_config_names = (
"text_config",
@@ -1717,6 +1840,10 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
force_dynamic_cache = kwargs.get(
"past_key_values"
) is None and _needs_bidirectional_multimodal_mask(self, kwargs)
+ # A static cache is sized for the longest reply up front, so on a tight or unified memory
+ # device it can OOM where a growing cache would not (#2590).
+ if not force_dynamic_cache and cache_implementation is not None:
+ force_dynamic_cache = _static_cache_does_not_fit(self, input_ids, kwargs)
if force_dynamic_cache:
cache_implementation = None
dynamic_implementation = _dynamic_cache_choice(kwargs)