Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: explain GGUFs the bundled llama.cpp cannot read, and default Ternary Bonsai to its loadable pack (#13304)
Daniel Han83a761ba72ed
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex a46f2576196..2c2f2ff91b3 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -3309,7 +3309,7 @@ _GGUF_KNOWN_QUANT_RE = re.compile(r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"r"|P?TQ[0-9]+_[0-9]+"r"|Q[0-9]+_K_[A-Z]+"- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"r"|Q[0-9]+_K"r"|BF16|F16|F32)",re.IGNORECASE,@@ -22010,6 +22010,29 @@ class LlamaCppBackend:secrets,)+ # A type this build lacks (prism-ml PQ2_0 / PTQ1_0 = 142 / 143) or packs differently (group-128 Q2_0, #11259).+ # A truncated download fails later instead, as "data is not within the file bounds".+ bad_type = re.search(r"tensor '([^']*)' has invalid ggml type (\d+)", scan_tail)+ bad_layout = re.search(r"tensor '[^']*' has offset \d+, expected \d+", scan_tail)+ if bad_type or bad_layout:+ detail = (+ f"tensor '{_short(bad_type.group(1))}' uses quantization type "+ f"{bad_type.group(2)}, which this build does not have"+ if bad_type+ else "its tensor layout does not match this build's quantization types"+ )+ return LlamaCppBackend._with_startup_diagnostics(+ "The installed llama.cpp cannot read this GGUF's weights "+ f"({detail}); this is not a memory problem. The file was likely made for "+ "a different llama.cpp build: a newer one, or a publisher's own fork. Pick "+ "another quant of this model, or use the llama.cpp build the "+ "model card names. If the model card says it runs on stock llama.cpp, "+ "the file may be damaged: download it again.",+ output,+ log_path,+ secrets,+ )+# 127 without any loader line above is not specific to a library: it is# also what a shell wrapper entrypoint reports when its exec target is# gone (_llama_lib_dir supports those, as does a customdiff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.pyindex 77b77a237f1..3ab9850cfeb 100644--- a/studio/backend/hub/services/models/gguf_variants.py+++ b/studio/backend/hub/services/models/gguf_variants.py@@ -847,7 +847,7 @@ _KNOWN_QUANT_RE = re.compile(r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"r"|P?TQ[0-9]+_[0-9]+"r"|Q[0-9]+_K_[A-Z]+"- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"r"|Q[0-9]+_K"r"|BF16|F16|F32)"r"(-[0-9]+(?:\.[0-9]+)?bpw)?",diff --git a/studio/backend/hub/tests/test_model_services.py b/studio/backend/hub/tests/test_model_services.pyindex a2e608934ec..616c7c4ef2a 100644--- a/studio/backend/hub/tests/test_model_services.py+++ b/studio/backend/hub/tests/test_model_services.py@@ -431,6 +431,7 @@ class TestExtractQuantToken:assert gguf.extract_quant_token("Ternary-Bonsai-1.7B-PQ2_0.gguf") == "PQ2_0"assert gguf.extract_quant_token("Ternary-Bonsai-1.7B-Q2_0.gguf") == "Q2_0"assert gguf.extract_quant_token("Ternary-Bonsai-1.7B-Q2_0_g64.gguf") == "Q2_0_g64"+ assert gguf.extract_quant_token("Ternary-Bonsai-27B-Q2_g64.gguf") == "Q2_g64"assert gguf.extract_quant_token("Ternary-Bonsai-2-27B-PTQ1_0.gguf") == "PTQ1_0"def test_precision_infix_variants_do_not_collapse(self):@@ -520,6 +521,72 @@ def test_pick_best_gguf_prefers_an_unlisted_quant_only_over_full_precision():assert gguf.pick_best_gguf(["model-APEX.gguf", "model-Q4_0.gguf"]) == "model-APEX.gguf"+def test_pick_best_gguf_skips_fork_only_packs_when_a_loadable_quant_exists():+ # prism-ml/Ternary-Bonsai-27B-gguf, largest first as the variant listing orders it: the+ # bundled llama.cpp rejects PQ2_0 (ggml type 142) and reads the group-64 Q2_g64 (#11259).+ assert (+ gguf.pick_best_gguf(+ [+ "Ternary-Bonsai-27B-F16.gguf",+ "Ternary-Bonsai-27B-PQ2_0.gguf",+ "Ternary-Bonsai-27B-Q2_g64.gguf",+ "Ternary-Bonsai-27B-Q2_0.gguf",+ ]+ )+ == "Ternary-Bonsai-27B-Q2_g64.gguf"+ )+ # A bare repo id hands over the Hub's alphabetical order, where the group-128 Q2_0 comes first.+ assert (+ gguf.pick_best_gguf(+ [+ "Ternary-Bonsai-27B-F16.gguf",+ "Ternary-Bonsai-27B-PQ2_0.gguf",+ "Ternary-Bonsai-27B-Q2_0.gguf",+ "Ternary-Bonsai-27B-Q2_g64.gguf",+ "Ternary-Bonsai-27B-dspark-Q4_1.gguf",+ ]+ )+ == "Ternary-Bonsai-27B-Q2_g64.gguf"+ )+ # Ready rows only, no full precision.+ assert (+ gguf.pick_best_gguf(["Ternary-Bonsai-27B-PQ2_0.gguf", "Ternary-Bonsai-27B-Q2_g64.gguf"])+ == "Ternary-Bonsai-27B-Q2_g64.gguf"+ )+ # Cached Q2_0 + Q2_g64 without PQ2_0, sorted: the plain group-128 pack still loses.+ assert (+ gguf.pick_best_gguf(["Ternary-Bonsai-27B-Q2_0.gguf", "Ternary-Bonsai-27B-Q2_g64.gguf"])+ == "Ternary-Bonsai-27B-Q2_g64.gguf"+ )+ # Otherwise the first-listed fallback stands.+ assert gguf.pick_best_gguf(["m-Q4_0.gguf", "m-Q2_g64.gguf", "m-F16.gguf"]) == "m-Q4_0.gguf"+ assert (+ gguf.pick_best_gguf(["m-Q4_0.gguf", "m-Q2_0.gguf", "m-Q2_g64.gguf", "m-F16.gguf"])+ == "m-Q4_0.gguf"+ )+ # Only Q2 has the fork's plain layout; a grouped Q4 hides no Q4_0.+ assert gguf.pick_best_gguf(["m-Q4_0.gguf", "m-Q4_g64.gguf"]) == "m-Q4_0.gguf"+ assert (+ gguf.pick_best_gguf(["base/Q2_0/model.gguf", "instruct/Q2_g64/model.gguf"])+ == "base/Q2_0/model.gguf"+ )+ assert (+ gguf.pick_best_gguf(["base-Q2_0.gguf", "instruct-Q2_g64.gguf", "base-F16.gguf"])+ == "base-Q2_0.gguf"+ )+ # Ternary-Bonsai-2-27B has nothing else quantized; its F16 needs the fork too, so keep the pack.+ assert (+ gguf.pick_best_gguf(+ [+ "Ternary-Bonsai-2-27B-F16.gguf",+ "Ternary-Bonsai-2-27B-PQ2_0.gguf",+ "Ternary-Bonsai-2-27B-PTQ1_0.gguf",+ ]+ )+ == "Ternary-Bonsai-2-27B-PQ2_0.gguf"+ )++def test_custom_inventory_filters_mtp_companions_at_registered_root(tmp_path, monkeypatch):root = tmp_path / "MTP"root.mkdir()diff --git a/studio/backend/hub/utils/gguf.py b/studio/backend/hub/utils/gguf.pyindex 1cca0f91a20..3ee869ecf37 100644--- a/studio/backend/hub/utils/gguf.py+++ b/studio/backend/hub/utils/gguf.py@@ -80,7 +80,7 @@ _GGUF_QUANT_RE = re.compile(r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"r"|P?TQ[0-9]+_[0-9]+"r"|Q[0-9]+_K_[A-Z]+"- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"r"|Q[0-9]+_K"r"|BF16|F16|F32)",re.IGNORECASE,@@ -284,17 +284,37 @@ def pick_best_gguf(filenames: list[str]) -> Optional[str]:continueif quant in _FLOAT_PRECISION_QUANTS:# The list leaves out quants such as Q4_0, Q3_K and TQ1_0; the first listed of them still beats full precision.- filename = next(- (- name- for name in by_quant.values()- if (token := extract_quant_token(name))- and token.upper() not in _FLOAT_PRECISION_QUANTS- ),- filename,- )+ quantized = [+ name+ for name in by_quant.values()+ if (token := extract_quant_token(name))+ and token.upper() not in _FLOAT_PRECISION_QUANTS+ ]+ filename = _first_mainline_loadable(quantized) or filenamereturn filename- return gguf_files[0]+ return _first_mainline_loadable(gguf_files)+++def _first_mainline_loadable(names: list[str]) -> Optional[str]:+ """First of *names*, skipping packs mainline llama.cpp cannot read while anything else is left."""+ tokens = {name: (extract_quant_token(name) or "").upper() for name in names}++ def _checkpoint(name: str) -> str:+ return (gguf_checkpoint_family(name) or "").upper()++ # A plain Q2_0 published beside the same checkpoint's grouped Q2_g64 / Q2_0_g64 is the fork's group-128 pack.+ shadowed = {+ (_checkpoint(name), f"{match.group(1)}_0")+ for name, token in tokens.items()+ if (match := _GROUPED_QUANT_RE.match(token))+ }+ loadable = [+ name+ for name in names+ if not _FORK_ONLY_QUANT_RE.match(tokens[name])+ and (_checkpoint(name), tokens[name]) not in shadowed+ ]+ return next(iter(loadable or names), None)def _gguf_stem(filename: str) -> str:@@ -303,6 +323,10 @@ def _gguf_stem(filename: str) -> str:_FLOAT_PRECISION_QUANTS = frozenset({"BF16", "F16", "F32"})+# prism-ml's PQ2_0 / PTQ1_0 are ggml types only its llama.cpp fork defines; the bundled build rejects them (#11259).+_FORK_ONLY_QUANT_RE = re.compile(r"PT?Q[0-9]", re.IGNORECASE)+# Q2 only: prism-ml's group-128 Q2_0 is the plain pack a grouped sibling marks as fork-only.+_GROUPED_QUANT_RE = re.compile(r"(Q2)(?:_0)?_G[0-9]+$", re.IGNORECASE)def _select_quant_match(text: str) -> Optional[re.Match]:diff --git a/studio/backend/tests/test_gguf_variant_rows.py b/studio/backend/tests/test_gguf_variant_rows.pyindex 9717dad7fe7..863f1224295 100644--- a/studio/backend/tests/test_gguf_variant_rows.py+++ b/studio/backend/tests/test_gguf_variant_rows.py@@ -148,6 +148,8 @@ def test_packed_and_grouped_quants_keep_their_own_key_in_both_copies():"Ternary-Bonsai-8B-PQ2_0.gguf": "PQ2_0","Ternary-Bonsai-8B-Q2_0.gguf": "Q2_0","Ternary-Bonsai-8B-Q2_0_g64.gguf": "Q2_0_g64",+ # prism-ml/Ternary-Bonsai-27B-gguf spells its group-64 pack without the "_0".+ "Ternary-Bonsai-27B-Q2_g64.gguf": "Q2_g64","Ternary-Bonsai-2-27B-PTQ1_0.gguf": "PTQ1_0",}for path, key in files.items():diff --git a/studio/backend/tests/test_llama_cpp_start_failure_classification.py b/studio/backend/tests/test_llama_cpp_start_failure_classification.pyindex cb3826e4ade..6f3d7b62d03 100644--- a/studio/backend/tests/test_llama_cpp_start_failure_classification.py+++ b/studio/backend/tests/test_llama_cpp_start_failure_classification.py@@ -209,6 +209,52 @@ class TestOllamaAndFallback:assert "GGUF file is valid" not in msgassert "minimax-m3.attention.indexer.head_count\n" in msg+ @pytest.mark.parametrize(+ "line, named",+ [+ # #11259 comment, prism-ml/Ternary-Bonsai-2-27B-gguf PQ2_0 on the bundled build.+ (+ "gguf_init_from_reader: tensor 'output.weight' has invalid ggml type 142. "+ "should be in [0, 67)\n"+ "gguf_init_from_reader: failed to read tensor info\n",+ "quantization type 142",+ ),+ # #11259 report: a group-128 Q2_0 tensor where the build defines group-64 Q2_0.+ (+ "gguf_init_from_reader: tensor 'dspark.fc.weight' has offset 337718592, "+ "expected 357584192\n"+ "gguf_init_from_reader: failed to read tensor data\n",+ "download it again",+ ),+ ],+ )+ def test_unreadable_tensor_type_names_the_build_not_memory(self, line, named):+ out = (+ "load_model: loading model 'Ternary-Bonsai-27B-PQ2_0.gguf'\n"+ + line+ + "llama_model_load: error loading model: llama_model_loader: failed to load model\n"+ "srv llama_server: exiting due to model loading error\n"+ )+ msg = _classify(+ out, "/models/Ternary-Bonsai-27B-PQ2_0.gguf", "prism-ml/Ternary-Bonsai-27B-gguf", 1+ )+ assert named in msg+ assert "download it again" in msg+ assert "not a memory problem" in msg+ assert "another quant" in msg+ assert "GGUF file is valid" not in msg++ def test_truncated_gguf_keeps_the_generic_message(self):+ # What a cut-off download prints: the header parses, the data does not fit.+ out = (+ "llama_model_load: error loading model: tensor 'output.weight' data is not "+ "within the file bounds, model is corrupted or incomplete\n"+ "llama_model_load_from_file_impl: failed to load model\n"+ )+ msg = _classify(out, "/models/m.gguf", "local/m", 1)+ assert "GGUF file is valid" in msg+ assert "cannot read this GGUF" not in msg+def test_generic_oom_keeps_memory_message(self):msg = _classify(_OOM_OUT, "/models/big.gguf", "local/big")assert "enough memory" in msg.lower()diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.pyindex 62158f11957..6b1c8535d00 100644--- a/studio/backend/utils/models/model_config.py+++ b/studio/backend/utils/models/model_config.py@@ -2772,7 +2772,7 @@ def _extract_quant_label(filename: str) -> str:r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?" # IQ variants: IQ4_XS, IQ4_NL, IQ1_Sr"|P?TQ[0-9]+_[0-9]+" # Ternary quant: TQ1_0, TQ2_0r"|Q[0-9]+_K_[A-Z]+" # K-quant: Q4_K_M, Q3_K_S- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?" # Standard/Packed: Q8_0, PQ2_0, Q2_0_g64+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)" # Standard/Packed/Grouped: Q8_0, PQ2_0, Q2_0_g64, Q2_g64r"|Q[0-9]+_K" # Short K-quant: Q6_Kr"|BF16|F16|F32)" # Full precision# Optional bits-per-weight modifier so repos shipping several files at one base quant@@ -2804,7 +2804,7 @@ _GGUF_KNOWN_QUANT_RE = re.compile(r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"r"|P?TQ[0-9]+_[0-9]+"r"|Q[0-9]+_K_[A-Z]+"- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"r"|Q[0-9]+_K"r"|BF16|F16|F32)",re.IGNORECASE,diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex d7790414804..6b2f10fe901 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -2549,7 +2549,7 @@ const MAX_AUTO_LOAD_ATTEMPTS = 3;const MAX_AUTO_VALIDATE_FAILURES = 12;const BIG_ENDIAN_GGUF_FILENAME_RE = /(^|[-_])be(?:[._-]|$)/gi;const GGUF_KNOWN_QUANT_RE =- /(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?|Q[0-9]+_K|BF16|F16|F32)/i;+ /(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)|Q[0-9]+_K|BF16|F16|F32)/i;type AutoLoadCandidate = {id: string;diff --git a/studio/frontend/src/features/model-picker/model-config/model-identity.ts b/studio/frontend/src/features/model-picker/model-config/model-identity.tsindex b8d2df1e1d2..2d8510d7543 100644--- a/studio/frontend/src/features/model-picker/model-config/model-identity.ts+++ b/studio/frontend/src/features/model-picker/model-config/model-identity.ts@@ -113,7 +113,7 @@ export function ggufVariantFromStorageKey(key: string): string | null {const BPW_SUFFIX = /-[0-9]+(?:\.[0-9]+)?bpw$/i;// One source for the anchored test and the scan below. Mirrors _GGUF_QUANT_RE in gguf.py.const QUANT_TOKEN_SOURCE =- "(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?|Q[0-9]+_K|BF16|F16|F32)";+ "(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)|Q[0-9]+_K|BF16|F16|F32)";const KNOWN_QUANT = new RegExp(`^${QUANT_TOKEN_SOURCE}$`, "i");const QUANT_TOKEN = new RegExp(QUANT_TOKEN_SOURCE, "gi");const MAX_QUANT_SUFFIX_LEN = 64;diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.pyindex e3cd495f7ad..8770f7b04c7 100644--- a/unsloth_cli/commands/start.py+++ b/unsloth_cli/commands/start.py@@ -2459,7 +2459,7 @@ _QUANT_LABEL_RE = re.compile(r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"r"|P?TQ[0-9]+_[0-9]+"r"|Q[0-9]+_K_[A-Z]+"- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"r"|Q[0-9]+_K"r"|BF16|F16|F32)"r"(-[0-9]+(?:\.[0-9]+)?bpw)?",diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.pyindex 4e27bc8f761..60288cd1466 100644--- a/unsloth_cli/tests/test_start.py+++ b/unsloth_cli/tests/test_start.py@@ -10517,6 +10517,7 @@ def test_direct_gguf_labels_keep_packed_and_grouped_quants():("Ternary-Bonsai-8B-PQ2_0.gguf", "PQ2_0"),("Ternary-Bonsai-8B-Q2_0.gguf", "Q2_0"),("Ternary-Bonsai-8B-Q2_0_g64.gguf", "Q2_0_g64"),+ ("Ternary-Bonsai-27B-Q2_g64.gguf", "Q2_g64"),("Ternary-Bonsai-2-27B-PTQ1_0.gguf", "PTQ1_0"),):assert start._direct_gguf_variant_labels(name)[1] == label