Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: explain GGUFs the bundled llama.cpp cannot read, and default Ternary Bonsai to its loadable pack (#13304)

Daniel Han83a761ba72ed
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index a46f2576196..2c2f2ff91b3 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -3309,7 +3309,7 @@ _GGUF_KNOWN_QUANT_RE = re.compile(
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
r"|P?TQ[0-9]+_[0-9]+"
r"|Q[0-9]+_K_[A-Z]+"
- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"
+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"
r"|Q[0-9]+_K"
r"|BF16|F16|F32)",
re.IGNORECASE,
@@ -22010,6 +22010,29 @@ class LlamaCppBackend:
secrets,
)
+ # A type this build lacks (prism-ml PQ2_0 / PTQ1_0 = 142 / 143) or packs differently (group-128 Q2_0, #11259).
+ # A truncated download fails later instead, as "data is not within the file bounds".
+ bad_type = re.search(r"tensor '([^']*)' has invalid ggml type (\d+)", scan_tail)
+ bad_layout = re.search(r"tensor '[^']*' has offset \d+, expected \d+", scan_tail)
+ if bad_type or bad_layout:
+ detail = (
+ f"tensor '{_short(bad_type.group(1))}' uses quantization type "
+ f"{bad_type.group(2)}, which this build does not have"
+ if bad_type
+ else "its tensor layout does not match this build's quantization types"
+ )
+ return LlamaCppBackend._with_startup_diagnostics(
+ "The installed llama.cpp cannot read this GGUF's weights "
+ f"({detail}); this is not a memory problem. The file was likely made for "
+ "a different llama.cpp build: a newer one, or a publisher's own fork. Pick "
+ "another quant of this model, or use the llama.cpp build the "
+ "model card names. If the model card says it runs on stock llama.cpp, "
+ "the file may be damaged: download it again.",
+ output,
+ log_path,
+ secrets,
+ )
+
# 127 without any loader line above is not specific to a library: it is
# also what a shell wrapper entrypoint reports when its exec target is
# gone (_llama_lib_dir supports those, as does a custom
diff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.py
index 77b77a237f1..3ab9850cfeb 100644
--- a/studio/backend/hub/services/models/gguf_variants.py
+++ b/studio/backend/hub/services/models/gguf_variants.py
@@ -847,7 +847,7 @@ _KNOWN_QUANT_RE = re.compile(
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
r"|P?TQ[0-9]+_[0-9]+"
r"|Q[0-9]+_K_[A-Z]+"
- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"
+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"
r"|Q[0-9]+_K"
r"|BF16|F16|F32)"
r"(-[0-9]+(?:\.[0-9]+)?bpw)?",
diff --git a/studio/backend/hub/tests/test_model_services.py b/studio/backend/hub/tests/test_model_services.py
index a2e608934ec..616c7c4ef2a 100644
--- a/studio/backend/hub/tests/test_model_services.py
+++ b/studio/backend/hub/tests/test_model_services.py
@@ -431,6 +431,7 @@ class TestExtractQuantToken:
assert gguf.extract_quant_token("Ternary-Bonsai-1.7B-PQ2_0.gguf") == "PQ2_0"
assert gguf.extract_quant_token("Ternary-Bonsai-1.7B-Q2_0.gguf") == "Q2_0"
assert gguf.extract_quant_token("Ternary-Bonsai-1.7B-Q2_0_g64.gguf") == "Q2_0_g64"
+ assert gguf.extract_quant_token("Ternary-Bonsai-27B-Q2_g64.gguf") == "Q2_g64"
assert gguf.extract_quant_token("Ternary-Bonsai-2-27B-PTQ1_0.gguf") == "PTQ1_0"
def test_precision_infix_variants_do_not_collapse(self):
@@ -520,6 +521,72 @@ def test_pick_best_gguf_prefers_an_unlisted_quant_only_over_full_precision():
assert gguf.pick_best_gguf(["model-APEX.gguf", "model-Q4_0.gguf"]) == "model-APEX.gguf"
+def test_pick_best_gguf_skips_fork_only_packs_when_a_loadable_quant_exists():
+ # prism-ml/Ternary-Bonsai-27B-gguf, largest first as the variant listing orders it: the
+ # bundled llama.cpp rejects PQ2_0 (ggml type 142) and reads the group-64 Q2_g64 (#11259).
+ assert (
+ gguf.pick_best_gguf(
+ [
+ "Ternary-Bonsai-27B-F16.gguf",
+ "Ternary-Bonsai-27B-PQ2_0.gguf",
+ "Ternary-Bonsai-27B-Q2_g64.gguf",
+ "Ternary-Bonsai-27B-Q2_0.gguf",
+ ]
+ )
+ == "Ternary-Bonsai-27B-Q2_g64.gguf"
+ )
+ # A bare repo id hands over the Hub's alphabetical order, where the group-128 Q2_0 comes first.
+ assert (
+ gguf.pick_best_gguf(
+ [
+ "Ternary-Bonsai-27B-F16.gguf",
+ "Ternary-Bonsai-27B-PQ2_0.gguf",
+ "Ternary-Bonsai-27B-Q2_0.gguf",
+ "Ternary-Bonsai-27B-Q2_g64.gguf",
+ "Ternary-Bonsai-27B-dspark-Q4_1.gguf",
+ ]
+ )
+ == "Ternary-Bonsai-27B-Q2_g64.gguf"
+ )
+ # Ready rows only, no full precision.
+ assert (
+ gguf.pick_best_gguf(["Ternary-Bonsai-27B-PQ2_0.gguf", "Ternary-Bonsai-27B-Q2_g64.gguf"])
+ == "Ternary-Bonsai-27B-Q2_g64.gguf"
+ )
+ # Cached Q2_0 + Q2_g64 without PQ2_0, sorted: the plain group-128 pack still loses.
+ assert (
+ gguf.pick_best_gguf(["Ternary-Bonsai-27B-Q2_0.gguf", "Ternary-Bonsai-27B-Q2_g64.gguf"])
+ == "Ternary-Bonsai-27B-Q2_g64.gguf"
+ )
+ # Otherwise the first-listed fallback stands.
+ assert gguf.pick_best_gguf(["m-Q4_0.gguf", "m-Q2_g64.gguf", "m-F16.gguf"]) == "m-Q4_0.gguf"
+ assert (
+ gguf.pick_best_gguf(["m-Q4_0.gguf", "m-Q2_0.gguf", "m-Q2_g64.gguf", "m-F16.gguf"])
+ == "m-Q4_0.gguf"
+ )
+ # Only Q2 has the fork's plain layout; a grouped Q4 hides no Q4_0.
+ assert gguf.pick_best_gguf(["m-Q4_0.gguf", "m-Q4_g64.gguf"]) == "m-Q4_0.gguf"
+ assert (
+ gguf.pick_best_gguf(["base/Q2_0/model.gguf", "instruct/Q2_g64/model.gguf"])
+ == "base/Q2_0/model.gguf"
+ )
+ assert (
+ gguf.pick_best_gguf(["base-Q2_0.gguf", "instruct-Q2_g64.gguf", "base-F16.gguf"])
+ == "base-Q2_0.gguf"
+ )
+ # Ternary-Bonsai-2-27B has nothing else quantized; its F16 needs the fork too, so keep the pack.
+ assert (
+ gguf.pick_best_gguf(
+ [
+ "Ternary-Bonsai-2-27B-F16.gguf",
+ "Ternary-Bonsai-2-27B-PQ2_0.gguf",
+ "Ternary-Bonsai-2-27B-PTQ1_0.gguf",
+ ]
+ )
+ == "Ternary-Bonsai-2-27B-PQ2_0.gguf"
+ )
+
+
def test_custom_inventory_filters_mtp_companions_at_registered_root(tmp_path, monkeypatch):
root = tmp_path / "MTP"
root.mkdir()
diff --git a/studio/backend/hub/utils/gguf.py b/studio/backend/hub/utils/gguf.py
index 1cca0f91a20..3ee869ecf37 100644
--- a/studio/backend/hub/utils/gguf.py
+++ b/studio/backend/hub/utils/gguf.py
@@ -80,7 +80,7 @@ _GGUF_QUANT_RE = re.compile(
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
r"|P?TQ[0-9]+_[0-9]+"
r"|Q[0-9]+_K_[A-Z]+"
- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"
+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"
r"|Q[0-9]+_K"
r"|BF16|F16|F32)",
re.IGNORECASE,
@@ -284,17 +284,37 @@ def pick_best_gguf(filenames: list[str]) -> Optional[str]:
continue
if quant in _FLOAT_PRECISION_QUANTS:
# The list leaves out quants such as Q4_0, Q3_K and TQ1_0; the first listed of them still beats full precision.
- filename = next(
- (
- name
- for name in by_quant.values()
- if (token := extract_quant_token(name))
- and token.upper() not in _FLOAT_PRECISION_QUANTS
- ),
- filename,
- )
+ quantized = [
+ name
+ for name in by_quant.values()
+ if (token := extract_quant_token(name))
+ and token.upper() not in _FLOAT_PRECISION_QUANTS
+ ]
+ filename = _first_mainline_loadable(quantized) or filename
return filename
- return gguf_files[0]
+ return _first_mainline_loadable(gguf_files)
+
+
+def _first_mainline_loadable(names: list[str]) -> Optional[str]:
+ """First of *names*, skipping packs mainline llama.cpp cannot read while anything else is left."""
+ tokens = {name: (extract_quant_token(name) or "").upper() for name in names}
+
+ def _checkpoint(name: str) -> str:
+ return (gguf_checkpoint_family(name) or "").upper()
+
+ # A plain Q2_0 published beside the same checkpoint's grouped Q2_g64 / Q2_0_g64 is the fork's group-128 pack.
+ shadowed = {
+ (_checkpoint(name), f"{match.group(1)}_0")
+ for name, token in tokens.items()
+ if (match := _GROUPED_QUANT_RE.match(token))
+ }
+ loadable = [
+ name
+ for name in names
+ if not _FORK_ONLY_QUANT_RE.match(tokens[name])
+ and (_checkpoint(name), tokens[name]) not in shadowed
+ ]
+ return next(iter(loadable or names), None)
def _gguf_stem(filename: str) -> str:
@@ -303,6 +323,10 @@ def _gguf_stem(filename: str) -> str:
_FLOAT_PRECISION_QUANTS = frozenset({"BF16", "F16", "F32"})
+# prism-ml's PQ2_0 / PTQ1_0 are ggml types only its llama.cpp fork defines; the bundled build rejects them (#11259).
+_FORK_ONLY_QUANT_RE = re.compile(r"PT?Q[0-9]", re.IGNORECASE)
+# Q2 only: prism-ml's group-128 Q2_0 is the plain pack a grouped sibling marks as fork-only.
+_GROUPED_QUANT_RE = re.compile(r"(Q2)(?:_0)?_G[0-9]+$", re.IGNORECASE)
def _select_quant_match(text: str) -> Optional[re.Match]:
diff --git a/studio/backend/tests/test_gguf_variant_rows.py b/studio/backend/tests/test_gguf_variant_rows.py
index 9717dad7fe7..863f1224295 100644
--- a/studio/backend/tests/test_gguf_variant_rows.py
+++ b/studio/backend/tests/test_gguf_variant_rows.py
@@ -148,6 +148,8 @@ def test_packed_and_grouped_quants_keep_their_own_key_in_both_copies():
"Ternary-Bonsai-8B-PQ2_0.gguf": "PQ2_0",
"Ternary-Bonsai-8B-Q2_0.gguf": "Q2_0",
"Ternary-Bonsai-8B-Q2_0_g64.gguf": "Q2_0_g64",
+ # prism-ml/Ternary-Bonsai-27B-gguf spells its group-64 pack without the "_0".
+ "Ternary-Bonsai-27B-Q2_g64.gguf": "Q2_g64",
"Ternary-Bonsai-2-27B-PTQ1_0.gguf": "PTQ1_0",
}
for path, key in files.items():
diff --git a/studio/backend/tests/test_llama_cpp_start_failure_classification.py b/studio/backend/tests/test_llama_cpp_start_failure_classification.py
index cb3826e4ade..6f3d7b62d03 100644
--- a/studio/backend/tests/test_llama_cpp_start_failure_classification.py
+++ b/studio/backend/tests/test_llama_cpp_start_failure_classification.py
@@ -209,6 +209,52 @@ class TestOllamaAndFallback:
assert "GGUF file is valid" not in msg
assert "minimax-m3.attention.indexer.head_count\n" in msg
+ @pytest.mark.parametrize(
+ "line, named",
+ [
+ # #11259 comment, prism-ml/Ternary-Bonsai-2-27B-gguf PQ2_0 on the bundled build.
+ (
+ "gguf_init_from_reader: tensor 'output.weight' has invalid ggml type 142. "
+ "should be in [0, 67)\n"
+ "gguf_init_from_reader: failed to read tensor info\n",
+ "quantization type 142",
+ ),
+ # #11259 report: a group-128 Q2_0 tensor where the build defines group-64 Q2_0.
+ (
+ "gguf_init_from_reader: tensor 'dspark.fc.weight' has offset 337718592, "
+ "expected 357584192\n"
+ "gguf_init_from_reader: failed to read tensor data\n",
+ "download it again",
+ ),
+ ],
+ )
+ def test_unreadable_tensor_type_names_the_build_not_memory(self, line, named):
+ out = (
+ "load_model: loading model 'Ternary-Bonsai-27B-PQ2_0.gguf'\n"
+ + line
+ + "llama_model_load: error loading model: llama_model_loader: failed to load model\n"
+ "srv llama_server: exiting due to model loading error\n"
+ )
+ msg = _classify(
+ out, "/models/Ternary-Bonsai-27B-PQ2_0.gguf", "prism-ml/Ternary-Bonsai-27B-gguf", 1
+ )
+ assert named in msg
+ assert "download it again" in msg
+ assert "not a memory problem" in msg
+ assert "another quant" in msg
+ assert "GGUF file is valid" not in msg
+
+ def test_truncated_gguf_keeps_the_generic_message(self):
+ # What a cut-off download prints: the header parses, the data does not fit.
+ out = (
+ "llama_model_load: error loading model: tensor 'output.weight' data is not "
+ "within the file bounds, model is corrupted or incomplete\n"
+ "llama_model_load_from_file_impl: failed to load model\n"
+ )
+ msg = _classify(out, "/models/m.gguf", "local/m", 1)
+ assert "GGUF file is valid" in msg
+ assert "cannot read this GGUF" not in msg
+
def test_generic_oom_keeps_memory_message(self):
msg = _classify(_OOM_OUT, "/models/big.gguf", "local/big")
assert "enough memory" in msg.lower()
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 62158f11957..6b1c8535d00 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -2772,7 +2772,7 @@ def _extract_quant_label(filename: str) -> str:
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?" # IQ variants: IQ4_XS, IQ4_NL, IQ1_S
r"|P?TQ[0-9]+_[0-9]+" # Ternary quant: TQ1_0, TQ2_0
r"|Q[0-9]+_K_[A-Z]+" # K-quant: Q4_K_M, Q3_K_S
- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?" # Standard/Packed: Q8_0, PQ2_0, Q2_0_g64
+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)" # Standard/Packed/Grouped: Q8_0, PQ2_0, Q2_0_g64, Q2_g64
r"|Q[0-9]+_K" # Short K-quant: Q6_K
r"|BF16|F16|F32)" # Full precision
# Optional bits-per-weight modifier so repos shipping several files at one base quant
@@ -2804,7 +2804,7 @@ _GGUF_KNOWN_QUANT_RE = re.compile(
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
r"|P?TQ[0-9]+_[0-9]+"
r"|Q[0-9]+_K_[A-Z]+"
- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"
+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"
r"|Q[0-9]+_K"
r"|BF16|F16|F32)",
re.IGNORECASE,
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index d7790414804..6b2f10fe901 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -2549,7 +2549,7 @@ const MAX_AUTO_LOAD_ATTEMPTS = 3;
const MAX_AUTO_VALIDATE_FAILURES = 12;
const BIG_ENDIAN_GGUF_FILENAME_RE = /(^|[-_])be(?:[._-]|$)/gi;
const GGUF_KNOWN_QUANT_RE =
- /(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?|Q[0-9]+_K|BF16|F16|F32)/i;
+ /(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)|Q[0-9]+_K|BF16|F16|F32)/i;
type AutoLoadCandidate = {
id: string;
diff --git a/studio/frontend/src/features/model-picker/model-config/model-identity.ts b/studio/frontend/src/features/model-picker/model-config/model-identity.ts
index b8d2df1e1d2..2d8510d7543 100644
--- a/studio/frontend/src/features/model-picker/model-config/model-identity.ts
+++ b/studio/frontend/src/features/model-picker/model-config/model-identity.ts
@@ -113,7 +113,7 @@ export function ggufVariantFromStorageKey(key: string): string | null {
const BPW_SUFFIX = /-[0-9]+(?:\.[0-9]+)?bpw$/i;
// One source for the anchored test and the scan below. Mirrors _GGUF_QUANT_RE in gguf.py.
const QUANT_TOKEN_SOURCE =
- "(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?|Q[0-9]+_K|BF16|F16|F32)";
+ "(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|P?TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)|Q[0-9]+_K|BF16|F16|F32)";
const KNOWN_QUANT = new RegExp(`^${QUANT_TOKEN_SOURCE}$`, "i");
const QUANT_TOKEN = new RegExp(QUANT_TOKEN_SOURCE, "gi");
const MAX_QUANT_SUFFIX_LEN = 64;
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index e3cd495f7ad..8770f7b04c7 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -2459,7 +2459,7 @@ _QUANT_LABEL_RE = re.compile(
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?"
r"|P?TQ[0-9]+_[0-9]+"
r"|Q[0-9]+_K_[A-Z]+"
- r"|P?Q[0-9]+_[0-9]+(?:_G[0-9]+)?"
+ r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)"
r"|Q[0-9]+_K"
r"|BF16|F16|F32)"
r"(-[0-9]+(?:\.[0-9]+)?bpw)?",
diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py
index 4e27bc8f761..60288cd1466 100644
--- a/unsloth_cli/tests/test_start.py
+++ b/unsloth_cli/tests/test_start.py
@@ -10517,6 +10517,7 @@ def test_direct_gguf_labels_keep_packed_and_grouped_quants():
("Ternary-Bonsai-8B-PQ2_0.gguf", "PQ2_0"),
("Ternary-Bonsai-8B-Q2_0.gguf", "Q2_0"),
("Ternary-Bonsai-8B-Q2_0_g64.gguf", "Q2_0_g64"),
+ ("Ternary-Bonsai-27B-Q2_g64.gguf", "Q2_g64"),
("Ternary-Bonsai-2-27B-PTQ1_0.gguf", "PTQ1_0"),
):
assert start._direct_gguf_variant_labels(name)[1] == label