Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: use --reasoning where available and name the compressed-tensors refusal (#11072)

Daniel Hane7becafe5ca7
diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml
index d78ee60aadf..ddc65f0ca6f 100644
--- a/.github/workflows/consolidated-tests-ci.yml
+++ b/.github/workflows/consolidated-tests-ci.yml
@@ -405,6 +405,10 @@ jobs:
# against simulated CPU counts and memory so the result is identical on
# a 4-vCPU runner and a 192-core box.
#
+ # The drift guard is named explicitly (these jobs collect paths, not
+ # trees): it is the only leg with a transformers past the `>=4.51,<5.5`
+ # pin, and the wording it exists to survive landed in 5.10.
+ #
# The Unsloth file runs here rather than in studio-backend-ci, which
# installs studio.txt and so has no unsloth_zoo at all. Note the policy
# module is not on unsloth_zoo main either until the companion PR lands,
@@ -417,7 +421,8 @@ jobs:
tests/utils/test_dataset_num_proc.py \
tests/utils/test_train_on_responses_only_num_proc.py
PYTHONPATH=studio/backend python -m pytest -v --tb=short \
- studio/backend/tests/test_dataset_map_num_proc.py
+ studio/backend/tests/test_dataset_map_num_proc.py \
+ studio/backend/tests/test_compressed_tensors_signature_drift.py
- name: unsloth Bucket-A — CPU tests not in Repo tests (CPU)
# CPU tests across 6 files under tests/saving/, tests/utils/, tests/python/
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index ead87a21835..3eda91ae8ce 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6256,6 +6256,40 @@ def _build_ngram_mod_flags(
return []
+def _build_launch_reasoning_args(
+ caps: Mapping[str, object], reasoning_kwargs: Mapping[str, object]
+) -> list[str]:
+ """The launch flags that carry ``reasoning_kwargs``, per key.
+
+ llama-server's --chat-template-kwargs deprecation is PER KEY (#7526), and only
+ ``enable_thinking`` has a flag of its own, so only that key moves; every other
+ template variable stays in --chat-template-kwargs, still the only channel for one.
+
+ Gated CLOSED on the probe: a build predating the flag exits with "error: invalid
+ argument: --reasoning" rather than starting without it, so an unknown answer keeps
+ main's argv byte for byte.
+
+ The two spellings are equivalent, not identical. common/arg.cpp's --reasoning
+ handler writes the same ``default_template_kwargs["enable_thinking"]`` entry AND
+ sets ``params.enable_reasoning``, whose only consumer (tools/server/server-context.cpp)
+ folds it into the per-server ``opt.enable_thinking`` default -- which
+ tools/server/server-common.cpp then overrides from the merged kwarg anyway. If
+ llama.cpp stops writing that kwarg, or stops letting it override, this becomes a
+ behaviour change rather than a substitution.
+
+ An exported LLAMA_ARG_REASONING is still overridden by the command line, exactly as
+ the kwargs channel overrides it on main; honouring it is #8521.
+ """
+ remaining = dict(reasoning_kwargs)
+ args: list[str] = []
+ if caps.get("supports_reasoning_flag") and isinstance(remaining.get("enable_thinking"), bool):
+ args.extend(["--reasoning", "on" if remaining.pop("enable_thinking") else "off"])
+ # `not args`: a caller passing `{}` would otherwise lose main's `-ctk {}` on an old build.
+ if remaining or not args:
+ args.extend(["--chat-template-kwargs", json.dumps(remaining)])
+ return args
+
+
def _build_reasoning_budget_flags(
caps: Mapping[str, object], reasoning_budget: int, reasoning_budget_message: str
) -> list[str]:
@@ -8709,6 +8743,8 @@ class LlamaCppBackend:
"supports_reasoning_budget": False,
"supports_reasoning_budget_message": False,
"reasoning_budget_probe_inconclusive": True,
+ # Fails CLOSED, unlike --jinja: a build without --reasoning exits on it.
+ "supports_reasoning_flag": False,
"supports_no_mmproj_offload": False,
"supports_video_fps": False,
"supports_load_mode": False,
@@ -8760,6 +8796,7 @@ class LlamaCppBackend:
supports_slot_save = False
supports_reasoning_budget = False
supports_reasoning_budget_message = False
+ supports_reasoning_flag = False
supports_no_mmproj_offload = False
supports_video_fps = False
supports_load_mode = False
@@ -8994,6 +9031,8 @@ class LlamaCppBackend:
supports_slot_save = _is_real("--slot-save-path")
supports_reasoning_budget = _is_real("--reasoning-budget")
supports_reasoning_budget_message = _is_real("--reasoning-budget-message")
+ # Catalogue, not help text: --reasoning-format / -budget / -preserve contain it.
+ supports_reasoning_flag = _is_real("--reasoning")
supports_no_mmproj_offload = _is_real("--no-mmproj-offload")
supports_video_fps = _is_real("--video-fps")
# --load-mode supersedes --mlock / --no-mmap, which are deprecated.
@@ -9091,6 +9130,7 @@ class LlamaCppBackend:
"supports_reasoning_budget": supports_reasoning_budget,
"supports_reasoning_budget_message": supports_reasoning_budget_message,
"reasoning_budget_probe_inconclusive": not (probe_ok and help_nonempty),
+ "supports_reasoning_flag": supports_reasoning_flag,
"supports_no_mmproj_offload": supports_no_mmproj_offload,
"supports_video_fps": supports_video_fps,
"supports_load_mode": supports_load_mode,
@@ -26315,13 +26355,12 @@ class LlamaCppBackend:
# frontend receives the same resolved value via load/status.
if self._supports_preserve_thinking:
reasoning_kw["preserve_thinking"] = self._preserve_thinking_default
- cmd.extend(
- [
- "--chat-template-kwargs",
- json.dumps(reasoning_kw),
- ]
+ _reasoning_args = _build_launch_reasoning_args(server_caps, reasoning_kw)
+ cmd.extend(_reasoning_args)
+ logger.info(
+ f"Reasoning model: {reasoning_kw} by default "
+ f"(launched as {' '.join(_reasoning_args)})"
)
- logger.info(f"Reasoning model: {reasoning_kw} by default")
reasoning_budget = resolve_reasoning_budget(extra_args, reasoning_budget)
reasoning_budget_message = resolve_reasoning_budget_message(
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index 4ab0f65c3fe..6817bd053c0 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -527,6 +527,10 @@ _TEMPLATE_FLAGS: frozenset[str] = frozenset(
"--chat-template",
"--chat-template-file",
"--chat-template-kwargs",
+ # enable_thinking's new spelling (#7526); a template override recomputes the default,
+ # so both must strip. Takes a value, so NOT in _BOOLEAN_SHADOWING_FLAGS.
+ "--reasoning",
+ "-rea",
"--jinja",
"--no-jinja",
}
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 918cdaeb8a0..a8d586ee925 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -14390,6 +14390,14 @@ _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE = (
"We are working on supporting NVFP4 inference. For now it is not supported"
)
+# Said instead of transformers' "pip install compressed-tensors", which no user can act on:
+# Studio runs from its own environment (#8246). Names no scheme: these errors carry none.
+_COMPRESSED_TENSORS_INFERENCE_UNSUPPORTED_MESSAGE = (
+ "This model is quantized with compressed-tensors, which Unsloth cannot run yet. "
+ "Installing compressed-tensors will not help. Try a GGUF or a bitsandbytes 4-bit "
+ "build of this model instead."
+)
+
def _diagnosis_text(msg: str) -> str:
"""``msg`` up to the startup-diagnostics block, which is not ours to read.
@@ -14417,6 +14425,35 @@ def _is_unsupported_nvfp4_inference_error(msg: str) -> bool:
return "nvfp4" in lower_msg and "per-module mlx quantization metadata" in lower_msg
+# The two transformers sites, reworded in 5.10 to add a version: match only what survived.
+_MISSING_COMPRESSED_TENSORS_SIGNATURES = (
+ "is required for compressed-tensors quantization",
+ "`compressed_tensors` quantized models requires",
+)
+
+
+def _is_missing_compressed_tensors_error(msg: str) -> bool:
+ """Whether ``msg`` says the checkpoint needs the compressed-tensors library.
+
+ The NVFP4 matcher above only fires on the MLX loader's metadata error, so it never
+ reaches a CUDA, ROCm or CPU host; there the load dies inside transformers (#8246).
+ """
+ lower_msg = _diagnosis_text(msg).lower()
+ return any(sig in lower_msg for sig in _MISSING_COMPRESSED_TENSORS_SIGNATURES)
+
+
+def _unsupported_quantization_detail(msg: str) -> Optional[str]:
+ """The refusal to show for ``msg``, or None when it is not about quantization.
+
+ One place to add a signature to, so load, native load and validate cannot drift.
+ """
+ if _is_unsupported_nvfp4_inference_error(msg):
+ return _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE
+ if _is_missing_compressed_tensors_error(msg):
+ return _COMPRESSED_TENSORS_INFERENCE_UNSUPPORTED_MESSAGE
+ return None
+
+
def _maybe_unsupported_message(msg: str) -> str:
"""Rewrite a load/validate error into the friendly "not supported yet"
message when it matches a known unsupported-model signature; otherwise
@@ -16219,14 +16256,16 @@ async def _load_model_impl(
raise
except ValueError as e:
redacted_msg = redact_native_paths(str(e))
- if _is_unsupported_nvfp4_inference_error(redacted_msg):
+ _unsupported_quantization = _unsupported_quantization_detail(redacted_msg)
+ if _unsupported_quantization is not None:
logger.warning(
- "NVFP4 inference is not supported yet while loading '%s'",
+ "Unsupported quantization while loading '%s': %s",
model_log_label,
+ _unsupported_quantization,
)
raise HTTPException(
status_code = 500,
- detail = _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE,
+ detail = _unsupported_quantization,
)
if native_grant_backed:
logger.warning(
@@ -16255,14 +16294,16 @@ async def _load_model_impl(
raise HTTPException(status_code = 409, detail = str(e))
# Friendlier message for models Unsloth cannot load.
redacted_msg = redact_native_paths(str(e))
- if _is_unsupported_nvfp4_inference_error(redacted_msg):
+ _unsupported_quantization = _unsupported_quantization_detail(redacted_msg)
+ if _unsupported_quantization is not None:
logger.warning(
- "NVFP4 inference is not supported yet while loading '%s'",
+ "Unsupported quantization while loading '%s': %s",
model_log_label,
+ _unsupported_quantization,
)
raise HTTPException(
status_code = 500,
- detail = _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE,
+ detail = _unsupported_quantization,
)
if native_grant_backed:
logger.error(
@@ -16805,14 +16846,16 @@ async def validate_model(
"in Settings, and confirm access to this gated repository."
),
)
- if _is_unsupported_nvfp4_inference_error(redacted_msg):
+ _unsupported_quantization = _unsupported_quantization_detail(redacted_msg)
+ if _unsupported_quantization is not None:
logger.warning(
- "NVFP4 inference is not supported yet while validating '%s'",
+ "Unsupported quantization while validating '%s': %s",
model_log_label,
+ _unsupported_quantization,
)
raise HTTPException(
status_code = 400,
- detail = _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE,
+ detail = _unsupported_quantization,
)
if native_grant_backed:
logger.error(
diff --git a/studio/backend/tests/test_compressed_tensors_load_error_message.py b/studio/backend/tests/test_compressed_tensors_load_error_message.py
new file mode 100644
index 00000000000..68716f1665b
--- /dev/null
+++ b/studio/backend/tests/test_compressed_tensors_load_error_message.py
@@ -0,0 +1,320 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A compressed-tensors checkpoint is refused on every host, not only on MLX.
+
+Studio only recognised the MLX loader's NVFP4 error, so on CUDA, ROCm and CPU the load
+died inside transformers telling the user to ``pip install compressed-tensors`` -- advice
+they cannot act on, since Studio runs from its own environment (#8246).
+
+The messages below are transformers' own: two sites, two wordings each, because both were
+reworded in 5.10 to name a minimum version. A later test re-derives the pair from the
+installed transformers, so a third rewording fails here.
+"""
+
+import asyncio
+import importlib.util
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+from fastapi import HTTPException
+
+from models.inference import LoadRequest, ValidateModelRequest
+
+_BACKEND_ROOT = Path(__file__).resolve().parent.parent
+
+EXPECTED = (
+ "This model is quantized with compressed-tensors, which Unsloth cannot run yet. "
+ "Installing compressed-tensors will not help. Try a GGUF or a bitsandbytes 4-bit "
+ "build of this model instead."
+)
+
+CONFIG_IMPORT_ERROR = (
+ "compressed_tensors is not installed and is required for compressed-tensors "
+ "quantization. Please install it with `pip install compressed-tensors`."
+)
+CONFIG_IMPORT_ERROR_5_10 = (
+ "compressed-tensors>=0.15.0 is required for compressed-tensors quantization. "
+ "Please install it with `pip install compressed-tensors>=0.15.0`."
+)
+QUANTIZER_IMPORT_ERROR = (
+ "Using `compressed_tensors` quantized models requires the compressed-tensors "
+ "library: `pip install compressed-tensors`"
+)
+QUANTIZER_IMPORT_ERROR_5_10 = (
+ "Using `compressed_tensors` quantized models requires compressed-tensors>=0.15.0: "
+ "`pip install compressed-tensors>=0.15.0`"
+)
+
+REFUSALS = {
+ "config-pre-5.10": CONFIG_IMPORT_ERROR,
+ "config-5.10-plus": CONFIG_IMPORT_ERROR_5_10,
+ "quantizer-pre-5.10": QUANTIZER_IMPORT_ERROR,
+ "quantizer-5.10-plus": QUANTIZER_IMPORT_ERROR_5_10,
+}
+
+
+def _load_route_module():
+ spec = importlib.util.spec_from_file_location(
+ "inference_route_compressed_tensors_error",
+ _BACKEND_ROOT / "routes/inference.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _load_failure(
+ message,
+ exception_type = ImportError,
+ native = False,
+) -> HTTPException:
+ inference_route = _load_route_module()
+ model_path = "unsloth/gemma-4-E2B-it-NVFP4"
+ model_label = "gemma-4-E2B-it-NVFP4" if native else model_path
+ request = LoadRequest(model_path = model_path)
+ backend = MagicMock(active_model_name = None)
+ with (
+ patch.object(
+ inference_route,
+ "_resolve_model_identifier_for_request",
+ return_value = (model_path, model_label, native),
+ ),
+ patch.object(
+ inference_route, "resolve_effective_chat_template_override", return_value = None
+ ),
+ patch.object(inference_route, "get_inference_backend", return_value = backend),
+ patch.object(inference_route, "get_llama_cpp_backend", return_value = MagicMock()),
+ patch.object(
+ inference_route.ModelConfig, "from_identifier", side_effect = exception_type(message)
+ ),
+ pytest.raises(HTTPException) as exc,
+ ):
+ asyncio.run(inference_route.load_model(request, MagicMock(), current_subject = "test-user"))
+ return exc.value
+
+
+def _validation_failure(
+ message,
+ exception_type = ImportError,
+ native = False,
+) -> HTTPException:
+ inference_route = _load_route_module()
+ model_path = "unsloth/gemma-4-E2B-it-NVFP4"
+ model_label = "gemma-4-E2B-it-NVFP4" if native else model_path
+ request = ValidateModelRequest(model_path = model_path)
+ with (
+ patch.object(
+ inference_route,
+ "_resolve_model_identifier_for_request",
+ return_value = (model_path, model_label, native),
+ ),
+ patch.object(
+ inference_route.ModelConfig, "from_identifier", side_effect = exception_type(message)
+ ),
+ pytest.raises(HTTPException) as exc,
+ ):
+ asyncio.run(inference_route.validate_model(request, current_subject = "test-user"))
+ return exc.value
+
+
[email protected]("message", REFUSALS.values(), ids = list(REFUSALS))
[email protected]("native", [False, True])
+def test_the_load_route_refuses_with_something_the_user_can_act_on(message, native):
+ error = _load_failure(message, native = native)
+
+ assert error.status_code == 500
+ assert error.detail == EXPECTED
+ assert "pip install" not in error.detail
+
+
[email protected]("message", REFUSALS.values(), ids = list(REFUSALS))
[email protected]("native", [False, True])
+def test_validate_refuses_the_same_way(message, native):
+ error = _validation_failure(message, native = native)
+
+ assert error.status_code == 400
+ assert error.detail == EXPECTED
+
+
+def test_no_signature_carries_a_version_number():
+ """The token that moved in 5.10 is the one a signature must not contain.
+
+ A signature pinning the minimum version matches exactly half the supported range.
+ """
+ inference_route = _load_route_module()
+
+ for signature in inference_route._MISSING_COMPRESSED_TENSORS_SIGNATURES:
+ assert ">=" not in signature
+ assert not any(character.isdigit() for character in signature)
+ assert len(signature) > 30
+
+
[email protected]("exception_type", [ImportError, RuntimeError, Exception])
+def test_the_exception_class_does_not_matter(exception_type):
+ """transformers raises ImportError today; the route must not depend on that."""
+ assert _load_failure(CONFIG_IMPORT_ERROR, exception_type = exception_type).detail == EXPECTED
+
+
+def _raise_with_compressed_tensors_absent(call):
+ """Run ``call`` with transformers believing the library is not installed.
+
+ Through ``is_compressed_tensors_available()``, so the real message is reached.
+ """
+ with pytest.raises(ImportError) as exc:
+ call()
+ return str(exc.value)
+
+
+def test_the_message_the_installed_transformers_really_raises_is_recognised():
+ """The drift guard. Whatever wording is installed here has to be matched.
+
+ Both sites, each through its own gate rather than a mock of the code under test.
+ """
+ transformers = pytest.importorskip("transformers")
+ inference_route = _load_route_module()
+
+ from transformers.quantizers import quantizer_compressed_tensors as quantizer_module
+ from transformers.utils import quantization_config as config_module
+
+ messages = {}
+ with patch.object(config_module, "is_compressed_tensors_available", return_value = False):
+ messages["config"] = _raise_with_compressed_tensors_absent(
+ lambda: config_module.CompressedTensorsConfig()
+ )
+ with patch.object(quantizer_module, "is_compressed_tensors_available", return_value = False):
+ quantizer = quantizer_module.CompressedTensorsHfQuantizer.__new__(
+ quantizer_module.CompressedTensorsHfQuantizer
+ )
+ messages["quantizer"] = _raise_with_compressed_tensors_absent(
+ lambda: quantizer.validate_environment()
+ )
+
+ for site, message in messages.items():
+ assert "pip install" in message, (site, message)
+ assert inference_route._unsupported_quantization_detail(message) == EXPECTED, (
+ f"transformers {transformers.__version__} reworded the {site} refusal: {message!r}. "
+ "Add the part that survived to _MISSING_COMPRESSED_TENSORS_SIGNATURES."
+ )
+
+
[email protected]("message", REFUSALS.values(), ids = list(REFUSALS))
+def test_the_message_survives_the_trip_out_of_the_inference_worker(message):
+ """The route can only match what reaches it, and two layers may rewrite it.
+
+ The load runs in a subprocess and its error passes through ``format_error_message``,
+ which rewrites 404s, auth failures and OOM into its own sentences. Not this one.
+ """
+ from utils.utils import format_error_message
+ assert format_error_message(ImportError(message), "unsloth/gemma-4-E2B-it-NVFP4") == message
+
+
+def _quantizer_refusal_messages():
+ """Every "you are missing a dependency" message transformers' quantizers raise.
+
+ Parsed from the installed package, so a family added upstream is covered the day it
+ lands. Not executed: most sites need a config object and a device.
+ """
+ import ast
+ import re
+ from pathlib import Path as _Path
+
+ import transformers.quantizers as quantizers_package
+
+ package_dir = _Path(quantizers_package.__file__).parent
+ out = []
+ for source_file in sorted(package_dir.glob("quantizer_*.py")):
+ tree = ast.parse(source_file.read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Raise) or node.exc is None:
+ continue
+ for literal in ast.walk(node.exc):
+ if not isinstance(literal, ast.Constant) or not isinstance(literal.value, str):
+ continue
+ text = literal.value
+ if not re.search(r"requires|install", text, re.IGNORECASE):
+ continue
+ out.append((source_file.stem, text))
+ return out
+
+
+def test_no_other_quantization_family_is_re_routed_into_this_refusal():
+ """The families that resolved correctly before must still resolve the same way.
+
+ A widened matcher is only safe if it stayed inside compressed-tensors, so every
+ quantizer transformers ships is walked and only its own message may match.
+ """
+ pytest.importorskip("transformers")
+ inference_route = _load_route_module()
+
+ messages = _quantizer_refusal_messages()
+ assert len({module for module, _ in messages}) >= 8, messages
+
+ for module, message in messages:
+ detail = inference_route._unsupported_quantization_detail(message)
+ if module == "quantizer_compressed_tensors":
+ continue
+ assert (
+ detail is None
+ ), f"{module} message now matches the compressed-tensors refusal: {message!r}"
+
+
+def test_the_mlx_refusal_is_unchanged():
+ """The signature this route already recognised keeps its own wording."""
+ error = _load_failure(
+ "Unsloth: 'unsloth/gemma-4-E2B-it-NVFP4' has per-module MLX quantization "
+ "metadata {'config_groups': {'group_1': {'format': 'nvfp4-pack-quantized'}}}",
+ exception_type = RuntimeError,
+ )
+
+ assert error.detail == (
+ "We are working on supporting NVFP4 inference. For now it is not supported"
+ )
+
+
+ "message",
+ [
+ "Network connection timed out",
+ # Names the library without being about it missing. Must not be swallowed.
+ "Failed to parse the config of this compressed-tensors export",
+ ],
+)
+def test_an_unrelated_failure_keeps_its_own_message(message):
+ error = _load_failure(message, exception_type = RuntimeError)
+
+ assert error.detail == f"Failed to load model: {message}"
+
+
+def test_the_child_process_diagnostics_block_is_not_matched():
+ """A llama-server tail is quoted from an untrusted process, so it is not ours to read."""
+ inference_route = _load_route_module()
+ msg = (
+ "llama-server exited before becoming healthy\n\nllama-server output:\n"
+ f"{CONFIG_IMPORT_ERROR}\n"
+ )
+
+ assert inference_route._unsupported_quantization_detail(msg) is None
+
+
+def test_the_refusal_names_no_quantization_scheme():
+ """The errors it fires on carry no scheme, so the text may not claim one.
+
+ Both wordings are raised before transformers looks at the compression config, so a
+ W4A16, W8A8, INT8 or MXFP4 checkpoint lands here with the NVFP4 message exactly.
+ """
+ inference_route = _load_route_module()
+ refusal = inference_route._COMPRESSED_TENSORS_INFERENCE_UNSUPPORTED_MESSAGE
+
+ for scheme in ("NVFP4", "FP8", "W4A16", "W8A8", "INT8", "MXFP4"):
+ assert scheme.lower() not in refusal.lower(), (
+ f"the compressed-tensors refusal names {scheme}, but the errors it matches "
+ f"do not identify the scheme"
+ )
+ assert "compressed-tensors" in refusal
+ assert "GGUF" in refusal
+ # Unqualified "4-bit" reads as a no-op to someone who already picked W4A16.
+ assert "bitsandbytes 4-bit" in refusal
+
+ assert "NVFP4" in inference_route._NVFP4_INFERENCE_UNSUPPORTED_MESSAGE
diff --git a/studio/backend/tests/test_compressed_tensors_signature_drift.py b/studio/backend/tests/test_compressed_tensors_signature_drift.py
new file mode 100644
index 00000000000..f3b40d78f2a
--- /dev/null
+++ b/studio/backend/tests/test_compressed_tensors_signature_drift.py
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The compressed-tensors drift guard, with nothing but transformers installed.
+
+`test_compressed_tensors_load_error_message.py` runs the same check, but only where the
+whole backend imports, and `studio-backend-ci.yml` pins that environment to
+`transformers>=4.51,<5.5` -- outside the 5.10 rewording the matcher exists to survive.
+This file reads the signatures and `_diagnosis_text` out of `routes/inference.py` with
+`ast` instead, so it needs no fastapi and runs on any transformers.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+_ROUTE = Path(__file__).resolve().parent.parent / "routes" / "inference.py"
+
+
+def _route_pieces():
+ """`_MISSING_COMPRESSED_TENSORS_SIGNATURES` and `_diagnosis_text`, without importing.
+
+ Lifting them out with `ast` is what lets this run beside a transformers the backend
+ environment cannot hold.
+ """
+ tree = ast.parse(_ROUTE.read_text(encoding = "utf-8"))
+ literals = {}
+ diagnosis = None
+ for node in tree.body:
+ if isinstance(node, ast.Assign):
+ for target in node.targets:
+ if isinstance(target, ast.Name) and target.id in (
+ "_MISSING_COMPRESSED_TENSORS_SIGNATURES",
+ "_COMPRESSED_TENSORS_INFERENCE_UNSUPPORTED_MESSAGE",
+ ):
+ literals[target.id] = ast.literal_eval(node.value)
+ elif isinstance(node, ast.FunctionDef) and node.name == "_diagnosis_text":
+ node.decorator_list = []
+ diagnosis = node
+ assert "_MISSING_COMPRESSED_TENSORS_SIGNATURES" in literals, _ROUTE
+ assert diagnosis is not None, "_diagnosis_text moved; this guard reads the wrong text"
+ namespace = {"re": re}
+ exec( # noqa: S102 - the source is this repository's own, read from disk above
+ compile(ast.Module(body = [diagnosis], type_ignores = []), str(_ROUTE), "exec"),
+ namespace,
+ )
+ return literals["_MISSING_COMPRESSED_TENSORS_SIGNATURES"], namespace["_diagnosis_text"]
+
+
+def _matches(message: str) -> bool:
+ signatures, diagnosis_text = _route_pieces()
+ lowered = diagnosis_text(message).lower()
+ return any(signature in lowered for signature in signatures)
+
+
+def _refusals_from_installed_transformers():
+ """The two refusals the INSTALLED transformers raises with the library absent.
+
+ Through `is_compressed_tensors_available`, so these are the real strings, not a copy.
+ """
+ pytest.importorskip("transformers")
+ from transformers.quantizers import quantizer_compressed_tensors as quantizer_module
+ from transformers.utils import quantization_config as config_module
+
+ messages = {}
+ with patch.object(config_module, "is_compressed_tensors_available", return_value = False):
+ with pytest.raises(ImportError) as raised:
+ config_module.CompressedTensorsConfig()
+ messages["config"] = str(raised.value)
+ with patch.object(quantizer_module, "is_compressed_tensors_available", return_value = False):
+ quantizer = quantizer_module.CompressedTensorsHfQuantizer.__new__(
+ quantizer_module.CompressedTensorsHfQuantizer
+ )
+ with pytest.raises(ImportError) as raised:
+ quantizer.validate_environment()
+ messages["quantizer"] = str(raised.value)
+ return messages
+
+
+def test_the_installed_transformers_refusals_are_both_recognised():
+ """The drift guard proper, runnable against any transformers on any host."""
+ import transformers
+ for site, message in _refusals_from_installed_transformers().items():
+ assert "pip install" in message, (site, message)
+ assert _matches(message), (
+ f"transformers {transformers.__version__} reworded the {site} "
+ f"compressed-tensors refusal: {message!r}. Add the part that survived to "
+ "_MISSING_COMPRESSED_TENSORS_SIGNATURES in studio/backend/routes/inference.py."
+ )
+
+
+def test_no_signature_pins_a_version_number():
+ """The version token is what moved in 5.10, so no signature may contain one."""
+ signatures, _ = _route_pieces()
+ assert signatures
+ for signature in signatures:
+ assert not re.search(r"\d+\.\d+", signature), signature
+ # Long enough to be this refusal and not an arbitrary sentence saying the words.
+ assert len(signature) >= 30, signature
+
+
+def test_no_other_quantizer_family_is_claimed():
+ """Every other quantizer's "you are missing a library" message must not match.
+
+ Parsed from the installed transformers, so a family added upstream is covered the day
+ it lands. The module count is asserted: a parse finding too little would pass empty.
+ """
+ pytest.importorskip("transformers")
+ import transformers.quantizers as quantizers_package
+
+ package_dir = Path(quantizers_package.__file__).parent
+ scanned, claimed = set(), []
+ for source_file in sorted(package_dir.glob("quantizer_*.py")):
+ tree = ast.parse(source_file.read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Raise) or node.exc is None:
+ continue
+ for literal in ast.walk(node.exc):
+ if not isinstance(literal, ast.Constant) or not isinstance(literal.value, str):
+ continue
+ if not re.search(r"requires|install", literal.value, re.IGNORECASE):
+ continue
+ scanned.add(source_file.stem)
+ if source_file.stem != "quantizer_compressed_tensors" and _matches(literal.value):
+ claimed.append((source_file.stem, literal.value))
+ assert len(scanned) >= 8, sorted(scanned)
+ assert not claimed, claimed
+
+
+def test_this_guard_needs_no_studio_backend_dependency():
+ """The property that makes it runnable outside the backend's pinned environment.
+
+ An import of fastapi or the route module puts it back inside the `<5.5` pin.
+ """
+ tree = ast.parse(Path(__file__).read_text(encoding = "utf-8"))
+ imported = set()
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Import):
+ imported.update(alias.name.split(".")[0] for alias in node.names)
+ elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
+ imported.add(node.module.split(".")[0])
+ forbidden = {"fastapi", "models", "routes", "core", "utils", "starlette", "jwt"}
+ assert not (imported & forbidden), sorted(imported & forbidden)
+
+
+def test_a_workflow_that_installs_a_modern_transformers_actually_collects_this_file():
+ """The guard above is worth nothing in an environment that never runs it.
+
+ Every auto-discovering job pins `transformers>=4.51,<5.5`; the jobs that install a
+ modern one collect listed paths, not trees, so an unlisted file never runs.
+ """
+ workflow = (
+ Path(__file__).resolve().parents[3] / ".github" / "workflows" / "consolidated-tests-ci.yml"
+ )
+ text = workflow.read_text(encoding = "utf-8")
+ assert (
+ "studio/backend/tests/test_compressed_tensors_signature_drift.py" in text
+ ), "the modern-transformers matrix does not collect this file"
+ assert 'transformers_spec: "transformers>=5,<6"' in text
diff --git a/studio/backend/tests/test_launch_reasoning_flag.py b/studio/backend/tests/test_launch_reasoning_flag.py
new file mode 100644
index 00000000000..923b6befa70
--- /dev/null
+++ b/studio/backend/tests/test_launch_reasoning_flag.py
@@ -0,0 +1,380 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The launch-time thinking default goes out as --reasoning where the build has it.
+
+llama-server deprecates enable_thinking via --chat-template-kwargs (#7526), per KEY, so
+only that key moves. The gate is the `llama-server --help` flag catalogue and is closed
+on anything short of a positive answer: a build predating --reasoning exits with
+"error: invalid argument: --reasoning" rather than starting without it. The help text
+below is verbatim from two real binaries, b10909 and b6277.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import types
+from pathlib import Path
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+import pytest # noqa: E402
+
+from test_llama_cpp_placement import _backend, _launch # noqa: E402
+
+import core.inference.llama_cpp as llama_cpp # noqa: E402
+from core.inference.llama_cpp import ( # noqa: E402
+ LlamaCppBackend,
+ _build_launch_reasoning_args,
+)
+from core.inference.llama_server_args import strip_shadowing_flags # noqa: E402
+
+# b10909. The name-alikes are the point: a substring match would report the flag wrongly.
+MODERN_HELP = """usage: llama-server [options]
+
+-m, --model FNAME model path
+--reasoning-format FORMAT controls whether thought tags are allowed and/or extracted from the
+ response, and in which format they're returned; one of:
+ - none: leaves thoughts unparsed in `message.content`
+-rea, --reasoning [on|off|auto] Use reasoning/thinking in the chat ('on', 'off', or 'auto', default:
+ 'auto' (detect from template))
+ (env: LLAMA_ARG_REASONING)
+--reasoning-effort LEVEL reasoning effort level given to the chat template: 'default' to keep
+ the template default,
+--reasoning-budget N token budget for thinking: -1 for unrestricted, 0 for immediate end,
+ N>0 for token budget (default: -1)
+ (env: LLAMA_ARG_THINK_BUDGET)
+--reasoning-preserve, --no-reasoning-preserve
+ preserve reasoning trace in the full history, not just the last
+ assistant message (default: enabled)
+--chat-template-kwargs STRING sets additional params for the json template parser, must be a valid
+ JSON object
+--jinja, --no-jinja whether to use jinja template engine for chat (default: disabled)
+"""
+
+OLD_HELP = """usage: llama-server [options]
+
+-m, --model FNAME model path
+--reasoning-format FORMAT controls whether thought tags are allowed and/or extracted from the
+ response, and in which format they're returned; one of:
+ - none: leaves thoughts unparsed in `message.content`
+--reasoning-budget N controls the amount of thinking allowed; currently only one of: -1 for
+ unrestricted thinking budget, or 0 to disable thinking (default: -1)
+ (env: LLAMA_ARG_THINK_BUDGET)
+--chat-template-kwargs STRING sets additional params for the json template parser
+--jinja, --no-jinja whether to use jinja template engine for chat (default: disabled)
+"""
+
+REMOVAL_STUB_HELP = """usage: llama-server [options]
+
+-m, --model FNAME model path
+-rea, --reasoning [on|off|auto] (argument has been removed)
+--chat-template-kwargs STRING sets additional params for the json template parser
+"""
+
+MODERN_CAPS = {"supports_reasoning_flag": True}
+OLD_CAPS = {"supports_reasoning_flag": False}
+
+
+def _probe(
+ tmp_path,
+ monkeypatch,
+ help_text,
+ returncode = 0,
+ stream = "stdout",
+):
+ """The real probe against a stubbed ``llama-server --help``."""
+ binary = tmp_path / "llama-server"
+ binary.write_text("")
+ completed = subprocess.CompletedProcess(
+ args = [str(binary), "--help"],
+ returncode = returncode,
+ stdout = help_text if stream == "stdout" else "",
+ stderr = help_text if stream == "stderr" else "",
+ )
+ monkeypatch.setattr(llama_cpp.subprocess, "run", lambda *a, **k: completed)
+ LlamaCppBackend._capability_cache.clear()
+ return LlamaCppBackend.probe_server_capabilities(str(binary))
+
+
+class TestTheProbeReadsTheFlagAndNotItsNamesakes:
+ def test_the_bundled_prebuilt_advertises_the_flag(self, tmp_path, monkeypatch):
+ assert _probe(tmp_path, monkeypatch, MODERN_HELP)["supports_reasoning_flag"] is True
+
+ def test_a_build_with_only_the_namesakes_does_not(self, tmp_path, monkeypatch):
+ """--reasoning-format / -budget / -preserve all contain the string."""
+ assert _probe(tmp_path, monkeypatch, OLD_HELP)["supports_reasoning_flag"] is False
+
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"help_text": "", "returncode": 0},
+ {"help_text": "", "returncode": 1},
+ {"help_text": "usage: llama-server [options]\n"},
+ {"help_text": REMOVAL_STUB_HELP},
+ ],
+ ids = ["empty-help", "failed-probe", "usage-only", "removal-stub"],
+ )
+ def test_the_gate_is_closed_on_anything_but_a_positive_answer(
+ self, tmp_path, monkeypatch, kwargs
+ ):
+ """Unlike --jinja, this gate must fail CLOSED: the flag aborts an old build."""
+ assert _probe(tmp_path, monkeypatch, **kwargs)["supports_reasoning_flag"] is False
+
+ def test_a_parsed_catalogue_counts_even_on_a_nonzero_exit(self, tmp_path, monkeypatch):
+ """Same policy as the sibling flags: a nonzero exit after a full help still named
+ the flags, and the doubt is carried separately in *_probe_inconclusive.
+ """
+ caps = _probe(tmp_path, monkeypatch, MODERN_HELP, returncode = 1)
+ assert caps["supports_reasoning_flag"] == caps["supports_reasoning_budget"] is True
+ assert caps["reasoning_budget_probe_inconclusive"] is True
+
+ def test_a_missing_binary_reports_it_unsupported(self, tmp_path):
+ caps = LlamaCppBackend.probe_server_capabilities(str(tmp_path / "nope"))
+ assert caps["supports_reasoning_flag"] is False
+
+ def test_the_flag_is_read_from_stderr_too(self, tmp_path, monkeypatch):
+ caps = _probe(tmp_path, monkeypatch, MODERN_HELP, stream = "stderr")
+ assert caps["supports_reasoning_flag"] is True
+
+
+class TestOnlyTheDeprecatedKeyMoves:
+ """The kwargs channel keeps carrying every key that has no flag of its own."""
+
+ def test_thinking_on_becomes_the_flag_and_nothing_else_is_emitted(self):
+ assert _build_launch_reasoning_args(MODERN_CAPS, {"enable_thinking": True}) == [
+ "--reasoning",
+ "on",
+ ]
+
+ def test_thinking_off_becomes_the_flag(self):
+ assert _build_launch_reasoning_args(MODERN_CAPS, {"enable_thinking": False}) == [
+ "--reasoning",
+ "off",
+ ]
+
+ @pytest.mark.parametrize("preserve", [True, False])
+ def test_preserve_thinking_stays_in_the_kwargs(self, preserve):
+ """It is a template variable with no flag: #9096's default has to survive."""
+ args = _build_launch_reasoning_args(
+ MODERN_CAPS, {"enable_thinking": True, "preserve_thinking": preserve}
+ )
+ assert args[:2] == ["--reasoning", "on"]
+ assert args[2] == "--chat-template-kwargs"
+ assert json.loads(args[3]) == {"preserve_thinking": preserve}
+
+ def test_a_reasoning_effort_ladder_is_untouched(self):
+ """gpt-oss style never sets enable_thinking, so nothing is deprecated there."""
+ kwargs = {"reasoning_effort": "high"}
+ assert _build_launch_reasoning_args(MODERN_CAPS, kwargs) == [
+ "--chat-template-kwargs",
+ json.dumps(kwargs),
+ ]
+
+ def test_an_unrelated_future_key_is_untouched(self):
+ kwargs = {"enable_thinking": False, "some_new_template_variable": "x"}
+ args = _build_launch_reasoning_args(MODERN_CAPS, kwargs)
+ assert args[:2] == ["--reasoning", "off"]
+ assert json.loads(args[3]) == {"some_new_template_variable": "x"}
+
+ def test_the_caller_s_dict_is_not_mutated(self):
+ kwargs = {"enable_thinking": True}
+ _build_launch_reasoning_args(MODERN_CAPS, kwargs)
+ assert kwargs == {"enable_thinking": True}
+
+ @pytest.mark.parametrize(
+ "caps",
+ [OLD_CAPS, {}, {"supports_reasoning_flag": None}],
+ ids = ["old-build", "nothing-known", "unset"],
+ )
+ def test_without_the_flag_the_argv_is_what_main_emits(self, caps):
+ kwargs = {"enable_thinking": True, "preserve_thinking": False}
+ assert _build_launch_reasoning_args(caps, kwargs) == [
+ "--chat-template-kwargs",
+ json.dumps(kwargs),
+ ]
+
+ @pytest.mark.parametrize(
+ "caps", [MODERN_CAPS, OLD_CAPS, {}], ids = ["modern", "old-build", "nothing-known"]
+ )
+ def test_an_empty_dict_still_goes_out_as_main_sends_it(self, caps):
+ """Nothing moved, so the argument main appends has to be appended.
+
+ Unreachable from the launcher today; held so the untouched path is identical
+ to main by construction rather than by luck.
+ """
+ assert _build_launch_reasoning_args(caps, {}) == ["--chat-template-kwargs", "{}"]
+
+ def test_an_empty_remainder_after_the_flag_appends_nothing(self):
+ """The other side of it: that argument did not exist on main either."""
+ assert _build_launch_reasoning_args(MODERN_CAPS, {"enable_thinking": True}) == [
+ "--reasoning",
+ "on",
+ ]
+
+
+# enable_thinking style, with preserve_thinking so both channels are exercised at once.
+THINKING_TEMPLATE = (
+ "{% if enable_thinking %}<think>\n{% endif %}"
+ "{% if preserve_thinking %}{{ messages[0].reasoning_content }}{% endif %}"
+ "{{ messages[0].content }}"
+)
+# gpt-oss style: a ladder, no on/off gate, nothing deprecated.
+EFFORT_TEMPLATE = "{{ reasoning_effort }}{% if reasoning_effort == 'high' %}<think>{% endif %}"
+
+OSES = {
+ "linux": ("linux", False),
+ "windows": ("win32", False),
+ "wsl": ("linux", True),
+ "macos": ("darwin", False),
+}
+VENDORS = ("nvidia", "amd", "cpu")
+
+
+def _fake_torch(*, rocm: bool, device_count: int) -> types.ModuleType:
+ torch = types.ModuleType("torch")
+ torch.__version__ = "2.11.0+rocm6.2" if rocm else "2.11.0+cu130"
+ version = types.ModuleType("torch.version")
+ version.hip = "6.2.0" if rocm else None
+ version.cuda = None if rocm else "13.0"
+ torch.version = version
+ torch.cuda = types.SimpleNamespace(
+ is_available = lambda: device_count > 0,
+ device_count = lambda: device_count,
+ )
+ return torch
+
+
+def _argv_for(tmp_path, monkeypatch, *, os_label, vendor, modern, template):
+ """The argv the real ``load_model`` builds on one host, for one vintage."""
+ platform, is_wsl = OSES[os_label]
+ monkeypatch.setattr(llama_cpp.sys, "platform", platform, raising = False)
+ monkeypatch.setattr(llama_cpp, "_is_wsl", lambda: is_wsl, raising = False)
+ monkeypatch.setitem(
+ sys.modules,
+ "torch",
+ _fake_torch(rocm = vendor == "amd", device_count = 0 if vendor == "cpu" else 1),
+ )
+ memory = [] if vendor == "cpu" else [(0, 24_000, 24_000)]
+ backend, gguf = _backend(tmp_path, vulkan = False, memory = memory)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(lambda cls, binary = None: dict(MODERN_CAPS if modern else OLD_CAPS)),
+ )
+ captured = _launch(
+ backend,
+ gguf,
+ n_ctx = 4096,
+ chat_template_override = template,
+ )
+ return captured["cmd"]
+
+
+def _reasoning_slice(cmd: list[str]) -> list[str]:
+ """Just the reasoning arguments, in order, out of the whole command line."""
+ out: list[str] = []
+ i = 0
+ while i < len(cmd):
+ if cmd[i] in {"--reasoning", "--chat-template-kwargs"}:
+ out.extend(cmd[i : i + 2])
+ i += 2
+ continue
+ i += 1
+ return out
+
+
+_HOSTS = [(o, v) for o in OSES for v in VENDORS]
+
+
+class TestTheRealLaunchOnEveryHost:
+ """The reasoning arguments are invariant under OS and device.
+
+ Drives the real ``load_model``, since placement appends into the same command.
+ """
+
+ @pytest.mark.parametrize("os_label,vendor", _HOSTS)
+ def test_a_modern_build_gets_the_flag_and_keeps_the_preserve_kwarg(
+ self, tmp_path, monkeypatch, os_label, vendor
+ ):
+ cmd = _argv_for(
+ tmp_path,
+ monkeypatch,
+ os_label = os_label,
+ vendor = vendor,
+ modern = True,
+ template = THINKING_TEMPLATE,
+ )
+ assert _reasoning_slice(cmd) == [
+ "--reasoning",
+ "on",
+ "--chat-template-kwargs",
+ '{"preserve_thinking": false}',
+ ]
+ assert cmd.count("--reasoning") == 1
+
+ @pytest.mark.parametrize("os_label,vendor", _HOSTS)
+ def test_an_old_build_never_sees_the_flag(self, tmp_path, monkeypatch, os_label, vendor):
+ cmd = _argv_for(
+ tmp_path,
+ monkeypatch,
+ os_label = os_label,
+ vendor = vendor,
+ modern = False,
+ template = THINKING_TEMPLATE,
+ )
+ assert "--reasoning" not in cmd
+ assert _reasoning_slice(cmd) == [
+ "--chat-template-kwargs",
+ '{"enable_thinking": true, "preserve_thinking": false}',
+ ]
+
+ @pytest.mark.parametrize("modern", [True, False], ids = ["modern", "old"])
+ def test_an_effort_ladder_model_is_unchanged_on_both_vintages(
+ self, tmp_path, monkeypatch, modern
+ ):
+ cmd = _argv_for(
+ tmp_path,
+ monkeypatch,
+ os_label = "linux",
+ vendor = "nvidia",
+ modern = modern,
+ template = EFFORT_TEMPLATE,
+ )
+ assert "--reasoning" not in cmd
+ assert _reasoning_slice(cmd)[0] == "--chat-template-kwargs"
+ assert "reasoning_effort" in _reasoning_slice(cmd)[1]
+
+
+class TestInheritedExtrasTreatBothSpellingsAlike:
+ """Applying a chat template override recomputes the reasoning default.
+
+ An inherited copy is appended after ours and would last-wins-override it, so both
+ spellings have to be stripped.
+ """
+
+ @pytest.mark.parametrize(
+ "args",
+ [
+ ["--reasoning", "off"],
+ ["--reasoning=off"],
+ ["-rea", "off"],
+ ["--chat-template-kwargs", '{"enable_thinking": false}'],
+ ],
+ )
+ def test_a_template_override_strips_the_inherited_default(self, args):
+ assert strip_shadowing_flags([*args, "--threads", "8"], strip_template = True) == [
+ "--threads",
+ "8",
+ ]
+
+ @pytest.mark.parametrize(
+ "args",
+ [["--reasoning", "off"], ["--chat-template-kwargs", '{"enable_thinking": false}']],
+ )
+ def test_without_a_template_override_both_survive(self, args):
+ assert strip_shadowing_flags([*args], strip_template = False) == args