Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: use --reasoning where available and name the compressed-tensors refusal (#11072)
Daniel Hane7becafe5ca7
diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.ymlindex d78ee60aadf..ddc65f0ca6f 100644--- a/.github/workflows/consolidated-tests-ci.yml+++ b/.github/workflows/consolidated-tests-ci.yml@@ -405,6 +405,10 @@ jobs:# against simulated CPU counts and memory so the result is identical on# a 4-vCPU runner and a 192-core box.#+ # The drift guard is named explicitly (these jobs collect paths, not+ # trees): it is the only leg with a transformers past the `>=4.51,<5.5`+ # pin, and the wording it exists to survive landed in 5.10.+ ## The Unsloth file runs here rather than in studio-backend-ci, which# installs studio.txt and so has no unsloth_zoo at all. Note the policy# module is not on unsloth_zoo main either until the companion PR lands,@@ -417,7 +421,8 @@ jobs:tests/utils/test_dataset_num_proc.py \tests/utils/test_train_on_responses_only_num_proc.pyPYTHONPATH=studio/backend python -m pytest -v --tb=short \- studio/backend/tests/test_dataset_map_num_proc.py+ studio/backend/tests/test_dataset_map_num_proc.py \+ studio/backend/tests/test_compressed_tensors_signature_drift.py- name: unsloth Bucket-A — CPU tests not in Repo tests (CPU)# CPU tests across 6 files under tests/saving/, tests/utils/, tests/python/diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex ead87a21835..3eda91ae8ce 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6256,6 +6256,40 @@ def _build_ngram_mod_flags(return []+def _build_launch_reasoning_args(+ caps: Mapping[str, object], reasoning_kwargs: Mapping[str, object]+) -> list[str]:+ """The launch flags that carry ``reasoning_kwargs``, per key.++ llama-server's --chat-template-kwargs deprecation is PER KEY (#7526), and only+ ``enable_thinking`` has a flag of its own, so only that key moves; every other+ template variable stays in --chat-template-kwargs, still the only channel for one.++ Gated CLOSED on the probe: a build predating the flag exits with "error: invalid+ argument: --reasoning" rather than starting without it, so an unknown answer keeps+ main's argv byte for byte.++ The two spellings are equivalent, not identical. common/arg.cpp's --reasoning+ handler writes the same ``default_template_kwargs["enable_thinking"]`` entry AND+ sets ``params.enable_reasoning``, whose only consumer (tools/server/server-context.cpp)+ folds it into the per-server ``opt.enable_thinking`` default -- which+ tools/server/server-common.cpp then overrides from the merged kwarg anyway. If+ llama.cpp stops writing that kwarg, or stops letting it override, this becomes a+ behaviour change rather than a substitution.++ An exported LLAMA_ARG_REASONING is still overridden by the command line, exactly as+ the kwargs channel overrides it on main; honouring it is #8521.+ """+ remaining = dict(reasoning_kwargs)+ args: list[str] = []+ if caps.get("supports_reasoning_flag") and isinstance(remaining.get("enable_thinking"), bool):+ args.extend(["--reasoning", "on" if remaining.pop("enable_thinking") else "off"])+ # `not args`: a caller passing `{}` would otherwise lose main's `-ctk {}` on an old build.+ if remaining or not args:+ args.extend(["--chat-template-kwargs", json.dumps(remaining)])+ return args++def _build_reasoning_budget_flags(caps: Mapping[str, object], reasoning_budget: int, reasoning_budget_message: str) -> list[str]:@@ -8709,6 +8743,8 @@ class LlamaCppBackend:"supports_reasoning_budget": False,"supports_reasoning_budget_message": False,"reasoning_budget_probe_inconclusive": True,+ # Fails CLOSED, unlike --jinja: a build without --reasoning exits on it.+ "supports_reasoning_flag": False,"supports_no_mmproj_offload": False,"supports_video_fps": False,"supports_load_mode": False,@@ -8760,6 +8796,7 @@ class LlamaCppBackend:supports_slot_save = Falsesupports_reasoning_budget = Falsesupports_reasoning_budget_message = False+ supports_reasoning_flag = Falsesupports_no_mmproj_offload = Falsesupports_video_fps = Falsesupports_load_mode = False@@ -8994,6 +9031,8 @@ class LlamaCppBackend:supports_slot_save = _is_real("--slot-save-path")supports_reasoning_budget = _is_real("--reasoning-budget")supports_reasoning_budget_message = _is_real("--reasoning-budget-message")+ # Catalogue, not help text: --reasoning-format / -budget / -preserve contain it.+ supports_reasoning_flag = _is_real("--reasoning")supports_no_mmproj_offload = _is_real("--no-mmproj-offload")supports_video_fps = _is_real("--video-fps")# --load-mode supersedes --mlock / --no-mmap, which are deprecated.@@ -9091,6 +9130,7 @@ class LlamaCppBackend:"supports_reasoning_budget": supports_reasoning_budget,"supports_reasoning_budget_message": supports_reasoning_budget_message,"reasoning_budget_probe_inconclusive": not (probe_ok and help_nonempty),+ "supports_reasoning_flag": supports_reasoning_flag,"supports_no_mmproj_offload": supports_no_mmproj_offload,"supports_video_fps": supports_video_fps,"supports_load_mode": supports_load_mode,@@ -26315,13 +26355,12 @@ class LlamaCppBackend:# frontend receives the same resolved value via load/status.if self._supports_preserve_thinking:reasoning_kw["preserve_thinking"] = self._preserve_thinking_default- cmd.extend(- [- "--chat-template-kwargs",- json.dumps(reasoning_kw),- ]+ _reasoning_args = _build_launch_reasoning_args(server_caps, reasoning_kw)+ cmd.extend(_reasoning_args)+ logger.info(+ f"Reasoning model: {reasoning_kw} by default "+ f"(launched as {' '.join(_reasoning_args)})")- logger.info(f"Reasoning model: {reasoning_kw} by default")reasoning_budget = resolve_reasoning_budget(extra_args, reasoning_budget)reasoning_budget_message = resolve_reasoning_budget_message(diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex 4ab0f65c3fe..6817bd053c0 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -527,6 +527,10 @@ _TEMPLATE_FLAGS: frozenset[str] = frozenset("--chat-template","--chat-template-file","--chat-template-kwargs",+ # enable_thinking's new spelling (#7526); a template override recomputes the default,+ # so both must strip. Takes a value, so NOT in _BOOLEAN_SHADOWING_FLAGS.+ "--reasoning",+ "-rea","--jinja","--no-jinja",}diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 918cdaeb8a0..a8d586ee925 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -14390,6 +14390,14 @@ _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE = ("We are working on supporting NVFP4 inference. For now it is not supported")+# Said instead of transformers' "pip install compressed-tensors", which no user can act on:+# Studio runs from its own environment (#8246). Names no scheme: these errors carry none.+_COMPRESSED_TENSORS_INFERENCE_UNSUPPORTED_MESSAGE = (+ "This model is quantized with compressed-tensors, which Unsloth cannot run yet. "+ "Installing compressed-tensors will not help. Try a GGUF or a bitsandbytes 4-bit "+ "build of this model instead."+)+def _diagnosis_text(msg: str) -> str:"""``msg`` up to the startup-diagnostics block, which is not ours to read.@@ -14417,6 +14425,35 @@ def _is_unsupported_nvfp4_inference_error(msg: str) -> bool:return "nvfp4" in lower_msg and "per-module mlx quantization metadata" in lower_msg+# The two transformers sites, reworded in 5.10 to add a version: match only what survived.+_MISSING_COMPRESSED_TENSORS_SIGNATURES = (+ "is required for compressed-tensors quantization",+ "`compressed_tensors` quantized models requires",+)+++def _is_missing_compressed_tensors_error(msg: str) -> bool:+ """Whether ``msg`` says the checkpoint needs the compressed-tensors library.++ The NVFP4 matcher above only fires on the MLX loader's metadata error, so it never+ reaches a CUDA, ROCm or CPU host; there the load dies inside transformers (#8246).+ """+ lower_msg = _diagnosis_text(msg).lower()+ return any(sig in lower_msg for sig in _MISSING_COMPRESSED_TENSORS_SIGNATURES)+++def _unsupported_quantization_detail(msg: str) -> Optional[str]:+ """The refusal to show for ``msg``, or None when it is not about quantization.++ One place to add a signature to, so load, native load and validate cannot drift.+ """+ if _is_unsupported_nvfp4_inference_error(msg):+ return _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE+ if _is_missing_compressed_tensors_error(msg):+ return _COMPRESSED_TENSORS_INFERENCE_UNSUPPORTED_MESSAGE+ return None++def _maybe_unsupported_message(msg: str) -> str:"""Rewrite a load/validate error into the friendly "not supported yet"message when it matches a known unsupported-model signature; otherwise@@ -16219,14 +16256,16 @@ async def _load_model_impl(raiseexcept ValueError as e:redacted_msg = redact_native_paths(str(e))- if _is_unsupported_nvfp4_inference_error(redacted_msg):+ _unsupported_quantization = _unsupported_quantization_detail(redacted_msg)+ if _unsupported_quantization is not None:logger.warning(- "NVFP4 inference is not supported yet while loading '%s'",+ "Unsupported quantization while loading '%s': %s",model_log_label,+ _unsupported_quantization,)raise HTTPException(status_code = 500,- detail = _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE,+ detail = _unsupported_quantization,)if native_grant_backed:logger.warning(@@ -16255,14 +16294,16 @@ async def _load_model_impl(raise HTTPException(status_code = 409, detail = str(e))# Friendlier message for models Unsloth cannot load.redacted_msg = redact_native_paths(str(e))- if _is_unsupported_nvfp4_inference_error(redacted_msg):+ _unsupported_quantization = _unsupported_quantization_detail(redacted_msg)+ if _unsupported_quantization is not None:logger.warning(- "NVFP4 inference is not supported yet while loading '%s'",+ "Unsupported quantization while loading '%s': %s",model_log_label,+ _unsupported_quantization,)raise HTTPException(status_code = 500,- detail = _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE,+ detail = _unsupported_quantization,)if native_grant_backed:logger.error(@@ -16805,14 +16846,16 @@ async def validate_model("in Settings, and confirm access to this gated repository."),)- if _is_unsupported_nvfp4_inference_error(redacted_msg):+ _unsupported_quantization = _unsupported_quantization_detail(redacted_msg)+ if _unsupported_quantization is not None:logger.warning(- "NVFP4 inference is not supported yet while validating '%s'",+ "Unsupported quantization while validating '%s': %s",model_log_label,+ _unsupported_quantization,)raise HTTPException(status_code = 400,- detail = _NVFP4_INFERENCE_UNSUPPORTED_MESSAGE,+ detail = _unsupported_quantization,)if native_grant_backed:logger.error(diff --git a/studio/backend/tests/test_compressed_tensors_load_error_message.py b/studio/backend/tests/test_compressed_tensors_load_error_message.pynew file mode 100644index 00000000000..68716f1665b--- /dev/null+++ b/studio/backend/tests/test_compressed_tensors_load_error_message.py@@ -0,0 +1,320 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A compressed-tensors checkpoint is refused on every host, not only on MLX.++Studio only recognised the MLX loader's NVFP4 error, so on CUDA, ROCm and CPU the load+died inside transformers telling the user to ``pip install compressed-tensors`` -- advice+they cannot act on, since Studio runs from its own environment (#8246).++The messages below are transformers' own: two sites, two wordings each, because both were+reworded in 5.10 to name a minimum version. A later test re-derives the pair from the+installed transformers, so a third rewording fails here.+"""++import asyncio+import importlib.util+from pathlib import Path+from unittest.mock import MagicMock, patch++import pytest+from fastapi import HTTPException++from models.inference import LoadRequest, ValidateModelRequest++_BACKEND_ROOT = Path(__file__).resolve().parent.parent++EXPECTED = (+ "This model is quantized with compressed-tensors, which Unsloth cannot run yet. "+ "Installing compressed-tensors will not help. Try a GGUF or a bitsandbytes 4-bit "+ "build of this model instead."+)++CONFIG_IMPORT_ERROR = (+ "compressed_tensors is not installed and is required for compressed-tensors "+ "quantization. Please install it with `pip install compressed-tensors`."+)+CONFIG_IMPORT_ERROR_5_10 = (+ "compressed-tensors>=0.15.0 is required for compressed-tensors quantization. "+ "Please install it with `pip install compressed-tensors>=0.15.0`."+)+QUANTIZER_IMPORT_ERROR = (+ "Using `compressed_tensors` quantized models requires the compressed-tensors "+ "library: `pip install compressed-tensors`"+)+QUANTIZER_IMPORT_ERROR_5_10 = (+ "Using `compressed_tensors` quantized models requires compressed-tensors>=0.15.0: "+ "`pip install compressed-tensors>=0.15.0`"+)++REFUSALS = {+ "config-pre-5.10": CONFIG_IMPORT_ERROR,+ "config-5.10-plus": CONFIG_IMPORT_ERROR_5_10,+ "quantizer-pre-5.10": QUANTIZER_IMPORT_ERROR,+ "quantizer-5.10-plus": QUANTIZER_IMPORT_ERROR_5_10,+}+++def _load_route_module():+ spec = importlib.util.spec_from_file_location(+ "inference_route_compressed_tensors_error",+ _BACKEND_ROOT / "routes/inference.py",+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _load_failure(+ message,+ exception_type = ImportError,+ native = False,+) -> HTTPException:+ inference_route = _load_route_module()+ model_path = "unsloth/gemma-4-E2B-it-NVFP4"+ model_label = "gemma-4-E2B-it-NVFP4" if native else model_path+ request = LoadRequest(model_path = model_path)+ backend = MagicMock(active_model_name = None)+ with (+ patch.object(+ inference_route,+ "_resolve_model_identifier_for_request",+ return_value = (model_path, model_label, native),+ ),+ patch.object(+ inference_route, "resolve_effective_chat_template_override", return_value = None+ ),+ patch.object(inference_route, "get_inference_backend", return_value = backend),+ patch.object(inference_route, "get_llama_cpp_backend", return_value = MagicMock()),+ patch.object(+ inference_route.ModelConfig, "from_identifier", side_effect = exception_type(message)+ ),+ pytest.raises(HTTPException) as exc,+ ):+ asyncio.run(inference_route.load_model(request, MagicMock(), current_subject = "test-user"))+ return exc.value+++def _validation_failure(+ message,+ exception_type = ImportError,+ native = False,+) -> HTTPException:+ inference_route = _load_route_module()+ model_path = "unsloth/gemma-4-E2B-it-NVFP4"+ model_label = "gemma-4-E2B-it-NVFP4" if native else model_path+ request = ValidateModelRequest(model_path = model_path)+ with (+ patch.object(+ inference_route,+ "_resolve_model_identifier_for_request",+ return_value = (model_path, model_label, native),+ ),+ patch.object(+ inference_route.ModelConfig, "from_identifier", side_effect = exception_type(message)+ ),+ pytest.raises(HTTPException) as exc,+ ):+ asyncio.run(inference_route.validate_model(request, current_subject = "test-user"))+ return exc.value++[email protected]("message", REFUSALS.values(), ids = list(REFUSALS))[email protected]("native", [False, True])+def test_the_load_route_refuses_with_something_the_user_can_act_on(message, native):+ error = _load_failure(message, native = native)++ assert error.status_code == 500+ assert error.detail == EXPECTED+ assert "pip install" not in error.detail++[email protected]("message", REFUSALS.values(), ids = list(REFUSALS))[email protected]("native", [False, True])+def test_validate_refuses_the_same_way(message, native):+ error = _validation_failure(message, native = native)++ assert error.status_code == 400+ assert error.detail == EXPECTED+++def test_no_signature_carries_a_version_number():+ """The token that moved in 5.10 is the one a signature must not contain.++ A signature pinning the minimum version matches exactly half the supported range.+ """+ inference_route = _load_route_module()++ for signature in inference_route._MISSING_COMPRESSED_TENSORS_SIGNATURES:+ assert ">=" not in signature+ assert not any(character.isdigit() for character in signature)+ assert len(signature) > 30++[email protected]("exception_type", [ImportError, RuntimeError, Exception])+def test_the_exception_class_does_not_matter(exception_type):+ """transformers raises ImportError today; the route must not depend on that."""+ assert _load_failure(CONFIG_IMPORT_ERROR, exception_type = exception_type).detail == EXPECTED+++def _raise_with_compressed_tensors_absent(call):+ """Run ``call`` with transformers believing the library is not installed.++ Through ``is_compressed_tensors_available()``, so the real message is reached.+ """+ with pytest.raises(ImportError) as exc:+ call()+ return str(exc.value)+++def test_the_message_the_installed_transformers_really_raises_is_recognised():+ """The drift guard. Whatever wording is installed here has to be matched.++ Both sites, each through its own gate rather than a mock of the code under test.+ """+ transformers = pytest.importorskip("transformers")+ inference_route = _load_route_module()++ from transformers.quantizers import quantizer_compressed_tensors as quantizer_module+ from transformers.utils import quantization_config as config_module++ messages = {}+ with patch.object(config_module, "is_compressed_tensors_available", return_value = False):+ messages["config"] = _raise_with_compressed_tensors_absent(+ lambda: config_module.CompressedTensorsConfig()+ )+ with patch.object(quantizer_module, "is_compressed_tensors_available", return_value = False):+ quantizer = quantizer_module.CompressedTensorsHfQuantizer.__new__(+ quantizer_module.CompressedTensorsHfQuantizer+ )+ messages["quantizer"] = _raise_with_compressed_tensors_absent(+ lambda: quantizer.validate_environment()+ )++ for site, message in messages.items():+ assert "pip install" in message, (site, message)+ assert inference_route._unsupported_quantization_detail(message) == EXPECTED, (+ f"transformers {transformers.__version__} reworded the {site} refusal: {message!r}. "+ "Add the part that survived to _MISSING_COMPRESSED_TENSORS_SIGNATURES."+ )++[email protected]("message", REFUSALS.values(), ids = list(REFUSALS))+def test_the_message_survives_the_trip_out_of_the_inference_worker(message):+ """The route can only match what reaches it, and two layers may rewrite it.++ The load runs in a subprocess and its error passes through ``format_error_message``,+ which rewrites 404s, auth failures and OOM into its own sentences. Not this one.+ """+ from utils.utils import format_error_message+ assert format_error_message(ImportError(message), "unsloth/gemma-4-E2B-it-NVFP4") == message+++def _quantizer_refusal_messages():+ """Every "you are missing a dependency" message transformers' quantizers raise.++ Parsed from the installed package, so a family added upstream is covered the day it+ lands. Not executed: most sites need a config object and a device.+ """+ import ast+ import re+ from pathlib import Path as _Path++ import transformers.quantizers as quantizers_package++ package_dir = _Path(quantizers_package.__file__).parent+ out = []+ for source_file in sorted(package_dir.glob("quantizer_*.py")):+ tree = ast.parse(source_file.read_text(encoding = "utf-8"))+ for node in ast.walk(tree):+ if not isinstance(node, ast.Raise) or node.exc is None:+ continue+ for literal in ast.walk(node.exc):+ if not isinstance(literal, ast.Constant) or not isinstance(literal.value, str):+ continue+ text = literal.value+ if not re.search(r"requires|install", text, re.IGNORECASE):+ continue+ out.append((source_file.stem, text))+ return out+++def test_no_other_quantization_family_is_re_routed_into_this_refusal():+ """The families that resolved correctly before must still resolve the same way.++ A widened matcher is only safe if it stayed inside compressed-tensors, so every+ quantizer transformers ships is walked and only its own message may match.+ """+ pytest.importorskip("transformers")+ inference_route = _load_route_module()++ messages = _quantizer_refusal_messages()+ assert len({module for module, _ in messages}) >= 8, messages++ for module, message in messages:+ detail = inference_route._unsupported_quantization_detail(message)+ if module == "quantizer_compressed_tensors":+ continue+ assert (+ detail is None+ ), f"{module} message now matches the compressed-tensors refusal: {message!r}"+++def test_the_mlx_refusal_is_unchanged():+ """The signature this route already recognised keeps its own wording."""+ error = _load_failure(+ "Unsloth: 'unsloth/gemma-4-E2B-it-NVFP4' has per-module MLX quantization "+ "metadata {'config_groups': {'group_1': {'format': 'nvfp4-pack-quantized'}}}",+ exception_type = RuntimeError,+ )++ assert error.detail == (+ "We are working on supporting NVFP4 inference. For now it is not supported"+ )+++ "message",+ [+ "Network connection timed out",+ # Names the library without being about it missing. Must not be swallowed.+ "Failed to parse the config of this compressed-tensors export",+ ],+)+def test_an_unrelated_failure_keeps_its_own_message(message):+ error = _load_failure(message, exception_type = RuntimeError)++ assert error.detail == f"Failed to load model: {message}"+++def test_the_child_process_diagnostics_block_is_not_matched():+ """A llama-server tail is quoted from an untrusted process, so it is not ours to read."""+ inference_route = _load_route_module()+ msg = (+ "llama-server exited before becoming healthy\n\nllama-server output:\n"+ f"{CONFIG_IMPORT_ERROR}\n"+ )++ assert inference_route._unsupported_quantization_detail(msg) is None+++def test_the_refusal_names_no_quantization_scheme():+ """The errors it fires on carry no scheme, so the text may not claim one.++ Both wordings are raised before transformers looks at the compression config, so a+ W4A16, W8A8, INT8 or MXFP4 checkpoint lands here with the NVFP4 message exactly.+ """+ inference_route = _load_route_module()+ refusal = inference_route._COMPRESSED_TENSORS_INFERENCE_UNSUPPORTED_MESSAGE++ for scheme in ("NVFP4", "FP8", "W4A16", "W8A8", "INT8", "MXFP4"):+ assert scheme.lower() not in refusal.lower(), (+ f"the compressed-tensors refusal names {scheme}, but the errors it matches "+ f"do not identify the scheme"+ )+ assert "compressed-tensors" in refusal+ assert "GGUF" in refusal+ # Unqualified "4-bit" reads as a no-op to someone who already picked W4A16.+ assert "bitsandbytes 4-bit" in refusal++ assert "NVFP4" in inference_route._NVFP4_INFERENCE_UNSUPPORTED_MESSAGEdiff --git a/studio/backend/tests/test_compressed_tensors_signature_drift.py b/studio/backend/tests/test_compressed_tensors_signature_drift.pynew file mode 100644index 00000000000..f3b40d78f2a--- /dev/null+++ b/studio/backend/tests/test_compressed_tensors_signature_drift.py@@ -0,0 +1,164 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The compressed-tensors drift guard, with nothing but transformers installed.++`test_compressed_tensors_load_error_message.py` runs the same check, but only where the+whole backend imports, and `studio-backend-ci.yml` pins that environment to+`transformers>=4.51,<5.5` -- outside the 5.10 rewording the matcher exists to survive.+This file reads the signatures and `_diagnosis_text` out of `routes/inference.py` with+`ast` instead, so it needs no fastapi and runs on any transformers.+"""++from __future__ import annotations++import ast+import re+from pathlib import Path+from unittest.mock import patch++import pytest++_ROUTE = Path(__file__).resolve().parent.parent / "routes" / "inference.py"+++def _route_pieces():+ """`_MISSING_COMPRESSED_TENSORS_SIGNATURES` and `_diagnosis_text`, without importing.++ Lifting them out with `ast` is what lets this run beside a transformers the backend+ environment cannot hold.+ """+ tree = ast.parse(_ROUTE.read_text(encoding = "utf-8"))+ literals = {}+ diagnosis = None+ for node in tree.body:+ if isinstance(node, ast.Assign):+ for target in node.targets:+ if isinstance(target, ast.Name) and target.id in (+ "_MISSING_COMPRESSED_TENSORS_SIGNATURES",+ "_COMPRESSED_TENSORS_INFERENCE_UNSUPPORTED_MESSAGE",+ ):+ literals[target.id] = ast.literal_eval(node.value)+ elif isinstance(node, ast.FunctionDef) and node.name == "_diagnosis_text":+ node.decorator_list = []+ diagnosis = node+ assert "_MISSING_COMPRESSED_TENSORS_SIGNATURES" in literals, _ROUTE+ assert diagnosis is not None, "_diagnosis_text moved; this guard reads the wrong text"+ namespace = {"re": re}+ exec( # noqa: S102 - the source is this repository's own, read from disk above+ compile(ast.Module(body = [diagnosis], type_ignores = []), str(_ROUTE), "exec"),+ namespace,+ )+ return literals["_MISSING_COMPRESSED_TENSORS_SIGNATURES"], namespace["_diagnosis_text"]+++def _matches(message: str) -> bool:+ signatures, diagnosis_text = _route_pieces()+ lowered = diagnosis_text(message).lower()+ return any(signature in lowered for signature in signatures)+++def _refusals_from_installed_transformers():+ """The two refusals the INSTALLED transformers raises with the library absent.++ Through `is_compressed_tensors_available`, so these are the real strings, not a copy.+ """+ pytest.importorskip("transformers")+ from transformers.quantizers import quantizer_compressed_tensors as quantizer_module+ from transformers.utils import quantization_config as config_module++ messages = {}+ with patch.object(config_module, "is_compressed_tensors_available", return_value = False):+ with pytest.raises(ImportError) as raised:+ config_module.CompressedTensorsConfig()+ messages["config"] = str(raised.value)+ with patch.object(quantizer_module, "is_compressed_tensors_available", return_value = False):+ quantizer = quantizer_module.CompressedTensorsHfQuantizer.__new__(+ quantizer_module.CompressedTensorsHfQuantizer+ )+ with pytest.raises(ImportError) as raised:+ quantizer.validate_environment()+ messages["quantizer"] = str(raised.value)+ return messages+++def test_the_installed_transformers_refusals_are_both_recognised():+ """The drift guard proper, runnable against any transformers on any host."""+ import transformers+ for site, message in _refusals_from_installed_transformers().items():+ assert "pip install" in message, (site, message)+ assert _matches(message), (+ f"transformers {transformers.__version__} reworded the {site} "+ f"compressed-tensors refusal: {message!r}. Add the part that survived to "+ "_MISSING_COMPRESSED_TENSORS_SIGNATURES in studio/backend/routes/inference.py."+ )+++def test_no_signature_pins_a_version_number():+ """The version token is what moved in 5.10, so no signature may contain one."""+ signatures, _ = _route_pieces()+ assert signatures+ for signature in signatures:+ assert not re.search(r"\d+\.\d+", signature), signature+ # Long enough to be this refusal and not an arbitrary sentence saying the words.+ assert len(signature) >= 30, signature+++def test_no_other_quantizer_family_is_claimed():+ """Every other quantizer's "you are missing a library" message must not match.++ Parsed from the installed transformers, so a family added upstream is covered the day+ it lands. The module count is asserted: a parse finding too little would pass empty.+ """+ pytest.importorskip("transformers")+ import transformers.quantizers as quantizers_package++ package_dir = Path(quantizers_package.__file__).parent+ scanned, claimed = set(), []+ for source_file in sorted(package_dir.glob("quantizer_*.py")):+ tree = ast.parse(source_file.read_text(encoding = "utf-8"))+ for node in ast.walk(tree):+ if not isinstance(node, ast.Raise) or node.exc is None:+ continue+ for literal in ast.walk(node.exc):+ if not isinstance(literal, ast.Constant) or not isinstance(literal.value, str):+ continue+ if not re.search(r"requires|install", literal.value, re.IGNORECASE):+ continue+ scanned.add(source_file.stem)+ if source_file.stem != "quantizer_compressed_tensors" and _matches(literal.value):+ claimed.append((source_file.stem, literal.value))+ assert len(scanned) >= 8, sorted(scanned)+ assert not claimed, claimed+++def test_this_guard_needs_no_studio_backend_dependency():+ """The property that makes it runnable outside the backend's pinned environment.++ An import of fastapi or the route module puts it back inside the `<5.5` pin.+ """+ tree = ast.parse(Path(__file__).read_text(encoding = "utf-8"))+ imported = set()+ for node in ast.walk(tree):+ if isinstance(node, ast.Import):+ imported.update(alias.name.split(".")[0] for alias in node.names)+ elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:+ imported.add(node.module.split(".")[0])+ forbidden = {"fastapi", "models", "routes", "core", "utils", "starlette", "jwt"}+ assert not (imported & forbidden), sorted(imported & forbidden)+++def test_a_workflow_that_installs_a_modern_transformers_actually_collects_this_file():+ """The guard above is worth nothing in an environment that never runs it.++ Every auto-discovering job pins `transformers>=4.51,<5.5`; the jobs that install a+ modern one collect listed paths, not trees, so an unlisted file never runs.+ """+ workflow = (+ Path(__file__).resolve().parents[3] / ".github" / "workflows" / "consolidated-tests-ci.yml"+ )+ text = workflow.read_text(encoding = "utf-8")+ assert (+ "studio/backend/tests/test_compressed_tensors_signature_drift.py" in text+ ), "the modern-transformers matrix does not collect this file"+ assert 'transformers_spec: "transformers>=5,<6"' in textdiff --git a/studio/backend/tests/test_launch_reasoning_flag.py b/studio/backend/tests/test_launch_reasoning_flag.pynew file mode 100644index 00000000000..923b6befa70--- /dev/null+++ b/studio/backend/tests/test_launch_reasoning_flag.py@@ -0,0 +1,380 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The launch-time thinking default goes out as --reasoning where the build has it.++llama-server deprecates enable_thinking via --chat-template-kwargs (#7526), per KEY, so+only that key moves. The gate is the `llama-server --help` flag catalogue and is closed+on anything short of a positive answer: a build predating --reasoning exits with+"error: invalid argument: --reasoning" rather than starting without it. The help text+below is verbatim from two real binaries, b10909 and b6277.+"""++from __future__ import annotations++import json+import subprocess+import sys+import types+from pathlib import Path++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++import pytest # noqa: E402++from test_llama_cpp_placement import _backend, _launch # noqa: E402++import core.inference.llama_cpp as llama_cpp # noqa: E402+from core.inference.llama_cpp import ( # noqa: E402+ LlamaCppBackend,+ _build_launch_reasoning_args,+)+from core.inference.llama_server_args import strip_shadowing_flags # noqa: E402++# b10909. The name-alikes are the point: a substring match would report the flag wrongly.+MODERN_HELP = """usage: llama-server [options]++-m, --model FNAME model path+--reasoning-format FORMAT controls whether thought tags are allowed and/or extracted from the+ response, and in which format they're returned; one of:+ - none: leaves thoughts unparsed in `message.content`+-rea, --reasoning [on|off|auto] Use reasoning/thinking in the chat ('on', 'off', or 'auto', default:+ 'auto' (detect from template))+ (env: LLAMA_ARG_REASONING)+--reasoning-effort LEVEL reasoning effort level given to the chat template: 'default' to keep+ the template default,+--reasoning-budget N token budget for thinking: -1 for unrestricted, 0 for immediate end,+ N>0 for token budget (default: -1)+ (env: LLAMA_ARG_THINK_BUDGET)+--reasoning-preserve, --no-reasoning-preserve+ preserve reasoning trace in the full history, not just the last+ assistant message (default: enabled)+--chat-template-kwargs STRING sets additional params for the json template parser, must be a valid+ JSON object+--jinja, --no-jinja whether to use jinja template engine for chat (default: disabled)+"""++OLD_HELP = """usage: llama-server [options]++-m, --model FNAME model path+--reasoning-format FORMAT controls whether thought tags are allowed and/or extracted from the+ response, and in which format they're returned; one of:+ - none: leaves thoughts unparsed in `message.content`+--reasoning-budget N controls the amount of thinking allowed; currently only one of: -1 for+ unrestricted thinking budget, or 0 to disable thinking (default: -1)+ (env: LLAMA_ARG_THINK_BUDGET)+--chat-template-kwargs STRING sets additional params for the json template parser+--jinja, --no-jinja whether to use jinja template engine for chat (default: disabled)+"""++REMOVAL_STUB_HELP = """usage: llama-server [options]++-m, --model FNAME model path+-rea, --reasoning [on|off|auto] (argument has been removed)+--chat-template-kwargs STRING sets additional params for the json template parser+"""++MODERN_CAPS = {"supports_reasoning_flag": True}+OLD_CAPS = {"supports_reasoning_flag": False}+++def _probe(+ tmp_path,+ monkeypatch,+ help_text,+ returncode = 0,+ stream = "stdout",+):+ """The real probe against a stubbed ``llama-server --help``."""+ binary = tmp_path / "llama-server"+ binary.write_text("")+ completed = subprocess.CompletedProcess(+ args = [str(binary), "--help"],+ returncode = returncode,+ stdout = help_text if stream == "stdout" else "",+ stderr = help_text if stream == "stderr" else "",+ )+ monkeypatch.setattr(llama_cpp.subprocess, "run", lambda *a, **k: completed)+ LlamaCppBackend._capability_cache.clear()+ return LlamaCppBackend.probe_server_capabilities(str(binary))+++class TestTheProbeReadsTheFlagAndNotItsNamesakes:+ def test_the_bundled_prebuilt_advertises_the_flag(self, tmp_path, monkeypatch):+ assert _probe(tmp_path, monkeypatch, MODERN_HELP)["supports_reasoning_flag"] is True++ def test_a_build_with_only_the_namesakes_does_not(self, tmp_path, monkeypatch):+ """--reasoning-format / -budget / -preserve all contain the string."""+ assert _probe(tmp_path, monkeypatch, OLD_HELP)["supports_reasoning_flag"] is False++ @pytest.mark.parametrize(+ "kwargs",+ [+ {"help_text": "", "returncode": 0},+ {"help_text": "", "returncode": 1},+ {"help_text": "usage: llama-server [options]\n"},+ {"help_text": REMOVAL_STUB_HELP},+ ],+ ids = ["empty-help", "failed-probe", "usage-only", "removal-stub"],+ )+ def test_the_gate_is_closed_on_anything_but_a_positive_answer(+ self, tmp_path, monkeypatch, kwargs+ ):+ """Unlike --jinja, this gate must fail CLOSED: the flag aborts an old build."""+ assert _probe(tmp_path, monkeypatch, **kwargs)["supports_reasoning_flag"] is False++ def test_a_parsed_catalogue_counts_even_on_a_nonzero_exit(self, tmp_path, monkeypatch):+ """Same policy as the sibling flags: a nonzero exit after a full help still named+ the flags, and the doubt is carried separately in *_probe_inconclusive.+ """+ caps = _probe(tmp_path, monkeypatch, MODERN_HELP, returncode = 1)+ assert caps["supports_reasoning_flag"] == caps["supports_reasoning_budget"] is True+ assert caps["reasoning_budget_probe_inconclusive"] is True++ def test_a_missing_binary_reports_it_unsupported(self, tmp_path):+ caps = LlamaCppBackend.probe_server_capabilities(str(tmp_path / "nope"))+ assert caps["supports_reasoning_flag"] is False++ def test_the_flag_is_read_from_stderr_too(self, tmp_path, monkeypatch):+ caps = _probe(tmp_path, monkeypatch, MODERN_HELP, stream = "stderr")+ assert caps["supports_reasoning_flag"] is True+++class TestOnlyTheDeprecatedKeyMoves:+ """The kwargs channel keeps carrying every key that has no flag of its own."""++ def test_thinking_on_becomes_the_flag_and_nothing_else_is_emitted(self):+ assert _build_launch_reasoning_args(MODERN_CAPS, {"enable_thinking": True}) == [+ "--reasoning",+ "on",+ ]++ def test_thinking_off_becomes_the_flag(self):+ assert _build_launch_reasoning_args(MODERN_CAPS, {"enable_thinking": False}) == [+ "--reasoning",+ "off",+ ]++ @pytest.mark.parametrize("preserve", [True, False])+ def test_preserve_thinking_stays_in_the_kwargs(self, preserve):+ """It is a template variable with no flag: #9096's default has to survive."""+ args = _build_launch_reasoning_args(+ MODERN_CAPS, {"enable_thinking": True, "preserve_thinking": preserve}+ )+ assert args[:2] == ["--reasoning", "on"]+ assert args[2] == "--chat-template-kwargs"+ assert json.loads(args[3]) == {"preserve_thinking": preserve}++ def test_a_reasoning_effort_ladder_is_untouched(self):+ """gpt-oss style never sets enable_thinking, so nothing is deprecated there."""+ kwargs = {"reasoning_effort": "high"}+ assert _build_launch_reasoning_args(MODERN_CAPS, kwargs) == [+ "--chat-template-kwargs",+ json.dumps(kwargs),+ ]++ def test_an_unrelated_future_key_is_untouched(self):+ kwargs = {"enable_thinking": False, "some_new_template_variable": "x"}+ args = _build_launch_reasoning_args(MODERN_CAPS, kwargs)+ assert args[:2] == ["--reasoning", "off"]+ assert json.loads(args[3]) == {"some_new_template_variable": "x"}++ def test_the_caller_s_dict_is_not_mutated(self):+ kwargs = {"enable_thinking": True}+ _build_launch_reasoning_args(MODERN_CAPS, kwargs)+ assert kwargs == {"enable_thinking": True}++ @pytest.mark.parametrize(+ "caps",+ [OLD_CAPS, {}, {"supports_reasoning_flag": None}],+ ids = ["old-build", "nothing-known", "unset"],+ )+ def test_without_the_flag_the_argv_is_what_main_emits(self, caps):+ kwargs = {"enable_thinking": True, "preserve_thinking": False}+ assert _build_launch_reasoning_args(caps, kwargs) == [+ "--chat-template-kwargs",+ json.dumps(kwargs),+ ]++ @pytest.mark.parametrize(+ "caps", [MODERN_CAPS, OLD_CAPS, {}], ids = ["modern", "old-build", "nothing-known"]+ )+ def test_an_empty_dict_still_goes_out_as_main_sends_it(self, caps):+ """Nothing moved, so the argument main appends has to be appended.++ Unreachable from the launcher today; held so the untouched path is identical+ to main by construction rather than by luck.+ """+ assert _build_launch_reasoning_args(caps, {}) == ["--chat-template-kwargs", "{}"]++ def test_an_empty_remainder_after_the_flag_appends_nothing(self):+ """The other side of it: that argument did not exist on main either."""+ assert _build_launch_reasoning_args(MODERN_CAPS, {"enable_thinking": True}) == [+ "--reasoning",+ "on",+ ]+++# enable_thinking style, with preserve_thinking so both channels are exercised at once.+THINKING_TEMPLATE = (+ "{% if enable_thinking %}<think>\n{% endif %}"+ "{% if preserve_thinking %}{{ messages[0].reasoning_content }}{% endif %}"+ "{{ messages[0].content }}"+)+# gpt-oss style: a ladder, no on/off gate, nothing deprecated.+EFFORT_TEMPLATE = "{{ reasoning_effort }}{% if reasoning_effort == 'high' %}<think>{% endif %}"++OSES = {+ "linux": ("linux", False),+ "windows": ("win32", False),+ "wsl": ("linux", True),+ "macos": ("darwin", False),+}+VENDORS = ("nvidia", "amd", "cpu")+++def _fake_torch(*, rocm: bool, device_count: int) -> types.ModuleType:+ torch = types.ModuleType("torch")+ torch.__version__ = "2.11.0+rocm6.2" if rocm else "2.11.0+cu130"+ version = types.ModuleType("torch.version")+ version.hip = "6.2.0" if rocm else None+ version.cuda = None if rocm else "13.0"+ torch.version = version+ torch.cuda = types.SimpleNamespace(+ is_available = lambda: device_count > 0,+ device_count = lambda: device_count,+ )+ return torch+++def _argv_for(tmp_path, monkeypatch, *, os_label, vendor, modern, template):+ """The argv the real ``load_model`` builds on one host, for one vintage."""+ platform, is_wsl = OSES[os_label]+ monkeypatch.setattr(llama_cpp.sys, "platform", platform, raising = False)+ monkeypatch.setattr(llama_cpp, "_is_wsl", lambda: is_wsl, raising = False)+ monkeypatch.setitem(+ sys.modules,+ "torch",+ _fake_torch(rocm = vendor == "amd", device_count = 0 if vendor == "cpu" else 1),+ )+ memory = [] if vendor == "cpu" else [(0, 24_000, 24_000)]+ backend, gguf = _backend(tmp_path, vulkan = False, memory = memory)+ monkeypatch.setattr(+ LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(lambda cls, binary = None: dict(MODERN_CAPS if modern else OLD_CAPS)),+ )+ captured = _launch(+ backend,+ gguf,+ n_ctx = 4096,+ chat_template_override = template,+ )+ return captured["cmd"]+++def _reasoning_slice(cmd: list[str]) -> list[str]:+ """Just the reasoning arguments, in order, out of the whole command line."""+ out: list[str] = []+ i = 0+ while i < len(cmd):+ if cmd[i] in {"--reasoning", "--chat-template-kwargs"}:+ out.extend(cmd[i : i + 2])+ i += 2+ continue+ i += 1+ return out+++_HOSTS = [(o, v) for o in OSES for v in VENDORS]+++class TestTheRealLaunchOnEveryHost:+ """The reasoning arguments are invariant under OS and device.++ Drives the real ``load_model``, since placement appends into the same command.+ """++ @pytest.mark.parametrize("os_label,vendor", _HOSTS)+ def test_a_modern_build_gets_the_flag_and_keeps_the_preserve_kwarg(+ self, tmp_path, monkeypatch, os_label, vendor+ ):+ cmd = _argv_for(+ tmp_path,+ monkeypatch,+ os_label = os_label,+ vendor = vendor,+ modern = True,+ template = THINKING_TEMPLATE,+ )+ assert _reasoning_slice(cmd) == [+ "--reasoning",+ "on",+ "--chat-template-kwargs",+ '{"preserve_thinking": false}',+ ]+ assert cmd.count("--reasoning") == 1++ @pytest.mark.parametrize("os_label,vendor", _HOSTS)+ def test_an_old_build_never_sees_the_flag(self, tmp_path, monkeypatch, os_label, vendor):+ cmd = _argv_for(+ tmp_path,+ monkeypatch,+ os_label = os_label,+ vendor = vendor,+ modern = False,+ template = THINKING_TEMPLATE,+ )+ assert "--reasoning" not in cmd+ assert _reasoning_slice(cmd) == [+ "--chat-template-kwargs",+ '{"enable_thinking": true, "preserve_thinking": false}',+ ]++ @pytest.mark.parametrize("modern", [True, False], ids = ["modern", "old"])+ def test_an_effort_ladder_model_is_unchanged_on_both_vintages(+ self, tmp_path, monkeypatch, modern+ ):+ cmd = _argv_for(+ tmp_path,+ monkeypatch,+ os_label = "linux",+ vendor = "nvidia",+ modern = modern,+ template = EFFORT_TEMPLATE,+ )+ assert "--reasoning" not in cmd+ assert _reasoning_slice(cmd)[0] == "--chat-template-kwargs"+ assert "reasoning_effort" in _reasoning_slice(cmd)[1]+++class TestInheritedExtrasTreatBothSpellingsAlike:+ """Applying a chat template override recomputes the reasoning default.++ An inherited copy is appended after ours and would last-wins-override it, so both+ spellings have to be stripped.+ """++ @pytest.mark.parametrize(+ "args",+ [+ ["--reasoning", "off"],+ ["--reasoning=off"],+ ["-rea", "off"],+ ["--chat-template-kwargs", '{"enable_thinking": false}'],+ ],+ )+ def test_a_template_override_strips_the_inherited_default(self, args):+ assert strip_shadowing_flags([*args, "--threads", "8"], strip_template = True) == [+ "--threads",+ "8",+ ]++ @pytest.mark.parametrize(+ "args",+ [["--reasoning", "off"], ["--chat-template-kwargs", '{"enable_thinking": false}']],+ )+ def test_without_a_template_override_both_survive(self, args):+ assert strip_shadowing_flags([*args], strip_template = False) == args