Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let Laya fine-tunes answer from their GGUF export (#16511)

Nilay8bf2d3c54847
diff --git a/studio/backend/core/systemone/catalog.py b/studio/backend/core/systemone/catalog.py
index b686c9d660e..86bfedcec2c 100644
--- a/studio/backend/core/systemone/catalog.py
+++ b/studio/backend/core/systemone/catalog.py
@@ -24,7 +24,7 @@ class Checkpoint:
# "laya" (rl_agent_config.json + encoder), "clef" (Qwen3.5 backbone + joint schema head), or
# "gguf" (a GGUF_COMPANIONS entry llama.cpp serves, with no PyTorch form).
layout: str = "laya"
- # "pytorch", "llama.cpp" for the GGUF a Clef entry is served from (see laya_runtime._native_target), or
+ # "pytorch", "llama.cpp" for the GGUF a Clef or Laya entry is served from (see laya_runtime._native_target), or
# "mlx" for the form unsloth-zoo's MLX engine serves on Apple Silicon (see laya_runtime._mlx_target).
backend: str = "pytorch"
# The served GGUF files of a local export, so a re-export is a different checkpoint to a resident server.
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index cf204a58edd..14b331f1d62 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import dataclasses
+import functools
import gc
import importlib.util
import logging
@@ -59,6 +60,10 @@ TRAINING_POLL_S = 2.0
IDLE_UNLOAD_S = 300.0
# Binaries (path, mtime) whose server answered without a "decisions" output.
_incapable: set[tuple[str, int]] = set()
+# (binary key, llama.cpp form of a Laya export) the build refused to load.
+_unloadable: set[tuple[tuple[str, int], Checkpoint]] = set()
+_LAYA_OPTION_TOKENS = 48
+_LOAD_REFUSALS = ("unknown model architecture", "error loading model vocabulary")
class Unavailable(Exception):
@@ -314,16 +319,21 @@ def _mlx_choice(
return target
+def _local_folder(checkpoint: Checkpoint) -> Path:
+ root = Path(checkpoint.source).expanduser()
+ return root / checkpoint.subfolder if checkpoint.subfolder else root
+
+
def _native_target(checkpoint: Checkpoint) -> Checkpoint | None:
- """The llama.cpp form of an entry: ggml-org's GGUF for a stock Clef, a current export for a folder, a GGUF entry itself."""
+ """The llama.cpp form of an entry: ggml-org's GGUF for a stock Clef, a current export for a Clef or Laya folder, a GGUF entry itself."""
if checkpoint.layout == GGUF:
return checkpoint
- if checkpoint.layout != "clef" or _is_native(checkpoint):
+ if _is_native(checkpoint):
return None
if checkpoint.is_local:
from .gguf_export_contract import served_files
- files = served_files(Path(checkpoint.source).expanduser(), "clef")
+ files = served_files(_local_folder(checkpoint), checkpoint.layout)
if files is None:
return None
return dataclasses.replace(
@@ -357,7 +367,7 @@ def _native_files(checkpoint: Checkpoint, local_only: bool) -> tuple[Path, Path
if checkpoint.is_local:
from .gguf_export_contract import served_files
- files = served_files(Path(checkpoint.source).expanduser(), "clef")
+ files = served_files(_local_folder(checkpoint), checkpoint.layout)
if files is None:
raise FileNotFoundError(f"No current GGUF export in {checkpoint.source}")
return files[1], files[2]
@@ -427,6 +437,8 @@ def _native_unavailable(checkpoint: Checkpoint, native: Checkpoint | None) -> st
return "llama-server is not installed."
if _binary_key(binary) in _incapable:
return "This llama.cpp build cannot serve decision models; update llama.cpp in Unsloth."
+ if (_binary_key(binary), native) in _unloadable:
+ return "This llama.cpp build cannot load this GGUF export."
return None
@@ -443,7 +455,7 @@ def select(
Auto takes llama.cpp when the entry has a GGUF and Studio's llama-server serves decisions, else PyTorch;
on Apple Silicon it answers through unsloth-zoo's MLX engine first.
Images are read by llama.cpp, and by MLX for a Clef. Audio is heard only by MLX, for a Clef with an audio tower.
- Laya is always served by PyTorch, a GGUF entry otherwise by llama.cpp.
+ Laya is served by PyTorch unless llama.cpp is chosen for a folder with a current export, a GGUF entry otherwise by llama.cpp.
"""
from utils.systemone_settings import get_backend
@@ -483,13 +495,14 @@ def select(
return mlx, None
if checkpoint.layout == GGUF:
return _select_gguf(checkpoint, questions, preference or get_backend()), None
+ preference = preference or get_backend()
if checkpoint.layout != "clef":
if images:
raise Unavailable(
400, "api_usage_error", "Images need a Clef model; Laya reads text only."
)
- return checkpoint, None
- preference = preference or get_backend()
+ if preference != LLAMA_CPP or _native_target(checkpoint) is None:
+ return checkpoint, None
if preference == "pytorch":
if images:
raise Unavailable(
@@ -642,9 +655,13 @@ def mlx_ready(checkpoint) -> bool:
)
+def has_native_form(checkpoint) -> bool:
+ return isinstance(checkpoint, Checkpoint) and _native_target(checkpoint) is not None
+
+
def native_ready(checkpoint) -> bool:
- """Whether llama.cpp could serve this Clef or GGUF entry here under the runtime setting."""
- if not isinstance(checkpoint, Checkpoint) or checkpoint.layout == "laya":
+ """Whether llama.cpp could serve this entry here under the runtime setting."""
+ if not has_native_form(checkpoint):
return False
from utils.systemone_settings import get_backend
@@ -670,7 +687,7 @@ def input_modalities(checkpoint) -> list[str]:
def _native_reads_images(native: Checkpoint) -> bool:
if native.is_local:
from .gguf_export_contract import served_files
- files = served_files(Path(native.source).expanduser(), "clef")
+ files = served_files(_local_folder(native), native.layout)
return files is not None and files[2] is not None
return bool(GGUF_COMPANIONS[native.name].mmproj)
@@ -1166,12 +1183,124 @@ def _build_laya(
return agent, str(agent.device.type)
+def _is_laya_export(checkpoint: Checkpoint) -> bool:
+ return checkpoint.layout == "laya" and _is_native(checkpoint)
+
+
+def _stock_laya(checkpoint) -> bool:
+ return checkpoint.layout == "laya" and not checkpoint.is_local
+
+
+def _laya_config(checkpoint: Checkpoint) -> dict[str, Any]:
+ import json
+ config = _local_folder(checkpoint) / "rl_agent_config.json"
+ return json.loads(config.read_text(encoding = "utf-8"))
+
+
[email protected]_cache(maxsize = 4)
+def _export_counter(checkpoint: Checkpoint):
+ """(token count, head_max_len) for an exported Laya folder, read without torch."""
+ import json
+
+ from tokenizers import Tokenizer
+
+ folder = _local_folder(checkpoint) / "tokenizer"
+ mask = json.loads((folder / "tokenizer_config.json").read_text(encoding = "utf-8"))["mask_token"]
+ tokenizer = Tokenizer.from_file(str(folder / "tokenizer.json"))
+
+ def count(text: str) -> int:
+ return len(tokenizer.encode(text.replace(mask, " "), add_special_tokens = False).ids)
+
+ return count, int(_laya_config(checkpoint).get("head_max_len", 192))
+
+
+def _export_criterion(value) -> str:
+ import json
+ if isinstance(value, str):
+ return value
+ return json.dumps(value, ensure_ascii = False, separators = (", ", ": "), default = str)
+
+
+def _export_options(question: dict[str, Any]) -> list[str]:
+ kind, criteria = question["type"], question.get("criteria")
+ if kind == "choice":
+ return [
+ key if value in (None, "") else f"{key}: {_export_criterion(value)}"
+ for key, value in criteria.items()
+ ]
+ if kind == "score":
+ return [f"level {i}: {_export_criterion(c)}" for i, c in enumerate(criteria)]
+ criteria = criteria or {}
+ no, yes = criteria.get("false"), criteria.get("true")
+ return [
+ f"false: {_export_criterion(no)}"
+ if no not in (None, "")
+ else "false: no, the statement does not hold",
+ f"true: {_export_criterion(yes)}"
+ if yes not in (None, "")
+ else "true: yes, the statement holds",
+ ]
+
+
+def _cut_problem(sizes: list[int], head: int, head_max_len: int) -> str | None:
+ """What build_sequence would cut: past the option budget each option keeps its share, instructions what is left."""
+ budget = head_max_len - 16
+ if sum(sizes) > budget and max(sizes) > max(4, budget // len(sizes)):
+ return f"options would be cut short to fit Laya's {budget}-token option budget. Use fewer or shorter criteria"
+ if head > max(8, head_max_len - sum(sizes)):
+ return (
+ f"instructions would be cut short to fit beside its options in Laya's {head_max_len}-token "
+ "question budget. Use shorter instructions or fewer criteria"
+ )
+ return None
+
+
[email protected]_cache(maxsize = 256)
+def _export_cut(checkpoint: Checkpoint, name: str, key: str) -> str | None:
+ import json
+
+ question = json.loads(key)
+ count, head_max_len = _export_counter(checkpoint)
+ sizes = [
+ 1 + min(_LAYA_OPTION_TOKENS, count(f" {option}")) for option in _export_options(question)
+ ]
+ # As native_worker._wire_questions sends it and laya's _to_internal reads it.
+ instructions = question.get("instructions")
+ if instructions in (None, ""):
+ instructions = name
+ elif not isinstance(instructions, str):
+ instructions = json.dumps(instructions)
+ return _cut_problem(sizes, count(f"{question['type']} question: {instructions}"), head_max_len)
+
+
+def _refuse_cut_questions(checkpoint: Checkpoint, questions: dict[str, dict[str, Any]]) -> None:
+ """llama.cpp cuts what does not fit Laya's question budget without saying so; refused here as PyTorch refuses it."""
+ import json
+
+ try:
+ _export_counter(checkpoint)
+ except Exception as exc:
+ raise Unavailable(
+ 503, "model_unavailable", f"Could not read the tokenizer of {checkpoint.name}: {exc}"
+ ) from None
+ for name, question in questions.items():
+ key = json.dumps(question, ensure_ascii = False)
+ if (problem := _export_cut(checkpoint, str(name), key)) is not None:
+ raise Unavailable(422, "invalid_request_error", f"Question {name!r} {problem}.")
+
+
def _load_native(checkpoint: Checkpoint):
global _gpu_load
from core.inference import gpu_arbiter
from utils.systemone_settings import get_native_ctx
- from .native_worker import NativeClefAgent, NativeError, NativeIncapable, resolve_binary
+ from .native_worker import (
+ NativeClefAgent,
+ NativeError,
+ NativeIncapable,
+ NativeLoadError,
+ resolve_binary,
+ )
# Fetched (pinned revision) before the eviction, so a failed download leaves the resident model serving.
model, mmproj = _native_files(checkpoint, local_only = False)
@@ -1191,7 +1320,9 @@ def _load_native(checkpoint: Checkpoint):
checkpoint.name,
gpu = gpu,
clef_answers = checkpoint.layout == "clef",
- ctx = get_native_ctx(),
+ ctx = min(get_native_ctx(), int(_laya_config(checkpoint).get("max_len", 512)))
+ if checkpoint.layout == "laya"
+ else get_native_ctx(),
cancelled = lambda: (gpu and _training_active()) or _unloaded(),
)
if gpu and _training_active():
@@ -1200,8 +1331,15 @@ def _load_native(checkpoint: Checkpoint):
f"{checkpoint.name} needs the GPU, which a training run took while it loaded."
)
except BaseException as exc:
- if isinstance(exc, NativeIncapable) and (binary := resolve_binary()):
- _incapable.add(_binary_key(binary))
+ if isinstance(exc, NativeError) and (binary := resolve_binary()):
+ if isinstance(exc, NativeIncapable):
+ _incapable.add(_binary_key(binary))
+ elif (
+ isinstance(exc, NativeLoadError)
+ and _is_laya_export(checkpoint)
+ and any(refusal in str(exc) for refusal in _LOAD_REFUSALS)
+ ):
+ _unloadable.add((_binary_key(binary), checkpoint))
if gpu:
gpu_arbiter.release(gpu_arbiter.DECISIONS)
raise
@@ -1570,7 +1708,7 @@ def _misplaced() -> bool:
def _clef_blocked_by_training(checkpoint: Checkpoint) -> None:
- if checkpoint.layout == "laya":
+ if checkpoint.layout == "laya" and not _is_laya_export(checkpoint):
return
from .catalog import clef_unsupported_reason
@@ -1582,7 +1720,11 @@ def _clef_blocked_by_training(checkpoint: Checkpoint) -> None:
return
# Clef has no CPU fallback: it waits for the GPU instead of taking it from the run.
# Only a resident on the GPU: a llama.cpp server on CPU keeps serving during the run.
- if _loaded is not None and _loaded.layout != "laya" and _device_name not in (None, "cpu"):
+ if (
+ _loaded is not None
+ and (_loaded.layout != "laya" or _is_laya_export(_loaded))
+ and _device_name not in (None, "cpu")
+ ):
_evict(external = True)
raise Unavailable(
503,
@@ -1704,23 +1846,14 @@ def _head(agent, question: dict[str, Any], max_len: int, head_max_len: int):
internal = agent._to_internal(question)
tok = agent.tok
options = [" " + o.replace(tok.mask_token, " ") for o in render_options(internal)]
- sizes = [1 + min(48, len(tok(o, add_special_tokens = False)["input_ids"])) for o in options]
- # build_sequence's rule: past this budget, each option keeps only its share of it.
- budget = head_max_len - 16
- share = max(4, budget // len(sizes))
- if sum(sizes) > budget and max(sizes) > share:
- raise ValueError(
- f"options would be cut short to fit Laya's {budget}-token option budget. "
- "Use fewer or shorter criteria"
- )
+ sizes = [
+ 1 + min(_LAYA_OPTION_TOKENS, len(tok(o, add_special_tokens = False)["input_ids"]))
+ for o in options
+ ]
ins = str(internal["ins"]).replace(tok.mask_token, " ")
head = tok(f"{internal['t']} question: {ins}", add_special_tokens = False)["input_ids"]
- if len(head) > max(8, head_max_len - sum(sizes)):
- raise ValueError(
- f"instructions would be cut short to fit beside its options in Laya's "
- f"{head_max_len}-token question budget. Use shorter instructions or fewer "
- "criteria"
- )
+ if (problem := _cut_problem(sizes, len(head), head_max_len)) is not None:
+ raise ValueError(problem)
ids, markers = build_sequence(tok, "", internal, max_len, head_max_len)
if len(markers) != len(sizes):
raise ValueError(
@@ -2281,9 +2414,11 @@ def _route(
hardware.get_device()
state_images = _state_has_images(state)
target, reason = select(checkpoint, images, questions, state_images = state_images, audio = audio)
- if checkpoint.layout != "laya":
+ if not _stock_laya(checkpoint):
_fallback_reason = reason
try:
+ if _is_laya_export(target):
+ _refuse_cut_questions(target, questions)
return _decide(target, state, questions, images, audio)
except NativeContextOverflow as exc:
# Auto answers on PyTorch only what its longer window can hold, and only from a download already made.
diff --git a/studio/backend/core/systemone/native_worker.py b/studio/backend/core/systemone/native_worker.py
index 3a68fac348c..efea4be23a9 100644
--- a/studio/backend/core/systemone/native_worker.py
+++ b/studio/backend/core/systemone/native_worker.py
@@ -57,6 +57,10 @@ class NativeIncapable(NativeError):
"""This llama-server build cannot serve decision models."""
+class NativeLoadError(NativeError):
+ """llama-server exited before it finished loading the model."""
+
+
class NativeInputError(NativeError):
"""The server refused this request; the server itself is fine."""
@@ -457,7 +461,7 @@ class NativeClefAgent:
raise NativeError("The llama.cpp decision server was stopped while it loaded.")
if not self.is_alive():
tail = self._log_tail()
- raise NativeError(
+ raise NativeLoadError(
f"llama.cpp exited while loading {self.alias} (code {self._process.returncode})"
+ (f":\n{tail}" if tail else ".")
)
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 0b95c35bf49..95dd115242f 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -748,6 +748,7 @@ class SystemOneSettingsResponse(BaseModel):
input_modalities: list[str] = ["text"]
# "laya", "clef" or "gguf" for the configured model, so env-configured local checkpoints get runtime controls.
layout: Optional[str] = None
+ has_gguf_form: bool = False
class SystemOneSettingsPayload(BaseModel):
@@ -1809,6 +1810,7 @@ def _systemone_response(request: Request) -> SystemOneSettingsResponse:
fallback_reason = fallback if effective == "pytorch" or effective is None else None,
input_modalities = laya_runtime.input_modalities(configured),
layout = getattr(configured, "layout", None),
+ has_gguf_form = laya_runtime.has_native_form(configured),
)
diff --git a/studio/backend/tests/test_decision_native.py b/studio/backend/tests/test_decision_native.py
index e5ab610f285..9f8bbbe22c4 100644
--- a/studio/backend/tests/test_decision_native.py
+++ b/studio/backend/tests/test_decision_native.py
@@ -62,6 +62,12 @@ log("start", argv=args, env={k: os.environ.get(k) for k in ("CUDA_VISIBLE_DEVICE
if mode == "exit":
print("stub: failed to load model", flush=True)
sys.exit(3)
+if mode == "unknown_arch":
+ print("error loading model architecture: unknown model architecture: 'laya'", flush=True)
+ sys.exit(1)
+if mode == "unknown_vocab":
+ print("error loading model vocabulary: unknown tokenizer: 'mmbert'", flush=True)
+ sys.exit(1)
key = opt("--api-key") or open(opt("--api-key-file"), encoding="utf-8").read().strip()
port, alias, ub, mmproj = int(opt("--port")), opt("--alias"), int(opt("-ub")), "--mmproj" in args
@@ -214,6 +220,7 @@ def home(tmp_path, monkeypatch):
):
monkeypatch.setattr(laya_runtime, name, None)
monkeypatch.setattr(laya_runtime, "_incapable", set())
+ monkeypatch.setattr(laya_runtime, "_unloadable", set(), raising = False)
monkeypatch.setattr(gpu_arbiter, "_owner", None)
monkeypatch.setattr(systemone_settings, "runtime_unavailable_reason", lambda: None)
monkeypatch.setattr(systemone_settings, "gpu_available", lambda: False)
@@ -329,6 +336,7 @@ def _export(
quants = ("Q8_0",),
mmproj = True,
fingerprint = None,
+ layout = "clef",
) -> None:
directory = folder / contract.EXPORT_DIR
directory.mkdir(exist_ok = True)
@@ -346,10 +354,10 @@ def _export(
{
"format": contract.FORMAT,
"version": 1,
- "layout": "clef",
+ "layout": layout,
"quantizations": list(quants),
"files": files,
- "source_fingerprint": fingerprint or contract.fingerprint(folder, "clef"),
+ "source_fingerprint": fingerprint or contract.fingerprint(folder, layout),
}
),
encoding = "utf-8",
@@ -1354,12 +1362,227 @@ def test_an_image_that_decodes_to_too_many_pixels_is_refused(home, client, stub)
assert stub.records("start") == []
+def _laya_fine_tune(
+ root: Path,
+ name = "laya_run",
+ exported = True,
+) -> Path:
+ # Every Studio install ships tokenizers; the export path reads the tokenizer with it.
+ tokenizers = pytest.importorskip("tokenizers")
+ folder = root / name
+ (folder / "encoder").mkdir(parents = True)
+ (folder / "tokenizer").mkdir()
+ tokenizer = tokenizers.Tokenizer(tokenizers.models.WordLevel({"[UNK]": 0}, unk_token = "[UNK]"))
+ tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()
+ tokenizer.save(str(folder / "tokenizer" / "tokenizer.json"))
+ (folder / "tokenizer" / "tokenizer_config.json").write_text(
+ '{"mask_token": "[MASK]"}', encoding = "utf-8"
+ )
+ (folder / "rl_agent_config.json").write_text(
+ '{"max_len": 1024, "head_max_len": 256}', encoding = "utf-8"
+ )
+ (folder / "model.safetensors").write_bytes(b"weights")
+ if exported:
+ _export(folder, mmproj = False, layout = "laya")
+ return folder
+
+
+def _torch_laya(monkeypatch) -> list:
+ loaded = []
+
+ def load(checkpoint):
+ if laya_runtime._is_native(checkpoint):
+ return _REAL_LOAD(checkpoint)
+ loaded.append(checkpoint)
+ return SimpleNamespace(), "cpu"
+
+ monkeypatch.setattr(laya_runtime, "_load_checkpoint", load)
+ monkeypatch.setattr(
+ laya_runtime,
+ "_predict",
+ lambda agent, state, questions: (
+ {
+ "answers": {n: {"type": "noul", "noul": 0.5, "confidence": 0.5} for n in questions},
+ "usage": {"input_tokens": 1, "output_tokens": 0},
+ },
+ False,
+ ),
+ )
+ return loaded
+
+
+NOUL = {"q": {"type": "noul", "instructions": "x"}}
+
+
+def test_an_exported_laya_fine_tune_answers_through_llama_cpp(home, client, stub):
+ folder = _laya_fine_tune(home.outputs)
+ settings = _put(client, enabled = True, model = "laya-ft:laya_run", backend = "llama.cpp")
+ assert settings["effective_backend"] == "llama.cpp" and settings["has_gguf_form"] is True
+ assert settings["input_modalities"] == ["text"]
+ response = _post(client, questions = NOUL)
+ assert response.status_code == 200, response.text
+ assert response.headers["x-unsloth-decision-backend"] == "llama.cpp"
+ assert response.json()["answers"]["q"]["noul"] == 0.7
+ argv = stub.records("start")[0]["argv"]
+ assert argv[argv.index("-m") + 1] == str(folder / "gguf" / "model-Q8_0.gguf")
+
+
+def test_a_laya_export_reads_no_more_than_pytorch_serves(home, client, stub):
+ _laya_fine_tune(home.outputs)
+ _put(client, enabled = True, model = "laya-ft:laya_run", backend = "llama.cpp")
+ assert _post(client, questions = NOUL).status_code == 200
+ argv = stub.records("start")[0]["argv"]
+ assert argv[argv.index("-c") + 1] == argv[argv.index("-ub") + 1] == "1024"
+ refused = _post(client, state = "word " * 1100, questions = NOUL)
+ assert refused.status_code == 422 and "at most 1024" in refused.json()["detail"]["message"]
+ assert laya_runtime._loaded.backend == "llama.cpp"
+
+
+def test_auto_keeps_an_exported_laya_fine_tune_on_pytorch(home, client, stub, monkeypatch):
+ loaded = _torch_laya(monkeypatch)
+ _laya_fine_tune(home.outputs)
+ settings = _put(client, enabled = True, model = "laya-ft:laya_run")
+ assert settings["effective_backend"] == "pytorch" and settings["has_gguf_form"] is True
+ response = _post(client, questions = NOUL)
+ assert response.status_code == 200, response.text
+ assert response.headers["x-unsloth-decision-backend"] == "pytorch"
+ assert len(loaded) == 1 and loaded[0].backend == "pytorch"
+ assert stub.records("start") == []
+
+
+def test_a_laya_fine_tune_stays_on_pytorch_without_a_usable_export(home, client, stub, monkeypatch):
+ laya = catalog.fine_tune(f"laya-ft:{_laya_fine_tune(home.outputs).name}")
+ assert laya_runtime.select(laya, preference = "pytorch") == (laya, None)
+ assert laya_runtime.select(laya, preference = "mlx") == (laya, None)
+ plain = catalog.fine_tune(f"laya-ft:{_laya_fine_tune(home.outputs, 'plain', False).name}")
+ assert laya_runtime.select(plain, preference = "llama.cpp") == (plain, None)
+ assert not laya_runtime.native_ready(plain)
+ (Path(laya.source) / "model.safetensors").write_bytes(b"retrained")
+ assert laya_runtime.select(laya) == (laya, None)
+ stock = catalog.CHECKPOINTS["laya-multilingual"]
+ assert laya_runtime.select(stock, preference = "llama.cpp") == (stock, None)
+
+
+def test_a_local_laya_export_is_read_from_its_subfolder(home, client, stub):
+ root = _laya_fine_tune(home.outputs, "local_root")
+ inner = _laya_fine_tune(root, "inner")
+ laya = catalog.Checkpoint(catalog.LOCAL_NAME, str(root), "inner", "Local Laya checkpoint.")
+ native = laya_runtime.select(laya, preference = "llama.cpp")[0]
+ assert native.backend == "llama.cpp"
+ assert (
+ laya_runtime._native_files(native, local_only = True)[0] == inner / "gguf" / "model-Q8_0.gguf"
+ )
+ (inner / "gguf" / contract.EXPORT_FILE).unlink()
+ assert laya_runtime.select(laya, preference = "llama.cpp") == (laya, None)
+
+
+def test_a_laya_export_refuses_what_llama_cpp_cannot_take(home, client, stub, monkeypatch):
+ laya = catalog.fine_tune(f"laya-ft:{_laya_fine_tune(home.outputs).name}")
+ gap = {"q": {"type": "score", "instructions": "x", "criteria": ["only"]}}
+ with pytest.raises(laya_runtime.Unavailable):
+ laya_runtime.select(laya, questions = gap, preference = "llama.cpp")
+ monkeypatch.setattr(native_worker, "resolve_binary", lambda: None)
+ with pytest.raises(laya_runtime.Unavailable, match = "not installed"):
+ laya_runtime.select(laya, preference = "llama.cpp")
+
+
+def test_a_laya_export_gives_way_to_a_training_run(home, client, stub, monkeypatch):
+ laya = catalog.fine_tune(f"laya-ft:{_laya_fine_tune(home.outputs).name}")
+ monkeypatch.setattr(laya_runtime, "_native_gpu", lambda: True)
+ forced = laya_runtime.select(laya, preference = "llama.cpp")[0]
+ assert forced.backend == "llama.cpp"
+ home.training = True
+ with pytest.raises(laya_runtime.Unavailable, match = "training run"):
+ laya_runtime._ensure_loading(forced)
+
+
[email protected]("mode", ["unknown_arch", "unknown_vocab"])
+def test_a_laya_export_llama_cpp_cannot_load_is_refused_and_remembered(home, client, stub, mode):
+ laya = catalog.fine_tune(f"laya-ft:{_laya_fine_tune(home.outputs).name}")
+ _put(client, enabled = True, model = laya.name, backend = "llama.cpp")
+ stub.set_mode(mode)
+ assert _post(client, questions = NOUL).status_code == 503
+ assert len(stub.records("start")) >= 1
+ with pytest.raises(laya_runtime.Unavailable, match = "cannot load"):
+ laya_runtime.select(laya, preference = "llama.cpp")
+ assert laya_runtime.select(laya, preference = "auto") == (laya, None)
+
+
+def test_a_laya_export_that_failed_for_memory_loads_again(home, client, stub):
+ _laya_fine_tune(home.outputs)
+ _put(client, enabled = True, model = "laya-ft:laya_run", backend = "llama.cpp")
+ stub.set_mode("exit")
+ assert _post(client, questions = NOUL).status_code == 503
+ stub.set_mode("ok")
+ laya_runtime.unload()
+ assert _post(client, questions = NOUL).status_code == 200
+
+
+LONG_OPTION = " ".join(f"word{i}" for i in range(60))
+
+
+ "question, refusal",
+ [
+ (
+ {"type": "score", "instructions": "x", "criteria": ["a " * 40] * 8},
+ "option budget",
+ ),
+ ({"type": "noul", "instructions": "word " * 300}, "instructions would be cut short"),
+ ],
+)
+def test_a_laya_export_refuses_what_llama_cpp_would_cut(home, client, stub, question, refusal):
+ _laya_fine_tune(home.outputs)
+ _put(client, enabled = True, model = "laya-ft:laya_run", backend = "llama.cpp")
+ assert _post(client, questions = NOUL).status_code == 200
+ sent = len(stub.records("request"))
+ response = _post(client, questions = {"q": question})
+ assert response.status_code == 422 and refusal in response.json()["detail"]["message"]
+ assert len(stub.records("request")) == sent
+
+
+def test_a_laya_export_answers_a_long_option_as_pytorch_does(home, client, stub):
+ _laya_fine_tune(home.outputs)
+ _put(client, enabled = True, model = "laya-ft:laya_run", backend = "llama.cpp")
+ question = {"type": "choice", "instructions": "x", "criteria": {"a": LONG_OPTION, "b": "short"}}
+ response = _post(client, questions = {"q": question})
+ assert response.status_code == 200
+ assert response.headers["x-unsloth-decision-backend"] == "llama.cpp"
+
+
+def test_a_laya_export_without_a_readable_tokenizer_is_refused(home, client, stub):
+ folder = _laya_fine_tune(home.outputs)
+ (folder / "tokenizer" / "tokenizer.json").unlink()
+ _put(client, enabled = True, model = "laya-ft:laya_run", backend = "llama.cpp")
+ response = _post(client, questions = NOUL)
+ assert response.status_code == 503
+ assert response.json()["detail"]["message"].startswith("Could not read the tokenizer")
+ assert stub.records("start") == []
+
+
+ "question",
+ [
+ {"type": "choice", "instructions": "x", "criteria": {"a": None, "b": "", "c": {"k": 1}}},
+ {"type": "score", "instructions": "x", "criteria": ["low", {"k": [1, 2]}]},
+ {"type": "noul", "instructions": "x"},
+ {"type": "noul", "instructions": "x", "criteria": {"true": "yes", "false": ""}},
+ ],
+)
+def test_export_options_read_like_the_vendored_laya(question):
+ common = laya_runtime._laya().common
+ internal = laya_runtime._laya().agent.Agent._to_internal(question)
+ assert laya_runtime._export_options(question) == common.render_options(internal)
+
+
def test_a_laya_model_never_reports_an_earlier_clefs_fallback_reason(monkeypatch):
from core.systemone import laya_runtime
monkeypatch.setattr(laya_runtime, "_fallback_reason", "The GGUF is not downloaded.")
clef = SimpleNamespace(name = "clef-flash", backend = "pytorch", layout = "clef")
- laya = SimpleNamespace(name = "laya-multilingual", backend = "pytorch", layout = "laya")
+ laya = SimpleNamespace(
+ name = "laya-multilingual", backend = "pytorch", layout = "laya", is_local = False
+ )
monkeypatch.setattr(laya_runtime, "_loaded", clef)
assert laya_runtime.status()["fallback_reason"] == "The GGUF is not downloaded."
monkeypatch.setattr(laya_runtime, "_loaded", laya)
diff --git a/studio/frontend/src/features/settings/api/systemone.ts b/studio/frontend/src/features/settings/api/systemone.ts
index 480569984bc..df921f2af88 100644
--- a/studio/frontend/src/features/settings/api/systemone.ts
+++ b/studio/frontend/src/features/settings/api/systemone.ts
@@ -51,6 +51,7 @@ export type SystemOneSettings = {
fallbackReason: string | null;
inputModalities: string[];
layout: string | null;
+ hasGgufForm: boolean;
};
export type SystemOneConnection = {
@@ -135,6 +136,8 @@ type ApiSystemOneSettings = {
// biome-ignore lint/style/useNamingConvention: API schema
input_modalities?: string[];
layout?: string | null;
+ // biome-ignore lint/style/useNamingConvention: API schema
+ has_gguf_form?: boolean;
};
type ApiSystemOneDownloadPlan = {
@@ -212,6 +215,7 @@ function fromApi(settings: ApiSystemOneSettings): SystemOneSettings {
fallbackReason: settings.fallback_reason ?? null,
inputModalities: settings.input_modalities ?? ["text"],
layout: settings.layout ?? null,
+ hasGgufForm: settings.has_gguf_form ?? false,
};
}
diff --git a/studio/frontend/src/features/settings/components/decision-api-section.tsx b/studio/frontend/src/features/settings/components/decision-api-section.tsx
index 8f06aa00d33..082ced885bc 100644
--- a/studio/frontend/src/features/settings/components/decision-api-section.tsx
+++ b/studio/frontend/src/features/settings/components/decision-api-section.tsx
@@ -443,10 +443,11 @@ export function DecisionApiSection({
const remote = connections?.find((c) => c.name === settings.model);
const knownModel = current !== undefined || isRemote;
const longLabel = isRemote || current?.kind === "fine_tune";
- // Laya is PyTorch only; Clef runs on any runtime, a GGUF-only model on llama.cpp or MLX.
+ // Laya is PyTorch only unless exported; Clef runs on any runtime, a GGUF-only model on llama.cpp or MLX.
const runtimeChoice =
isClefDecisionModel(settings.model) ||
settings.layout === "clef" ||
+ settings.hasGgufForm ||
!!current?.llamaCppOnly;
const connectionGroups = [
...new Set(connections?.map((c) => c.providerId)),
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index b09e5b148a2..b1caa3f0198 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -3497,7 +3497,7 @@ export const en = {
backend: "Runtime",
backendAuto: "Auto",
backendDescription:
- "Auto serves Clef through llama.cpp when the model has a GGUF build and falls back to PyTorch otherwise. llama.cpp also reads images.",
+ "Auto serves Clef through llama.cpp when the model has a GGUF build and falls back to PyTorch otherwise. Exported Laya fine-tunes use llama.cpp when it is chosen here. llama.cpp also reads images.",
backendDescriptionMlx:
"Auto serves Clef through MLX on Apple Silicon and through llama.cpp when only the model's GGUF build is loaded or downloaded. MLX reads images for Clef models only. llama.cpp also reads images.",
backendStatus: "Runtime: {backend}",