Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: vendor laya 0.3.5 and hold its weights in float16 (#12202)

Daniel Han4ab5c77ee03d
diff --git a/pyproject.toml b/pyproject.toml
index 221ea4003ee..112ab8b9ea9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -66,7 +66,7 @@ studio = [
"frontend/*.yaml",
"frontend/.git*",
"backend/requirements/**/*",
- # Vendored truststore files.
+ # Vendored truststore and laya files.
"backend/vendor/**/*",
"backend/integrations/blender/*.py",
"backend/plugins/**/*",
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index ad7bf68f000..493dadb289d 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import gc
+import importlib.util
import logging
import sys
import threading
@@ -13,7 +14,7 @@ import time
from pathlib import Path
from typing import Any
-from .catalog import LOCAL_NAME, Checkpoint
+from .catalog import CHECKPOINTS, LOCAL_NAME, Checkpoint
logger = logging.getLogger(__name__)
@@ -23,8 +24,8 @@ RUN_WAIT_S = 30.0
MAX_PENDING = 8
FAILURE_BACKOFF_S = 60.0
_REQUIRED_DIRS = ("encoder", "tokenizer")
-LAYA_REQUIREMENT = "laya==0.3.5"
-INSTALL_TIMEOUT_S = 300
+# laya 0.3.5 ships inside Studio (see vendor/README.md), so the Decision API never installs anything.
+_VENDORED_LAYA = Path(__file__).resolve().parent.parent.parent / "vendor" / "laya"
_state_lock = threading.Lock()
_run_lock = threading.Lock()
@@ -35,9 +36,7 @@ _device_name: str | None = None
_loader: threading.Thread | None = None
_loading: Checkpoint | None = None
_failure: tuple[Checkpoint, str, float] | None = None
-_install_lock = threading.Lock()
-_installer: threading.Thread | None = None
-_install_failure: tuple[str, float] | None = None
+_import_lock = threading.Lock()
class Unavailable(Exception):
@@ -129,95 +128,30 @@ def _checkpoint_dir(checkpoint: Checkpoint, *, local_only: bool = False) -> Path
return root
-def package_available() -> bool:
- import importlib.util
- return importlib.util.find_spec("laya") is not None
-
-
-def _install_command() -> list[str]:
- import sys
-
- from utils.mlx_repair import _uv_executable
-
- # No deps: laya's deps are all Studio pins, so the install can never move them.
- uv = _uv_executable()
- if uv:
- return [uv, "pip", "install", "--python", sys.executable, "--no-deps", LAYA_REQUIREMENT]
- return [sys.executable, "-m", "pip", "install", "--no-deps", LAYA_REQUIREMENT]
-
-
-def ensure_package() -> None:
- global _install_failure
- if package_available():
- _install_failure = None
- return
- with _install_lock:
- if package_available():
- return
- if _install_failure and time.monotonic() < _install_failure[1]:
- raise RuntimeError(_install_failure[0])
- import importlib
- import os
- import subprocess
-
- from utils.mlx_repair import _MLX_ENV_ALLOWLIST, _venv_root
-
- # Same allowlisted env as the MLX self-heal: secrets/package-source vars cannot steer the install.
- env = {key: os.environ[key] for key in _MLX_ENV_ALLOWLIST if key in os.environ}
- if (venv_root := _venv_root()) is not None:
- env["VIRTUAL_ENV"] = venv_root
- logger.info("Installing %s for the Decision API", LAYA_REQUIREMENT)
- try:
- result = subprocess.run(
- _install_command(),
- env = env,
- capture_output = True,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- timeout = INSTALL_TIMEOUT_S,
- )
- detail = ((result.stderr or result.stdout).strip().splitlines() or [""])[-1]
- ok = result.returncode == 0
- except (OSError, subprocess.TimeoutExpired) as exc:
- detail, ok = type(exc).__name__, False
- importlib.invalidate_caches()
- if ok and package_available():
- _install_failure = None
- return
- message = (
- f"Could not install {LAYA_REQUIREMENT}. Check the internet connection. {detail}".strip()
- )
- logger.warning("Decision API install failed: %s", message)
- _install_failure = (message, time.monotonic() + FAILURE_BACKOFF_S)
- raise RuntimeError(message)
-
-
-def install_in_background() -> None:
- global _installer, _install_failure
- if package_available():
- _install_failure = None
- return
- with _state_lock:
- if _installer is not None and _installer.is_alive():
- return
- if _install_failure and time.monotonic() < _install_failure[1]:
- return
- _installer = threading.Thread(
- target = _install_quietly, name = "systemone-install", daemon = True
+def _laya():
+ """The vendored laya package, registered as top-level ``laya`` (its modules import each other relatively).
+
+ Loaded by file path, not from ``sys.path``: a laya installed in the venv (Studio pinned one before
+ vendoring it) must not replace this copy, since this module drives laya internals.
+ """
+ if (module := sys.modules.get("laya")) is not None:
+ return module
+ with _import_lock:
+ if (module := sys.modules.get("laya")) is not None:
+ return module
+ init = _VENDORED_LAYA / "__init__.py"
+ spec = importlib.util.spec_from_file_location(
+ "laya", init, submodule_search_locations = [str(_VENDORED_LAYA)]
)
- _installer.start()
-
-
-def _install_quietly() -> None:
- try:
- ensure_package()
- except RuntimeError:
- pass
-
-
-def installing() -> bool:
- return _install_lock.locked() or (_installer is not None and _installer.is_alive())
+ module = importlib.util.module_from_spec(spec)
+ sys.modules["laya"] = module
+ try:
+ spec.loader.exec_module(module)
+ except BaseException:
+ for name in [n for n in sys.modules if n == "laya" or n.startswith("laya.")]:
+ del sys.modules[name]
+ raise
+ return module
def is_cached(checkpoint: Checkpoint) -> bool:
@@ -293,11 +227,11 @@ class _MLXAgent:
def __init__(self, folder: Path):
import json
- from laya.agent import Agent
- from laya.common import clamp_temperature
from transformers import AutoTokenizer
from unsloth_zoo.mlx.decision import load_decision_model
+ laya = _laya()
+ clamp_temperature = laya.common.clamp_temperature
self.folder = folder
self.cfg = json.loads((folder / "rl_agent_config.json").read_text(encoding = "utf-8"))
self.tok = AutoTokenizer.from_pretrained(str(folder / "tokenizer"))
@@ -307,21 +241,133 @@ class _MLXAgent:
self.temperature_by_options = {
k: clamp_temperature(v) for k, v in self.cfg.get("temperature_by_options", {}).items()
}
- self._to_internal = Agent._to_internal
+ self._to_internal = laya.agent.Agent._to_internal
self.model = load_decision_model(folder)
def _load_checkpoint(checkpoint: Checkpoint):
root = _checkpoint_dir(checkpoint)
- import laya
+ laya = _laya()
# Evict only once the new checkpoint is on disk, so a long or failed download leaves the resident model serving.
_evict()
device = _device()
+ folder = root / checkpoint.subfolder if checkpoint.subfolder else root
if device == "mlx":
- folder = root / checkpoint.subfolder if checkpoint.subfolder else root
return _MLXAgent(folder), device
- return laya.load(str(root), subfolder = checkpoint.subfolder, device = device), device
+ if device not in ("cuda", "cpu"):
+ return laya.load(str(root), subfolder = checkpoint.subfolder, device = device), device
+ import torch
+
+ # Built on CPU and cast before the move, so the device never holds laya's fp32 copy.
+ agent = laya.load(str(root), subfolder = checkpoint.subfolder, device = "cpu")
+ fp16_checkpoint = checkpoint.name in CHECKPOINTS or _stored_fp16(folder)
+ _place(agent, torch.device(device), fp16_checkpoint)
+ return agent, str(agent.device.type)
+
+
+def _stored_fp16(folder: Path) -> bool:
+ """Whether the checkpoint's weights are saved as float16, as all three published Laya checkpoints are."""
+ import json
+ import math
+ import struct
+
+ try:
+ with open(folder / "model.safetensors", "rb") as f:
+ header = json.loads(f.read(struct.unpack("<Q", f.read(8))[0]))
+ except (OSError, ValueError, struct.error):
+ return False
+ sizes: dict[str, int] = {}
+ for name, meta in header.items():
+ if name != "__metadata__" and isinstance(meta, dict):
+ sizes[meta["dtype"]] = sizes.get(meta["dtype"], 0) + math.prod(meta["shape"])
+ return bool(sizes) and max(sizes, key = sizes.get) == "F16"
+
+
+def _precision(device, fp16_checkpoint: bool):
+ """(weight dtype, compute dtype) for ``device``; ``(None, None)`` keeps laya's fp32 weights and compute.
+
+ An fp16 checkpoint held in fp16 is exact (fp16 -> fp32 is lossless), and fp16 compute tracked fp32 about
+ 10x closer than bf16 on both GPU and CPU. Anything else follows the device: bf16 where it is native, else fp16.
+ """
+ import os
+ import platform
+
+ import torch
+
+ if os.environ.get("UNSLOTH_SYSTEMONE_FP32", "") == "1":
+ return None, None
+ if device.type == "cuda":
+ if fp16_checkpoint:
+ return torch.float16, torch.float16
+ if torch.version.hip:
+ bf16 = torch.cuda.is_bf16_supported()
+ else:
+ # By capability: pre-Ampere NVIDIA reports is_bf16_supported() through slow emulation.
+ bf16 = torch.cuda.get_device_capability(device)[0] >= 8
+ dtype = torch.bfloat16 if bf16 else torch.float16
+ return dtype, dtype
+ # CPU only where the ISA has the format natively (AVX512-FP16 / AMX); emulated fp16 or bf16 is slower than fp32.
+ # Measured on x86 only, so other CPUs keep fp32.
+ if device.type == "cpu" and platform.machine().lower() in ("x86_64", "amd64"):
+ mkldnn = getattr(torch.ops, "mkldnn", None)
+ try:
+ if fp16_checkpoint and mkldnn._is_mkldnn_fp16_supported():
+ return torch.float16, torch.float16
+ if not fp16_checkpoint and mkldnn._is_mkldnn_bf16_supported():
+ return torch.bfloat16, torch.bfloat16
+ except (AttributeError, RuntimeError):
+ pass
+ return None, None
+
+
+def _fp32_output(module, args, output):
+ return output.float()
+
+
+def _cast_matmul_weights(model, dtype) -> None:
+ """Cast the matmul and lookup weights only.
+
+ Norms stay fp32 (CPU layer_norm rejects low-precision parameters) and embedding outputs return to fp32,
+ so the residual stream keeps laya's precision. With bf16 compute this is bit-identical to fp32 weights,
+ since autocast rounds each weight to the same bf16 either way.
+ """
+ import torch.nn as nn
+ for module in model.modules():
+ if isinstance(module, (nn.Linear, nn.Embedding)):
+ module.to(dtype)
+ elif isinstance(module, nn.MultiheadAttention):
+ module.in_proj_weight.data = module.in_proj_weight.data.to(dtype)
+ if module.in_proj_bias is not None:
+ module.in_proj_bias.data = module.in_proj_bias.data.to(dtype)
+ if isinstance(module, nn.Embedding) and not getattr(module, "_unsloth_fp32_output", False):
+ module.register_forward_hook(_fp32_output)
+ module._unsloth_fp32_output = True
+
+
+def _place(agent, device, fp16_checkpoint: bool) -> None:
+ """Move a CPU-built laya agent to ``device`` in the precision :func:`_precision` picks for it."""
+ import torch
+
+ agent.__dict__["_unsloth_fp16_checkpoint"] = fp16_checkpoint
+ weights, compute = _precision(device, fp16_checkpoint)
+ # Cast on the CPU side of the move: before moving to an accelerator, after coming back from one.
+ if device.type == "cpu":
+ agent.model.to(device)
+ if weights is None:
+ agent.model.float()
+ else:
+ _cast_matmul_weights(agent.model, weights)
+ if device.type != "cpu":
+ try:
+ agent.model.to(device)
+ except (RuntimeError, torch.cuda.OutOfMemoryError):
+ # Same fallback as laya's own loader: a device that cannot hold the model leaves it on CPU.
+ logger.warning("Laya could not be placed on %s; running it on CPU", device)
+ _place(agent, torch.device("cpu"), fp16_checkpoint)
+ _release_memory()
+ return
+ agent.device, agent.dtype = device, compute or torch.float32
def _hub_download_active(checkpoint: Checkpoint) -> bool:
@@ -348,14 +394,9 @@ def _load(checkpoint: Checkpoint) -> None:
global _agent, _loaded, _device_name, _loading, _failure
started = time.monotonic()
try:
- ensure_package()
agent, device = _load_checkpoint(checkpoint)
except Exception as exc:
- message = (
- str(exc)
- if isinstance(exc, RuntimeError) and _install_failure
- else f"Could not load {checkpoint.name}: {type(exc).__name__}: {exc}"
- )
+ message = f"Could not load {checkpoint.name}: {type(exc).__name__}: {exc}"
logger.warning("System One load failed: %s", message)
with _state_lock:
_failure = (checkpoint, message, time.monotonic() + FAILURE_BACKOFF_S)
@@ -431,7 +472,7 @@ _HEAD_CACHE_SIZE = 1024
def _state_ids(tok, state, room: int) -> tuple[list[int], bool]:
- from laya.common import serialize_state
+ serialize_state = _laya().common.serialize_state
text = serialize_state(state).replace(tok.mask_token, " ")
chars = max(4096, room * 16)
@@ -447,7 +488,9 @@ def _state_ids(tok, state, room: int) -> tuple[list[int], bool]:
def _head(agent, question: dict[str, Any], max_len: int, head_max_len: int):
import json
- from laya.common import build_sequence, render_options
+ common = _laya().common
+ build_sequence = common.build_sequence
+ render_options = common.render_options
cache = agent.__dict__.setdefault("_unsloth_heads", {})
key = json.dumps(question, ensure_ascii = False)
@@ -464,7 +507,10 @@ def _head(agent, question: dict[str, Any], max_len: int, head_max_len: int):
def _predict(agent, state, questions: dict[str, dict[str, Any]]) -> tuple[dict[str, Any], bool]:
import numpy as np
- from laya.common import QTYPES, confidence_from_probs
+
+ common = _laya().common
+ QTYPES = common.QTYPES
+ confidence_from_probs = common.confidence_from_probs
max_len = int(agent.cfg.get("max_len", 512))
head_max_len = int(agent.cfg.get("head_max_len", 192))
@@ -525,7 +571,8 @@ def _predict(agent, state, questions: dict[str, dict[str, Any]]) -> tuple[dict[s
def _forward(agent, items: list[dict[str, Any]]):
global _agent, _loaded, _device_name
import torch
- from laya.common import collate_items
+
+ collate_items = _laya().common.collate_items
batch = collate_items([items], agent.tok.pad_token_id)
if agent.device == "mlx":
@@ -537,13 +584,14 @@ def _forward(agent, items: list[dict[str, Any]]):
if "memory" not in reason and "allocate" not in reason:
raise
# Past the handler, so the traceback no longer keeps the MLX arrays alive while the CPU copy loads.
- import laya
+ laya = _laya()
logger.warning("Laya ran out of GPU memory; moving it to CPU")
agent.model = None
_release_memory()
try:
cpu = laya.load(str(agent.folder), device = "cpu")
+ _place(cpu, torch.device("cpu"), _stored_fp16(Path(agent.folder)))
except Exception:
# Callers hold _run_lock, so drop the half-moved agent here; the next request loads it again.
_agent = _loaded = _device_name = None
@@ -558,11 +606,16 @@ def _forward(agent, items: list[dict[str, Any]]):
if agent.device.type == "cpu" or ("memory" not in reason and "cuda" not in reason):
raise
logger.warning("Laya ran out of GPU memory; moving it to CPU")
- agent.device, agent.dtype = torch.device("cpu"), torch.float32
- agent.model.to(agent.device)
+ _place(agent, torch.device("cpu"), agent.__dict__.get("_unsloth_fp16_checkpoint", False))
_device_name = "cpu"
_release_memory()
logits = _run_model(agent, batch)
+ if agent.dtype == torch.float16 and not bool(torch.isfinite(logits).all()):
+ # fp16 activations overflowed; the fp16 weights widen to fp32 exactly, so rerun at full precision.
+ logger.warning("Laya overflowed in float16; continuing in float32")
+ agent.model.float()
+ agent.dtype = torch.float32
+ logits = _run_model(agent, batch)
return logits.float().cpu().numpy(), int(batch["attention_mask"].sum())
@@ -572,7 +625,11 @@ def _run_model(agent, batch):
device = agent.device
with (
torch.inference_mode(),
- torch.autocast(device_type = device.type, dtype = agent.dtype, enabled = device.type == "cuda"),
+ torch.autocast(
+ device_type = device.type,
+ dtype = agent.dtype,
+ enabled = device.type in ("cuda", "cpu") and agent.dtype != torch.float32,
+ ),
):
logits, _ = agent.model(
batch["input_ids"].to(device),
@@ -586,7 +643,8 @@ def _run_model(agent, batch):
def _probabilities(agent, row, k: int, qtype: int):
import numpy as np
- from laya.common import temp_bucket
+
+ temp_bucket = _laya().common.temp_bucket
scale = agent.temperature_by_options.get(temp_bucket(qtype, k), agent.temperature[qtype])
z = row[:k] / scale
@@ -654,14 +712,15 @@ def status() -> dict[str, Any]:
"loaded_model": _loaded.name if _loaded else None,
"device": _device_name,
"loading_model": _loading.name if _loading else None,
- "installing": installing(),
- "error": failure[1] if failure else (_install_failure[0] if _install_failure else None),
+ # Kept for the settings API: laya is vendored, so there is never an install in flight.
+ "installing": False,
+ "error": failure[1] if failure else None,
"error_model": failure[0].name if failure else None,
}
def unload() -> bool:
- global _agent, _loaded, _device_name, _failure, _install_failure
+ global _agent, _loaded, _device_name, _failure
with _state_lock:
# _loading: a load claimed but not yet started would otherwise land after this unload.
if _loading is not None or (_loader is not None and _loader.is_alive()):
@@ -669,6 +728,6 @@ def unload() -> bool:
with _run_lock:
was_loaded = _agent is not None
_agent = _loaded = _device_name = None
- _failure = _install_failure = None
+ _failure = None
_release_memory()
return was_loaded
diff --git a/studio/backend/requirements/extras-no-deps.txt b/studio/backend/requirements/extras-no-deps.txt
index ad2c62fc866..1b61721f43e 100644
--- a/studio/backend/requirements/extras-no-deps.txt
+++ b/studio/backend/requirements/extras-no-deps.txt
@@ -10,10 +10,6 @@ snac==1.2.1
# installing with --no-deps to avoid pulling in torch>=0.11.0
peft==0.18.1
-# Laya System One runtime for /v1/systemone. Pure Python over torch, transformers,
-# safetensors, huggingface_hub and numpy, all already pinned elsewhere.
-laya==0.3.5; python_version >= "3.10"
-
# TRL and related packages
trl==0.23.1
# executorch>=1.0.1 # 41.5 MB - no imports in unsloth/zoo/studio
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 428b9c201f9..534ff033925 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -1372,8 +1372,6 @@ def _systemone_response() -> SystemOneSettingsResponse:
from core.systemone import catalog, laya_runtime
enabled = systemone_settings.get_enabled()
- if enabled:
- laya_runtime.install_in_background()
runtime = laya_runtime.status()
model = catalog.default_checkpoint().name
error = runtime["error"]
diff --git a/studio/backend/tests/test_systemone.py b/studio/backend/tests/test_systemone.py
index 02f0adef3a9..3136afe66e3 100644
--- a/studio/backend/tests/test_systemone.py
+++ b/studio/backend/tests/test_systemone.py
@@ -91,8 +91,6 @@ def runtime(monkeypatch):
("_loader", None),
("_loading", None),
("_failure", None),
- ("_installer", None),
- ("_install_failure", None),
):
monkeypatch.setattr(laya_runtime, name, value)
for name in (
@@ -117,7 +115,6 @@ def runtime(monkeypatch):
laya_runtime, "_predict", lambda agent, state, qs: (agent.predict(state, qs), False)
)
monkeypatch.setattr(systemone_settings, "runtime_unavailable_reason", lambda: None)
- monkeypatch.setattr(laya_runtime, "package_available", lambda: True)
yield loads
if laya_runtime._loader is not None:
laya_runtime._loader.join(5)
@@ -291,91 +288,26 @@ def test_failed_load_backs_off_and_reports_why(client, monkeypatch):
assert attempts == ["laya-multilingual"]
-class FakePip:
- def __init__(
- self,
- monkeypatch,
- returncode = 0,
- stderr = "",
- release = None,
- ):
- import subprocess
-
- self.calls = []
- self.installed = False
- self.returncode, self.stderr, self.release = returncode, stderr, release
- monkeypatch.setattr(laya_runtime, "package_available", lambda: self.installed)
- monkeypatch.setattr(subprocess, "run", self.run)
-
- def run(self, cmd, **kwargs):
- self.calls.append((cmd, kwargs))
- if self.release is not None:
- self.release.wait(5)
- self.installed = self.returncode == 0
- return SimpleNamespace(returncode = self.returncode, stdout = "", stderr = self.stderr)
-
-
-def test_missing_package_is_installed_automatically(client, monkeypatch, runtime):
- monkeypatch.setenv("UNSLOTH_TEST_SECRET", "do-not-forward")
- pip = FakePip(monkeypatch)
- response = _post(client)
- assert response.status_code == 200, response.text
- [(cmd, kwargs)] = pip.calls
- assert cmd[-2:] == ["--no-deps", laya_runtime.LAYA_REQUIREMENT]
- assert "UNSLOTH_TEST_SECRET" not in kwargs["env"]
- assert runtime == ["laya-multilingual"]
-
-
-def test_failed_install_says_why_and_backs_off(client, monkeypatch, runtime):
- pip = FakePip(monkeypatch, returncode = 1, stderr = "error: Failed to fetch laya\n")
- first = _post(client)
- assert first.status_code == 503
- message = first.json()["detail"]["message"]
- assert "Could not install laya==0.3.5" in message and "Failed to fetch laya" in message
- assert _post(client).status_code == 503
- assert len(pip.calls) == 1
- assert runtime == []
- assert client.get("/api/settings/systemone").json()["error"] == message
-
-
-def test_a_package_installed_by_hand_clears_the_old_install_error(client, monkeypatch):
- monkeypatch.setattr(laya_runtime, "FAILURE_BACKOFF_S", 0.0)
- pip = FakePip(monkeypatch, returncode = 1, stderr = "offline")
- assert _post(client).status_code == 503
- laya_runtime._loader.join(5)
- pip.installed = True
- assert _post(client).status_code == 200
- body = client.get("/api/settings/systemone").json()
- assert body["error"] is None and body["loaded_model"] == "laya-multilingual"
-
-
-def test_turning_it_back_on_retries_a_failed_install(client, monkeypatch):
- pip = FakePip(monkeypatch, returncode = 1, stderr = "offline")
- assert _post(client).status_code == 503
- client.put("/api/settings/systemone", json = {"enabled": False})
- pip.returncode = 0
- client.put("/api/settings/systemone", json = {"enabled": True})
- laya_runtime._installer.join(5)
- assert pip.installed is True
- assert _post(client).status_code == 200
-
-
-def test_settings_install_in_the_background_and_say_so(client, monkeypatch):
- release = threading.Event()
- pip = FakePip(monkeypatch, release = release)
- assert client.get("/api/settings/systemone").json()["installing"] is True
- release.set()
- laya_runtime._installer.join(5)
+def test_settings_never_report_an_install(client):
body = client.get("/api/settings/systemone").json()
assert body["installing"] is False and body["error"] is None
- assert len(pip.calls) == 1
-def test_nothing_installs_while_it_is_off(client, monkeypatch):
- monkeypatch.setattr(systemone_settings, "_owner_setting", {}.get)
- pip = FakePip(monkeypatch)
- assert client.get("/api/settings/systemone").json()["installing"] is False
- assert pip.calls == []
+def test_the_vendored_laya_wins_over_an_installed_one(monkeypatch, tmp_path):
+ pytest.importorskip("torch")
+ from pathlib import Path
+
+ for name in [n for n in sys.modules if n == "laya" or n.startswith("laya.")]:
+ monkeypatch.delitem(sys.modules, name)
+ decoy = tmp_path / "laya"
+ decoy.mkdir()
+ (decoy / "__init__.py").write_text("raise ImportError('installed laya was imported')\n")
+ monkeypatch.syspath_prepend(str(tmp_path))
+ laya = laya_runtime._laya()
+ assert laya_runtime._laya() is laya and sys.modules["laya"] is laya
+ assert Path(laya.__file__).parent == laya_runtime._VENDORED_LAYA
+ assert Path(sys.modules["laya.common"].__file__).parent == laya_runtime._VENDORED_LAYA
+ assert callable(laya.load) and callable(laya.common.collate_items)
def test_slow_load_answers_retry_after_instead_of_hanging(client, monkeypatch):
@@ -679,7 +611,7 @@ def test_mlx_answers_match_laya_on_cpu(monkeypatch):
path = os.environ.get("SYSTEMONE_TEST_LAYA")
if not path:
pytest.skip("set SYSTEMONE_TEST_LAYA to a downloaded convaiinnovations/laya snapshot")
- laya = pytest.importorskip("laya")
+ laya = laya_runtime._laya()
if not laya_runtime._mlx_available():
pytest.skip("needs unsloth_zoo.mlx.decision on Apple Silicon")
from pathlib import Path
@@ -925,7 +857,6 @@ def test_real_laya_answers_through_the_route(client, monkeypatch):
path = os.environ.get("SYSTEMONE_TEST_LAYA")
if not path:
pytest.skip("set SYSTEMONE_TEST_LAYA to a downloaded convaiinnovations/laya snapshot")
- pytest.importorskip("laya")
monkeypatch.setattr(laya_runtime, "_load_checkpoint", _REAL_LOAD)
monkeypatch.setattr(laya_runtime, "_predict", _REAL_PREDICT)
monkeypatch.setenv("UNSLOTH_SYSTEMONE_MODEL", path)
@@ -975,7 +906,7 @@ class WordTokenizer:
def test_long_state_is_tokenized_only_as_far_as_the_model_reads():
- pytest.importorskip("laya")
+ pytest.importorskip("torch") # laya imports torch
tok = WordTokenizer()
ids, cut = laya_runtime._state_ids(tok, "word " * 100_000, 900)
assert (ids, cut) == ([4] * 900, True)
@@ -984,7 +915,7 @@ def test_long_state_is_tokenized_only_as_far_as_the_model_reads():
@pytest.mark.parametrize("words, cut", [(900, False), (901, True), (5000, True)])
def test_state_cut_matches_the_full_tokenization(words, cut):
- pytest.importorskip("laya")
+ pytest.importorskip("torch")
ids, was_cut = laya_runtime._state_ids(WordTokenizer(), "word " * words, 900)
assert (len(ids), was_cut) == (min(words, 900), cut)
@@ -997,12 +928,19 @@ class MovableModel:
self.moved_to.append(device.type)
return self
+ def float(self):
+ return self
+
@pytest.fixture
def gpu_agent(monkeypatch):
torch = pytest.importorskip("torch")
+ # A CPU without native fp16/bf16: the fallback lands in fp32 whatever this host supports.
+ monkeypatch.setattr(laya_runtime, "_precision", lambda device, fp16_checkpoint: (None, None))
collate = lambda groups, pad: {"attention_mask": torch.ones(1, len(groups[0][0]["ids"]))}
- monkeypatch.setitem(sys.modules, "laya.common", SimpleNamespace(collate_items = collate))
+ monkeypatch.setitem(
+ sys.modules, "laya", SimpleNamespace(common = SimpleNamespace(collate_items = collate))
+ )
return SimpleNamespace(
device = torch.device("cuda"),
dtype = torch.float16,
@@ -1092,7 +1030,7 @@ def test_mlx_out_of_memory_falls_back_to_cpu(monkeypatch, gpu_agent):
monkeypatch.setattr(laya_runtime, "_loaded", "checkpoint")
return laya_runtime._forward(agent, _items())
- monkeypatch.setitem(sys.modules, "laya", SimpleNamespace(load = load))
+ monkeypatch.setattr(sys.modules["laya"], "load", load, raising = False)
monkeypatch.setitem(
sys.modules, "mlx.core", SimpleNamespace(clear_cache = lambda: cleared.append(True))
)
@@ -1136,7 +1074,7 @@ def test_fast_path_matches_laya_predict():
path = os.environ.get("SYSTEMONE_TEST_LAYA")
if not path:
pytest.skip("set SYSTEMONE_TEST_LAYA to a downloaded convaiinnovations/laya snapshot")
- laya = pytest.importorskip("laya")
+ laya = laya_runtime._laya()
agent = laya.load(path, subfolder = "multilingual", device = "cpu")
questions = {name: laya_runtime._to_laya(q) for name, q in QUESTIONS.items()}
questions["zeta"] = {
@@ -1158,3 +1096,99 @@ def test_fast_path_matches_laya_predict():
if "probabilities" in answer:
assert list(answer["probabilities"]) == list(want["probabilities"])
assert truncated == (len(state) > 200 if isinstance(state, str) else False)
+
+
+def _save_checkpoint(folder, dtype):
+ torch = pytest.importorskip("torch")
+ from safetensors.torch import save_file
+
+ folder.mkdir(parents = True, exist_ok = True)
+ save_file(
+ {"big.weight": torch.zeros(64, 64, dtype = dtype), "temperature": torch.ones(3)},
+ str(folder / "model.safetensors"),
+ )
+ return folder
+
+
+def test_stored_dtype_is_read_from_the_safetensors_header(tmp_path):
+ torch = pytest.importorskip("torch")
+ assert laya_runtime._stored_fp16(_save_checkpoint(tmp_path / "half", torch.float16))
+ assert not laya_runtime._stored_fp16(_save_checkpoint(tmp_path / "full", torch.float32))
+ assert not laya_runtime._stored_fp16(tmp_path / "missing")
+
+
+def test_precision_follows_the_checkpoint_then_the_device(monkeypatch):
+ torch = pytest.importorskip("torch")
+ cuda = torch.device("cuda")
+ monkeypatch.setattr(torch.version, "hip", None)
+ assert laya_runtime._precision(cuda, True) == (torch.float16, torch.float16)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device = None: (7, 5))
+ assert laya_runtime._precision(cuda, False) == (torch.float16, torch.float16)
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device = None: (9, 0))
+ assert laya_runtime._precision(cuda, False) == (torch.bfloat16, torch.bfloat16)
+ monkeypatch.setenv("UNSLOTH_SYSTEMONE_FP32", "1")
+ assert laya_runtime._precision(cuda, True) == (None, None)
+ monkeypatch.delenv("UNSLOTH_SYSTEMONE_FP32")
+ import platform
+
+ monkeypatch.setattr(platform, "machine", lambda: "arm64")
+ assert laya_runtime._precision(torch.device("cpu"), True) == (None, None)
+ assert laya_runtime._precision(torch.device("mps"), True) == (None, None)
+
+
+def _tiny_decision_model(torch):
+ nn = torch.nn
+ torch.manual_seed(0)
+
+ class Tiny(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.emb = nn.Embedding(50, 32)
+ self.norm = nn.LayerNorm(32)
+ self.layer = nn.TransformerEncoderLayer(
+ 32, 4, 64, 0.0, batch_first = True, norm_first = True
+ )
+ self.out = nn.Linear(32, 1)
+
+ def forward(self, ids):
+ return self.out(self.layer(self.norm(self.emb(ids)))).float()
+
+ model = Tiny().eval()
+ # Weights exactly representable in fp16, as a checkpoint saved in fp16 loads into fp32.
+ for param in model.parameters():
+ param.data = param.data.half().float()
+ return model
+
+
+def test_fp16_weights_are_bit_identical_under_bf16_autocast():
+ torch = pytest.importorskip("torch")
+ import copy
+
+ reference = _tiny_decision_model(torch)
+ half = copy.deepcopy(reference)
+ laya_runtime._cast_matmul_weights(half, torch.float16)
+ assert (
+ half.emb.weight.dtype == half.out.weight.dtype == half.layer.self_attn.in_proj_weight.dtype
+ )
+ assert half.emb.weight.dtype == torch.float16
+ assert half.norm.weight.dtype == half.layer.norm1.weight.dtype == torch.float32
+ ids = torch.randint(0, 50, (3, 17))
+ assert half.emb(ids).dtype == torch.float32
+ with torch.inference_mode(), torch.autocast(device_type = "cpu", dtype = torch.bfloat16):
+ assert torch.equal(reference(ids), half(ids))
+
+
+def test_fp16_overflow_reruns_in_fp32(monkeypatch, gpu_agent):
+ torch = pytest.importorskip("torch")
+ agent = gpu_agent
+ calls = []
+
+ def run(agent, batch):
+ calls.append(agent.dtype)
+ value = float("inf") if agent.dtype == torch.float16 else 1.0
+ return torch.tensor([[value, 0.5]])
+
+ monkeypatch.setattr(laya_runtime, "_run_model", run)
+ logits, _ = laya_runtime._forward(agent, _items())
+ assert calls == [torch.float16, torch.float32] and agent.dtype == torch.float32
+ assert logits.tolist() == [[1.0, 0.5]]
diff --git a/studio/backend/tests/test_vendored_truststore.py b/studio/backend/tests/test_vendored_truststore.py
index 42ae190a668..b07cf04b431 100644
--- a/studio/backend/tests/test_vendored_truststore.py
+++ b/studio/backend/tests/test_vendored_truststore.py
@@ -1,12 +1,12 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""The vendored truststore stays byte-identical to the release it came from.
+"""The vendored truststore and laya stay byte-identical to the releases they came from.
-It is a static copy: nothing refreshes it, so any change to these bytes is either
-a deliberate version bump that must update the manifest with it, or an accident.
-The accident is the dangerous one, since it means Unsloth verifies certificates
-with code no upstream release ever shipped.
+They are static copies: nothing refreshes them, so any change to these bytes is
+either a deliberate version bump that must update the manifest with it, or an
+accident. For truststore the accident is the dangerous one, since it means
+Unsloth verifies certificates with code no upstream release ever shipped.
"""
from __future__ import annotations
@@ -20,12 +20,22 @@ import pytest
_BACKEND = Path(__file__).resolve().parent.parent
_VENDOR = _BACKEND / "vendor"
-_MANIFEST = json.loads((_VENDOR / "truststore_manifest.json").read_text(encoding = "utf-8"))
-
-# Everything the vendor directory is allowed to hold, beyond the package itself.
-_SIDECARS = {"LICENSE", "README.md", "truststore_manifest.json"}
-# Ours, not upstream's: prose we may reword, and the manifest cannot hash itself.
-_UNPINNED = {"README.md", "truststore_manifest.json"}
+_MANIFESTS = {
+ path.name.removesuffix("_manifest.json"): json.loads(path.read_text(encoding = "utf-8"))
+ for path in sorted(_VENDOR.glob("*_manifest.json"))
+}
+_MANIFEST = _MANIFESTS["truststore"]
+_PACKAGES = {"truststore", "laya"}
+
+# Everything the vendor directory is allowed to hold, beyond the packages themselves.
+_SIDECARS = {
+ "LICENSE",
+ "LICENSE.laya",
+ "README.md",
+ *(f"{name}_manifest.json" for name in _PACKAGES),
+}
+# Ours, not upstream's: prose we may reword, and the manifests cannot hash themselves.
+_UNPINNED = {"README.md", *(f"{name}_manifest.json" for name in _PACKAGES)}
def _tracked_files() -> dict[str, Path]:
@@ -37,12 +47,18 @@ def _tracked_files() -> dict[str, Path]:
}
-def test_vendored_tree_matches_the_manifest():
+def test_every_vendored_package_has_a_manifest():
+ assert set(_MANIFESTS) == _PACKAGES
+ assert all(manifest["package"] == name for name, manifest in _MANIFESTS.items())
+
+
+def test_vendored_tree_matches_the_manifests():
found = _tracked_files()
- recorded = _MANIFEST["files"]
+ recorded = {name: digest for m in _MANIFESTS.values() for name, digest in m["files"].items()}
+ assert sum(len(m["files"]) for m in _MANIFESTS.values()) == len(recorded)
assert set(found) == set(recorded), (
- "the vendored tree gained or lost a file; it is a static copy of upstream "
- f"{_MANIFEST['version']}, so update truststore_manifest.json in the same commit"
+ "the vendored tree gained or lost a file; each package is a static copy of an upstream "
+ "release, so update its <package>_manifest.json in the same commit"
)
drifted = [
name
@@ -50,7 +66,7 @@ def test_vendored_tree_matches_the_manifest():
if hashlib.sha256(path.read_bytes()).hexdigest() != recorded[name]
]
assert not drifted, (
- f"vendored files no longer match upstream {_MANIFEST['version']}: {', '.join(drifted)}. "
+ f"vendored files no longer match upstream: {', '.join(drifted)}. "
"A formatter most likely rewrote them; check the vendor excludes in pyproject.toml "
"and .pre-commit-config.yaml"
)
@@ -66,11 +82,11 @@ def test_no_symlinks_or_special_files():
assert not offenders, f"vendor tree must be plain files: {offenders}"
-def test_vendor_holds_nothing_but_truststore():
+def test_vendor_holds_nothing_but_the_vendored_packages():
"""The gate appends this directory to sys.path, so anything else here is importable."""
top_level = {path.name for path in _VENDOR.iterdir()} - _SIDECARS
- assert top_level == {"truststore"}, (
- f"unexpected entries in the vendor directory: {sorted(top_level - {'truststore'})}. "
+ assert top_level == _PACKAGES, (
+ f"unexpected entries in the vendor directory: {sorted(top_level - _PACKAGES)}. "
"Appending it to sys.path would make them importable as top-level modules"
)
@@ -105,8 +121,9 @@ def test_nothing_imports_the_vendor_path_directly():
if any("vendor" in alias.name.split(".") for alias in node.names):
offenders.append(f"{path.relative_to(studio)}:{node.lineno}")
assert not offenders, (
- "import the vendored package as top-level `truststore` after appending the vendor "
- f"directory to sys.path, never by its dotted path: {offenders}"
+ "import truststore as top-level `truststore` after appending the vendor directory to "
+ "sys.path, and laya through core.systemone.laya_runtime._laya(), never by a dotted "
+ f"vendor path: {offenders}"
)
@@ -124,7 +141,15 @@ def test_vendored_version_is_the_one_recorded():
), f"vendored truststore is {version} but the manifest records {_MANIFEST['version']}"
[email protected]("relative", ["LICENSE", "README.md"])
+def test_vendored_laya_version_is_the_one_recorded():
+ from core.systemone.laya_runtime import _VENDORED_LAYA
+
+ assert _VENDORED_LAYA == _VENDOR / "laya"
+ init = (_VENDORED_LAYA / "__init__.py").read_text(encoding = "utf-8")
+ assert f'__version__ = "{_MANIFESTS["laya"]["version"]}"' in init
+
+
[email protected]("relative", ["LICENSE", "LICENSE.laya", "README.md"])
def test_provenance_files_are_present(relative):
- """MIT requires the licence to travel with the copy."""
+ """MIT and Apache-2.0 both require the licence to travel with the copy."""
assert (_VENDOR / relative).read_text(encoding = "utf-8").strip()
diff --git a/studio/backend/utils/systemone_settings.py b/studio/backend/utils/systemone_settings.py
index 429488cd157..8c963d79905 100644
--- a/studio/backend/utils/systemone_settings.py
+++ b/studio/backend/utils/systemone_settings.py
@@ -51,7 +51,7 @@ def device_locked() -> bool:
def runtime_unavailable_reason() -> str | None:
- # laya is installed only on Python 3.10+ (extras-no-deps.txt) and needs torch (absent on --no-torch).
+ # The vendored laya needs Python 3.10+ and torch (absent on --no-torch).
if sys.version_info < (3, 10):
return "The Decision API needs Python 3.10 or newer."
try:
diff --git a/studio/backend/vendor/LICENSE.laya b/studio/backend/vendor/LICENSE.laya
new file mode 100644
index 00000000000..d9a10c0d8e8
--- /dev/null
+++ b/studio/backend/vendor/LICENSE.laya
@@ -0,0 +1,176 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
diff --git a/studio/backend/vendor/README.md b/studio/backend/vendor/README.md
index c0b189823cc..924354147c7 100644
--- a/studio/backend/vendor/README.md
+++ b/studio/backend/vendor/README.md
@@ -47,3 +47,32 @@ To move to another release, replace `truststore/` with that version's wheel cont
`truststore_manifest.json`. `tests/test_vendored_truststore.py` fails until the manifest matches, so
a half-finished swap cannot land. Read upstream's changelog first: a 0.x minor is where truststore
has changed verification behaviour, which here applies process-wide.
+
+## laya 0.3.5 (Apache-2.0)
+
+- Upstream: https://huggingface.co/convaiinnovations/laya
+- Release: https://pypi.org/project/laya/0.3.5/
+- Taken from `laya-0.3.5-py3-none-any.whl`,
+ sha256 `4c57f64cbaf893bb5c7b4affddc2bf21a819f55df51941689f11868583be2903`
+- Licence: `LICENSE.laya` beside this file, copied unmodified from the wheel. The wheel ships no
+ `NOTICE` file, and the source files are unmodified.
+
+`core/systemone/laya_runtime.py` uses it to serve the Decision API (`POST /v1/systemone`). It is pure
+Python over torch, transformers, safetensors, huggingface_hub and numpy, all of which Studio already
+installs, so shipping the source replaces the `laya` pin in `requirements/extras-no-deps.txt` and the
+runtime `pip install` the Decision API used to fall back to.
+
+### How it is imported
+
+Only through `laya_runtime._laya()`, which loads `vendor/laya/__init__.py` by file path and registers
+it as the top-level `laya` module (its files import each other relatively). It never goes through
+`sys.path`, so a `laya` left in the venv by an older Studio, or installed by the user, does not replace
+this copy: `laya_runtime` drives laya internals (`Agent._to_internal`, `laya.common.collate_items`),
+so the version it runs must be the one it was written against.
+
+### Updating
+
+Replace `laya/` with the new wheel's `laya/` directory, copy its licence to `LICENSE.laya`, and update
+`version`, `wheel`, `wheel_sha256` and the per-file hashes in `laya_manifest.json`. Then check every
+laya internal `laya_runtime.py` reaches into, and run `SYSTEMONE_TEST_LAYA=<snapshot> pytest
+tests/test_systemone.py`, which compares the fast path against `laya.Agent.predict`.
diff --git a/studio/backend/vendor/laya/__init__.py b/studio/backend/vendor/laya/__init__.py
new file mode 100644
index 00000000000..f6805a5d043
--- /dev/null
+++ b/studio/backend/vendor/laya/__init__.py
@@ -0,0 +1,55 @@
+"""Laya: Fast, non-autoregressive System 1 decision engine with calibrated probabilities."""
+
+from .agent import Agent, RLAgent, load
+from .common import (
+ QTYPES,
+ QTYPE_NAMES,
+ confidence_from_probs,
+ ece_score,
+ proper_reward,
+ render_options,
+ td_lambda_targets,
+)
+from .email import clean_email_body, email_state
+from .lang import analyse as detect_language
+from .lang import detect_script, is_english
+from .presets import (
+ email_questions,
+ guard_questions,
+ moderation_questions,
+ router_questions,
+ triage_questions,
+)
+from .router import DEFAULT_MODELS, RouteDecision, Router
+from .shortlist import embed_fn_from_agent, predict_shortlist, shortlist_choice
+
+__version__ = "0.3.5"
+__all__ = [
+ "Agent",
+ "RLAgent",
+ "load",
+ "Router",
+ "RouteDecision",
+ "DEFAULT_MODELS",
+ "shortlist_choice",
+ "predict_shortlist",
+ "embed_fn_from_agent",
+ "detect_language",
+ "detect_script",
+ "is_english",
+ "clean_email_body",
+ "email_questions",
+ "email_state",
+ "guard_questions",
+ "moderation_questions",
+ "router_questions",
+ "triage_questions",
+ "proper_reward",
+ "td_lambda_targets",
+ "ece_score",
+ "confidence_from_probs",
+ "render_options",
+ "QTYPES",
+ "QTYPE_NAMES",
+ "__version__",
+]
diff --git a/studio/backend/vendor/laya/agent.py b/studio/backend/vendor/laya/agent.py
new file mode 100644
index 00000000000..15e2eb5666f
--- /dev/null
+++ b/studio/backend/vendor/laya/agent.py
@@ -0,0 +1,385 @@
+"""High-level inference runtime for laya System 1 decision models."""
+import json
+import os
+import warnings
+from typing import Any, Dict, Optional, Union
+
+import numpy as np
+import torch
+
+from .common import (
+ QTYPES,
+ TEMP_MAX,
+ TEMP_MIN,
+ amp_dtype,
+ build_model,
+ build_sequence,
+ clamp_temperature,
+ collate_items,
+ confidence_from_probs,
+ render_options,
+ temp_bucket,
+)
+
+
+def _fix_tokenizer_config(path: str):
+ """Ensure tokenizer_config.json can be loaded across all transformers versions."""
+ cfg_file = os.path.join(path, "tokenizer", "tokenizer_config.json")
+ if not os.path.exists(cfg_file):
+ return
+ try:
+ with open(cfg_file) as f:
+ tcfg = json.load(f)
+ changed = False
+ if tcfg.get("tokenizer_class") in (None, "TokenizersBackend"):
+ tcfg["tokenizer_class"] = "PreTrainedTokenizerFast"
+ tcfg.pop("backend", None)
+ tcfg.pop("is_local", None)
+ changed = True
+ # Checkpoints built on the mmBERT/Gemma tokenizer store extra_special_tokens as a list;
+ # transformers expects a mapping and raises "'list' object has no attribute 'keys'",
+ # which makes AutoTokenizer -- and so the whole model -- fail to load.
+ extra = tcfg.get("extra_special_tokens")
+ if isinstance(extra, list):
+ tcfg["extra_special_tokens"] = {"extra_%d" % i: t for i, t in enumerate(extra)}
+ changed = True
+ if changed:
+ with open(cfg_file, "w") as f:
+ json.dump(tcfg, f, indent=2)
+ except Exception:
+ pass
+
+
+def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str, torch.Tensor], model_id: str):
+ """Verify that the loaded checkpoint weights and config strictly match the expected architecture."""
+ # 1. Verify required configuration attributes
+ required_cfg = ["encoder", "head_layers"]
+ missing_cfg = [k for k in required_cfg if k not in cfg]
+ if missing_cfg:
+ raise ValueError(
+ f"Incompatible model config for {model_id!r}: missing configuration keys {missing_cfg}. "
+ f"Ensure this is a valid RL Agent decision model."
+ )
+
+ # 2. Check for required component prefixes
+ required_prefixes = ("encoder.", "type_emb.", "scorer.", "act_head.")
+ for prefix in required_prefixes:
+ if not any(k.startswith(prefix) for k in weights.keys()):
+ raise ValueError(
+ f"Incompatible model weights for {model_id!r}: checkpoint is missing '{prefix}' parameters. "
+ f"Expected an RL Agent decision model with encoder and decision heads."
+ )
+
+ # 3. Check for parameter shape mismatches
+ model_sd = model.state_dict()
+ shape_mismatches = []
+ missing_keys = []
+
+ for name, param in model.named_parameters():
+ if name not in weights:
+ missing_keys.append(name)
+ elif tuple(weights[name].shape) != tuple(param.shape):
+ shape_mismatches.append(f" - {name}: expected {tuple(param.shape)}, found {tuple(weights[name].shape)}")
+
+ if shape_mismatches:
+ err_details = "\n".join(shape_mismatches[:5])
+ if len(shape_mismatches) > 5:
+ err_details += f"\n ... and {len(shape_mismatches) - 5} more mismatched layers."
+ raise ValueError(
+ f"Model architecture mismatch for {model_id!r}:\n{err_details}\n"
+ f"The checkpoint weights do not match the configured model architecture."
+ )
+
+ if missing_keys:
+ raise ValueError(
+ f"Model weights incomplete for {model_id!r}: missing {len(missing_keys)} parameter tensors "
+ f"(e.g. {missing_keys[:3]})."
+ )
+
+
+class Agent:
+ """System 1 decision model runtime: fast, non-autoregressive, calibrated decisions."""
+
+ def __init__(
+ self,
+ model_id_or_path: str = "convaiinnovations/laya",
+ device: Optional[str] = None,
+ token: Optional[str] = None,
+ subfolder: Optional[str] = None,
+ ):
+ """Load a Laya checkpoint.
+
+ `subfolder` selects one checkpoint from a repo that bundles several, e.g.
+ `Agent("convaiinnovations/laya", subfolder="multilingual")`. Only that subfolder is
+ downloaded, so bundling does not cost every user the whole family.
+ """
+ from safetensors.torch import load_file
+ from transformers import AutoTokenizer
+
+ model_dir = model_id_or_path
+ if not os.path.exists(model_dir):
+ if model_id_or_path.startswith(("/", "./", "../")) or os.path.isabs(model_id_or_path):
+ raise FileNotFoundError(
+ f"Local model path not found: {model_id_or_path!r}. "
+ f"Check that the directory exists and that training saved the model successfully."
+ )
+ from huggingface_hub import snapshot_download
+
+ # Restrict root checkpoints too: the default repo also contains sibling
+ # checkpoints, which an unfiltered snapshot would unnecessarily download.
+ prefix = f"{subfolder}/" if subfolder else ""
+ kw = {
+ "token": token or os.environ.get("HF_TOKEN"),
+ "allow_patterns": [prefix + name for name in (
+ "rl_agent_config.json", "model.safetensors", "tokenizer/*", "encoder/*",
+ )],
+ }
+ model_dir = snapshot_download(model_id_or_path, **kw)
+
+ if subfolder:
+ model_dir = os.path.join(model_dir, subfolder)
+ if not os.path.isdir(model_dir):
+ raise FileNotFoundError(
+ f"Subfolder {subfolder!r} not found in {model_id_or_path!r}."
+ )
+
+ _fix_tokenizer_config(model_dir)
+
+ cfg_path = os.path.join(model_dir, "rl_agent_config.json")
+ if not os.path.exists(cfg_path):
+ raise FileNotFoundError(
+ f"Incompatible model: {model_id_or_path!r} does not contain 'rl_agent_config.json'. "
+ f"That file ships with the weights of a Laya checkpoint, so load one of those "
+ f"(e.g. 'convaiinnovations/laya') or a directory your own training run wrote."
+ )
+
+ with open(cfg_path) as f:
+ self.cfg = json.load(f)
+
+ weights_path = os.path.join(model_dir, "model.safetensors")
+ if not os.path.exists(weights_path):
+ raise FileNotFoundError(
+ f"Incompatible model: 'model.safetensors' not found in {model_id_or_path!r}."
+ )
+
+ # 1. Device resolution with automatic fallback
+ if device is not None:
+ target_device = torch.device(device)
+ if target_device.type == "cuda" and not torch.cuda.is_available():
+ print("Warning: CUDA requested but not available. Falling back to CPU.")
+ self.device = torch.device("cpu")
+ elif target_device.type == "mps" and not (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()):
+ print("Warning: MPS requested but not available. Falling back to CPU.")
+ self.device = torch.device("cpu")
+ else:
+ self.device = target_device
+ else:
+ if torch.cuda.is_available():
+ self.device = torch.device("cuda")
+ elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+ self.device = torch.device("mps")
+ else:
+ self.device = torch.device("cpu")
+
+ tok_dir = os.path.join(model_dir, "tokenizer")
+ self.tok = AutoTokenizer.from_pretrained(tok_dir if os.path.exists(tok_dir) else self.cfg.get("encoder"))
+
+ enc_dir = os.path.join(model_dir, "encoder")
+ self.model = build_model(self.cfg, encoder_dir=enc_dir if os.path.exists(enc_dir) else None)
+
+ # Load weights and verify architectural compatibility
+ weights = load_file(weights_path)
+ _verify_compatibility(self.model, self.cfg, weights, model_id_or_path)
+
+ self.model.load_state_dict(weights, strict=True)
+
+ # ModernBERT's reference_compile defaults to "auto" and will torch.compile the encoder.
+ # That is a loss for the batch sizes Laya runs (a handful of questions per call) and can
+ # hang on some platforms, so keep the eager path.
+ try:
+ self.model.encoder.config.reference_compile = False
+ except Exception:
+ pass
+
+ # Keep what the checkpoint shipped for inspection, but only ever apply clamped values:
+ # some buckets are fitted to sharpen rather than soften (see clamp_temperature).
+ self.temperature_raw = self.cfg.get("temperature", [1.0, 1.0, 1.0])
+ self.temperature_by_options_raw = self.cfg.get("temperature_by_options", {})
+ self.temperature = [clamp_temperature(t) for t in self.temperature_raw]
+ self.temperature_by_options = {k: clamp_temperature(v)
+ for k, v in self.temperature_by_options_raw.items()}
+ rejected = ["%s=%.4g" % (k, float(v)) for k, v in self.temperature_by_options_raw.items()
+ if clamp_temperature(v) != float(v)]
+ rejected += ["temperature[%d]=%.4g" % (i, float(t)) for i, t in enumerate(self.temperature_raw)
+ if clamp_temperature(t) != float(t)]
+ if rejected:
+ warnings.warn(
+ "laya: this checkpoint ships temperatures outside [%g, %g] which would distort "
+ "confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."
+ % (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
+ RuntimeWarning, stacklevel=2)
+ self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))
+
+ if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device)[0] < 8:
+ self.dtype = torch.float16
+ elif self.device.type in ("cpu", "mps"):
+ self.dtype = torch.float32
+
+ # 2. Place on device with graceful fallback to CPU on memory error
+ fell_back_from = fell_back_why = None
+ try:
+ self.model.to(self.device).eval()
+ except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
+ if self.device.type != "cpu":
+ # Record what actually went wrong: the reason matters more than the symptom,
+ # and it is the only place the underlying exception is ever surfaced.
+ fell_back_from, fell_back_why = self.device, e
+ self.device = torch.device("cpu")
+ self.dtype = torch.float32
+ self.model.to(self.device).eval()
+ else:
+ raise e
+
+ if fell_back_from is not None:
+ print(
+ "\n[laya] Warning: could not place the model on %s, so it is running on CPU.\n"
+ " Reason: %s\n"
+ " Inference will be roughly 10-15x slower (~200-500 ms rather than ~35 ms).\n"
+ " If this is a newer NVIDIA GPU (Blackwell / RTX 50-series), your PyTorch build\n"
+ " may not support its CUDA architecture:\n"
+ " pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128\n"
+ " See https://pytorch.org/get-started/locally/\n"
+ % (fell_back_from, fell_back_why), flush=True)
+
+ @staticmethod
+ def _to_internal(qdef: Dict) -> Dict:
+ t = qdef["type"]
+ crit = qdef.get("criteria")
+ if t == "choice" and isinstance(crit, list):
+ crit = {c: None for c in crit}
+ ins = qdef["instructions"]
+ if not isinstance(ins, str):
+ ins = json.dumps(ins)
+ return {"t": t, "ins": ins, "crit": crit}
+
+ @torch.no_grad()
+ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
+ """Evaluate typed questions across state in a single, parallel forward pass.
+
+ Args:
+ state: Text string, JSON dict, or conversation turn list.
+ questions: Dictionary mapping question_id -> question definition.
+ - choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}}
+ - score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]}
+ - noul: {"type": "noul", "instructions": "..."}
+
+ Returns:
+ Dictionary with answers, probabilities, calibrated confidence, and token usage.
+ """
+ ids = list(questions.keys())
+ items = []
+ max_len = self.cfg.get("max_len", 512)
+ head_max_len = self.cfg.get("head_max_len", 192)
+
+ for qid in ids:
+ q = self._to_internal(questions[qid])
+ seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len)
+ if len(markers) != len(render_options(q)):
+ raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})
+
+ b = collate_items([items], self.tok.pad_token_id)
+ use_amp = self.device.type == "cuda"
+
+ try:
+ with torch.autocast(device_type=self.device.type, dtype=self.dtype, enabled=use_amp):
+ logits, act = self.model(
+ b["input_ids"].to(self.device),
+ b["attention_mask"].to(self.device),
+ b["marker_pos"].to(self.device),
+ b["marker_mask"].to(self.device),
+ b["qtype"].to(self.device),
+ )
+ except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
+ if self.device.type != "cpu" and ("memory" in str(e).lower() or "cuda" in str(e).lower()):
+ print("Warning: GPU memory exceeded during inference. Falling back to CPU...")
+ self.device = torch.device("cpu")
+ self.dtype = torch.float32
+ self.model.to(self.device)
+ logits, act = self.model(
+ b["input_ids"].to(self.device),
+ b["attention_mask"].to(self.device),
+ b["marker_pos"].to(self.device),
+ b["marker_mask"].to(self.device),
+ b["qtype"].to(self.device),
+ )
+ else:
+ raise e
+
+ logits = logits.float().cpu().numpy()
+ act = torch.softmax(act.float(), -1).cpu().numpy()
+
+ answers = {}
+ n_tokens = int(b["attention_mask"].sum())
+
+ for r, qid in enumerate(ids):
+ q = self._to_internal(questions[qid])
+ k = len(items[r]["markers"])
+ qt = QTYPES[q["t"]]
+ t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])
+ z = logits[r, :k] / t_scale
+ p = np.exp(z - z.max())
+ p = p / p.sum()
+
+ conf_score = round(confidence_from_probs(p, k), 4)
+ ext = {"act_probability": round(float(act[r, 0]), 4)}
+
+ if q["t"] == "choice":
+ keys = list(q["crit"].keys())
+ answers[qid] = {
+ "type": "choice",
+ "choice": keys[int(p.argmax())],
+ "probabilities": {kk: round(float(v), 4) for kk, v in zip(keys, p)},
+ "confidence": conf_score,
+ "action": ext,
+ }
+ elif q["t"] == "score":
+ exp_score = float((np.arange(k) * p).sum())
+ answers[qid] = {
+ "type": "score",
+ "score": round(exp_score, 4),
+ "legend": {str(i): c for i, c in enumerate(q["crit"])},
+ "probabilities": {str(i): round(float(v), 4) for i, v in enumerate(p)},
+ "confidence": conf_score,
+ "action": ext,
+ }
+ else:
+ answers[qid] = {
+ "type": "noul",
+ "noul": round(float(p[1]), 4),
+ "confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),
+ "action": ext,
+ }
+
+ return {
+ "model": "laya-rl-agent",
+ "answers": answers,
+ "usage": {"input_tokens": n_tokens, "output_tokens": 0},
+ }
+
+ predict = system_one
+
+
+RLAgent = Agent
+
+
+def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str] = None,
+ token: Optional[str] = None, subfolder: Optional[str] = None) -> Agent:
+ """Load a Laya agent.
+
+ `subfolder` picks one checkpoint out of a repo that bundles several:
+
+ laya.load("convaiinnovations/laya") # English (repo root)
+ laya.load("convaiinnovations/laya", subfolder="multilingual")
+ """
+ return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder)
diff --git a/studio/backend/vendor/laya/common.py b/studio/backend/vendor/laya/common.py
new file mode 100644
index 00000000000..fb900053da6
--- /dev/null
+++ b/studio/backend/vendor/laya/common.py
@@ -0,0 +1,280 @@
+"""Core model architecture, token sequence construction, and confidence estimation for laya."""
+import json
+import math
+import os
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+QTYPES = {"choice": 0, "score": 1, "noul": 2}
+QTYPE_NAMES = {v: k for k, v in QTYPES.items()}
+
+
+def serialize_state(state: Union[str, dict, list]) -> str:
+ if isinstance(state, str):
+ return state
+ return json.dumps(state, ensure_ascii=False)
+
+
+def render_criterion(value) -> str:
+ """Render one criterion value as text.
+
+ Strings pass through; anything structured (dict, list, number) becomes compact JSON, so a
+ rubric reads as JSON rather than a Python repr. Without this a dict-valued criterion
+ crashed `noul` outright and leaked `{'desc': ...}` into `choice` and `score` prompts.
+ """
+ if isinstance(value, str):
+ return value
+ return json.dumps(value, ensure_ascii=False, separators=(", ", ": "), default=str)
+
+
+def render_options(q: Dict) -> List[str]:
+ """Render option texts in label-index order. Noul is always [false, true]."""
+ t, crit = q["t"], q.get("crit")
+ if t == "choice":
+ # only None/"" mean "no description"; 0 and False are legitimate criterion values
+ return [k if v is None or v == "" else "%s: %s" % (k, render_criterion(v)) for k, v in crit.items()]
+ if t == "score":
+ return ["level %d: %s" % (i, render_criterion(c)) for i, c in enumerate(crit)]
+ crit = crit or {}
+ false_crit, true_crit = crit.get("false"), crit.get("true")
+ return [
+ "false: " + (render_criterion(false_crit) if false_crit not in (None, "") else "no, the statement does not hold"),
+ "true: " + (render_criterion(true_crit) if true_crit not in (None, "") else "yes, the statement holds"),
+ ]
+
+
+def build_sequence(
+ tok,
+ state: Union[str, dict, list],
+ q: Dict,
+ max_len: int = 512,
+ head_max_len: int = 192,
+ option_order: Optional[List[int]] = None,
+ truncate_left: bool = False,
+):
+ """Format: [CLS] <type> instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP]."""
+ mask_tok = tok.mask_token
+ opts = render_options(q)
+ order = option_order if option_order is not None else list(range(len(opts)))
+ ins = str(q["ins"]).replace(mask_tok, " ")
+ head_ids = tok("%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]
+ opt_ids = []
+ for i in order:
+ opt_ids.append(
+ [tok.mask_token_id]
+ + tok(" " + opts[i].replace(mask_tok, " "), add_special_tokens=False)["input_ids"][:48]
+ )
+ opt_budget = head_max_len - sum(len(o) for o in opt_ids)
+ if opt_budget < 16:
+ per = max(4, (head_max_len - 16) // max(1, len(opt_ids)))
+ opt_ids = [o[:per] for o in opt_ids]
+ opt_budget = head_max_len - sum(len(o) for o in opt_ids)
+ head_ids = head_ids[: max(8, opt_budget)]
+ ids = [tok.cls_token_id] + head_ids + [tok.sep_token_id]
+ markers = []
+ for o in opt_ids:
+ markers.append(len(ids))
+ ids.extend(o)
+ ids.append(tok.sep_token_id)
+ room = max(0, max_len - len(ids) - 1)
+ st = tok(serialize_state(state).replace(mask_tok, " "), add_special_tokens=False)["input_ids"]
+ st = st[-room:] if truncate_left else st[:room]
+ ids = ids + st + [tok.sep_token_id]
+ return ids[:max_len], [m for m in markers if m < max_len]
+
+
+class DecisionModel(nn.Module):
+ """Bidirectional transformer encoder backbone + typed decision head."""
+
+ def __init__(self, encoder: nn.Module, head_layers: int = 2, n_act: int = 2, dropout: float = 0.1):
+ super().__init__()
+ self.encoder = encoder
+ d = encoder.config.hidden_size
+ nhead = max(1, d // 64)
+ layer = nn.TransformerEncoderLayer(d, nhead, 4 * d, dropout, batch_first=True, norm_first=True)
+ self.head = nn.TransformerEncoder(layer, head_layers, enable_nested_tensor=False) if head_layers > 0 else None
+ self.type_emb = nn.Embedding(3, d)
+ self.scorer = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, d), nn.GELU(), nn.Linear(d, 1))
+ self.act_head = nn.Sequential(nn.Linear(d + 4, 256), nn.GELU(), nn.Linear(256, n_act))
+ self.register_buffer("temperature", torch.ones(3))
+ self.head_checkpointing = False
+
+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder: bool = False):
+ h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
+ if detach_encoder:
+ h = h.detach()
+ h = h + self.type_emb(qtype)[:, None, :]
+ if self.head is not None:
+ pad = ~attention_mask.bool()
+ for layer in self.head.layers:
+ h = layer(h, src_key_padding_mask=pad)
+ idx = marker_pos.clamp(min=0)[:, :, None].expand(-1, -1, h.size(-1))
+ m = torch.gather(h, 1, idx)
+ logits = self.scorer(m).squeeze(-1).float()
+ logits = logits.masked_fill(~marker_mask, -1e4)
+
+ p = torch.softmax(logits.detach(), -1)
+ k = marker_mask.sum(-1).clamp(min=2).float()
+ ent = -(p * torch.log(p.clamp_min(1e-9))).sum(-1) / torch.log(k)
+ if p.size(-1) >= 2:
+ top2 = p.topk(2, -1).values
+ else:
+ # A single-option question has exactly one marker, so p.topk(2, ...)
+ # has nothing to select for the second slot and raises. The answer
+ # is still well-defined: softmax over one logit is 1.0 regardless of
+ # its value, so pad the missing second entry with 0.0 - that gives
+ # the act head top1 - top2 == 1.0, the same "fully decided" signal
+ # it would see for any other unambiguous top-1-vs-rest gap.
+ top1 = p.topk(1, -1).values
+ top2 = torch.cat([top1, torch.zeros_like(top1)], dim=-1)
+ feats = torch.stack([top2[:, 0], top2[:, 0] - top2[:, 1], ent, k / 255.0], -1)
+ pooled = h[:, 0].float()
+ act_logits = self.act_head(torch.cat([pooled, feats], -1))
+ return logits, act_logits
+
+
+def build_model(cfg: Dict, encoder_dir: Optional[str] = None) -> DecisionModel:
+ from transformers import AutoConfig, AutoModel
+
+ if encoder_dir and os.path.exists(encoder_dir):
+ ecfg = AutoConfig.from_pretrained(encoder_dir)
+ enc = AutoModel.from_config(ecfg, attn_implementation="sdpa")
+ else:
+ enc = AutoModel.from_pretrained(cfg["encoder"], attn_implementation="sdpa")
+ return DecisionModel(enc, cfg.get("head_layers", 2), len(cfg.get("act_costs", {})) + 1)
+
+
+def proper_reward(
+ q: torch.Tensor,
+ target: torch.Tensor,
+ qtype: torch.Tensor,
+ mask: torch.Tensor,
+ w_sph: float = 0.5,
+ w_rps: float = 1.0,
+ log_floor: float = -9.21,
+) -> torch.Tensor:
+ """Strictly proper scoring rule reward: log score + spherical score + ranked probability score.
+
+ q: [..., N, K] reported distributions
+ target: [N, K] (one-hot or soft target distributions)
+ """
+ q = q * mask
+ logq = torch.log(q.clamp_min(1e-12)).clamp_min(log_floor)
+ log_score = (target * logq).sum(-1)
+ sph = (target * q).sum(-1) / q.norm(dim=-1).clamp_min(1e-9)
+ r = log_score + w_sph * sph
+ is_score = (qtype == QTYPES["score"]).float()
+ if is_score.any():
+ k = mask.sum(-1).clamp(min=2).float()
+ cdf_q = torch.cumsum(q, -1)
+ cdf_t = torch.cumsum(target, -1)
+ rps = (((cdf_q - cdf_t) ** 2) * mask).sum(-1) / (k - 1)
+ r = r - w_rps * rps * is_score
+ return r
+
+
+def td_lambda_targets(p_true: torch.Tensor, batch: Dict, lam: float = 1.0) -> torch.Tensor:
+ """TD(lambda) targets for multi-turn conversation trajectories."""
+ target = batch["target"].clone()
+ groups = batch.get("ep_group")
+ if groups is None:
+ return target
+ for g in torch.unique(groups[groups >= 0]).tolist():
+ idx = (groups == g).nonzero(as_tuple=True)[0]
+ idx = idx[torch.argsort(batch["ep_step"][idx])]
+ y = batch["target"][idx[-1], 1]
+ G = y
+ for j in range(len(idx) - 1, -1, -1):
+ if j < len(idx) - 1:
+ G = (1 - lam) * p_true[idx[j + 1]] + lam * G
+ target[idx[j], 0], target[idx[j], 1] = 1 - G, G
+ return target
+
+
+def ece_score(conf: np.ndarray, correct: np.ndarray, bins: int = 15) -> float:
+ """Expected Calibration Error across confidence bins."""
+ if len(conf) == 0:
+ return float("nan")
+ edges = np.linspace(0, 1, bins + 1)
+ e = 0.0
+ for lo, hi in zip(edges[:-1], edges[1:]):
+ sel = (conf > lo) & (conf <= hi)
+ if sel.any():
+ e += sel.mean() * abs(conf[sel].mean() - correct[sel].mean())
+ return float(e)
+
+
+def confidence_from_probs(p: np.ndarray, k: int) -> float:
+ """Normalized Shannon entropy confidence: 1 - H(p) / log(k)."""
+ if k < 2:
+ return 1.0
+ p = p[:k]
+ ent = -(p * np.log(np.clip(p, 1e-12, 1.0))).sum()
+ return float(np.clip(1.0 - ent / math.log(k), 0.0, 1.0))
+
+
+def temp_bucket(qtype: int, k: int) -> str:
+ size = "2" if k <= 2 else "3-5" if k <= 5 else "6-10" if k <= 10 else "11+"
+ return "%s:%s" % (QTYPE_NAMES[int(qtype)], size)
+
+
+# A fitted temperature below 1 sharpens the logits instead of softening them. The shipped
+# `choice:11+` bucket is 0.1006, which multiplies them ~10x: a 0.24 top probability is published as
+# 0.99, so a caller gating on confidence is told a coin flip is a certainty. No honest calibration
+# needs to sharpen this hard, so refuse to apply one that does.
+TEMP_MIN = 0.5
+TEMP_MAX = 5.0
+
+
+def clamp_temperature(t, lo: float = TEMP_MIN, hi: float = TEMP_MAX) -> float:
+ """A usable temperature: `t` confined to [lo, hi], falling back to 1.0 if it is not a number."""
+ try:
+ t = float(t)
+ except (TypeError, ValueError):
+ return 1.0
+ if t != t or t in (float("inf"), float("-inf")): # NaN / inf
+ return 1.0
+ return min(hi, max(lo, t))
+
+
+def amp_dtype(name: Optional[str]) -> torch.dtype:
+ return torch.bfloat16 if name == "bf16" else torch.float16
+
+
+def collate_items(batch, pad_id: int):
+ items = [it for group in batch for it in group]
+ if not items:
+ return None
+ n, L = len(items), max(len(it["ids"]) for it in items)
+ kmax = max(len(it["markers"]) for it in items)
+ ids = torch.full((n, L), pad_id, dtype=torch.long)
+ att = torch.zeros((n, L), dtype=torch.long)
+ mpos = torch.zeros((n, kmax), dtype=torch.long)
+ mmask = torch.zeros((n, kmax), dtype=torch.bool)
+ has_target = any("target" in it for it in items)
+ target = torch.zeros((n, kmax), dtype=torch.float32) if has_target else None
+
+ for i, it in enumerate(items):
+ ids[i, : len(it["ids"])] = torch.tensor(it["ids"])
+ att[i, : len(it["ids"])] = 1
+ k = len(it["markers"])
+ mpos[i, :k] = torch.tensor(it["markers"])
+ mmask[i, :k] = True
+ if has_target and "target" in it:
+ target[i, : len(it["target"])] = torch.tensor(it["target"], dtype=torch.float32)
+
+ res = {
+ "input_ids": ids,
+ "attention_mask": att,
+ "marker_pos": mpos,
+ "marker_mask": mmask,
+ "qtype": torch.tensor([it["qtype"] for it in items]),
+ "label": torch.tensor([it.get("label", -1) for it in items]),
+ "meta": [{k: it[k] for k in it if k not in ("ids", "markers", "target")} for it in items],
+ }
+ if target is not None:
+ res["target"] = target
+ return res
diff --git a/studio/backend/vendor/laya/email.py b/studio/backend/vendor/laya/email.py
new file mode 100644
index 00000000000..5512741739f
--- /dev/null
+++ b/studio/backend/vendor/laya/email.py
@@ -0,0 +1,104 @@
+"""Email utilities for cleaning and structuring email inputs in laya."""
+import re
+from typing import Dict, Optional
+
+_QUOTE_HEADERS = [
+ re.compile(r"^\s*On .{0,300}wrote:\s*$", re.I),
+ re.compile(r"^\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}", re.I),
+ re.compile(r"^\s*_{8,}\s*$"),
+ re.compile(r"^\s*From:\s.+$", re.I),
+]
+_SIGNATURE_MARKERS = [
+ re.compile(r"^\s*--\s*$"),
+ re.compile(r"^\s*(best|kind|warm|many thanks|thanks|thank you|regards|cheers|sincerely)[\w ,!.]*$", re.I),
+ re.compile(r"^\s*sent from my (iphone|android|mobile|ipad)", re.I),
+]
+_DISCLAIMER = re.compile(
+ r"(confidential|intended (solely )?for the (use of the )?(named )?(addressee|recipient)|"
+ r"if you (have )?received this (e-?mail|message) in error)",
+ re.I,
+)
+_SENTENCE = re.compile(r"(?<=[.!?])\s+")
+
+
+def _strip_disclaimer(paragraph: str) -> str:
+ """Drop boilerplate disclaimer text from one paragraph.
+
+ A paragraph is dropped whole only when *every* sentence in it is boilerplate; otherwise only
+ the boilerplate sentences go. A footer that runs on without a blank line used to take the
+ sender's actual request with it, which is worse than leaving one boilerplate line behind.
+ """
+ if not _DISCLAIMER.search(paragraph):
+ return paragraph # nothing to do: keep the original line structure
+ parts = [p.strip() for p in _SENTENCE.split(paragraph) if p.strip()]
+ return " ".join(p for p in parts if not _DISCLAIMER.search(p))
+
+
+def clean_email_body(body: str, max_chars: int = 3000) -> str:
+ """Remove quoted email history, signatures and disclaimers to keep input focused."""
+ text = (body or "").replace("\r\n", "\n").replace("\r", "\n").replace("\\n", "\n")
+ lines = []
+ for line in text.split("\n"):
+ if any(p.match(line) for p in _QUOTE_HEADERS) and lines:
+ break
+ if line.lstrip().startswith(">"):
+ continue
+ lines.append(line.rstrip())
+ cut = len(lines)
+ for i in range(max(1, min(int(len(lines) * 0.6), len(lines) - 8)), len(lines)):
+ if len(lines[i].strip()) <= 40 and any(p.match(lines[i]) for p in _SIGNATURE_MARKERS):
+ cut = i
+ break
+ lines = lines[:cut]
+ paragraphs = [_strip_disclaimer(p) for p in re.split(r"\n\s*\n", "\n".join(lines))]
+ text = re.sub(r"[ \t]+", " ", "\n\n".join(p.strip() for p in paragraphs if p.strip()))
+ return text[:max_chars]
+
+
+def email_state(subject: str, body: str, sender: Optional[str] = None, clean: bool = True, **extra) -> Dict:
+ """Construct a clean state dictionary for email classification."""
+ state = {
+ "subject": (subject or "").strip(),
+ "body": clean_email_body(body) if clean else (body or ""),
+ }
+ if sender:
+ state["from"] = sender
+ state.update({k: v for k, v in extra.items() if v is not None})
+ return state
+
+
+def email_questions(categories: Optional[Dict[str, str]] = None) -> Dict:
+ """Standard pre-built questions for email triage."""
+ categories = categories or {
+ "billing": "invoices, payments, refunds",
+ "technical": "bugs, outages, integrations",
+ "sales": "pricing, demos, new purchases",
+ "security": "phishing, scams, account compromise",
+ "hr": "hiring, leave, payroll",
+ "other": "none of the above",
+ }
+ return {
+ "category": {
+ "type": "choice",
+ "instructions": "Which team should handle the email in `body`?",
+ "criteria": categories,
+ },
+ "is_spam": {
+ "type": "noul",
+ "instructions": "Is this email unsolicited spam or bulk marketing?",
+ },
+ "is_phishing": {
+ "type": "noul",
+ "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?",
+ "criteria": {"true": "phishing, scam, or fraud", "false": "a legitimate email"},
+ },
+ "urgency": {
+ "type": "score",
+ "instructions": "How urgent is the request in `body`?",
+ "criteria": ["no time pressure", "needs attention soon", "blocking issue or hard deadline"],
+ },
+ "needs_reply": {
+ "type": "noul",
+ "instructions": "Does the sender expect a reply?",
+ },
+ }
diff --git a/studio/backend/vendor/laya/lang.py b/studio/backend/vendor/laya/lang.py
new file mode 100644
index 00000000000..0d1ed7bc014
--- /dev/null
+++ b/studio/backend/vendor/laya/lang.py
@@ -0,0 +1,242 @@
+"""Dependency-free language/script detection used to route between Laya checkpoints.
+
+Routing only needs one decision: *is this English Latin text, or is it something the English
+checkpoint cannot read?* Benchmarks on MASSIVE (14 languages) showed the English checkpoint
+collapsing to near-random on non-Latin scripts (Hindi 0.100, Korean 0.103, Swahili 0.103,
+Tamil 0.113 at 20 options, where random is 0.050), while holding up far better on Latin-script
+languages (French 0.487, Spanish 0.480). So the signal that matters most is *script*, and the
+secondary signal is whether Latin text is English.
+
+Script detection is exact. The Latin-script language guess is a stopword/diacritic heuristic and
+is explicitly best-effort: pass an explicit model or `lang=` when you already know the language.
+"""
+import re
+from typing import Dict, List, Optional, Union
+
+# Unicode blocks that the English (ModernBERT-large, 50k English BPE) checkpoint cannot read.
+_SCRIPT_RANGES = [
+ ("greek", ((0x0370, 0x03FF), (0x1F00, 0x1FFF))),
+ ("cyrillic", ((0x0400, 0x052F), (0x2DE0, 0x2DFF), (0xA640, 0xA69F))),
+ ("armenian", ((0x0530, 0x058F),)),
+ ("hebrew", ((0x0590, 0x05FF),)),
+ ("arabic", ((0x0600, 0x06FF), (0x0750, 0x077F), (0x08A0, 0x08FF), (0xFB50, 0xFDFF), (0xFE70, 0xFEFF))),
+ ("devanagari", ((0x0900, 0x097F), (0xA8E0, 0xA8FF))),
+ ("bengali", ((0x0980, 0x09FF),)),
+ ("gurmukhi", ((0x0A00, 0x0A7F),)),
+ ("gujarati", ((0x0A80, 0x0AFF),)),
+ ("oriya", ((0x0B00, 0x0B7F),)),
+ ("tamil", ((0x0B80, 0x0BFF),)),
+ ("telugu", ((0x0C00, 0x0C7F),)),
+ ("kannada", ((0x0C80, 0x0CFF),)),
+ ("malayalam", ((0x0D00, 0x0D7F),)),
+ ("sinhala", ((0x0D80, 0x0DFF),)),
+ ("thai", ((0x0E00, 0x0E7F),)),
+ ("lao", ((0x0E80, 0x0EFF),)),
+ ("tibetan", ((0x0F00, 0x0FFF),)),
+ ("myanmar", ((0x1000, 0x109F),)),
+ ("georgian", ((0x10A0, 0x10FF),)),
+ ("ethiopic", ((0x1200, 0x137F),)),
+ ("khmer", ((0x1780, 0x17FF),)),
+ ("hangul", ((0x1100, 0x11FF), (0x3130, 0x318F), (0xAC00, 0xD7AF))),
+ ("kana", ((0x3040, 0x309F), (0x30A0, 0x30FF), (0x31F0, 0x31FF))),
+ ("han", ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF))),
+]
+
+# Function words. Latin-script languages overlap heavily (de/la/le/un/e/que), so each hit is
+# weighted and a margin is required before calling something non-English.
+_STOP = {
+ "en": {"the", "and", "is", "are", "was", "were", "to", "of", "in", "for", "with", "that",
+ "this", "it", "you", "have", "has", "not", "but", "on", "at", "be", "as", "from",
+ "will", "can", "would", "there", "their", "what", "which", "please", "we", "i"},
+ "fr": {"le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "avec", "sur",
+ "pas", "plus", "nous", "vous", "être", "cette", "mais", "sont", "ont", "aux", "ce"},
+ "de": {"der", "die", "das", "und", "ist", "ein", "eine", "den", "dem", "nicht", "mit", "für",
+ "auf", "von", "zu", "sich", "auch", "werden", "wurde", "haben", "sind", "oder", "aber"},
+ "es": {"el", "los", "las", "que", "por", "con", "para", "una", "es", "se", "del", "como",
+ "pero", "son", "está", "este", "esta", "todo", "más", "muy", "hay", "sus"},
+ "pt": {"os", "as", "que", "em", "um", "uma", "para", "com", "não", "é", "se", "do", "da",
+ "dos", "das", "mas", "são", "está", "este", "esta", "muito", "pelo", "pela"},
+ "it": {"il", "lo", "gli", "che", "di", "per", "con", "non", "è", "si", "del", "della", "sono",
+ "questo", "questa", "anche", "come", "più", "sono", "nella", "alla"},
+ "nl": {"het", "een", "van", "is", "op", "te", "dat", "niet", "met", "voor", "zijn", "aan",
+ "door", "maar", "ook", "worden", "deze", "naar", "wordt"},
+ # Romanian words that its Romance neighbours do not share, so adding `ro` cannot steal a
+ # French/Spanish/Italian/Portuguese state: `la`, `o`, `un`, `de`, `pe`, `ca` are deliberately
+ # left out for that reason, and the diacritic signal below carries the rest.
+ "ro": {"și", "să", "este", "sunt", "care", "pentru", "din", "dar", "după", "până", "fără",
+ "ale", "lui", "în", "fost", "acum", "vreau", "trebuie", "foarte", "acest", "această",
+ "acesta", "aceasta", "mi", "ți", "vă", "nu"},
+}
+# Letters that ordinary English does not use. This is the signal that catches a Latin-script
+# language we hold no stopwords for at all (Romanian, Polish, Czech, Turkish, Baltic, ...),
+# which is the difference between routing it to the multilingual checkpoint and silently
+# handing it to the English one.
+_NON_EN_DIACRITICS = set(
+ "àâäãáåçéèêëíìîïñóòôöõøúùûüýÿßæœ" # Western European
+ "ăâîșțşţ" # Romanian
+ "ąćęłńśźż" # Polish
+ "čďěňřšťůž" # Czech / Slovak
+ "őű" # Hungarian
+ "ğı" # Turkish (text is lowercased before matching)
+ "āēģīķļņūž" # Baltic
+ "đ" # Serbo-Croatian / Vietnamese
+)
+_WORD = re.compile(r"[^\W\d_]+", re.UNICODE)
+
+
+def _iter_text(state: Union[str, dict, list, None], _depth: int = 0) -> List[str]:
+ """Collect the string leaves of a state (str / dict / list), so detection sees real content."""
+ if _depth > 6 or state is None:
+ return []
+ if isinstance(state, str):
+ return [state]
+ if isinstance(state, dict):
+ out = []
+ for v in state.values():
+ out.extend(_iter_text(v, _depth + 1))
+ return out
+ if isinstance(state, (list, tuple)):
+ out = []
+ for v in state:
+ out.extend(_iter_text(v, _depth + 1))
+ return out
+ return []
+
+
+def state_text(state: Union[str, dict, list, None], max_chars: int = 4000) -> str:
+ """Flatten a state into the text used for detection (keys are ignored: they are usually English)."""
+ return " ".join(_iter_text(state))[:max_chars]
+
+
+def detect_script(text: str) -> str:
+ """Dominant script of `text`: 'latin', 'han', 'devanagari', ... or 'unknown' if there are no letters."""
+ counts: Dict[str, int] = {}
+ latin = 0
+ for ch in text:
+ if not ch.isalpha():
+ continue
+ cp = ord(ch)
+ if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF: # Latin + Latin Extended Additional
+ latin += 1
+ continue
+ for name, ranges in _SCRIPT_RANGES:
+ if any(lo <= cp <= hi for lo, hi in ranges):
+ counts[name] = counts.get(name, 0) + 1
+ break
+ counts["latin"] = latin
+ total = sum(counts.values())
+ if total == 0:
+ return "unknown"
+ return max(counts.items(), key=lambda kv: kv[1])[0]
+
+
+def script_profile(text: str) -> Dict[str, float]:
+ """Fraction of alphabetic characters belonging to each detected script."""
+ counts: Dict[str, int] = {"latin": 0}
+ for ch in text:
+ if not ch.isalpha():
+ continue
+ cp = ord(ch)
+ if cp < 0x0250 or 0x1E00 <= cp <= 0x1EFF:
+ counts["latin"] += 1
+ continue
+ for name, ranges in _SCRIPT_RANGES:
+ if any(lo <= cp <= hi for lo, hi in ranges):
+ counts[name] = counts.get(name, 0) + 1
+ break
+ total = sum(counts.values())
+ if not total:
+ return {}
+ return {k: v / total for k, v in counts.items() if v}
+
+
+# A diacritic rate above this is taken as evidence the text is not English, even when no
+# stopword list matches it.
+NON_EN_DIACRITIC_RATE = 0.02
+
+
+def latin_profile(text: str) -> Dict[str, object]:
+ """Evidence behind the Latin-script language guess.
+
+ Returns `language` (may be None when undecided), `english_hits`, `diacritic_rate` and
+ `looks_non_english`. `analyse` needs the evidence and not just the verdict, because
+ "undecided" and "English" are different answers and only one of them is safe to send to the
+ English checkpoint.
+ """
+ words = [w.lower() for w in _WORD.findall(text)]
+ lowered = text.lower()
+ diac = sum(1 for ch in lowered if ch in _NON_EN_DIACRITICS)
+ diac_rate = diac / max(1, len(lowered))
+ non_english = diac_rate >= NON_EN_DIACRITIC_RATE
+ if len(words) < 4:
+ return {"language": None, "english_hits": 0, "diacritic_rate": diac_rate,
+ "looks_non_english": non_english}
+
+ scores = {lg: sum(1 for w in words if w in sw) for lg, sw in _STOP.items()}
+ en = scores.get("en", 0)
+ best_lg, best = max(((lg, s) for lg, s in scores.items() if lg != "en"),
+ key=lambda kv: kv[1], default=(None, 0))
+ # No stopword hit for any non-English language is no evidence for a *particular* one. Naming
+ # the winner of a 0-0 tie invented a language (Romanian text was reported as French), so stay
+ # undecided and let the diacritic rate speak.
+ if best == 0:
+ best_lg = None
+
+ lang = None
+ if best_lg and best >= max(2, en + 2):
+ # a non-English language needs a clear margin over English function words
+ lang = best_lg
+ elif best_lg and non_english and best >= max(2, en):
+ # Needs two hits here too. One shared function word ("para" in Turkish text) named Spanish
+ # on the strength of the diacritics alone, which is a guess dressed as a detection.
+ lang = best_lg
+ elif en and not non_english:
+ lang = "en"
+ return {"language": lang, "english_hits": en, "diacritic_rate": diac_rate,
+ "looks_non_english": non_english}
+
+
+def guess_latin_language(text: str) -> Optional[str]:
+ """Best-effort language code for Latin-script text, or None when undecided.
+
+ Scores function-word hits per language and requires the winner to beat English by a margin,
+ so ordinary English is never misrouted. Short inputs usually return None on purpose.
+ """
+ return latin_profile(text)["language"]
+
+
+def analyse(state: Union[str, dict, list, None]) -> Dict[str, object]:
+ """Full detection result for a state.
+
+ Returns `script`, `script_profile`, `language` (best effort, may be None),
+ `is_english` and `non_latin_fraction`.
+ """
+ text = state_text(state)
+ prof = script_profile(text)
+ script = detect_script(text)
+ non_latin = round(1.0 - prof.get("latin", 0.0), 4) if prof else 0.0
+ if script == "unknown":
+ return {"script": "unknown", "script_profile": prof, "language": None,
+ "is_english": True, "language_undecided": True, "diacritic_rate": 0.0,
+ "non_latin_fraction": 0.0}
+ if script != "latin":
+ return {"script": script, "script_profile": prof, "language": None,
+ "is_english": False, "language_undecided": True, "diacritic_rate": 0.0,
+ "non_latin_fraction": non_latin}
+ prof_lat = latin_profile(text)
+ lang = prof_lat["language"]
+ # Undecided is not English. Treating it as English sent every Latin-script language we hold no
+ # stopwords for to the checkpoint that cannot read it, silently. When nothing identifies the
+ # language, non-English letters are enough to prefer the multilingual checkpoint; text with no
+ # such letters (including short English) still goes to the English one.
+ undecided = lang is None
+ english = lang == "en" or (undecided and not prof_lat["looks_non_english"])
+ return {"script": "latin", "script_profile": prof, "language": lang,
+ "is_english": english, "language_undecided": undecided,
+ "diacritic_rate": round(float(prof_lat["diacritic_rate"]), 4),
+ "non_latin_fraction": non_latin}
+
+
+def is_english(state: Union[str, dict, list, None]) -> bool:
+ """True when the English checkpoint can be expected to read this state."""
+ return bool(analyse(state)["is_english"])
diff --git a/studio/backend/vendor/laya/presets.py b/studio/backend/vendor/laya/presets.py
new file mode 100644
index 00000000000..ea2efd385e8
--- /dev/null
+++ b/studio/backend/vendor/laya/presets.py
@@ -0,0 +1,187 @@
+"""Ready-to-use question presets for common production decision workflows."""
+from typing import Dict, Optional
+
+
+def triage_questions() -> Dict:
+ """Preset questions for customer support ticket triage."""
+ return {
+ "intent": {
+ "type": "choice",
+ "instructions": "What does the customer want in `message`?",
+ "criteria": {
+ "refund": "money returned or a duplicate charge reversed",
+ "technical_help": "a bug, outage or integration problem",
+ "billing_question": "a question about an invoice, plan or payment method",
+ "information": "general information, pricing or how-to",
+ "cancellation": "wants to cancel or downgrade",
+ "other": "none of the other options fits",
+ },
+ },
+ "is_urgent": {
+ "type": "noul",
+ "instructions": "Does `message` communicate time pressure or a deadline?",
+ },
+ "frustration": {
+ "type": "score",
+ "instructions": "How frustrated does the customer sound in `message`?",
+ "criteria": [
+ "calm and neutral",
+ "concerned but civil",
+ "clearly annoyed",
+ "very angry or using strong language",
+ ],
+ },
+ "refund_requested": {
+ "type": "noul",
+ "instructions": "Does the customer ask for money back?",
+ },
+ "churn_risk": {
+ "type": "noul",
+ "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?",
+ },
+ }
+
+
+def email_questions(categories: Optional[Dict[str, str]] = None) -> Dict:
+ """Preset questions for inbound email triage and threat filtering."""
+ categories = categories or {
+ "billing": "invoices, payments, refunds",
+ "technical": "bugs, outages, integrations",
+ "sales": "pricing, demos, new purchases",
+ "security": "phishing, scams, account compromise",
+ "hr": "hiring, leave, payroll",
+ "other": "none of the above",
+ }
+ return {
+ "category": {
+ "type": "choice",
+ "instructions": "Which team should handle the email in `body`?",
+ "criteria": categories,
+ },
+ "is_spam": {
+ "type": "noul",
+ "instructions": "Is this email unsolicited spam or bulk marketing?",
+ },
+ "is_phishing": {
+ "type": "noul",
+ "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?",
+ "criteria": {"true": "phishing, scam, or fraud", "false": "a legitimate email"},
+ },
+ "urgency": {
+ "type": "score",
+ "instructions": "How urgent is the request in `body`?",
+ "criteria": ["no time pressure", "needs attention soon", "blocking issue or hard deadline"],
+ },
+ "needs_reply": {
+ "type": "noul",
+ "instructions": "Does the sender expect a reply?",
+ },
+ }
+
+
+def guard_questions() -> Dict:
+ """Preset questions for real-time LLM input guardrails."""
+ return {
+ "jailbreak": {
+ "type": "noul",
+ "instructions": "Does `prompt` try to make an AI assistant ignore its rules, policies or system instructions?",
+ },
+ "prompt_injection": {
+ "type": "noul",
+ "instructions": "Does `prompt` contain instructions aimed at the AI system rather than a genuine user request?",
+ },
+ "sensitive_data": {
+ "type": "noul",
+ "instructions": "Does `prompt` contain credentials, personal data or other sensitive information?",
+ },
+ "harm_severity": {
+ "type": "score",
+ "instructions": "How much harm would complying with `prompt` cause?",
+ "criteria": [
+ "none: ordinary request",
+ "minor: mildly inappropriate",
+ "serious: unsafe advice or abuse",
+ "severe: dangerous or illegal",
+ ],
+ },
+ "topic": {
+ "type": "choice",
+ "instructions": "What is `prompt` about?",
+ "criteria": {
+ "product_support": None,
+ "coding": None,
+ "general_knowledge": None,
+ "personal_advice": None,
+ "security_testing": None,
+ "other": None,
+ },
+ },
+ }
+
+
+def moderation_questions() -> Dict:
+ """Preset questions for content safety and moderation."""
+ return {
+ "toxic": {
+ "type": "noul",
+ "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?",
+ },
+ "harassment": {
+ "type": "noul",
+ "instructions": "Does `post` target or harass a specific person?",
+ },
+ "threat": {
+ "type": "noul",
+ "instructions": "Does `post` threaten violence, harm or intimidation?",
+ },
+ "spam": {
+ "type": "noul",
+ "instructions": "Is `post` spam or advertising?",
+ },
+ "severity": {
+ "type": "score",
+ "instructions": "How severe is any rule-breaking in `post`?",
+ "criteria": [
+ "no rule-breaking: ordinary on-topic post",
+ "mild: rude tone or off-topic, no target",
+ "clear violation: insults, harassment or spam aimed at someone",
+ "severe: threats, hate speech or calls for violence",
+ ],
+ },
+ }
+
+
+def router_questions() -> Dict:
+ """Preset questions for intelligent model routing."""
+ return {
+ "difficulty": {
+ "type": "score",
+ "instructions": "How hard is `request` for a language model?",
+ "criteria": [
+ "trivial: a lookup or one-liner",
+ "easy: short answer, no reasoning",
+ "moderate: several steps",
+ "hard: long multi-step reasoning or specialist knowledge",
+ ],
+ },
+ "domain": {
+ "type": "choice",
+ "instructions": "What domain does `request` belong to?",
+ "criteria": {
+ "code": "software engineering, programming, refactoring, architecture, debugging",
+ "math_or_logic": "mathematics, logic puzzles, proofs, complex calculation",
+ "writing": "creative writing, essays, emails, blog posts, copywriting",
+ "factual_lookup": "facts, definitions, trivia, history",
+ "data_analysis": "statistics, SQL, data manipulation, metrics",
+ "chitchat": "casual conversation, greetings, small talk",
+ },
+ },
+ "needs_tools": {
+ "type": "noul",
+ "instructions": "Does answering `request` require external tools, search or private data?",
+ },
+ "is_sensitive": {
+ "type": "noul",
+ "instructions": "Does `request` involve money, legal, medical or safety consequences?",
+ },
+ }
diff --git a/studio/backend/vendor/laya/router.py b/studio/backend/vendor/laya/router.py
new file mode 100644
index 00000000000..6577bec9e00
--- /dev/null
+++ b/studio/backend/vendor/laya/router.py
@@ -0,0 +1,336 @@
+"""Route a request to the Laya checkpoint best suited to it.
+
+Three checkpoints, measured on a shared benchmark (17,416 questions, one T4, identical questions
+per model -- see the repository's benchmark notebook):
+
+ english convaiinnovations/laya 421M ModernBERT-large, 512 tokens
+ multilingual convaiinnovations/laya-multilingual 322M mmBERT-base, 1024 tokens, 100+ langs
+ typed-decisions convaiinnovations/laya-typed-decisions 421M ModernBERT-large, 1024 tokens,
+ fine-tuned on the typed-decisions
+ workflows
+
+Why routing is worth it -- accuracy by language family:
+
+ english multilingual
+ MASSIVE intent en 0.783 0.657 <- English checkpoint wins
+ MASSIVE intent non-en 0.306 0.451
+ XNLI en 0.860 0.843
+ XNLI non-en 0.521 0.731 <- +21 points for multilingual
+ English suites 0.684 0.619
+
+The English checkpoint does not gently degrade off English, it collapses: on 20-option MASSIVE
+intent it scores 0.100 on Hindi and 0.103 on Korean, against 0.050 for random guessing -- and it
+reports high confidence while doing so (ECE 0.855 on Hindi). Script detection is therefore the
+primary routing signal.
+
+`typed-decisions` is never selected automatically unless you opt in with
+`auto_task_detection=True` or pass `task="typed_decisions"`: it is fine-tuned on four specific
+synthetic workflows and should not be a silent default.
+"""
+import os
+import threading
+from typing import Any, Dict, List, Optional, Union
+
+from .lang import analyse
+
+# The hub repo bundles all three checkpoints; only the requested subfolder is downloaded.
+BUNDLE_REPO = "convaiinnovations/laya"
+DEFAULT_MODELS = {
+ "english": (BUNDLE_REPO, None),
+ "multilingual": (BUNDLE_REPO, "multilingual"),
+ "typed-decisions": (BUNDLE_REPO, "typed-decisions"),
+}
+
+# The same checkpoints also live in their own repos, for anyone who prefers them.
+STANDALONE_MODELS = {
+ "english": "convaiinnovations/laya",
+ "multilingual": "convaiinnovations/laya-multilingual",
+ "typed-decisions": "convaiinnovations/laya-typed-decisions",
+}
+
+
+def _repo_str(spec):
+ """Human-readable id for a model spec: 'repo' or 'repo/subfolder'."""
+ repo, sub = _split(spec)
+ return "%s/%s" % (repo, sub) if sub else repo
+
+
+def _split(spec):
+ """Normalise a model spec to (repo_or_path, subfolder)."""
+ if isinstance(spec, (tuple, list)):
+ repo, sub = (list(spec) + [None])[:2]
+ return repo, sub
+ return spec, None
+
+# Aliases people are likely to type.
+_ALIASES = {
+ "en": "english", "laya": "english", "default": "english",
+ "multi": "multilingual", "ml": "multilingual", "laya-multilingual": "multilingual",
+ "typed": "typed-decisions", "typed_decisions": "typed-decisions",
+ "laya-typed-decisions": "typed-decisions", "decisions": "typed-decisions",
+}
+
+# Question-id signatures of the four typed-decisions workflows, used only when
+# auto_task_detection is enabled.
+_TYPED_DECISION_WORKFLOWS = {
+ "agent_trace_observability": {"action", "needs_review", "outcome", "risk", "urgency"},
+ "customer_service": {"action", "category", "churn_risk", "needs_human", "urgency"},
+ "invoice_processing": {"discrepancy_severity", "disposition", "duplicate", "matches_order", "urgency"},
+ "security_incidents": {"credential_compromise", "disposition", "severity", "true_positive", "urgency"},
+}
+
+
+class RouteDecision(dict):
+ """The routing outcome: which model, why, and what was detected.
+
+ Behaves as a dict so it serialises straight into an API response.
+ """
+
+ @property
+ def model(self) -> str:
+ return self["model"]
+
+ @property
+ def reason(self) -> str:
+ return self["reason"]
+
+ def __repr__(self):
+ return "RouteDecision(model=%r, reason=%r)" % (self["model"], self["reason"])
+
+
+def normalise_name(name: str) -> str:
+ key = str(name).strip().lower()
+ key = _ALIASES.get(key, key)
+ if key not in DEFAULT_MODELS:
+ raise ValueError("unknown model %r; choose one of %s (or an alias: %s)"
+ % (name, sorted(DEFAULT_MODELS), sorted(_ALIASES)))
+ return key
+
+
+def match_typed_decisions_workflow(questions: Dict[str, Any]) -> Optional[str]:
+ """Name of the typed-decisions workflow whose question ids these are, else None.
+
+ Requires an exact id-set match, so an unrelated schema that happens to contain 'urgency'
+ is never captured.
+ """
+ ids = set(questions or {})
+ for wf, sig in _TYPED_DECISION_WORKFLOWS.items():
+ if ids == sig:
+ return wf
+ return None
+
+
+class Router:
+ """Lazily loads Laya checkpoints and sends each request to the right one.
+
+ from laya import Router
+
+ r = Router()
+ r.predict({"message": "Mein Konto wurde zweimal belastet"}, questions) # -> multilingual
+ r.predict({"message": "I was charged twice"}, questions) # -> english
+ r.predict(state, questions, model="typed-decisions") # explicit
+
+ Models are downloaded and built on first use. `max_loaded` caps how many stay resident
+ (least-recently-used is evicted), because all three together are ~1.16B parameters.
+
+ For a server or a demo, preload instead: a cold load costs seconds, while detection costs
+ microseconds, so anything that alternates languages at `max_loaded=1` reloads on every
+ request.
+
+ r = Router(preload=True) # all three resident, routing is free
+ r = Router(preload=True, device="cuda")
+ r.preload(["english", "multilingual"]) # or just the two you serve
+ """
+
+ def __init__(
+ self,
+ models: Optional[Dict[str, str]] = None,
+ device: Optional[str] = None,
+ token: Optional[str] = None,
+ max_loaded: int = 1,
+ default: str = "english",
+ auto_task_detection: bool = False,
+ standalone_repos: bool = False,
+ preload: bool = False,
+ ):
+ self.models = dict(STANDALONE_MODELS if standalone_repos else DEFAULT_MODELS)
+ if models:
+ self.models.update({normalise_name(k): v for k, v in models.items()})
+ self.device = device
+ self.token = token or os.environ.get("HF_TOKEN")
+ self.max_loaded = max(1, int(max_loaded))
+ self.default = normalise_name(default)
+ self.auto_task_detection = bool(auto_task_detection)
+ self._agents: Dict[str, Any] = {}
+ self._order: List[str] = [] # least-recently-used first
+ # Re-entrant lock guarding model lifecycle (load/unload/attach/preload) and the
+ # LRU bookkeeping. RLock so the public methods can call the private `_touch`/`_evict`
+ # helpers without deadlocking. Inference (`Agent.system_one`) is deliberately left
+ # outside the lock so concurrent predictions share a checkpoint without serialising.
+ self._lock = threading.RLock()
+ if preload:
+ self.preload()
+
+ # ------------------------------------------------------------------ loading
+ def load(self, name: str):
+ """Return the Agent for `name`, downloading and building it on first use.
+
+ Concurrent callers share a single Agent instead of building duplicates.
+ """
+ key = normalise_name(name)
+ with self._lock:
+ if key in self._agents:
+ self._touch(key)
+ return self._agents[key]
+ from .agent import Agent
+ repo, sub = _split(self.models[key])
+ agent = Agent(repo, device=self.device, token=self.token, subfolder=sub)
+ self._agents[key] = agent
+ self._order.append(key)
+ self._evict()
+ return agent
+
+ def _touch(self, key: str):
+ with self._lock:
+ if key in self._order:
+ self._order.remove(key)
+ self._order.append(key)
+
+ def _evict(self):
+ with self._lock:
+ while len(self._order) > self.max_loaded:
+ victim = self._order.pop(0)
+ self._agents.pop(victim, None)
+ if len(self._order) < len(self._agents): # keep the two views consistent
+ for k in list(self._agents):
+ if k not in self._order:
+ self._agents.pop(k, None)
+
+ def attach(self, name: str, agent: Any):
+ """Register an already-built Agent under `name` instead of loading a second copy.
+
+ Useful when the process has a checkpoint loaded for other reasons: a demo that already
+ built `convaiinnovations/laya` can hand it to the router rather than pay for -- and hold
+ in memory -- a duplicate 421M parameters.
+ """
+ key = normalise_name(name)
+ with self._lock:
+ self._agents[key] = agent
+ self._touch(key)
+ self.max_loaded = max(self.max_loaded, len(self._agents))
+ return agent
+
+ def preload(self, names: Optional[List[str]] = None):
+ """Download and build checkpoints up front so no request ever pays a model load.
+
+ A cold load costs seconds; language detection costs microseconds. With every
+ checkpoint resident, routing is effectively free -- which is what you want in a
+ server or a demo. `max_loaded` is raised to fit whatever is preloaded, otherwise
+ the LRU would immediately evict what this just built.
+ """
+ names = [normalise_name(n) for n in (names or list(self.models))]
+ with self._lock:
+ self.max_loaded = max(self.max_loaded, len(names), len(self._agents))
+ for n in names:
+ if n not in self._agents: # an attached agent is already built
+ self.load(n)
+ return self
+
+ def unload(self, name: Optional[str] = None):
+ """Free one model, or all of them."""
+ with self._lock:
+ if name is None:
+ self._agents.clear()
+ self._order.clear()
+ else:
+ key = normalise_name(name)
+ self._agents.pop(key, None)
+ if key in self._order:
+ self._order.remove(key)
+
+ @property
+ def loaded(self) -> List[str]:
+ with self._lock:
+ return list(self._order)
+
+ # ------------------------------------------------------------------ routing
+ def route(
+ self,
+ state: Union[str, dict, list, None],
+ questions: Optional[Dict[str, Any]] = None,
+ model: Optional[str] = None,
+ task: Optional[str] = None,
+ lang: Optional[str] = None,
+ ) -> RouteDecision:
+ """Decide which checkpoint to use, without loading or running anything.
+
+ Precedence: explicit `model` > explicit `task` > detected workflow (opt-in) >
+ explicit `lang` > detected script/language > default.
+ """
+ if model is not None:
+ key = normalise_name(model)
+ return RouteDecision(model=key, repo=_repo_str(self.models[key]), reason="explicit model=%r" % model,
+ detection=None, workflow=None)
+
+ if task is not None:
+ key = normalise_name("typed-decisions" if str(task).lower().replace("-", "_") == "typed_decisions" else task)
+ return RouteDecision(model=key, repo=_repo_str(self.models[key]), reason="explicit task=%r" % task,
+ detection=None, workflow=None)
+
+ workflow = match_typed_decisions_workflow(questions or {})
+ if workflow and self.auto_task_detection:
+ return RouteDecision(model="typed-decisions", repo=self.models["typed-decisions"],
+ reason="question ids match the %r typed-decisions workflow" % workflow,
+ detection=None, workflow=workflow)
+
+ if lang is not None:
+ key = "english" if str(lang).lower().split("-")[0] in ("en", "eng", "english") else "multilingual"
+ return RouteDecision(model=key, repo=_repo_str(self.models[key]), reason="explicit lang=%r" % lang,
+ detection=None, workflow=workflow)
+
+ det = analyse(state)
+ if det["script"] == "unknown":
+ key = self.default
+ reason = "no letters detected in state; using default (%s)" % key
+ elif det["script"] != "latin":
+ key = "multilingual"
+ reason = "non-Latin script (%s, %.0f%% of letters); the English checkpoint cannot read it" % (
+ det["script"], 100 * float(det["non_latin_fraction"]))
+ elif not det["is_english"]:
+ key = "multilingual"
+ if det["language"]:
+ reason = "Latin script but language looks like %r, not English" % det["language"]
+ else:
+ # Unidentified Latin-script language: routed on the non-English letters alone,
+ # because no stopword list here covers it.
+ reason = ("Latin script, language not identified but %.0f%% non-English letters; "
+ "not safe for the English checkpoint" % (100 * float(det["diacritic_rate"])))
+ else:
+ key = "english"
+ reason = "English Latin text"
+ return RouteDecision(model=key, repo=_repo_str(self.models[key]), reason=reason,
+ detection=det, workflow=workflow)
+
+ # ------------------------------------------------------------------ running
+ def predict(
+ self,
+ state: Union[str, dict, list],
+ questions: Dict[str, Any],
+ model: Optional[str] = None,
+ task: Optional[str] = None,
+ lang: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Route, then answer every question in one forward pass on the chosen checkpoint.
+
+ The result is the usual `system_one` payload plus a `routing` key recording the decision.
+ """
+ decision = self.route(state, questions, model=model, task=task, lang=lang)
+ agent = self.load(decision["model"])
+ result = agent.system_one(state, questions)
+ result["routing"] = dict(decision)
+ return result
+
+ system_one = predict
+
+ def __repr__(self):
+ return "Router(loaded=%s, max_loaded=%d, default=%r)" % (self.loaded, self.max_loaded, self.default)
diff --git a/studio/backend/vendor/laya/shortlist.py b/studio/backend/vendor/laya/shortlist.py
new file mode 100644
index 00000000000..b37638e9372
--- /dev/null
+++ b/studio/backend/vendor/laya/shortlist.py
@@ -0,0 +1,272 @@
+"""Opt-in embedding shortlist for high-cardinality choice questions.
+
+Choice options share one ``head_max_len`` budget, so a large label set leaves only a few
+tokens per label. ``predict_shortlist`` embeds the state and each option with a
+caller-supplied ``embed_fn``, keeps the top ``k``, and runs a single ``predict`` (or
+``system_one``) on that reduced criteria set.
+
+``Agent.predict`` and ``Agent.system_one`` are separate: they still score every criterion
+they are given. This module does not change ``DecisionModel.forward`` and does not add a
+second decision-model pass.
+
+The coarse-to-fine pattern is the one the README recommends and the one reported in
+https://github.com/NandhaKishorM/laya/issues/102. Ranking here is cosine similarity on
+whatever vectors ``embed_fn`` returns. Issue #102's BANKING77 figures belong to that
+report; this module does not measure them.
+"""
+import json
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+import numpy as np
+
+from .common import render_options, serialize_state
+
+DEFAULT_SHORTLIST_K = 20
+
+
+def shortlist_choice(
+ state: Any,
+ criteria: Any,
+ embed_fn: Callable[[Sequence[str]], Any],
+ k: int = DEFAULT_SHORTLIST_K,
+ *,
+ instructions: Optional[str] = None,
+) -> List[Any]:
+ """Return the top-``k`` choice labels for ``state``.
+
+ ``embed_fn`` maps a list of strings to an array of shape ``(len(texts), dim)``.
+ It is called once, with the query text first and then one string per option in
+ criteria order. Option strings match ``render_options`` for a choice question.
+
+ When ``k`` is at least the number of labels, every label is returned in its
+ original order and ``embed_fn`` is not called.
+
+ Ties keep the earlier label. A zero vector scores 0 and does not outrank a
+ label that came before it.
+ """
+ labels, _scores, _passthrough, _n = _rank(state, criteria, embed_fn, k, instructions)
+ return labels
+
+
+def predict_shortlist(
+ agent: Any,
+ state: Any,
+ questions: Dict[str, Dict[str, Any]],
+ embed_fn: Callable[[Sequence[str]], Any],
+ k: int = DEFAULT_SHORTLIST_K,
+ **predict_kwargs: Any,
+) -> Dict[str, Any]:
+ """Shortlist each choice question, then call ``predict`` or ``system_one`` once.
+
+ Non-choice questions are forwarded unchanged. A choice whose label count is
+ ``<= k`` is forwarded unchanged and does not call ``embed_fn``. The caller's
+ ``questions`` dict is not mutated.
+
+ The returned dict is the model result plus a ``shortlist`` entry. Probabilities
+ on a shortlisted choice are over the kept labels only. ``shortlist[qid]`` holds
+ ``labels`` (rank order), ``scores`` (cosine, or ``None`` when nothing was
+ dropped), ``k``, ``n``, and ``passthrough``.
+
+ Extra keyword arguments are forwarded to ``predict`` / ``system_one`` (for
+ example ``model=`` on a ``Router``).
+ """
+ if not isinstance(questions, dict):
+ raise TypeError("questions must be a dict of question id -> definition")
+ checked = _check_k(k)
+ reduced: Dict[str, Any] = {}
+ meta: Dict[str, Dict[str, Any]] = {}
+ for qid, qdef in questions.items():
+ if not isinstance(qdef, dict) or qdef.get("type") != "choice":
+ reduced[qid] = qdef
+ continue
+ if "criteria" not in qdef:
+ raise ValueError("question %r is a choice but has no criteria" % (qid,))
+ labels, scores, passthrough, n = _rank(
+ state, qdef["criteria"], embed_fn, checked, qdef.get("instructions")
+ )
+ meta[qid] = {
+ "labels": list(labels),
+ "scores": scores,
+ "k": checked,
+ "n": n,
+ "passthrough": passthrough,
+ }
+ if passthrough:
+ reduced[qid] = qdef
+ continue
+ updated = dict(qdef)
+ updated["criteria"] = _subset_criteria(qdef["criteria"], labels)
+ reduced[qid] = updated
+
+ result = _call_predict(agent, state, reduced, **predict_kwargs)
+ if not isinstance(result, dict):
+ raise TypeError(
+ "predict/system_one must return a dict, got %s" % type(result).__name__
+ )
+ out = dict(result)
+ out["shortlist"] = meta
+ return out
+
+
+def embed_fn_from_agent(
+ agent: Any,
+ max_length: int = 512,
+ batch_size: int = 32,
+) -> Callable[[Sequence[str]], np.ndarray]:
+ """Mean-pool the checkpoint encoder already loaded on ``agent``.
+
+ The callable embeds a list of strings with ``agent.tok`` and ``agent.model.encoder``.
+ It does not run the decision head and does not download weights. A dedicated
+ bi-encoder passed as ``embed_fn`` will usually shortlist better; this helper is
+ for callers who only have the Laya checkpoint in memory.
+
+ Padding positions are excluded from the mean. The encoder's train/eval flag is
+ left as the caller set it (a loaded ``Agent`` is already in eval).
+ """
+ if isinstance(max_length, bool) or not isinstance(max_length, int) or max_length < 1:
+ raise ValueError("max_length must be a positive integer, got %r" % (max_length,))
+ if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size < 1:
+ raise ValueError("batch_size must be a positive integer, got %r" % (batch_size,))
+
+ import torch
+
+ tok = agent.tok
+ encoder = agent.model.encoder
+ device = agent.device
+
+ def embed_fn(texts: Sequence[str]) -> np.ndarray:
+ rows = ["" if text is None else str(text) for text in texts]
+ hidden = _hidden_size(encoder)
+ if not rows:
+ return np.zeros((0, hidden), dtype=np.float32)
+ parts: List[np.ndarray] = []
+ for start in range(0, len(rows), batch_size):
+ chunk = rows[start : start + batch_size]
+ encoded = tok(
+ chunk,
+ padding=True,
+ truncation=True,
+ max_length=max_length,
+ return_tensors="pt",
+ )
+ input_ids = encoded["input_ids"].to(device)
+ attention_mask = encoded["attention_mask"].to(device)
+ with torch.inference_mode():
+ hidden_states = encoder(
+ input_ids=input_ids, attention_mask=attention_mask
+ ).last_hidden_state
+ mask = attention_mask.unsqueeze(-1).to(dtype=hidden_states.dtype)
+ pooled = (hidden_states * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1.0)
+ parts.append(pooled.float().cpu().numpy())
+ return np.concatenate(parts, axis=0)
+
+ return embed_fn
+
+
+def _rank(state, criteria, embed_fn, k, instructions):
+ checked = _check_k(k)
+ items = _criteria_items(criteria)
+ n = len(items)
+ keys = [key for key, _value in items]
+ if checked >= n:
+ return list(keys), None, True, n
+ query = _query_text(state, instructions)
+ matrix = _embeddings(embed_fn, [query] + _option_texts(items))
+ sims = _cosine(matrix[0], matrix[1:])
+ order = np.argsort(-sims, kind="mergesort")[:checked]
+ labels = [keys[int(i)] for i in order]
+ scores = [float(sims[int(i)]) for i in order]
+ return labels, scores, False, n
+
+
+def _check_k(k: int) -> int:
+ if isinstance(k, bool) or not isinstance(k, int) or k < 1:
+ raise ValueError("k must be a positive integer, got %r" % (k,))
+ return k
+
+
+def _criteria_items(criteria):
+ if isinstance(criteria, dict):
+ items = list(criteria.items())
+ elif isinstance(criteria, list):
+ items = [(item, None) for item in criteria]
+ else:
+ raise TypeError(
+ "choice criteria must be a dict or list, got %s" % type(criteria).__name__
+ )
+ if not items:
+ raise ValueError("choice criteria must contain at least one option")
+ seen = set()
+ for key, _value in items:
+ if key in seen:
+ raise ValueError("choice criteria label %r is duplicated" % (key,))
+ seen.add(key)
+ return items
+
+
+def _option_texts(items) -> List[str]:
+ crit = {key: value for key, value in items}
+ rendered = render_options({"t": "choice", "ins": "", "crit": crit})
+ texts = [piece if isinstance(piece, str) else str(piece) for piece in rendered]
+ if len(texts) != len(items):
+ raise ValueError("could not render every choice option")
+ return texts
+
+
+def _query_text(state, instructions) -> str:
+ body = serialize_state(state)
+ if instructions is None or instructions == "":
+ return body
+ if not isinstance(instructions, str):
+ instructions = json.dumps(instructions, ensure_ascii=False)
+ return "%s\n%s" % (instructions, body)
+
+
+def _subset_criteria(criteria, labels):
+ if isinstance(criteria, dict):
+ return {label: criteria[label] for label in labels}
+ return list(labels)
+
+
+def _embeddings(embed_fn, texts: Sequence[str]) -> np.ndarray:
+ if not callable(embed_fn):
+ raise TypeError("embed_fn must be callable")
+ raw = embed_fn(list(texts))
+ if hasattr(raw, "detach"):
+ raw = raw.detach().float().cpu().numpy()
+ arr = np.asarray(raw, dtype=np.float64)
+ if arr.ndim != 2 or arr.shape[0] != len(texts) or arr.shape[1] < 1:
+ raise ValueError(
+ "embed_fn must return an array of shape (%d, dim), got %s"
+ % (len(texts), tuple(arr.shape))
+ )
+ return np.nan_to_num(arr, copy=True, nan=0.0, posinf=0.0, neginf=0.0)
+
+
+def _cosine(query: np.ndarray, docs: np.ndarray) -> np.ndarray:
+ qn = float(np.linalg.norm(query))
+ dn = np.linalg.norm(docs, axis=1)
+ sims = np.zeros(docs.shape[0], dtype=np.float64)
+ if qn == 0.0:
+ return sims
+ denom = dn * qn
+ ok = denom > 0.0
+ if np.any(ok):
+ sims[ok] = docs[ok] @ query / denom[ok]
+ return sims
+
+
+def _call_predict(agent, state, questions, **predict_kwargs):
+ fn = getattr(agent, "predict", None)
+ if fn is None:
+ fn = getattr(agent, "system_one", None)
+ if fn is None:
+ raise TypeError("agent must provide predict or system_one")
+ return fn(state, questions, **predict_kwargs)
+
+
+def _hidden_size(encoder) -> int:
+ size = getattr(getattr(encoder, "config", None), "hidden_size", None)
+ if isinstance(size, bool) or not isinstance(size, int) or size < 1:
+ return 0
+ return size
diff --git a/studio/backend/vendor/laya_manifest.json b/studio/backend/vendor/laya_manifest.json
new file mode 100644
index 00000000000..511b48622b3
--- /dev/null
+++ b/studio/backend/vendor/laya_manifest.json
@@ -0,0 +1,19 @@
+{
+ "files": {
+ "LICENSE.laya": "a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9",
+ "laya/__init__.py": "43e8c7d02969b37381915850cb4970e62b6ea0a328ac26c1efc9fb3fca0916d7",
+ "laya/agent.py": "128567096446c5d39af8e4a3a7c4dd9e32a134a1b099ce5a5eed383beeff1b89",
+ "laya/common.py": "f231d42fcec84da203222fcaa89c083b22776e00341e66e118183d754e1dcabf",
+ "laya/email.py": "481440b9f4dfc1d8c0c297322581ec2ba567f4286c1988c6e0b5163889f32088",
+ "laya/lang.py": "8e84fca6e3fefeeb5ac625733ba12c4be01bf6bb140ada2dea4a21ca52cd2a26",
+ "laya/presets.py": "2a3370f587cae29696bfa487edcc51caf47a211518733fdb192cf6d4eddbf1ea",
+ "laya/router.py": "1bdb5f3eda41a9dafddc3cd0cde150538b1bd36fbc1dba9f9d06ea00c774efa1",
+ "laya/shortlist.py": "0d5a2a0f59ceb3e7fbf1f087864fcce41bb73eb039cf73357792976c10bfdb76"
+ },
+ "license": "Apache-2.0",
+ "package": "laya",
+ "source": "https://pypi.org/project/laya/0.3.5/",
+ "version": "0.3.5",
+ "wheel": "laya-0.3.5-py3-none-any.whl",
+ "wheel_sha256": "4c57f64cbaf893bb5c7b4affddc2bf21a819f55df51941689f11868583be2903"
+}