Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: train compatible queued fine-tuning jobs together with multi-LoRA (#15344)
Daniel Han3e3c2fd53937
diff --git a/studio/backend/core/training/multi_lora_trainer.py b/studio/backend/core/training/multi_lora_trainer.pynew file mode 100644index 00000000000..979932fdcaa--- /dev/null+++ b/studio/backend/core/training/multi_lora_trainer.py@@ -0,0 +1,1084 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Several queued LoRA jobs trained together on one base model with the fused multi-LoRA kernels.++Each member keeps its own dataset, rank, alpha, learning rate, epochs, seed, optimizer and output, and+reports its own events (``member_run_id``) with the same shapes a lone job sends."""++from __future__ import annotations++import copy+import math+import os+import queue+import time+import traceback+import types+import warnings+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Iterator, Optional++from loggers import get_logger++logger = get_logger(__name__)++_DEFAULT_TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]+_OPTIMIZERS = ("adamw_8bit", "adamw_torch", "adamw_torch_fused")+++@dataclass+class _Member:+ slot: int+ run_id: str+ config: dict+ tokenizer: Any = None+ rows: list = field(default_factory = list)+ output_dir: str = ""+ total_steps: int = 0+ steps_per_epoch: int = 1+ step: int = 0+ num_tokens: int = 0+ windows: Optional[Iterator] = None+ optimizer: Any = None+ scheduler: Any = None+ params: list = field(default_factory = list)+ done: bool = False+++def _send(+ event_queue,+ kind: str,+ member: Optional[_Member] = None,+ **payload,+) -> None:+ event = {"type": kind, **payload, "ts": time.time()}+ if member is not None:+ event["member_run_id"] = member.run_id+ event_queue.put(event)+++def _new_trainer():+ from core.training.trainer import UnslothTrainer+ return UnslothTrainer()+++def _detect_model(trainer, config: dict, hf_token) -> None:+ from core.training import worker++ model_load_name = worker._resolve_cached_model_load_name(config)+ local_only = worker._model_local_files_only(config)+ worker._pre_detect_training_model(+ trainer,+ config,+ config["model_name"],+ hf_token,+ model_load_name,+ local_only,+ None if local_only else config.get("model_revision"),+ )+++def _format_dataset(trainer, config: dict, hf_token):+ from core.training import worker++ hf_dataset = config.get("hf_dataset", "")+ result = trainer.load_and_format_dataset(+ dataset_source = hf_dataset if hf_dataset and hf_dataset.strip() else None,+ format_type = config.get("format_type", ""),+ local_datasets = config.get("local_datasets") or None,+ custom_format_mapping = config.get("custom_format_mapping"),+ subset = config.get("subset"),+ train_split = config.get("train_split", "train"),+ dataset_slice_start = config.get("dataset_slice_start"),+ dataset_slice_end = config.get("dataset_slice_end"),+ s3_config = config.get("s3_config"),+ dataset_local_files_only = worker._dataset_local_files_only(config),+ dataset_local_path = config.get("dataset_snapshot_path"),+ dataset_revision = config.get("dataset_revision"),+ hf_token = hf_token,+ max_train_rows_seed = config.get("random_seed", 3407),+ )+ if result is None:+ raise RuntimeError(trainer.training_progress.error or "Failed to load dataset")+ return result[0] if isinstance(result, tuple) else result+++def _load_base(trainer, config: dict, hf_token, event_queue) -> Optional[bool]:+ """The single job's LoRA load; returns the effective 4-bit flag, or None when stopped."""+ from core.training import worker+ from utils.hf_xet_fallback import start_watchdog++ model_name = config["model_name"]+ model_load_name = worker._resolve_cached_model_load_name(config)+ local_only = worker._model_local_files_only(config)+ base_error = worker._adapter_base_changed_error(config) or worker._escaping_weights_error(+ config+ )+ if base_error:+ raise RuntimeError(base_error)+ event_queue.put({"type": "model_load_started", "ts": time.time()})+ watchdog = start_watchdog(+ repo_ids = [model_name],+ on_stall = lambda msg: event_queue.put({"type": "stall", "message": msg, "ts": time.time()}),+ xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",+ )++ def report_repo(repo_id):+ if not os.path.isdir(os.path.expanduser(repo_id)):+ event_queue.put({"type": "model_load_resolved", "repo_id": repo_id, "ts": time.time()})++ try:+ load_in_4bit = worker._effective_training_load_in_4bit(config, model_load_name, hf_token)+ success = trainer.load_model(+ on_model_resolved = report_repo,+ model_name = model_name,+ max_seq_length = config["max_seq_length"],+ load_in_4bit = load_in_4bit,+ full_finetuning = False,+ hf_token = hf_token,+ trust_remote_code = config.get("trust_remote_code", False),+ gpu_ids = config.get("resolved_gpu_ids"),+ model_load_name = model_load_name,+ local_files_only = local_only,+ actual_model_repo_id = config.get("actual_model_repo_id"),+ model_revision = None if local_only else config.get("model_revision"),+ use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),+ offload_layers = config.get("offload_layers", 0),+ prefetch_depth = config.get("prefetch_depth", 2),+ offload_plan_shape = worker._offload_plan_shape(config),+ )+ finally:+ watchdog.set()+ event_queue.put({"type": "model_load_completed", "ts": time.time()})+ if trainer.should_stop:+ return None+ if not success:+ raise RuntimeError(+ worker._with_vram_budget_hint(+ config, trainer.training_progress.error or "Failed to load model"+ )+ )+ worker._emit_resource_provenance(+ event_queue,+ config,+ trainer.model,+ model_load_target = model_load_name,+ model_load_in_4bit = load_in_4bit,+ dataset_loaded_from_exact_snapshot = bool(+ getattr(trainer, "dataset_loaded_from_exact_snapshot", False)+ ),+ )+ return load_in_4bit+++def _is_moe(model) -> bool:+ """unsloth.models._utils.is_moe_model, without importing unsloth's package root."""+ config = getattr(model, "config", None)+ for cfg in (config, getattr(config, "text_config", None)):+ count = next(+ (+ getattr(cfg, k)+ for k in ("num_experts", "n_routed_experts", "num_local_experts")+ if getattr(cfg, k, None) is not None+ ),+ None,+ )+ if count is not None:+ return isinstance(count, int) and count > 0+ return False+++def _resolve_targets(model, target_modules) -> list[str]:+ """Leaf names as PEFT resolves them: an explicit list, the Studio default, or every linear layer+ except the output head for ``all-linear``."""+ if isinstance(target_modules, str):+ target_modules = [target_modules]+ target_modules = list(target_modules or [])+ # As the single job: explicit names beside "all-linear" win; "all-linear" alone means every linear layer.+ explicit = [m for m in target_modules if m != "all-linear"]+ if explicit or not target_modules:+ return explicit or list(_DEFAULT_TARGETS)+ head = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None+ names = {+ name.rsplit(".", 1)[-1]+ for name, module in model.named_modules()+ if name+ and module is not head+ and hasattr(module, "in_features")+ and hasattr(module, "weight")+ and not list(module.children())+ }+ return sorted(names)+++def _multi_lora_layers(model):+ from unsloth.kernels.multi_lora import MultiLoraLinear+ return [(n, m) for n, m in model.named_modules() if isinstance(m, MultiLoraLinear)]+++def _seed_adapter(model, slot: int, seed: int) -> None:+ """Kaiming A from the member's own seed, B zero: the same init whatever the batch order."""+ import torch++ generator = torch.Generator().manual_seed(int(seed))+ with torch.no_grad():+ for _, layer in _multi_lora_layers(model):+ A = torch.empty(layer.lora_A[slot].shape, dtype = torch.float32)+ torch.nn.init.kaiming_uniform_(A, a = math.sqrt(5), generator = generator)+ layer.lora_A[slot].copy_(A)+ layer.lora_B[slot].zero_()+++def _enable_training(model, gradient_checkpointing, max_seq_length: int) -> None:+ """What get_peft_model does for checkpointing on a LoRA model, minus the PEFT wrapper."""+ from unsloth.models._utils import (+ apply_unsloth_gradient_checkpointing,+ prepare_model_for_kbit_training,+ )++ dtype = model.get_input_embeddings().weight.dtype+ gradient_checkpointing = apply_unsloth_gradient_checkpointing(+ gradient_checkpointing, max_seq_length, dtype+ )+ model._unsloth_gradient_checkpointing = gradient_checkpointing+ prepare_model_for_kbit_training(+ model, use_gradient_checkpointing = gradient_checkpointing, use_reentrant = True+ )+ for_training = getattr(model, "for_training", None)+ if callable(for_training):+ for_training(use_gradient_checkpointing = gradient_checkpointing)+ else:+ model.train()+++def _check_routed(model, device) -> None:+ """One tiny forward must reach every injected layer, else the patched forwards bypass them."""+ import torch+ from unsloth.kernels.multi_lora import set_multi_lora_batch++ layers = _multi_lora_layers(model)+ hits = set()+ handles = [+ layer.register_forward_hook(lambda m, i, o, name = name: hits.add(name))+ for name, layer in layers+ ]+ try:+ ids = torch.zeros(1, 2, dtype = torch.long, device = device)+ set_multi_lora_batch(model, torch.zeros(1, dtype = torch.long, device = device), seq_len = 2)+ # No grad here, so checkpointed layers warn that their inputs need none.+ with torch.no_grad(), warnings.catch_warnings():+ warnings.simplefilter("ignore", UserWarning)+ model(input_ids = ids, attention_mask = torch.ones_like(ids), use_cache = False)+ finally:+ for handle in handles:+ handle.remove()+ for _, layer in layers:+ layer.batch = layer.seen = layer.trained = None+ missed = [name for name, _ in layers if name not in hits]+ if missed:+ raise RuntimeError(+ f"Batched training cannot run on this model: its forward skips {len(missed)} adapter "+ f"layers (e.g. {missed[0]})."+ )+++def _member_tokenizer(tokenizer, model_name: str, final_format: str):+ from utils.datasets.chat_templates import get_training_chat_template+ if getattr(tokenizer, "chat_template", None) or final_format not in (+ "alpaca",+ "chatml_messages",+ "chatml_conversations",+ ):+ return tokenizer+ return get_training_chat_template(copy.deepcopy(tokenizer), model_name, final_format)+++def _tokenize_member(+ dataset_info,+ tokenizer,+ config: dict,+ model_name: str,+ max_length: int,+ vocab_size = None,+):+ """(tokenizer to save, [(input_ids, labels)]) with the single SFT path's tokenization and masking."""+ from datasets import Dataset++ from utils.datasets.completion_masking import apply_completion_masking+ from utils.datasets.online_tokenization import (+ build_tokenizing_transform,+ first_sample_text,+ resolve_add_special_tokens,+ )+ from utils.datasets.pretokenized import (+ PRETOKENIZED_FORMAT,+ _collator_masks_user_labels,+ prepare_pretokenized_dataset,+ )++ dataset = dataset_info["dataset"] if isinstance(dataset_info, dict) else dataset_info+ final_format = (+ str(dataset_info.get("final_format", "")).lower() if isinstance(dataset_info, dict) else ""+ )+ member_tok = _member_tokenizer(tokenizer, model_name, final_format)+ labels = None+ if final_format == PRETOKENIZED_FORMAT:+ # The lone job's checks (ids inside the vocabulary, mask and label lengths) and truncation: a bad+ # dataset then fails only its own member, never the shared forward.+ dataset = prepare_pretokenized_dataset(+ dataset, max_seq_length = max_length, vocab_size = vocab_size+ )+ masks = dataset["assistant_masks"] if "assistant_masks" in dataset.column_names else None+ input_ids = [list(x)[:max_length] for x in dataset["input_ids"]]+ if "labels" in dataset.column_names:+ labels = [list(x)[:max_length] for x in dataset["labels"]]+ if not _collator_masks_user_labels():+ masks = None+ if masks is not None:+ # TRL's collator trains only the assistant tokens, on top of user labels before TRL 1.7.+ labels = [+ [t if m else -100 for t, m in zip(row, mask)]+ for row, mask in zip(labels if labels is not None else input_ids, masks)+ ]+ else:+ add_special_tokens = resolve_add_special_tokens(+ member_tok, first_sample_text(dataset, "text")+ )+ encode = build_tokenizing_transform(member_tok, "text", max_length, add_special_tokens)+ input_ids = [list(x) for x in encode({"text": list(dataset["text"])})["input_ids"]]+ raw = final_format == "raw_text" or config.get("format_type") == "raw"+ if labels is None and config.get("train_on_completions") and not raw:++ def build_masker(holder, **kwargs):+ from unsloth.chat_templates import train_on_responses_only++ kwargs.pop("num_proc", None)+ masker = train_on_responses_only(+ None, tokenizer = holder.processing_class, return_function = True, **kwargs+ )+ holder.train_dataset = holder.train_dataset.map(masker, batched = True)+ return holder++ holder, applied = apply_completion_masking(+ types.SimpleNamespace(+ processing_class = member_tok,+ train_dataset = Dataset.from_dict({"input_ids": input_ids}),+ eval_dataset = None,+ ),+ model_name,+ build_masker,+ notify = lambda level, message: logger.info(message),+ dataset_template = "alpaca" if final_format == "alpaca" else None,+ )+ if applied:+ labels = [list(x) for x in holder.train_dataset["labels"]]+ kept = [i for i, row in enumerate(labels) if any(t != -100 for t in row[1:])]+ dropped = len(labels) - len(kept)+ drop_pct = round(100 * dropped / len(labels), 1) if labels else 0+ if not kept or drop_pct > 30:+ raise ValueError(+ f"{dropped}/{len(labels)} samples ({drop_pct}%) were dropped after applying "+ f"'Train on completions': after masking, those rows had no trainable response "+ f"tokens left. The usual cause is that this model's response template was not "+ f"found in the formatted samples, so 'Train on completions' should be turned "+ f"off for this dataset."+ )+ input_ids = [input_ids[i] for i in kept]+ labels = [labels[i] for i in kept]+ if labels is None:+ labels = [list(x) for x in input_ids]+ rows = [(ids, lab) for ids, lab in zip(input_ids, labels) if len(ids) > 1]+ if not rows:+ raise ValueError("The dataset has no rows to train on after tokenization.")+ return member_tok, rows+++def _windows(n_rows: int, batch_size: int, grad_accum: int, seed: int):+ """Endless (epoch, [microbatch row indices] * <= grad_accum); windows never span an epoch."""+ import torch++ epoch = 0+ while True:+ order = torch.randperm(n_rows, generator = torch.Generator().manual_seed(seed + epoch))+ order = order.tolist()+ micro = [order[i : i + batch_size] for i in range(0, n_rows, batch_size)]+ for i in range(0, len(micro), grad_accum):+ yield epoch, micro[i : i + grad_accum]+ epoch += 1+++def _member_scheduler(config: dict, lr: float, total_steps: int):+ """The single job's schedule for this member, on a stand-in optimizer: it only yields learning rates."""+ import torch+ from transformers import get_scheduler++ warmup_ratio, warmup_steps = config.get("warmup_ratio"), config.get("warmup_steps")+ if warmup_ratio is not None:+ warmup = math.ceil(total_steps * float(warmup_ratio))+ elif warmup_steps is not None:+ warmup = int(warmup_steps)+ else:+ warmup = 5+ stand_in = torch.optim.SGD([torch.zeros(1, requires_grad = True)], lr = lr)+ return get_scheduler(+ config.get("lr_scheduler_type", "linear") or "linear",+ stand_in,+ num_warmup_steps = warmup,+ num_training_steps = total_steps,+ )+++class _PerMemberLR:+ """Steps every member's own schedule and writes its rate into that member's parameter group."""++ def __init__(self, optimizer, schedulers: list):+ self.optimizer = optimizer+ self.schedulers = schedulers+ self._apply()++ def _apply(self) -> None:+ for group, scheduler in zip(self.optimizer.param_groups, self.schedulers):+ group["lr"] = scheduler.get_last_lr()[0]++ def step(self) -> None:+ for scheduler in self.schedulers:+ scheduler.step()+ self._apply()++ def get_last_lr(self) -> list:+ return [group["lr"] for group in self.optimizer.param_groups]++ def state_dict(self) -> dict:+ return {}++ def load_state_dict(self, state: dict) -> None:+ pass+++def _collate(rows: list, slots: list, pad_id: int) -> dict:+ import torch++ T = max(len(ids) for ids, _ in rows)+ input_ids = torch.full((len(rows), T), pad_id, dtype = torch.long)+ labels = torch.full((len(rows), T), -100, dtype = torch.long)+ attention = torch.zeros((len(rows), T), dtype = torch.long)+ for i, (ids, lab) in enumerate(rows):+ input_ids[i, : len(ids)] = torch.tensor(ids)+ labels[i, : len(lab)] = torch.tensor(lab)+ attention[i, : len(ids)] = 1+ return {+ "input_ids": input_ids,+ "attention_mask": attention,+ "labels": labels,+ "adapter_ids": torch.tensor(slots, dtype = torch.long),+ }+++def _unique_output_dir(config: dict, model_name: str, taken: set) -> str:+ from utils.paths import resolve_output_dir+ from utils.training_runs import build_default_output_dir_name++ explicit = config.get("output_dir")+ stamp = int(time.time())+ for attempt in range(1000):+ if explicit:+ name = explicit if attempt == 0 else f"{explicit}-{attempt}"+ else:+ # A later timestamp, not a suffix: the folder name is parsed back by its trailing timestamp.+ name = build_default_output_dir_name(+ model_name, config.get("project_name"), timestamp = stamp + attempt+ )+ path = str(resolve_output_dir(name))+ if path not in taken and not os.path.exists(path):+ taken.add(path)+ return path+ raise RuntimeError("Could not reserve an output folder for a batched job.")+++def _export(+ model,+ member: _Member,+ trainer,+ targets: list,+ use_rslora: bool,+ path: str = "",+) -> None:+ from peft import LoraConfig+ from safetensors.torch import save_file++ from core.training.trainer import UnslothTrainer+ from unsloth.kernels.multi_lora import adapter_state_dict+ from utils.paths import ensure_dir++ path = path or member.output_dir+ ensure_dir(Path(path))+ state = {+ k: v.detach().to("cpu").contiguous()+ for k, v in adapter_state_dict(model, member.slot).items()+ }+ save_file(+ state,+ os.path.join(path, "adapter_model.safetensors"),+ metadata = {"format": "pt"},+ )+ LoraConfig(+ r = int(member.config.get("lora_r", 16)),+ lora_alpha = member.config.get("lora_alpha", 16),+ lora_dropout = 0.0,+ bias = "none",+ target_modules = targets,+ use_rslora = use_rslora,+ task_type = "CAUSAL_LM",+ base_model_name_or_path = getattr(model.config, "_name_or_path", None),+ ).save_pretrained(path)+ member.tokenizer.save_pretrained(path)+ UnslothTrainer._patch_adapter_config(trainer, path)+++def _status_forwarder(event_queue):+ from core.training.worker import _send_status++ sent: set = set()++ def forward(progress) -> None:+ if progress.status_message:+ _send_status(event_queue, progress.status_message)+ for message in progress.warnings:+ if message not in sent:+ sent.add(message)+ event_queue.put({"type": "warning", "message": message, "ts": time.time()})++ return forward+++def _start_stop_reader(stop_queue, inbox: "queue.Queue", trainer) -> None:+ from utils.account_context import account_thread+ def read():+ while True:+ try:+ message = stop_queue.get(timeout = 1.0)+ except queue.Empty:+ continue+ except (EOFError, OSError, ValueError):+ return+ if not isinstance(message, dict):+ continue+ if message.get("type") == "stop":+ trainer.should_stop = True+ trainer.save_on_stop = bool(message.get("save", True))+ if message.get("type") in ("stop", "stop_member"):+ inbox.put(message)++ account_thread(target = read, daemon = True).start()+++def _oom_message(exc: BaseException) -> Optional[str]:+ text = str(exc).lower()+ if "out of memory" in text or type(exc).__name__ == "OutOfMemoryError":+ return (+ "GPU ran out of VRAM while training the batched jobs.\n"+ "To fix: reduce max_seq_length or the batch size of the queued jobs."+ )+ return None+++def run_multi_lora_training(event_queue: Any, stop_queue: Any, config: dict) -> None:+ """Worker entry for ``config["batch_members"]`` (same event and stop contract as one job)."""+ members = [+ _Member(slot = -1, run_id = m["run_id"], config = m["config"]) for m in config["batch_members"]+ ]+ try:+ _run(event_queue, stop_queue, config, members)+ except Exception as exc:+ message = _oom_message(exc) or str(exc)+ stack = traceback.format_exc(limit = 20)+ logger.error("Batched training failed: %s", exc)+ for member in members:+ if not member.done:+ member.done = True+ _send(event_queue, "error", member, error = message, stack = stack)+ event_queue.put({"type": "error", "error": message, "stack": stack, "ts": time.time()})+++def _finish_stopped(event_queue, members, save: bool, export) -> None:+ for member in members:+ if member.done:+ continue+ member.done = True+ saved = bool(save and member.step > 0 and member.output_dir)+ if saved:+ export(member)+ _send(+ event_queue,+ "stopped",+ member,+ output_dir = member.output_dir if saved else None,+ status_message = (+ f"Training stopped. Model saved to {member.output_dir}"+ if saved+ else "Training cancelled."+ ),+ saved = saved,+ )+++def _run(event_queue, stop_queue, config: dict, members: list) -> None:+ from core.import_guards import ensure_real_packages++ ensure_real_packages("unsloth_zoo", "unsloth")+ import torch++ from core.training.trainer import normalize_gradient_checkpointing+ from core.training.worker import _send_status, _worker_hf_token+ from unsloth.kernels.multi_lora import inject_multi_lora++ trainer = _new_trainer()+ if hasattr(trainer, "add_progress_callback"):+ trainer.add_progress_callback(_status_forwarder(event_queue))+ inbox: queue.Queue = queue.Queue()+ _start_stop_reader(stop_queue, inbox, trainer)++ def stopped_early() -> bool:+ if not trainer.should_stop:+ return False+ _finish_stopped(event_queue, members, False, None)+ event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})+ return True++ hf_token = _worker_hf_token(config)+ model_name = config["model_name"]+ _send_status(event_queue, "Detecting model type...")+ _detect_model(trainer, config, hf_token)+ if stopped_early():+ return++ datasets = {}+ for i, member in enumerate(members):+ _send_status(event_queue, f"Loading and formatting dataset {i + 1}/{len(members)}...")+ try:+ datasets[member.run_id] = _format_dataset(trainer, member.config, hf_token)+ except Exception as exc:+ member.done = True+ _send(+ event_queue, "error", member, error = str(exc), stack = traceback.format_exc(limit = 20)+ )+ if stopped_early():+ return+ if not datasets:+ raise RuntimeError("None of the batched jobs could load its dataset.")++ _send_status(event_queue, "Loading model...")+ load_in_4bit = _load_base(trainer, config, hf_token, event_queue)+ if stopped_early():+ return+ if getattr(trainer, "diffusion_lm_profile", None):+ raise RuntimeError(+ "Batched training does not support text-diffusion models; train these jobs one at a time."+ )+ if _is_moe(trainer.model):+ # The single-job path trains fused experts through PEFT target_parameters; these layers cannot.+ raise RuntimeError(+ "Batched training does not support MoE models; train these jobs one at a time."+ )+ if getattr(trainer.model, "peft_config", None):+ # A saved LoRA loads as a PeftModel; the multi-LoRA layers would train fresh adapters and drop its weights.+ raise RuntimeError(+ "Batched training cannot continue an existing LoRA adapter; train these jobs one at a time."+ )+ trainer.load_in_4bit = load_in_4bit+ trainer.is_cpt = False+ model, tokenizer = trainer.model, trainer.tokenizer+ max_length = int(config.get("max_seq_length") or 2048)+ model_cap = getattr(model, "max_seq_length", None)+ if isinstance(model_cap, int) and 0 < model_cap < max_length:+ max_length = model_cap++ _send_status(event_queue, "Tokenizing datasets...")+ for member in members:+ if member.done:+ continue+ try:+ member.tokenizer, member.rows = _tokenize_member(+ datasets.pop(member.run_id),+ tokenizer,+ member.config,+ model_name,+ max_length,+ vocab_size = int(model.get_input_embeddings().num_embeddings),+ )+ except Exception as exc:+ member.done = True+ _send(+ event_queue, "error", member, error = str(exc), stack = traceback.format_exc(limit = 20)+ )+ live = [m for m in members if not m.done]+ if not live:+ raise RuntimeError("None of the batched jobs has rows to train on.")+ for slot, member in enumerate(live):+ member.slot = slot+ U = len(live)++ _send_status(event_queue, f"Configuring {U} LoRA adapters...")+ targets = _resolve_targets(model, config.get("target_modules"))+ use_rslora = bool(config.get("use_rslora", False))+ injected = inject_multi_lora(+ model,+ targets,+ U,+ r = [int(m.config.get("lora_r", 16)) for m in live],+ lora_alpha = [float(m.config.get("lora_alpha", 16)) for m in live],+ use_rslora = use_rslora,+ )+ targets = sorted({name.rsplit(".", 1)[-1] for name in injected})+ for member in live:+ _seed_adapter(model, member.slot, member.config.get("random_seed", 3407))+ _enable_training(+ model,+ normalize_gradient_checkpointing(config.get("gradient_checkpointing", "unsloth")),+ max_length,+ )+ device = model.get_input_embeddings().weight.device+ _check_routed(model, device)++ layers = _multi_lora_layers(model)+ grad_accum = max(int(config.get("gradient_accumulation_steps", 4) or 1), 1)+ taken: set = set()+ for member in live:+ cfg = member.config+ batch_size = max(int(cfg.get("batch_size", 2) or 1), 1)+ n_micro = math.ceil(len(member.rows) / batch_size)+ member.steps_per_epoch = max(math.ceil(n_micro / grad_accum), 1)+ max_steps = int(cfg.get("max_steps") or 0)+ member.total_steps = (+ max_steps+ if max_steps > 0+ else member.steps_per_epoch * int(cfg.get("num_epochs", 3) or 1)+ )+ member.windows = _windows(+ len(member.rows), batch_size, grad_accum, int(cfg.get("random_seed", 3407))+ )+ member.params = [+ p for _, layer in layers for p in (layer.lora_A[member.slot], layer.lora_B[member.slot])+ ]+ member.output_dir = _unique_output_dir(cfg, model_name, taken)+ _send(event_queue, "output_dir", member, output_dir = member.output_dir)++ def export(member: _Member, path: str = "") -> None:+ _export(model, member, trainer, targets, use_rslora, path)++ def take_messages() -> Optional[bool]:+ """Applies queued member stops; returns the save flag of a whole-batch stop, if one arrived."""+ save_all = None+ while not inbox.empty():+ message = inbox.get_nowait()+ if message.get("type") == "stop":+ save_all = bool(message.get("save", True))+ continue+ target = next(+ (m for m in live if m.run_id == message.get("run_id") and not m.done), None+ )+ if target is not None:+ _finish_stopped(event_queue, [target], bool(message.get("save", True)), export)+ return save_all++ def stop_all(save: bool) -> None:+ _finish_stopped(event_queue, live, save, export)+ event_queue.put(+ {+ "type": "complete",+ "output_dir": None,+ "status_message": "Training stopped",+ "ts": time.time(),+ }+ )++ save_all = take_messages()+ if save_all is not None:+ stop_all(save_all)+ return+ if all(m.done for m in live):+ event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})+ return++ sft = _build_sft_trainer(+ model,+ tokenizer,+ config,+ live,+ grad_accum,+ device,+ event_queue,+ export,+ take_messages,+ stop_all,+ )+ for member in live:+ if not member.done:+ sft.progress(member, None, None, 0.0, sft.step_lrs[member.slot])+ _send_status(event_queue, f'Training {U} jobs together on "{model_name}"')+ sft.train()+ if not sft.stopped:+ event_queue.put(+ {+ "type": "complete",+ "output_dir": None,+ "status_message": "Training completed",+ "ts": time.time(),+ }+ )+++def _build_sft_trainer(+ model, tokenizer, config, live, grad_accum, device, event_queue, export, take_messages, stop_all+):+ """TRL's SFTTrainer over the batch: each optimizer step gives every running member its own next+ window, losses and gradient clipping stay per adapter, and each member keeps its own learning rate+ schedule (one parameter group each)."""+ import torch+ from datasets import Dataset+ from torch.utils.data import DataLoader+ from transformers import TrainerCallback+ from trl import SFTConfig, SFTTrainer++ from core.training.resume import session_eta_seconds+ from core.training.training import apply_save_strategy+ from unsloth.kernels.multi_lora import (+ adapter_token_counts,+ clip_grad_norm_per_adapter_,+ drop_absent_adapter_grads_,+ per_adapter_loss,+ set_multi_lora_batch,+ )++ U = len(live)+ total = max(m.total_steps for m in live if not m.done)+ pad_id = (+ tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id+ )+ pad_id = 0 if pad_id is None else int(pad_id)+ bf16 = fp16 = False+ if device.type == "cuda":+ from unsloth import is_bfloat16_supported+ bf16 = is_bfloat16_supported()+ fp16 = not bf16+ args = {+ "output_dir": live[0].output_dir,+ "per_device_train_batch_size": 1,+ "gradient_accumulation_steps": grad_accum,+ "max_steps": total,+ "learning_rate": float(live[0].config.get("learning_rate", 2e-4)),+ "max_grad_norm": float(config.get("max_grad_norm", 1.0) or 1.0),+ "bf16": bf16,+ "fp16": fp16,+ "logging_strategy": "no",+ "report_to": "none",+ "disable_tqdm": True,+ "seed": int(live[0].config.get("random_seed", 3407)),+ "remove_unused_columns": False,+ "gradient_checkpointing": False,+ "dataset_kwargs": {"skip_prepare_dataset": True},+ "dataloader_num_workers": 0,+ }+ apply_save_strategy(args, config.get("save_steps", 0))+ micro_total = total * grad_accum++ class _Steps(TrainerCallback):+ def on_step_end(self, args, state, control, **kwargs):+ sft.after_step(control)++ class MultiLoraSFTTrainer(SFTTrainer):+ def __init__(self, **kwargs):+ # Trainer refuses a quantized base without a PEFT wrapper; the adapters here are injected.+ loaded = getattr(model, "_hf_peft_config_loaded", None)+ model._hf_peft_config_loaded = True+ try:+ super().__init__(**kwargs)+ finally:+ if loaded is None:+ del model._hf_peft_config_loaded+ else:+ model._hf_peft_config_loaded = loaded+ # compute_loss already normalizes each adapter by its own window's tokens.+ self.model_accepts_loss_kwargs = True+ # Every Trainer clips through this one call (4.x inline, 5.x via _clip_grad_norm): clip per member.+ self.accelerator.clip_grad_norm_ = self._clip_members+ self.stopped = False+ self.round = []+ self.round_losses = torch.zeros(U)+ self.member_norms = torch.zeros(U)+ self.step_lrs = [float(m.config.get("learning_rate", 2e-4)) for m in live]+ self.start = time.time()++ def get_train_dataloader(self):+ return DataLoader(range(micro_total), batch_size = 1)++ def create_optimizer(self, model = None):+ if self.optimizer is None:+ groups = [+ {+ "params": m.params,+ "lr": float(m.config.get("learning_rate", 2e-4)),+ "weight_decay": float(m.config.get("weight_decay", 0.001) or 0.0),+ }+ for m in live+ ]+ optim = config.get("optim", "adamw_8bit") or "adamw_8bit"+ if optim == "adamw_8bit":+ import bitsandbytes as bnb+ self.optimizer = bnb.optim.AdamW8bit(groups)+ elif optim in ("adamw_torch", "adamw_torch_fused"):+ fused = optim == "adamw_torch_fused" and device.type == "cuda"+ self.optimizer = torch.optim.AdamW(groups, fused = fused)+ else:+ raise ValueError(f"Batched training does not support the optimizer {optim!r}.")+ return self.optimizer++ def create_scheduler(+ self,+ num_training_steps,+ optimizer = None,+ ):+ if self.lr_scheduler is None:+ self.lr_scheduler = _PerMemberLR(+ optimizer or self.optimizer,+ [+ _member_scheduler(+ m.config, float(m.config.get("learning_rate", 2e-4)), m.total_steps+ )+ for m in live+ ],+ )+ return self.lr_scheduler++ def get_batch_samples(self, epoch_iterator, num_batches, device):+ """One optimizer step: micro-batch k holds every running member's k-th micro-batch."""+ self.round = [m for m in live if not m.done]+ if not self.round:+ return [], None+ windows = {m.slot: next(m.windows)[1] for m in self.round}+ samples = []+ counts = None+ for k in range(num_batches):+ rows, slots = [], []+ for member in self.round:+ window = windows[member.slot]+ if k < len(window):+ picked = [member.rows[r] for r in window[k]]+ member.num_tokens += sum(len(ids) for ids, _ in picked)+ rows.extend(picked)+ slots.extend([member.slot] * len(picked))+ if not rows:+ # Every member's window is shorter here (an epoch end); the step keeps its size.+ samples.append({"multi_lora_empty": torch.zeros(1)})+ continue+ batch = _collate(rows, slots, pad_id)+ found = adapter_token_counts(batch["labels"], batch["adapter_ids"], U).cpu()+ counts = found if counts is None else counts + found+ samples.append(batch)+ self.round_losses = torch.zeros(U)+ return samples, counts.to(device)++ def compute_loss(+ self,+ model,+ inputs,+ return_outputs = False,+ num_items_in_batch = None,+ ):+ if "multi_lora_empty" in inputs:+ return torch.zeros((), device = self.args.device, requires_grad = True)+ ids = inputs["adapter_ids"]+ set_multi_lora_batch(self.model, ids, seq_len = inputs["input_ids"].shape[1])+ logits = model(+ input_ids = inputs["input_ids"],+ attention_mask = inputs["attention_mask"],+ use_cache = False,+ ).logits+ loss, per = per_adapter_loss(+ logits,+ inputs["labels"],+ ids,+ U,+ model = self.model,+ num_items_in_batch = num_items_in_batch,+ )+ self.round_losses += per.detach().float().cpu()+ return loss++ def _clip_members(+ self,+ parameters,+ max_norm,+ norm_type = 2,+ **kwargs,+ ):+ # The rate this step uses, as a lone job reports it (read before the scheduler moves).+ self.step_lrs = [group["lr"] for group in self.optimizer.param_groups]+ drop_absent_adapter_grads_(self.model)+ self.accelerator.unscale_gradients()+ self.member_norms = clip_grad_norm_per_adapter_(self.model, max_norm).float().cpu()+ return self.member_norms.norm()++ def _save_checkpoint(self, model, trial):+ name = f"checkpoint-{self.state.global_step}"+ for member in self.round:+ export(member, os.path.join(member.output_dir, name))++ def progress(self, member, loss, grad_norm, elapsed, lr) -> None:+ _send(+ event_queue,+ "progress",+ member,+ step = member.step,+ epoch = round(member.step / member.steps_per_epoch, 2),+ loss = loss,+ learning_rate = lr,+ total_steps = member.total_steps,+ elapsed_seconds = elapsed,+ eta_seconds = session_eta_seconds(elapsed, member.step, 0, member.total_steps),+ session_start_step = 0,+ grad_norm = grad_norm,+ num_tokens = member.num_tokens,+ eval_loss = None,+ status_message = "",+ )++ def after_step(self, control) -> None:+ elapsed = time.time() - self.start+ for member in self.round:+ member.step += 1+ self.progress(+ member,+ float(self.round_losses[member.slot]),+ float(self.member_norms[member.slot]),+ elapsed,+ self.step_lrs[member.slot],+ )+ if member.step >= member.total_steps and not member.done:+ export(member)+ member.done = True+ _send(+ event_queue,+ "complete",+ member,+ output_dir = member.output_dir,+ status_message = f"Training completed! Model saved to {member.output_dir}",+ )+ save_all = take_messages()+ if save_all is not None:+ stop_all(save_all)+ self.stopped = True+ if self.stopped or all(m.done for m in live):+ control.should_training_stop = True++ sft = MultiLoraSFTTrainer(+ model = model,+ args = SFTConfig(**args),+ train_dataset = Dataset.from_dict({"input_ids": [[pad_id]] * micro_total}),+ processing_class = tokenizer,+ data_collator = lambda rows: rows,+ callbacks = [_Steps()],+ )+ return sftdiff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex c0659ddd232..b6e714e2b4c 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -655,6 +655,14 @@ def _apply_cache_pins(config: dict[str, Any]) -> None:config["cache_pin_warnings"] = warnings+def _finite_or_none(value: Any) -> Optional[float]:+ try:+ value = float(value)+ except (TypeError, ValueError):+ return None+ return value if math.isfinite(value) else None++def _s3_dataset_name(s3_dataset: Any) -> Optional[str]:if not isinstance(s3_dataset, dict):return None@@ -1275,6 +1283,8 @@ class TrainingBackend:self._db_total_steps_set: bool = Falseself._db_config: Optional[dict] = Noneself._db_started_at: Optional[str] = None+ # Multi-LoRA batch: run_id -> member state; the batch id itself never gets a DB row.+ self._batch_members: dict[str, dict] = {}# Xet -> HTTP model-load fallback state (config kept for the respawn).self._last_full_config: Optional[dict] = None@@ -1707,10 +1717,13 @@ class TrainingBackend:before_spawn = None,resume_source_run_id: Optional[str] = None,start_request_id: Optional[str] = None,+ batch_members: Optional[list] = None,**kwargs,) -> bool:# Reserve first: compare-and-set stops two concurrent requests reaching the spawn.validate_job_paths(kwargs, cached_resources = True)+ for member in batch_members or ():+ validate_job_paths(member, cached_resources = True)with self._new_job_spawn_reservation(job_id) as spawn_reserved:if not spawn_reserved:logger.warning("Training subprocess already running")@@ -1734,6 +1747,7 @@ class TrainingBackend:resume_source_run_id = resume_source_run_id,start_request_id = start_request_id,spawn_already_reserved = True,+ batch_members = batch_members,**kwargs,)@@ -1745,6 +1759,7 @@ class TrainingBackend:resume_source_run_id: Optional[str] = None,start_request_id: Optional[str] = None,spawn_already_reserved: bool = False,+ batch_members: Optional[list] = None,**kwargs,) -> bool:"""before_spawn runs after validation but before GPU auto-selection, so refused starts free@@ -1773,6 +1788,19 @@ class TrainingBackend:self._pump_running = Falseconfig = _build_training_worker_config(kwargs)+ if batch_members:+ config["batch_id"] = job_id+ config["batch_members"] = [+ {+ "run_id": member["run_id"],+ "config": _sanitize_db_config(+ _build_training_worker_config(+ {k: v for k, v in member.items() if k != "run_id"}+ )+ ),+ }+ for member in batch_members+ ]hf_token = hf_token_arg(config["hf_token"], allow_ambient_token = config.get("allow_ambient", True))@@ -1792,9 +1820,14 @@ class TrainingBackend:hf_token = hf_token,training_type = config["training_type"],load_in_4bit = config["load_in_4bit"],- batch_size = config.get("batch_size", 4),+ # A batch (its members include this first one) trains every member's rows and adapter at once.+ batch_size = sum(int(m.get("batch_size") or 4) for m in batch_members)+ if batch_members+ else config.get("batch_size", 4),max_seq_length = config.get("max_seq_length", 2048),- lora_rank = config.get("lora_r", 16),+ lora_rank = sum(int(m.get("lora_r") or 16) for m in batch_members)+ if batch_members+ else config.get("lora_r", 16),target_modules = config.get("target_modules"),gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),optimizer = config.get("optim", DEFAULT_TRAINING_OPTIMIZER),@@ -1991,8 +2024,12 @@ class TrainingBackend:self._db_run_created = Falseself._db_create_in_progress = Falseself._db_total_steps_set = False- self._db_config = _sanitize_db_config(config)+ self._db_config = None if batch_members else _sanitize_db_config(config)self._db_started_at = datetime.now(timezone.utc).isoformat()+ self._batch_members = {+ member["run_id"]: self._new_batch_member(member["config"], self._db_started_at)+ for member in config.get("batch_members") or ()+ }# Start each job Xet-first; keep config so a stall can respawn over HTTP.self._last_full_config = configself._last_hf_cache_env = cache_env@@ -2060,7 +2097,14 @@ class TrainingBackend:if not expected_job_id or self.current_job_id != expected_job_id:return Falserun_id = self.current_job_id- if not save and run_id:+ batch = bool(self._batch_members)+ if not save and run_id and batch:+ self._cancel_batch_members()+ with self._lock:+ if self.current_job_id != run_id:+ return False+ self._should_stop = self._cancel_requested = True+ elif not save and run_id:persist_error: Optional[Exception] = Nonefor attempt in range(_DB_FINALIZE_RETRIES):try:@@ -2288,6 +2332,10 @@ class TrainingBackend:elif status != "completed":self._progress.status_message = "Training stopped."# A completed run keeps its message; reaping a wedged worker must not relabel it.+ if self._batch_members:+ self._finalize_batch_members(status, error_message, clear_output_dir)+ # Create the row if a start-time create failed; skipped while the pump is mid-create, whose create-+ # then-finalize records the run instead.self._ensure_db_run_created()with self._provenance_lock:with self._lock:@@ -2669,6 +2717,10 @@ class TrainingBackend:self._model_download_repo_id = Noneself._event_queue = event_queueself._stop_queue = stop_queue+ # Member stops sent to the stalled worker's queue died with it; under the lock no new one is lost.+ for run_id, member in self._batch_members.items():+ if member["stop_requested"] and not member["terminal"]:+ stop_queue.put({"type": "stop_member", "run_id": run_id, "save": True})self._proc = new_procself._spawn_in_progress = Falseself._pump_thread = new_pump@@ -3088,6 +3140,9 @@ class TrainingBackend:if etype == "stall":self._handle_stall_event(event)return+ if event.get("member_run_id"):+ self._handle_member_event(event)+ returnwith self._lock:if etype == "progress":@@ -3505,6 +3560,12 @@ class TrainingBackend:"""Claims the finalize under the lock so the watchdog and pump cannot both finalize the same run."""if account_is_retired():return+ if self._batch_members:+ with self._lock:+ if expected_job_id is not None and self.current_job_id != expected_job_id:+ return+ self._finalize_batch_members(status, error_message, clear_output_dir)+ returnwith self._provenance_lock:with self._lock:if expected_job_id is not None and self.current_job_id != expected_job_id:@@ -3592,6 +3653,249 @@ class TrainingBackend:self._metric_buffer[:0] = batchlogger.warning("Failed to flush metrics to DB", exc_info = True)+ @staticmethod+ def _new_batch_member(config: dict, started_at: str) -> dict:+ return {+ "config": config,+ "started_at": started_at,+ "output_dir": None,+ "created": False,+ "terminal": False,+ "stop_requested": False,+ "cancelled": False,+ "total_steps": None,+ "total_steps_set": False,+ "step": 0,+ "loss": None,+ "elapsed": None,+ "loss_history": [],+ "metrics": [],+ }++ def batch_member_ids(self) -> list[str]:+ with self._lock:+ return [run_id for run_id, m in self._batch_members.items() if not m["terminal"]]++ @job_control+ def stop_member(self, run_id: str) -> bool:+ with self._lock:+ member = self._batch_members.get(run_id)+ if member is None or member["terminal"] or self._stop_queue is None:+ return False+ try:+ self._stop_queue.put({"type": "stop_member", "run_id": run_id, "save": True})+ except (OSError, ValueError):+ return False+ member["stop_requested"] = True+ return True++ def _cancel_batch_members(self) -> None:+ from storage.studio_db import mark_run_cancel_requested+ with self._lock:+ running = [+ (run_id, m) for run_id, m in self._batch_members.items() if not m["terminal"]+ ]+ for _, member in running:+ member["cancelled"] = True+ for run_id, member in running:+ if member["created"]:+ mark_run_cancel_requested(run_id)++ def _handle_member_event(self, event: dict) -> None:+ etype = event.get("type")+ run_id = event["member_run_id"]+ with self._lock:+ member = self._batch_members.get(run_id)+ if member is None or member["terminal"]:+ return+ if etype == "output_dir":+ member["output_dir"] = event.get("output_dir")+ elif etype == "progress":+ step = event.get("step", 0)+ loss = _finite_or_none(event.get("loss"))+ if step > 0 and loss is not None:+ member["loss_history"].append(loss)+ member["step"] = step+ member["loss"] = loss+ member["elapsed"] = event.get("elapsed_seconds", member["elapsed"])+ member["total_steps"] = event.get("total_steps") or member["total_steps"]+ member["metrics"].append(+ {+ "step": step,+ "loss": loss,+ "learning_rate": _finite_or_none(event.get("learning_rate")),+ "grad_norm": _finite_or_none(event.get("grad_norm")),+ "eval_loss": None,+ "epoch": event.get("epoch"),+ "num_tokens": event.get("num_tokens"),+ "elapsed_seconds": event.get("elapsed_seconds"),+ }+ )+ p = self._progress+ p.is_training = True+ p.step = step+ p.epoch = event.get("epoch", p.epoch)+ p.loss = loss+ p.total_steps = event.get("total_steps", p.total_steps)+ p.elapsed_seconds = event.get("elapsed_seconds", p.elapsed_seconds)+ p.eta_seconds = event.get("eta_seconds", p.eta_seconds)+ if event.get("status_message"):+ p.status_message = event["status_message"]+ if etype == "output_dir":+ self._persist_member(run_id, output_dir_changed = True)+ elif etype == "progress":+ self._persist_member(run_id, flush = len(member["metrics"]) >= self.FLUSH_THRESHOLD)+ elif etype == "complete":+ status = "stopped" if member["stop_requested"] else "completed"+ self._finish_member(run_id, status, output_dir = event.get("output_dir"))+ elif etype == "stopped":+ self._finish_member(+ run_id,+ "stopped",+ output_dir = event.get("output_dir"),+ clear_output_dir = not event.get("saved", True),+ )+ elif etype == "error":+ logger.error("Training error for batch member %s: %s", run_id, event.get("error"))+ self._finish_member(run_id, "error", error_message = event.get("error", "Unknown error"))++ def _persist_member(+ self,+ run_id: str,+ flush: bool = False,+ output_dir_changed: bool = False,+ ) -> None:+ if account_is_retired():+ return+ from storage.studio_db import (+ create_run,+ insert_metrics_batch,+ update_run_output_dir,+ update_run_progress,+ update_run_total_steps,+ )++ member = self._batch_members[run_id]+ try:+ if not member["created"]:+ if not member["metrics"] and not member["terminal"]:+ return+ config = member["config"]+ create_run(+ id = run_id,+ model_name = config["model_name"],+ dataset_name = config.get("hf_dataset")+ or next(iter(config.get("local_datasets") or []), None)+ or "unknown",+ config_json = _json.dumps(config),+ started_at = member["started_at"],+ total_steps = member["total_steps"],+ output_dir = member["output_dir"],+ cancel_requested = member["cancelled"],+ )+ member["created"] = True+ member["total_steps_set"] = bool(member["total_steps"])+ flush = True+ elif output_dir_changed and member["output_dir"] and not member["cancelled"]:+ update_run_output_dir(run_id, member["output_dir"])+ if member["total_steps"] and not member["total_steps_set"]:+ update_run_total_steps(run_id, member["total_steps"])+ member["total_steps_set"] = True+ if flush and member["metrics"]:+ with self._lock:+ batch = list(member["metrics"])+ del member["metrics"][: len(batch)]+ step, loss, elapsed = member["step"], member["loss"], member["elapsed"]+ try:+ insert_metrics_batch(run_id, batch)+ update_run_progress(id = run_id, step = step, loss = loss, duration_seconds = elapsed)+ except Exception:+ with self._lock:+ member["metrics"][:0] = batch+ raise+ except Exception:+ logger.warning("Failed to persist batch member %s", run_id, exc_info = True)++ def _finish_member(+ self,+ run_id: str,+ status: str,+ *,+ output_dir: Optional[str] = None,+ error_message: Optional[str] = None,+ clear_output_dir: bool = False,+ ) -> None:+ with self._lock:+ member = self._batch_members.get(run_id)+ if member is None or member["terminal"]:+ return+ member["terminal"] = True+ if output_dir:+ member["output_dir"] = output_dir+ clear_output_dir = clear_output_dir or member["cancelled"]+ # Retried like the finalize below: a member that fails before its first step needs its row created.+ for attempt in range(_DB_FINALIZE_RETRIES):+ self._persist_member(run_id, flush = True)+ if member["created"] or account_is_retired() or attempt + 1 == _DB_FINALIZE_RETRIES:+ break+ time.sleep(_DB_FINALIZE_RETRY_S)+ if not member["created"] or account_is_retired():+ return+ from storage.studio_db import finish_run+ from utils.downsample import downsample++ # Retried like a lone run's finalize: a transient SQLite lock must not leave the row running.+ for attempt in range(_DB_FINALIZE_RETRIES):+ try:+ finish_run(+ id = run_id,+ status = status,+ ended_at = datetime.now(timezone.utc).isoformat(),+ final_step = member["step"],+ final_loss = member["loss"],+ duration_seconds = member["elapsed"],+ loss_sparkline = _json.dumps(downsample(member["loss_history"], 50)),+ output_dir = None if clear_output_dir else member["output_dir"],+ error_message = error_message,+ clear_output_dir = clear_output_dir,+ )+ return+ except Exception:+ if attempt + 1 < _DB_FINALIZE_RETRIES:+ time.sleep(_DB_FINALIZE_RETRY_S)+ continue+ logger.warning("Failed to finalize batch member %s", run_id, exc_info = True)++ def _finalize_batch_members(+ self,+ status: str,+ error_message: Optional[str],+ clear_output_dir: bool = False,+ ) -> None:+ # Process-level ending: only members without their own terminal event are finalized here.+ with self._lock:+ running = [+ (run_id, m) for run_id, m in self._batch_members.items() if not m["terminal"]+ ]+ for run_id, member in running:+ saved = bool(member["output_dir"]) and os.path.isfile(+ os.path.join(member["output_dir"], "adapter_config.json")+ )+ if status == "error":+ self._finish_member(+ run_id,+ "error",+ error_message = error_message or "Training process terminated unexpectedly",+ )+ elif clear_output_dir or member["cancelled"]:+ self._finish_member(run_id, "stopped", clear_output_dir = True)+ elif saved:+ self._finish_member(run_id, status)+ else:+ self._finish_member(+ run_id, "error", error_message = "Training ended without saving this job."+ )+@staticmethoddef _read_queue(q: Any, timeout_sec: float) -> Optional[dict]:try:diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.pyindex 64d6cad1eca..027ea83f941 100644--- a/studio/backend/core/training/worker.py+++ b/studio/backend/core/training/worker.py@@ -4701,6 +4701,21 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->)return+ if config.get("batch_members"):+ try:+ from core.training.multi_lora_trainer import run_multi_lora_training+ return run_multi_lora_training(event_queue, stop_queue, config)+ except Exception as exc:+ event_queue.put(+ {+ "type": "error",+ "error": str(exc),+ "stack": traceback.format_exc(limit = 20),+ "ts": time.time(),+ }+ )+ return+# Embedding models use a different pipeline (FastSentenceTransformer + SentenceTransformerTrainer +# MultipleNegativesRankingLoss), so branch early.if config.get("is_embedding", False):diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.pyindex db854385283..511b3838ae4 100644--- a/studio/backend/routes/openai_fine_tuning.py+++ b/studio/backend/routes/openai_fine_tuning.py@@ -15,7 +15,9 @@ training is idle. The queue lives in this process: queued jobs do not survive afrom __future__ import annotationsimport asyncio+import importlib.utilimport json+import osimport reimport secretsimport weakref@@ -280,6 +282,9 @@ _STARTED: dict[str, tuple[dict[str, Any], int]] = {}_STARTED_LIMIT = 16# Cancelled here while the worker finishes its step: the run row reads "running" until it stops._CANCELLING: set[str] = set()+# Multi-LoRA batches started here: member run id -> batch job id, batch job id -> member run ids.+_BATCH_OF: dict[str, str] = {}+_BATCHES: dict[str, list[str]] = {}def _remember_start(run_id: str, config: dict[str, Any], created_at: int) -> None:@@ -288,6 +293,14 @@ def _remember_start(run_id: str, config: dict[str, Any], created_at: int) -> Non_STARTED.pop(next(iter(_STARTED)))+def _remember_batch(batch_id: str, run_ids: list[str]) -> None:+ _BATCHES[batch_id] = list(run_ids)+ _BATCH_OF.update({run_id: batch_id for run_id in run_ids})+ while len(_BATCHES) > _STARTED_LIMIT:+ for run_id in _BATCHES.pop(next(iter(_BATCHES))):+ _BATCH_OF.pop(run_id, None)++def _for_caller(job: dict[str, Any], via_api_key: bool) -> dict[str, Any]:"""An API key never sees host paths: a local model becomes its handle, an error loses its paths."""if host_paths_visible(via_api_key):@@ -320,12 +333,16 @@ async def _job(run_id: str, current_subject: str, via_api_key: bool) -> dict[strreturn _for_caller(await asyncio.to_thread(_job_from_run, run), via_api_key)# A started job has no history row until its worker is up; the live status still knows it.status = await _current_status(current_subject, via_api_key)- if status is not None and _status_field(status, "job_id") == run_id:+ if status is not None and _status_field(status, "job_id") == _BATCH_OF.get(run_id, run_id):phase = _status_field(status, "phase")failed = _status_field(status, "start_request_state") == "rejected" or phase == "error"message = _status_field(status, "error") or _status_field(status, "message")config, created_at = _STARTED.get(run_id, ({}, None))state = _PHASE_STATUS.get(phase, "queued")+ if run_id in _BATCH_OF and state in ("queued", "running"):+ # A member's row appears with its first step; until then the batch's phase (read from its+ # setup messages) would flip it between running and validating_files.+ state = "validating_files"if failed:state = "failed"elif run_id in _CANCELLING:@@ -425,6 +442,26 @@ _TRANSIENT_CONFLICTS = ("is replacing the latest sidecar",)_ALIAS_TABLE = "openai_fine_tuning_aliases"+# Jobs that may differ only in these train as one multi-LoRA batch; every other training setting must match.+_BATCH_MEMBER_FIELDS = frozenset(+ {+ "local_datasets",+ "learning_rate",+ "num_epochs",+ "batch_size",+ "lora_r",+ "lora_alpha",+ "random_seed",+ "project_name",+ "max_steps",+ # Each member has its own optimizer group and scheduler, so these stay per job too.+ "weight_decay",+ "warmup_steps",+ "warmup_ratio",+ }+)+_BATCH_OPTIMIZERS = ("adamw_8bit", "adamw_torch")+_BATCH_MAX_DEFAULT = 4@dataclass@@ -544,6 +581,129 @@ def _queued_job(entry: _Queued) -> dict[str, Any]:return {**job, "id": entry.alias}+def _batch_max() -> int:+ try:+ return max(1, int(os.environ.get("UNSLOTH_STUDIO_FT_BATCH_MAX", _BATCH_MAX_DEFAULT)))+ except ValueError:+ return _BATCH_MAX_DEFAULT+++_multi_lora: Optional[bool] = None+++def _multi_lora_available() -> bool:+ """Whether the installed Unsloth ships the multi-LoRA kernels; looked up on disk, so torch is not imported."""+ global _multi_lora+ if _multi_lora is None:+ try:+ spec = importlib.util.find_spec("unsloth")+ roots = list(spec.submodule_search_locations or []) if spec is not None else []+ except (ImportError, ValueError):+ roots = []+ _multi_lora = any(+ (Path(root) / "kernels" / "multi_lora.py").is_file()+ or (Path(root) / "kernels" / "multi_lora" / "__init__.py").is_file()+ for root in roots+ )+ return _multi_lora+++def _batchable(request: TrainingStartRequest) -> bool:+ """Text LoRA/QLoRA on local files with nothing the batched trainer does not run; the rest trains alone."""+ from core.training.eval_dataset import evaluation_enabled+ from core.training.training import should_use_mlx_training_backend++ return (+ not should_use_mlx_training_backend()+ and request.training_type == "LoRA/QLoRA"+ and request.use_lora+ and bool(request.local_datasets)+ and not (request.hf_dataset or request.dataset_streaming or request.s3_config is not None)+ and not (request.resume_from_checkpoint or request.start_request_id)+ and not (request.local_eval_datasets or evaluation_enabled(request.eval_steps))+ and not request.packing+ and request.lora_dropout == 0+ and not (request.use_dora or request.use_loftq)+ and not (+ request.enable_wandb or request.enable_tensorboard or request.early_stopping_patience+ )+ and request.lr_scheduler_type != "reduce_lr_on_plateau"+ and not request.offload_layers+ and not (+ request.is_dataset_image+ or request.is_dataset_audio+ or request.is_embedding+ or request.is_decision+ or request.finetune_vision_layers+ )+ and not (request.drafter is not None and request.drafter.enabled)+ and request.optim in _BATCH_OPTIMIZERS+ and len(request.gpu_ids or []) <= 1+ )+++def _batch_key(entry: _Queued) -> Optional[tuple]:+ if not _batchable(entry.request):+ return None+ shared = entry.request.model_dump(mode = "json", exclude = set(_BATCH_MEMBER_FIELDS))+ return (+ entry.account.account_id,+ entry.subject,+ entry.via_api_key,+ json.dumps(shared, sort_keys = True),+ )+++_MOE_COUNT_KEYS = ("num_experts", "n_routed_experts", "num_local_experts")+++def _moe_config(config: dict) -> bool:+ for cfg in (config, config.get("text_config") or {}):+ count = next((cfg[k] for k in _MOE_COUNT_KEYS if cfg.get(k) is not None), None)+ if count is not None:+ return isinstance(count, int) and count > 0+ return False+++async def _trains_alone(request: TrainingStartRequest) -> bool:+ """Outside what the batched trainer runs: text-diffusion models (own denoising objective), saved LoRA+ adapters (continued in place alone) and MoE models (fused experts train via target_parameters alone)."""+ from utils.models.diffusion_lm import diffusion_lm_profile_for_model+ from utils.models.model_config import _read_raw_config_json, get_base_model_from_lora_identifier++ def probe() -> bool:+ if diffusion_lm_profile_for_model(request.model_name, request.hf_token):+ return True+ if get_base_model_from_lora_identifier(request.model_name, request.hf_token) is not None:+ return True+ try:+ config = _read_raw_config_json(request.model_name, request.hf_token)+ except Exception:+ return True+ return config is None or _moe_config(config)++ return await asyncio.to_thread(probe)+++def _batch_for(head: _Queued) -> list[_Queued]:+ """The queue head plus the compatible jobs waiting behind it, in queue order, up to the cap."""+ cap = _batch_max()+ key = _batch_key(head) if cap > 1 and not head.cancel_requested else None+ if key is None:+ return [head]+ group = [head]+ for entry in _QUEUE:+ if len(group) >= cap:+ break+ if entry is head or entry.status != "queued" or entry.cancel_requested:+ continue+ if _batch_key(entry) == key:+ group.append(entry)+ if len(group) > 1 and not _multi_lora_available():+ return [head]+ return group++def _finish(entry: _Queued,status: str,@@ -568,6 +728,13 @@ async def _start_queued(entry: _Queued) -> bool:return Trueif _training_busy():return False+ group = _batch_for(entry)+ if len(group) > 1 and await _trains_alone(entry.request):+ group = [entry]+ if len(group) > 1:+ outcome = await _start_batch(group)+ if outcome is not None:+ return outcomeentry.status = "starting"try:started = await arun_as(@@ -614,6 +781,57 @@ async def _start_queued(entry: _Queued) -> bool:return True+async def _start_batch(group: list[_Queued]) -> Optional[bool]:+ """Start ``group`` as one multi-LoRA run, each job with its own run. None: start the head alone instead,+ which reports any refusal exactly as an unbatched start does; False: training is busy."""+ from core.training import get_training_backend+ from routes import training as training_routes++ head = group[0]+ for member in group:+ member.status = "starting"+ try:+ started = await arun_as(+ head.account,+ training_routes.start_training_batch(+ [member.request for member in group],+ current_subject = head.subject,+ via_api_key = head.via_api_key,+ ),+ )+ except Exception as exc: # noqa: BLE001+ logger.info("Fine-tuning batch did not start (%s); starting %s alone", exc, head.alias)+ started = None+ if started is None or any(r.status == "error" or not r.job_id for r in started):+ for member in group:+ member.status = "queued"+ if member.cancel_requested and member is not head:+ _finish(member, "cancelled")+ if started is not None and (started[0].error in _BUSY_ERRORS or _training_busy()):+ if head.cancel_requested:+ _finish(head, "cancelled")+ return True+ return False+ return None+ run_ids = [r.job_id for r in started]+ batch_id = getattr(get_training_backend(), "current_job_id", None)+ if batch_id:+ _remember_batch(batch_id, run_ids)+ for member, run_id in zip(group, run_ids):+ await arun_as(+ member.account,+ asyncio.to_thread(+ _save_alias, member.alias, run_id, member.account.account_id, member.created_at+ ),+ )+ _remember_start(run_id, member.request.model_dump(), member.created_at)+ _finish(member, "started")+ for member, run_id in zip(group, run_ids):+ if member.cancel_requested:+ await arun_as(member.account, _cancel_run(run_id, member.subject))+ return True++async def _drain_once() -> None:if not _QUEUE:return@@ -684,6 +902,34 @@ async def _stop_run(run_id: str, current_subject: str) -> Any:return stopped+async def _cancel_run(run_id: str, current_subject: str) -> Any:+ """A job training in a batch beside others still running stops alone; otherwise the whole run stops."""+ batch_id = _BATCH_OF.get(run_id)+ if batch_id is None:+ return await _stop_run(run_id, current_subject)+ from core.training import get_training_backend+ from core.training.account_jobs import require_job_owner+ from models.responses import TrainingStopResponse++ backend = get_training_backend()+ require_job_owner(backend)+ running = await asyncio.to_thread(backend.batch_member_ids)+ if run_id in running and len(running) > 1:+ if await asyncio.to_thread(backend.stop_member, run_id):+ _CANCELLING.add(run_id)+ return TrainingStopResponse(+ status = "stopped",+ message = "Stop requested. This job will stop at the next safe step.",+ )+ running = await asyncio.to_thread(backend.batch_member_ids)+ if run_id not in running:+ return await _stop_run(run_id, current_subject)+ stopped = await _stop_run(batch_id, current_subject)+ if stopped.status != "idle":+ _CANCELLING.add(run_id)+ return stopped++@router.post("/fine_tuning/jobs")async def create_fine_tuning_job(body: FineTuningJobCreate,@@ -780,18 +1026,24 @@ async def list_fine_tuning_jobs(pending = []after_run = await _resolve(after) if after else Noneruns = await asyncio.to_thread(_runs_after, after_run, limit + 1)- if after_run and not runs:- # The live job has no history row yet and is listed first, so the history follows it.- status = await _current_status(current_subject, via_api_key)- if status is not None and _status_field(status, "job_id") == after_run:- runs = await asyncio.to_thread(_runs_after, None, limit + 1)+ status = await _current_status(current_subject, via_api_key)+ live_id = _status_field(status, "job_id") if status is not None else None+ # Live jobs without a history row yet (a batch's members while it sets up) are listed first.+ live = []+ if live_id and _status_field(status, "phase") != "idle":+ live = [+ r+ for r in reversed(_BATCHES.get(live_id, [live_id]))+ if await asyncio.to_thread(get_run, r) is None+ ]+ if after_run in live:+ # The history follows the live jobs, so a cursor on one continues with the rest of them.+ live = live[live.index(after_run) + 1 :]+ runs = await asyncio.to_thread(_runs_after, None, limit + 1)+ elif after_run:+ live = []jobs = [_for_caller(await asyncio.to_thread(_job_from_run, run), via_api_key) for run in runs]- if after is None:- status = await _current_status(current_subject, via_api_key)- live_id = _status_field(status, "job_id") if status is not None else None- live_phase = _status_field(status, "phase") if status is not None else None- if live_id and live_phase != "idle" and all(run["id"] != live_id for run in runs):- jobs.insert(0, await _job(live_id, current_subject, via_api_key))+ jobs = [await _job(r, current_subject, via_api_key) for r in live] + jobs# A job that ended before it started sorts by its creation; once a newer job ran, it is only retrievable.newest = max((job["created_at"] for job in jobs), default = 0)jobs = [@@ -1007,7 +1259,7 @@ async def cancel_fine_tuning_job(job = await _job(run_id, current_subject, via_api_key)if job["status"] in ("succeeded", "failed", "cancelled"):raise _bad_request(f"Job {job['id']} has already {job['status']}.")- stopped = await _stop_run(run_id, current_subject)+ stopped = await _cancel_run(run_id, current_subject)if stopped.status == "idle":raise _bad_request(f"Job {job['id']} is not running.", 409)return {**job, "status": "cancelled"}diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.pyindex d7737da72ba..2aef44a987d 100644--- a/studio/backend/routes/training.py+++ b/studio/backend/routes/training.py@@ -16,6 +16,7 @@ from core.training.account_jobs import (validate_job_paths,)import contextlib+import contextvarsimport jsonimport osimport re@@ -1898,18 +1899,7 @@ def _background_video_generation_active() -> bool:return False[email protected]("/start", responses = _TRAINING_START_ERROR_RESPONSES)-async def start_training(- request: TrainingStartRequest,- current_subject: str = Depends(get_current_subject),- via_api_key: bool = Depends(authenticated_via_api_key),-):- """- Start a training job.-- Initiates training in the background and returns immediately. Use /status- to check progress.- """+async def _prepare_training_request(request: TrainingStartRequest, via_api_key: bool) -> None:await asyncio.to_thread(_validate_decision_request, request, via_api_key)if request.drafter is not None and request.drafter.enabled:from core.training.training import should_use_mlx_training_backend@@ -1929,6 +1919,382 @@ async def start_training(if not directory or not Path(directory).is_absolute():request.tensorboard_dir = str(tensorboard_root() / (directory or ""))validate_job_paths(request.model_dump())+++@dataclass+class _AdmittedTrainingRequest:+ training_kwargs: dict+ hf_token: Optional[str]+ resume_run: Optional[dict]+++async def _admit_training_request(+ request: TrainingStartRequest, current_subject: str, via_api_key: bool+) -> _AdmittedTrainingRequest:+ """Per-request validation and worker kwargs of a start; raises the start route's own errors."""+ resume_output_dir: Optional[str] = None+ resume_run: Optional[dict] = None+ resume_actual_model_repo_id: Optional[str] = None+ resume_model_load_mode: Optional[str] = None+ resume_requires_exact_resources = False+ resume_requires_exact_model = False+ resume_requires_exact_dataset = False+ if request.resume_from_checkpoint:+ try:+ resume_output_dir = await asyncio.to_thread(+ normalize_resume_output_dir,+ request.resume_from_checkpoint,+ )+ except ValueError as e:+ # Deliberate user-facing validation message.+ validation_message = str(e)+ raise HTTPException(status_code = 400, detail = validation_message)++ resume_run = await asyncio.to_thread(+ get_resumable_run_by_output_dir,+ resume_output_dir,+ )+ if not resume_run or not await asyncio.to_thread(can_resume_run, resume_run):+ detail = "Resume checkpoint must belong to a stopped or errored run with complete saved trainer state."+ # Only when the checkpoint itself is intact: can_resume_run refuses for several reasons, so asking+ # unconditionally answers a provenance sentence when the checkpoint is what is missing.+ # has_resume_state is the discriminator can_resume_run itself short-circuits on.+ if resume_run and await asyncio.to_thread(+ has_resume_state, resume_run.get("output_dir")+ ):+ from core.training.provenance import (+ resource_provenance_resume_blocker,+ )+ blocker = await asyncio.to_thread(+ resource_provenance_resume_blocker,+ training_run_config(resume_run),+ )+ if blocker:+ detail = blocker+ raise HTTPException(status_code = 400, detail = detail)+ resume_checkpoint = await asyncio.to_thread(+ get_resume_checkpoint_path,+ resume_output_dir,+ )+ if not resume_checkpoint:+ raise HTTPException(+ status_code = 400,+ detail = "Resume checkpoint must include saved trainer state.",+ )+ request.resume_from_checkpoint = resume_checkpoint+ (+ resume_actual_model_repo_id,+ resume_requires_exact_model,+ resume_requires_exact_dataset,+ resume_requires_exact_resources,+ resume_model_load_mode,+ ) = await asyncio.to_thread(+ _prepare_resume_resource_provenance,+ request,+ resume_run,+ )++ if request.local_datasets:+ request.local_datasets = _validate_local_dataset_paths(+ request.local_datasets, "Local dataset"+ )+ # Not `eval_steps > 0`: inf passes that but reads as disabled everywhere in the trainer.+ if request.local_eval_datasets and evaluation_enabled(request.eval_steps):+ request.local_eval_datasets = _validate_local_dataset_paths(+ request.local_eval_datasets, "Local eval dataset"+ )++ from utils.hardware import hardware as _hw+ from utils.hardware import ensure_hardware_detected++ await asyncio.to_thread(ensure_hardware_detected)+ _validate_training_platform(request)++ if request.dataset_streaming:+ if not request.hf_dataset:+ raise HTTPException(+ status_code = 400,+ detail = "dataset_streaming requires hf_dataset; streaming is not supported for local datasets.",+ )+ if request.is_dataset_image or request.is_dataset_audio:+ raise HTTPException(+ status_code = 400,+ detail = "dataset_streaming is not supported for vision or audio datasets.",+ )+ if request.is_embedding:+ raise HTTPException(+ status_code = 400,+ detail = "dataset_streaming is not supported for embedding training; the embedding loader needs the full dataset.",+ )+ if _hw.DEVICE == _hw.DeviceType.MLX:+ raise HTTPException(+ status_code = 400,+ detail = "dataset_streaming is not yet supported on Apple Silicon (MLX); the MLX loader materializes the full dataset.",+ )+ if request.max_steps is None or request.max_steps <= 0:+ raise HTTPException(+ status_code = 422,+ detail = "dataset_streaming requires max_steps > 0 because streaming datasets have no known length.",+ )+ if request.train_on_completions:+ raise HTTPException(+ status_code = 422,+ detail = "dataset_streaming is not supported with train_on_completions yet.",+ )+ if evaluation_enabled(request.eval_steps):+ train_split = request.train_split or "train"+ if not request.eval_split or request.eval_split == train_split:+ raise HTTPException(+ status_code = 422,+ detail = "dataset_streaming with evaluation requires a separate eval_split.",+ )+ # Streaming is HF-only: reject when the request also carries a local dataset path or an+ # S3 config, since those sources cannot be streamed via HF's loader.+ if request.local_datasets:+ raise HTTPException(+ status_code = 400,+ detail = (+ "dataset_streaming is HF-only; remove local_datasets / S3 source. "+ "Streaming is not supported with local file paths."+ ),+ )+ if request.s3_config is not None:+ raise HTTPException(+ status_code = 400,+ detail = (+ "dataset_streaming is HF-only; remove local_datasets / S3 source. "+ "Streaming is not supported with S3 datasets."+ ),+ )+ if request.dataset_known_cached or request.dataset_local_path:+ raise HTTPException(+ status_code = 422,+ detail = (+ "dataset_streaming streams from the Hub and cannot use the local "+ "dataset cache; disable streaming to train from the cached copy."+ ),+ )+ allow_ambient = via_api_key is not True+ hf_token = hf_token_arg(request.hf_token, allow_ambient_token = allow_ambient)+ # The local dataset paths were resolved above for existence only; authorize them now that+ # the caller's token is known, before anything reads them.+ # Eval paths only while evaluation is on, matching the validation above and the trainer,+ # which reads them under the same condition: a stale eval path nothing will open must not+ # refuse an otherwise valid run.+ for _local_paths, _label in (+ (request.local_datasets, "dataset"),+ (+ request.local_eval_datasets if evaluation_enabled(request.eval_steps) else [],+ "eval_dataset",+ ),+ ):+ if _local_paths:+ await asyncio.to_thread(+ _refuse_unauthorized_cached_local_paths, _local_paths, hf_token, _label+ )+ model_preflight = await asyncio.to_thread(+ _reject_untrainable_model_request,+ request,+ resume_actual_model_repo_id,+ hf_token,+ )+ cached_model_pin = model_preflight.cached_model_pin+ training_actual_model_repo_id = resume_actual_model_repo_id+ training_model_snapshot_path = request.model_snapshot_path+ if cached_model_pin is not None:+ training_actual_model_repo_id, training_model_snapshot_path = cached_model_pin++ if request.hf_dataset:+ await asyncio.to_thread(_preflight_hf_dataset_request, request)+ # After the preflight: a cache it pinned is still on disk for this scan to refuse.+ # Not for a streaming start: it reads the Hub (load_dataset(streaming = True)) and never+ # the materialized cache, the preflight above always verified the repo remotely, and+ # the route already refuses streaming with a pinned cache. Asking anyway would refuse a+ # valid start over an unrelated cached copy whenever /auth-check cannot be reached.+ # And only when the Hub repo is the EFFECTIVE source: load_and_format_dataset takes+ # local_datasets (and s3_config below it) ahead of dataset_source, so a payload carrying+ # both never reads the Hub cache, and refusing over it fails a run on a dataset it does+ # not use. The local paths took their own authorization pass above.+ if not request.dataset_streaming and _hf_dataset_is_the_source(request):+ await asyncio.to_thread(_refuse_unauthorized_cached_dataset, request, hf_token)++ device_backend = getattr(_hw.DEVICE, "value", "") or ""+ training_optimizer = normalize_training_optimizer_for_device(+ request.optim,+ device_backend = device_backend,+ )++ training_kwargs = {+ "model_name": model_preflight.model_name,+ "project_name": request.project_name,+ "training_type": request.training_type,+ "hf_token": (request.hf_token or "").strip(),+ "allow_ambient": allow_ambient,+ "load_in_4bit": request.load_in_4bit,+ "max_seq_length": request.max_seq_length,+ "vision_image_size": request.vision_image_size,+ "hf_dataset": request.hf_dataset or "",+ "model_known_cached": request.model_known_cached,+ "model_local_path": model_preflight.model_local_path,+ "model_format": request.model_format,+ "model_snapshot_path": training_model_snapshot_path,+ "actual_model_repo_id": training_actual_model_repo_id,+ "resume_model_load_mode": resume_model_load_mode,+ "dataset_known_cached": request.dataset_known_cached,+ "dataset_local_path": request.dataset_local_path,+ "dataset_snapshot_path": request.dataset_snapshot_path,+ "local_datasets": request.local_datasets,+ "local_eval_datasets": request.local_eval_datasets,+ "format_type": request.format_type,+ "subset": request.subset,+ "train_split": request.train_split,+ "dataset_streaming": request.dataset_streaming,+ "eval_split": request.eval_split,+ "eval_steps": request.eval_steps,+ "early_stopping_patience": request.early_stopping_patience,+ "drafter": (+ request.drafter.model_dump()+ if request.drafter is not None and request.drafter.enabled+ else None+ ),+ "dataset_slice_start": request.dataset_slice_start,+ "dataset_slice_end": request.dataset_slice_end,+ "custom_format_mapping": request.custom_format_mapping,+ "num_epochs": request.num_epochs,+ "learning_rate": request.learning_rate,+ "embedding_learning_rate": request.embedding_learning_rate,+ "batch_size": request.batch_size,+ "gradient_accumulation_steps": request.gradient_accumulation_steps,+ "warmup_steps": request.warmup_steps,+ "warmup_ratio": request.warmup_ratio,+ "max_steps": request.max_steps,+ "save_steps": request.save_steps,+ "weight_decay": request.weight_decay,+ "max_grad_norm": request.max_grad_norm,+ "max_grad_value": request.max_grad_value,+ "max_grad_leaf_norm": request.max_grad_leaf_norm,+ "cast_norm_output_to_input_dtype": request.cast_norm_output_to_input_dtype,+ "random_seed": request.random_seed,+ "packing": request.packing,+ "optim": training_optimizer,+ "lr_scheduler_type": request.lr_scheduler_type,+ "use_lora": request.use_lora,+ "lora_r": request.lora_r,+ "lora_alpha": request.lora_alpha,+ "lora_dropout": request.lora_dropout,+ "target_modules": request.target_modules if request.target_modules else None,+ "gradient_checkpointing": request.gradient_checkpointing.strip()+ if request.gradient_checkpointing and request.gradient_checkpointing.strip()+ else "unsloth",+ "offload_layers": request.offload_layers,+ "offload_vram_gb": request.offload_vram_gb,+ "offload_vram_gb_per_device": request.offload_vram_gb_per_device,+ "prefetch_depth": request.prefetch_depth,+ "use_rslora": request.use_rslora,+ "use_loftq": request.use_loftq,+ "use_dora": request.use_dora,+ "train_on_completions": request.train_on_completions,+ "finetune_vision_layers": request.finetune_vision_layers,+ "finetune_language_layers": request.finetune_language_layers,+ "finetune_attention_modules": request.finetune_attention_modules,+ "finetune_mlp_modules": request.finetune_mlp_modules,+ "is_dataset_image": request.is_dataset_image,+ "is_dataset_audio": request.is_dataset_audio,+ "is_embedding": request.is_embedding,+ "is_decision": request.is_decision,+ "model_subfolder": request.model_subfolder,+ "decision_layout": request.decision_layout,+ "enable_wandb": request.enable_wandb,+ "wandb_token": request.wandb_token or "",+ "wandb_project": request.wandb_project or "",+ "enable_tensorboard": request.enable_tensorboard,+ "tensorboard_dir": request.tensorboard_dir or "",+ "output_dir": resume_output_dir,+ "resume_from_checkpoint": request.resume_from_checkpoint,+ "require_exact_resume_resources": resume_requires_exact_resources,+ "require_exact_model_resource": resume_requires_exact_model,+ "require_exact_dataset_resource": resume_requires_exact_dataset,+ "require_validated_model_snapshot": cached_model_pin is not None,+ "adapter_base_pin": getattr(model_preflight, "adapter_base_pin", None),+ # A managed account never forwards remote code to the worker.+ "trust_remote_code": request.trust_remote_code and not managed_remote_code_refused(),+ "approved_remote_code_fingerprint": request.approved_remote_code_fingerprint,+ "subject": current_subject,+ "gpu_ids": request.gpu_ids,+ "s3_config": request.s3_config.model_dump() if request.s3_config else None,+ }++ # Latest-sidecar models size and train 16-bit (same flip as chat load): 4-bit is disabled for brand-new+ # architectures, so VRAM checks must not underestimate a load the worker refuses.+ from core.training.provenance import (+ ExactResumeResourcesUnavailable,+ effective_training_load_in_4bit,+ )++ if training_kwargs["load_in_4bit"]:+ from core.training.training import resolve_training_model_load_target++ try:+ model_load_target = await asyncio.to_thread(+ resolve_training_model_load_target,+ training_kwargs,+ )+ except ExactResumeResourcesUnavailable as exc:+ raise HTTPException(status_code = 409, detail = str(exc))+ try:+ effective_load_in_4bit = await asyncio.to_thread(+ effective_training_load_in_4bit,+ training_kwargs,+ model_load_target,+ hf_token,+ )+ except ExactResumeResourcesUnavailable as exc:+ raise HTTPException(status_code = 409, detail = str(exc))+ if not effective_load_in_4bit:+ training_kwargs["load_in_4bit"] = False+ logger.info(+ "Latest-transformers sidecar active for %s - sizing and "+ "training in 16-bit (4-bit is disabled for brand-new "+ "architectures)",+ model_load_target,+ )++ # Training page has no trust_remote_code toggle.+ if not training_kwargs["trust_remote_code"]:+ from utils.security.trusted_org import is_trusted_org_repo++ model_defaults = load_model_defaults(request.model_name)+ yaml_trust = model_defaults.get("training", {}).get("trust_remote_code", False)+ if yaml_trust and is_trusted_org_repo(request.model_name, hf_token = hf_token):+ logger.info(f"YAML config sets trust_remote_code=True for {request.model_name}")+ training_kwargs["trust_remote_code"] = True+ elif yaml_trust:+ logger.warning(+ "YAML sets trust_remote_code=True for %s but it is not a trusted "+ "first-party repo; leaving disabled (user can opt in explicitly).",+ request.model_name,+ )++ return _AdmittedTrainingRequest(+ training_kwargs = training_kwargs,+ hf_token = hf_token,+ resume_run = resume_run,+ )++[email protected]("/start", responses = _TRAINING_START_ERROR_RESPONSES)+async def start_training(+ request: TrainingStartRequest,+ current_subject: str = Depends(get_current_subject),+ via_api_key: bool = Depends(authenticated_via_api_key),+):+ """+ Start a training job.++ Initiates training in the background and returns immediately. Use /status+ to check progress.+ """+ await _prepare_training_request(request, via_api_key)backend = Nonereserved_start_request_id = Nonestart_task: Optional[asyncio.Task[bool]] = None@@ -2023,333 +2389,12 @@ async def start_training(error = "Diffusion training already active",)- resume_output_dir: Optional[str] = None- resume_run: Optional[dict] = None- resume_actual_model_repo_id: Optional[str] = None- resume_model_load_mode: Optional[str] = None- resume_requires_exact_resources = False- resume_requires_exact_model = False- resume_requires_exact_dataset = False- if request.resume_from_checkpoint:- try:- resume_output_dir = await asyncio.to_thread(- normalize_resume_output_dir,- request.resume_from_checkpoint,- )- except ValueError as e:- # Deliberate user-facing validation message.- validation_message = str(e)- raise HTTPException(status_code = 400, detail = validation_message)-- resume_run = await asyncio.to_thread(- get_resumable_run_by_output_dir,- resume_output_dir,- )- if not resume_run or not await asyncio.to_thread(can_resume_run, resume_run):- detail = "Resume checkpoint must belong to a stopped or errored run with complete saved trainer state."- # Only when the checkpoint is intact, else the answer is a misleading provenance sentence.- if resume_run and await asyncio.to_thread(- has_resume_state, resume_run.get("output_dir")- ):- from core.training.provenance import (- resource_provenance_resume_blocker,- )- blocker = await asyncio.to_thread(- resource_provenance_resume_blocker,- training_run_config(resume_run),- )- if blocker:- detail = blocker- raise HTTPException(status_code = 400, detail = detail)- resume_checkpoint = await asyncio.to_thread(- get_resume_checkpoint_path,- resume_output_dir,- )- if not resume_checkpoint:- raise HTTPException(- status_code = 400,- detail = "Resume checkpoint must include saved trainer state.",- )- request.resume_from_checkpoint = resume_checkpoint- (- resume_actual_model_repo_id,- resume_requires_exact_model,- resume_requires_exact_dataset,- resume_requires_exact_resources,- resume_model_load_mode,- ) = await asyncio.to_thread(- _prepare_resume_resource_provenance,- request,- resume_run,- )-- if request.local_datasets:- request.local_datasets = _validate_local_dataset_paths(- request.local_datasets, "Local dataset"- )- # Not `eval_steps > 0`: inf passes that but reads as disabled everywhere in the trainer.- if request.local_eval_datasets and evaluation_enabled(request.eval_steps):- request.local_eval_datasets = _validate_local_dataset_paths(- request.local_eval_datasets, "Local eval dataset"- )-- from utils.hardware import hardware as _hw- from utils.hardware import ensure_hardware_detected-- await asyncio.to_thread(ensure_hardware_detected)- _validate_training_platform(request)-- if request.dataset_streaming:- if not request.hf_dataset:- raise HTTPException(- status_code = 400,- detail = "dataset_streaming requires hf_dataset; streaming is not supported for local datasets.",- )- if request.is_dataset_image or request.is_dataset_audio:- raise HTTPException(- status_code = 400,- detail = "dataset_streaming is not supported for vision or audio datasets.",- )- if request.is_embedding:- raise HTTPException(- status_code = 400,- detail = "dataset_streaming is not supported for embedding training; the embedding loader needs the full dataset.",- )- if _hw.DEVICE == _hw.DeviceType.MLX:- raise HTTPException(- status_code = 400,- detail = "dataset_streaming is not yet supported on Apple Silicon (MLX); the MLX loader materializes the full dataset.",- )- if request.max_steps is None or request.max_steps <= 0:- raise HTTPException(- status_code = 422,- detail = "dataset_streaming requires max_steps > 0 because streaming datasets have no known length.",- )- if request.train_on_completions:- raise HTTPException(- status_code = 422,- detail = "dataset_streaming is not supported with train_on_completions yet.",- )- if evaluation_enabled(request.eval_steps):- train_split = request.train_split or "train"- if not request.eval_split or request.eval_split == train_split:- raise HTTPException(- status_code = 422,- detail = "dataset_streaming with evaluation requires a separate eval_split.",- )- # Streaming is HF-only; a local dataset or S3 config in the same request cannot be streamed.- if request.local_datasets:- raise HTTPException(- status_code = 400,- detail = (- "dataset_streaming is HF-only; remove local_datasets / S3 source. "- "Streaming is not supported with local file paths."- ),- )- if request.s3_config is not None:- raise HTTPException(- status_code = 400,- detail = (- "dataset_streaming is HF-only; remove local_datasets / S3 source. "- "Streaming is not supported with S3 datasets."- ),- )- if request.dataset_known_cached or request.dataset_local_path:- raise HTTPException(- status_code = 422,- detail = (- "dataset_streaming streams from the Hub and cannot use the local "- "dataset cache; disable streaming to train from the cached copy."- ),- )- allow_ambient = via_api_key is not True- hf_token = hf_token_arg(request.hf_token, allow_ambient_token = allow_ambient)- # Authorize local paths now the caller's token is known. Eval paths only while evaluation is on.- for _local_paths, _label in (- (request.local_datasets, "dataset"),- (- request.local_eval_datasets if evaluation_enabled(request.eval_steps) else [],- "eval_dataset",- ),- ):- if _local_paths:- await asyncio.to_thread(- _refuse_unauthorized_cached_local_paths, _local_paths, hf_token, _label- )- model_preflight = await asyncio.to_thread(- _reject_untrainable_model_request,- request,- resume_actual_model_repo_id,- hf_token,- )- cached_model_pin = model_preflight.cached_model_pin- training_actual_model_repo_id = resume_actual_model_repo_id- training_model_snapshot_path = request.model_snapshot_path- if cached_model_pin is not None:- training_actual_model_repo_id, training_model_snapshot_path = cached_model_pin-- if request.hf_dataset:- await asyncio.to_thread(_preflight_hf_dataset_request, request)- # Not for streaming (reads the Hub, never the cache), and only when the Hub repo is the effective- # source: local_datasets / s3_config take precedence in load_and_format_dataset.- if not request.dataset_streaming and _hf_dataset_is_the_source(request):- await asyncio.to_thread(_refuse_unauthorized_cached_dataset, request, hf_token)-- device_backend = getattr(_hw.DEVICE, "value", "") or ""- training_optimizer = normalize_training_optimizer_for_device(- request.optim,- device_backend = device_backend,- )-- training_kwargs = {- "model_name": model_preflight.model_name,- "project_name": request.project_name,- "training_type": request.training_type,- "hf_token": (request.hf_token or "").strip(),- "allow_ambient": allow_ambient,- "load_in_4bit": request.load_in_4bit,- "max_seq_length": request.max_seq_length,- "vision_image_size": request.vision_image_size,- "hf_dataset": request.hf_dataset or "",- "model_known_cached": request.model_known_cached,- "model_local_path": model_preflight.model_local_path,- "model_format": request.model_format,- "model_snapshot_path": training_model_snapshot_path,- "actual_model_repo_id": training_actual_model_repo_id,- "resume_model_load_mode": resume_model_load_mode,- "dataset_known_cached": request.dataset_known_cached,- "dataset_local_path": request.dataset_local_path,- "dataset_snapshot_path": request.dataset_snapshot_path,- "local_datasets": request.local_datasets,- "local_eval_datasets": request.local_eval_datasets,- "format_type": request.format_type,- "subset": request.subset,- "train_split": request.train_split,- "dataset_streaming": request.dataset_streaming,- "eval_split": request.eval_split,- "eval_steps": request.eval_steps,- "early_stopping_patience": request.early_stopping_patience,- "drafter": (- request.drafter.model_dump()- if request.drafter is not None and request.drafter.enabled- else None- ),- "dataset_slice_start": request.dataset_slice_start,- "dataset_slice_end": request.dataset_slice_end,- "custom_format_mapping": request.custom_format_mapping,- "num_epochs": request.num_epochs,- "learning_rate": request.learning_rate,- "embedding_learning_rate": request.embedding_learning_rate,- "batch_size": request.batch_size,- "gradient_accumulation_steps": request.gradient_accumulation_steps,- "warmup_steps": request.warmup_steps,- "warmup_ratio": request.warmup_ratio,- "max_steps": request.max_steps,- "save_steps": request.save_steps,- "weight_decay": request.weight_decay,- "max_grad_norm": request.max_grad_norm,- "max_grad_value": request.max_grad_value,- "max_grad_leaf_norm": request.max_grad_leaf_norm,- "cast_norm_output_to_input_dtype": request.cast_norm_output_to_input_dtype,- "random_seed": request.random_seed,- "packing": request.packing,- "optim": training_optimizer,- "lr_scheduler_type": request.lr_scheduler_type,- "use_lora": request.use_lora,- "lora_r": request.lora_r,- "lora_alpha": request.lora_alpha,- "lora_dropout": request.lora_dropout,- "target_modules": request.target_modules if request.target_modules else None,- "gradient_checkpointing": request.gradient_checkpointing.strip()- if request.gradient_checkpointing and request.gradient_checkpointing.strip()- else "unsloth",- "offload_layers": request.offload_layers,- "offload_vram_gb": request.offload_vram_gb,- "offload_vram_gb_per_device": request.offload_vram_gb_per_device,- "prefetch_depth": request.prefetch_depth,- "use_rslora": request.use_rslora,- "use_loftq": request.use_loftq,- "use_dora": request.use_dora,- "train_on_completions": request.train_on_completions,- "finetune_vision_layers": request.finetune_vision_layers,- "finetune_language_layers": request.finetune_language_layers,- "finetune_attention_modules": request.finetune_attention_modules,- "finetune_mlp_modules": request.finetune_mlp_modules,- "is_dataset_image": request.is_dataset_image,- "is_dataset_audio": request.is_dataset_audio,- "is_embedding": request.is_embedding,- "is_decision": request.is_decision,- "model_subfolder": request.model_subfolder,- "decision_layout": request.decision_layout,- "enable_wandb": request.enable_wandb,- "wandb_token": request.wandb_token or "",- "wandb_project": request.wandb_project or "",- "enable_tensorboard": request.enable_tensorboard,- "tensorboard_dir": request.tensorboard_dir or "",- "output_dir": resume_output_dir,- "resume_from_checkpoint": request.resume_from_checkpoint,- "require_exact_resume_resources": resume_requires_exact_resources,- "require_exact_model_resource": resume_requires_exact_model,- "require_exact_dataset_resource": resume_requires_exact_dataset,- "require_validated_model_snapshot": cached_model_pin is not None,- "adapter_base_pin": getattr(model_preflight, "adapter_base_pin", None),- # A managed account never forwards remote code to the worker.- "trust_remote_code": request.trust_remote_code and not managed_remote_code_refused(),- "approved_remote_code_fingerprint": request.approved_remote_code_fingerprint,- "subject": current_subject,- "gpu_ids": request.gpu_ids,- "s3_config": request.s3_config.model_dump() if request.s3_config else None,- }-- # Latest-sidecar models train 16-bit (4-bit disabled for new archs); size VRAM accordingly.- from core.training.provenance import (- ExactResumeResourcesUnavailable,- effective_training_load_in_4bit,- )-- if training_kwargs["load_in_4bit"]:- from core.training.training import resolve_training_model_load_target-- try:- model_load_target = await asyncio.to_thread(- resolve_training_model_load_target,- training_kwargs,- )- except ExactResumeResourcesUnavailable as exc:- raise HTTPException(status_code = 409, detail = str(exc))- try:- effective_load_in_4bit = await asyncio.to_thread(- effective_training_load_in_4bit,- training_kwargs,- model_load_target,- hf_token,- )- except ExactResumeResourcesUnavailable as exc:- raise HTTPException(status_code = 409, detail = str(exc))- if not effective_load_in_4bit:- training_kwargs["load_in_4bit"] = False- logger.info(- "Latest-transformers sidecar active for %s - sizing and "- "training in 16-bit (4-bit is disabled for brand-new "- "architectures)",- model_load_target,- )-- # Training page has no trust_remote_code toggle.- if not training_kwargs["trust_remote_code"]:- from utils.security.trusted_org import is_trusted_org_repo-- model_defaults = load_model_defaults(request.model_name)- yaml_trust = model_defaults.get("training", {}).get("trust_remote_code", False)- if yaml_trust and is_trusted_org_repo(request.model_name, hf_token = hf_token):- logger.info(f"YAML config sets trust_remote_code=True for {request.model_name}")- training_kwargs["trust_remote_code"] = True- elif yaml_trust:- logger.warning(- "YAML sets trust_remote_code=True for %s but it is not a trusted "- "first-party repo; leaving disabled (user can opt in explicitly).",- request.model_name,- )+ admitted = await _admit_training_request(request, current_subject, via_api_key)+ training_kwargs = admitted.training_kwargs+ hf_token = admitted.hf_token+ resume_run = admitted.resume_run+ # Set only by start_training_batch: this start becomes one multi-LoRA batch.+ batch_members = _batch_members_for_start(training_kwargs)# before_spawn hook: runs only after start_training's guards pass.def _free_vram_for_training() -> None:@@ -2419,14 +2464,15 @@ async def start_training()def _can_keep_resident_models():+ members = batch_members or [training_kwargs]return can_keep_chat_during_training(model_name = training_kwargs["model_name"],hf_token = hf_token,training_type = training_kwargs["training_type"],load_in_4bit = training_kwargs["load_in_4bit"],- batch_size = training_kwargs["batch_size"],+ batch_size = sum(m["batch_size"] for m in members),max_seq_length = training_kwargs["max_seq_length"],- lora_rank = training_kwargs["lora_r"],+ lora_rank = sum(m["lora_r"] for m in members),target_modules = training_kwargs["target_modules"],gradient_checkpointing = training_kwargs["gradient_checkpointing"],optimizer = training_kwargs["optim"],@@ -2443,6 +2489,7 @@ async def start_training(# The hook runs only once start guards pass -> VRAM freed iff training starts.from core.inference.gpu_arbiter import SegmenterStillRunningError+ from core.training.provenance import ExactResumeResourcesUnavailablefrom routes.training_vram import ManagedEngineStillRunningfrom utils.transformers_version import SidecarSwapInProgress@@ -2453,6 +2500,7 @@ async def start_training(start_request_id = request.start_request_id,before_spawn = _free_vram_for_training,resume_source_run_id = resume_run["id"] if resume_run else None,+ **({"batch_members": batch_members} if batch_members else {}),**training_kwargs,)except (@@ -2581,6 +2629,64 @@ async def start_training()+# (first member's run id, the other members' entries) while start_training_batch drives start_training.+_PENDING_BATCH: contextvars.ContextVar[Optional[tuple[str, list[dict]]]] = contextvars.ContextVar(+ "studio_training_pending_batch", default = None+)+++def _batch_members_for_start(training_kwargs: dict) -> Optional[list[dict]]:+ pending = _PENDING_BATCH.get()+ if pending is None:+ return None+ first_run_id, others = pending+ return [{"run_id": first_run_id, **training_kwargs}, *others]+++async def start_training_batch(+ requests: list[TrainingStartRequest], current_subject: str, via_api_key: bool+) -> list[TrainingJobResponse]:+ """Start compatible LoRA requests as one multi-LoRA run; internal, not a route.++ Every member takes every per-request check; the shared ones (busy, inference conflict, install, VRAM, launch)+ run once, through start_training for the first member, whose job id becomes the batch id. One response per+ member, carrying that member's own run id.+ """+ if len(requests) < 2:+ raise ValueError("A training batch needs at least two requests.")+ if any(request.resume_from_checkpoint or request.start_request_id for request in requests):+ raise ValueError("A training batch cannot resume or carry a start_request_id.")+ run_ids = [+ f"job_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{_uuid.uuid4().hex[:8]}" for _ in requests+ ]+ others = []+ for run_id, request in zip(run_ids[1:], requests[1:]):+ await _prepare_training_request(request, via_api_key)+ try:+ admitted = await _admit_training_request(request, current_subject, via_api_key)+ except ValueError as e:+ raise HTTPException(status_code = 400, detail = str(e))+ others.append({"run_id": run_id, **admitted.training_kwargs})+ token = _PENDING_BATCH.set((run_ids[0], others))+ try:+ started = await start_training(+ requests[0], current_subject = current_subject, via_api_key = via_api_key+ )+ finally:+ _PENDING_BATCH.reset(token)+ if started.status == "error" or not started.job_id:+ return [started.model_copy() for _ in requests]+ return [+ TrainingJobResponse(+ job_id = run_id,+ status = started.status,+ message = started.message,+ error = started.error,+ )+ for run_id in run_ids+ ]++@router.post("/stop", response_model = TrainingStopResponse)async def stop_training(body: TrainingStopRequest, current_subject: str = Depends(get_current_subject)diff --git a/studio/backend/tests/test_multi_lora_trainer.py b/studio/backend/tests/test_multi_lora_trainer.pynew file mode 100644index 00000000000..91880562e0c--- /dev/null+++ b/studio/backend/tests/test_multi_lora_trainer.py@@ -0,0 +1,530 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Batched multi-LoRA worker on a tiny random Qwen3 (CPU, PyTorch fallback of the kernels)."""++from __future__ import annotations++import copy+import json+import os+import queue+import types+from pathlib import Path++import sys++import pytest++# The repo root (pytest pythonpath) shadows PYTHONPATH.+if "unsloth" not in sys.modules:+ for _entry in os.environ.get("PYTHONPATH", "").split(os.pathsep)[::-1]:+ if _entry and os.path.isfile(os.path.join(_entry, "unsloth", "kernels", "multi_lora.py")):+ sys.path.insert(0, _entry)++try:+ import torch+ from unsloth.kernels import multi_lora as ml+except Exception as exc: # noqa: BLE001 - no multi-LoRA kernels or no importable unsloth here+ pytest.skip(f"unsloth.kernels.multi_lora unavailable: {exc}", allow_module_level = True)++from datasets import Dataset+from safetensors.torch import load_file+from tokenizers import Tokenizer, models, pre_tokenizers+from transformers import PreTrainedTokenizerFast, Qwen3Config, Qwen3ForCausalLM++from core.training import multi_lora_trainer as mlt++WORDS = [f"w{i}" for i in range(56)]+++def _tokenizer():+ vocab = {"[PAD]": 0, "[EOS]": 1, "[U]": 2, "[B]": 3}+ vocab.update({w: i + 4 for i, w in enumerate(WORDS)})+ tok = Tokenizer(models.WordLevel(vocab = vocab, unk_token = "[PAD]"))+ tok.pre_tokenizer = pre_tokenizers.WhitespaceSplit()+ return PreTrainedTokenizerFast(tokenizer_object = tok, pad_token = "[PAD]", eos_token = "[EOS]")+++def _base():+ torch.manual_seed(0)+ config = Qwen3Config(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ max_position_embeddings = 128,+ use_cache = False,+ tie_word_embeddings = False,+ )+ return Qwen3ForCausalLM(config).float()+++def _texts(n, seed):+ g = torch.Generator().manual_seed(seed)+ out = []+ for _ in range(n):+ k = int(torch.randint(4, 12, (1,), generator = g))+ out.append(+ " ".join(WORDS[int(i)] for i in torch.randint(0, 56, (k,), generator = g)) + " [EOS]"+ )+ return out+++MEMBERS = [+ dict(run_id = "job_a", rows = 6, batch_size = 2, num_epochs = 1, lr = 1e-2, r = 4, alpha = 8, seed = 11),+ dict(run_id = "job_b", rows = 12, batch_size = 2, num_epochs = 2, lr = 5e-3, r = 8, alpha = 16, seed = 22),+ dict(+ run_id = "job_c",+ rows = 9,+ batch_size = 3,+ num_epochs = 1,+ lr = 2e-2,+ r = 2,+ alpha = 4,+ seed = 33,+ max_steps = 4,+ ),+]+EXPECTED_STEPS = {"job_a": 2, "job_b": 6, "job_c": 4}+++def _member_config(m):+ return {+ "model_name": "tiny/qwen3",+ "local_datasets": [m["run_id"]],+ "format_type": "",+ "learning_rate": m["lr"],+ "num_epochs": m["num_epochs"],+ "batch_size": m["batch_size"],+ "lora_r": m["r"],+ "lora_alpha": m["alpha"],+ "random_seed": m["seed"],+ "max_steps": m.get("max_steps", 0),+ "optim": "adamw_torch",+ "warmup_steps": m.get("warmup_steps", 0),+ "weight_decay": m.get("weight_decay", 0.0),+ "gradient_accumulation_steps": 2,+ }+++class _Events:+ def __init__(self, on_event = None):+ self.items = []+ self.on_event = on_event++ def put(self, event):+ self.items.append(event)+ if self.on_event is not None:+ self.on_event(event)+++def _run(+ monkeypatch,+ members,+ on_event = None,+ trainer_attrs = None,+ datasets = None,+ model_attrs = None,+ **extra,+):+ base = _base()+ for name, value in (model_attrs or {}).items():+ setattr(base, name, value)+ pristine = copy.deepcopy(base)+ tokenizer = _tokenizer()+ trainer = types.SimpleNamespace(+ model = None,+ tokenizer = None,+ should_stop = False,+ save_on_stop = True,+ load_in_4bit = False,+ is_cpt = False,+ **(trainer_attrs or {}),+ )+ data = {+ m["run_id"]: {+ "dataset": Dataset.from_dict({"text": _texts(m["rows"], m["seed"])}),+ "final_format": "",+ }+ for m in members+ }+ data.update(datasets or {})+ inboxes = []++ def load_base(trainer, config, hf_token, event_queue):+ trainer.model, trainer.tokenizer = base, tokenizer+ return False++ def stop_reader(stop_queue, inbox, trainer):+ inboxes.append(inbox)++ monkeypatch.setattr(mlt, "_new_trainer", lambda: trainer)+ monkeypatch.setattr(mlt, "_detect_model", lambda trainer, config, hf_token: None)+ monkeypatch.setattr(mlt, "_format_dataset", lambda t, cfg, tok: data[cfg["local_datasets"][0]])+ monkeypatch.setattr(mlt, "_load_base", load_base)+ monkeypatch.setattr(mlt, "_start_stop_reader", stop_reader)+ configs = [_member_config(m) for m in members]+ config = {+ **configs[0],+ "max_seq_length": 64,+ "gradient_checkpointing": "unsloth",+ "target_modules": ["all-linear"],+ "batch_members": [{"run_id": m["run_id"], "config": c} for m, c in zip(members, configs)],+ **extra,+ }+ events = _Events(None if on_event is None else lambda e: on_event(e, inboxes[0]))+ mlt.run_multi_lora_training(events, queue.Queue(), config)+ return events.items, base, pristine+++def _member_events(events, run_id):+ return [e for e in events if e.get("member_run_id") == run_id]+++def _adapter_logits(model, slot, input_ids):+ ml.set_multi_lora_batch(+ model, torch.full((input_ids.shape[0],), slot), seq_len = input_ids.shape[1]+ )+ with torch.no_grad():+ return model(input_ids = input_ids, use_cache = False).logits++[email protected](scope = "module")+def batched_run(tmp_path_factory):+ monkeypatch = pytest.MonkeyPatch()+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path_factory.mktemp("studio_home")))+ try:+ yield _run(monkeypatch, MEMBERS)+ finally:+ monkeypatch.undo()+++def test_each_member_reports_and_exports_on_its_own(batched_run):+ events, model, pristine = batched_run+ assert [e["type"] for e in events if "member_run_id" not in e][-1] == "complete"+ dirs = set()+ for m in MEMBERS:+ mine = _member_events(events, m["run_id"])+ kinds = [e["type"] for e in mine]+ assert kinds[0] == "output_dir" and kinds[-1] == "complete"+ assert set(kinds[1:-1]) == {"progress"}+ steps = [e["step"] for e in mine if e["type"] == "progress"]+ assert steps == list(range(EXPECTED_STEPS[m["run_id"]] + 1))+ tokens = [e["num_tokens"] for e in mine if e["type"] == "progress"]+ assert tokens == sorted(tokens) and tokens[-1] > 0+ assert all(+ e["total_steps"] == EXPECTED_STEPS[m["run_id"]] for e in mine if e["type"] == "progress"+ )+ out = mine[0]["output_dir"]+ assert mine[-1]["output_dir"] == out+ dirs.add(out)+ config = json.load(open(os.path.join(out, "adapter_config.json")))+ assert config["r"] == m["r"] and config["lora_alpha"] == m["alpha"]+ assert config["unsloth_training_method"] == "lora"+ assert os.path.exists(os.path.join(out, "tokenizer.json"))+ assert len(dirs) == len(MEMBERS)+++def test_each_step_reports_the_rate_it_trained_with(batched_run):+ events, _model, _pristine = batched_run+ for m in MEMBERS:+ total = EXPECTED_STEPS[m["run_id"]]+ reported = [+ e["learning_rate"]+ for e in _member_events(events, m["run_id"])+ if e["type"] == "progress" and e["step"] > 0+ ]+ # Linear decay without warmup, read before each step: lr, then lr * (1 - 1/total), ...+ assert reported == pytest.approx([m["lr"] * (1 - k / total) for k in range(total)])+++def test_exported_adapter_reloads_with_peft_and_matches(batched_run):+ from peft import PeftModel++ events, model, pristine = batched_run+ model.eval()+ input_ids = torch.randint(4, 60, (2, 10), generator = torch.Generator().manual_seed(5))+ for slot, m in enumerate(MEMBERS):+ out = _member_events(events, m["run_id"])[0]["output_dir"]+ state = load_file(os.path.join(out, "adapter_model.safetensors"))+ assert any(v.abs().sum() > 0 for k, v in state.items() if "lora_B" in k)+ peft_model = PeftModel.from_pretrained(copy.deepcopy(pristine), out).eval()+ with torch.no_grad():+ expected = peft_model(input_ids = input_ids).logits+ torch.testing.assert_close(+ _adapter_logits(model, slot, input_ids), expected, atol = 1e-5, rtol = 1e-5+ )+++def test_finished_member_gets_no_more_updates(batched_run):+ events, model, pristine = batched_run+ out = _member_events(events, "job_a")[0]["output_dir"]+ exported = load_file(os.path.join(out, "adapter_model.safetensors"))+ current = ml.adapter_state_dict(model, 0)+ assert exported.keys() == current.keys()+ for key in exported:+ torch.testing.assert_close(current[key], exported[key], atol = 0, rtol = 0)+++def test_stop_member_stops_only_that_member(monkeypatch):+ def on_event(event, inbox):+ if (+ event.get("member_run_id") == "job_b"+ and event["type"] == "progress"+ and event["step"] == 2+ ):+ inbox.put({"type": "stop_member", "run_id": "job_b", "save": True})++ events, model, pristine = _run(monkeypatch, MEMBERS, on_event)+ b = _member_events(events, "job_b")+ assert b[-1]["type"] == "stopped" and b[-1]["saved"] is True+ assert max(e["step"] for e in b if e["type"] == "progress") == 2+ assert os.path.exists(os.path.join(b[-1]["output_dir"], "adapter_model.safetensors"))+ for run_id in ("job_a", "job_c"):+ mine = _member_events(events, run_id)+ assert mine[-1]["type"] == "complete"+ assert max(e["step"] for e in mine if e["type"] == "progress") == EXPECTED_STEPS[run_id]+ assert events[-1]["type"] == "complete" and "member_run_id" not in events[-1]+++def test_seed_init_is_independent_of_batch_order():+ first, second = _base(), _base()+ ml.inject_multi_lora(first, ["q_proj", "down_proj"], 2, r = [4, 8], lora_alpha = [8, 16])+ ml.inject_multi_lora(second, ["q_proj", "down_proj"], 2, r = [8, 4], lora_alpha = [16, 8])+ mlt._seed_adapter(first, 0, 11)+ mlt._seed_adapter(first, 1, 22)+ mlt._seed_adapter(second, 0, 22)+ mlt._seed_adapter(second, 1, 11)+ a, b = ml.adapter_state_dict(first, 0), ml.adapter_state_dict(second, 1)+ for key in a:+ torch.testing.assert_close(a[key], b[key], atol = 0, rtol = 0)+ assert all(v.abs().sum() == 0 for k, v in a.items() if "lora_B" in k)+++def test_completion_masking_masks_the_prompt():+ tokenizer = _tokenizer()+ tokenizer._unsloth_input_part = "[U]"+ tokenizer._unsloth_output_part = "[B]"+ info = {+ "dataset": Dataset.from_dict({"text": ["[U] w1 w2 w3 [B] w4 w5 [EOS]"] * 3}),+ "final_format": "",+ }+ tok, rows = mlt._tokenize_member(+ info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 64+ )+ assert tok is tokenizer+ ids, labels = rows[0]+ response = ids.index(tokenizer.convert_tokens_to_ids("[B]")) + 1+ assert all(t == -100 for t in labels[:response])+ assert labels[response:] == ids[response:]+++def test_batch_trains_through_sft_trainer_and_saves_checkpoints(monkeypatch, tmp_path):+ from trl import SFTTrainer++ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ built = []+ original = mlt._build_sft_trainer++ def spy(*args, **kwargs):+ built.append(original(*args, **kwargs))+ return built[-1]++ monkeypatch.setattr(mlt, "_build_sft_trainer", spy)+ events, model, pristine = _run(monkeypatch, MEMBERS[:2], save_steps = 2)+ assert isinstance(built[0], SFTTrainer) and built[0].state.global_step == 6+ found = {}+ for m in MEMBERS[:2]:+ out = _member_events(events, m["run_id"])[0]["output_dir"]+ found[m["run_id"]] = sorted(p.name for p in Path(out).glob("checkpoint-*"))+ last = os.path.join(out, found[m["run_id"]][-1], "adapter_model.safetensors")+ final = load_file(os.path.join(out, "adapter_model.safetensors"))+ for key, value in load_file(last).items():+ torch.testing.assert_close(value, final[key], atol = 0, rtol = 0)+ assert found == {+ "job_a": ["checkpoint-2"],+ "job_b": ["checkpoint-2", "checkpoint-4", "checkpoint-6"],+ }+++def test_gradients_are_clipped_per_member_every_step(monkeypatch, tmp_path):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ seen = []+ real = ml.clip_grad_norm_per_adapter_++ def spy(model, max_norm):+ norms = real(model, max_norm)+ seen.append((max_norm, norms.clone()))+ return norms++ monkeypatch.setattr(ml, "clip_grad_norm_per_adapter_", spy)+ events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], max_grad_norm = 0.01)+ assert len(seen) == EXPECTED_STEPS["job_b"]+ assert all(max_norm == 0.01 for max_norm, _ in seen)+ # Each member reports its own pre-clip norm, not one shared global norm.+ a = [+ e["grad_norm"]+ for e in _member_events(events, "job_a")+ if e["type"] == "progress" and e["step"]+ ]+ b = [+ e["grad_norm"]+ for e in _member_events(events, "job_b")+ if e["type"] == "progress" and e["step"]+ ]+ assert a == pytest.approx([float(n[0]) for _, n in seen[:2]])+ assert b == pytest.approx([float(n[1]) for _, n in seen])+++def test_each_member_keeps_its_own_weight_decay_and_warmup(monkeypatch, tmp_path):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ members = [+ {**MEMBERS[0], "weight_decay": 0.1, "warmup_steps": 1},+ {**MEMBERS[1], "weight_decay": 0.0, "warmup_steps": 0},+ ]+ built = []+ original = mlt._build_sft_trainer+ monkeypatch.setattr(+ mlt, "_build_sft_trainer", lambda *a, **k: built.append(original(*a, **k)) or built[-1]+ )+ events, _model, _pristine = _run(monkeypatch, members)+ assert [g["weight_decay"] for g in built[0].optimizer.param_groups] == [0.1, 0.0]+ for m in members:+ total = EXPECTED_STEPS[m["run_id"]]+ reported = [+ e["learning_rate"]+ for e in _member_events(events, m["run_id"])+ if e["type"] == "progress" and e["step"]+ ]+ warmup = m["warmup_steps"]+ # transformers' linear schedule: ramp over the member's own warmup, then decay over its own length.+ expected = [+ m["lr"] * (k / warmup if k < warmup else (total - k) / (total - warmup))+ for k in range(total)+ ]+ assert reported == pytest.approx(expected)+++def test_targets_follow_the_single_job_rules():+ model = _base()+ every_linear = mlt._resolve_targets(model, ["all-linear"])+ assert (+ "q_proj" in every_linear and "down_proj" in every_linear and "lm_head" not in every_linear+ )+ # Explicit names beside "all-linear" win, as UnslothTrainer.prepare_model_for_training does.+ assert mlt._resolve_targets(model, ["all-linear", "q_proj"]) == ["q_proj"]+ assert mlt._resolve_targets(model, None) == mlt._DEFAULT_TARGETS+++def test_a_text_diffusion_model_never_trains_batched(monkeypatch, tmp_path):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ events, _model, _pristine = _run(+ monkeypatch, MEMBERS[:2], trainer_attrs = {"diffusion_lm_profile": "dream"}+ )+ assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]+ assert "text-diffusion" in events[-1]["error"]+++def test_an_existing_lora_adapter_never_trains_batched(monkeypatch, tmp_path):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ events, _model, _pristine = _run(+ monkeypatch, MEMBERS[:2], model_attrs = {"peft_config": {"default": object()}}+ )+ assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]+ assert "existing LoRA adapter" in events[-1]["error"]+++def test_a_moe_model_never_trains_batched(monkeypatch, tmp_path):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ config = copy.deepcopy(_base().config)+ config.num_experts = 8+ events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], model_attrs = {"config": config})+ assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]+ assert "MoE" in events[-1]["error"]+++def test_pretokenized_assistant_masks_become_labels():+ ids = [[5, 6, 7, 8], [9, 10, 11]]+ info = {+ "dataset": Dataset.from_dict(+ {"input_ids": ids, "assistant_masks": [[0, 0, 1, 1], [0, 1, 1]]}+ ),+ "final_format": "pretokenized",+ }+ _tok, rows = mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)+ assert [labels for _, labels in rows] == [[-100, -100, 7, 8], [-100, 10, 11]]+++def test_a_pretokenized_mask_of_the_wrong_length_is_refused():+ info = {+ "dataset": Dataset.from_dict({"input_ids": [[5, 6, 7, 8]], "assistant_masks": [[0, 1]]}),+ "final_format": "pretokenized",+ }+ with pytest.raises(ValueError, match = "assistant_masks"):+ mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)+++def test_a_pretokenized_token_outside_the_vocabulary_is_refused():+ info = {+ "dataset": Dataset.from_dict({"input_ids": [[5, 6, 900]]}),+ "final_format": "pretokenized",+ }+ with pytest.raises(ValueError, match = "outside this model's vocabulary"):+ mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64, vocab_size = 64)+++def test_one_members_bad_dataset_fails_only_that_member(monkeypatch, tmp_path):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ bad = {+ "dataset": Dataset.from_dict({"input_ids": [[5, 6, 900]] * 4}),+ "final_format": "pretokenized",+ }+ events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], datasets = {"job_a": bad})+ a, b = _member_events(events, "job_a"), _member_events(events, "job_b")+ assert [e["type"] for e in a] == ["error"] and "vocabulary" in a[0]["error"]+ assert b[-1]["type"] == "complete"+ assert events[-1]["type"] == "complete" and "member_run_id" not in events[-1]++[email protected]("collator_masks_labels", [True, False])+def test_pretokenized_labels_and_assistant_masks(monkeypatch, collator_masks_labels):+ import utils.datasets.pretokenized as pretokenized++ monkeypatch.setattr(pretokenized, "_collator_masks_user_labels", lambda: collator_masks_labels)+ info = {+ "dataset": Dataset.from_dict(+ {+ "input_ids": [[5, 6, 7, 8]],+ "labels": [[5, 6, 7, 8]],+ "assistant_masks": [[0, 0, 1, 1]],+ }+ ),+ "final_format": "pretokenized",+ }+ _tok, rows = mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)+ # The single-job collator masks user labels only before TRL 1.7.+ assert rows[0][1] == ([-100, -100, 7, 8] if collator_masks_labels else [5, 6, 7, 8])+++def test_batched_member_trains_as_it_would_alone(batched_run, monkeypatch):+ events, model, pristine = batched_run+ batched = load_file(+ os.path.join(_member_events(events, "job_b")[0]["output_dir"], "adapter_model.safetensors")+ )+ alone_events, _, _ = _run(monkeypatch, [MEMBERS[1]])+ alone = load_file(+ os.path.join(+ _member_events(alone_events, "job_b")[0]["output_dir"], "adapter_model.safetensors"+ )+ )+ batched_loss = [e["loss"] for e in _member_events(events, "job_b") if e["type"] == "progress"]+ alone_loss = [+ e["loss"] for e in _member_events(alone_events, "job_b") if e["type"] == "progress"+ ]+ assert batched_loss[1:] == pytest.approx(alone_loss[1:], rel = 1e-4)+ for key in alone:+ torch.testing.assert_close(batched[key], alone[key], atol = 1e-5, rtol = 1e-4)diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.pyindex e7351d4cb59..2a899f9ac19 100644--- a/studio/backend/tests/test_openai_fine_tuning.py+++ b/studio/backend/tests/test_openai_fine_tuning.py@@ -19,6 +19,9 @@ from auth.authentication import authenticated_via_api_key, get_current_subjectfrom models.training import TrainingJobResponse # noqa: E402from routes import openai_fine_tuning, training as training_routes # noqa: E402from storage import studio_db # noqa: E402+import core.training.training as _training_core # noqa: E402++_REAL_TRAINS_ALONE = openai_fine_tuning._trains_aloneCHATS = (b'{"messages": [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello!"}]}\n'@@ -61,6 +64,8 @@ def studio(monkeypatch):monkeypatch.setattr(openai_fine_tuning, "_STARTED", {})monkeypatch.setattr(openai_fine_tuning, "_CANCELLING", set())+ monkeypatch.setattr(openai_fine_tuning, "_BATCH_OF", {})+ monkeypatch.setattr(openai_fine_tuning, "_BATCHES", {})monkeypatch.setattr(training_routes, "start_training", start_training)monkeypatch.setattr(training_routes, "stop_training", stop_training)monkeypatch.setattr(training_routes, "get_training_status", get_training_status)@@ -388,6 +393,8 @@ def queue(studio, monkeypatch):monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())monkeypatch.setattr(openai_fine_tuning, "_ensure_drainer", lambda: None)+ # Without the multi-LoRA kernels: every queued job starts alone, as before batching existed.+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora_available", lambda: False)gpu = {"busy": True}monkeypatch.setattr(openai_fine_tuning, "_training_busy", lambda: gpu["busy"])return client, calls, live, gpu@@ -828,3 +835,603 @@ def test_a_sidecar_swap_conflict_queues_and_a_crash_after_cancel_stays_cancelledqueued, after = _run(flow())assert queued.status == "queued" and after.status == "cancelled"+++def _entry(+ tmp_path,+ name = "a",+ subject = "unsloth",+ **fields,+):+ from utils.account_context import current_account++ dataset = tmp_path / f"{name}.jsonl"+ dataset.write_bytes(CHATS)+ config = {+ "model_name": "unsloth/Qwen3-0.6B",+ "training_type": "LoRA/QLoRA",+ "format_type": "auto",+ "local_datasets": [str(dataset)],+ **fields,+ }+ return openai_fine_tuning._Queued(+ alias = f"ftjob_q{name}",+ account = current_account(),+ subject = subject,+ via_api_key = True,+ request = openai_fine_tuning.TrainingStartRequest.model_validate(config),+ created_at = 1_700_000_000,+ )+++def batching(monkeypatch):+ monkeypatch.setattr(openai_fine_tuning, "_QUEUE", [])+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora_available", lambda: True)+ monkeypatch.setattr(_training_core, "should_use_mlx_training_backend", lambda: False)+ monkeypatch.delenv("UNSLOTH_STUDIO_FT_BATCH_MAX", raising = False)+ return openai_fine_tuning._QUEUE+++def test_jobs_differing_only_in_member_fields_batch_in_queue_order(batching, tmp_path):+ head = _entry(tmp_path, "a")+ other = _entry(+ tmp_path,+ "b",+ learning_rate = "1e-4",+ num_epochs = 3,+ batch_size = 4,+ lora_r = 32,+ lora_alpha = 64,+ random_seed = 7,+ project_name = "other",+ max_steps = 30,+ weight_decay = 0.05,+ warmup_steps = 10,+ )+ odd = _entry(tmp_path, "c", max_seq_length = 4096)+ last = _entry(tmp_path, "d", lora_r = 8, warmup_ratio = 0.1)+ batching.extend([head, other, odd, last])+ assert openai_fine_tuning._batch_for(head) == [head, other, last]+++ "fields",+ [+ {"resume_from_checkpoint": "/runs/old"},+ {"eval_steps": 0.1},+ {"packing": True},+ {"lora_dropout": 0.05},+ {"use_dora": True},+ {"use_loftq": True},+ {"enable_wandb": True},+ {"enable_tensorboard": True},+ {"lr_scheduler_type": "reduce_lr_on_plateau"},+ {"offload_layers": "auto"},+ {"offload_layers": 4},+ {"training_type": "Full Finetuning"},+ {"training_type": "Continued Pretraining"},+ {"is_dataset_image": True},+ {"is_embedding": True},+ {"optim": "adamw_torch_fused"},+ {"gpu_ids": [0, 1]},+ {"start_request_id": "retry-1"},+ ],+)+def test_an_excluded_setting_trains_alone(batching, tmp_path, fields):+ head = _entry(tmp_path, "a", **fields)+ other = _entry(tmp_path, "b", **fields)+ batching.extend([head, other])+ assert openai_fine_tuning._batch_for(head) == [head]+ # A compatible head never pulls the excluded job in either.+ plain = _entry(tmp_path, "c")+ batching.insert(0, plain)+ assert openai_fine_tuning._batch_for(plain) == [plain]+++ "fields",+ [+ {"gradient_accumulation_steps": 4},+ {"load_in_4bit": False},+ {"model_name": "unsloth/Qwen3-1.7B"},+ {"target_modules": ["q_proj"]},+ {"lr_scheduler_type": "cosine"},+ {"optim": "adamw_torch"},+ {"use_rslora": True},+ ],+)+def test_shared_settings_must_match(batching, tmp_path, fields):+ head = _entry(tmp_path, "a")+ other = _entry(tmp_path, "b", **fields)+ batching.extend([head, other])+ assert openai_fine_tuning._batch_for(head) == [head]+++def test_another_subject_or_account_is_never_batched(batching, tmp_path):+ import dataclasses++ head = _entry(tmp_path, "a")+ other_subject = _entry(tmp_path, "b", subject = "bob")+ other_account = _entry(tmp_path, "c")+ other_account.account = dataclasses.replace(head.account, account_id = "acct-bob")+ batching.extend([head, other_subject, other_account])+ assert openai_fine_tuning._batch_for(head) == [head]+++def test_the_cap_and_the_kill_switch(batching, tmp_path, monkeypatch):+ entries = [_entry(tmp_path, name) for name in "abcdef"]+ batching.extend(entries)+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:4]+ monkeypatch.setenv("UNSLOTH_STUDIO_FT_BATCH_MAX", "2")+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:2]+ monkeypatch.setenv("UNSLOTH_STUDIO_FT_BATCH_MAX", "1")+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:1]+ monkeypatch.setenv("UNSLOTH_STUDIO_FT_BATCH_MAX", "many")+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:4]+++def test_without_the_kernels_every_job_trains_alone(batching, tmp_path, monkeypatch):+ entries = [_entry(tmp_path, name) for name in "ab"]+ batching.extend(entries)+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora_available", lambda: False)+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:1]+++def test_the_kernel_lookup_reads_the_package_on_disk_once(tmp_path, monkeypatch):+ import importlib.machinery++ from routes.openai_fine_tuning import _multi_lora_available++ package = tmp_path / "unsloth"+ (package / "kernels").mkdir(parents = True)+ lookups = []++ def find_spec(name):+ lookups.append(name)+ spec = importlib.machinery.ModuleSpec(name, None, is_package = True)+ spec.submodule_search_locations = [str(package)]+ return spec++ monkeypatch.setattr(openai_fine_tuning.importlib.util, "find_spec", find_spec)+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora", None)+ assert _multi_lora_available() is False+ (package / "kernels" / "multi_lora.py").write_text("")+ assert _multi_lora_available() is False # cached+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora", None)+ assert _multi_lora_available() is True+ # Only the top-level package is looked up: finding a submodule would import unsloth (and torch).+ assert lookups == ["unsloth", "unsloth"]+++class _BatchBackend:+ def __init__(self):+ self.current_job_id = ""+ self.running: list[str] = []+ self.stopped: list[str] = []++ def batch_member_ids(self):+ return list(self.running)++ def stop_member(self, run_id):+ if run_id not in self.running:+ return False+ self.stopped.append(run_id)+ return True+++def batch_queue(queue, monkeypatch):+ """The queue with the multi-LoRA kernels present and a stubbed start_training_batch."""+ import core.training++ client, calls, live, gpu = queue+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora_available", lambda: True)++ async def batchable_model(request):+ return False++ monkeypatch.setattr(openai_fine_tuning, "_trains_alone", batchable_model)+ monkeypatch.setattr(_training_core, "should_use_mlx_training_backend", lambda: False)+ monkeypatch.delenv("UNSLOTH_STUDIO_FT_BATCH_MAX", raising = False)+ backend = _BatchBackend()+ monkeypatch.setattr(core.training, "get_training_backend", lambda: backend)+ calls["batch"] = []++ async def start_training_batch(requests, current_subject, via_api_key):+ calls["batch"].append((list(requests), current_subject, via_api_key))+ backend.current_job_id = "job_20261021_101500_batch000"+ run_ids = [f"job_20261021_101500_member0{i}" for i in range(len(requests))]+ backend.running = list(run_ids)+ live["status"] = _Status(backend.current_job_id, "loading_model")+ return [+ TrainingJobResponse(job_id = run_id, status = "queued", message = "queued")+ for run_id in run_ids+ ]++ monkeypatch.setattr(training_routes, "start_training_batch", start_training_batch)+ return client, calls, live, gpu, backend+++def test_compatible_queued_jobs_start_as_one_batch(batch_queue):+ client, calls, live, gpu, _backend = batch_queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [+ await _create(client, uploaded.id, suffix = "a", seed = 1),+ await _create(client, uploaded.id, suffix = "b", hyperparameters = {"n_epochs": 3}),+ await _create(client, uploaded.id, suffix = "solo", extra_body = {"packing": True}),+ await _create(client, uploaded.id, suffix = "c", extra_body = {"lora_r": 64}),+ ]+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ retrieved = [await client.fine_tuning.jobs.retrieve(job.id) for job in jobs]+ listed = await client.fine_tuning.jobs.list()+ return jobs, retrieved, listed++ jobs, retrieved, listed = _run(flow())+ [(requests, subject, via_api_key)] = calls["batch"]+ assert [r.project_name for r in requests] == ["a", "b", "c"]+ assert (subject, via_api_key) == ("unsloth", True)+ assert calls["start"] == []+ batched = [retrieved[0], retrieved[1], retrieved[3]]+ # Each job keeps the id it was given and its own settings, read from the batch's live status.+ assert [j.id for j in batched] == [jobs[0].id, jobs[1].id, jobs[3].id]+ assert [j.status for j in batched] == ["validating_files"] * 3+ assert [j.suffix for j in batched] == ["a", "b", "c"]+ assert batched[0].seed == 1 and batched[1].hyperparameters.n_epochs == 3+ assert retrieved[2].status == "queued"+ run_ids = [openai_fine_tuning._ALIAS_RUN[j.id][0] for j in batched]+ assert run_ids == [f"job_20261021_101500_member0{i}" for i in range(3)]+ # The batch itself is no job: the list shows its members and the job still waiting.+ listed_ids = [j.id for j in listed.data]+ assert sorted(listed_ids) == sorted(j.id for j in jobs)+ assert live["status"].job_id not in {openai_fine_tuning._run_id(i) for i in listed_ids}+++def test_listing_pages_through_a_batch_that_is_still_loading(batch_queue):+ client, _calls, _live, gpu, _backend = batch_queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "abc"]+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ seen, after = [], None+ for _ in range(5):+ page = await client.fine_tuning.jobs.list(+ limit = 1, **({"after": after} if after else {})+ )+ seen += [j.id for j in page.data]+ if not page.has_more:+ break+ after = page.data[-1].id+ return jobs, seen++ jobs, seen = _run(flow())+ assert sorted(seen) == sorted(j.id for j in jobs) and len(seen) == 3+++ "model_kind", ["diffusion", "lora_adapter", "moe", "moe_text_config", "dense"]+)+def test_only_plain_dense_models_batch(batch_queue, monkeypatch, model_kind):+ import utils.models.diffusion_lm as diffusion_lm+ import utils.models.model_config as model_config++ client, calls, _live, gpu, _backend = batch_queue+ monkeypatch.setattr(openai_fine_tuning, "_trains_alone", _REAL_TRAINS_ALONE)+ monkeypatch.setattr(+ diffusion_lm,+ "diffusion_lm_profile_for_model",+ lambda *a, **k: "dream" if model_kind == "diffusion" else None,+ )+ monkeypatch.setattr(+ model_config,+ "get_base_model_from_lora_identifier",+ lambda *a, **k: "unsloth/Qwen3-0.6B" if model_kind == "lora_adapter" else None,+ )+ configs = {+ "moe": {"model_type": "qwen3_moe", "num_experts": 128},+ "moe_text_config": {"text_config": {"num_local_experts": 8}},+ }+ monkeypatch.setattr(+ model_config,+ "_read_raw_config_json",+ lambda *a, **k: configs.get(model_kind, {"model_type": "qwen3", "num_experts": None}),+ )++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ for s in "ab":+ await _create(client, uploaded.id, suffix = s)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()++ _run(flow())+ if model_kind == "dense":+ assert len(calls["batch"]) == 1 and calls["start"] == []+ else:+ assert calls["batch"] == []+ assert [r.project_name for r in calls["start"]] == ["a"]+++def test_mlx_jobs_never_batch(batch_queue, monkeypatch):+ client, calls, _live, gpu, _backend = batch_queue+ monkeypatch.setattr(_training_core, "should_use_mlx_training_backend", lambda: True)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ for s in "ab":+ await _create(client, uploaded.id, suffix = s)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()++ _run(flow())+ assert calls["batch"] == []+ assert [r.project_name for r in calls["start"]] == ["a"]+++def test_batch_members_stay_validating_until_their_first_step(batch_queue):+ client, _calls, live, gpu, _backend = batch_queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "ab"]+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ seen = []+ # The batch's setup messages move its phase through all of these before any member has a row.+ for phase in ("loading_model", "training", "configuring", "loading_dataset", "training"):+ live["status"].phase = phase+ seen.append([(await client.fine_tuning.jobs.retrieve(j.id)).status for j in jobs])+ live["status"].phase = "error"+ failed = [(await client.fine_tuning.jobs.retrieve(j.id)).status for j in jobs]+ return seen, failed++ seen, failed = _run(flow())+ assert seen == [["validating_files"] * 2] * 5+ assert failed == ["failed"] * 2+++def test_a_single_or_incompatible_job_starts_exactly_as_before(batch_queue):+ client, calls, _live, gpu, _backend = batch_queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ first = await _create(client, uploaded.id, suffix = "first", extra_body = {"packing": True})+ second = await _create(client, uploaded.id, suffix = "second")+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ queued = list(openai_fine_tuning._QUEUE)+ await openai_fine_tuning._drain_once()+ return first, second, queued++ first, second, queued = _run(flow())+ assert calls["batch"] == []+ assert [q.alias for q in queued] == [second.id]+ assert [(r.project_name, r.packing) for r in calls["start"]] == [+ ("first", True),+ ("second", False),+ ]+ assert queued[0].request is calls["start"][1]+++def test_a_refused_batch_starts_the_head_alone(batch_queue, monkeypatch):+ client, calls, _live, gpu, _backend = batch_queue++ async def refuse(requests, current_subject, via_api_key):+ calls["batch"].append(requests)+ raise openai_fine_tuning.HTTPException(status_code = 400, detail = "Dataset not found")++ monkeypatch.setattr(training_routes, "start_training_batch", refuse)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ first = await _create(client, uploaded.id, suffix = "first")+ second = await _create(client, uploaded.id, suffix = "second")+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return (+ first,+ second,+ [await client.fine_tuning.jobs.retrieve(j.id) for j in (first, second)],+ )++ first, second, (started, waiting) = _run(flow())+ assert len(calls["batch"]) == 1+ assert [r.project_name for r in calls["start"]] == ["first"]+ assert started.id == first.id and started.status == "validating_files"+ assert waiting.status == "queued"+++def test_a_busy_gpu_keeps_the_whole_batch_queued(batch_queue, monkeypatch):+ client, calls, _live, gpu, _backend = batch_queue++ async def busy(requests, current_subject, via_api_key):+ calls["batch"].append(requests)+ return [+ TrainingJobResponse(+ job_id = "",+ status = "error",+ message = "Training is already in progress.",+ error = "Training already active",+ )+ for _ in requests+ ]++ monkeypatch.setattr(training_routes, "start_training_batch", busy)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = name) for name in ("a", "b")]+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return [await client.fine_tuning.jobs.retrieve(j.id) for j in jobs]++ assert [j.status for j in _run(flow())] == ["queued", "queued"]+ assert calls["start"] == [] and len(openai_fine_tuning._QUEUE) == 2+++def test_cancelling_a_batched_job_stops_only_that_adapter(batch_queue):+ client, calls, _live, gpu, backend = batch_queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = name) for name in ("a", "b", "c")]+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ cancelled = await client.fine_tuning.jobs.cancel(jobs[1].id)+ after = [await client.fine_tuning.jobs.retrieve(j.id) for j in jobs]+ return jobs, cancelled, after++ jobs, cancelled, after = _run(flow())+ assert backend.stopped == ["job_20261021_101500_member01"]+ assert calls["stop"] == []+ assert cancelled.status == "cancelled"+ assert [j.status for j in after] == ["validating_files", "cancelled", "validating_files"]+++def test_cancelling_the_last_running_batched_job_stops_the_run(batch_queue):+ client, calls, live, gpu, backend = batch_queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = name) for name in ("a", "b")]+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ backend.running = ["job_20261021_101500_member01"] # the other adapter already finished+ cancelled = await client.fine_tuning.jobs.cancel(jobs[1].id)+ return cancelled, await client.fine_tuning.jobs.retrieve(jobs[1].id)++ cancelled, retrieved = _run(flow())+ assert backend.stopped == []+ [stop] = calls["stop"]+ # The whole run is the batch, so the stop names the batch's job.+ assert stop.expected_job_id == live["status"].job_id == "job_20261021_101500_batch000"+ assert cancelled.status == retrieved.status == "cancelled"+++def test_a_batched_job_cancelled_while_its_start_was_in_flight_stops_alone(+ batch_queue, monkeypatch+):+ client, calls, _live, gpu, backend = batch_queue+ start = training_routes.start_training_batch+ release, entered = asyncio.Event(), asyncio.Event()++ async def slow(requests, current_subject, via_api_key):+ entered.set()+ await release.wait()+ return await start(requests, current_subject = current_subject, via_api_key = via_api_key)++ monkeypatch.setattr(training_routes, "start_training_batch", slow)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = name) for name in ("a", "b")]+ gpu["busy"] = False+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())+ await entered.wait()+ await client.fine_tuning.jobs.cancel(jobs[0].id)+ release.set()+ await drain+ return [await client.fine_tuning.jobs.retrieve(j.id) for j in jobs]++ assert [j.status for j in _run(flow())] == ["cancelled", "validating_files"]+ assert backend.stopped == ["job_20261021_101500_member00"] and calls["stop"] == []+++def test_start_training_batch_runs_one_start_carrying_every_member(monkeypatch):+ seen = []++ async def prepare(request, via_api_key):+ seen.append(("prepare", request.project_name))++ async def admit(request, current_subject, via_api_key):+ seen.append(("admit", request.project_name))+ return training_routes._AdmittedTrainingRequest(+ training_kwargs = {"project_name": request.project_name, "lora_r": request.lora_r},+ hf_token = None,+ resume_run = None,+ )++ async def start_training(request, current_subject, via_api_key):+ # Inside start_training, after its own admission of the first member.+ members = training_routes._batch_members_for_start({"project_name": request.project_name})+ seen.append(("start", request.project_name, members))+ return TrainingJobResponse(job_id = "job_batch", status = "queued", message = "queued")++ monkeypatch.setattr(training_routes, "_prepare_training_request", prepare)+ monkeypatch.setattr(training_routes, "_admit_training_request", admit)+ monkeypatch.setattr(training_routes, "start_training", start_training)+ requests = [+ openai_fine_tuning.TrainingStartRequest(+ model_name = "unsloth/Qwen3-0.6B",+ training_type = "LoRA/QLoRA",+ format_type = "auto",+ project_name = name,+ lora_r = r,+ )+ for name, r in (("a", 8), ("b", 16), ("c", 32))+ ]+ responses = _run(+ training_routes.start_training_batch(requests, current_subject = "unsloth", via_api_key = True)+ )+ assert seen[:4] == [("prepare", "b"), ("admit", "b"), ("prepare", "c"), ("admit", "c")]+ [(_, first, members)] = seen[4:]+ assert first == "a"+ run_ids = [r.job_id for r in responses]+ assert len(set(run_ids)) == 3 and "job_batch" not in run_ids+ assert all(r.status == "queued" for r in responses)+ assert members == [+ {"run_id": run_ids[0], "project_name": "a"},+ {"run_id": run_ids[1], "project_name": "b", "lora_r": 16},+ {"run_id": run_ids[2], "project_name": "c", "lora_r": 32},+ ]+ assert training_routes._batch_members_for_start({"project_name": "a"}) is None+++def test_start_training_batch_refuses_resume_and_reports_a_busy_gpu_per_member(monkeypatch):+ async def busy(request, current_subject, via_api_key):+ return TrainingJobResponse(+ job_id = "job_running",+ status = "error",+ message = "Training is already in progress.",+ error = "Training already active",+ )++ async def admit(request, current_subject, via_api_key):+ return training_routes._AdmittedTrainingRequest({}, None, None)++ async def prepare(request, via_api_key):+ return None++ monkeypatch.setattr(training_routes, "_prepare_training_request", prepare)+ monkeypatch.setattr(training_routes, "_admit_training_request", admit)+ monkeypatch.setattr(training_routes, "start_training", busy)++ def request(**fields):+ return openai_fine_tuning.TrainingStartRequest(+ model_name = "unsloth/Qwen3-0.6B",+ training_type = "LoRA/QLoRA",+ format_type = "auto",+ **fields,+ )++ with pytest.raises(ValueError):+ _run(+ training_routes.start_training_batch(+ [request(), request(resume_from_checkpoint = "/runs/old")],+ current_subject = "unsloth",+ via_api_key = True,+ )+ )+ responses = _run(+ training_routes.start_training_batch(+ [request(), request()], current_subject = "unsloth", via_api_key = True+ )+ )+ assert [(r.status, r.error, r.job_id) for r in responses] == [+ ("error", "Training already active", "job_running")+ ] * 2diff --git a/studio/backend/tests/test_startup_defers_torch.py b/studio/backend/tests/test_startup_defers_torch.pyindex 4872200e237..6802c5d2fc8 100644--- a/studio/backend/tests/test_startup_defers_torch.py+++ b/studio/backend/tests/test_startup_defers_torch.py@@ -266,7 +266,7 @@ def test_a_failing_warm_stage_is_reported_not_swallowed(monkeypatch, capsys, cap_OFFLOAD_REQUIRED = [("main.py", "get_gpu_visibility", "get_backend_visible_gpu_info"),("routes/training.py", "get_hardware_utilization", "get_gpu_utilization"),- ("routes/training.py", "start_training", "ensure_hardware_detected"),+ ("routes/training.py", "_admit_training_request", "ensure_hardware_detected"),("routes/export.py", "_ensure_export_supported", "export_capability"),("routes/models.py", "list_models", "get_inference_backend"),# get_status only peeks now; test_async_singleton_access.py guards it.diff --git a/studio/backend/tests/test_training_batch_backend.py b/studio/backend/tests/test_training_batch_backend.pynew file mode 100644index 00000000000..ac5b1f8686e--- /dev/null+++ b/studio/backend/tests/test_training_batch_backend.py@@ -0,0 +1,445 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A multi-LoRA batch: one training_runs row per member, the batch id itself never gets one."""++from __future__ import annotations++import json+import queue+from unittest.mock import patch++import pytest++import core.training.training as training_mod+from core.training.training import TrainingBackend, TrainingProgress++BATCH = "job_batch"+MEMBERS = ("job_a", "job_b", "job_c")+++def db(monkeypatch, tmp_path):+ from storage import studio_db++ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))+ monkeypatch.setattr(studio_db, "_schema_ready", set())+ return studio_db+++class _AliveProc:+ pid = 4321++ def is_alive(self):+ return True+++def _member_config(run_id):+ return training_mod._sanitize_db_config(+ training_mod._build_training_worker_config(+ {+ "model_name": "unsloth/tiny",+ "local_datasets": [f"/data/{run_id}.jsonl"],+ "hf_token": "hf_secret",+ "wandb_token": "wb_secret",+ "learning_rate": "1e-4",+ }+ )+ )+++def _batch_backend(tmp_path):+ b = TrainingBackend()+ b.current_job_id = BATCH+ b._proc = _AliveProc()+ b._stop_queue = queue.Queue()+ b._start_stop_watchdog = lambda **kw: None+ b._progress = TrainingProgress(is_training = True)+ b._db_config = None+ b._batch_members = {+ run_id: b._new_batch_member(_member_config(run_id), "2026-10-26T00:00:00+00:00")+ for run_id in MEMBERS+ }+ return b+++def _progress(run_id, step, loss):+ return {+ "type": "progress",+ "member_run_id": run_id,+ "step": step,+ "loss": loss,+ "learning_rate": 1e-4,+ "total_steps": 4,+ "epoch": 0.5,+ "num_tokens": step * 10,+ "elapsed_seconds": float(step),+ }+++def _exported(tmp_path, run_id):+ out = tmp_path / "out" / run_id+ out.mkdir(parents = True, exist_ok = True)+ (out / "adapter_config.json").write_text("{}")+ return str(out)+++def test_member_events_get_their_own_rows_metrics_and_terminal_states(db, tmp_path):+ b = _batch_backend(tmp_path)+ outs = {r: _exported(tmp_path, r) for r in MEMBERS}+ for run_id in MEMBERS:+ b._handle_event({"type": "output_dir", "member_run_id": run_id, "output_dir": outs[run_id]})+ for step in (1, 2):+ for i, run_id in enumerate(MEMBERS):+ b._handle_event(_progress(run_id, step, 1.0 + i + step / 10))+ assert b.batch_member_ids() == list(MEMBERS)++ b._handle_event({"type": "complete", "member_run_id": "job_a", "output_dir": outs["job_a"]})+ assert b.batch_member_ids() == ["job_b", "job_c"]+ b._handle_event({"type": "error", "member_run_id": "job_a", "error": "late"})+ b._handle_event(_progress("job_a", 3, 9.0))+ b._handle_event(+ {"type": "stopped", "member_run_id": "job_b", "output_dir": outs["job_b"], "saved": True}+ )+ assert b.batch_member_ids() == ["job_c"]+ b._handle_event({"type": "error", "error": "CUDA error: boom"})+ assert b.batch_member_ids() == []++ assert db.get_run(BATCH) is None+ rows = {r: db.get_run(r) for r in MEMBERS}+ assert rows["job_a"]["status"] == "completed"+ assert rows["job_a"]["error_message"] is None+ assert rows["job_b"]["status"] == "stopped"+ assert rows["job_c"]["status"] == "error"+ assert rows["job_c"]["error_message"] == "CUDA error: boom"+ for i, run_id in enumerate(MEMBERS):+ row = rows[run_id]+ assert row["dataset_name"] == f"/data/{run_id}.jsonl"+ config = json.loads(row["config_json"])+ assert config["local_datasets"] == [f"/data/{run_id}.jsonl"]+ assert "hf_token" not in config and "wandb_token" not in config+ assert "hf_secret" not in row["config_json"]+ assert row["final_step"] == 2+ metrics = db.get_run_metrics(run_id)+ assert metrics["step_history"] == [1, 2]+ assert metrics["loss_history"] == pytest.approx([1.1 + i, 1.2 + i])+ assert rows["job_a"]["output_dir"] == outs["job_a"]+ assert rows["job_b"]["output_dir"] == outs["job_b"]+++def test_process_complete_finalizes_only_running_members(db, tmp_path):+ b = _batch_backend(tmp_path)+ saved = _exported(tmp_path, "job_b")+ for run_id in MEMBERS:+ b._handle_event(_progress(run_id, 1, 2.0))+ b._handle_event({"type": "error", "member_run_id": "job_a", "error": "bad dataset"})+ b._handle_event({"type": "output_dir", "member_run_id": "job_b", "output_dir": saved})+ b._handle_event(+ {"type": "complete", "output_dir": None, "status_message": "Training completed"}+ )++ assert db.get_run(BATCH) is None+ assert db.get_run("job_a")["status"] == "error"+ assert db.get_run("job_a")["error_message"] == "bad dataset"+ assert db.get_run("job_b")["status"] == "completed"+ assert db.get_run("job_c")["status"] == "error"+ assert "without saving" in db.get_run("job_c")["error_message"]+++def test_member_finalize_survives_a_transient_db_error(db, tmp_path, monkeypatch):+ b = _batch_backend(tmp_path)+ out = _exported(tmp_path, "job_a")+ b._handle_event({"type": "output_dir", "member_run_id": "job_a", "output_dir": out})+ b._handle_event(_progress("job_a", 1, 2.0))+ real, calls = db.finish_run, []++ def flaky(**kwargs):+ calls.append(kwargs["id"])+ if len(calls) == 1:+ raise RuntimeError("database is locked")+ return real(**kwargs)++ monkeypatch.setattr(db, "finish_run", flaky)+ monkeypatch.setattr(training_mod, "_DB_FINALIZE_RETRY_S", 0)+ b._handle_event({"type": "complete", "member_run_id": "job_a", "output_dir": out})+ assert calls == ["job_a", "job_a"]+ assert db.get_run("job_a")["status"] == "completed"+++def test_member_error_before_progress_survives_a_transient_db_error(db, tmp_path, monkeypatch):+ b = _batch_backend(tmp_path)+ real, calls = db.create_run, []++ def flaky(**kwargs):+ calls.append(kwargs["id"])+ if len(calls) == 1:+ raise RuntimeError("database is locked")+ return real(**kwargs)++ monkeypatch.setattr(db, "create_run", flaky)+ monkeypatch.setattr(training_mod, "_DB_FINALIZE_RETRY_S", 0)+ b._handle_event({"type": "error", "member_run_id": "job_c", "error": "empty dataset"})+ assert calls == ["job_c", "job_c"]+ row = db.get_run("job_c")+ assert row["status"] == "error" and row["error_message"] == "empty dataset"+++def test_member_error_before_progress_still_records_its_row(db, tmp_path):+ b = _batch_backend(tmp_path)+ b._handle_event({"type": "error", "member_run_id": "job_c", "error": "empty dataset"})+ row = db.get_run("job_c")+ assert row["status"] == "error" and row["error_message"] == "empty dataset"+ assert db.get_run("job_a") is None+ assert b.batch_member_ids() == ["job_a", "job_b"]+++def test_stop_member_sends_member_stop_only(db, tmp_path):+ b = _batch_backend(tmp_path)+ assert b.stop_member("job_b") is True+ assert b._stop_queue.get_nowait() == {"type": "stop_member", "run_id": "job_b", "save": True}+ assert b._stop_queue.empty()+ assert b._should_stop is False and b._cancel_requested is False+ assert b.stop_member("job_unknown") is False+ b._handle_event({"type": "complete", "member_run_id": "job_a"})+ assert b.stop_member("job_a") is False+ assert b._stop_queue.empty()+ b._handle_event(_progress("job_b", 1, 1.0))+ b._handle_event({"type": "complete", "member_run_id": "job_b", "output_dir": "/x"})+ assert db.get_run("job_b")["status"] == "stopped"+++def test_batch_cancel_marks_member_rows(db, tmp_path):+ b = _batch_backend(tmp_path)+ b._handle_event(_progress("job_a", 1, 1.0))+ assert b.stop_training(save = False, expected_job_id = BATCH) is True+ assert b._stop_queue.get_nowait() == {"type": "stop", "save": False}+ assert db.get_run("job_a")["resume_blocked"] == 1+ b._handle_event({"type": "complete", "status_message": "Training cancelled."})+ assert db.get_run(BATCH) is None+ for run_id in ("job_a",):+ row = db.get_run(run_id)+ assert row["status"] == "stopped" and row["output_dir"] is None+++def test_single_job_events_unchanged(db, tmp_path):+ b = TrainingBackend()+ b.current_job_id = "job_single"+ b._proc = _AliveProc()+ b._start_stop_watchdog = lambda **kw: None+ b._progress = TrainingProgress(is_training = True)+ b._db_config = {"model_name": "unsloth/tiny", "local_datasets": ["/d.jsonl"]}+ b._handle_event({"type": "progress", "step": 1, "loss": 1.5, "total_steps": 2})+ b._handle_event(+ {"type": "complete", "output_dir": "/o", "status_message": "Training completed"}+ )+ row = db.get_run("job_single")+ assert row["status"] == "completed"+ assert db.get_run_metrics("job_single")["step_history"] == [1]+++class _DummyProcess:+ pid = 4321++ def start(self):+ return None++ def is_alive(self):+ return True+++class _DummyThread:+ def start(self):+ return None+++def test_start_training_builds_batch_members_config(db, tmp_path):+ captured = {}+ real_spec = training_mod.account_process_spec++ def _spy(module, fn, cache_env, kwargs):+ captured["config"] = kwargs["config"]+ return real_spec(module, fn, cache_env, kwargs)++ def _select(gpu_ids, **kwargs):+ captured["sizing"] = kwargs+ return [0], {}++ backend = TrainingBackend()+ with (+ patch("core.training.training.prepare_gpu_selection", side_effect = _select),+ patch("core.training.training._CTX.Queue", side_effect = [object(), object()]),+ patch("core.training.training._CTX.Process", return_value = _DummyProcess()),+ patch("core.training.training.threading.Thread", return_value = _DummyThread()),+ patch("core.training.training.account_process_spec", side_effect = _spy),+ ):+ ok = backend.start_training(+ job_id = BATCH,+ batch_members = [+ {+ "run_id": run_id,+ "model_name": "unsloth/test",+ "training_type": "LoRA/QLoRA",+ "hf_token": "hf_secret",+ "local_datasets": [f"/data/{run_id}.jsonl"],+ "lora_r": 8 * (i + 1),+ "batch_size": i + 1,+ }+ for i, run_id in enumerate(MEMBERS)+ ],+ model_name = "unsloth/test",+ training_type = "LoRA/QLoRA",+ hf_token = "hf_secret",+ local_datasets = ["/data/job_a.jsonl"],+ lora_r = 8,+ gpu_ids = [0],+ )+ assert ok+ # The GPU is chosen for every member's rows and adapters together, not the first member's.+ assert captured["sizing"]["batch_size"] == 1 + 2 + 3+ assert captured["sizing"]["lora_rank"] == 8 + 16 + 24+ config = captured["config"]+ assert config["batch_id"] == BATCH+ assert config["hf_token"] == "hf_secret"+ assert [m["run_id"] for m in config["batch_members"]] == list(MEMBERS)+ for i, member in enumerate(config["batch_members"]):+ assert member["config"]["lora_r"] == 8 * (i + 1)+ assert member["config"]["local_datasets"] == [f"/data/{MEMBERS[i]}.jsonl"]+ assert "hf_token" not in member["config"]+ assert "subject" not in member["config"]+ assert backend.batch_member_ids() == list(MEMBERS)+ assert backend._db_config is None+ assert db.get_run(BATCH) is None+++def test_start_training_without_batch_is_unchanged(db, tmp_path):+ captured = {}+ real_spec = training_mod.account_process_spec++ def _spy(module, fn, cache_env, kwargs):+ captured["config"] = kwargs["config"]+ return real_spec(module, fn, cache_env, kwargs)++ backend = TrainingBackend()+ with (+ patch("core.training.training.prepare_gpu_selection", return_value = ([0], {})),+ patch("core.training.training._CTX.Queue", side_effect = [object(), object()]),+ patch("core.training.training._CTX.Process", return_value = _DummyProcess()),+ patch("core.training.training.threading.Thread", return_value = _DummyThread()),+ patch("core.training.training.account_process_spec", side_effect = _spy),+ ):+ assert backend.start_training(+ job_id = "job_single", model_name = "unsloth/test", training_type = "LoRA/QLoRA"+ )+ assert "batch_members" not in captured["config"]+ assert "batch_id" not in captured["config"]+ assert backend.batch_member_ids() == []+ assert db.get_run("job_single")["status"] == "running"+++def test_batch_start_sizes_chat_coexistence_for_every_member(monkeypatch):+ import asyncio+ import contextlib++ import routes.training as tr+ import routes.training_vram as vram+ from models import TrainingStartRequest++ class _Backend:+ current_job_id = None++ def is_training_active(self):+ return False++ def start_training(+ self,+ job_id,+ *,+ before_spawn = None,+ **kwargs,+ ):+ self.current_job_id = job_id+ before_spawn()+ return True++ asked = []+ monkeypatch.setattr(tr, "get_training_backend", lambda: _Backend())+ monkeypatch.setattr(tr, "_diffusion_training_active", lambda: False)+ monkeypatch.setattr(tr, "_diffusion_gpu_admission", contextlib.nullcontext)+ monkeypatch.setattr(tr, "hf_env_offline", lambda: False)+ monkeypatch.setattr(tr, "_hub_unreachable", lambda: False)+ monkeypatch.setattr(tr, "_remote_untrainable_model_format", lambda *a, **k: None)+ monkeypatch.setattr("utils.hardware.ensure_hardware_detected", lambda: None)+ monkeypatch.setattr(vram, "coordinate_models_for_training", lambda keep: [keep()])+ monkeypatch.setattr(+ vram, "can_keep_chat_during_training", lambda **kw: asked.append(kw) or True+ )++ def request(r, batch_size):+ return TrainingStartRequest(+ model_name = "unsloth/tiny-model",+ training_type = "LoRA/QLoRA",+ format_type = "alpaca",+ load_in_4bit = False,+ lora_r = r,+ batch_size = batch_size,+ )++ started = asyncio.run(+ tr.start_training_batch(+ [request(8, 2), request(16, 4), request(32, 1)],+ current_subject = "alice",+ via_api_key = True,+ )+ )+ assert [s.status for s in started] == ["queued"] * 3+ assert [(kw["batch_size"], kw["lora_rank"]) for kw in asked] == [(2 + 4 + 1, 8 + 16 + 32)]+++def test_xet_respawn_replays_pending_member_stops(monkeypatch, tmp_path):+ class _Queue:+ def __init__(self):+ self.items = []++ def put(self, item, *a, **k):+ self.items.append(item)++ class _Proc:+ pid = 4322++ def __init__(self, **kwargs):+ self.kwargs = kwargs++ def start(self):+ pass++ def is_alive(self):+ return False++ def join(self, timeout = None):+ pass++ class _Ctx:+ def __init__(self):+ self.spawned = []++ def Queue(self):+ return _Queue()++ def Process(self, **kwargs):+ self.spawned.append(kwargs)+ return _Proc(**kwargs)++ b = _batch_backend(tmp_path)+ b._last_full_config = {"model_name": "unsloth/tiny", "batch_members": []}+ b._last_hf_cache_env = {"HF_HOME": str(tmp_path / "hf")}+ b._proc = _Proc()+ b._batch_members["job_a"]["stop_requested"] = True+ b._batch_members["job_c"].update(stop_requested = True, terminal = True)+ ctx = _Ctx()+ monkeypatch.setattr(training_mod, "_CTX", ctx)+ monkeypatch.setattr(b, "_pump_loop", lambda: None)+ monkeypatch.setattr(b, "_open_worker_stderr_capture", lambda: None)++ assert b._respawn_worker_disable_xet() is True+ stop_queue = b._stop_queue+ assert stop_queue.items == [{"type": "stop_member", "run_id": "job_a", "save": True}]diff --git a/studio/backend/tests/test_training_cached_repo_authorization.py b/studio/backend/tests/test_training_cached_repo_authorization.pyindex 17b473d8801..5f6fea9bf41 100644--- a/studio/backend/tests/test_training_cached_repo_authorization.py+++ b/studio/backend/tests/test_training_cached_repo_authorization.py@@ -468,7 +468,7 @@ def test_a_disabled_eval_path_is_not_authorized(monkeypatch, cache_root):import routes.training as training_routes- source = inspect.getsource(training_routes.start_training)+ source = inspect.getsource(training_routes._admit_training_request)marker = "request.local_eval_datasets if evaluation_enabled(request.eval_steps) else []"assert marker in source@@ -511,7 +511,7 @@ def test_a_streaming_start_skips_the_cached_dataset_check():import routes.training as training_routes- source = inspect.getsource(training_routes.start_training)+ source = inspect.getsource(training_routes._admit_training_request)assert "not request.dataset_streaming" in sourceguarded = source.split("not request.dataset_streaming", 1)[1]assert "_refuse_unauthorized_cached_dataset" in guarded.split("\n\n", 1)[0]@@ -557,7 +557,7 @@ def test_the_shadowed_dataset_gate_is_wired_into_the_start():import routes.training as training_routes- source = inspect.getsource(training_routes.start_training)+ source = inspect.getsource(training_routes._admit_training_request)assert "_hf_dataset_is_the_source(request)" in sourceguarded = source.split("_hf_dataset_is_the_source(request)", 1)[1]assert "_refuse_unauthorized_cached_dataset" in guarded.split("\n\n", 1)[0]