Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: train compatible queued fine-tuning jobs together with multi-LoRA (#15344)

Daniel Han3e3c2fd53937
diff --git a/studio/backend/core/training/multi_lora_trainer.py b/studio/backend/core/training/multi_lora_trainer.py
new file mode 100644
index 00000000000..979932fdcaa
--- /dev/null
+++ b/studio/backend/core/training/multi_lora_trainer.py
@@ -0,0 +1,1084 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Several queued LoRA jobs trained together on one base model with the fused multi-LoRA kernels.
+
+Each member keeps its own dataset, rank, alpha, learning rate, epochs, seed, optimizer and output, and
+reports its own events (``member_run_id``) with the same shapes a lone job sends."""
+
+from __future__ import annotations
+
+import copy
+import math
+import os
+import queue
+import time
+import traceback
+import types
+import warnings
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterator, Optional
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+_DEFAULT_TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+_OPTIMIZERS = ("adamw_8bit", "adamw_torch", "adamw_torch_fused")
+
+
+@dataclass
+class _Member:
+ slot: int
+ run_id: str
+ config: dict
+ tokenizer: Any = None
+ rows: list = field(default_factory = list)
+ output_dir: str = ""
+ total_steps: int = 0
+ steps_per_epoch: int = 1
+ step: int = 0
+ num_tokens: int = 0
+ windows: Optional[Iterator] = None
+ optimizer: Any = None
+ scheduler: Any = None
+ params: list = field(default_factory = list)
+ done: bool = False
+
+
+def _send(
+ event_queue,
+ kind: str,
+ member: Optional[_Member] = None,
+ **payload,
+) -> None:
+ event = {"type": kind, **payload, "ts": time.time()}
+ if member is not None:
+ event["member_run_id"] = member.run_id
+ event_queue.put(event)
+
+
+def _new_trainer():
+ from core.training.trainer import UnslothTrainer
+ return UnslothTrainer()
+
+
+def _detect_model(trainer, config: dict, hf_token) -> None:
+ from core.training import worker
+
+ model_load_name = worker._resolve_cached_model_load_name(config)
+ local_only = worker._model_local_files_only(config)
+ worker._pre_detect_training_model(
+ trainer,
+ config,
+ config["model_name"],
+ hf_token,
+ model_load_name,
+ local_only,
+ None if local_only else config.get("model_revision"),
+ )
+
+
+def _format_dataset(trainer, config: dict, hf_token):
+ from core.training import worker
+
+ hf_dataset = config.get("hf_dataset", "")
+ result = trainer.load_and_format_dataset(
+ dataset_source = hf_dataset if hf_dataset and hf_dataset.strip() else None,
+ format_type = config.get("format_type", ""),
+ local_datasets = config.get("local_datasets") or None,
+ custom_format_mapping = config.get("custom_format_mapping"),
+ subset = config.get("subset"),
+ train_split = config.get("train_split", "train"),
+ dataset_slice_start = config.get("dataset_slice_start"),
+ dataset_slice_end = config.get("dataset_slice_end"),
+ s3_config = config.get("s3_config"),
+ dataset_local_files_only = worker._dataset_local_files_only(config),
+ dataset_local_path = config.get("dataset_snapshot_path"),
+ dataset_revision = config.get("dataset_revision"),
+ hf_token = hf_token,
+ max_train_rows_seed = config.get("random_seed", 3407),
+ )
+ if result is None:
+ raise RuntimeError(trainer.training_progress.error or "Failed to load dataset")
+ return result[0] if isinstance(result, tuple) else result
+
+
+def _load_base(trainer, config: dict, hf_token, event_queue) -> Optional[bool]:
+ """The single job's LoRA load; returns the effective 4-bit flag, or None when stopped."""
+ from core.training import worker
+ from utils.hf_xet_fallback import start_watchdog
+
+ model_name = config["model_name"]
+ model_load_name = worker._resolve_cached_model_load_name(config)
+ local_only = worker._model_local_files_only(config)
+ base_error = worker._adapter_base_changed_error(config) or worker._escaping_weights_error(
+ config
+ )
+ if base_error:
+ raise RuntimeError(base_error)
+ event_queue.put({"type": "model_load_started", "ts": time.time()})
+ watchdog = start_watchdog(
+ repo_ids = [model_name],
+ on_stall = lambda msg: event_queue.put({"type": "stall", "message": msg, "ts": time.time()}),
+ xet_disabled = os.environ.get("HF_HUB_DISABLE_XET") == "1",
+ )
+
+ def report_repo(repo_id):
+ if not os.path.isdir(os.path.expanduser(repo_id)):
+ event_queue.put({"type": "model_load_resolved", "repo_id": repo_id, "ts": time.time()})
+
+ try:
+ load_in_4bit = worker._effective_training_load_in_4bit(config, model_load_name, hf_token)
+ success = trainer.load_model(
+ on_model_resolved = report_repo,
+ model_name = model_name,
+ max_seq_length = config["max_seq_length"],
+ load_in_4bit = load_in_4bit,
+ full_finetuning = False,
+ hf_token = hf_token,
+ trust_remote_code = config.get("trust_remote_code", False),
+ gpu_ids = config.get("resolved_gpu_ids"),
+ model_load_name = model_load_name,
+ local_files_only = local_only,
+ actual_model_repo_id = config.get("actual_model_repo_id"),
+ model_revision = None if local_only else config.get("model_revision"),
+ use_gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
+ offload_layers = config.get("offload_layers", 0),
+ prefetch_depth = config.get("prefetch_depth", 2),
+ offload_plan_shape = worker._offload_plan_shape(config),
+ )
+ finally:
+ watchdog.set()
+ event_queue.put({"type": "model_load_completed", "ts": time.time()})
+ if trainer.should_stop:
+ return None
+ if not success:
+ raise RuntimeError(
+ worker._with_vram_budget_hint(
+ config, trainer.training_progress.error or "Failed to load model"
+ )
+ )
+ worker._emit_resource_provenance(
+ event_queue,
+ config,
+ trainer.model,
+ model_load_target = model_load_name,
+ model_load_in_4bit = load_in_4bit,
+ dataset_loaded_from_exact_snapshot = bool(
+ getattr(trainer, "dataset_loaded_from_exact_snapshot", False)
+ ),
+ )
+ return load_in_4bit
+
+
+def _is_moe(model) -> bool:
+ """unsloth.models._utils.is_moe_model, without importing unsloth's package root."""
+ config = getattr(model, "config", None)
+ for cfg in (config, getattr(config, "text_config", None)):
+ count = next(
+ (
+ getattr(cfg, k)
+ for k in ("num_experts", "n_routed_experts", "num_local_experts")
+ if getattr(cfg, k, None) is not None
+ ),
+ None,
+ )
+ if count is not None:
+ return isinstance(count, int) and count > 0
+ return False
+
+
+def _resolve_targets(model, target_modules) -> list[str]:
+ """Leaf names as PEFT resolves them: an explicit list, the Studio default, or every linear layer
+ except the output head for ``all-linear``."""
+ if isinstance(target_modules, str):
+ target_modules = [target_modules]
+ target_modules = list(target_modules or [])
+ # As the single job: explicit names beside "all-linear" win; "all-linear" alone means every linear layer.
+ explicit = [m for m in target_modules if m != "all-linear"]
+ if explicit or not target_modules:
+ return explicit or list(_DEFAULT_TARGETS)
+ head = model.get_output_embeddings() if hasattr(model, "get_output_embeddings") else None
+ names = {
+ name.rsplit(".", 1)[-1]
+ for name, module in model.named_modules()
+ if name
+ and module is not head
+ and hasattr(module, "in_features")
+ and hasattr(module, "weight")
+ and not list(module.children())
+ }
+ return sorted(names)
+
+
+def _multi_lora_layers(model):
+ from unsloth.kernels.multi_lora import MultiLoraLinear
+ return [(n, m) for n, m in model.named_modules() if isinstance(m, MultiLoraLinear)]
+
+
+def _seed_adapter(model, slot: int, seed: int) -> None:
+ """Kaiming A from the member's own seed, B zero: the same init whatever the batch order."""
+ import torch
+
+ generator = torch.Generator().manual_seed(int(seed))
+ with torch.no_grad():
+ for _, layer in _multi_lora_layers(model):
+ A = torch.empty(layer.lora_A[slot].shape, dtype = torch.float32)
+ torch.nn.init.kaiming_uniform_(A, a = math.sqrt(5), generator = generator)
+ layer.lora_A[slot].copy_(A)
+ layer.lora_B[slot].zero_()
+
+
+def _enable_training(model, gradient_checkpointing, max_seq_length: int) -> None:
+ """What get_peft_model does for checkpointing on a LoRA model, minus the PEFT wrapper."""
+ from unsloth.models._utils import (
+ apply_unsloth_gradient_checkpointing,
+ prepare_model_for_kbit_training,
+ )
+
+ dtype = model.get_input_embeddings().weight.dtype
+ gradient_checkpointing = apply_unsloth_gradient_checkpointing(
+ gradient_checkpointing, max_seq_length, dtype
+ )
+ model._unsloth_gradient_checkpointing = gradient_checkpointing
+ prepare_model_for_kbit_training(
+ model, use_gradient_checkpointing = gradient_checkpointing, use_reentrant = True
+ )
+ for_training = getattr(model, "for_training", None)
+ if callable(for_training):
+ for_training(use_gradient_checkpointing = gradient_checkpointing)
+ else:
+ model.train()
+
+
+def _check_routed(model, device) -> None:
+ """One tiny forward must reach every injected layer, else the patched forwards bypass them."""
+ import torch
+ from unsloth.kernels.multi_lora import set_multi_lora_batch
+
+ layers = _multi_lora_layers(model)
+ hits = set()
+ handles = [
+ layer.register_forward_hook(lambda m, i, o, name = name: hits.add(name))
+ for name, layer in layers
+ ]
+ try:
+ ids = torch.zeros(1, 2, dtype = torch.long, device = device)
+ set_multi_lora_batch(model, torch.zeros(1, dtype = torch.long, device = device), seq_len = 2)
+ # No grad here, so checkpointed layers warn that their inputs need none.
+ with torch.no_grad(), warnings.catch_warnings():
+ warnings.simplefilter("ignore", UserWarning)
+ model(input_ids = ids, attention_mask = torch.ones_like(ids), use_cache = False)
+ finally:
+ for handle in handles:
+ handle.remove()
+ for _, layer in layers:
+ layer.batch = layer.seen = layer.trained = None
+ missed = [name for name, _ in layers if name not in hits]
+ if missed:
+ raise RuntimeError(
+ f"Batched training cannot run on this model: its forward skips {len(missed)} adapter "
+ f"layers (e.g. {missed[0]})."
+ )
+
+
+def _member_tokenizer(tokenizer, model_name: str, final_format: str):
+ from utils.datasets.chat_templates import get_training_chat_template
+ if getattr(tokenizer, "chat_template", None) or final_format not in (
+ "alpaca",
+ "chatml_messages",
+ "chatml_conversations",
+ ):
+ return tokenizer
+ return get_training_chat_template(copy.deepcopy(tokenizer), model_name, final_format)
+
+
+def _tokenize_member(
+ dataset_info,
+ tokenizer,
+ config: dict,
+ model_name: str,
+ max_length: int,
+ vocab_size = None,
+):
+ """(tokenizer to save, [(input_ids, labels)]) with the single SFT path's tokenization and masking."""
+ from datasets import Dataset
+
+ from utils.datasets.completion_masking import apply_completion_masking
+ from utils.datasets.online_tokenization import (
+ build_tokenizing_transform,
+ first_sample_text,
+ resolve_add_special_tokens,
+ )
+ from utils.datasets.pretokenized import (
+ PRETOKENIZED_FORMAT,
+ _collator_masks_user_labels,
+ prepare_pretokenized_dataset,
+ )
+
+ dataset = dataset_info["dataset"] if isinstance(dataset_info, dict) else dataset_info
+ final_format = (
+ str(dataset_info.get("final_format", "")).lower() if isinstance(dataset_info, dict) else ""
+ )
+ member_tok = _member_tokenizer(tokenizer, model_name, final_format)
+ labels = None
+ if final_format == PRETOKENIZED_FORMAT:
+ # The lone job's checks (ids inside the vocabulary, mask and label lengths) and truncation: a bad
+ # dataset then fails only its own member, never the shared forward.
+ dataset = prepare_pretokenized_dataset(
+ dataset, max_seq_length = max_length, vocab_size = vocab_size
+ )
+ masks = dataset["assistant_masks"] if "assistant_masks" in dataset.column_names else None
+ input_ids = [list(x)[:max_length] for x in dataset["input_ids"]]
+ if "labels" in dataset.column_names:
+ labels = [list(x)[:max_length] for x in dataset["labels"]]
+ if not _collator_masks_user_labels():
+ masks = None
+ if masks is not None:
+ # TRL's collator trains only the assistant tokens, on top of user labels before TRL 1.7.
+ labels = [
+ [t if m else -100 for t, m in zip(row, mask)]
+ for row, mask in zip(labels if labels is not None else input_ids, masks)
+ ]
+ else:
+ add_special_tokens = resolve_add_special_tokens(
+ member_tok, first_sample_text(dataset, "text")
+ )
+ encode = build_tokenizing_transform(member_tok, "text", max_length, add_special_tokens)
+ input_ids = [list(x) for x in encode({"text": list(dataset["text"])})["input_ids"]]
+ raw = final_format == "raw_text" or config.get("format_type") == "raw"
+ if labels is None and config.get("train_on_completions") and not raw:
+
+ def build_masker(holder, **kwargs):
+ from unsloth.chat_templates import train_on_responses_only
+
+ kwargs.pop("num_proc", None)
+ masker = train_on_responses_only(
+ None, tokenizer = holder.processing_class, return_function = True, **kwargs
+ )
+ holder.train_dataset = holder.train_dataset.map(masker, batched = True)
+ return holder
+
+ holder, applied = apply_completion_masking(
+ types.SimpleNamespace(
+ processing_class = member_tok,
+ train_dataset = Dataset.from_dict({"input_ids": input_ids}),
+ eval_dataset = None,
+ ),
+ model_name,
+ build_masker,
+ notify = lambda level, message: logger.info(message),
+ dataset_template = "alpaca" if final_format == "alpaca" else None,
+ )
+ if applied:
+ labels = [list(x) for x in holder.train_dataset["labels"]]
+ kept = [i for i, row in enumerate(labels) if any(t != -100 for t in row[1:])]
+ dropped = len(labels) - len(kept)
+ drop_pct = round(100 * dropped / len(labels), 1) if labels else 0
+ if not kept or drop_pct > 30:
+ raise ValueError(
+ f"{dropped}/{len(labels)} samples ({drop_pct}%) were dropped after applying "
+ f"'Train on completions': after masking, those rows had no trainable response "
+ f"tokens left. The usual cause is that this model's response template was not "
+ f"found in the formatted samples, so 'Train on completions' should be turned "
+ f"off for this dataset."
+ )
+ input_ids = [input_ids[i] for i in kept]
+ labels = [labels[i] for i in kept]
+ if labels is None:
+ labels = [list(x) for x in input_ids]
+ rows = [(ids, lab) for ids, lab in zip(input_ids, labels) if len(ids) > 1]
+ if not rows:
+ raise ValueError("The dataset has no rows to train on after tokenization.")
+ return member_tok, rows
+
+
+def _windows(n_rows: int, batch_size: int, grad_accum: int, seed: int):
+ """Endless (epoch, [microbatch row indices] * <= grad_accum); windows never span an epoch."""
+ import torch
+
+ epoch = 0
+ while True:
+ order = torch.randperm(n_rows, generator = torch.Generator().manual_seed(seed + epoch))
+ order = order.tolist()
+ micro = [order[i : i + batch_size] for i in range(0, n_rows, batch_size)]
+ for i in range(0, len(micro), grad_accum):
+ yield epoch, micro[i : i + grad_accum]
+ epoch += 1
+
+
+def _member_scheduler(config: dict, lr: float, total_steps: int):
+ """The single job's schedule for this member, on a stand-in optimizer: it only yields learning rates."""
+ import torch
+ from transformers import get_scheduler
+
+ warmup_ratio, warmup_steps = config.get("warmup_ratio"), config.get("warmup_steps")
+ if warmup_ratio is not None:
+ warmup = math.ceil(total_steps * float(warmup_ratio))
+ elif warmup_steps is not None:
+ warmup = int(warmup_steps)
+ else:
+ warmup = 5
+ stand_in = torch.optim.SGD([torch.zeros(1, requires_grad = True)], lr = lr)
+ return get_scheduler(
+ config.get("lr_scheduler_type", "linear") or "linear",
+ stand_in,
+ num_warmup_steps = warmup,
+ num_training_steps = total_steps,
+ )
+
+
+class _PerMemberLR:
+ """Steps every member's own schedule and writes its rate into that member's parameter group."""
+
+ def __init__(self, optimizer, schedulers: list):
+ self.optimizer = optimizer
+ self.schedulers = schedulers
+ self._apply()
+
+ def _apply(self) -> None:
+ for group, scheduler in zip(self.optimizer.param_groups, self.schedulers):
+ group["lr"] = scheduler.get_last_lr()[0]
+
+ def step(self) -> None:
+ for scheduler in self.schedulers:
+ scheduler.step()
+ self._apply()
+
+ def get_last_lr(self) -> list:
+ return [group["lr"] for group in self.optimizer.param_groups]
+
+ def state_dict(self) -> dict:
+ return {}
+
+ def load_state_dict(self, state: dict) -> None:
+ pass
+
+
+def _collate(rows: list, slots: list, pad_id: int) -> dict:
+ import torch
+
+ T = max(len(ids) for ids, _ in rows)
+ input_ids = torch.full((len(rows), T), pad_id, dtype = torch.long)
+ labels = torch.full((len(rows), T), -100, dtype = torch.long)
+ attention = torch.zeros((len(rows), T), dtype = torch.long)
+ for i, (ids, lab) in enumerate(rows):
+ input_ids[i, : len(ids)] = torch.tensor(ids)
+ labels[i, : len(lab)] = torch.tensor(lab)
+ attention[i, : len(ids)] = 1
+ return {
+ "input_ids": input_ids,
+ "attention_mask": attention,
+ "labels": labels,
+ "adapter_ids": torch.tensor(slots, dtype = torch.long),
+ }
+
+
+def _unique_output_dir(config: dict, model_name: str, taken: set) -> str:
+ from utils.paths import resolve_output_dir
+ from utils.training_runs import build_default_output_dir_name
+
+ explicit = config.get("output_dir")
+ stamp = int(time.time())
+ for attempt in range(1000):
+ if explicit:
+ name = explicit if attempt == 0 else f"{explicit}-{attempt}"
+ else:
+ # A later timestamp, not a suffix: the folder name is parsed back by its trailing timestamp.
+ name = build_default_output_dir_name(
+ model_name, config.get("project_name"), timestamp = stamp + attempt
+ )
+ path = str(resolve_output_dir(name))
+ if path not in taken and not os.path.exists(path):
+ taken.add(path)
+ return path
+ raise RuntimeError("Could not reserve an output folder for a batched job.")
+
+
+def _export(
+ model,
+ member: _Member,
+ trainer,
+ targets: list,
+ use_rslora: bool,
+ path: str = "",
+) -> None:
+ from peft import LoraConfig
+ from safetensors.torch import save_file
+
+ from core.training.trainer import UnslothTrainer
+ from unsloth.kernels.multi_lora import adapter_state_dict
+ from utils.paths import ensure_dir
+
+ path = path or member.output_dir
+ ensure_dir(Path(path))
+ state = {
+ k: v.detach().to("cpu").contiguous()
+ for k, v in adapter_state_dict(model, member.slot).items()
+ }
+ save_file(
+ state,
+ os.path.join(path, "adapter_model.safetensors"),
+ metadata = {"format": "pt"},
+ )
+ LoraConfig(
+ r = int(member.config.get("lora_r", 16)),
+ lora_alpha = member.config.get("lora_alpha", 16),
+ lora_dropout = 0.0,
+ bias = "none",
+ target_modules = targets,
+ use_rslora = use_rslora,
+ task_type = "CAUSAL_LM",
+ base_model_name_or_path = getattr(model.config, "_name_or_path", None),
+ ).save_pretrained(path)
+ member.tokenizer.save_pretrained(path)
+ UnslothTrainer._patch_adapter_config(trainer, path)
+
+
+def _status_forwarder(event_queue):
+ from core.training.worker import _send_status
+
+ sent: set = set()
+
+ def forward(progress) -> None:
+ if progress.status_message:
+ _send_status(event_queue, progress.status_message)
+ for message in progress.warnings:
+ if message not in sent:
+ sent.add(message)
+ event_queue.put({"type": "warning", "message": message, "ts": time.time()})
+
+ return forward
+
+
+def _start_stop_reader(stop_queue, inbox: "queue.Queue", trainer) -> None:
+ from utils.account_context import account_thread
+ def read():
+ while True:
+ try:
+ message = stop_queue.get(timeout = 1.0)
+ except queue.Empty:
+ continue
+ except (EOFError, OSError, ValueError):
+ return
+ if not isinstance(message, dict):
+ continue
+ if message.get("type") == "stop":
+ trainer.should_stop = True
+ trainer.save_on_stop = bool(message.get("save", True))
+ if message.get("type") in ("stop", "stop_member"):
+ inbox.put(message)
+
+ account_thread(target = read, daemon = True).start()
+
+
+def _oom_message(exc: BaseException) -> Optional[str]:
+ text = str(exc).lower()
+ if "out of memory" in text or type(exc).__name__ == "OutOfMemoryError":
+ return (
+ "GPU ran out of VRAM while training the batched jobs.\n"
+ "To fix: reduce max_seq_length or the batch size of the queued jobs."
+ )
+ return None
+
+
+def run_multi_lora_training(event_queue: Any, stop_queue: Any, config: dict) -> None:
+ """Worker entry for ``config["batch_members"]`` (same event and stop contract as one job)."""
+ members = [
+ _Member(slot = -1, run_id = m["run_id"], config = m["config"]) for m in config["batch_members"]
+ ]
+ try:
+ _run(event_queue, stop_queue, config, members)
+ except Exception as exc:
+ message = _oom_message(exc) or str(exc)
+ stack = traceback.format_exc(limit = 20)
+ logger.error("Batched training failed: %s", exc)
+ for member in members:
+ if not member.done:
+ member.done = True
+ _send(event_queue, "error", member, error = message, stack = stack)
+ event_queue.put({"type": "error", "error": message, "stack": stack, "ts": time.time()})
+
+
+def _finish_stopped(event_queue, members, save: bool, export) -> None:
+ for member in members:
+ if member.done:
+ continue
+ member.done = True
+ saved = bool(save and member.step > 0 and member.output_dir)
+ if saved:
+ export(member)
+ _send(
+ event_queue,
+ "stopped",
+ member,
+ output_dir = member.output_dir if saved else None,
+ status_message = (
+ f"Training stopped. Model saved to {member.output_dir}"
+ if saved
+ else "Training cancelled."
+ ),
+ saved = saved,
+ )
+
+
+def _run(event_queue, stop_queue, config: dict, members: list) -> None:
+ from core.import_guards import ensure_real_packages
+
+ ensure_real_packages("unsloth_zoo", "unsloth")
+ import torch
+
+ from core.training.trainer import normalize_gradient_checkpointing
+ from core.training.worker import _send_status, _worker_hf_token
+ from unsloth.kernels.multi_lora import inject_multi_lora
+
+ trainer = _new_trainer()
+ if hasattr(trainer, "add_progress_callback"):
+ trainer.add_progress_callback(_status_forwarder(event_queue))
+ inbox: queue.Queue = queue.Queue()
+ _start_stop_reader(stop_queue, inbox, trainer)
+
+ def stopped_early() -> bool:
+ if not trainer.should_stop:
+ return False
+ _finish_stopped(event_queue, members, False, None)
+ event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
+ return True
+
+ hf_token = _worker_hf_token(config)
+ model_name = config["model_name"]
+ _send_status(event_queue, "Detecting model type...")
+ _detect_model(trainer, config, hf_token)
+ if stopped_early():
+ return
+
+ datasets = {}
+ for i, member in enumerate(members):
+ _send_status(event_queue, f"Loading and formatting dataset {i + 1}/{len(members)}...")
+ try:
+ datasets[member.run_id] = _format_dataset(trainer, member.config, hf_token)
+ except Exception as exc:
+ member.done = True
+ _send(
+ event_queue, "error", member, error = str(exc), stack = traceback.format_exc(limit = 20)
+ )
+ if stopped_early():
+ return
+ if not datasets:
+ raise RuntimeError("None of the batched jobs could load its dataset.")
+
+ _send_status(event_queue, "Loading model...")
+ load_in_4bit = _load_base(trainer, config, hf_token, event_queue)
+ if stopped_early():
+ return
+ if getattr(trainer, "diffusion_lm_profile", None):
+ raise RuntimeError(
+ "Batched training does not support text-diffusion models; train these jobs one at a time."
+ )
+ if _is_moe(trainer.model):
+ # The single-job path trains fused experts through PEFT target_parameters; these layers cannot.
+ raise RuntimeError(
+ "Batched training does not support MoE models; train these jobs one at a time."
+ )
+ if getattr(trainer.model, "peft_config", None):
+ # A saved LoRA loads as a PeftModel; the multi-LoRA layers would train fresh adapters and drop its weights.
+ raise RuntimeError(
+ "Batched training cannot continue an existing LoRA adapter; train these jobs one at a time."
+ )
+ trainer.load_in_4bit = load_in_4bit
+ trainer.is_cpt = False
+ model, tokenizer = trainer.model, trainer.tokenizer
+ max_length = int(config.get("max_seq_length") or 2048)
+ model_cap = getattr(model, "max_seq_length", None)
+ if isinstance(model_cap, int) and 0 < model_cap < max_length:
+ max_length = model_cap
+
+ _send_status(event_queue, "Tokenizing datasets...")
+ for member in members:
+ if member.done:
+ continue
+ try:
+ member.tokenizer, member.rows = _tokenize_member(
+ datasets.pop(member.run_id),
+ tokenizer,
+ member.config,
+ model_name,
+ max_length,
+ vocab_size = int(model.get_input_embeddings().num_embeddings),
+ )
+ except Exception as exc:
+ member.done = True
+ _send(
+ event_queue, "error", member, error = str(exc), stack = traceback.format_exc(limit = 20)
+ )
+ live = [m for m in members if not m.done]
+ if not live:
+ raise RuntimeError("None of the batched jobs has rows to train on.")
+ for slot, member in enumerate(live):
+ member.slot = slot
+ U = len(live)
+
+ _send_status(event_queue, f"Configuring {U} LoRA adapters...")
+ targets = _resolve_targets(model, config.get("target_modules"))
+ use_rslora = bool(config.get("use_rslora", False))
+ injected = inject_multi_lora(
+ model,
+ targets,
+ U,
+ r = [int(m.config.get("lora_r", 16)) for m in live],
+ lora_alpha = [float(m.config.get("lora_alpha", 16)) for m in live],
+ use_rslora = use_rslora,
+ )
+ targets = sorted({name.rsplit(".", 1)[-1] for name in injected})
+ for member in live:
+ _seed_adapter(model, member.slot, member.config.get("random_seed", 3407))
+ _enable_training(
+ model,
+ normalize_gradient_checkpointing(config.get("gradient_checkpointing", "unsloth")),
+ max_length,
+ )
+ device = model.get_input_embeddings().weight.device
+ _check_routed(model, device)
+
+ layers = _multi_lora_layers(model)
+ grad_accum = max(int(config.get("gradient_accumulation_steps", 4) or 1), 1)
+ taken: set = set()
+ for member in live:
+ cfg = member.config
+ batch_size = max(int(cfg.get("batch_size", 2) or 1), 1)
+ n_micro = math.ceil(len(member.rows) / batch_size)
+ member.steps_per_epoch = max(math.ceil(n_micro / grad_accum), 1)
+ max_steps = int(cfg.get("max_steps") or 0)
+ member.total_steps = (
+ max_steps
+ if max_steps > 0
+ else member.steps_per_epoch * int(cfg.get("num_epochs", 3) or 1)
+ )
+ member.windows = _windows(
+ len(member.rows), batch_size, grad_accum, int(cfg.get("random_seed", 3407))
+ )
+ member.params = [
+ p for _, layer in layers for p in (layer.lora_A[member.slot], layer.lora_B[member.slot])
+ ]
+ member.output_dir = _unique_output_dir(cfg, model_name, taken)
+ _send(event_queue, "output_dir", member, output_dir = member.output_dir)
+
+ def export(member: _Member, path: str = "") -> None:
+ _export(model, member, trainer, targets, use_rslora, path)
+
+ def take_messages() -> Optional[bool]:
+ """Applies queued member stops; returns the save flag of a whole-batch stop, if one arrived."""
+ save_all = None
+ while not inbox.empty():
+ message = inbox.get_nowait()
+ if message.get("type") == "stop":
+ save_all = bool(message.get("save", True))
+ continue
+ target = next(
+ (m for m in live if m.run_id == message.get("run_id") and not m.done), None
+ )
+ if target is not None:
+ _finish_stopped(event_queue, [target], bool(message.get("save", True)), export)
+ return save_all
+
+ def stop_all(save: bool) -> None:
+ _finish_stopped(event_queue, live, save, export)
+ event_queue.put(
+ {
+ "type": "complete",
+ "output_dir": None,
+ "status_message": "Training stopped",
+ "ts": time.time(),
+ }
+ )
+
+ save_all = take_messages()
+ if save_all is not None:
+ stop_all(save_all)
+ return
+ if all(m.done for m in live):
+ event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
+ return
+
+ sft = _build_sft_trainer(
+ model,
+ tokenizer,
+ config,
+ live,
+ grad_accum,
+ device,
+ event_queue,
+ export,
+ take_messages,
+ stop_all,
+ )
+ for member in live:
+ if not member.done:
+ sft.progress(member, None, None, 0.0, sft.step_lrs[member.slot])
+ _send_status(event_queue, f'Training {U} jobs together on "{model_name}"')
+ sft.train()
+ if not sft.stopped:
+ event_queue.put(
+ {
+ "type": "complete",
+ "output_dir": None,
+ "status_message": "Training completed",
+ "ts": time.time(),
+ }
+ )
+
+
+def _build_sft_trainer(
+ model, tokenizer, config, live, grad_accum, device, event_queue, export, take_messages, stop_all
+):
+ """TRL's SFTTrainer over the batch: each optimizer step gives every running member its own next
+ window, losses and gradient clipping stay per adapter, and each member keeps its own learning rate
+ schedule (one parameter group each)."""
+ import torch
+ from datasets import Dataset
+ from torch.utils.data import DataLoader
+ from transformers import TrainerCallback
+ from trl import SFTConfig, SFTTrainer
+
+ from core.training.resume import session_eta_seconds
+ from core.training.training import apply_save_strategy
+ from unsloth.kernels.multi_lora import (
+ adapter_token_counts,
+ clip_grad_norm_per_adapter_,
+ drop_absent_adapter_grads_,
+ per_adapter_loss,
+ set_multi_lora_batch,
+ )
+
+ U = len(live)
+ total = max(m.total_steps for m in live if not m.done)
+ pad_id = (
+ tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id
+ )
+ pad_id = 0 if pad_id is None else int(pad_id)
+ bf16 = fp16 = False
+ if device.type == "cuda":
+ from unsloth import is_bfloat16_supported
+ bf16 = is_bfloat16_supported()
+ fp16 = not bf16
+ args = {
+ "output_dir": live[0].output_dir,
+ "per_device_train_batch_size": 1,
+ "gradient_accumulation_steps": grad_accum,
+ "max_steps": total,
+ "learning_rate": float(live[0].config.get("learning_rate", 2e-4)),
+ "max_grad_norm": float(config.get("max_grad_norm", 1.0) or 1.0),
+ "bf16": bf16,
+ "fp16": fp16,
+ "logging_strategy": "no",
+ "report_to": "none",
+ "disable_tqdm": True,
+ "seed": int(live[0].config.get("random_seed", 3407)),
+ "remove_unused_columns": False,
+ "gradient_checkpointing": False,
+ "dataset_kwargs": {"skip_prepare_dataset": True},
+ "dataloader_num_workers": 0,
+ }
+ apply_save_strategy(args, config.get("save_steps", 0))
+ micro_total = total * grad_accum
+
+ class _Steps(TrainerCallback):
+ def on_step_end(self, args, state, control, **kwargs):
+ sft.after_step(control)
+
+ class MultiLoraSFTTrainer(SFTTrainer):
+ def __init__(self, **kwargs):
+ # Trainer refuses a quantized base without a PEFT wrapper; the adapters here are injected.
+ loaded = getattr(model, "_hf_peft_config_loaded", None)
+ model._hf_peft_config_loaded = True
+ try:
+ super().__init__(**kwargs)
+ finally:
+ if loaded is None:
+ del model._hf_peft_config_loaded
+ else:
+ model._hf_peft_config_loaded = loaded
+ # compute_loss already normalizes each adapter by its own window's tokens.
+ self.model_accepts_loss_kwargs = True
+ # Every Trainer clips through this one call (4.x inline, 5.x via _clip_grad_norm): clip per member.
+ self.accelerator.clip_grad_norm_ = self._clip_members
+ self.stopped = False
+ self.round = []
+ self.round_losses = torch.zeros(U)
+ self.member_norms = torch.zeros(U)
+ self.step_lrs = [float(m.config.get("learning_rate", 2e-4)) for m in live]
+ self.start = time.time()
+
+ def get_train_dataloader(self):
+ return DataLoader(range(micro_total), batch_size = 1)
+
+ def create_optimizer(self, model = None):
+ if self.optimizer is None:
+ groups = [
+ {
+ "params": m.params,
+ "lr": float(m.config.get("learning_rate", 2e-4)),
+ "weight_decay": float(m.config.get("weight_decay", 0.001) or 0.0),
+ }
+ for m in live
+ ]
+ optim = config.get("optim", "adamw_8bit") or "adamw_8bit"
+ if optim == "adamw_8bit":
+ import bitsandbytes as bnb
+ self.optimizer = bnb.optim.AdamW8bit(groups)
+ elif optim in ("adamw_torch", "adamw_torch_fused"):
+ fused = optim == "adamw_torch_fused" and device.type == "cuda"
+ self.optimizer = torch.optim.AdamW(groups, fused = fused)
+ else:
+ raise ValueError(f"Batched training does not support the optimizer {optim!r}.")
+ return self.optimizer
+
+ def create_scheduler(
+ self,
+ num_training_steps,
+ optimizer = None,
+ ):
+ if self.lr_scheduler is None:
+ self.lr_scheduler = _PerMemberLR(
+ optimizer or self.optimizer,
+ [
+ _member_scheduler(
+ m.config, float(m.config.get("learning_rate", 2e-4)), m.total_steps
+ )
+ for m in live
+ ],
+ )
+ return self.lr_scheduler
+
+ def get_batch_samples(self, epoch_iterator, num_batches, device):
+ """One optimizer step: micro-batch k holds every running member's k-th micro-batch."""
+ self.round = [m for m in live if not m.done]
+ if not self.round:
+ return [], None
+ windows = {m.slot: next(m.windows)[1] for m in self.round}
+ samples = []
+ counts = None
+ for k in range(num_batches):
+ rows, slots = [], []
+ for member in self.round:
+ window = windows[member.slot]
+ if k < len(window):
+ picked = [member.rows[r] for r in window[k]]
+ member.num_tokens += sum(len(ids) for ids, _ in picked)
+ rows.extend(picked)
+ slots.extend([member.slot] * len(picked))
+ if not rows:
+ # Every member's window is shorter here (an epoch end); the step keeps its size.
+ samples.append({"multi_lora_empty": torch.zeros(1)})
+ continue
+ batch = _collate(rows, slots, pad_id)
+ found = adapter_token_counts(batch["labels"], batch["adapter_ids"], U).cpu()
+ counts = found if counts is None else counts + found
+ samples.append(batch)
+ self.round_losses = torch.zeros(U)
+ return samples, counts.to(device)
+
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ return_outputs = False,
+ num_items_in_batch = None,
+ ):
+ if "multi_lora_empty" in inputs:
+ return torch.zeros((), device = self.args.device, requires_grad = True)
+ ids = inputs["adapter_ids"]
+ set_multi_lora_batch(self.model, ids, seq_len = inputs["input_ids"].shape[1])
+ logits = model(
+ input_ids = inputs["input_ids"],
+ attention_mask = inputs["attention_mask"],
+ use_cache = False,
+ ).logits
+ loss, per = per_adapter_loss(
+ logits,
+ inputs["labels"],
+ ids,
+ U,
+ model = self.model,
+ num_items_in_batch = num_items_in_batch,
+ )
+ self.round_losses += per.detach().float().cpu()
+ return loss
+
+ def _clip_members(
+ self,
+ parameters,
+ max_norm,
+ norm_type = 2,
+ **kwargs,
+ ):
+ # The rate this step uses, as a lone job reports it (read before the scheduler moves).
+ self.step_lrs = [group["lr"] for group in self.optimizer.param_groups]
+ drop_absent_adapter_grads_(self.model)
+ self.accelerator.unscale_gradients()
+ self.member_norms = clip_grad_norm_per_adapter_(self.model, max_norm).float().cpu()
+ return self.member_norms.norm()
+
+ def _save_checkpoint(self, model, trial):
+ name = f"checkpoint-{self.state.global_step}"
+ for member in self.round:
+ export(member, os.path.join(member.output_dir, name))
+
+ def progress(self, member, loss, grad_norm, elapsed, lr) -> None:
+ _send(
+ event_queue,
+ "progress",
+ member,
+ step = member.step,
+ epoch = round(member.step / member.steps_per_epoch, 2),
+ loss = loss,
+ learning_rate = lr,
+ total_steps = member.total_steps,
+ elapsed_seconds = elapsed,
+ eta_seconds = session_eta_seconds(elapsed, member.step, 0, member.total_steps),
+ session_start_step = 0,
+ grad_norm = grad_norm,
+ num_tokens = member.num_tokens,
+ eval_loss = None,
+ status_message = "",
+ )
+
+ def after_step(self, control) -> None:
+ elapsed = time.time() - self.start
+ for member in self.round:
+ member.step += 1
+ self.progress(
+ member,
+ float(self.round_losses[member.slot]),
+ float(self.member_norms[member.slot]),
+ elapsed,
+ self.step_lrs[member.slot],
+ )
+ if member.step >= member.total_steps and not member.done:
+ export(member)
+ member.done = True
+ _send(
+ event_queue,
+ "complete",
+ member,
+ output_dir = member.output_dir,
+ status_message = f"Training completed! Model saved to {member.output_dir}",
+ )
+ save_all = take_messages()
+ if save_all is not None:
+ stop_all(save_all)
+ self.stopped = True
+ if self.stopped or all(m.done for m in live):
+ control.should_training_stop = True
+
+ sft = MultiLoraSFTTrainer(
+ model = model,
+ args = SFTConfig(**args),
+ train_dataset = Dataset.from_dict({"input_ids": [[pad_id]] * micro_total}),
+ processing_class = tokenizer,
+ data_collator = lambda rows: rows,
+ callbacks = [_Steps()],
+ )
+ return sft
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index c0659ddd232..b6e714e2b4c 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -655,6 +655,14 @@ def _apply_cache_pins(config: dict[str, Any]) -> None:
config["cache_pin_warnings"] = warnings
+def _finite_or_none(value: Any) -> Optional[float]:
+ try:
+ value = float(value)
+ except (TypeError, ValueError):
+ return None
+ return value if math.isfinite(value) else None
+
+
def _s3_dataset_name(s3_dataset: Any) -> Optional[str]:
if not isinstance(s3_dataset, dict):
return None
@@ -1275,6 +1283,8 @@ class TrainingBackend:
self._db_total_steps_set: bool = False
self._db_config: Optional[dict] = None
self._db_started_at: Optional[str] = None
+ # Multi-LoRA batch: run_id -> member state; the batch id itself never gets a DB row.
+ self._batch_members: dict[str, dict] = {}
# Xet -> HTTP model-load fallback state (config kept for the respawn).
self._last_full_config: Optional[dict] = None
@@ -1707,10 +1717,13 @@ class TrainingBackend:
before_spawn = None,
resume_source_run_id: Optional[str] = None,
start_request_id: Optional[str] = None,
+ batch_members: Optional[list] = None,
**kwargs,
) -> bool:
# Reserve first: compare-and-set stops two concurrent requests reaching the spawn.
validate_job_paths(kwargs, cached_resources = True)
+ for member in batch_members or ():
+ validate_job_paths(member, cached_resources = True)
with self._new_job_spawn_reservation(job_id) as spawn_reserved:
if not spawn_reserved:
logger.warning("Training subprocess already running")
@@ -1734,6 +1747,7 @@ class TrainingBackend:
resume_source_run_id = resume_source_run_id,
start_request_id = start_request_id,
spawn_already_reserved = True,
+ batch_members = batch_members,
**kwargs,
)
@@ -1745,6 +1759,7 @@ class TrainingBackend:
resume_source_run_id: Optional[str] = None,
start_request_id: Optional[str] = None,
spawn_already_reserved: bool = False,
+ batch_members: Optional[list] = None,
**kwargs,
) -> bool:
"""before_spawn runs after validation but before GPU auto-selection, so refused starts free
@@ -1773,6 +1788,19 @@ class TrainingBackend:
self._pump_running = False
config = _build_training_worker_config(kwargs)
+ if batch_members:
+ config["batch_id"] = job_id
+ config["batch_members"] = [
+ {
+ "run_id": member["run_id"],
+ "config": _sanitize_db_config(
+ _build_training_worker_config(
+ {k: v for k, v in member.items() if k != "run_id"}
+ )
+ ),
+ }
+ for member in batch_members
+ ]
hf_token = hf_token_arg(
config["hf_token"], allow_ambient_token = config.get("allow_ambient", True)
)
@@ -1792,9 +1820,14 @@ class TrainingBackend:
hf_token = hf_token,
training_type = config["training_type"],
load_in_4bit = config["load_in_4bit"],
- batch_size = config.get("batch_size", 4),
+ # A batch (its members include this first one) trains every member's rows and adapter at once.
+ batch_size = sum(int(m.get("batch_size") or 4) for m in batch_members)
+ if batch_members
+ else config.get("batch_size", 4),
max_seq_length = config.get("max_seq_length", 2048),
- lora_rank = config.get("lora_r", 16),
+ lora_rank = sum(int(m.get("lora_r") or 16) for m in batch_members)
+ if batch_members
+ else config.get("lora_r", 16),
target_modules = config.get("target_modules"),
gradient_checkpointing = config.get("gradient_checkpointing", "unsloth"),
optimizer = config.get("optim", DEFAULT_TRAINING_OPTIMIZER),
@@ -1991,8 +2024,12 @@ class TrainingBackend:
self._db_run_created = False
self._db_create_in_progress = False
self._db_total_steps_set = False
- self._db_config = _sanitize_db_config(config)
+ self._db_config = None if batch_members else _sanitize_db_config(config)
self._db_started_at = datetime.now(timezone.utc).isoformat()
+ self._batch_members = {
+ member["run_id"]: self._new_batch_member(member["config"], self._db_started_at)
+ for member in config.get("batch_members") or ()
+ }
# Start each job Xet-first; keep config so a stall can respawn over HTTP.
self._last_full_config = config
self._last_hf_cache_env = cache_env
@@ -2060,7 +2097,14 @@ class TrainingBackend:
if not expected_job_id or self.current_job_id != expected_job_id:
return False
run_id = self.current_job_id
- if not save and run_id:
+ batch = bool(self._batch_members)
+ if not save and run_id and batch:
+ self._cancel_batch_members()
+ with self._lock:
+ if self.current_job_id != run_id:
+ return False
+ self._should_stop = self._cancel_requested = True
+ elif not save and run_id:
persist_error: Optional[Exception] = None
for attempt in range(_DB_FINALIZE_RETRIES):
try:
@@ -2288,6 +2332,10 @@ class TrainingBackend:
elif status != "completed":
self._progress.status_message = "Training stopped."
# A completed run keeps its message; reaping a wedged worker must not relabel it.
+ if self._batch_members:
+ self._finalize_batch_members(status, error_message, clear_output_dir)
+ # Create the row if a start-time create failed; skipped while the pump is mid-create, whose create-
+ # then-finalize records the run instead.
self._ensure_db_run_created()
with self._provenance_lock:
with self._lock:
@@ -2669,6 +2717,10 @@ class TrainingBackend:
self._model_download_repo_id = None
self._event_queue = event_queue
self._stop_queue = stop_queue
+ # Member stops sent to the stalled worker's queue died with it; under the lock no new one is lost.
+ for run_id, member in self._batch_members.items():
+ if member["stop_requested"] and not member["terminal"]:
+ stop_queue.put({"type": "stop_member", "run_id": run_id, "save": True})
self._proc = new_proc
self._spawn_in_progress = False
self._pump_thread = new_pump
@@ -3088,6 +3140,9 @@ class TrainingBackend:
if etype == "stall":
self._handle_stall_event(event)
return
+ if event.get("member_run_id"):
+ self._handle_member_event(event)
+ return
with self._lock:
if etype == "progress":
@@ -3505,6 +3560,12 @@ class TrainingBackend:
"""Claims the finalize under the lock so the watchdog and pump cannot both finalize the same run."""
if account_is_retired():
return
+ if self._batch_members:
+ with self._lock:
+ if expected_job_id is not None and self.current_job_id != expected_job_id:
+ return
+ self._finalize_batch_members(status, error_message, clear_output_dir)
+ return
with self._provenance_lock:
with self._lock:
if expected_job_id is not None and self.current_job_id != expected_job_id:
@@ -3592,6 +3653,249 @@ class TrainingBackend:
self._metric_buffer[:0] = batch
logger.warning("Failed to flush metrics to DB", exc_info = True)
+ @staticmethod
+ def _new_batch_member(config: dict, started_at: str) -> dict:
+ return {
+ "config": config,
+ "started_at": started_at,
+ "output_dir": None,
+ "created": False,
+ "terminal": False,
+ "stop_requested": False,
+ "cancelled": False,
+ "total_steps": None,
+ "total_steps_set": False,
+ "step": 0,
+ "loss": None,
+ "elapsed": None,
+ "loss_history": [],
+ "metrics": [],
+ }
+
+ def batch_member_ids(self) -> list[str]:
+ with self._lock:
+ return [run_id for run_id, m in self._batch_members.items() if not m["terminal"]]
+
+ @job_control
+ def stop_member(self, run_id: str) -> bool:
+ with self._lock:
+ member = self._batch_members.get(run_id)
+ if member is None or member["terminal"] or self._stop_queue is None:
+ return False
+ try:
+ self._stop_queue.put({"type": "stop_member", "run_id": run_id, "save": True})
+ except (OSError, ValueError):
+ return False
+ member["stop_requested"] = True
+ return True
+
+ def _cancel_batch_members(self) -> None:
+ from storage.studio_db import mark_run_cancel_requested
+ with self._lock:
+ running = [
+ (run_id, m) for run_id, m in self._batch_members.items() if not m["terminal"]
+ ]
+ for _, member in running:
+ member["cancelled"] = True
+ for run_id, member in running:
+ if member["created"]:
+ mark_run_cancel_requested(run_id)
+
+ def _handle_member_event(self, event: dict) -> None:
+ etype = event.get("type")
+ run_id = event["member_run_id"]
+ with self._lock:
+ member = self._batch_members.get(run_id)
+ if member is None or member["terminal"]:
+ return
+ if etype == "output_dir":
+ member["output_dir"] = event.get("output_dir")
+ elif etype == "progress":
+ step = event.get("step", 0)
+ loss = _finite_or_none(event.get("loss"))
+ if step > 0 and loss is not None:
+ member["loss_history"].append(loss)
+ member["step"] = step
+ member["loss"] = loss
+ member["elapsed"] = event.get("elapsed_seconds", member["elapsed"])
+ member["total_steps"] = event.get("total_steps") or member["total_steps"]
+ member["metrics"].append(
+ {
+ "step": step,
+ "loss": loss,
+ "learning_rate": _finite_or_none(event.get("learning_rate")),
+ "grad_norm": _finite_or_none(event.get("grad_norm")),
+ "eval_loss": None,
+ "epoch": event.get("epoch"),
+ "num_tokens": event.get("num_tokens"),
+ "elapsed_seconds": event.get("elapsed_seconds"),
+ }
+ )
+ p = self._progress
+ p.is_training = True
+ p.step = step
+ p.epoch = event.get("epoch", p.epoch)
+ p.loss = loss
+ p.total_steps = event.get("total_steps", p.total_steps)
+ p.elapsed_seconds = event.get("elapsed_seconds", p.elapsed_seconds)
+ p.eta_seconds = event.get("eta_seconds", p.eta_seconds)
+ if event.get("status_message"):
+ p.status_message = event["status_message"]
+ if etype == "output_dir":
+ self._persist_member(run_id, output_dir_changed = True)
+ elif etype == "progress":
+ self._persist_member(run_id, flush = len(member["metrics"]) >= self.FLUSH_THRESHOLD)
+ elif etype == "complete":
+ status = "stopped" if member["stop_requested"] else "completed"
+ self._finish_member(run_id, status, output_dir = event.get("output_dir"))
+ elif etype == "stopped":
+ self._finish_member(
+ run_id,
+ "stopped",
+ output_dir = event.get("output_dir"),
+ clear_output_dir = not event.get("saved", True),
+ )
+ elif etype == "error":
+ logger.error("Training error for batch member %s: %s", run_id, event.get("error"))
+ self._finish_member(run_id, "error", error_message = event.get("error", "Unknown error"))
+
+ def _persist_member(
+ self,
+ run_id: str,
+ flush: bool = False,
+ output_dir_changed: bool = False,
+ ) -> None:
+ if account_is_retired():
+ return
+ from storage.studio_db import (
+ create_run,
+ insert_metrics_batch,
+ update_run_output_dir,
+ update_run_progress,
+ update_run_total_steps,
+ )
+
+ member = self._batch_members[run_id]
+ try:
+ if not member["created"]:
+ if not member["metrics"] and not member["terminal"]:
+ return
+ config = member["config"]
+ create_run(
+ id = run_id,
+ model_name = config["model_name"],
+ dataset_name = config.get("hf_dataset")
+ or next(iter(config.get("local_datasets") or []), None)
+ or "unknown",
+ config_json = _json.dumps(config),
+ started_at = member["started_at"],
+ total_steps = member["total_steps"],
+ output_dir = member["output_dir"],
+ cancel_requested = member["cancelled"],
+ )
+ member["created"] = True
+ member["total_steps_set"] = bool(member["total_steps"])
+ flush = True
+ elif output_dir_changed and member["output_dir"] and not member["cancelled"]:
+ update_run_output_dir(run_id, member["output_dir"])
+ if member["total_steps"] and not member["total_steps_set"]:
+ update_run_total_steps(run_id, member["total_steps"])
+ member["total_steps_set"] = True
+ if flush and member["metrics"]:
+ with self._lock:
+ batch = list(member["metrics"])
+ del member["metrics"][: len(batch)]
+ step, loss, elapsed = member["step"], member["loss"], member["elapsed"]
+ try:
+ insert_metrics_batch(run_id, batch)
+ update_run_progress(id = run_id, step = step, loss = loss, duration_seconds = elapsed)
+ except Exception:
+ with self._lock:
+ member["metrics"][:0] = batch
+ raise
+ except Exception:
+ logger.warning("Failed to persist batch member %s", run_id, exc_info = True)
+
+ def _finish_member(
+ self,
+ run_id: str,
+ status: str,
+ *,
+ output_dir: Optional[str] = None,
+ error_message: Optional[str] = None,
+ clear_output_dir: bool = False,
+ ) -> None:
+ with self._lock:
+ member = self._batch_members.get(run_id)
+ if member is None or member["terminal"]:
+ return
+ member["terminal"] = True
+ if output_dir:
+ member["output_dir"] = output_dir
+ clear_output_dir = clear_output_dir or member["cancelled"]
+ # Retried like the finalize below: a member that fails before its first step needs its row created.
+ for attempt in range(_DB_FINALIZE_RETRIES):
+ self._persist_member(run_id, flush = True)
+ if member["created"] or account_is_retired() or attempt + 1 == _DB_FINALIZE_RETRIES:
+ break
+ time.sleep(_DB_FINALIZE_RETRY_S)
+ if not member["created"] or account_is_retired():
+ return
+ from storage.studio_db import finish_run
+ from utils.downsample import downsample
+
+ # Retried like a lone run's finalize: a transient SQLite lock must not leave the row running.
+ for attempt in range(_DB_FINALIZE_RETRIES):
+ try:
+ finish_run(
+ id = run_id,
+ status = status,
+ ended_at = datetime.now(timezone.utc).isoformat(),
+ final_step = member["step"],
+ final_loss = member["loss"],
+ duration_seconds = member["elapsed"],
+ loss_sparkline = _json.dumps(downsample(member["loss_history"], 50)),
+ output_dir = None if clear_output_dir else member["output_dir"],
+ error_message = error_message,
+ clear_output_dir = clear_output_dir,
+ )
+ return
+ except Exception:
+ if attempt + 1 < _DB_FINALIZE_RETRIES:
+ time.sleep(_DB_FINALIZE_RETRY_S)
+ continue
+ logger.warning("Failed to finalize batch member %s", run_id, exc_info = True)
+
+ def _finalize_batch_members(
+ self,
+ status: str,
+ error_message: Optional[str],
+ clear_output_dir: bool = False,
+ ) -> None:
+ # Process-level ending: only members without their own terminal event are finalized here.
+ with self._lock:
+ running = [
+ (run_id, m) for run_id, m in self._batch_members.items() if not m["terminal"]
+ ]
+ for run_id, member in running:
+ saved = bool(member["output_dir"]) and os.path.isfile(
+ os.path.join(member["output_dir"], "adapter_config.json")
+ )
+ if status == "error":
+ self._finish_member(
+ run_id,
+ "error",
+ error_message = error_message or "Training process terminated unexpectedly",
+ )
+ elif clear_output_dir or member["cancelled"]:
+ self._finish_member(run_id, "stopped", clear_output_dir = True)
+ elif saved:
+ self._finish_member(run_id, status)
+ else:
+ self._finish_member(
+ run_id, "error", error_message = "Training ended without saving this job."
+ )
+
@staticmethod
def _read_queue(q: Any, timeout_sec: float) -> Optional[dict]:
try:
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index 64d6cad1eca..027ea83f941 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -4701,6 +4701,21 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
)
return
+ if config.get("batch_members"):
+ try:
+ from core.training.multi_lora_trainer import run_multi_lora_training
+ return run_multi_lora_training(event_queue, stop_queue, config)
+ except Exception as exc:
+ event_queue.put(
+ {
+ "type": "error",
+ "error": str(exc),
+ "stack": traceback.format_exc(limit = 20),
+ "ts": time.time(),
+ }
+ )
+ return
+
# Embedding models use a different pipeline (FastSentenceTransformer + SentenceTransformerTrainer +
# MultipleNegativesRankingLoss), so branch early.
if config.get("is_embedding", False):
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.py
index db854385283..511b3838ae4 100644
--- a/studio/backend/routes/openai_fine_tuning.py
+++ b/studio/backend/routes/openai_fine_tuning.py
@@ -15,7 +15,9 @@ training is idle. The queue lives in this process: queued jobs do not survive a
from __future__ import annotations
import asyncio
+import importlib.util
import json
+import os
import re
import secrets
import weakref
@@ -280,6 +282,9 @@ _STARTED: dict[str, tuple[dict[str, Any], int]] = {}
_STARTED_LIMIT = 16
# Cancelled here while the worker finishes its step: the run row reads "running" until it stops.
_CANCELLING: set[str] = set()
+# Multi-LoRA batches started here: member run id -> batch job id, batch job id -> member run ids.
+_BATCH_OF: dict[str, str] = {}
+_BATCHES: dict[str, list[str]] = {}
def _remember_start(run_id: str, config: dict[str, Any], created_at: int) -> None:
@@ -288,6 +293,14 @@ def _remember_start(run_id: str, config: dict[str, Any], created_at: int) -> Non
_STARTED.pop(next(iter(_STARTED)))
+def _remember_batch(batch_id: str, run_ids: list[str]) -> None:
+ _BATCHES[batch_id] = list(run_ids)
+ _BATCH_OF.update({run_id: batch_id for run_id in run_ids})
+ while len(_BATCHES) > _STARTED_LIMIT:
+ for run_id in _BATCHES.pop(next(iter(_BATCHES))):
+ _BATCH_OF.pop(run_id, None)
+
+
def _for_caller(job: dict[str, Any], via_api_key: bool) -> dict[str, Any]:
"""An API key never sees host paths: a local model becomes its handle, an error loses its paths."""
if host_paths_visible(via_api_key):
@@ -320,12 +333,16 @@ async def _job(run_id: str, current_subject: str, via_api_key: bool) -> dict[str
return _for_caller(await asyncio.to_thread(_job_from_run, run), via_api_key)
# A started job has no history row until its worker is up; the live status still knows it.
status = await _current_status(current_subject, via_api_key)
- if status is not None and _status_field(status, "job_id") == run_id:
+ if status is not None and _status_field(status, "job_id") == _BATCH_OF.get(run_id, run_id):
phase = _status_field(status, "phase")
failed = _status_field(status, "start_request_state") == "rejected" or phase == "error"
message = _status_field(status, "error") or _status_field(status, "message")
config, created_at = _STARTED.get(run_id, ({}, None))
state = _PHASE_STATUS.get(phase, "queued")
+ if run_id in _BATCH_OF and state in ("queued", "running"):
+ # A member's row appears with its first step; until then the batch's phase (read from its
+ # setup messages) would flip it between running and validating_files.
+ state = "validating_files"
if failed:
state = "failed"
elif run_id in _CANCELLING:
@@ -425,6 +442,26 @@ _TRANSIENT_CONFLICTS = (
"is replacing the latest sidecar",
)
_ALIAS_TABLE = "openai_fine_tuning_aliases"
+# Jobs that may differ only in these train as one multi-LoRA batch; every other training setting must match.
+_BATCH_MEMBER_FIELDS = frozenset(
+ {
+ "local_datasets",
+ "learning_rate",
+ "num_epochs",
+ "batch_size",
+ "lora_r",
+ "lora_alpha",
+ "random_seed",
+ "project_name",
+ "max_steps",
+ # Each member has its own optimizer group and scheduler, so these stay per job too.
+ "weight_decay",
+ "warmup_steps",
+ "warmup_ratio",
+ }
+)
+_BATCH_OPTIMIZERS = ("adamw_8bit", "adamw_torch")
+_BATCH_MAX_DEFAULT = 4
@dataclass
@@ -544,6 +581,129 @@ def _queued_job(entry: _Queued) -> dict[str, Any]:
return {**job, "id": entry.alias}
+def _batch_max() -> int:
+ try:
+ return max(1, int(os.environ.get("UNSLOTH_STUDIO_FT_BATCH_MAX", _BATCH_MAX_DEFAULT)))
+ except ValueError:
+ return _BATCH_MAX_DEFAULT
+
+
+_multi_lora: Optional[bool] = None
+
+
+def _multi_lora_available() -> bool:
+ """Whether the installed Unsloth ships the multi-LoRA kernels; looked up on disk, so torch is not imported."""
+ global _multi_lora
+ if _multi_lora is None:
+ try:
+ spec = importlib.util.find_spec("unsloth")
+ roots = list(spec.submodule_search_locations or []) if spec is not None else []
+ except (ImportError, ValueError):
+ roots = []
+ _multi_lora = any(
+ (Path(root) / "kernels" / "multi_lora.py").is_file()
+ or (Path(root) / "kernels" / "multi_lora" / "__init__.py").is_file()
+ for root in roots
+ )
+ return _multi_lora
+
+
+def _batchable(request: TrainingStartRequest) -> bool:
+ """Text LoRA/QLoRA on local files with nothing the batched trainer does not run; the rest trains alone."""
+ from core.training.eval_dataset import evaluation_enabled
+ from core.training.training import should_use_mlx_training_backend
+
+ return (
+ not should_use_mlx_training_backend()
+ and request.training_type == "LoRA/QLoRA"
+ and request.use_lora
+ and bool(request.local_datasets)
+ and not (request.hf_dataset or request.dataset_streaming or request.s3_config is not None)
+ and not (request.resume_from_checkpoint or request.start_request_id)
+ and not (request.local_eval_datasets or evaluation_enabled(request.eval_steps))
+ and not request.packing
+ and request.lora_dropout == 0
+ and not (request.use_dora or request.use_loftq)
+ and not (
+ request.enable_wandb or request.enable_tensorboard or request.early_stopping_patience
+ )
+ and request.lr_scheduler_type != "reduce_lr_on_plateau"
+ and not request.offload_layers
+ and not (
+ request.is_dataset_image
+ or request.is_dataset_audio
+ or request.is_embedding
+ or request.is_decision
+ or request.finetune_vision_layers
+ )
+ and not (request.drafter is not None and request.drafter.enabled)
+ and request.optim in _BATCH_OPTIMIZERS
+ and len(request.gpu_ids or []) <= 1
+ )
+
+
+def _batch_key(entry: _Queued) -> Optional[tuple]:
+ if not _batchable(entry.request):
+ return None
+ shared = entry.request.model_dump(mode = "json", exclude = set(_BATCH_MEMBER_FIELDS))
+ return (
+ entry.account.account_id,
+ entry.subject,
+ entry.via_api_key,
+ json.dumps(shared, sort_keys = True),
+ )
+
+
+_MOE_COUNT_KEYS = ("num_experts", "n_routed_experts", "num_local_experts")
+
+
+def _moe_config(config: dict) -> bool:
+ for cfg in (config, config.get("text_config") or {}):
+ count = next((cfg[k] for k in _MOE_COUNT_KEYS if cfg.get(k) is not None), None)
+ if count is not None:
+ return isinstance(count, int) and count > 0
+ return False
+
+
+async def _trains_alone(request: TrainingStartRequest) -> bool:
+ """Outside what the batched trainer runs: text-diffusion models (own denoising objective), saved LoRA
+ adapters (continued in place alone) and MoE models (fused experts train via target_parameters alone)."""
+ from utils.models.diffusion_lm import diffusion_lm_profile_for_model
+ from utils.models.model_config import _read_raw_config_json, get_base_model_from_lora_identifier
+
+ def probe() -> bool:
+ if diffusion_lm_profile_for_model(request.model_name, request.hf_token):
+ return True
+ if get_base_model_from_lora_identifier(request.model_name, request.hf_token) is not None:
+ return True
+ try:
+ config = _read_raw_config_json(request.model_name, request.hf_token)
+ except Exception:
+ return True
+ return config is None or _moe_config(config)
+
+ return await asyncio.to_thread(probe)
+
+
+def _batch_for(head: _Queued) -> list[_Queued]:
+ """The queue head plus the compatible jobs waiting behind it, in queue order, up to the cap."""
+ cap = _batch_max()
+ key = _batch_key(head) if cap > 1 and not head.cancel_requested else None
+ if key is None:
+ return [head]
+ group = [head]
+ for entry in _QUEUE:
+ if len(group) >= cap:
+ break
+ if entry is head or entry.status != "queued" or entry.cancel_requested:
+ continue
+ if _batch_key(entry) == key:
+ group.append(entry)
+ if len(group) > 1 and not _multi_lora_available():
+ return [head]
+ return group
+
+
def _finish(
entry: _Queued,
status: str,
@@ -568,6 +728,13 @@ async def _start_queued(entry: _Queued) -> bool:
return True
if _training_busy():
return False
+ group = _batch_for(entry)
+ if len(group) > 1 and await _trains_alone(entry.request):
+ group = [entry]
+ if len(group) > 1:
+ outcome = await _start_batch(group)
+ if outcome is not None:
+ return outcome
entry.status = "starting"
try:
started = await arun_as(
@@ -614,6 +781,57 @@ async def _start_queued(entry: _Queued) -> bool:
return True
+async def _start_batch(group: list[_Queued]) -> Optional[bool]:
+ """Start ``group`` as one multi-LoRA run, each job with its own run. None: start the head alone instead,
+ which reports any refusal exactly as an unbatched start does; False: training is busy."""
+ from core.training import get_training_backend
+ from routes import training as training_routes
+
+ head = group[0]
+ for member in group:
+ member.status = "starting"
+ try:
+ started = await arun_as(
+ head.account,
+ training_routes.start_training_batch(
+ [member.request for member in group],
+ current_subject = head.subject,
+ via_api_key = head.via_api_key,
+ ),
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.info("Fine-tuning batch did not start (%s); starting %s alone", exc, head.alias)
+ started = None
+ if started is None or any(r.status == "error" or not r.job_id for r in started):
+ for member in group:
+ member.status = "queued"
+ if member.cancel_requested and member is not head:
+ _finish(member, "cancelled")
+ if started is not None and (started[0].error in _BUSY_ERRORS or _training_busy()):
+ if head.cancel_requested:
+ _finish(head, "cancelled")
+ return True
+ return False
+ return None
+ run_ids = [r.job_id for r in started]
+ batch_id = getattr(get_training_backend(), "current_job_id", None)
+ if batch_id:
+ _remember_batch(batch_id, run_ids)
+ for member, run_id in zip(group, run_ids):
+ await arun_as(
+ member.account,
+ asyncio.to_thread(
+ _save_alias, member.alias, run_id, member.account.account_id, member.created_at
+ ),
+ )
+ _remember_start(run_id, member.request.model_dump(), member.created_at)
+ _finish(member, "started")
+ for member, run_id in zip(group, run_ids):
+ if member.cancel_requested:
+ await arun_as(member.account, _cancel_run(run_id, member.subject))
+ return True
+
+
async def _drain_once() -> None:
if not _QUEUE:
return
@@ -684,6 +902,34 @@ async def _stop_run(run_id: str, current_subject: str) -> Any:
return stopped
+async def _cancel_run(run_id: str, current_subject: str) -> Any:
+ """A job training in a batch beside others still running stops alone; otherwise the whole run stops."""
+ batch_id = _BATCH_OF.get(run_id)
+ if batch_id is None:
+ return await _stop_run(run_id, current_subject)
+ from core.training import get_training_backend
+ from core.training.account_jobs import require_job_owner
+ from models.responses import TrainingStopResponse
+
+ backend = get_training_backend()
+ require_job_owner(backend)
+ running = await asyncio.to_thread(backend.batch_member_ids)
+ if run_id in running and len(running) > 1:
+ if await asyncio.to_thread(backend.stop_member, run_id):
+ _CANCELLING.add(run_id)
+ return TrainingStopResponse(
+ status = "stopped",
+ message = "Stop requested. This job will stop at the next safe step.",
+ )
+ running = await asyncio.to_thread(backend.batch_member_ids)
+ if run_id not in running:
+ return await _stop_run(run_id, current_subject)
+ stopped = await _stop_run(batch_id, current_subject)
+ if stopped.status != "idle":
+ _CANCELLING.add(run_id)
+ return stopped
+
+
@router.post("/fine_tuning/jobs")
async def create_fine_tuning_job(
body: FineTuningJobCreate,
@@ -780,18 +1026,24 @@ async def list_fine_tuning_jobs(
pending = []
after_run = await _resolve(after) if after else None
runs = await asyncio.to_thread(_runs_after, after_run, limit + 1)
- if after_run and not runs:
- # The live job has no history row yet and is listed first, so the history follows it.
- status = await _current_status(current_subject, via_api_key)
- if status is not None and _status_field(status, "job_id") == after_run:
- runs = await asyncio.to_thread(_runs_after, None, limit + 1)
+ status = await _current_status(current_subject, via_api_key)
+ live_id = _status_field(status, "job_id") if status is not None else None
+ # Live jobs without a history row yet (a batch's members while it sets up) are listed first.
+ live = []
+ if live_id and _status_field(status, "phase") != "idle":
+ live = [
+ r
+ for r in reversed(_BATCHES.get(live_id, [live_id]))
+ if await asyncio.to_thread(get_run, r) is None
+ ]
+ if after_run in live:
+ # The history follows the live jobs, so a cursor on one continues with the rest of them.
+ live = live[live.index(after_run) + 1 :]
+ runs = await asyncio.to_thread(_runs_after, None, limit + 1)
+ elif after_run:
+ live = []
jobs = [_for_caller(await asyncio.to_thread(_job_from_run, run), via_api_key) for run in runs]
- if after is None:
- status = await _current_status(current_subject, via_api_key)
- live_id = _status_field(status, "job_id") if status is not None else None
- live_phase = _status_field(status, "phase") if status is not None else None
- if live_id and live_phase != "idle" and all(run["id"] != live_id for run in runs):
- jobs.insert(0, await _job(live_id, current_subject, via_api_key))
+ jobs = [await _job(r, current_subject, via_api_key) for r in live] + jobs
# A job that ended before it started sorts by its creation; once a newer job ran, it is only retrievable.
newest = max((job["created_at"] for job in jobs), default = 0)
jobs = [
@@ -1007,7 +1259,7 @@ async def cancel_fine_tuning_job(
job = await _job(run_id, current_subject, via_api_key)
if job["status"] in ("succeeded", "failed", "cancelled"):
raise _bad_request(f"Job {job['id']} has already {job['status']}.")
- stopped = await _stop_run(run_id, current_subject)
+ stopped = await _cancel_run(run_id, current_subject)
if stopped.status == "idle":
raise _bad_request(f"Job {job['id']} is not running.", 409)
return {**job, "status": "cancelled"}
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index d7737da72ba..2aef44a987d 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -16,6 +16,7 @@ from core.training.account_jobs import (
validate_job_paths,
)
import contextlib
+import contextvars
import json
import os
import re
@@ -1898,18 +1899,7 @@ def _background_video_generation_active() -> bool:
return False
[email protected]("/start", responses = _TRAINING_START_ERROR_RESPONSES)
-async def start_training(
- request: TrainingStartRequest,
- current_subject: str = Depends(get_current_subject),
- via_api_key: bool = Depends(authenticated_via_api_key),
-):
- """
- Start a training job.
-
- Initiates training in the background and returns immediately. Use /status
- to check progress.
- """
+async def _prepare_training_request(request: TrainingStartRequest, via_api_key: bool) -> None:
await asyncio.to_thread(_validate_decision_request, request, via_api_key)
if request.drafter is not None and request.drafter.enabled:
from core.training.training import should_use_mlx_training_backend
@@ -1929,6 +1919,382 @@ async def start_training(
if not directory or not Path(directory).is_absolute():
request.tensorboard_dir = str(tensorboard_root() / (directory or ""))
validate_job_paths(request.model_dump())
+
+
+@dataclass
+class _AdmittedTrainingRequest:
+ training_kwargs: dict
+ hf_token: Optional[str]
+ resume_run: Optional[dict]
+
+
+async def _admit_training_request(
+ request: TrainingStartRequest, current_subject: str, via_api_key: bool
+) -> _AdmittedTrainingRequest:
+ """Per-request validation and worker kwargs of a start; raises the start route's own errors."""
+ resume_output_dir: Optional[str] = None
+ resume_run: Optional[dict] = None
+ resume_actual_model_repo_id: Optional[str] = None
+ resume_model_load_mode: Optional[str] = None
+ resume_requires_exact_resources = False
+ resume_requires_exact_model = False
+ resume_requires_exact_dataset = False
+ if request.resume_from_checkpoint:
+ try:
+ resume_output_dir = await asyncio.to_thread(
+ normalize_resume_output_dir,
+ request.resume_from_checkpoint,
+ )
+ except ValueError as e:
+ # Deliberate user-facing validation message.
+ validation_message = str(e)
+ raise HTTPException(status_code = 400, detail = validation_message)
+
+ resume_run = await asyncio.to_thread(
+ get_resumable_run_by_output_dir,
+ resume_output_dir,
+ )
+ if not resume_run or not await asyncio.to_thread(can_resume_run, resume_run):
+ detail = "Resume checkpoint must belong to a stopped or errored run with complete saved trainer state."
+ # Only when the checkpoint itself is intact: can_resume_run refuses for several reasons, so asking
+ # unconditionally answers a provenance sentence when the checkpoint is what is missing.
+ # has_resume_state is the discriminator can_resume_run itself short-circuits on.
+ if resume_run and await asyncio.to_thread(
+ has_resume_state, resume_run.get("output_dir")
+ ):
+ from core.training.provenance import (
+ resource_provenance_resume_blocker,
+ )
+ blocker = await asyncio.to_thread(
+ resource_provenance_resume_blocker,
+ training_run_config(resume_run),
+ )
+ if blocker:
+ detail = blocker
+ raise HTTPException(status_code = 400, detail = detail)
+ resume_checkpoint = await asyncio.to_thread(
+ get_resume_checkpoint_path,
+ resume_output_dir,
+ )
+ if not resume_checkpoint:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Resume checkpoint must include saved trainer state.",
+ )
+ request.resume_from_checkpoint = resume_checkpoint
+ (
+ resume_actual_model_repo_id,
+ resume_requires_exact_model,
+ resume_requires_exact_dataset,
+ resume_requires_exact_resources,
+ resume_model_load_mode,
+ ) = await asyncio.to_thread(
+ _prepare_resume_resource_provenance,
+ request,
+ resume_run,
+ )
+
+ if request.local_datasets:
+ request.local_datasets = _validate_local_dataset_paths(
+ request.local_datasets, "Local dataset"
+ )
+ # Not `eval_steps > 0`: inf passes that but reads as disabled everywhere in the trainer.
+ if request.local_eval_datasets and evaluation_enabled(request.eval_steps):
+ request.local_eval_datasets = _validate_local_dataset_paths(
+ request.local_eval_datasets, "Local eval dataset"
+ )
+
+ from utils.hardware import hardware as _hw
+ from utils.hardware import ensure_hardware_detected
+
+ await asyncio.to_thread(ensure_hardware_detected)
+ _validate_training_platform(request)
+
+ if request.dataset_streaming:
+ if not request.hf_dataset:
+ raise HTTPException(
+ status_code = 400,
+ detail = "dataset_streaming requires hf_dataset; streaming is not supported for local datasets.",
+ )
+ if request.is_dataset_image or request.is_dataset_audio:
+ raise HTTPException(
+ status_code = 400,
+ detail = "dataset_streaming is not supported for vision or audio datasets.",
+ )
+ if request.is_embedding:
+ raise HTTPException(
+ status_code = 400,
+ detail = "dataset_streaming is not supported for embedding training; the embedding loader needs the full dataset.",
+ )
+ if _hw.DEVICE == _hw.DeviceType.MLX:
+ raise HTTPException(
+ status_code = 400,
+ detail = "dataset_streaming is not yet supported on Apple Silicon (MLX); the MLX loader materializes the full dataset.",
+ )
+ if request.max_steps is None or request.max_steps <= 0:
+ raise HTTPException(
+ status_code = 422,
+ detail = "dataset_streaming requires max_steps > 0 because streaming datasets have no known length.",
+ )
+ if request.train_on_completions:
+ raise HTTPException(
+ status_code = 422,
+ detail = "dataset_streaming is not supported with train_on_completions yet.",
+ )
+ if evaluation_enabled(request.eval_steps):
+ train_split = request.train_split or "train"
+ if not request.eval_split or request.eval_split == train_split:
+ raise HTTPException(
+ status_code = 422,
+ detail = "dataset_streaming with evaluation requires a separate eval_split.",
+ )
+ # Streaming is HF-only: reject when the request also carries a local dataset path or an
+ # S3 config, since those sources cannot be streamed via HF's loader.
+ if request.local_datasets:
+ raise HTTPException(
+ status_code = 400,
+ detail = (
+ "dataset_streaming is HF-only; remove local_datasets / S3 source. "
+ "Streaming is not supported with local file paths."
+ ),
+ )
+ if request.s3_config is not None:
+ raise HTTPException(
+ status_code = 400,
+ detail = (
+ "dataset_streaming is HF-only; remove local_datasets / S3 source. "
+ "Streaming is not supported with S3 datasets."
+ ),
+ )
+ if request.dataset_known_cached or request.dataset_local_path:
+ raise HTTPException(
+ status_code = 422,
+ detail = (
+ "dataset_streaming streams from the Hub and cannot use the local "
+ "dataset cache; disable streaming to train from the cached copy."
+ ),
+ )
+ allow_ambient = via_api_key is not True
+ hf_token = hf_token_arg(request.hf_token, allow_ambient_token = allow_ambient)
+ # The local dataset paths were resolved above for existence only; authorize them now that
+ # the caller's token is known, before anything reads them.
+ # Eval paths only while evaluation is on, matching the validation above and the trainer,
+ # which reads them under the same condition: a stale eval path nothing will open must not
+ # refuse an otherwise valid run.
+ for _local_paths, _label in (
+ (request.local_datasets, "dataset"),
+ (
+ request.local_eval_datasets if evaluation_enabled(request.eval_steps) else [],
+ "eval_dataset",
+ ),
+ ):
+ if _local_paths:
+ await asyncio.to_thread(
+ _refuse_unauthorized_cached_local_paths, _local_paths, hf_token, _label
+ )
+ model_preflight = await asyncio.to_thread(
+ _reject_untrainable_model_request,
+ request,
+ resume_actual_model_repo_id,
+ hf_token,
+ )
+ cached_model_pin = model_preflight.cached_model_pin
+ training_actual_model_repo_id = resume_actual_model_repo_id
+ training_model_snapshot_path = request.model_snapshot_path
+ if cached_model_pin is not None:
+ training_actual_model_repo_id, training_model_snapshot_path = cached_model_pin
+
+ if request.hf_dataset:
+ await asyncio.to_thread(_preflight_hf_dataset_request, request)
+ # After the preflight: a cache it pinned is still on disk for this scan to refuse.
+ # Not for a streaming start: it reads the Hub (load_dataset(streaming = True)) and never
+ # the materialized cache, the preflight above always verified the repo remotely, and
+ # the route already refuses streaming with a pinned cache. Asking anyway would refuse a
+ # valid start over an unrelated cached copy whenever /auth-check cannot be reached.
+ # And only when the Hub repo is the EFFECTIVE source: load_and_format_dataset takes
+ # local_datasets (and s3_config below it) ahead of dataset_source, so a payload carrying
+ # both never reads the Hub cache, and refusing over it fails a run on a dataset it does
+ # not use. The local paths took their own authorization pass above.
+ if not request.dataset_streaming and _hf_dataset_is_the_source(request):
+ await asyncio.to_thread(_refuse_unauthorized_cached_dataset, request, hf_token)
+
+ device_backend = getattr(_hw.DEVICE, "value", "") or ""
+ training_optimizer = normalize_training_optimizer_for_device(
+ request.optim,
+ device_backend = device_backend,
+ )
+
+ training_kwargs = {
+ "model_name": model_preflight.model_name,
+ "project_name": request.project_name,
+ "training_type": request.training_type,
+ "hf_token": (request.hf_token or "").strip(),
+ "allow_ambient": allow_ambient,
+ "load_in_4bit": request.load_in_4bit,
+ "max_seq_length": request.max_seq_length,
+ "vision_image_size": request.vision_image_size,
+ "hf_dataset": request.hf_dataset or "",
+ "model_known_cached": request.model_known_cached,
+ "model_local_path": model_preflight.model_local_path,
+ "model_format": request.model_format,
+ "model_snapshot_path": training_model_snapshot_path,
+ "actual_model_repo_id": training_actual_model_repo_id,
+ "resume_model_load_mode": resume_model_load_mode,
+ "dataset_known_cached": request.dataset_known_cached,
+ "dataset_local_path": request.dataset_local_path,
+ "dataset_snapshot_path": request.dataset_snapshot_path,
+ "local_datasets": request.local_datasets,
+ "local_eval_datasets": request.local_eval_datasets,
+ "format_type": request.format_type,
+ "subset": request.subset,
+ "train_split": request.train_split,
+ "dataset_streaming": request.dataset_streaming,
+ "eval_split": request.eval_split,
+ "eval_steps": request.eval_steps,
+ "early_stopping_patience": request.early_stopping_patience,
+ "drafter": (
+ request.drafter.model_dump()
+ if request.drafter is not None and request.drafter.enabled
+ else None
+ ),
+ "dataset_slice_start": request.dataset_slice_start,
+ "dataset_slice_end": request.dataset_slice_end,
+ "custom_format_mapping": request.custom_format_mapping,
+ "num_epochs": request.num_epochs,
+ "learning_rate": request.learning_rate,
+ "embedding_learning_rate": request.embedding_learning_rate,
+ "batch_size": request.batch_size,
+ "gradient_accumulation_steps": request.gradient_accumulation_steps,
+ "warmup_steps": request.warmup_steps,
+ "warmup_ratio": request.warmup_ratio,
+ "max_steps": request.max_steps,
+ "save_steps": request.save_steps,
+ "weight_decay": request.weight_decay,
+ "max_grad_norm": request.max_grad_norm,
+ "max_grad_value": request.max_grad_value,
+ "max_grad_leaf_norm": request.max_grad_leaf_norm,
+ "cast_norm_output_to_input_dtype": request.cast_norm_output_to_input_dtype,
+ "random_seed": request.random_seed,
+ "packing": request.packing,
+ "optim": training_optimizer,
+ "lr_scheduler_type": request.lr_scheduler_type,
+ "use_lora": request.use_lora,
+ "lora_r": request.lora_r,
+ "lora_alpha": request.lora_alpha,
+ "lora_dropout": request.lora_dropout,
+ "target_modules": request.target_modules if request.target_modules else None,
+ "gradient_checkpointing": request.gradient_checkpointing.strip()
+ if request.gradient_checkpointing and request.gradient_checkpointing.strip()
+ else "unsloth",
+ "offload_layers": request.offload_layers,
+ "offload_vram_gb": request.offload_vram_gb,
+ "offload_vram_gb_per_device": request.offload_vram_gb_per_device,
+ "prefetch_depth": request.prefetch_depth,
+ "use_rslora": request.use_rslora,
+ "use_loftq": request.use_loftq,
+ "use_dora": request.use_dora,
+ "train_on_completions": request.train_on_completions,
+ "finetune_vision_layers": request.finetune_vision_layers,
+ "finetune_language_layers": request.finetune_language_layers,
+ "finetune_attention_modules": request.finetune_attention_modules,
+ "finetune_mlp_modules": request.finetune_mlp_modules,
+ "is_dataset_image": request.is_dataset_image,
+ "is_dataset_audio": request.is_dataset_audio,
+ "is_embedding": request.is_embedding,
+ "is_decision": request.is_decision,
+ "model_subfolder": request.model_subfolder,
+ "decision_layout": request.decision_layout,
+ "enable_wandb": request.enable_wandb,
+ "wandb_token": request.wandb_token or "",
+ "wandb_project": request.wandb_project or "",
+ "enable_tensorboard": request.enable_tensorboard,
+ "tensorboard_dir": request.tensorboard_dir or "",
+ "output_dir": resume_output_dir,
+ "resume_from_checkpoint": request.resume_from_checkpoint,
+ "require_exact_resume_resources": resume_requires_exact_resources,
+ "require_exact_model_resource": resume_requires_exact_model,
+ "require_exact_dataset_resource": resume_requires_exact_dataset,
+ "require_validated_model_snapshot": cached_model_pin is not None,
+ "adapter_base_pin": getattr(model_preflight, "adapter_base_pin", None),
+ # A managed account never forwards remote code to the worker.
+ "trust_remote_code": request.trust_remote_code and not managed_remote_code_refused(),
+ "approved_remote_code_fingerprint": request.approved_remote_code_fingerprint,
+ "subject": current_subject,
+ "gpu_ids": request.gpu_ids,
+ "s3_config": request.s3_config.model_dump() if request.s3_config else None,
+ }
+
+ # Latest-sidecar models size and train 16-bit (same flip as chat load): 4-bit is disabled for brand-new
+ # architectures, so VRAM checks must not underestimate a load the worker refuses.
+ from core.training.provenance import (
+ ExactResumeResourcesUnavailable,
+ effective_training_load_in_4bit,
+ )
+
+ if training_kwargs["load_in_4bit"]:
+ from core.training.training import resolve_training_model_load_target
+
+ try:
+ model_load_target = await asyncio.to_thread(
+ resolve_training_model_load_target,
+ training_kwargs,
+ )
+ except ExactResumeResourcesUnavailable as exc:
+ raise HTTPException(status_code = 409, detail = str(exc))
+ try:
+ effective_load_in_4bit = await asyncio.to_thread(
+ effective_training_load_in_4bit,
+ training_kwargs,
+ model_load_target,
+ hf_token,
+ )
+ except ExactResumeResourcesUnavailable as exc:
+ raise HTTPException(status_code = 409, detail = str(exc))
+ if not effective_load_in_4bit:
+ training_kwargs["load_in_4bit"] = False
+ logger.info(
+ "Latest-transformers sidecar active for %s - sizing and "
+ "training in 16-bit (4-bit is disabled for brand-new "
+ "architectures)",
+ model_load_target,
+ )
+
+ # Training page has no trust_remote_code toggle.
+ if not training_kwargs["trust_remote_code"]:
+ from utils.security.trusted_org import is_trusted_org_repo
+
+ model_defaults = load_model_defaults(request.model_name)
+ yaml_trust = model_defaults.get("training", {}).get("trust_remote_code", False)
+ if yaml_trust and is_trusted_org_repo(request.model_name, hf_token = hf_token):
+ logger.info(f"YAML config sets trust_remote_code=True for {request.model_name}")
+ training_kwargs["trust_remote_code"] = True
+ elif yaml_trust:
+ logger.warning(
+ "YAML sets trust_remote_code=True for %s but it is not a trusted "
+ "first-party repo; leaving disabled (user can opt in explicitly).",
+ request.model_name,
+ )
+
+ return _AdmittedTrainingRequest(
+ training_kwargs = training_kwargs,
+ hf_token = hf_token,
+ resume_run = resume_run,
+ )
+
+
[email protected]("/start", responses = _TRAINING_START_ERROR_RESPONSES)
+async def start_training(
+ request: TrainingStartRequest,
+ current_subject: str = Depends(get_current_subject),
+ via_api_key: bool = Depends(authenticated_via_api_key),
+):
+ """
+ Start a training job.
+
+ Initiates training in the background and returns immediately. Use /status
+ to check progress.
+ """
+ await _prepare_training_request(request, via_api_key)
backend = None
reserved_start_request_id = None
start_task: Optional[asyncio.Task[bool]] = None
@@ -2023,333 +2389,12 @@ async def start_training(
error = "Diffusion training already active",
)
- resume_output_dir: Optional[str] = None
- resume_run: Optional[dict] = None
- resume_actual_model_repo_id: Optional[str] = None
- resume_model_load_mode: Optional[str] = None
- resume_requires_exact_resources = False
- resume_requires_exact_model = False
- resume_requires_exact_dataset = False
- if request.resume_from_checkpoint:
- try:
- resume_output_dir = await asyncio.to_thread(
- normalize_resume_output_dir,
- request.resume_from_checkpoint,
- )
- except ValueError as e:
- # Deliberate user-facing validation message.
- validation_message = str(e)
- raise HTTPException(status_code = 400, detail = validation_message)
-
- resume_run = await asyncio.to_thread(
- get_resumable_run_by_output_dir,
- resume_output_dir,
- )
- if not resume_run or not await asyncio.to_thread(can_resume_run, resume_run):
- detail = "Resume checkpoint must belong to a stopped or errored run with complete saved trainer state."
- # Only when the checkpoint is intact, else the answer is a misleading provenance sentence.
- if resume_run and await asyncio.to_thread(
- has_resume_state, resume_run.get("output_dir")
- ):
- from core.training.provenance import (
- resource_provenance_resume_blocker,
- )
- blocker = await asyncio.to_thread(
- resource_provenance_resume_blocker,
- training_run_config(resume_run),
- )
- if blocker:
- detail = blocker
- raise HTTPException(status_code = 400, detail = detail)
- resume_checkpoint = await asyncio.to_thread(
- get_resume_checkpoint_path,
- resume_output_dir,
- )
- if not resume_checkpoint:
- raise HTTPException(
- status_code = 400,
- detail = "Resume checkpoint must include saved trainer state.",
- )
- request.resume_from_checkpoint = resume_checkpoint
- (
- resume_actual_model_repo_id,
- resume_requires_exact_model,
- resume_requires_exact_dataset,
- resume_requires_exact_resources,
- resume_model_load_mode,
- ) = await asyncio.to_thread(
- _prepare_resume_resource_provenance,
- request,
- resume_run,
- )
-
- if request.local_datasets:
- request.local_datasets = _validate_local_dataset_paths(
- request.local_datasets, "Local dataset"
- )
- # Not `eval_steps > 0`: inf passes that but reads as disabled everywhere in the trainer.
- if request.local_eval_datasets and evaluation_enabled(request.eval_steps):
- request.local_eval_datasets = _validate_local_dataset_paths(
- request.local_eval_datasets, "Local eval dataset"
- )
-
- from utils.hardware import hardware as _hw
- from utils.hardware import ensure_hardware_detected
-
- await asyncio.to_thread(ensure_hardware_detected)
- _validate_training_platform(request)
-
- if request.dataset_streaming:
- if not request.hf_dataset:
- raise HTTPException(
- status_code = 400,
- detail = "dataset_streaming requires hf_dataset; streaming is not supported for local datasets.",
- )
- if request.is_dataset_image or request.is_dataset_audio:
- raise HTTPException(
- status_code = 400,
- detail = "dataset_streaming is not supported for vision or audio datasets.",
- )
- if request.is_embedding:
- raise HTTPException(
- status_code = 400,
- detail = "dataset_streaming is not supported for embedding training; the embedding loader needs the full dataset.",
- )
- if _hw.DEVICE == _hw.DeviceType.MLX:
- raise HTTPException(
- status_code = 400,
- detail = "dataset_streaming is not yet supported on Apple Silicon (MLX); the MLX loader materializes the full dataset.",
- )
- if request.max_steps is None or request.max_steps <= 0:
- raise HTTPException(
- status_code = 422,
- detail = "dataset_streaming requires max_steps > 0 because streaming datasets have no known length.",
- )
- if request.train_on_completions:
- raise HTTPException(
- status_code = 422,
- detail = "dataset_streaming is not supported with train_on_completions yet.",
- )
- if evaluation_enabled(request.eval_steps):
- train_split = request.train_split or "train"
- if not request.eval_split or request.eval_split == train_split:
- raise HTTPException(
- status_code = 422,
- detail = "dataset_streaming with evaluation requires a separate eval_split.",
- )
- # Streaming is HF-only; a local dataset or S3 config in the same request cannot be streamed.
- if request.local_datasets:
- raise HTTPException(
- status_code = 400,
- detail = (
- "dataset_streaming is HF-only; remove local_datasets / S3 source. "
- "Streaming is not supported with local file paths."
- ),
- )
- if request.s3_config is not None:
- raise HTTPException(
- status_code = 400,
- detail = (
- "dataset_streaming is HF-only; remove local_datasets / S3 source. "
- "Streaming is not supported with S3 datasets."
- ),
- )
- if request.dataset_known_cached or request.dataset_local_path:
- raise HTTPException(
- status_code = 422,
- detail = (
- "dataset_streaming streams from the Hub and cannot use the local "
- "dataset cache; disable streaming to train from the cached copy."
- ),
- )
- allow_ambient = via_api_key is not True
- hf_token = hf_token_arg(request.hf_token, allow_ambient_token = allow_ambient)
- # Authorize local paths now the caller's token is known. Eval paths only while evaluation is on.
- for _local_paths, _label in (
- (request.local_datasets, "dataset"),
- (
- request.local_eval_datasets if evaluation_enabled(request.eval_steps) else [],
- "eval_dataset",
- ),
- ):
- if _local_paths:
- await asyncio.to_thread(
- _refuse_unauthorized_cached_local_paths, _local_paths, hf_token, _label
- )
- model_preflight = await asyncio.to_thread(
- _reject_untrainable_model_request,
- request,
- resume_actual_model_repo_id,
- hf_token,
- )
- cached_model_pin = model_preflight.cached_model_pin
- training_actual_model_repo_id = resume_actual_model_repo_id
- training_model_snapshot_path = request.model_snapshot_path
- if cached_model_pin is not None:
- training_actual_model_repo_id, training_model_snapshot_path = cached_model_pin
-
- if request.hf_dataset:
- await asyncio.to_thread(_preflight_hf_dataset_request, request)
- # Not for streaming (reads the Hub, never the cache), and only when the Hub repo is the effective
- # source: local_datasets / s3_config take precedence in load_and_format_dataset.
- if not request.dataset_streaming and _hf_dataset_is_the_source(request):
- await asyncio.to_thread(_refuse_unauthorized_cached_dataset, request, hf_token)
-
- device_backend = getattr(_hw.DEVICE, "value", "") or ""
- training_optimizer = normalize_training_optimizer_for_device(
- request.optim,
- device_backend = device_backend,
- )
-
- training_kwargs = {
- "model_name": model_preflight.model_name,
- "project_name": request.project_name,
- "training_type": request.training_type,
- "hf_token": (request.hf_token or "").strip(),
- "allow_ambient": allow_ambient,
- "load_in_4bit": request.load_in_4bit,
- "max_seq_length": request.max_seq_length,
- "vision_image_size": request.vision_image_size,
- "hf_dataset": request.hf_dataset or "",
- "model_known_cached": request.model_known_cached,
- "model_local_path": model_preflight.model_local_path,
- "model_format": request.model_format,
- "model_snapshot_path": training_model_snapshot_path,
- "actual_model_repo_id": training_actual_model_repo_id,
- "resume_model_load_mode": resume_model_load_mode,
- "dataset_known_cached": request.dataset_known_cached,
- "dataset_local_path": request.dataset_local_path,
- "dataset_snapshot_path": request.dataset_snapshot_path,
- "local_datasets": request.local_datasets,
- "local_eval_datasets": request.local_eval_datasets,
- "format_type": request.format_type,
- "subset": request.subset,
- "train_split": request.train_split,
- "dataset_streaming": request.dataset_streaming,
- "eval_split": request.eval_split,
- "eval_steps": request.eval_steps,
- "early_stopping_patience": request.early_stopping_patience,
- "drafter": (
- request.drafter.model_dump()
- if request.drafter is not None and request.drafter.enabled
- else None
- ),
- "dataset_slice_start": request.dataset_slice_start,
- "dataset_slice_end": request.dataset_slice_end,
- "custom_format_mapping": request.custom_format_mapping,
- "num_epochs": request.num_epochs,
- "learning_rate": request.learning_rate,
- "embedding_learning_rate": request.embedding_learning_rate,
- "batch_size": request.batch_size,
- "gradient_accumulation_steps": request.gradient_accumulation_steps,
- "warmup_steps": request.warmup_steps,
- "warmup_ratio": request.warmup_ratio,
- "max_steps": request.max_steps,
- "save_steps": request.save_steps,
- "weight_decay": request.weight_decay,
- "max_grad_norm": request.max_grad_norm,
- "max_grad_value": request.max_grad_value,
- "max_grad_leaf_norm": request.max_grad_leaf_norm,
- "cast_norm_output_to_input_dtype": request.cast_norm_output_to_input_dtype,
- "random_seed": request.random_seed,
- "packing": request.packing,
- "optim": training_optimizer,
- "lr_scheduler_type": request.lr_scheduler_type,
- "use_lora": request.use_lora,
- "lora_r": request.lora_r,
- "lora_alpha": request.lora_alpha,
- "lora_dropout": request.lora_dropout,
- "target_modules": request.target_modules if request.target_modules else None,
- "gradient_checkpointing": request.gradient_checkpointing.strip()
- if request.gradient_checkpointing and request.gradient_checkpointing.strip()
- else "unsloth",
- "offload_layers": request.offload_layers,
- "offload_vram_gb": request.offload_vram_gb,
- "offload_vram_gb_per_device": request.offload_vram_gb_per_device,
- "prefetch_depth": request.prefetch_depth,
- "use_rslora": request.use_rslora,
- "use_loftq": request.use_loftq,
- "use_dora": request.use_dora,
- "train_on_completions": request.train_on_completions,
- "finetune_vision_layers": request.finetune_vision_layers,
- "finetune_language_layers": request.finetune_language_layers,
- "finetune_attention_modules": request.finetune_attention_modules,
- "finetune_mlp_modules": request.finetune_mlp_modules,
- "is_dataset_image": request.is_dataset_image,
- "is_dataset_audio": request.is_dataset_audio,
- "is_embedding": request.is_embedding,
- "is_decision": request.is_decision,
- "model_subfolder": request.model_subfolder,
- "decision_layout": request.decision_layout,
- "enable_wandb": request.enable_wandb,
- "wandb_token": request.wandb_token or "",
- "wandb_project": request.wandb_project or "",
- "enable_tensorboard": request.enable_tensorboard,
- "tensorboard_dir": request.tensorboard_dir or "",
- "output_dir": resume_output_dir,
- "resume_from_checkpoint": request.resume_from_checkpoint,
- "require_exact_resume_resources": resume_requires_exact_resources,
- "require_exact_model_resource": resume_requires_exact_model,
- "require_exact_dataset_resource": resume_requires_exact_dataset,
- "require_validated_model_snapshot": cached_model_pin is not None,
- "adapter_base_pin": getattr(model_preflight, "adapter_base_pin", None),
- # A managed account never forwards remote code to the worker.
- "trust_remote_code": request.trust_remote_code and not managed_remote_code_refused(),
- "approved_remote_code_fingerprint": request.approved_remote_code_fingerprint,
- "subject": current_subject,
- "gpu_ids": request.gpu_ids,
- "s3_config": request.s3_config.model_dump() if request.s3_config else None,
- }
-
- # Latest-sidecar models train 16-bit (4-bit disabled for new archs); size VRAM accordingly.
- from core.training.provenance import (
- ExactResumeResourcesUnavailable,
- effective_training_load_in_4bit,
- )
-
- if training_kwargs["load_in_4bit"]:
- from core.training.training import resolve_training_model_load_target
-
- try:
- model_load_target = await asyncio.to_thread(
- resolve_training_model_load_target,
- training_kwargs,
- )
- except ExactResumeResourcesUnavailable as exc:
- raise HTTPException(status_code = 409, detail = str(exc))
- try:
- effective_load_in_4bit = await asyncio.to_thread(
- effective_training_load_in_4bit,
- training_kwargs,
- model_load_target,
- hf_token,
- )
- except ExactResumeResourcesUnavailable as exc:
- raise HTTPException(status_code = 409, detail = str(exc))
- if not effective_load_in_4bit:
- training_kwargs["load_in_4bit"] = False
- logger.info(
- "Latest-transformers sidecar active for %s - sizing and "
- "training in 16-bit (4-bit is disabled for brand-new "
- "architectures)",
- model_load_target,
- )
-
- # Training page has no trust_remote_code toggle.
- if not training_kwargs["trust_remote_code"]:
- from utils.security.trusted_org import is_trusted_org_repo
-
- model_defaults = load_model_defaults(request.model_name)
- yaml_trust = model_defaults.get("training", {}).get("trust_remote_code", False)
- if yaml_trust and is_trusted_org_repo(request.model_name, hf_token = hf_token):
- logger.info(f"YAML config sets trust_remote_code=True for {request.model_name}")
- training_kwargs["trust_remote_code"] = True
- elif yaml_trust:
- logger.warning(
- "YAML sets trust_remote_code=True for %s but it is not a trusted "
- "first-party repo; leaving disabled (user can opt in explicitly).",
- request.model_name,
- )
+ admitted = await _admit_training_request(request, current_subject, via_api_key)
+ training_kwargs = admitted.training_kwargs
+ hf_token = admitted.hf_token
+ resume_run = admitted.resume_run
+ # Set only by start_training_batch: this start becomes one multi-LoRA batch.
+ batch_members = _batch_members_for_start(training_kwargs)
# before_spawn hook: runs only after start_training's guards pass.
def _free_vram_for_training() -> None:
@@ -2419,14 +2464,15 @@ async def start_training(
)
def _can_keep_resident_models():
+ members = batch_members or [training_kwargs]
return can_keep_chat_during_training(
model_name = training_kwargs["model_name"],
hf_token = hf_token,
training_type = training_kwargs["training_type"],
load_in_4bit = training_kwargs["load_in_4bit"],
- batch_size = training_kwargs["batch_size"],
+ batch_size = sum(m["batch_size"] for m in members),
max_seq_length = training_kwargs["max_seq_length"],
- lora_rank = training_kwargs["lora_r"],
+ lora_rank = sum(m["lora_r"] for m in members),
target_modules = training_kwargs["target_modules"],
gradient_checkpointing = training_kwargs["gradient_checkpointing"],
optimizer = training_kwargs["optim"],
@@ -2443,6 +2489,7 @@ async def start_training(
# The hook runs only once start guards pass -> VRAM freed iff training starts.
from core.inference.gpu_arbiter import SegmenterStillRunningError
+ from core.training.provenance import ExactResumeResourcesUnavailable
from routes.training_vram import ManagedEngineStillRunning
from utils.transformers_version import SidecarSwapInProgress
@@ -2453,6 +2500,7 @@ async def start_training(
start_request_id = request.start_request_id,
before_spawn = _free_vram_for_training,
resume_source_run_id = resume_run["id"] if resume_run else None,
+ **({"batch_members": batch_members} if batch_members else {}),
**training_kwargs,
)
except (
@@ -2581,6 +2629,64 @@ async def start_training(
)
+# (first member's run id, the other members' entries) while start_training_batch drives start_training.
+_PENDING_BATCH: contextvars.ContextVar[Optional[tuple[str, list[dict]]]] = contextvars.ContextVar(
+ "studio_training_pending_batch", default = None
+)
+
+
+def _batch_members_for_start(training_kwargs: dict) -> Optional[list[dict]]:
+ pending = _PENDING_BATCH.get()
+ if pending is None:
+ return None
+ first_run_id, others = pending
+ return [{"run_id": first_run_id, **training_kwargs}, *others]
+
+
+async def start_training_batch(
+ requests: list[TrainingStartRequest], current_subject: str, via_api_key: bool
+) -> list[TrainingJobResponse]:
+ """Start compatible LoRA requests as one multi-LoRA run; internal, not a route.
+
+ Every member takes every per-request check; the shared ones (busy, inference conflict, install, VRAM, launch)
+ run once, through start_training for the first member, whose job id becomes the batch id. One response per
+ member, carrying that member's own run id.
+ """
+ if len(requests) < 2:
+ raise ValueError("A training batch needs at least two requests.")
+ if any(request.resume_from_checkpoint or request.start_request_id for request in requests):
+ raise ValueError("A training batch cannot resume or carry a start_request_id.")
+ run_ids = [
+ f"job_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{_uuid.uuid4().hex[:8]}" for _ in requests
+ ]
+ others = []
+ for run_id, request in zip(run_ids[1:], requests[1:]):
+ await _prepare_training_request(request, via_api_key)
+ try:
+ admitted = await _admit_training_request(request, current_subject, via_api_key)
+ except ValueError as e:
+ raise HTTPException(status_code = 400, detail = str(e))
+ others.append({"run_id": run_id, **admitted.training_kwargs})
+ token = _PENDING_BATCH.set((run_ids[0], others))
+ try:
+ started = await start_training(
+ requests[0], current_subject = current_subject, via_api_key = via_api_key
+ )
+ finally:
+ _PENDING_BATCH.reset(token)
+ if started.status == "error" or not started.job_id:
+ return [started.model_copy() for _ in requests]
+ return [
+ TrainingJobResponse(
+ job_id = run_id,
+ status = started.status,
+ message = started.message,
+ error = started.error,
+ )
+ for run_id in run_ids
+ ]
+
+
@router.post("/stop", response_model = TrainingStopResponse)
async def stop_training(
body: TrainingStopRequest, current_subject: str = Depends(get_current_subject)
diff --git a/studio/backend/tests/test_multi_lora_trainer.py b/studio/backend/tests/test_multi_lora_trainer.py
new file mode 100644
index 00000000000..91880562e0c
--- /dev/null
+++ b/studio/backend/tests/test_multi_lora_trainer.py
@@ -0,0 +1,530 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Batched multi-LoRA worker on a tiny random Qwen3 (CPU, PyTorch fallback of the kernels)."""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+import queue
+import types
+from pathlib import Path
+
+import sys
+
+import pytest
+
+# The repo root (pytest pythonpath) shadows PYTHONPATH.
+if "unsloth" not in sys.modules:
+ for _entry in os.environ.get("PYTHONPATH", "").split(os.pathsep)[::-1]:
+ if _entry and os.path.isfile(os.path.join(_entry, "unsloth", "kernels", "multi_lora.py")):
+ sys.path.insert(0, _entry)
+
+try:
+ import torch
+ from unsloth.kernels import multi_lora as ml
+except Exception as exc: # noqa: BLE001 - no multi-LoRA kernels or no importable unsloth here
+ pytest.skip(f"unsloth.kernels.multi_lora unavailable: {exc}", allow_module_level = True)
+
+from datasets import Dataset
+from safetensors.torch import load_file
+from tokenizers import Tokenizer, models, pre_tokenizers
+from transformers import PreTrainedTokenizerFast, Qwen3Config, Qwen3ForCausalLM
+
+from core.training import multi_lora_trainer as mlt
+
+WORDS = [f"w{i}" for i in range(56)]
+
+
+def _tokenizer():
+ vocab = {"[PAD]": 0, "[EOS]": 1, "[U]": 2, "[B]": 3}
+ vocab.update({w: i + 4 for i, w in enumerate(WORDS)})
+ tok = Tokenizer(models.WordLevel(vocab = vocab, unk_token = "[PAD]"))
+ tok.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
+ return PreTrainedTokenizerFast(tokenizer_object = tok, pad_token = "[PAD]", eos_token = "[EOS]")
+
+
+def _base():
+ torch.manual_seed(0)
+ config = Qwen3Config(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ max_position_embeddings = 128,
+ use_cache = False,
+ tie_word_embeddings = False,
+ )
+ return Qwen3ForCausalLM(config).float()
+
+
+def _texts(n, seed):
+ g = torch.Generator().manual_seed(seed)
+ out = []
+ for _ in range(n):
+ k = int(torch.randint(4, 12, (1,), generator = g))
+ out.append(
+ " ".join(WORDS[int(i)] for i in torch.randint(0, 56, (k,), generator = g)) + " [EOS]"
+ )
+ return out
+
+
+MEMBERS = [
+ dict(run_id = "job_a", rows = 6, batch_size = 2, num_epochs = 1, lr = 1e-2, r = 4, alpha = 8, seed = 11),
+ dict(run_id = "job_b", rows = 12, batch_size = 2, num_epochs = 2, lr = 5e-3, r = 8, alpha = 16, seed = 22),
+ dict(
+ run_id = "job_c",
+ rows = 9,
+ batch_size = 3,
+ num_epochs = 1,
+ lr = 2e-2,
+ r = 2,
+ alpha = 4,
+ seed = 33,
+ max_steps = 4,
+ ),
+]
+EXPECTED_STEPS = {"job_a": 2, "job_b": 6, "job_c": 4}
+
+
+def _member_config(m):
+ return {
+ "model_name": "tiny/qwen3",
+ "local_datasets": [m["run_id"]],
+ "format_type": "",
+ "learning_rate": m["lr"],
+ "num_epochs": m["num_epochs"],
+ "batch_size": m["batch_size"],
+ "lora_r": m["r"],
+ "lora_alpha": m["alpha"],
+ "random_seed": m["seed"],
+ "max_steps": m.get("max_steps", 0),
+ "optim": "adamw_torch",
+ "warmup_steps": m.get("warmup_steps", 0),
+ "weight_decay": m.get("weight_decay", 0.0),
+ "gradient_accumulation_steps": 2,
+ }
+
+
+class _Events:
+ def __init__(self, on_event = None):
+ self.items = []
+ self.on_event = on_event
+
+ def put(self, event):
+ self.items.append(event)
+ if self.on_event is not None:
+ self.on_event(event)
+
+
+def _run(
+ monkeypatch,
+ members,
+ on_event = None,
+ trainer_attrs = None,
+ datasets = None,
+ model_attrs = None,
+ **extra,
+):
+ base = _base()
+ for name, value in (model_attrs or {}).items():
+ setattr(base, name, value)
+ pristine = copy.deepcopy(base)
+ tokenizer = _tokenizer()
+ trainer = types.SimpleNamespace(
+ model = None,
+ tokenizer = None,
+ should_stop = False,
+ save_on_stop = True,
+ load_in_4bit = False,
+ is_cpt = False,
+ **(trainer_attrs or {}),
+ )
+ data = {
+ m["run_id"]: {
+ "dataset": Dataset.from_dict({"text": _texts(m["rows"], m["seed"])}),
+ "final_format": "",
+ }
+ for m in members
+ }
+ data.update(datasets or {})
+ inboxes = []
+
+ def load_base(trainer, config, hf_token, event_queue):
+ trainer.model, trainer.tokenizer = base, tokenizer
+ return False
+
+ def stop_reader(stop_queue, inbox, trainer):
+ inboxes.append(inbox)
+
+ monkeypatch.setattr(mlt, "_new_trainer", lambda: trainer)
+ monkeypatch.setattr(mlt, "_detect_model", lambda trainer, config, hf_token: None)
+ monkeypatch.setattr(mlt, "_format_dataset", lambda t, cfg, tok: data[cfg["local_datasets"][0]])
+ monkeypatch.setattr(mlt, "_load_base", load_base)
+ monkeypatch.setattr(mlt, "_start_stop_reader", stop_reader)
+ configs = [_member_config(m) for m in members]
+ config = {
+ **configs[0],
+ "max_seq_length": 64,
+ "gradient_checkpointing": "unsloth",
+ "target_modules": ["all-linear"],
+ "batch_members": [{"run_id": m["run_id"], "config": c} for m, c in zip(members, configs)],
+ **extra,
+ }
+ events = _Events(None if on_event is None else lambda e: on_event(e, inboxes[0]))
+ mlt.run_multi_lora_training(events, queue.Queue(), config)
+ return events.items, base, pristine
+
+
+def _member_events(events, run_id):
+ return [e for e in events if e.get("member_run_id") == run_id]
+
+
+def _adapter_logits(model, slot, input_ids):
+ ml.set_multi_lora_batch(
+ model, torch.full((input_ids.shape[0],), slot), seq_len = input_ids.shape[1]
+ )
+ with torch.no_grad():
+ return model(input_ids = input_ids, use_cache = False).logits
+
+
[email protected](scope = "module")
+def batched_run(tmp_path_factory):
+ monkeypatch = pytest.MonkeyPatch()
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path_factory.mktemp("studio_home")))
+ try:
+ yield _run(monkeypatch, MEMBERS)
+ finally:
+ monkeypatch.undo()
+
+
+def test_each_member_reports_and_exports_on_its_own(batched_run):
+ events, model, pristine = batched_run
+ assert [e["type"] for e in events if "member_run_id" not in e][-1] == "complete"
+ dirs = set()
+ for m in MEMBERS:
+ mine = _member_events(events, m["run_id"])
+ kinds = [e["type"] for e in mine]
+ assert kinds[0] == "output_dir" and kinds[-1] == "complete"
+ assert set(kinds[1:-1]) == {"progress"}
+ steps = [e["step"] for e in mine if e["type"] == "progress"]
+ assert steps == list(range(EXPECTED_STEPS[m["run_id"]] + 1))
+ tokens = [e["num_tokens"] for e in mine if e["type"] == "progress"]
+ assert tokens == sorted(tokens) and tokens[-1] > 0
+ assert all(
+ e["total_steps"] == EXPECTED_STEPS[m["run_id"]] for e in mine if e["type"] == "progress"
+ )
+ out = mine[0]["output_dir"]
+ assert mine[-1]["output_dir"] == out
+ dirs.add(out)
+ config = json.load(open(os.path.join(out, "adapter_config.json")))
+ assert config["r"] == m["r"] and config["lora_alpha"] == m["alpha"]
+ assert config["unsloth_training_method"] == "lora"
+ assert os.path.exists(os.path.join(out, "tokenizer.json"))
+ assert len(dirs) == len(MEMBERS)
+
+
+def test_each_step_reports_the_rate_it_trained_with(batched_run):
+ events, _model, _pristine = batched_run
+ for m in MEMBERS:
+ total = EXPECTED_STEPS[m["run_id"]]
+ reported = [
+ e["learning_rate"]
+ for e in _member_events(events, m["run_id"])
+ if e["type"] == "progress" and e["step"] > 0
+ ]
+ # Linear decay without warmup, read before each step: lr, then lr * (1 - 1/total), ...
+ assert reported == pytest.approx([m["lr"] * (1 - k / total) for k in range(total)])
+
+
+def test_exported_adapter_reloads_with_peft_and_matches(batched_run):
+ from peft import PeftModel
+
+ events, model, pristine = batched_run
+ model.eval()
+ input_ids = torch.randint(4, 60, (2, 10), generator = torch.Generator().manual_seed(5))
+ for slot, m in enumerate(MEMBERS):
+ out = _member_events(events, m["run_id"])[0]["output_dir"]
+ state = load_file(os.path.join(out, "adapter_model.safetensors"))
+ assert any(v.abs().sum() > 0 for k, v in state.items() if "lora_B" in k)
+ peft_model = PeftModel.from_pretrained(copy.deepcopy(pristine), out).eval()
+ with torch.no_grad():
+ expected = peft_model(input_ids = input_ids).logits
+ torch.testing.assert_close(
+ _adapter_logits(model, slot, input_ids), expected, atol = 1e-5, rtol = 1e-5
+ )
+
+
+def test_finished_member_gets_no_more_updates(batched_run):
+ events, model, pristine = batched_run
+ out = _member_events(events, "job_a")[0]["output_dir"]
+ exported = load_file(os.path.join(out, "adapter_model.safetensors"))
+ current = ml.adapter_state_dict(model, 0)
+ assert exported.keys() == current.keys()
+ for key in exported:
+ torch.testing.assert_close(current[key], exported[key], atol = 0, rtol = 0)
+
+
+def test_stop_member_stops_only_that_member(monkeypatch):
+ def on_event(event, inbox):
+ if (
+ event.get("member_run_id") == "job_b"
+ and event["type"] == "progress"
+ and event["step"] == 2
+ ):
+ inbox.put({"type": "stop_member", "run_id": "job_b", "save": True})
+
+ events, model, pristine = _run(monkeypatch, MEMBERS, on_event)
+ b = _member_events(events, "job_b")
+ assert b[-1]["type"] == "stopped" and b[-1]["saved"] is True
+ assert max(e["step"] for e in b if e["type"] == "progress") == 2
+ assert os.path.exists(os.path.join(b[-1]["output_dir"], "adapter_model.safetensors"))
+ for run_id in ("job_a", "job_c"):
+ mine = _member_events(events, run_id)
+ assert mine[-1]["type"] == "complete"
+ assert max(e["step"] for e in mine if e["type"] == "progress") == EXPECTED_STEPS[run_id]
+ assert events[-1]["type"] == "complete" and "member_run_id" not in events[-1]
+
+
+def test_seed_init_is_independent_of_batch_order():
+ first, second = _base(), _base()
+ ml.inject_multi_lora(first, ["q_proj", "down_proj"], 2, r = [4, 8], lora_alpha = [8, 16])
+ ml.inject_multi_lora(second, ["q_proj", "down_proj"], 2, r = [8, 4], lora_alpha = [16, 8])
+ mlt._seed_adapter(first, 0, 11)
+ mlt._seed_adapter(first, 1, 22)
+ mlt._seed_adapter(second, 0, 22)
+ mlt._seed_adapter(second, 1, 11)
+ a, b = ml.adapter_state_dict(first, 0), ml.adapter_state_dict(second, 1)
+ for key in a:
+ torch.testing.assert_close(a[key], b[key], atol = 0, rtol = 0)
+ assert all(v.abs().sum() == 0 for k, v in a.items() if "lora_B" in k)
+
+
+def test_completion_masking_masks_the_prompt():
+ tokenizer = _tokenizer()
+ tokenizer._unsloth_input_part = "[U]"
+ tokenizer._unsloth_output_part = "[B]"
+ info = {
+ "dataset": Dataset.from_dict({"text": ["[U] w1 w2 w3 [B] w4 w5 [EOS]"] * 3}),
+ "final_format": "",
+ }
+ tok, rows = mlt._tokenize_member(
+ info, tokenizer, {"train_on_completions": True}, "tiny/qwen3", 64
+ )
+ assert tok is tokenizer
+ ids, labels = rows[0]
+ response = ids.index(tokenizer.convert_tokens_to_ids("[B]")) + 1
+ assert all(t == -100 for t in labels[:response])
+ assert labels[response:] == ids[response:]
+
+
+def test_batch_trains_through_sft_trainer_and_saves_checkpoints(monkeypatch, tmp_path):
+ from trl import SFTTrainer
+
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ built = []
+ original = mlt._build_sft_trainer
+
+ def spy(*args, **kwargs):
+ built.append(original(*args, **kwargs))
+ return built[-1]
+
+ monkeypatch.setattr(mlt, "_build_sft_trainer", spy)
+ events, model, pristine = _run(monkeypatch, MEMBERS[:2], save_steps = 2)
+ assert isinstance(built[0], SFTTrainer) and built[0].state.global_step == 6
+ found = {}
+ for m in MEMBERS[:2]:
+ out = _member_events(events, m["run_id"])[0]["output_dir"]
+ found[m["run_id"]] = sorted(p.name for p in Path(out).glob("checkpoint-*"))
+ last = os.path.join(out, found[m["run_id"]][-1], "adapter_model.safetensors")
+ final = load_file(os.path.join(out, "adapter_model.safetensors"))
+ for key, value in load_file(last).items():
+ torch.testing.assert_close(value, final[key], atol = 0, rtol = 0)
+ assert found == {
+ "job_a": ["checkpoint-2"],
+ "job_b": ["checkpoint-2", "checkpoint-4", "checkpoint-6"],
+ }
+
+
+def test_gradients_are_clipped_per_member_every_step(monkeypatch, tmp_path):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ seen = []
+ real = ml.clip_grad_norm_per_adapter_
+
+ def spy(model, max_norm):
+ norms = real(model, max_norm)
+ seen.append((max_norm, norms.clone()))
+ return norms
+
+ monkeypatch.setattr(ml, "clip_grad_norm_per_adapter_", spy)
+ events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], max_grad_norm = 0.01)
+ assert len(seen) == EXPECTED_STEPS["job_b"]
+ assert all(max_norm == 0.01 for max_norm, _ in seen)
+ # Each member reports its own pre-clip norm, not one shared global norm.
+ a = [
+ e["grad_norm"]
+ for e in _member_events(events, "job_a")
+ if e["type"] == "progress" and e["step"]
+ ]
+ b = [
+ e["grad_norm"]
+ for e in _member_events(events, "job_b")
+ if e["type"] == "progress" and e["step"]
+ ]
+ assert a == pytest.approx([float(n[0]) for _, n in seen[:2]])
+ assert b == pytest.approx([float(n[1]) for _, n in seen])
+
+
+def test_each_member_keeps_its_own_weight_decay_and_warmup(monkeypatch, tmp_path):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ members = [
+ {**MEMBERS[0], "weight_decay": 0.1, "warmup_steps": 1},
+ {**MEMBERS[1], "weight_decay": 0.0, "warmup_steps": 0},
+ ]
+ built = []
+ original = mlt._build_sft_trainer
+ monkeypatch.setattr(
+ mlt, "_build_sft_trainer", lambda *a, **k: built.append(original(*a, **k)) or built[-1]
+ )
+ events, _model, _pristine = _run(monkeypatch, members)
+ assert [g["weight_decay"] for g in built[0].optimizer.param_groups] == [0.1, 0.0]
+ for m in members:
+ total = EXPECTED_STEPS[m["run_id"]]
+ reported = [
+ e["learning_rate"]
+ for e in _member_events(events, m["run_id"])
+ if e["type"] == "progress" and e["step"]
+ ]
+ warmup = m["warmup_steps"]
+ # transformers' linear schedule: ramp over the member's own warmup, then decay over its own length.
+ expected = [
+ m["lr"] * (k / warmup if k < warmup else (total - k) / (total - warmup))
+ for k in range(total)
+ ]
+ assert reported == pytest.approx(expected)
+
+
+def test_targets_follow_the_single_job_rules():
+ model = _base()
+ every_linear = mlt._resolve_targets(model, ["all-linear"])
+ assert (
+ "q_proj" in every_linear and "down_proj" in every_linear and "lm_head" not in every_linear
+ )
+ # Explicit names beside "all-linear" win, as UnslothTrainer.prepare_model_for_training does.
+ assert mlt._resolve_targets(model, ["all-linear", "q_proj"]) == ["q_proj"]
+ assert mlt._resolve_targets(model, None) == mlt._DEFAULT_TARGETS
+
+
+def test_a_text_diffusion_model_never_trains_batched(monkeypatch, tmp_path):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ events, _model, _pristine = _run(
+ monkeypatch, MEMBERS[:2], trainer_attrs = {"diffusion_lm_profile": "dream"}
+ )
+ assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]
+ assert "text-diffusion" in events[-1]["error"]
+
+
+def test_an_existing_lora_adapter_never_trains_batched(monkeypatch, tmp_path):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ events, _model, _pristine = _run(
+ monkeypatch, MEMBERS[:2], model_attrs = {"peft_config": {"default": object()}}
+ )
+ assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]
+ assert "existing LoRA adapter" in events[-1]["error"]
+
+
+def test_a_moe_model_never_trains_batched(monkeypatch, tmp_path):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ config = copy.deepcopy(_base().config)
+ config.num_experts = 8
+ events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], model_attrs = {"config": config})
+ assert [e["type"] for e in events if e.get("member_run_id")] == ["error", "error"]
+ assert "MoE" in events[-1]["error"]
+
+
+def test_pretokenized_assistant_masks_become_labels():
+ ids = [[5, 6, 7, 8], [9, 10, 11]]
+ info = {
+ "dataset": Dataset.from_dict(
+ {"input_ids": ids, "assistant_masks": [[0, 0, 1, 1], [0, 1, 1]]}
+ ),
+ "final_format": "pretokenized",
+ }
+ _tok, rows = mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)
+ assert [labels for _, labels in rows] == [[-100, -100, 7, 8], [-100, 10, 11]]
+
+
+def test_a_pretokenized_mask_of_the_wrong_length_is_refused():
+ info = {
+ "dataset": Dataset.from_dict({"input_ids": [[5, 6, 7, 8]], "assistant_masks": [[0, 1]]}),
+ "final_format": "pretokenized",
+ }
+ with pytest.raises(ValueError, match = "assistant_masks"):
+ mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)
+
+
+def test_a_pretokenized_token_outside_the_vocabulary_is_refused():
+ info = {
+ "dataset": Dataset.from_dict({"input_ids": [[5, 6, 900]]}),
+ "final_format": "pretokenized",
+ }
+ with pytest.raises(ValueError, match = "outside this model's vocabulary"):
+ mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64, vocab_size = 64)
+
+
+def test_one_members_bad_dataset_fails_only_that_member(monkeypatch, tmp_path):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ bad = {
+ "dataset": Dataset.from_dict({"input_ids": [[5, 6, 900]] * 4}),
+ "final_format": "pretokenized",
+ }
+ events, _model, _pristine = _run(monkeypatch, MEMBERS[:2], datasets = {"job_a": bad})
+ a, b = _member_events(events, "job_a"), _member_events(events, "job_b")
+ assert [e["type"] for e in a] == ["error"] and "vocabulary" in a[0]["error"]
+ assert b[-1]["type"] == "complete"
+ assert events[-1]["type"] == "complete" and "member_run_id" not in events[-1]
+
+
[email protected]("collator_masks_labels", [True, False])
+def test_pretokenized_labels_and_assistant_masks(monkeypatch, collator_masks_labels):
+ import utils.datasets.pretokenized as pretokenized
+
+ monkeypatch.setattr(pretokenized, "_collator_masks_user_labels", lambda: collator_masks_labels)
+ info = {
+ "dataset": Dataset.from_dict(
+ {
+ "input_ids": [[5, 6, 7, 8]],
+ "labels": [[5, 6, 7, 8]],
+ "assistant_masks": [[0, 0, 1, 1]],
+ }
+ ),
+ "final_format": "pretokenized",
+ }
+ _tok, rows = mlt._tokenize_member(info, _tokenizer(), {}, "tiny/qwen3", 64)
+ # The single-job collator masks user labels only before TRL 1.7.
+ assert rows[0][1] == ([-100, -100, 7, 8] if collator_masks_labels else [5, 6, 7, 8])
+
+
+def test_batched_member_trains_as_it_would_alone(batched_run, monkeypatch):
+ events, model, pristine = batched_run
+ batched = load_file(
+ os.path.join(_member_events(events, "job_b")[0]["output_dir"], "adapter_model.safetensors")
+ )
+ alone_events, _, _ = _run(monkeypatch, [MEMBERS[1]])
+ alone = load_file(
+ os.path.join(
+ _member_events(alone_events, "job_b")[0]["output_dir"], "adapter_model.safetensors"
+ )
+ )
+ batched_loss = [e["loss"] for e in _member_events(events, "job_b") if e["type"] == "progress"]
+ alone_loss = [
+ e["loss"] for e in _member_events(alone_events, "job_b") if e["type"] == "progress"
+ ]
+ assert batched_loss[1:] == pytest.approx(alone_loss[1:], rel = 1e-4)
+ for key in alone:
+ torch.testing.assert_close(batched[key], alone[key], atol = 1e-5, rtol = 1e-4)
diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.py
index e7351d4cb59..2a899f9ac19 100644
--- a/studio/backend/tests/test_openai_fine_tuning.py
+++ b/studio/backend/tests/test_openai_fine_tuning.py
@@ -19,6 +19,9 @@ from auth.authentication import authenticated_via_api_key, get_current_subject
from models.training import TrainingJobResponse # noqa: E402
from routes import openai_fine_tuning, training as training_routes # noqa: E402
from storage import studio_db # noqa: E402
+import core.training.training as _training_core # noqa: E402
+
+_REAL_TRAINS_ALONE = openai_fine_tuning._trains_alone
CHATS = (
b'{"messages": [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello!"}]}\n'
@@ -61,6 +64,8 @@ def studio(monkeypatch):
monkeypatch.setattr(openai_fine_tuning, "_STARTED", {})
monkeypatch.setattr(openai_fine_tuning, "_CANCELLING", set())
+ monkeypatch.setattr(openai_fine_tuning, "_BATCH_OF", {})
+ monkeypatch.setattr(openai_fine_tuning, "_BATCHES", {})
monkeypatch.setattr(training_routes, "start_training", start_training)
monkeypatch.setattr(training_routes, "stop_training", stop_training)
monkeypatch.setattr(training_routes, "get_training_status", get_training_status)
@@ -388,6 +393,8 @@ def queue(studio, monkeypatch):
monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})
monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())
monkeypatch.setattr(openai_fine_tuning, "_ensure_drainer", lambda: None)
+ # Without the multi-LoRA kernels: every queued job starts alone, as before batching existed.
+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora_available", lambda: False)
gpu = {"busy": True}
monkeypatch.setattr(openai_fine_tuning, "_training_busy", lambda: gpu["busy"])
return client, calls, live, gpu
@@ -828,3 +835,603 @@ def test_a_sidecar_swap_conflict_queues_and_a_crash_after_cancel_stays_cancelled
queued, after = _run(flow())
assert queued.status == "queued" and after.status == "cancelled"
+
+
+def _entry(
+ tmp_path,
+ name = "a",
+ subject = "unsloth",
+ **fields,
+):
+ from utils.account_context import current_account
+
+ dataset = tmp_path / f"{name}.jsonl"
+ dataset.write_bytes(CHATS)
+ config = {
+ "model_name": "unsloth/Qwen3-0.6B",
+ "training_type": "LoRA/QLoRA",
+ "format_type": "auto",
+ "local_datasets": [str(dataset)],
+ **fields,
+ }
+ return openai_fine_tuning._Queued(
+ alias = f"ftjob_q{name}",
+ account = current_account(),
+ subject = subject,
+ via_api_key = True,
+ request = openai_fine_tuning.TrainingStartRequest.model_validate(config),
+ created_at = 1_700_000_000,
+ )
+
+
+def batching(monkeypatch):
+ monkeypatch.setattr(openai_fine_tuning, "_QUEUE", [])
+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora_available", lambda: True)
+ monkeypatch.setattr(_training_core, "should_use_mlx_training_backend", lambda: False)
+ monkeypatch.delenv("UNSLOTH_STUDIO_FT_BATCH_MAX", raising = False)
+ return openai_fine_tuning._QUEUE
+
+
+def test_jobs_differing_only_in_member_fields_batch_in_queue_order(batching, tmp_path):
+ head = _entry(tmp_path, "a")
+ other = _entry(
+ tmp_path,
+ "b",
+ learning_rate = "1e-4",
+ num_epochs = 3,
+ batch_size = 4,
+ lora_r = 32,
+ lora_alpha = 64,
+ random_seed = 7,
+ project_name = "other",
+ max_steps = 30,
+ weight_decay = 0.05,
+ warmup_steps = 10,
+ )
+ odd = _entry(tmp_path, "c", max_seq_length = 4096)
+ last = _entry(tmp_path, "d", lora_r = 8, warmup_ratio = 0.1)
+ batching.extend([head, other, odd, last])
+ assert openai_fine_tuning._batch_for(head) == [head, other, last]
+
+
+ "fields",
+ [
+ {"resume_from_checkpoint": "/runs/old"},
+ {"eval_steps": 0.1},
+ {"packing": True},
+ {"lora_dropout": 0.05},
+ {"use_dora": True},
+ {"use_loftq": True},
+ {"enable_wandb": True},
+ {"enable_tensorboard": True},
+ {"lr_scheduler_type": "reduce_lr_on_plateau"},
+ {"offload_layers": "auto"},
+ {"offload_layers": 4},
+ {"training_type": "Full Finetuning"},
+ {"training_type": "Continued Pretraining"},
+ {"is_dataset_image": True},
+ {"is_embedding": True},
+ {"optim": "adamw_torch_fused"},
+ {"gpu_ids": [0, 1]},
+ {"start_request_id": "retry-1"},
+ ],
+)
+def test_an_excluded_setting_trains_alone(batching, tmp_path, fields):
+ head = _entry(tmp_path, "a", **fields)
+ other = _entry(tmp_path, "b", **fields)
+ batching.extend([head, other])
+ assert openai_fine_tuning._batch_for(head) == [head]
+ # A compatible head never pulls the excluded job in either.
+ plain = _entry(tmp_path, "c")
+ batching.insert(0, plain)
+ assert openai_fine_tuning._batch_for(plain) == [plain]
+
+
+ "fields",
+ [
+ {"gradient_accumulation_steps": 4},
+ {"load_in_4bit": False},
+ {"model_name": "unsloth/Qwen3-1.7B"},
+ {"target_modules": ["q_proj"]},
+ {"lr_scheduler_type": "cosine"},
+ {"optim": "adamw_torch"},
+ {"use_rslora": True},
+ ],
+)
+def test_shared_settings_must_match(batching, tmp_path, fields):
+ head = _entry(tmp_path, "a")
+ other = _entry(tmp_path, "b", **fields)
+ batching.extend([head, other])
+ assert openai_fine_tuning._batch_for(head) == [head]
+
+
+def test_another_subject_or_account_is_never_batched(batching, tmp_path):
+ import dataclasses
+
+ head = _entry(tmp_path, "a")
+ other_subject = _entry(tmp_path, "b", subject = "bob")
+ other_account = _entry(tmp_path, "c")
+ other_account.account = dataclasses.replace(head.account, account_id = "acct-bob")
+ batching.extend([head, other_subject, other_account])
+ assert openai_fine_tuning._batch_for(head) == [head]
+
+
+def test_the_cap_and_the_kill_switch(batching, tmp_path, monkeypatch):
+ entries = [_entry(tmp_path, name) for name in "abcdef"]
+ batching.extend(entries)
+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:4]
+ monkeypatch.setenv("UNSLOTH_STUDIO_FT_BATCH_MAX", "2")
+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:2]
+ monkeypatch.setenv("UNSLOTH_STUDIO_FT_BATCH_MAX", "1")
+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:1]
+ monkeypatch.setenv("UNSLOTH_STUDIO_FT_BATCH_MAX", "many")
+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:4]
+
+
+def test_without_the_kernels_every_job_trains_alone(batching, tmp_path, monkeypatch):
+ entries = [_entry(tmp_path, name) for name in "ab"]
+ batching.extend(entries)
+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora_available", lambda: False)
+ assert openai_fine_tuning._batch_for(entries[0]) == entries[:1]
+
+
+def test_the_kernel_lookup_reads_the_package_on_disk_once(tmp_path, monkeypatch):
+ import importlib.machinery
+
+ from routes.openai_fine_tuning import _multi_lora_available
+
+ package = tmp_path / "unsloth"
+ (package / "kernels").mkdir(parents = True)
+ lookups = []
+
+ def find_spec(name):
+ lookups.append(name)
+ spec = importlib.machinery.ModuleSpec(name, None, is_package = True)
+ spec.submodule_search_locations = [str(package)]
+ return spec
+
+ monkeypatch.setattr(openai_fine_tuning.importlib.util, "find_spec", find_spec)
+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora", None)
+ assert _multi_lora_available() is False
+ (package / "kernels" / "multi_lora.py").write_text("")
+ assert _multi_lora_available() is False # cached
+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora", None)
+ assert _multi_lora_available() is True
+ # Only the top-level package is looked up: finding a submodule would import unsloth (and torch).
+ assert lookups == ["unsloth", "unsloth"]
+
+
+class _BatchBackend:
+ def __init__(self):
+ self.current_job_id = ""
+ self.running: list[str] = []
+ self.stopped: list[str] = []
+
+ def batch_member_ids(self):
+ return list(self.running)
+
+ def stop_member(self, run_id):
+ if run_id not in self.running:
+ return False
+ self.stopped.append(run_id)
+ return True
+
+
+def batch_queue(queue, monkeypatch):
+ """The queue with the multi-LoRA kernels present and a stubbed start_training_batch."""
+ import core.training
+
+ client, calls, live, gpu = queue
+ monkeypatch.setattr(openai_fine_tuning, "_multi_lora_available", lambda: True)
+
+ async def batchable_model(request):
+ return False
+
+ monkeypatch.setattr(openai_fine_tuning, "_trains_alone", batchable_model)
+ monkeypatch.setattr(_training_core, "should_use_mlx_training_backend", lambda: False)
+ monkeypatch.delenv("UNSLOTH_STUDIO_FT_BATCH_MAX", raising = False)
+ backend = _BatchBackend()
+ monkeypatch.setattr(core.training, "get_training_backend", lambda: backend)
+ calls["batch"] = []
+
+ async def start_training_batch(requests, current_subject, via_api_key):
+ calls["batch"].append((list(requests), current_subject, via_api_key))
+ backend.current_job_id = "job_20261021_101500_batch000"
+ run_ids = [f"job_20261021_101500_member0{i}" for i in range(len(requests))]
+ backend.running = list(run_ids)
+ live["status"] = _Status(backend.current_job_id, "loading_model")
+ return [
+ TrainingJobResponse(job_id = run_id, status = "queued", message = "queued")
+ for run_id in run_ids
+ ]
+
+ monkeypatch.setattr(training_routes, "start_training_batch", start_training_batch)
+ return client, calls, live, gpu, backend
+
+
+def test_compatible_queued_jobs_start_as_one_batch(batch_queue):
+ client, calls, live, gpu, _backend = batch_queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [
+ await _create(client, uploaded.id, suffix = "a", seed = 1),
+ await _create(client, uploaded.id, suffix = "b", hyperparameters = {"n_epochs": 3}),
+ await _create(client, uploaded.id, suffix = "solo", extra_body = {"packing": True}),
+ await _create(client, uploaded.id, suffix = "c", extra_body = {"lora_r": 64}),
+ ]
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ retrieved = [await client.fine_tuning.jobs.retrieve(job.id) for job in jobs]
+ listed = await client.fine_tuning.jobs.list()
+ return jobs, retrieved, listed
+
+ jobs, retrieved, listed = _run(flow())
+ [(requests, subject, via_api_key)] = calls["batch"]
+ assert [r.project_name for r in requests] == ["a", "b", "c"]
+ assert (subject, via_api_key) == ("unsloth", True)
+ assert calls["start"] == []
+ batched = [retrieved[0], retrieved[1], retrieved[3]]
+ # Each job keeps the id it was given and its own settings, read from the batch's live status.
+ assert [j.id for j in batched] == [jobs[0].id, jobs[1].id, jobs[3].id]
+ assert [j.status for j in batched] == ["validating_files"] * 3
+ assert [j.suffix for j in batched] == ["a", "b", "c"]
+ assert batched[0].seed == 1 and batched[1].hyperparameters.n_epochs == 3
+ assert retrieved[2].status == "queued"
+ run_ids = [openai_fine_tuning._ALIAS_RUN[j.id][0] for j in batched]
+ assert run_ids == [f"job_20261021_101500_member0{i}" for i in range(3)]
+ # The batch itself is no job: the list shows its members and the job still waiting.
+ listed_ids = [j.id for j in listed.data]
+ assert sorted(listed_ids) == sorted(j.id for j in jobs)
+ assert live["status"].job_id not in {openai_fine_tuning._run_id(i) for i in listed_ids}
+
+
+def test_listing_pages_through_a_batch_that_is_still_loading(batch_queue):
+ client, _calls, _live, gpu, _backend = batch_queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "abc"]
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ seen, after = [], None
+ for _ in range(5):
+ page = await client.fine_tuning.jobs.list(
+ limit = 1, **({"after": after} if after else {})
+ )
+ seen += [j.id for j in page.data]
+ if not page.has_more:
+ break
+ after = page.data[-1].id
+ return jobs, seen
+
+ jobs, seen = _run(flow())
+ assert sorted(seen) == sorted(j.id for j in jobs) and len(seen) == 3
+
+
+ "model_kind", ["diffusion", "lora_adapter", "moe", "moe_text_config", "dense"]
+)
+def test_only_plain_dense_models_batch(batch_queue, monkeypatch, model_kind):
+ import utils.models.diffusion_lm as diffusion_lm
+ import utils.models.model_config as model_config
+
+ client, calls, _live, gpu, _backend = batch_queue
+ monkeypatch.setattr(openai_fine_tuning, "_trains_alone", _REAL_TRAINS_ALONE)
+ monkeypatch.setattr(
+ diffusion_lm,
+ "diffusion_lm_profile_for_model",
+ lambda *a, **k: "dream" if model_kind == "diffusion" else None,
+ )
+ monkeypatch.setattr(
+ model_config,
+ "get_base_model_from_lora_identifier",
+ lambda *a, **k: "unsloth/Qwen3-0.6B" if model_kind == "lora_adapter" else None,
+ )
+ configs = {
+ "moe": {"model_type": "qwen3_moe", "num_experts": 128},
+ "moe_text_config": {"text_config": {"num_local_experts": 8}},
+ }
+ monkeypatch.setattr(
+ model_config,
+ "_read_raw_config_json",
+ lambda *a, **k: configs.get(model_kind, {"model_type": "qwen3", "num_experts": None}),
+ )
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ for s in "ab":
+ await _create(client, uploaded.id, suffix = s)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+
+ _run(flow())
+ if model_kind == "dense":
+ assert len(calls["batch"]) == 1 and calls["start"] == []
+ else:
+ assert calls["batch"] == []
+ assert [r.project_name for r in calls["start"]] == ["a"]
+
+
+def test_mlx_jobs_never_batch(batch_queue, monkeypatch):
+ client, calls, _live, gpu, _backend = batch_queue
+ monkeypatch.setattr(_training_core, "should_use_mlx_training_backend", lambda: True)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ for s in "ab":
+ await _create(client, uploaded.id, suffix = s)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+
+ _run(flow())
+ assert calls["batch"] == []
+ assert [r.project_name for r in calls["start"]] == ["a"]
+
+
+def test_batch_members_stay_validating_until_their_first_step(batch_queue):
+ client, _calls, live, gpu, _backend = batch_queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "ab"]
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ seen = []
+ # The batch's setup messages move its phase through all of these before any member has a row.
+ for phase in ("loading_model", "training", "configuring", "loading_dataset", "training"):
+ live["status"].phase = phase
+ seen.append([(await client.fine_tuning.jobs.retrieve(j.id)).status for j in jobs])
+ live["status"].phase = "error"
+ failed = [(await client.fine_tuning.jobs.retrieve(j.id)).status for j in jobs]
+ return seen, failed
+
+ seen, failed = _run(flow())
+ assert seen == [["validating_files"] * 2] * 5
+ assert failed == ["failed"] * 2
+
+
+def test_a_single_or_incompatible_job_starts_exactly_as_before(batch_queue):
+ client, calls, _live, gpu, _backend = batch_queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ first = await _create(client, uploaded.id, suffix = "first", extra_body = {"packing": True})
+ second = await _create(client, uploaded.id, suffix = "second")
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ queued = list(openai_fine_tuning._QUEUE)
+ await openai_fine_tuning._drain_once()
+ return first, second, queued
+
+ first, second, queued = _run(flow())
+ assert calls["batch"] == []
+ assert [q.alias for q in queued] == [second.id]
+ assert [(r.project_name, r.packing) for r in calls["start"]] == [
+ ("first", True),
+ ("second", False),
+ ]
+ assert queued[0].request is calls["start"][1]
+
+
+def test_a_refused_batch_starts_the_head_alone(batch_queue, monkeypatch):
+ client, calls, _live, gpu, _backend = batch_queue
+
+ async def refuse(requests, current_subject, via_api_key):
+ calls["batch"].append(requests)
+ raise openai_fine_tuning.HTTPException(status_code = 400, detail = "Dataset not found")
+
+ monkeypatch.setattr(training_routes, "start_training_batch", refuse)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ first = await _create(client, uploaded.id, suffix = "first")
+ second = await _create(client, uploaded.id, suffix = "second")
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return (
+ first,
+ second,
+ [await client.fine_tuning.jobs.retrieve(j.id) for j in (first, second)],
+ )
+
+ first, second, (started, waiting) = _run(flow())
+ assert len(calls["batch"]) == 1
+ assert [r.project_name for r in calls["start"]] == ["first"]
+ assert started.id == first.id and started.status == "validating_files"
+ assert waiting.status == "queued"
+
+
+def test_a_busy_gpu_keeps_the_whole_batch_queued(batch_queue, monkeypatch):
+ client, calls, _live, gpu, _backend = batch_queue
+
+ async def busy(requests, current_subject, via_api_key):
+ calls["batch"].append(requests)
+ return [
+ TrainingJobResponse(
+ job_id = "",
+ status = "error",
+ message = "Training is already in progress.",
+ error = "Training already active",
+ )
+ for _ in requests
+ ]
+
+ monkeypatch.setattr(training_routes, "start_training_batch", busy)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = name) for name in ("a", "b")]
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return [await client.fine_tuning.jobs.retrieve(j.id) for j in jobs]
+
+ assert [j.status for j in _run(flow())] == ["queued", "queued"]
+ assert calls["start"] == [] and len(openai_fine_tuning._QUEUE) == 2
+
+
+def test_cancelling_a_batched_job_stops_only_that_adapter(batch_queue):
+ client, calls, _live, gpu, backend = batch_queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = name) for name in ("a", "b", "c")]
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ cancelled = await client.fine_tuning.jobs.cancel(jobs[1].id)
+ after = [await client.fine_tuning.jobs.retrieve(j.id) for j in jobs]
+ return jobs, cancelled, after
+
+ jobs, cancelled, after = _run(flow())
+ assert backend.stopped == ["job_20261021_101500_member01"]
+ assert calls["stop"] == []
+ assert cancelled.status == "cancelled"
+ assert [j.status for j in after] == ["validating_files", "cancelled", "validating_files"]
+
+
+def test_cancelling_the_last_running_batched_job_stops_the_run(batch_queue):
+ client, calls, live, gpu, backend = batch_queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = name) for name in ("a", "b")]
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ backend.running = ["job_20261021_101500_member01"] # the other adapter already finished
+ cancelled = await client.fine_tuning.jobs.cancel(jobs[1].id)
+ return cancelled, await client.fine_tuning.jobs.retrieve(jobs[1].id)
+
+ cancelled, retrieved = _run(flow())
+ assert backend.stopped == []
+ [stop] = calls["stop"]
+ # The whole run is the batch, so the stop names the batch's job.
+ assert stop.expected_job_id == live["status"].job_id == "job_20261021_101500_batch000"
+ assert cancelled.status == retrieved.status == "cancelled"
+
+
+def test_a_batched_job_cancelled_while_its_start_was_in_flight_stops_alone(
+ batch_queue, monkeypatch
+):
+ client, calls, _live, gpu, backend = batch_queue
+ start = training_routes.start_training_batch
+ release, entered = asyncio.Event(), asyncio.Event()
+
+ async def slow(requests, current_subject, via_api_key):
+ entered.set()
+ await release.wait()
+ return await start(requests, current_subject = current_subject, via_api_key = via_api_key)
+
+ monkeypatch.setattr(training_routes, "start_training_batch", slow)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = name) for name in ("a", "b")]
+ gpu["busy"] = False
+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())
+ await entered.wait()
+ await client.fine_tuning.jobs.cancel(jobs[0].id)
+ release.set()
+ await drain
+ return [await client.fine_tuning.jobs.retrieve(j.id) for j in jobs]
+
+ assert [j.status for j in _run(flow())] == ["cancelled", "validating_files"]
+ assert backend.stopped == ["job_20261021_101500_member00"] and calls["stop"] == []
+
+
+def test_start_training_batch_runs_one_start_carrying_every_member(monkeypatch):
+ seen = []
+
+ async def prepare(request, via_api_key):
+ seen.append(("prepare", request.project_name))
+
+ async def admit(request, current_subject, via_api_key):
+ seen.append(("admit", request.project_name))
+ return training_routes._AdmittedTrainingRequest(
+ training_kwargs = {"project_name": request.project_name, "lora_r": request.lora_r},
+ hf_token = None,
+ resume_run = None,
+ )
+
+ async def start_training(request, current_subject, via_api_key):
+ # Inside start_training, after its own admission of the first member.
+ members = training_routes._batch_members_for_start({"project_name": request.project_name})
+ seen.append(("start", request.project_name, members))
+ return TrainingJobResponse(job_id = "job_batch", status = "queued", message = "queued")
+
+ monkeypatch.setattr(training_routes, "_prepare_training_request", prepare)
+ monkeypatch.setattr(training_routes, "_admit_training_request", admit)
+ monkeypatch.setattr(training_routes, "start_training", start_training)
+ requests = [
+ openai_fine_tuning.TrainingStartRequest(
+ model_name = "unsloth/Qwen3-0.6B",
+ training_type = "LoRA/QLoRA",
+ format_type = "auto",
+ project_name = name,
+ lora_r = r,
+ )
+ for name, r in (("a", 8), ("b", 16), ("c", 32))
+ ]
+ responses = _run(
+ training_routes.start_training_batch(requests, current_subject = "unsloth", via_api_key = True)
+ )
+ assert seen[:4] == [("prepare", "b"), ("admit", "b"), ("prepare", "c"), ("admit", "c")]
+ [(_, first, members)] = seen[4:]
+ assert first == "a"
+ run_ids = [r.job_id for r in responses]
+ assert len(set(run_ids)) == 3 and "job_batch" not in run_ids
+ assert all(r.status == "queued" for r in responses)
+ assert members == [
+ {"run_id": run_ids[0], "project_name": "a"},
+ {"run_id": run_ids[1], "project_name": "b", "lora_r": 16},
+ {"run_id": run_ids[2], "project_name": "c", "lora_r": 32},
+ ]
+ assert training_routes._batch_members_for_start({"project_name": "a"}) is None
+
+
+def test_start_training_batch_refuses_resume_and_reports_a_busy_gpu_per_member(monkeypatch):
+ async def busy(request, current_subject, via_api_key):
+ return TrainingJobResponse(
+ job_id = "job_running",
+ status = "error",
+ message = "Training is already in progress.",
+ error = "Training already active",
+ )
+
+ async def admit(request, current_subject, via_api_key):
+ return training_routes._AdmittedTrainingRequest({}, None, None)
+
+ async def prepare(request, via_api_key):
+ return None
+
+ monkeypatch.setattr(training_routes, "_prepare_training_request", prepare)
+ monkeypatch.setattr(training_routes, "_admit_training_request", admit)
+ monkeypatch.setattr(training_routes, "start_training", busy)
+
+ def request(**fields):
+ return openai_fine_tuning.TrainingStartRequest(
+ model_name = "unsloth/Qwen3-0.6B",
+ training_type = "LoRA/QLoRA",
+ format_type = "auto",
+ **fields,
+ )
+
+ with pytest.raises(ValueError):
+ _run(
+ training_routes.start_training_batch(
+ [request(), request(resume_from_checkpoint = "/runs/old")],
+ current_subject = "unsloth",
+ via_api_key = True,
+ )
+ )
+ responses = _run(
+ training_routes.start_training_batch(
+ [request(), request()], current_subject = "unsloth", via_api_key = True
+ )
+ )
+ assert [(r.status, r.error, r.job_id) for r in responses] == [
+ ("error", "Training already active", "job_running")
+ ] * 2
diff --git a/studio/backend/tests/test_startup_defers_torch.py b/studio/backend/tests/test_startup_defers_torch.py
index 4872200e237..6802c5d2fc8 100644
--- a/studio/backend/tests/test_startup_defers_torch.py
+++ b/studio/backend/tests/test_startup_defers_torch.py
@@ -266,7 +266,7 @@ def test_a_failing_warm_stage_is_reported_not_swallowed(monkeypatch, capsys, cap
_OFFLOAD_REQUIRED = [
("main.py", "get_gpu_visibility", "get_backend_visible_gpu_info"),
("routes/training.py", "get_hardware_utilization", "get_gpu_utilization"),
- ("routes/training.py", "start_training", "ensure_hardware_detected"),
+ ("routes/training.py", "_admit_training_request", "ensure_hardware_detected"),
("routes/export.py", "_ensure_export_supported", "export_capability"),
("routes/models.py", "list_models", "get_inference_backend"),
# get_status only peeks now; test_async_singleton_access.py guards it.
diff --git a/studio/backend/tests/test_training_batch_backend.py b/studio/backend/tests/test_training_batch_backend.py
new file mode 100644
index 00000000000..ac5b1f8686e
--- /dev/null
+++ b/studio/backend/tests/test_training_batch_backend.py
@@ -0,0 +1,445 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A multi-LoRA batch: one training_runs row per member, the batch id itself never gets one."""
+
+from __future__ import annotations
+
+import json
+import queue
+from unittest.mock import patch
+
+import pytest
+
+import core.training.training as training_mod
+from core.training.training import TrainingBackend, TrainingProgress
+
+BATCH = "job_batch"
+MEMBERS = ("job_a", "job_b", "job_c")
+
+
+def db(monkeypatch, tmp_path):
+ from storage import studio_db
+
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ monkeypatch.setattr(studio_db, "_schema_ready", set())
+ return studio_db
+
+
+class _AliveProc:
+ pid = 4321
+
+ def is_alive(self):
+ return True
+
+
+def _member_config(run_id):
+ return training_mod._sanitize_db_config(
+ training_mod._build_training_worker_config(
+ {
+ "model_name": "unsloth/tiny",
+ "local_datasets": [f"/data/{run_id}.jsonl"],
+ "hf_token": "hf_secret",
+ "wandb_token": "wb_secret",
+ "learning_rate": "1e-4",
+ }
+ )
+ )
+
+
+def _batch_backend(tmp_path):
+ b = TrainingBackend()
+ b.current_job_id = BATCH
+ b._proc = _AliveProc()
+ b._stop_queue = queue.Queue()
+ b._start_stop_watchdog = lambda **kw: None
+ b._progress = TrainingProgress(is_training = True)
+ b._db_config = None
+ b._batch_members = {
+ run_id: b._new_batch_member(_member_config(run_id), "2026-10-26T00:00:00+00:00")
+ for run_id in MEMBERS
+ }
+ return b
+
+
+def _progress(run_id, step, loss):
+ return {
+ "type": "progress",
+ "member_run_id": run_id,
+ "step": step,
+ "loss": loss,
+ "learning_rate": 1e-4,
+ "total_steps": 4,
+ "epoch": 0.5,
+ "num_tokens": step * 10,
+ "elapsed_seconds": float(step),
+ }
+
+
+def _exported(tmp_path, run_id):
+ out = tmp_path / "out" / run_id
+ out.mkdir(parents = True, exist_ok = True)
+ (out / "adapter_config.json").write_text("{}")
+ return str(out)
+
+
+def test_member_events_get_their_own_rows_metrics_and_terminal_states(db, tmp_path):
+ b = _batch_backend(tmp_path)
+ outs = {r: _exported(tmp_path, r) for r in MEMBERS}
+ for run_id in MEMBERS:
+ b._handle_event({"type": "output_dir", "member_run_id": run_id, "output_dir": outs[run_id]})
+ for step in (1, 2):
+ for i, run_id in enumerate(MEMBERS):
+ b._handle_event(_progress(run_id, step, 1.0 + i + step / 10))
+ assert b.batch_member_ids() == list(MEMBERS)
+
+ b._handle_event({"type": "complete", "member_run_id": "job_a", "output_dir": outs["job_a"]})
+ assert b.batch_member_ids() == ["job_b", "job_c"]
+ b._handle_event({"type": "error", "member_run_id": "job_a", "error": "late"})
+ b._handle_event(_progress("job_a", 3, 9.0))
+ b._handle_event(
+ {"type": "stopped", "member_run_id": "job_b", "output_dir": outs["job_b"], "saved": True}
+ )
+ assert b.batch_member_ids() == ["job_c"]
+ b._handle_event({"type": "error", "error": "CUDA error: boom"})
+ assert b.batch_member_ids() == []
+
+ assert db.get_run(BATCH) is None
+ rows = {r: db.get_run(r) for r in MEMBERS}
+ assert rows["job_a"]["status"] == "completed"
+ assert rows["job_a"]["error_message"] is None
+ assert rows["job_b"]["status"] == "stopped"
+ assert rows["job_c"]["status"] == "error"
+ assert rows["job_c"]["error_message"] == "CUDA error: boom"
+ for i, run_id in enumerate(MEMBERS):
+ row = rows[run_id]
+ assert row["dataset_name"] == f"/data/{run_id}.jsonl"
+ config = json.loads(row["config_json"])
+ assert config["local_datasets"] == [f"/data/{run_id}.jsonl"]
+ assert "hf_token" not in config and "wandb_token" not in config
+ assert "hf_secret" not in row["config_json"]
+ assert row["final_step"] == 2
+ metrics = db.get_run_metrics(run_id)
+ assert metrics["step_history"] == [1, 2]
+ assert metrics["loss_history"] == pytest.approx([1.1 + i, 1.2 + i])
+ assert rows["job_a"]["output_dir"] == outs["job_a"]
+ assert rows["job_b"]["output_dir"] == outs["job_b"]
+
+
+def test_process_complete_finalizes_only_running_members(db, tmp_path):
+ b = _batch_backend(tmp_path)
+ saved = _exported(tmp_path, "job_b")
+ for run_id in MEMBERS:
+ b._handle_event(_progress(run_id, 1, 2.0))
+ b._handle_event({"type": "error", "member_run_id": "job_a", "error": "bad dataset"})
+ b._handle_event({"type": "output_dir", "member_run_id": "job_b", "output_dir": saved})
+ b._handle_event(
+ {"type": "complete", "output_dir": None, "status_message": "Training completed"}
+ )
+
+ assert db.get_run(BATCH) is None
+ assert db.get_run("job_a")["status"] == "error"
+ assert db.get_run("job_a")["error_message"] == "bad dataset"
+ assert db.get_run("job_b")["status"] == "completed"
+ assert db.get_run("job_c")["status"] == "error"
+ assert "without saving" in db.get_run("job_c")["error_message"]
+
+
+def test_member_finalize_survives_a_transient_db_error(db, tmp_path, monkeypatch):
+ b = _batch_backend(tmp_path)
+ out = _exported(tmp_path, "job_a")
+ b._handle_event({"type": "output_dir", "member_run_id": "job_a", "output_dir": out})
+ b._handle_event(_progress("job_a", 1, 2.0))
+ real, calls = db.finish_run, []
+
+ def flaky(**kwargs):
+ calls.append(kwargs["id"])
+ if len(calls) == 1:
+ raise RuntimeError("database is locked")
+ return real(**kwargs)
+
+ monkeypatch.setattr(db, "finish_run", flaky)
+ monkeypatch.setattr(training_mod, "_DB_FINALIZE_RETRY_S", 0)
+ b._handle_event({"type": "complete", "member_run_id": "job_a", "output_dir": out})
+ assert calls == ["job_a", "job_a"]
+ assert db.get_run("job_a")["status"] == "completed"
+
+
+def test_member_error_before_progress_survives_a_transient_db_error(db, tmp_path, monkeypatch):
+ b = _batch_backend(tmp_path)
+ real, calls = db.create_run, []
+
+ def flaky(**kwargs):
+ calls.append(kwargs["id"])
+ if len(calls) == 1:
+ raise RuntimeError("database is locked")
+ return real(**kwargs)
+
+ monkeypatch.setattr(db, "create_run", flaky)
+ monkeypatch.setattr(training_mod, "_DB_FINALIZE_RETRY_S", 0)
+ b._handle_event({"type": "error", "member_run_id": "job_c", "error": "empty dataset"})
+ assert calls == ["job_c", "job_c"]
+ row = db.get_run("job_c")
+ assert row["status"] == "error" and row["error_message"] == "empty dataset"
+
+
+def test_member_error_before_progress_still_records_its_row(db, tmp_path):
+ b = _batch_backend(tmp_path)
+ b._handle_event({"type": "error", "member_run_id": "job_c", "error": "empty dataset"})
+ row = db.get_run("job_c")
+ assert row["status"] == "error" and row["error_message"] == "empty dataset"
+ assert db.get_run("job_a") is None
+ assert b.batch_member_ids() == ["job_a", "job_b"]
+
+
+def test_stop_member_sends_member_stop_only(db, tmp_path):
+ b = _batch_backend(tmp_path)
+ assert b.stop_member("job_b") is True
+ assert b._stop_queue.get_nowait() == {"type": "stop_member", "run_id": "job_b", "save": True}
+ assert b._stop_queue.empty()
+ assert b._should_stop is False and b._cancel_requested is False
+ assert b.stop_member("job_unknown") is False
+ b._handle_event({"type": "complete", "member_run_id": "job_a"})
+ assert b.stop_member("job_a") is False
+ assert b._stop_queue.empty()
+ b._handle_event(_progress("job_b", 1, 1.0))
+ b._handle_event({"type": "complete", "member_run_id": "job_b", "output_dir": "/x"})
+ assert db.get_run("job_b")["status"] == "stopped"
+
+
+def test_batch_cancel_marks_member_rows(db, tmp_path):
+ b = _batch_backend(tmp_path)
+ b._handle_event(_progress("job_a", 1, 1.0))
+ assert b.stop_training(save = False, expected_job_id = BATCH) is True
+ assert b._stop_queue.get_nowait() == {"type": "stop", "save": False}
+ assert db.get_run("job_a")["resume_blocked"] == 1
+ b._handle_event({"type": "complete", "status_message": "Training cancelled."})
+ assert db.get_run(BATCH) is None
+ for run_id in ("job_a",):
+ row = db.get_run(run_id)
+ assert row["status"] == "stopped" and row["output_dir"] is None
+
+
+def test_single_job_events_unchanged(db, tmp_path):
+ b = TrainingBackend()
+ b.current_job_id = "job_single"
+ b._proc = _AliveProc()
+ b._start_stop_watchdog = lambda **kw: None
+ b._progress = TrainingProgress(is_training = True)
+ b._db_config = {"model_name": "unsloth/tiny", "local_datasets": ["/d.jsonl"]}
+ b._handle_event({"type": "progress", "step": 1, "loss": 1.5, "total_steps": 2})
+ b._handle_event(
+ {"type": "complete", "output_dir": "/o", "status_message": "Training completed"}
+ )
+ row = db.get_run("job_single")
+ assert row["status"] == "completed"
+ assert db.get_run_metrics("job_single")["step_history"] == [1]
+
+
+class _DummyProcess:
+ pid = 4321
+
+ def start(self):
+ return None
+
+ def is_alive(self):
+ return True
+
+
+class _DummyThread:
+ def start(self):
+ return None
+
+
+def test_start_training_builds_batch_members_config(db, tmp_path):
+ captured = {}
+ real_spec = training_mod.account_process_spec
+
+ def _spy(module, fn, cache_env, kwargs):
+ captured["config"] = kwargs["config"]
+ return real_spec(module, fn, cache_env, kwargs)
+
+ def _select(gpu_ids, **kwargs):
+ captured["sizing"] = kwargs
+ return [0], {}
+
+ backend = TrainingBackend()
+ with (
+ patch("core.training.training.prepare_gpu_selection", side_effect = _select),
+ patch("core.training.training._CTX.Queue", side_effect = [object(), object()]),
+ patch("core.training.training._CTX.Process", return_value = _DummyProcess()),
+ patch("core.training.training.threading.Thread", return_value = _DummyThread()),
+ patch("core.training.training.account_process_spec", side_effect = _spy),
+ ):
+ ok = backend.start_training(
+ job_id = BATCH,
+ batch_members = [
+ {
+ "run_id": run_id,
+ "model_name": "unsloth/test",
+ "training_type": "LoRA/QLoRA",
+ "hf_token": "hf_secret",
+ "local_datasets": [f"/data/{run_id}.jsonl"],
+ "lora_r": 8 * (i + 1),
+ "batch_size": i + 1,
+ }
+ for i, run_id in enumerate(MEMBERS)
+ ],
+ model_name = "unsloth/test",
+ training_type = "LoRA/QLoRA",
+ hf_token = "hf_secret",
+ local_datasets = ["/data/job_a.jsonl"],
+ lora_r = 8,
+ gpu_ids = [0],
+ )
+ assert ok
+ # The GPU is chosen for every member's rows and adapters together, not the first member's.
+ assert captured["sizing"]["batch_size"] == 1 + 2 + 3
+ assert captured["sizing"]["lora_rank"] == 8 + 16 + 24
+ config = captured["config"]
+ assert config["batch_id"] == BATCH
+ assert config["hf_token"] == "hf_secret"
+ assert [m["run_id"] for m in config["batch_members"]] == list(MEMBERS)
+ for i, member in enumerate(config["batch_members"]):
+ assert member["config"]["lora_r"] == 8 * (i + 1)
+ assert member["config"]["local_datasets"] == [f"/data/{MEMBERS[i]}.jsonl"]
+ assert "hf_token" not in member["config"]
+ assert "subject" not in member["config"]
+ assert backend.batch_member_ids() == list(MEMBERS)
+ assert backend._db_config is None
+ assert db.get_run(BATCH) is None
+
+
+def test_start_training_without_batch_is_unchanged(db, tmp_path):
+ captured = {}
+ real_spec = training_mod.account_process_spec
+
+ def _spy(module, fn, cache_env, kwargs):
+ captured["config"] = kwargs["config"]
+ return real_spec(module, fn, cache_env, kwargs)
+
+ backend = TrainingBackend()
+ with (
+ patch("core.training.training.prepare_gpu_selection", return_value = ([0], {})),
+ patch("core.training.training._CTX.Queue", side_effect = [object(), object()]),
+ patch("core.training.training._CTX.Process", return_value = _DummyProcess()),
+ patch("core.training.training.threading.Thread", return_value = _DummyThread()),
+ patch("core.training.training.account_process_spec", side_effect = _spy),
+ ):
+ assert backend.start_training(
+ job_id = "job_single", model_name = "unsloth/test", training_type = "LoRA/QLoRA"
+ )
+ assert "batch_members" not in captured["config"]
+ assert "batch_id" not in captured["config"]
+ assert backend.batch_member_ids() == []
+ assert db.get_run("job_single")["status"] == "running"
+
+
+def test_batch_start_sizes_chat_coexistence_for_every_member(monkeypatch):
+ import asyncio
+ import contextlib
+
+ import routes.training as tr
+ import routes.training_vram as vram
+ from models import TrainingStartRequest
+
+ class _Backend:
+ current_job_id = None
+
+ def is_training_active(self):
+ return False
+
+ def start_training(
+ self,
+ job_id,
+ *,
+ before_spawn = None,
+ **kwargs,
+ ):
+ self.current_job_id = job_id
+ before_spawn()
+ return True
+
+ asked = []
+ monkeypatch.setattr(tr, "get_training_backend", lambda: _Backend())
+ monkeypatch.setattr(tr, "_diffusion_training_active", lambda: False)
+ monkeypatch.setattr(tr, "_diffusion_gpu_admission", contextlib.nullcontext)
+ monkeypatch.setattr(tr, "hf_env_offline", lambda: False)
+ monkeypatch.setattr(tr, "_hub_unreachable", lambda: False)
+ monkeypatch.setattr(tr, "_remote_untrainable_model_format", lambda *a, **k: None)
+ monkeypatch.setattr("utils.hardware.ensure_hardware_detected", lambda: None)
+ monkeypatch.setattr(vram, "coordinate_models_for_training", lambda keep: [keep()])
+ monkeypatch.setattr(
+ vram, "can_keep_chat_during_training", lambda **kw: asked.append(kw) or True
+ )
+
+ def request(r, batch_size):
+ return TrainingStartRequest(
+ model_name = "unsloth/tiny-model",
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ load_in_4bit = False,
+ lora_r = r,
+ batch_size = batch_size,
+ )
+
+ started = asyncio.run(
+ tr.start_training_batch(
+ [request(8, 2), request(16, 4), request(32, 1)],
+ current_subject = "alice",
+ via_api_key = True,
+ )
+ )
+ assert [s.status for s in started] == ["queued"] * 3
+ assert [(kw["batch_size"], kw["lora_rank"]) for kw in asked] == [(2 + 4 + 1, 8 + 16 + 32)]
+
+
+def test_xet_respawn_replays_pending_member_stops(monkeypatch, tmp_path):
+ class _Queue:
+ def __init__(self):
+ self.items = []
+
+ def put(self, item, *a, **k):
+ self.items.append(item)
+
+ class _Proc:
+ pid = 4322
+
+ def __init__(self, **kwargs):
+ self.kwargs = kwargs
+
+ def start(self):
+ pass
+
+ def is_alive(self):
+ return False
+
+ def join(self, timeout = None):
+ pass
+
+ class _Ctx:
+ def __init__(self):
+ self.spawned = []
+
+ def Queue(self):
+ return _Queue()
+
+ def Process(self, **kwargs):
+ self.spawned.append(kwargs)
+ return _Proc(**kwargs)
+
+ b = _batch_backend(tmp_path)
+ b._last_full_config = {"model_name": "unsloth/tiny", "batch_members": []}
+ b._last_hf_cache_env = {"HF_HOME": str(tmp_path / "hf")}
+ b._proc = _Proc()
+ b._batch_members["job_a"]["stop_requested"] = True
+ b._batch_members["job_c"].update(stop_requested = True, terminal = True)
+ ctx = _Ctx()
+ monkeypatch.setattr(training_mod, "_CTX", ctx)
+ monkeypatch.setattr(b, "_pump_loop", lambda: None)
+ monkeypatch.setattr(b, "_open_worker_stderr_capture", lambda: None)
+
+ assert b._respawn_worker_disable_xet() is True
+ stop_queue = b._stop_queue
+ assert stop_queue.items == [{"type": "stop_member", "run_id": "job_a", "save": True}]
diff --git a/studio/backend/tests/test_training_cached_repo_authorization.py b/studio/backend/tests/test_training_cached_repo_authorization.py
index 17b473d8801..5f6fea9bf41 100644
--- a/studio/backend/tests/test_training_cached_repo_authorization.py
+++ b/studio/backend/tests/test_training_cached_repo_authorization.py
@@ -468,7 +468,7 @@ def test_a_disabled_eval_path_is_not_authorized(monkeypatch, cache_root):
import routes.training as training_routes
- source = inspect.getsource(training_routes.start_training)
+ source = inspect.getsource(training_routes._admit_training_request)
marker = "request.local_eval_datasets if evaluation_enabled(request.eval_steps) else []"
assert marker in source
@@ -511,7 +511,7 @@ def test_a_streaming_start_skips_the_cached_dataset_check():
import routes.training as training_routes
- source = inspect.getsource(training_routes.start_training)
+ source = inspect.getsource(training_routes._admit_training_request)
assert "not request.dataset_streaming" in source
guarded = source.split("not request.dataset_streaming", 1)[1]
assert "_refuse_unauthorized_cached_dataset" in guarded.split("\n\n", 1)[0]
@@ -557,7 +557,7 @@ def test_the_shadowed_dataset_gate_is_wired_into_the_start():
import routes.training as training_routes
- source = inspect.getsource(training_routes.start_training)
+ source = inspect.getsource(training_routes._admit_training_request)
assert "_hf_dataset_is_the_source(request)" in source
guarded = source.split("_hf_dataset_is_the_source(request)", 1)[1]
assert "_refuse_unauthorized_cached_dataset" in guarded.split("\n\n", 1)[0]