Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: save sampling defaults per model and use them for API requests (#14515)
Daniel Han3ea01403de81
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex c24aaf5471d..b5caa2be215 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -3010,6 +3010,54 @@ def update_openai_auto_switch_override(return ModelOverridesResponse(overrides = get_model_overrides(), removed_keys = removed_keys)+class ModelSamplingDefaultsPayload(BaseModel):+ """One model's saved sampling defaults. A null or missing field follows the model's recommendation;+ all of them null forgets the model."""++ model_id: str = Field(..., min_length = 1, max_length = MAX_MODEL_OVERRIDE_KEY_LEN)+ sampling: Optional[dict[str, Optional[float]]] = None+++class ModelSamplingDefaultsResponse(BaseModel):+ defaults: dict[str, dict]+ # What a request naming the given model gets, resolved here so the browser need not mirror the key rules.+ resolved: Optional[dict] = None+++@_account_settings_router.get(+ "/model-sampling-defaults", response_model = ModelSamplingDefaultsResponse+)+def get_model_sampling_defaults_route(+ model_id: Optional[str] = None, current_subject: str = Depends(get_current_subject)+) -> ModelSamplingDefaultsResponse:+ from utils.model_sampling_defaults import get_model_sampling_defaults, saved_sampling_defaults+ return ModelSamplingDefaultsResponse(+ defaults = get_model_sampling_defaults(),+ resolved = saved_sampling_defaults(model_id) if model_id else None,+ )+++@_account_settings_router.put(+ "/model-sampling-defaults", response_model = ModelSamplingDefaultsResponse+)+def update_model_sampling_defaults_route(+ payload: ModelSamplingDefaultsPayload, current_subject: str = Depends(get_current_subject)+) -> ModelSamplingDefaultsResponse:+ from utils.model_sampling_defaults import (+ get_model_sampling_defaults,+ saved_sampling_defaults,+ set_model_sampling_defaults,+ )+ try:+ set_model_sampling_defaults(payload.model_id, payload.sampling)+ except ValueError as exc:+ raise HTTPException(status_code = 400, detail = str(exc)) from exc+ return ModelSamplingDefaultsResponse(+ defaults = get_model_sampling_defaults(),+ resolved = saved_sampling_defaults(payload.model_id),+ )++class EmbeddingModelPayload(BaseModel):embedding_model: str = Field(..., min_length = 1, max_length = MAX_EMBEDDING_MODEL_LENGTH)# The repo /resolve named, stored so the loader opens what was downloaded.diff --git a/studio/backend/tests/test_model_sampling_defaults.py b/studio/backend/tests/test_model_sampling_defaults.pynew file mode 100644index 00000000000..1941c98bfdb--- /dev/null+++ b/studio/backend/tests/test_model_sampling_defaults.py@@ -0,0 +1,173 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Sampling defaults saved for one model (#10631): a /v1 request that leaves a field unset gets+the saved value, an explicit value and an operator pin still win, and nobody who never saved one+sees any change."""++import asyncio+import math+import os+import sys++import pytest++_backend = os.path.join(os.path.dirname(__file__), "..")+sys.path.insert(0, _backend)++from routes import settings as settings_routes+from utils import openai_auto_switch_settings as auto_switch+from utils.inference import inference_config as ic+from utils.model_sampling_defaults import (+ get_model_sampling_defaults,+ saved_sampling_defaults,+ set_model_sampling_defaults,+)++MODEL = "unsloth/Qwen3.5-9B-GGUF"++[email protected](autouse = True)+def _fresh(monkeypatch):+ for field in ic.SAMPLING_FIELD_NAMES:+ monkeypatch.delenv(ic._SAMPLING_FIELDS[field][0], raising = False)+ monkeypatch.setattr(ic, "load_inference_config", lambda mid: {"temperature": 1.0, "top_k": 64})+ ic._recommended_sampling.cache_clear()+ auto_switch._cache.clear()+ yield+ ic._recommended_sampling.cache_clear()+ auto_switch._cache.clear()+++def _omitted():+ return {field: None for field in ic.SAMPLING_FIELD_NAMES}+++def _put(model_id, sampling):+ return settings_routes.update_model_sampling_defaults_route(+ settings_routes.ModelSamplingDefaultsPayload(model_id = model_id, sampling = sampling),+ "tester",+ )+++def test_nothing_saved_resolves_exactly_as_before():+ assert get_model_sampling_defaults() == {}+ assert ic.resolve_effective_sampling(MODEL, _omitted()) == {+ "temperature": 1.0,+ "top_p": 0.95,+ "top_k": 64,+ "min_p": 0.01,+ "repetition_penalty": 1.0,+ "presence_penalty": 0.0,+ }+++def test_saved_defaults_fill_what_the_request_omits():+ _put(MODEL, {"temperature": 0.3, "top_p": 0.8, "repetition_penalty": 1.1})+ eff = ic.resolve_effective_sampling(MODEL, _omitted())+ assert eff["temperature"] == 0.3+ assert eff["top_p"] == 0.8+ assert eff["repetition_penalty"] == 1.1+ # Unsaved fields keep the recommendation, then the schema default.+ assert eff["top_k"] == 64+ assert eff["min_p"] == 0.01+++def test_an_explicit_request_value_and_an_operator_pin_still_win(monkeypatch):+ _put(MODEL, {"temperature": 0.3, "top_p": 0.8})+ eff = ic.resolve_effective_sampling(MODEL, {**_omitted(), "temperature": 0.9})+ assert eff["temperature"] == 0.9+ monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_P", "0.5")+ assert ic.resolve_effective_sampling(MODEL, _omitted())["top_p"] == 0.5+++def test_passthrough_fill_sends_only_saved_and_recommended_fields():+ _put(MODEL, {"repetition_penalty": 1.2})+ eff = ic.resolve_effective_sampling(MODEL, _omitted(), fill_defaults = False)+ assert eff == {"temperature": 1.0, "top_k": 64, "repetition_penalty": 1.2}+++def test_saved_defaults_beat_the_qwen3_thinking_table():+ model = "unsloth/Qwen3-4B-GGUF"+ _put(model, {"temperature": 0.2})+ assert ic.resolve_effective_sampling(model, _omitted(), thinking = True)["temperature"] == 0.2+++def test_a_cached_snapshot_path_finds_the_repo_row():+ _put("unsloth/Foo-GGUF", {"min_p": 0.05})+ path = "/home/u/.cache/huggingface/hub/models--unsloth--Foo-GGUF/snapshots/abc123"+ assert saved_sampling_defaults(path) == {"min_p": 0.05}+++def test_a_snapshot_path_save_is_stored_under_the_repo_id():+ old = "/home/u/.cache/huggingface/hub/models--unsloth--Foo-GGUF/snapshots/abc123"+ new = "/home/u/.cache/huggingface/hub/models--unsloth--Foo-GGUF/snapshots/def456"+ response = _put(old + ":Q4_K_M", {"top_k": 40})+ assert response.defaults == {"unsloth/Foo-GGUF": {"top_k": 40}}+ assert response.resolved == {"top_k": 40}+ assert saved_sampling_defaults(new) == {"top_k": 40}+++def test_a_quant_qualified_id_reads_the_repo_row():+ _put("unsloth/Foo-GGUF", {"top_k": 40})+ assert saved_sampling_defaults("unsloth/Foo-GGUF:Q4_K_M") == {"top_k": 40}+++def test_lookup_folds_case_but_not_ambiguously():+ _put("unsloth/foo-gguf", {"top_k": 40})+ assert saved_sampling_defaults("unsloth/Foo-GGUF") == {"top_k": 40}+ _put("Unsloth/FOO-gguf", {"top_k": 10})+ assert saved_sampling_defaults("UNSLOTH/foo-GGUF") == {}+++def test_a_posix_path_never_matches_another_casing():+ _put("/models/Foo.gguf", {"top_k": 40})+ assert saved_sampling_defaults("/models/Foo.gguf") == {"top_k": 40}+ assert saved_sampling_defaults("/models/foo.gguf") == {}+++def test_a_windows_path_matches_another_casing_and_slash():+ _put("C:\\Models\\Foo.gguf", {"top_k": 40})+ assert saved_sampling_defaults("c:/models/foo.gguf") == {"top_k": 40}+++def test_out_of_range_and_non_finite_values_are_dropped():+ _put(MODEL, {"temperature": 5.0, "top_p": math.nan, "top_k": 30, "min_p": -1})+ assert get_model_sampling_defaults() == {MODEL: {"top_k": 30}}+++def test_save_is_idempotent_and_clearing_forgets_the_model():+ first = _put(MODEL, {"temperature": 0.4})+ second = _put(MODEL, {"temperature": 0.4})+ assert first.defaults == second.defaults == {MODEL: {"temperature": 0.4}}+ assert _put(MODEL, {"temperature": None}).defaults == {}+ assert _put(MODEL, None).defaults == {}+ assert ic.resolve_effective_sampling(MODEL, _omitted())["temperature"] == 1.0+++def test_an_unreadable_store_never_fails_a_request(monkeypatch):+ def boom(*_args, **_kwargs):+ raise RuntimeError("db gone")++ monkeypatch.setattr("utils.model_sampling_defaults.saved_sampling_defaults", boom, raising = True)+ assert ic.resolve_effective_sampling(MODEL, _omitted())["temperature"] == 1.0+++def test_the_sampling_defaults_route_shows_what_a_request_gets():+ from routes import inference as inference_routes++ _put(MODEL, {"temperature": 0.35})+ got = asyncio.run(inference_routes.get_sampling_defaults(model = MODEL, current_subject = "owner"))+ assert got["temperature"] == 0.35+++def test_completions_passthrough_body_gets_the_saved_repeat_penalty():+ from routes import inference as inference_routes++ _put(MODEL, {"repetition_penalty": 1.15})+ body = {"prompt": "hi"}+ inference_routes._fill_recommended_sampling_completions(body, MODEL)+ assert body["repeat_penalty"] == 1.15+ explicit = {"prompt": "hi", "repeat_penalty": 1.0}+ inference_routes._fill_recommended_sampling_completions(explicit, MODEL)+ assert explicit["repeat_penalty"] == 1.0diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.pyindex 3633e8e1109..887eb7e9f76 100644--- a/studio/backend/utils/inference/inference_config.py+++ b/studio/backend/utils/inference/inference_config.py@@ -205,6 +205,18 @@ def _recommended_sampling(model_id: str) -> Dict[str, Any]:return recommended+def _saved_sampling(model_id: Optional[str]) -> Dict[str, Any]:+ """The user's saved defaults for this model; a store that cannot be read means none, never a failed request."""+ if not model_id:+ return {}+ try:+ from utils.model_sampling_defaults import saved_sampling_defaults+ return saved_sampling_defaults(model_id)+ except Exception as e:+ logger.debug(f"Could not read saved sampling defaults for '{model_id}': {e}")+ return {}++def resolve_effective_sampling(model_id: Optional[str],explicit: Dict[str, Any],@@ -212,10 +224,13 @@ def resolve_effective_sampling(fill_defaults: bool = True,thinking: Optional[bool] = None,) -> Dict[str, Any]:- """Resolve the effective sampling params for a request. ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent value, or ``None`` when the client omitted it. Precedence, highest first: an operator ``UNSLOTH_SAMPLING_*`` pin, the client's explicit value, the per-model recommendation, then the static schema default. When ``fill_defaults`` is False a field with none of the first three is omitted rather than set to the static default, so a raw proxy body (``/v1/completions``) keeps llama-server's own default for that field."""+ """Resolve the effective sampling params for a request. ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent value, or ``None`` when the client omitted it. Precedence, highest first: an operator ``UNSLOTH_SAMPLING_*`` pin, the client's explicit value, the defaults the user saved for this model, the per-model recommendation, then the static schema default. When ``fill_defaults`` is False a field with none of the first four is omitted rather than set to the static default, so a raw proxy body (``/v1/completions``) keeps llama-server's own default for that field."""recommended = _recommended_sampling(model_id or "")if thinking is not None and "qwen3" in (model_id or "").lower():recommended = {**recommended, **_QWEN3_THINKING_SAMPLING[thinking]}+ saved = _saved_sampling(model_id)+ if saved:+ recommended = {**recommended, **saved}effective: Dict[str, Any] = {}for field, (_env, default, _lo, _hi, _int) in _SAMPLING_FIELDS.items():override = _operator_sampling_override(field)diff --git a/studio/backend/utils/model_sampling_defaults.py b/studio/backend/utils/model_sampling_defaults.pynew file mode 100644index 00000000000..c50f4637c4a--- /dev/null+++ b/studio/backend/utils/model_sampling_defaults.py@@ -0,0 +1,143 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Per-model sampling defaults a user saved; a request that leaves a field unset gets them before the+model's recommendation. Not stored in the launch overrides: a row there counts as a load override+and shadows a broader one."""++from __future__ import annotations++import threading+from typing import Any, Mapping, Optional++from utils.account_context import current_account+from utils.openai_auto_switch_settings import (+ _cached_setting,+ _fold_case_insensitive_path,+ _invalidate,+ _looks_like_filesystem_path,+ split_quant_suffix,+)++MODEL_SAMPLING_DEFAULTS_SETTING_KEY = "model_sampling_defaults"+++def normalize_sampling_defaults(payload: Optional[Mapping[str, Any]]) -> dict[str, Any]:+ """The usable fields of ``payload``, each validated the way a request value is; anything else is dropped."""+ from utils.inference.inference_config import SAMPLING_FIELD_NAMES, _clean_sampling_value++ if not isinstance(payload, Mapping):+ return {}+ cleaned: dict[str, Any] = {}+ for field in SAMPLING_FIELD_NAMES:+ value = _clean_sampling_value(field, payload.get(field))+ if value is not None:+ cleaned[field] = value+ return cleaned+++def _stored(account) -> Any:+ return _cached_setting(MODEL_SAMPLING_DEFAULTS_SETTING_KEY, None, account)+++def get_model_sampling_defaults(account = None) -> dict[str, dict]:+ raw = _stored(account or current_account())+ if not isinstance(raw, dict):+ return {}+ return {+ key: value for key, value in raw.items() if isinstance(key, str) and isinstance(value, dict)+ }+++# Rebuilt only when the cached setting is re-read, so a request does dict lookups, not a scan.+_index_lock = threading.Lock()+_index_cache: dict[str, tuple[Any, dict[str, dict], dict[str, Optional[dict]]]] = {}+++def _fold(key: str) -> Optional[str]:+ """The launch overrides' folding rule: repo ids and Windows / UNC / WSL paths fold, a POSIX path does not."""+ if _looks_like_filesystem_path(key):+ return _fold_case_insensitive_path(key)+ return key.casefold()+++def _index(account) -> tuple[dict[str, dict], dict[str, Optional[dict]]]:+ raw = _stored(account)+ with _index_lock:+ hit = _index_cache.get(account.account_id)+ if hit is not None and hit[0] is raw:+ return hit[1], hit[2]+ exact: dict[str, dict] = {}+ folded: dict[str, Optional[dict]] = {}+ if isinstance(raw, dict):+ for key, value in raw.items():+ if not isinstance(key, str) or not isinstance(value, dict):+ continue+ entry = normalize_sampling_defaults(value)+ exact[key] = entry+ # An ambiguous fold matches nothing.+ fold = _fold(key)+ if fold is not None:+ folded[fold] = None if fold in folded else entry+ with _index_lock:+ _index_cache[account.account_id] = (raw, exact, folded)+ return exact, folded+++def _lookup(+ exact: dict[str, dict], folded: dict[str, Optional[dict]], candidates: list[str]+) -> Optional[dict]:+ for candidate in candidates:+ if candidate in exact:+ return exact[candidate]+ for candidate in candidates:+ fold = _fold(candidate)+ found = folded.get(fold) if fold is not None else None+ if found is not None:+ return found+ return None+++def sampling_defaults_key(model_id: str) -> str:+ """The key one model's defaults are stored under: per model, not per quant, and a cached repo by+ its repo id rather than a snapshot path that a new revision would orphan."""+ from core.inference.model_ids import hf_cache_repo_id++ key = model_id.strip()+ split = split_quant_suffix(key)+ if split is not None:+ key = split[0]+ return hf_cache_repo_id(key) or key+++def saved_sampling_defaults(model_id: Optional[str]) -> dict[str, Any]:+ """The saved defaults for ``model_id``, or ``{}``."""+ if not model_id or not isinstance(model_id, str):+ return {}+ exact, folded = _index(current_account())+ if not exact:+ return {}+ candidates = [model_id]+ key = sampling_defaults_key(model_id)+ if key not in candidates:+ candidates.append(key)+ found = _lookup(exact, folded, candidates)+ return dict(found) if found is not None else {}+++def set_model_sampling_defaults(+ model_id: str, sampling: Optional[Mapping[str, Any]]+) -> dict[str, Any]:+ """Replace one model's saved defaults; nothing usable removes the row. Returns what was stored."""+ if not model_id or not model_id.strip():+ raise ValueError("model_id is required.")+ entry = normalize_sampling_defaults(sampling)+ from storage.studio_db import upsert_app_setting_map_entry++ key = sampling_defaults_key(model_id)+ upsert_app_setting_map_entry(MODEL_SAMPLING_DEFAULTS_SETTING_KEY, key, entry or None)+ # A row an earlier save left under the literal spelling would otherwise outrank this one.+ if model_id.strip() != key:+ upsert_app_setting_map_entry(MODEL_SAMPLING_DEFAULTS_SETTING_KEY, model_id.strip(), None)+ _invalidate(MODEL_SAMPLING_DEFAULTS_SETTING_KEY, current_account())+ return entrydiff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsxindex 0cb2f2dd0cd..d8d16c8cc91 100644--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx@@ -71,6 +71,8 @@ import { Braces, ChevronDown, ExternalLink } from "lucide-react";import { Tooltip as TooltipPrimitive } from "radix-ui";import { type CSSProperties, Fragment, type ReactNode } from "react";import {+ lazy,+ Suspense,useCallback,useEffect,useMemo,@@ -132,6 +134,10 @@ import {import { effectiveMinPMode, isMinPMode } from "./lib/min-p-policy";import { mlxSpecFallbackMessage } from "./lib/mlx-spec-fallback";+const SaveForModelButton = lazy(+ () => import("./components/save-for-model-button"),+);+export { defaultInferenceParams, type Preset } from "./presets/preset-policy";export type { InferenceParams } from "./types/runtime";@@ -1736,6 +1742,12 @@ export function ChatSettingsPanel({<RetrievalSettingsSection /></CollapsibleSection>)}++ {!isExternalModel && currentCheckpoint ? (+ <Suspense fallback={null}>+ <SaveForModelButton store={useChatRuntimeStore} notify={toast} />+ </Suspense>+ ) : null}</div></div></div>diff --git a/studio/frontend/src/features/chat/components/save-for-model-button.tsx b/studio/frontend/src/features/chat/components/save-for-model-button.tsxnew file mode 100644index 00000000000..8e826a710ed--- /dev/null+++ b/studio/frontend/src/features/chat/components/save-for-model-button.tsx@@ -0,0 +1,91 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { Button } from "@/components/ui/button";+import { authFetch } from "@/features/auth";+import { readFastApiError } from "@/lib/format-fastapi-error";+import type { toast } from "@/lib/toast";+import type { useChatRuntimeStore } from "../stores/chat-runtime-store";++// Passed in, not imported: importing them here splits them out of the startup bundle into extra chunks.+type Deps = { store: typeof useChatRuntimeStore; notify: typeof toast };++/** Store the sampling on screen as the loaded model's defaults: new chats with it start from them, and API+ * requests that leave a value out get them. Min P on the server's choice stays unsaved. */+async function saveSamplingForCurrentModel({+ store,+ notify,+}: Deps): Promise<void> {+ const current = store.getState().params;+ const modelId = current.checkpoint;+ const res = await authFetch("/api/settings/model-sampling-defaults", {+ method: "PUT",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ // biome-ignore lint/style/useNamingConvention: API schema+ model_id: modelId,+ sampling: {+ temperature: current.temperature,+ // biome-ignore lint/style/useNamingConvention: API schema+ top_p: current.topP,+ // biome-ignore lint/style/useNamingConvention: API schema+ top_k: current.topK,+ // biome-ignore lint/style/useNamingConvention: API schema+ repetition_penalty: current.repetitionPenalty,+ // biome-ignore lint/style/useNamingConvention: API schema+ presence_penalty: current.presencePenalty,+ ...(current.minPMode === "custom"+ ? // biome-ignore lint/style/useNamingConvention: API schema+ { min_p: current.minP }+ : {}),+ },+ }),+ });+ if (!res.ok) {+ throw new Error(+ await readFastApiError(res, "Failed to save sampling for this model"),+ );+ }+ const sampling = {+ temperature: current.temperature,+ topP: current.topP,+ topK: current.topK,+ minP: current.minP,+ minPMode: current.minPMode,+ repetitionPenalty: current.repetitionPenalty,+ presencePenalty: current.presencePenalty,+ };+ // Writing the memory also sets the live sliders: skip it if one moved during the save.+ const live = store.getState().params;+ if (+ live.checkpoint === modelId &&+ Object.entries(sampling).every(+ ([key, value]) => live[key as keyof typeof sampling] === value,+ )+ ) {+ store.getState().setRememberedParamsForModel(modelId, sampling);+ }+ notify.success("Sampling saved for this model.");+}++export default function SaveForModelButton(deps: Deps) {+ return (+ <div className="border-t border-[rgb(0_0_0_/_calc(0.13*var(--contrast-edge-gain,1)))] py-5 dark:border-[rgb(255_255_255_/_calc(0.09*var(--contrast-edge-gain,1)))]">+ <Button+ type="button"+ variant="outline"+ size="sm"+ className="w-full"+ onClick={() => {+ saveSamplingForCurrentModel(deps).catch((error: unknown) => {+ deps.notify.error(+ error instanceof Error ? error.message : String(error),+ );+ });+ }}+ >+ Save for this model+ </Button>+ </div>+ );+}