Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""keep_resident: weights never return to system RAM while loaded, no idle auto-unload, --mlock;
no_ram_reserve avoids locked or reserved weight buffers; both on = VRAM only, and no_ram_reserve
wins over --mlock."""
from __future__ import annotations
import threading
import time
from typing import Any, Optional
from utils.account_context import OWNER, run_as
KEEP_RESIDENT_SETTING_KEY = "model_memory_keep_resident"
NO_RAM_RESERVE_SETTING_KEY = "model_memory_no_ram_reserve"
DEFAULT_KEEP_RESIDENT = False
DEFAULT_NO_RAM_RESERVE = False
# Read on hot paths, so memo briefly to spare SQLite.
_CACHE_TTL_S = 2.0
_cache_lock = threading.Lock()
_cache: dict[tuple[str, str], tuple[float, Any]] = {}
# Bumped on every write so an earlier read cannot cache the old value; else a load could launch contradicting the new setting for the rest of the TTL.
_generation: dict[tuple[str, str], int] = {}
def _coerce_bool(value: Any) -> Optional[bool]:
if isinstance(value, bool):
return value
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off", ""}:
return False
return None
# Bound only guards against a pathological write storm.
_MAX_REREADS = 3
def _cached_setting(key: str) -> Any:
cache_key = (OWNER.account_id, key)
for _attempt in range(_MAX_REREADS):
with _cache_lock:
hit = _cache.get(cache_key)
if hit is not None and time.monotonic() - hit[0] < _CACHE_TTL_S:
return hit[1]
generation = _generation.get(cache_key, 0)
try:
from storage.studio_db import get_app_setting
stored = run_as(OWNER, get_app_setting, key, None)
except Exception:
# An unreadable DB must not fail a load; fall back to the default.
return None
with _cache_lock:
if _generation.get(cache_key, 0) == generation:
_cache[cache_key] = (time.monotonic(), stored)
return stored
# A write landed mid-read, so stored predates it; returning it would let a load launch with flags contradicting the setting just saved. Read again.
return stored
def _invalidate(*keys: str) -> None:
"""Drop all keys in one acquisition so a load cannot pair new keep_resident with stale
no_ram_reserve."""
account_id = OWNER.account_id
with _cache_lock:
for key in keys:
cache_key = (account_id, key)
_cache.pop(cache_key, None)
_generation[cache_key] = _generation.get(cache_key, 0) + 1
def get_keep_resident() -> bool:
"""True when the loaded model must stay in GPU memory while it is loaded."""
parsed = _coerce_bool(_cached_setting(KEEP_RESIDENT_SETTING_KEY))
return parsed if parsed is not None else DEFAULT_KEEP_RESIDENT
def get_no_ram_reserve() -> bool:
"""True when no full host-RAM copy of the weights may be held."""
parsed = _coerce_bool(_cached_setting(NO_RAM_RESERVE_SETTING_KEY))
return parsed if parsed is not None else DEFAULT_NO_RAM_RESERVE
def should_mlock() -> bool:
"""Needs keep_resident on and no_ram_reserve off, since mlock pins the whole model in host RAM."""
keep_resident, no_ram_reserve = get_model_memory_settings()
return keep_resident and not no_ram_reserve
def _pair_generations() -> tuple[int, int]:
with _cache_lock:
return (
_generation.get((OWNER.account_id, KEEP_RESIDENT_SETTING_KEY), 0),
_generation.get((OWNER.account_id, NO_RAM_RESERVE_SETTING_KEY), 0),
)
def capture_model_memory_settings(publish) -> tuple[bool, bool]:
"""Publishes the pair, rereading if a save bumped the generation mid-read; else reload_required=false for a child on pre-save flags. No lock across DB I/O."""
for _attempt in range(_MAX_REREADS):
before = _pair_generations()
pair = get_model_memory_settings()
publish(pair)
if _pair_generations() == before:
return pair
return pair
def get_model_memory_settings() -> tuple[bool, bool]:
"""One coherent snapshot of both flags: a save between separate reads could give a never-stored pair."""
pair = (get_keep_resident(), get_no_ram_reserve())
for _attempt in range(_MAX_REREADS):
before = _pair_generations()
pair = (get_keep_resident(), get_no_ram_reserve())
if _pair_generations() == before:
return pair
return pair
def set_model_memory_settings(
keep_resident: Any = None, no_ram_reserve: Any = None
) -> tuple[bool, bool]:
"""One-transaction write; ``None`` leaves a stored value untouched."""
updates: dict[str, bool] = {}
if keep_resident is not None:
parsed = _coerce_bool(keep_resident)
if parsed is None:
raise ValueError("Keep model in GPU memory must be true or false.")
updates[KEEP_RESIDENT_SETTING_KEY] = parsed
if no_ram_reserve is not None:
parsed = _coerce_bool(no_ram_reserve)
if parsed is None:
raise ValueError("Do not reserve system RAM must be true or false.")
updates[NO_RAM_RESERVE_SETTING_KEY] = parsed
if updates:
from storage.studio_db import upsert_app_settings
upsert_app_settings(updates)
_invalidate(*updates)
return get_keep_resident(), get_no_ram_reserve()
def memlock_limit_bytes() -> Optional[int]:
"""Soft RLIMIT_MEMLOCK in bytes, None when unlimited or unavailable; the Linux 8 MB default makes llama.cpp mlock fail silently, disabling residency."""
try:
import resource
except ImportError:
return None
try:
soft, _hard = resource.getrlimit(resource.RLIMIT_MEMLOCK)
except (AttributeError, ValueError, OSError):
return None
if soft < 0 or soft == resource.RLIM_INFINITY:
return None
return int(soft)