Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Trim comments in studio/backend core services and the unsloth package (#10116)
Daniel Han115339ba0c07
diff --git a/studio/backend/_platform_compat.py b/studio/backend/_platform_compat.pyindex 013667d5cae..95f6cacce34 100644--- a/studio/backend/_platform_compat.py+++ b/studio/backend/_platform_compat.py@@ -39,12 +39,12 @@ def _seed_sys_version_cache() -> None:cleaned = m.group(0) + cleaned[p:]if cleaned == raw:- return # Nothing to fix+ returntry:result = platform._sys_version(cleaned)except ValueError:- return # Still unparsable; don't make things worse+ return# Seed the cache so future calls with the raw string skip parsingcache = getattr(platform, "_sys_version_cache", None)diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.pyindex 28b4cadef59..a107b916aad 100644--- a/studio/backend/auth/authentication.py+++ b/studio/backend/auth/authentication.py@@ -187,7 +187,7 @@ class _BearerOrKeyless(HTTPBearer):# scheme_name pinned so the OpenAPI securitySchemes entry keeps its published name-security = _BearerOrKeyless(scheme_name = "HTTPBearer") # Reads Authorization: Bearer <token>+security = _BearerOrKeyless(scheme_name = "HTTPBearer")def _get_secret_for_subject(subject: str) -> str:@@ -365,9 +365,7 @@ async def credentials_for_token("""from utils.keyless_api_access import APPROVED_DUMMY_BEARERS, keyless_request_allowed- # A real token is authoritative and never needs keyless classification. The- # remaining settings/listener reads use SQLite and DNS, so keep them off the- # event loop just like the normal credential lookup path.+ # Settings/listener reads hit SQLite and DNS, so keep them off the event loop.if token and token not in APPROVED_DUMMY_BEARERS:return HTTPAuthorizationCredentials(scheme = "Bearer", credentials = token)eligible = await run_in_threadpool(keyless_request_allowed, request)@@ -487,7 +485,6 @@ async def _get_current_credential(token = credentials.credentials- # --- API key path (sk-unsloth-...) ---if token.startswith(API_KEY_PREFIX):verified = await run_in_threadpool(validate_api_key_with_credential, token)if verified is None:@@ -498,7 +495,6 @@ async def _get_current_credential(username, secret = verifiedreturn username, credential_generation(secret)- # --- JWT path ---subject = _decode_subject_without_verification(token)if subject is None:raise HTTPException(diff --git a/studio/backend/auth/bootstrap_timeout.py b/studio/backend/auth/bootstrap_timeout.pyindex 2653c6e9857..57e24101538 100644--- a/studio/backend/auth/bootstrap_timeout.py+++ b/studio/backend/auth/bootstrap_timeout.py@@ -130,7 +130,7 @@ def enforce_bootstrap_password_deadline(except Exception:return Falseif not still_default:- return False # password changed in time -> leave Unsloth running+ return Falsemessage = ("\nUnsloth Studio was exposed on the network but its default admin "diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.pyindex 2756e334154..fb10ecba168 100644--- a/studio/backend/auth/storage.py+++ b/studio/backend/auth/storage.py@@ -21,12 +21,11 @@ from utils.paths import auth_db_path, ensure_dirDB_PATH = auth_db_path()DEFAULT_ADMIN_USERNAME = "unsloth"-# Single source for the password policy; models/auth.py ChangePasswordRequest-# and the terminal prompt both enforce it. Keep the unsloth_cli mirror in sync.+# Single source for the password policy; models/auth.py ChangePasswordRequest and the terminal+# prompt both enforce it. Keep the unsloth_cli mirror in sync.MIN_PASSWORD_LENGTH = 8-# Plaintext bootstrap password file beside auth.db, deleted on first password-# change so the credential never lingers on disk.+# Plaintext bootstrap password file, deleted on first password change._BOOTSTRAP_PW_PATH = DB_PATH.parent / ".bootstrap_password"# In-process cache to avoid re-reading the file on every HTML serve.@@ -81,8 +80,7 @@ def _normalise_bootstrap_file(raw: bytes, password: str) -> None:if raw != password.encode("utf-8"):return- # O_BINARY: without it Windows opens in text mode and turns the LF straight- # back into CRLF, the bug being fixed.+ # O_BINARY: Windows text mode turns the LF back into CRLF, the bug being fixed.fd = os.open(_BOOTSTRAP_PW_PATH,os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0),@@ -103,9 +101,7 @@ def _read_persisted_bootstrap_password() -> Optional[str]:if not _BOOTSTRAP_PW_PATH.is_file():return None- # No caller handles a raise, so an unreadable file has to mean "no bootstrap- # password", not a dead backend. We write UTF-8, so undecodable bytes are- # damage whose plaintext is worthless anyway.+ # An unreadable file must mean "no bootstrap password"; no caller handles a raise.try:raw = _BOOTSTRAP_PW_PATH.read_bytes()password = raw.decode("utf-8").strip()@@ -114,8 +110,7 @@ def _read_persisted_bootstrap_password() -> Optional[str]:if not password:return None- # Older releases wrote no terminator; best-effort, a read-only auth dir must- # not fail startup.+ # Older releases wrote no terminator; a read-only auth dir must not fail startup.if raw != _bootstrap_file_bytes(password):try:_normalise_bootstrap_file(raw, password)@@ -185,10 +180,7 @@ def clear_bootstrap_password() -> None:try:_BOOTSTRAP_PW_PATH.unlink(missing_ok = True)except OSError as e:- # Removal failed (Windows AV, read-only auth dir). The hash is already- # committed, so don't fail the change -- but truncate the file so its- # stale plaintext can't be re-seeded by generate_bootstrap_password()- # if auth.db is ever recreated.+ # Truncate when removal fails: stale plaintext would otherwise be re-seeded if auth.db is recreated.try:_BOOTSTRAP_PW_PATH.write_text("", encoding = "utf-8")cleared = True@@ -202,8 +194,7 @@ def clear_bootstrap_password() -> None:"cleared its contents so the old bootstrap password cannot be reused.")else:- # Neither removed nor truncated: stale plaintext is still on disk- # and would be reused if auth.db is reset. Don't claim otherwise.+ # Stale plaintext is still on disk and would be reused if auth.db is reset.message = (f"Warning: could not delete or clear {_BOOTSTRAP_PW_PATH.name} ({e}); ""its old bootstrap password is still on disk. Remove it manually to "@@ -272,21 +263,15 @@ def get_connection() -> sqlite3.Connection:"""Get a connection to the auth database, creating tables if needed."""ensure_dir(DB_PATH.parent)conn = sqlite3.connect(DB_PATH)- # Keep the auth dir + DB private (they hold the JWT/identity secrets and- # password hashes); sqlite3.connect would otherwise create the DB 0644 under- # a 022 umask, letting another OS user read the identity secret and forge proofs.+ # sqlite3.connect would create the DB 0644 under a 022 umask, exposing identity secrets and password hashes.for _path, _mode in ((DB_PATH.parent, 0o700), (DB_PATH, 0o600)):try:os.chmod(_path, _mode)except OSError:passconn.row_factory = sqlite3.Row- # WAL lets token reads run concurrently with refresh-token writes;- # busy_timeout bounds lock waits. Matches the other Unsloth SQLite stores.- # Set busy_timeout first: switching journal_mode needs a lock, so if a- # refresh-token write already holds one, journal_mode=WAL raises SQLITE_BUSY;- # with busy_timeout already in effect it waits instead of failing and leaving- # this connection on SQLite's default zero lock wait.+ # Set busy_timeout before journal_mode=WAL: switching journal mode needs a lock and would otherwise+ # raise SQLITE_BUSY.try:conn.execute("PRAGMA busy_timeout=5000")conn.execute("PRAGMA journal_mode=WAL")@@ -357,13 +342,8 @@ def get_connection() -> sqlite3.Connection:return conn+# No lock needed: INSERT OR IGNORE is atomic and concurrent populations converge on the same value.# ── API-key PBKDF2 salt ────────────────────────────────────────────────-#-# Module-level cache for the persistent API-key PBKDF2 salt, populated lazily-# via ``_get_or_create_api_key_pbkdf2_salt``. No lock needed: (a) ``INSERT OR-# IGNORE`` is atomic at the SQLite layer and (b) concurrent populations-# converge on the same value, so the worst case is a harmless duplicate read-# on startup._api_key_pbkdf2_salt_cache: Optional[bytes] = None@@ -385,7 +365,7 @@ def _get_or_create_api_key_pbkdf2_salt() -> bytes:)row = cur.fetchone()if row is None:- new_value = secrets.token_hex(32) # 32 bytes -> 64 hex chars+ new_value = secrets.token_hex(32)conn.execute("INSERT OR IGNORE INTO app_secrets (key, value) VALUES (?, ?)",("api_key_pbkdf2_salt", new_value),@@ -404,9 +384,7 @@ def _get_or_create_api_key_pbkdf2_salt() -> bytes:return salt-# Secret answering the /api/auth/identity challenge (HMAC(secret, nonce)). Lives-# in this same-user DB so a port squatter or remote/fake server can't forge a-# proof. Separate from the per-user JWT secret.+# Identity-challenge secret lives in auth.db so a port squatter cannot forge a proof; separate from the JWT secret._IDENTITY_SECRET_DB_KEY = "studio_identity_secret"_identity_secret_cache: Optional[bytes] = None@@ -441,9 +419,8 @@ def get_or_create_identity_secret() -> bytes:return secret-# Dedicated AES-256 key used to encrypt Unsloth credentials in studio.db.-# It intentionally lives in auth.db so copying studio.db alone does not expose-# provider or Hugging Face tokens, and survives password changes/resets.+# Dedicated AES-256 key: lives in auth.db so copying studio.db alone does not expose provider/HF+# tokens, and survives password resets._CREDENTIAL_ENCRYPTION_KEY_DB_KEY = "credential_encryption_key_v1"_credential_encryption_key_cache: Optional[bytes] = None@@ -487,17 +464,14 @@ def compute_identity_proof(nonce: bytes, host: str, port: int) -> str:real one, e.g. localhost resolving to ::1 while Unsloth is on 127.0.0.1) wascomputed for that other endpoint and won't match the one the client dialed."""try:- host = ipaddress.ip_address(host).compressed # normalise 127.0.0.1 / ::1 forms+ host = ipaddress.ip_address(host).compressedexcept ValueError:host = (host or "").lower()msg = b"|".join([nonce, host.encode(), str(int(port)).encode()])return hmac.new(get_or_create_identity_secret(), msg, hashlib.sha256).hexdigest()-# Capability secret for public ``/p`` preview share links. HMAC(secret, ref)-# turns the deterministic preview ref into an unguessable bearer capability, so a-# guessed run/checkpoint name can't reach inference. Dedicated (not the per-user-# JWT secret) so rotating it revokes every shared link without touching logins.+# Dedicated secret so rotating it revokes every shared preview link without touching logins._PREVIEW_LINK_SECRET_DB_KEY = "preview_link_secret"_preview_link_secret_cache: Optional[bytes] = None@@ -579,11 +553,8 @@ def _pbkdf2_desktop_secret(raw_secret: str) -> str:return _pbkdf2_api_key(raw_secret)-# Memoize the deterministic raw-key -> PBKDF2-hash derivation so the 100k-round-# KDF runs once per key instead of on every authenticated request. Keyed by a-# salted HMAC of the key (not the key itself); revocation/expiry are still-# enforced by the SQLite read on every call, so a cache hit only skips the KDF.-# Only keys present in the DB are cached, so unknown-key spam can't grow it.+# Keyed by a salted HMAC, not the key; revocation/expiry are still enforced by the SQLite read, and+# only known keys are cached._api_key_hash_cache: dict[str, str] = {}# Whether each memoized key was minted internally; set once, since minting decides it._api_key_internal_cache: dict[str, bool] = {}@@ -873,9 +844,8 @@ def consume_refresh_token(token: str) -> Optional[Tuple[str, bool, str]]:now = datetime.now(timezone.utc).isoformat()conn = get_connection()try:- # One transaction with the delete: an unstamped legacy row has no- # generation to compare, so reading the credential after committing would- # hand a reset's new secret to a token issued before it.+ # One transaction with the delete: an unstamped legacy row has no generation, so a later read could+ # hand a reset's new secret to an older token.conn.execute("BEGIN IMMEDIATE")conn.execute("DELETE FROM refresh_tokens WHERE expires_at < ?",@@ -914,7 +884,6 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]:token_hash = _hash_token(token)conn = get_connection()try:- # Opportunistically clean up expired tokensconn.execute("DELETE FROM refresh_tokens WHERE expires_at < ?",(datetime.now(timezone.utc).isoformat(),),@@ -939,7 +908,6 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]:conn.commit()return None- # Check expiryexpires_at = datetime.fromisoformat(row["expires_at"])if datetime.now(timezone.utc) > expires_at:conn.execute("DELETE FROM refresh_tokens WHERE id = ?", (row["id"],))@@ -1031,19 +999,12 @@ def clear_desktop_secret() -> None:conn.close()-# ----------------------------------------------------------------------------# API key management# ---------------------------------------------------------------------------API_KEY_PREFIX = "sk-unsloth-"-# The ``name`` a workflow mints its internal key under. Unsloth mints internal-# keys for more than one workflow and they do not carry the same authority, so-# the name is the only thing that tells them apart after the fact. Deep Research-# is durable: its hops outlive the session that started them, so they carry a key-# instead of a JWT and still have to reach the saved connection the run was-# created with. A data-recipe key is handed to a user-authored recipe subprocess-# and needs nothing but this host's local /v1, so it must never gain that reach.+# The name is the only thing distinguishing internal keys by authority: Deep Research keys must+# reach the saved connection, data-recipe keys only local /v1.DEEP_RESEARCH_WORKFLOW_KEY_NAME = "deep-research workflow"@@ -1274,9 +1235,8 @@ def validate_api_key_with_credential(with _api_key_hash_cache_lock:if len(_api_key_hash_cache) >= _API_KEY_HASH_CACHE_MAX:_api_key_hash_cache.clear()- # is_internal_api_key sizes itself against the hash cache, so clearing one- # without the other lets the origin cache grow past the bound. Deep Research- # mints a fresh internal key per model call, so that adds up.+ # Clear both caches together: the origin cache is sized against the hash cache and would otherwise+ # grow past its bound._api_key_internal_cache.clear()_api_key_hash_cache[cache_id] = key_hashif not row["is_active"]:diff --git a/studio/backend/auth/terminal_prompt.py b/studio/backend/auth/terminal_prompt.pyindex 925404f47df..988a857ca46 100644--- a/studio/backend/auth/terminal_prompt.py+++ b/studio/backend/auth/terminal_prompt.py@@ -24,8 +24,7 @@ _CTRL_Z = "\x1a"_BACKSPACES = ("\x7f", "\x08")_SUBMITS = ("\r", "\n")-# Env var that supplies the initial admin password non-interactively (mirror in-# unsloth_cli/commands/_password_prompt.py). Keep the name in sync.+# Mirror in unsloth_cli/commands/_password_prompt.py; keep the name in sync.SUPPLIED_PASSWORD_ENV = "UNSLOTH_STUDIO_PASSWORD"@@ -33,8 +32,7 @@ def _getch_windows() -> str: # pragma: no cover - exercised via fake on Linux Cimport msvcrtch = msvcrt.getwch()- # Function/arrow keys arrive as a two-wchar \x00/\xe0 sequence; consume the- # second half and report a no-op control char.+ # Function/arrow keys arrive as a two-wchar \x00/\xe0 sequence; consume the second half.if ch in ("\x00", "\xe0"):msvcrt.getwch()return "\x00"@@ -69,7 +67,7 @@ class _RestoreTtyOnSignals:continuetry:self._previous.append((sig, signal.signal(sig, _restore_and_reraise)))- except (ValueError, OSError): # non-main thread / unsupported+ except (ValueError, OSError):passreturn self@@ -99,20 +97,18 @@ class _prompt_raw_mode:try:import termiosimport tty- except ImportError: # non-POSIX (Windows uses msvcrt, no mode to hold)+ except ImportError:return selftry:fd = sys.stdin.fileno()old_attrs = termios.tcgetattr(fd)except (AttributeError, ValueError, OSError, termios.error):- return self # redirected / captured stdin (tests): nothing to hold+ return selfself._fd = fdself._old_attrs = old_attrsself._signals = _RestoreTtyOnSignals(fd, old_attrs)self._signals.__enter__()- # cbreak (not raw) keeps output post-processing while disabling echo/line- # buffering. It leaves ISIG on, so clear it and surface Ctrl-C as \x03 to- # the caller loop, which restores the tty itself.+ # cbreak leaves ISIG on, so clear it and surface Ctrl-C as \x03 for the caller loop to restore the tty.tty.setcbreak(fd, termios.TCSADRAIN)new_attrs = termios.tcgetattr(fd)new_attrs[3] &= ~termios.ISIG@@ -131,9 +127,7 @@ class _prompt_raw_mode:def _getch_posix() -> str: # pragma: no cover - needs a real tty- # Terminal already in cbreak+no-echo for the whole line (_prompt_raw_mode),- # so just read. Byte-at-a-time incremental decode so a multi-byte UTF-8 char- # straddling a read boundary isn't dropped.+ # Byte-at-a-time decode so a multi-byte UTF-8 char straddling a read boundary is not dropped.import codecsfd = sys.stdin.fileno()@@ -141,7 +135,7 @@ def _getch_posix() -> str: # pragma: no cover - needs a real ttywhile True:b = os.read(fd, 1)if not b:- return "" # stream EOF; caller raises EOFError+ return ""ch = decoder.decode(b)if ch:return ch@@ -164,11 +158,11 @@ def _read_password(prompt: str, *, out: "TextIO | None" = None) -> str:with _prompt_raw_mode():while True:key = _getch()- if key == "": # stream ended mid-line: abort, don't submit a partial+ if key == "":out.write("\n")out.flush()raise EOFError- for ch in key: # a paste can deliver several chars per read+ for ch in key:if ch in _SUBMITS:out.write("\n")out.flush()@@ -182,14 +176,14 @@ def _read_password(prompt: str, *, out: "TextIO | None" = None) -> str:out.write("\n")out.flush()raise EOFError- continue # ignore mid-input+ continueif ch in _BACKSPACES:if chars:chars.pop()out.write("\b \b")out.flush()continue- if ch < " ": # other control characters (tab, escape, ...)+ if ch < " ":continuechars.append(ch)out.write("*")diff --git a/studio/backend/cloudflare_tunnel.py b/studio/backend/cloudflare_tunnel.pyindex a9577559fc8..85265cc2c70 100644--- a/studio/backend/cloudflare_tunnel.py+++ b/studio/backend/cloudflare_tunnel.py@@ -24,25 +24,19 @@ import timefrom pathlib import Pathfrom typing import Callable, Optional, Tuple-# cloudflared logs the quick-tunnel URL; match only the URL so we do not depend-# on the surrounding wording, which Cloudflare may change. The negative lookahead-# drops cloudflared's own API host, which appears in failure lines such as-# failed to request quick Tunnel: Post "https://api.trycloudflare.com/tunnel"-# and must never be mistaken for a usable tunnel URL.+# Match only the URL; the negative lookahead drops cloudflared's own api.trycloudflare.com host from failure lines._URL_RE = re.compile(r"https://(?!api\.)[A-Za-z0-9-]+\.trycloudflare\.com")-# cloudflared logs this once per edge connection it establishes. Until at least-# one appears the quick-tunnel URL returns Cloudflare error 1033 (HTTP 530), so-# we wait for it before advertising the URL.+# Until an edge connection registers, the quick-tunnel URL returns Cloudflare error 1033 (HTTP 530)._REGISTERED_MARKER = "Registered tunnel connection"_RELEASE_BASE = "https://github.com/cloudflare/cloudflared/releases/latest/download"-_READY_TIMEOUT = 15.0 # seconds to wait for the URL + a registered edge connection-_DOWNLOAD_TIMEOUT = 60 # urlopen timeout for the one-time binary download+_READY_TIMEOUT = 15.0+_DOWNLOAD_TIMEOUT = 60-# A registered edge connection does not mean the hostname resolves yet, so the-# URL is fetched once before it is advertised.+# A registered edge connection does not mean the hostname resolves yet, so the URL is fetched once before it is+# advertised._PUBLIC_PROBE_PATH = "/api/health"_PUBLIC_PROBE_MARKER = "Unsloth UI Backend"# One deadline for DNS propagation + the health probe, bounding the startup stall.@@ -50,21 +44,18 @@ _PUBLIC_PROBE_TIMEOUT = 45.0_PUBLIC_PROBE_ATTEMPT_TIMEOUT = 5.0_PUBLIC_PROBE_RETRY_DELAY = 1.0-# Wait for the hostname via DoH first: an early OS lookup negative-caches the-# NXDOMAIN for up to 30 min.+# Resolve via DoH first: an early OS lookup negative-caches the NXDOMAIN for up to 30 min._DNS_POLL_DELAY = 2.0# Retry transient DoH failures, but give up fast when DoH is blocked outright._DNS_MAX_DOH_ERRORS = 3_DOH_URL = "https://cloudflare-dns.com/dns-query?name={host}&type=A"-# The resolver negative-caches a miss of its own, so a query sent before the-# record can exist blinds the poll for that cache's lifetime. Hold off first.+# The resolver negative-caches a miss of its own, so a query sent before the record can exist blinds the poll+# for that cache's lifetime. Hold off first._DNS_INITIAL_GRACE = 3.0# A blinded poll cannot recover, so bound its share of the shared deadline._DNS_WAIT_MAX = 20.0-# Cloudflare's edge routes by TLS SNI, so it serves the tunnel as soon as the-# connection registers -- before the hostname resolves anywhere. Probing there-# keeps DNS off the startup path entirely.+# Cloudflare's edge routes by TLS SNI, so it serves the tunnel before the hostname resolves anywhere._EDGE_HOST = "trycloudflare.com"_EDGE_PROBE_RETRY_DELAY = 0.5# Bound the wait so the hostname fallback keeps most of the shared deadline.@@ -147,7 +138,7 @@ def _asset_name() -> Optional[Tuple[str, bool]]:def _cache_path() -> Optional[Path]:"""studio_bin_root()/cloudflared(.exe), or None if the studio home is unresolvable."""try:- from utils.paths.storage_roots import studio_bin_root # lazy: backend-only import+ from utils.paths.storage_roots import studio_bin_rootexcept Exception:return Nonename = "cloudflared.exe" if sys.platform == "win32" else "cloudflared"@@ -293,8 +284,7 @@ def _edge_addresses() -> list:return addressesfor info in resolved:address = info[4][0]- # macOS reports the A records as IPv4-mapped under AF_INET6. The mapped- # and bare forms are one frontend, and probing it twice buys nothing.+ # macOS reports the A records as IPv4-mapped under AF_INET6; the mapped and bare forms are one frontend.if address.startswith("::ffff:"):address = address[len("::ffff:") :]if address not in addresses:@@ -371,12 +361,14 @@ def verify_public_url(url: str, timeout: float = _PUBLIC_PROBE_TIMEOUT) -> bool:if host:if _verify_through_edge(host, deadline):return True- # The edge never served the tunnel, so fall back to the hostname and pay- # the DoH wait that keeps an early OS lookup from caching the miss.+ # The edge never served the tunnel, so fall back to the hostname and pay the DoH wait that keeps an+ # early OS lookup from caching the miss._wait_for_dns(host, deadline)probe_url = f"{url.rstrip('/')}{_PUBLIC_PROBE_PATH}"while True:+ # Drain cloudflared's output: capture the first trycloudflare URL and the first edge-connection+ # registration, and keep draining so it never blocks on a full pipe.try:req = urllib.request.Request(probe_url, headers = {"User-Agent": "unsloth-studio"})with urllib.request.urlopen(req, timeout = _PUBLIC_PROBE_ATTEMPT_TIMEOUT) as response:@@ -422,8 +414,7 @@ class CloudflareTunnel:self.port = portself.binary = binaryself.origin_host = origin_host- # None lets cloudflared pick its default (quic, with its own http2- # fallback); set to "http2" to force it when quic is blocked.+ # None lets cloudflared pick quic; set "http2" to force it when quic is blocked.self.protocol = protocolself._proc: Optional[subprocess.Popen] = Noneself._lock = threading.Lock()@@ -448,15 +439,13 @@ class CloudflareTunnel:if self.protocol:cmd += ["--protocol", self.protocol]with self._lock:- # A stop() that landed before us (e.g. a shutdown in the caller's- # register->start window) marks the tunnel stopped; spawning now would- # orphan a process nobody owns, so refuse.+ # Refuse to spawn once a stop() has marked the tunnel stopped: it would orphan a process nobody owns.if self._stopped:return_set_studio_tunnel_runtime_active(self, True)try:- # PDEATHSIG binds to the forking thread, so spawning from the- # settings start worker would kill cloudflared when it returned.+ # PDEATHSIG binds to the forking thread, so spawning from the settings worker would kill cloudflared+ # when it returns.proc = _spawn_child(lambda: subprocess.Popen(cmd,@@ -474,9 +463,8 @@ class CloudflareTunnel:except Exception:_set_studio_tunnel_runtime_active(self, False)raise- # Adopted before the lock drops: a stop() that got in first would- # otherwise reap and forget it while nothing was tracked, and this- # would then record whatever inherited the pid.+ # Adopt before dropping the lock: a racing stop() would otherwise forget it and this would record+ # whatever inherited the pid._adopt_pid(proc.pid)self._proc = procthreading.Thread(@@ -484,12 +472,11 @@ class CloudflareTunnel:).start()def _reader(self, proc: subprocess.Popen) -> None:- # Drain cloudflared's output: capture the first trycloudflare URL and the- # first edge-connection registration, and keep draining so it never- # blocks on a full pipe.try:if proc.stdout is not None:for line in proc.stdout:+ # stdout closed -> cloudflared has exited. Record why, and unblock any waiters at once+ # instead of letting them wait out the full timeout.if self.url is None:match = _URL_RE.search(line)if match:@@ -501,8 +488,6 @@ class CloudflareTunnel:except Exception:passfinally:- # stdout closed -> cloudflared has exited. Record why, and unblock any- # waiters at once instead of letting them wait out the full timeout.if self.url is None:self.error = "cloudflared exited before emitting a tunnel URL"elif not self.ready:@@ -557,8 +542,8 @@ class CloudflareTunnel:_set_studio_tunnel_runtime_active(self, False)return Trueelse:- # Preserve both the stop handle and the fail-closed trust state when- # termination could not be confirmed. A later stop can retry.+ # Preserve both the stop handle and the fail-closed trust state when termination could not be+ # confirmed. A later stop can retry.with self._lock:if self._proc is None:self._proc = proc@@ -593,13 +578,12 @@ class CloudflareTunnel:return running-# Single serving process per Unsloth launch, so one module-level tunnel handle is-# enough; the lock guards the start/stop/shutdown races.+# Single serving process per Unsloth launch, so one module-level tunnel handle is enough; the lock guards the+# start/stop/shutdown races._active_tunnel: Optional[CloudflareTunnel] = None_active_lock = threading.Lock()_start_lock = threading.Lock()-# Latched by stop_studio_tunnel so a shutdown landing *between* a start's retry-# attempts aborts the loop instead of starting a tunnel nobody will ever stop.+# Latched by stop_studio_tunnel so a shutdown landing between retries cannot start a tunnel nobody will stop._shutdown_requested = False_tunnel_generation = 0_tunnel_lifecycle = 0@@ -873,9 +857,8 @@ def start_studio_tunnel(or _shutdown_requestedor _active_tunnel is not tunnel):- # Stop/shutdown landed while coming up. Detach AND tear- # down: returning this URL would leave a live public- # tunnel no later stop_studio_tunnel() can reach.+ # Detach and tear down: returning this URL would leave a live public tunnel no later+ # stop_studio_tunnel() can reach.aborted = Truewas_active = Falseif _active_tunnel is tunnel:@@ -922,9 +905,8 @@ def start_studio_tunnel(elif generation == _tunnel_generation and _active_tunnel is tunnel:if stopped:_active_tunnel = None- # Attempt 1's teardown parked the state at "stopping".- # Leaving it there for the http2 retry makes- # stop_studio_tunnel() early-return and stop nothing.+ # Reset from "stopping" before the http2 retry, or stop_studio_tunnel() early-returns and stops+ # nothing.if _tunnel_state == "stopping" and protocol is None:_tunnel_state = "starting"else:@@ -953,11 +935,11 @@ def stop_studio_tunnel(*, admission: Optional[Tuple[int, int]] = None) -> None:if admission is not None and admission != (_tunnel_lifecycle, _tunnel_generation):returnif _tunnel_state == "stopping":+ # Latch so an in-flight start_studio_tunnel won't start a fresh tunnel (e.g. its http2 retry) after+ # we have already torn down._shutdown_requested = True_tunnel_generation += 1return- # Latch so an in-flight start_studio_tunnel won't start a fresh tunnel- # (e.g. its http2 retry) after we have already torn down._shutdown_requested = True_tunnel_generation += 1stop_generation = _tunnel_generationdiff --git a/studio/backend/core/__init__.py b/studio/backend/core/__init__.pyindex 30ecb27f810..e92cbd1d950 100644--- a/studio/backend/core/__init__.py+++ b/studio/backend/core/__init__.py@@ -12,28 +12,23 @@ torch) before the version-activation code runs.import sysfrom pathlib import Path-# Add backend dir to sys.path so bare "from utils.*" imports work when core-# is imported as a package.+# Add backend dir to sys.path so bare "from utils.*" imports work when core is imported as a package._backend_dir = str(Path(__file__).resolve().parent.parent)if _backend_dir not in sys.path:sys.path.insert(0, _backend_dir)__all__ = [- # Inference"InferenceBackend","get_inference_backend",- # Training"get_training_backend","TrainingBackend","TrainingProgress",- # Config"ModelConfig","is_vision_model","scan_trained_models","scan_trained_loras","load_model_defaults","get_base_model_from_lora",- # Utils"format_and_template_dataset","normalize_path","is_local_path",@@ -50,7 +45,6 @@ __all__ = [def __getattr__(name):- # Inferenceif name in ("InferenceBackend", "get_inference_backend"):from .inference import InferenceBackend, get_inference_backend@@ -58,7 +52,6 @@ def __getattr__(name):globals()["get_inference_backend"] = get_inference_backendreturn globals()[name]- # Trainingif name in ("TrainingBackend", "get_training_backend", "TrainingProgress"):from .training import TrainingBackend, get_training_backend, TrainingProgress@@ -67,7 +60,6 @@ def __getattr__(name):globals()["TrainingProgress"] = TrainingProgressreturn globals()[name]- # Config (utils.models)if name in ("is_vision_model","ModelConfig",@@ -92,7 +84,6 @@ def __getattr__(name):globals()["get_base_model_from_lora"] = get_base_model_from_lorareturn globals()[name]- # Pathsif name in ("normalize_path", "is_local_path", "is_model_cached"):from utils.paths import normalize_path, is_local_path, is_model_cached@@ -101,7 +92,6 @@ def __getattr__(name):globals()["is_model_cached"] = is_model_cachedreturn globals()[name]- # Utilsif name in ("without_hf_auth", "format_error_message"):from utils.utils import without_hf_auth, format_error_message@@ -109,7 +99,6 @@ def __getattr__(name):globals()["format_error_message"] = format_error_messagereturn globals()[name]- # Hardwareif name in ("get_device","is_apple_silicon",@@ -135,7 +124,6 @@ def __getattr__(name):globals()["DeviceType"] = DeviceTypereturn globals()[name]- # Datasetsif name == "format_and_template_dataset":from utils.datasets import format_and_template_datasetglobals()["format_and_template_dataset"] = format_and_template_datasetdiff --git a/studio/backend/core/_torchao_stub.py b/studio/backend/core/_torchao_stub.pyindex 7dd758f854a..a915ec074d8 100644--- a/studio/backend/core/_torchao_stub.py+++ b/studio/backend/core/_torchao_stub.py@@ -28,9 +28,7 @@ from typing import Optional_STUB_SENTINEL = object()-# Metaclass for stub types so isinstance(x, StubClass) returns False instead of-# raising TypeError -- peft's lora/torchao.py does isinstance() against torchao-# types, which fails if those names resolve to stub modules rather than types.+# isinstance() against a stub module raises TypeError; peft's lora/torchao.py needs it to return False.class _StubTypeMeta(type):def __instancecheck__(cls, instance):return False@@ -122,6 +120,7 @@ _HIP_LINE_RE = re.compile(r"^hip\s*(?::[^=]*)?=\s*(.+?)\s*$", re.MULTILINE)def _version_is_rocm_tagged() -> Optional[bool]:"""Whether the installed wheel's version carries a rocm tag. None if unreadable."""+ # Neither on-disk signal was readable, so importing is the only way left.try:from importlib.metadata import versionreturn "rocm" in version("torch").lower()@@ -177,7 +176,6 @@ def torch_is_rocm() -> bool:verdict = _installed_torch_is_rocm()if verdict is not None:return verdict- # Neither on-disk signal was readable, so importing is the only way left.try:import torchexcept Exception:diff --git a/studio/backend/core/data_recipe/jobs/constants.py b/studio/backend/core/data_recipe/jobs/constants.pyindex 0045276e205..b919ea18f23 100644--- a/studio/backend/core/data_recipe/jobs/constants.py+++ b/studio/backend/core/data_recipe/jobs/constants.py@@ -3,7 +3,6 @@from __future__ import annotations-# stages parsed from data-designer logsSTAGE_CREATE = "create"STAGE_PREVIEW = "preview"STAGE_DAG = "dag"@@ -25,7 +24,6 @@ USAGE_RESET_STAGES = {STAGE_PROFILING,}-# job event types emitted by worker/managerEVENT_JOB_ENQUEUED = "job.enqueued"EVENT_JOB_STARTED = "job.started"EVENT_JOB_CANCELLING = "job.cancelling"diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.pyindex 135c9fccf64..45306e769d8 100644--- a/studio/backend/core/data_recipe/jobs/manager.py+++ b/studio/backend/core/data_recipe/jobs/manager.py@@ -186,7 +186,7 @@ class JobManager:proc.start()from utils.process_lifetime import adopt_pid- adopt_pid(proc.pid) # bind to parent lifetime (Windows job / sweep)+ adopt_pid(proc.pid)self._mp_q = mp_qself._proc = proc@@ -462,6 +462,7 @@ class JobManager:def _safe_handle_event(self, job: Job, event: dict) -> None:"""Apply one event, swallowing any handler error so the pump can't die."""+ # Worker exited: drain + finalize, guarded so an error can't strand the run "active".try:self._handle_event(job, event)except Exception:@@ -483,8 +484,7 @@ class JobManager:try:event = self._read_queue_with_timeout(mp_q, timeout_sec = 0.25)except Exception:- # If a read keeps raising after the worker died, finalize instead- # of spinning forever; only retry while the worker is still alive.+ # Only retry while the worker is alive; otherwise finalize instead of spinning forever.logger.exception("Data-recipe job pump: queue read failed; continuing")if proc.is_alive():time.sleep(0.1)@@ -498,7 +498,6 @@ class JobManager:if proc.is_alive():continue- # Worker exited: drain + finalize, guarded so an error can't strand the run "active".try:for e in self._drain_queue(mp_q):self._safe_handle_event(job, e)diff --git a/studio/backend/core/data_recipe/jobs/parse.py b/studio/backend/core/data_recipe/jobs/parse.pyindex 47f0f936d38..c4e26349b97 100644--- a/studio/backend/core/data_recipe/jobs/parse.py+++ b/studio/backend/core/data_recipe/jobs/parse.py@@ -50,8 +50,7 @@ class ParsedUpdate:_RE_SAMPLERS = re.compile(r"Preparing samplers to generate (?P<rows>\d+) records across (?P<cols>\d+) columns")-# Current data-designer phrasing for the same sampling stage (the line above is-# kept for older builds). No column count is reported here, so only rows is set.+# Newer data-designer phrasing for the same stage; no column count is reported here._RE_SAMPLING_SEED = re.compile(r"Sampling (?P<rows>\d+) records from .*?seed dataset", re.IGNORECASE)diff --git a/studio/backend/core/data_recipe/jobs/types.py b/studio/backend/core/data_recipe/jobs/types.pyindex c30606b2d12..8dc693fb0ef 100644--- a/studio/backend/core/data_recipe/jobs/types.py+++ b/studio/backend/core/data_recipe/jobs/types.py@@ -90,8 +90,7 @@ class Job:progress_columns_total: int | None = Nonesource_progress_estimated_total: int | None = Nonecompleted_columns: list[str] = field(default_factory = list)- # Id of the internal sk-unsloth-* API key minted for a local-model workflow.- # Revoked when the job ends so the key's window matches the run, not its 24h TTL.+ # Internal sk-unsloth-* key id; revoked when the job ends so its window matches the run, not the 24h TTL.internal_api_key_id: int | None = None_current_usage_model: str | None = None_in_usage_summary: bool = Falsediff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.pyindex 79943b3b708..9ffc8282e96 100644--- a/studio/backend/core/data_recipe/jobs/worker.py+++ b/studio/backend/core/data_recipe/jobs/worker.py@@ -82,7 +82,7 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])"""Subprocess entrypoint. Sends events to `event_queue`."""import os- os.environ["PYTHONWARNINGS"] = "ignore" # suppress C-level warnings before imports+ os.environ["PYTHONWARNINGS"] = "ignore"import warningsfrom loggers.config import LogConfig@@ -118,8 +118,8 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])builder = build_config_builder(recipe)designer = create_data_designer(recipe, artifact_path = str(_ARTIFACT_ROOT))- # DataDesigner resets root logging in __init__; attach the queue handler- # to the named loggers directly so parser events survive.+ # DataDesigner resets root logging in __init__; attach the queue handler to the named loggers so+ # parser events survive.handler = _QueueLogHandler(event_queue)handler.setLevel(logging.INFO)for logger_name in (@@ -193,8 +193,7 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:parquet_dir = base_dataset_path / "parquet-files"- # Counted, so a companion makes a single-batch job take the merge path and rewrite a- # dataset that needed none.+ # Counted, so a companion would make a single-batch job take the merge path and rewrite a dataset that needed none.parquet_files = drop_appledouble_metadata(sorted(parquet_dir.glob("*.parquet")))if len(parquet_files) <= 1:returndiff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.pyindex dc0deafdf5c..e6abd84be96 100644--- a/studio/backend/core/data_recipe/local_callable_validators.py+++ b/studio/backend/core/data_recipe/local_callable_validators.py@@ -186,7 +186,7 @@ def _build_oxc_validation_function(lang: str, validation_mode: str, code_shape:normalized_code_shape = code_shape if code_shape in _OXC_CODE_SHAPES else "auto"def _validator(df):- import pandas as pd # lazy import for local callable runtime+ import pandas as pdrow_count = int(len(df.index))if row_count == 0:@@ -235,8 +235,7 @@ def _run_oxc_batch(# The wrapper bounds oxlint with this; a kill here would leave the grandchild."timeout_ms": int(_OXC_TIMEOUT_S * 1000),}- # Resolve a usable Node (system or the isolated install, which is not on the- # user's PATH); a bare "node" would fail for isolated-Node users.+ # Resolve a usable Node: a bare "node" fails for isolated-Node users, whose install is not on PATH.node_executable = resolve_node_executable()if not node_executable:return _fallback_results(diff --git a/studio/backend/core/data_recipe/oxc-validator/validate.mjs b/studio/backend/core/data_recipe/oxc-validator/validate.mjsindex 9eb6565d7dd..d51921fca13 100644--- a/studio/backend/core/data_recipe/oxc-validator/validate.mjs+++ b/studio/backend/core/data_recipe/oxc-validator/validate.mjs@@ -22,11 +22,8 @@ const SNIPPET_PREFIX = "(() => {\n";const SNIPPET_SUFFIX = "\n})();\nexport {};\n";const OXLINT_SUPPRESSED_RULES = ["no-unused-vars", "no-new-array"];// The caller kills only this wrapper, so a wedged oxlint has to die here or it is orphaned.-// It gets the caller's budget minus what this process spent, on the same monotonic basis.const OXLINT_DEFAULT_BUDGET_MS = 30_000;-// Covers the spawn, which precedes this process, plus its own wind-down.const OXLINT_BUDGET_MARGIN_MS = 2_000;-// Below this, starting oxlint would only hand the caller's kill an orphan.const OXLINT_MIN_TIMEOUT_MS = 1_000;const TOOL_DIR = dirname(fileURLToPath(import.meta.url));diff --git a/studio/backend/core/data_recipe/service.py b/studio/backend/core/data_recipe/service.pyindex 3c2545bf19a..553c102d86b 100644--- a/studio/backend/core/data_recipe/service.py+++ b/studio/backend/core/data_recipe/service.py@@ -181,8 +181,7 @@ def recipe_has_stdio_mcp(recipe: dict[str, Any]) -> bool:def build_mcp_providers(recipe: dict[str, Any]) -> list:from data_designer.config.mcp import LocalStdioMCPProvider, MCPProvider # pyright: ignore[reportMissingImports]- # Same gate as the chat MCP path: stdio providers spawn a local subprocess,- # so build them only when this host allows it (desktop / explicit opt-in).+ # Stdio providers spawn a local subprocess, so build them only when this host allows it.from core.inference.mcp_client import stdio_mcp_enabledstdio_allowed = stdio_mcp_enabled()@@ -270,8 +269,8 @@ def build_config_builder(recipe: dict[str, Any]):specs = oxc_local_callable_specs,)- # DataDesignerConfigBuilder.from_config currently skips processors.- # Re-attach so drop_columns/schema_transform survive the API payload.+ # DataDesignerConfigBuilder.from_config skips processors; re-attach so drop_columns/schema_transform+ # survive the API payload.for processor in recipe_core.get("processors") or []:if not isinstance(processor, dict):continue@@ -292,16 +291,15 @@ def create_data_designer(recipe: dict[str, Any], *, artifact_path: str | None =from data_designer.interface.data_designer import DataDesigner # pyright: ignore[reportMissingImports]if artifact_path is None:- # DataDesigner defaults to cwd/artifacts; packaged Unsloth can run with- # cwd=/, so keep default callers on Unsloth's writable recipe artifact root.+ # DataDesigner defaults to cwd/artifacts and packaged Unsloth can run with cwd=/, so pin the+ # writable recipe artifact root.artifact_path = str(recipe_datasets_root())recipe = _strip_frontend_model_config_metadata(recipe)model_providers = build_model_providers(recipe)_validate_recipe_runtime_support(recipe, model_providers)- # DataDesigner requires >=1 model provider even with no LLM columns; stub- # one so sampler/expression-only recipes run without a real provider.+ # DataDesigner requires >=1 model provider even with no LLM columns.if not model_providers:from data_designer.config.models import ModelProvider # pyright: ignore[reportMissingImports]model_providers = [diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.pyindex cebf669a15c..7ed67132f6b 100644--- a/studio/backend/core/export/export.py+++ b/studio/backend/core/export/export.py@@ -13,12 +13,12 @@ import contextlibfrom pathlib import Pathfrom typing import Optional, Tuple, List-# unsloth imports torch on non-MLX hosts, so a --no-torch install raises here. Stay importable-# (null the classes) so exports return a clean "PyTorch is not installed" error.+# unsloth imports torch, so a --no-torch install raises here; stay importable and return a clean+# "PyTorch is not installed" error.try:from unsloth import FastLanguageModel, FastVisionModel, _IS_MLX_UNSLOTH_IMPORT_ERROR = None-except Exception as _unsloth_exc: # ImportError (e.g. missing torch) or a broken native load+except Exception as _unsloth_exc:FastLanguageModel = NoneFastVisionModel = None_IS_MLX = False@@ -47,7 +47,7 @@ if not _IS_MLX:from peft import PeftModel, PeftModelForCausalLMfrom transformers.modeling_utils import PushToHubMixinimport torch- except Exception as _torch_exc: # ImportError, or a broken native torch load+ except Exception as _torch_exc:_TORCH_IMPORT_ERROR = _torch_exclogger = get_logger(__name__)@@ -102,8 +102,7 @@ def _multi_gpu_device_map_kwargs() -> dict:device_map = get_device_map(None)else:return {}- # Every sharding answer `get_device_map` gives; a "balanced"-only whitelist- # dropped the map the moment CUDA began asking for a planned one.+ # A "balanced"-only whitelist dropped the map the moment CUDA began asking for a planned one.if device_map in ("balanced", "unsloth", "unsloth_balanced"):return {"device_map": device_map}except Exception as exc:@@ -259,7 +258,7 @@ def _materialized_imatrix_path(model_dir, imatrix_file):(see `_is_imatrix`): a basename match would suppress a real output of the same name."""if imatrix_file is True:- name = "imatrix_unsloth.gguf" # the upstream imatrix_unsloth.gguf_file, renamed+ name = "imatrix_unsloth.gguf"elif isinstance(imatrix_file, (str, os.PathLike)):base = os.path.basename(os.fspath(imatrix_file))if not base.endswith(".gguf_file"):@@ -333,7 +332,7 @@ def _hf_offline(timeout = 3):):return Trueif os.environ.get("UNSLOTH_OFFLINE_PROBE", "1").strip().lower() in {"0", "false", "no", "off"}:- return False # probe disabled -> assume online; loads still pass local_files_only on env+ return False# Shared bounded, proxy-aware probe (also used by the export worker before version activation).from utils.transformers_version import hf_endpoint_unreachable@@ -344,7 +343,6 @@ def _hf_offline(timeout = 3):return False-# Reuse Unsloth's lock-guarded forced-offline context; no-op fallback if it moves.try:from unsloth.models.loader_utils import _force_hf_offlineexcept Exception:@@ -488,7 +486,6 @@ class ExportBackend:checkpoint_path_obj = Path(checkpoint_path)- # Model identity for type detectionadapter_config = checkpoint_path_obj / "adapter_config.json"base_model = Noneif adapter_config.exists():@@ -508,9 +505,9 @@ class ExportBackend:else _device_map_override)- # Run the type-detection probes in the forced-offline window (else a gated- # base 404s); it covers is_vision_model's Hub reads + the transformers-5- # subprocess, and local_files_only makes detect_audio_type's requests.get skip.+ # Run the type-detection probes inside the forced-offline window, else a gated base 404s: it+ # covers is_vision_model's Hub reads, and local_files_only makes detect_audio_type's requests.get+ # skip.with _offline_window_if(local_files_only):self._audio_type = detect_audio_type(model_id, hf_token = token, local_files_only = local_files_only@@ -604,7 +601,7 @@ class ExportBackend:local_files_only = local_files_only,**_device_map_kw,)- tokenizer = processor # vision: processor acts as tokenizer+ tokenizer = processorelse:logger.info("Loading as text model...")@@ -619,15 +616,14 @@ class ExportBackend:**_device_map_kw,)- # Only when we asked for the multi-GPU map: a single-GPU host has no second- # placement to retry on, so leave its behaviour untouched.+ # Only for the multi-GPU map: a single-GPU host has no second placement to retry on._offloaded = _cpu_offloaded_modules(model) if _device_map_kw else 0if _device_map_override is None and _offloaded:del modelraise _CpuSpillRetry(f"{_offloaded} module(s) offloaded to CPU/disk")if _IS_MLX:- # MLX doesn't use PeftModel — detect LoRA via adapter_config.json+ # MLX doesn't use PeftModel - detect LoRA via adapter_config.jsonself.is_peft = adapter_config.exists()else:self.is_peft = isinstance(model, (PeftModel, PeftModelForCausalLM))@@ -654,11 +650,8 @@ class ExportBackend:return True, f"Loaded {model_type} model{peft_info} successfully"except Exception as e:- # Sharding is an optimisation, never a requirement. "balanced" budgets from the- # free memory read BEFORE this process opens a CUDA context on each GPU, so when- # a training or chat job already owns the others the shard can OOM, or spill to- # CPU and be refused by bitsandbytes, where the old single-device load succeeded.- # Fall back once before giving up.+ # "balanced" budgets from free memory read BEFORE this process opens a CUDA context, so the shard+ # can OOM when another job owns the other GPUs; fall back once.if (_device_map_override is Noneand (@@ -669,8 +662,8 @@ class ExportBackend:)and _multi_gpu_device_map_kwargs()):- # Retry outside this block: the live traceback pins the half-built model's- # frames, so an in-block retry inherits the exhausted device.+ # Retry outside this block: the live traceback pins the half-built model's frames, so an in-block+ # retry inherits the exhausted device.retry_reason = str(e)else:logger.error(f"Error loading checkpoint: {e}")@@ -690,9 +683,8 @@ class ExportBackend:load_in_4bit = load_in_4bit,trust_remote_code = trust_remote_code,hf_token = hf_token,- # Named, not omitted: an omitted map is unsloth's DEFAULT_DEVICE_MAP, which- # `requested_device_map` upgrades back to the planner, re-running the- # placement that just failed. A typed "sequential" passes through.+ # Name the map: an omitted one is unsloth's DEFAULT_DEVICE_MAP, which requested_device_map+ # upgrades back to the planner, re-running the placement that just failed._device_map_override = {"device_map": "sequential"},)@@ -744,13 +736,11 @@ class ExportBackend:if not self.current_model or not self.current_tokenizer:return False, "No model loaded. Please select a checkpoint first.", None- # Merged export works for PEFT adapters and non-PEFT Local/HF base models alike- # (save_pretrained_merged is a no-op merge that just saves the base).+ # save_pretrained_merged is a no-op merge for non-PEFT base models, so one path covers both.output_path: Optional[str] = None- # Quantized formats save to a sibling "<dir>-<suffix>". Two backends: compressed-tensors- # (llm-compressor, NVIDIA-only) and portable torchao FP8/INT8 (device-agnostic). The alias- # comes from `compressed_method` (the "all formats" dropdown) or the `format_type` label.+ # Two backends: compressed-tensors (llm-compressor, NVIDIA-only) and portable torchao FP8/INT8.+ # The alias comes from compressed_method (the "all formats" dropdown) or the format_type label._LABEL_TO_ALIAS = {"FP8 (compressed-tensors)": "fp8","NVFP4 (compressed-tensors)": "nvfp4",@@ -798,9 +788,8 @@ class ExportBackend:)import unsloth.save as _us- # Prefer the llm-compressor-main shadow (transformers 5.x): it quantizes newer models- # (Qwen3.5, Gemma-4, ...) the shipped 0.10.x cannot. Route all compressed exports- # through it when available; else fall back to the workspace 0.10.x path below.+ # Prefer the llm-compressor-main shadow (transformers 5.x): the shipped 0.10.x cannot quantize newer+ # models._shadow_pp = Nonetry:from utils.transformers_version import llmcompressor_shadow_pythonpath@@ -810,8 +799,7 @@ class ExportBackend:if _shadow_pp:os.environ[_us._COMPRESSED_QUANTIZE_PYTHONPATH_ENV] = _shadow_ppelse:- # No shadow (disabled/offline/failed): the workspace 0.10.x cannot exceed its- # transformers ceiling, so fail fast for sidecar models; default-tier still works.+ # The workspace 0.10.x cannot exceed its transformers ceiling, so fail fast for sidecar models.os.environ.pop(_us._COMPRESSED_QUANTIZE_PYTHONPATH_ENV, None)_exceeds, _tf_ver = _us._transformers_exceeds_llm_compressor_ceiling()if _exceeds:@@ -908,8 +896,8 @@ class ExportBackend:private = private,)elif (is_compressed or is_torchao) and output_path and Path(output_path).is_dir():- # Already built in output_path; upload it directly instead of re-running the- # expensive quantization that push_to_hub_merged(save_method=...) would redo.+ # Upload the artifact already built in output_path; push_to_hub_merged(save_method=...) would+ # redo the expensive quantization.hf_api = HfApi(token = hf_token)repo_id = PushToHubMixin._create_repo(PushToHubMixin,@@ -987,8 +975,7 @@ class ExportBackend:ensure_dir(Path(save_directory))if _IS_MLX:- # MLX: save_pretrained_merged handles non-LoRA models too- # (fuse() is a no-op without LoRA layers)+ # fuse() is a no-op without LoRA layers, so this handles non-LoRA models too.self.current_model.save_pretrained_merged(save_directory,self.current_tokenizer,@@ -998,7 +985,6 @@ class ExportBackend:self.current_model.save_pretrained(save_directory)self.current_tokenizer.save_pretrained(save_directory)- # Write export metadata so the Chat page can identify the base modelself._write_export_metadata(save_directory)logger.info(f"Model saved successfully to {save_directory}")output_path = str(Path(save_directory).resolve())@@ -1037,7 +1023,6 @@ class ExportBackend:private = private,)else:- # Base model name from request or model configbase_model = (base_model_id or self.current_model.config._name_or_path or "unknown")@@ -1118,8 +1103,7 @@ class ExportBackend:if not self.current_model or not self.current_tokenizer:return False, "No model loaded. Please select a checkpoint first.", None- # Only forward imatrix_file to an unsloth build that accepts it, else older builds raise- # an unexpected-keyword error even for a plain no-imatrix export.+ # Older unsloth builds raise on an unexpected imatrix_file kwarg even for a plain no-imatrix export.if imatrix_file and not _imatrix_export_supported(self.current_model.save_pretrained_gguf):return (False,@@ -1144,8 +1128,8 @@ class ExportBackend:None,)shard_kw = {"gguf_shard_size": gguf_shard_size} if gguf_shard_size is not None else {}- # Resolution reads a Hub repo, so the local save needs the token too -- kept out of- # imatrix_kw, which the push below shares and already names token= itself.+ # Resolution reads a Hub repo, so the local save needs the token; kept out of imatrix_kw, which the+ # push shares and names token= itself.local_token_kw = ({"token": hf_token}if imatrix_file@@ -1165,8 +1149,8 @@ class ExportBackend:quant_methods = ["q4_k_m"]quant_method = quant_methods if len(quant_methods) > 1 else quant_methods[0]- # Pin convert_hf_to_gguf.py to setup.sh's tagged llama.cpp ref so it- # can't drift past the pinned llama-quantize binary's gguf API.+ # Pin convert_hf_to_gguf.py to setup.sh's llama.cpp ref so it cannot drift past the pinned llama-+ # quantize gguf API.global _LLAMA_CPP_SCRIPTS_WARNING_EMITTEDtry:from unsloth_zoo.llama_cpp import (@@ -1175,8 +1159,8 @@ class ExportBackend:)os.environ.setdefault("UNSLOTH_LLAMA_CPP_SCRIPTS_DIR", LLAMA_CPP_DEFAULT_DIR)except Exception:- # Not just ImportError: a half-built unsloth_zoo raises RuntimeError or- # AttributeError, and this pin is an optimisation, not worth failing an export.+ # Not just ImportError: a half-built unsloth_zoo raises RuntimeError or AttributeError, and this pin+ # is only an optimisation.if not _LLAMA_CPP_SCRIPTS_WARNING_EMITTED:logger.warning("Unsloth: installed unsloth_zoo does not honor "@@ -1202,7 +1186,7 @@ class ExportBackend:model_tmp_root = tempfile.mkdtemp(prefix = "_tmp_model_", dir = abs_save_dir)model_tmp_path = Path(model_tmp_root)_model_tmp = os.path.join(model_tmp_root, "model")- # Needed by the cleanup below too, so resolve it before anything can raise.+ # Resolve before anything can raise; the cleanup below needs it too.imatrix_path = _materialized_imatrix_path(_model_tmp, imatrix_file)try:result = self.current_model.save_pretrained_gguf(@@ -1247,16 +1231,17 @@ class ExportBackend:"GGUF conversion produced a symlinked Modelfile, "f"refusing to relocate it: {modelfile}")- # Optional artifact: unsloth generates it best-effort, so a locked or- # read-only destination must not fail an export whose GGUFs all landed.+ # Optional artifact: a locked or read-only destination must not fail an export whose GGUFs all+ # landed.+ # Optional artifact: unsloth generates it best-effort, so a locked or read-only+ # destination must not fail an export whose GGUFs all landed.try:shutil.move(str(modelfile), os.path.join(abs_save_dir, "Modelfile"))logger.info(f"Relocated Modelfile → {abs_save_dir}/")except OSError as exception:logger.warning(f"Could not relocate the Modelfile: {exception}")finally:- # Preserve any GGUF that could not be relocated. The imatrix is an input,- # so counting it would retain the merged checkpoint on every such export.+ # The imatrix is an input, so counting it would retain the merged checkpoint on every such export.unrelocated = []if model_tmp_path.is_dir():unrelocated = sorted(@@ -1272,9 +1257,8 @@ class ExportBackend:else:shutil.rmtree(model_tmp_root, ignore_errors = True)- # iterdir, not glob.glob: glob hides dot-leading names, so an empty- # model stem's ".Q4_K_M.gguf" got reported as "(none)". This list is the- # success gate, so a leftover companion would report a run that wrote nothing.+ # iterdir, not glob.glob: glob hides dot-leading names, so an empty model stem's ".Q4_K_M.gguf"+ # read as "(none)". This list is the success gate.final_ggufs = sorted(str(p)for p in drop_appledouble_metadata(list(Path(abs_save_dir).iterdir()))@@ -1287,7 +1271,6 @@ class ExportBackend:)if not final_ggufs:# Reporting success over an empty directory is what hid #7897.- # The owned temp root is already gone: the finally above drops it.return (False,f"GGUF conversion reported success but wrote no .gguf file to "@@ -1373,11 +1356,8 @@ class ExportBackend:"Use the safetensors adapter instead.",None,)- # llama.cpp's convert_lora_to_gguf.py has no concept of DoRA's- # lora_magnitude_vector tensors: it only reads the standard- # lora_A/lora_B delta, so exporting a DoRA adapter would silently- # drop the magnitude rescaling and produce a GGUF LoRA file that- # loads fine but no longer matches the trained model.+ # convert_lora_to_gguf.py reads only the standard lora_A/lora_B delta and ignores DoRA's+ # lora_magnitude_vector, so a DoRA export silently drops magnitude rescaling._peft_config = getattr(self.current_model, "peft_config", {}).get("default")if getattr(_peft_config, "use_dora", False):return (@@ -1396,8 +1376,7 @@ class ExportBackend:f"Choose one of {', '.join(_GGUF_LORA_OUTTYPES)}.",None,)- # getattr so an older build without save_pretrained_gguf returns a clean message- # instead of an AttributeError (a generic 500).+ # getattr so an older build without save_pretrained_gguf returns a clean message instead of a generic 500._save_gguf_fn = getattr(self.current_model, "save_pretrained_gguf", None)if _save_gguf_fn is None or not _supports_kwarg(_save_gguf_fn, "save_method"):return (@@ -1437,7 +1416,6 @@ class ExportBackend:"\n ".join(os.path.basename(f) for f in final_ggufs) or "(none)",)elif _IS_MLX:- # MLX: save adapters.safetensors + tokenizer filesself.current_model.save_lora_adapters(save_directory)self.current_tokenizer.save_pretrained(save_directory)else:@@ -1457,8 +1435,7 @@ class ExportBackend:logger.info(f"Pushing LoRA adapter to Hub: {repo_id}")if gguf:- # Upload the locally-built GGUF folder; needs a local save_directory so the- # conversion is not re-run.+ # Needs a local save_directory so the conversion is not re-run.if not (output_path and Path(output_path).is_dir()):return (False,@@ -1499,7 +1476,6 @@ class ExportBackend:return False, f"Adapter export failed: {str(e)}", None-# Global export backend instance_export_backend = Nonediff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.pyindex e8f258ebe84..d357b1e0998 100644--- a/studio/backend/core/export/orchestrator.py+++ b/studio/backend/core/export/orchestrator.py@@ -58,20 +58,16 @@ class ExportOrchestrator:self._log_lock = threading.Lock()# Monotonic seq, never reset, so SSE clients have a stable cursor across clear_logs().self._log_seq: int = 0- # _log_seq snapshot at the current run's start; SSE defaults its cursor here so a- # late-connecting client still sees the full run. Current run has seq > this.+ # SSE defaults its cursor here so a late-connecting client still sees the full run.self._run_start_seq: int = 0# True while an export op runs; SSE ends the stream 1s after this flips False.self._export_active: bool = False- # Set by cancel_export(); reset when a new load/export run starts. Lets the- # caller distinguish a user cancel from a genuine subprocess crash.+ # Set by cancel_export(); reset when a new load/export run starts. Lets the caller distinguish a+ # user cancel from a genuine subprocess crash.self._cancel_requested: bool = False- # Last finished operation, so a client whose blocking POST was cut off by a- # Cloudflare tunnel timeout (524 at ~100s, while the op runs for minutes) can- # poll /api/export/status and still learn the real outcome. Guarded by- # _op_lock. `_op_seq` is a monotonic counter the client uses as a baseline to- # tell "my op finished" (seq grew) from a stale previous result.+ # Kept so a client whose blocking POST was cut by a tunnel 524 can poll /api/export/status; _op_seq+ # is the monotonic baseline for "my op finished".self._op_lock = threading.Lock()self._op_seq: int = 0self._active_op_kind: Optional[str] = None@@ -81,8 +77,6 @@ class ExportOrchestrator:logger.info("ExportOrchestrator initialized (subprocess mode)")# ------------------------------------------------------------------- # Live log capture helpers- # ------------------------------------------------------------------def _append_log(self, entry: Dict[str, Any]) -> None:"""Append a worker log line to the buffer, stamped with a monotonic seq."""@@ -204,22 +198,18 @@ class ExportOrchestrator:return True# ------------------------------------------------------------------- # Subprocess lifecycle- # ------------------------------------------------------------------def _spawn_subprocess(self, config: dict) -> None:"""Spawn a new export subprocess."""- # Last-resort recheck for spawns outside an active op. Inside an op, _export_active is set and- # load_checkpoint already rechecked, so a reservation here is an install about to observe- # is_export_active() and abort; raising would kill this export for an install that never proceeds.+ # Inside an op a reservation is an install about to abort on is_export_active(), so raising here+ # would kill the export for an install that never proceeds.from utils.transformers_version import sidecar_swap_in_progressfrom utils.transformers_version import sidecar_swap_kind_swap_kind = sidecar_swap_kind()- # Inside an active op an INSTALL reservation is about to abort on the- # is_export_active check, but a lazy REPAIR has no such check and can be- # rebuilding the sidecar right now, so it must always refuse the spawn.+ # An INSTALL reservation aborts on the is_export_active check, but a lazy REPAIR has none and may+ # be rebuilding the sidecar right now, so always refuse the spawn.if _swap_kind == "repair" or (_swap_kind is not None and not self._export_active):from utils.transformers_version import SidecarSwapInProgressraise SidecarSwapInProgress(@@ -254,7 +244,7 @@ class ExportOrchestrator:self._proc.start()from utils.process_lifetime import adopt_pid- adopt_pid(self._proc.pid) # bind to parent lifetime (Windows job / sweep)+ adopt_pid(self._proc.pid)logger.info("Export subprocess started (pid=%s)", self._proc.pid)def _shutdown_subprocess(self, timeout: float = 10.0) -> bool:@@ -298,8 +288,7 @@ class ExportOrchestrator:passif self._proc is not None and self._proc.is_alive():- # Survived SIGKILL (uninterruptible syscall): keep the handle so callers- # and the pre-swap guard see a live worker rather than a nulled one.+ # Survived SIGKILL: keep the handle so callers and the pre-swap guard see a live worker.logger.error("Export subprocess still alive after terminate/kill; ""preserving its handle for the pre-swap liveness check"@@ -321,8 +310,6 @@ class ExportOrchestrator:return self._proc is not None and self._proc.is_alive()# ------------------------------------------------------------------- # Queue helpers- # ------------------------------------------------------------------def _send_cmd(self, cmd: dict) -> None:"""Send a command to the subprocess."""@@ -381,8 +368,8 @@ class ExportOrchestrator:if rtype == "status":message = resp.get("message", "")- # One structured export_progress line per phase (consolidated in the- # server log, like training/download progress); also shown live.+ # One structured export_progress line per phase (consolidated in the server log, like+ # training/download progress); also shown live.if message:logger.info("export_progress", phase = message)self._append_log(@@ -394,7 +381,7 @@ class ExportOrchestrator:)continue- # Other response types during wait — skip.+ # Other response types during wait - skip.logger.debug("Skipping response type '%s' while waiting for '%s'",rtype,@@ -417,8 +404,6 @@ class ExportOrchestrator:return events# ------------------------------------------------------------------- # Public API — same interface as ExportBackend- # ------------------------------------------------------------------def load_checkpoint(self,@@ -452,9 +437,8 @@ class ExportOrchestrator:self._export_active = Trueop_success, op_message = False, ""try:- # Handshake with the sidecar install route: _export_active is set above, so either this- # recheck refuses BEFORE tearing down the old worker (keeping the loaded checkpoint), or- # the install sees is_export_active() and 409s. The spawn-time recheck stays as a last resort.+ # Handshake with the sidecar install route (see load_checkpoint): either this recheck refuses+ # before tearing down the old worker, or the install sees is_export_active() and 409s.from utils.transformers_version import sidecar_swap_in_progressif sidecar_swap_in_progress():@@ -467,10 +451,9 @@ class ExportOrchestrator:# Always kill any existing subprocess and spawn fresh.if self._ensure_subprocess_alive():if self._shutdown_subprocess() is False:- # Survivor still holds GPU memory (a wedged CUDA syscall outliving- # SIGKILL); its handle is kept so is_worker_alive() and the pre-swap- # guard still see it. Do not spawn a second worker over it -- fail so- # the load can retry once it exits.+ # A survivor still holds GPU memory (a wedged CUDA syscall outliving SIGKILL) and its handle is+ # kept so is_worker_alive() still sees it, so do not spawn a second worker over it; fail so the+ # load can retry once it exits.op_message = ("The current export worker did not exit and still holds GPU ""memory; not starting a new checkpoint load over it. Retry shortly."@@ -483,9 +466,8 @@ class ExportOrchestrator:try:self._spawn_subprocess(sub_config)except Exception:- # The old worker is already gone; a stale current_checkpoint- # would make the Export page claim a loaded checkpoint that- # the next op then fails on with "no subprocess running".+ # A stale current_checkpoint would make the Export page claim a loaded checkpoint the next op then+ # fails on.self.current_checkpoint = Noneself.is_vision = Falseself.is_peft = False@@ -637,9 +619,7 @@ class ExportOrchestrator:self._export_active = Trueop_success, op_message, op_output_path = False, "", Nonetry:- # Handshake with the sidecar install route (see load_checkpoint): _export_active is set- # above, so this recheck refuses before the command is sent, or the install sees the active- # op and 409s. Without it, an install would block in cleanup_memory behind a long export op.+ # Recheck before sending, else an install blocks in cleanup_memory behind a long export op.from utils.transformers_version import sidecar_swap_in_progressif sidecar_swap_in_progress():@@ -652,8 +632,8 @@ class ExportOrchestrator:cmd = {"type": "export", "export_type": export_type, **params}try:self._send_cmd(cmd)- # GGUF for 30B+ models can take 30+ min per quant; a multi-quant list runs them- # all in one op off a single merge, so scale the timeout by the quant count.+ # GGUF for a 30B+ model can take 30+ min per quant, and a multi-quant list runs every quant in one+ # op off a single merge, so scale the timeout by quant count._qm = params.get("quantization_method")_n = len(_qm) if isinstance(_qm, (list, tuple)) and _qm else 1resp = self._wait_response(@@ -692,7 +672,7 @@ class ExportOrchestrator:except RuntimeError:success = False- # Shut down subprocess after cleanup — no model loaded.+ # Shut down subprocess after cleanup - no model loaded.self._shutdown_subprocess()self.current_checkpoint = Nonediff --git a/studio/backend/core/rag/captioner.py b/studio/backend/core/rag/captioner.pyindex ccc150c3c76..b6c301c03c2 100644--- a/studio/backend/core/rag/captioner.py+++ b/studio/backend/core/rag/captioner.py@@ -54,7 +54,7 @@ def _collapse_runaway(for line in text.splitlines():key = line.strip()if not key:- if prev == "": # collapse runs of blank lines to a single separator+ if prev == "":continueprev = ""out.append("")diff --git a/studio/backend/core/rag/chunking.py b/studio/backend/core/rag/chunking.pyindex c64acb4c60f..cbb4530b1ad 100644--- a/studio/backend/core/rag/chunking.py+++ b/studio/backend/core/rag/chunking.py@@ -36,7 +36,7 @@ def _split(text: str, seps: tuple[str, ...], max_tokens: int, count: TokenCounteparts = list(text) if sep == "" else text.split(sep)if len(parts) <= 1:continue- if sep: # re-attach the separator+ if sep:parts = [p + sep for p in parts[:-1]] + parts[-1:]out: list[str] = []for p in parts:@@ -74,8 +74,8 @@ def _merge(pt = count(piece)if buf and buf_tok + pt > max_tokens:_flush()- # Bound the carry so carry + this piece fits max_tokens; else a full- # overlap before a near-max piece overflows the embedder.+ # Bound the carry so carry + this piece fits max_tokens; else a full overlap before a near-max piece+ # overflows the embedder.carry_budget = min(overlap, max(0, max_tokens - pt))carry, carry_starts, run = [], [], 0for prev, prev_start in zip(reversed(buf), reversed(buf_starts)):diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.pyindex 169bc5dd010..cac9c4201ff 100644--- a/studio/backend/core/rag/config.py+++ b/studio/backend/core/rag/config.py@@ -10,8 +10,7 @@ import reDEFAULT_EMBEDDING_MODEL = "unsloth/bge-small-en-v1.5"EMBEDDING_MODEL = os.environ.get("RAG_EMBEDDING_MODEL", DEFAULT_EMBEDDING_MODEL)-# Under bge's 512 limit, leaving headroom for the 2 special tokens (else overflow:-# llama-server 500s, ST truncates). Keep <= embedder_max - ~12.+# Keep <= embedder_max - ~12: bge's 512 limit plus 2 special tokens (llama-server 500s on overflow, ST truncates).CHUNK_TOKENS = int(os.environ.get("RAG_CHUNK_TOKENS", "500"))CHUNK_OVERLAP = int(os.environ.get("RAG_CHUNK_OVERLAP", "64"))TOP_K_LEXICAL = int(os.environ.get("RAG_TOP_K_LEXICAL", "30"))@@ -19,80 +18,55 @@ TOP_K_DENSE = int(os.environ.get("RAG_TOP_K_DENSE", "30"))TOP_K_HYBRID = int(os.environ.get("RAG_TOP_K_HYBRID", "10"))RRF_K = int(os.environ.get("RAG_RRF_K", "60"))-# Whole-document context: a thread-attached file under the token budget is injected-# in full (every chunk, in order) instead of top-K retrieval; above it, use retrieval.+# Whole-document context: a file under this budget is injected in full instead of top-K retrieval.THREAD_WHOLE_DOC = os.environ.get("RAG_THREAD_WHOLE_DOC", "1") == "1"WHOLE_DOC_MAX_TOKENS = int(os.environ.get("RAG_WHOLE_DOC_MAX_TOKENS", "6000"))-# Conversation archive: turns evicted by the rolling context window go to a per-thread-# searchable scope and the relevant ones are recalled on the turn that evicted them. Off,-# evicted turns are simply dropped again, and the recall reserve is not taken. Only applies-# once the window evicts, which is itself opt-in per request via-# context_overflow="truncate_oldest".-#-# It does NOT turn the rolling window back into what it was before: the compaction headroom-# and the sticky boundary belong to the window, not to the archive, and have their own knob-# (ROLLING_COMPACTION_HEADROOM_RATIO). Gating them here instead would make a host without-# sqlite-vec silently compact differently.+# Off, evicted turns are simply dropped and the recall reserve is not taken. Only applies once the+# window evicts, itself opt-in per request via context_overflow="truncate_oldest"; the compaction+# headroom and sticky boundary belong to the window (ROLLING_COMPACTION_HEADROOM_RATIO), not here.CONVERSATION_ARCHIVE = os.environ.get("RAG_CONVERSATION_ARCHIVE", "1") == "1"CONVERSATION_ARCHIVE_TOP_K = int(os.environ.get("RAG_CONVERSATION_ARCHIVE_TOP_K", "4"))-# Room held back during the fit for the turns recalled straight after it. Sized to-# CONVERSATION_ARCHIVE_TOP_K * CHUNK_TOKENS with slack for the wrapper text.+# Sized to CONVERSATION_ARCHIVE_TOP_K * CHUNK_TOKENS with slack for the wrapper text.CONVERSATION_RECALL_RESERVE_TOKENS = int(os.environ.get("RAG_CONVERSATION_RECALL_RESERVE_TOKENS", "2048"))-# Shape the archive's lexical query: require identifier-like tokens first, drop function-# words from the fallback. Off restores the plain OR-of-every-token other scopes use.+# Off restores the plain OR-of-every-token query other scopes use.CONVERSATION_QUERY_FOCUS = os.environ.get("RAG_CONVERSATION_QUERY_FOCUS", "1") == "1"-# "chronological" presents recalled turns oldest first, labelled, with later superseding-# earlier; "relevance" restores the previous rendering. Presentation only: neither changes-# which turns are selected.+# Presentation only: neither ordering changes which turns are selected.CONVERSATION_RECALL_ORDER = os.environ.get("RAG_CONVERSATION_RECALL_ORDER", "chronological")-# Cosine floor for the automatic recall only, never for a search the model asked for.-# Default 0.0 (off): a weak match is often still the right turn in one's own conversation.-# Raise it when the automatic block does more harm than good.+# Automatic recall only, never a search the model asked for; default 0.0 since a weak match is often+# still the right turn.CONVERSATION_FORCED_MIN_SCORE = float(os.environ.get("RAG_CONVERSATION_FORCED_MIN_SCORE", "0.0"))UPLOAD_EXTS = {".pdf", ".txt", ".md", ".markdown", ".docx", ".html", ".htm"}-# Reject uploads larger than this, so one pathological file can't drive unbounded parse-# + vision work at ingest. 0 disables the cap. Default 200 MB.+# 0 disables the cap; bounds parse + vision work at ingest.MAX_UPLOAD_BYTES = int(os.environ.get("RAG_MAX_UPLOAD_BYTES", str(200 * 1024 * 1024)))-# Linked folders use periodic full reconciliation. Metadata-only comparisons keep-# unchanged passes cheap; caps prevent an accidentally broad folder from becoming-# an unbounded ingestion queue.+# Caps prevent an accidentally broad linked folder from becoming an unbounded ingestion queue.FOLDER_SYNC_INTERVAL_S = float(os.environ.get("RAG_FOLDER_SYNC_INTERVAL_S", "30"))FOLDER_MAX_FILES = int(os.environ.get("RAG_FOLDER_MAX_FILES", "10000"))FOLDER_JOB_HISTORY_LIMIT = int(os.environ.get("RAG_FOLDER_JOB_HISTORY_LIMIT", "200"))-# Extract PDF text as layout-aware Markdown (pymupdf4llm) instead of flat text, so-# tables, headings and lists survive into chunks and retrieval. Falls back to plain-# PyMuPDF text when off, when pymupdf4llm is missing, or when extraction fails.+# Falls back to plain PyMuPDF text when off, when pymupdf4llm is missing, or when extraction fails.PDF_MARKDOWN = os.environ.get("RAG_PDF_MARKDOWN", "1") == "1"-# Figure captioning via the loaded vision model: detected figures are transcribed +-# described so they become searchable. On by default, a no-op without a vision model;-# the chat's "Describe figures & charts" toggle overrides it per upload.+# No-op without a vision model; the chat's "Describe figures & charts" toggle overrides it per upload.CAPTION_IMAGES = os.environ.get("RAG_CAPTION_IMAGES", "1") == "1"# Total per-document tile budget (figure-bearing pages are tiled, see below).CAPTION_MAX_IMAGES = int(os.environ.get("RAG_CAPTION_MAX_IMAGES", "24"))CAPTION_TIMEOUT_S = float(os.environ.get("RAG_CAPTION_TIMEOUT_S", "60"))-# Larger than a one-line caption since captions transcribe every label. FIGURE_DPI is-# high enough to keep small box/axis labels legible when tiles are rendered.+# Captions transcribe every label, and FIGURE_DPI keeps small box/axis labels legible when tiled.CAPTION_MAX_TOKENS = int(os.environ.get("RAG_CAPTION_MAX_TOKENS", "768"))FIGURE_DPI = int(os.environ.get("RAG_FIGURE_DPI", "200"))-# Figure pages are tiled into an overlapping ROWS x COLS grid of high-DPI tiles (plus-# an optional full page), so small labels and every sub-figure are covered without-# exact region detection. MAX_PAGES bounds figure pages; MAX_IMAGES bounds total tiles.+# Overlapping tile grid covers sub-figures and small labels without exact region detection.FIGURE_TILE_ROWS = int(os.environ.get("RAG_FIGURE_TILE_ROWS", "2"))FIGURE_TILE_COLS = int(os.environ.get("RAG_FIGURE_TILE_COLS", "2"))FIGURE_TILE_OVERLAP = float(os.environ.get("RAG_FIGURE_TILE_OVERLAP", "0.12"))FIGURE_FULLPAGE = os.environ.get("RAG_FIGURE_FULLPAGE", "1") == "1"CAPTION_MAX_PAGES = int(os.environ.get("RAG_CAPTION_MAX_PAGES", "4"))-# Scanned-PDF OCR: a page with little extractable text is rendered and transcribed by-# the vision model so it becomes searchable. Needs a vision model, else skipped (page-# stays empty). MIN_CHARS is the text length below which a page is treated as scanned.+# Needs a vision model, else the page stays empty; MIN_CHARS is the text length below which a page counts as scanned.OCR_SCANNED = os.environ.get("RAG_OCR_SCANNED", "1") == "1"OCR_MIN_CHARS = int(os.environ.get("RAG_OCR_MIN_CHARS", "16"))OCR_MAX_PAGES = int(os.environ.get("RAG_OCR_MAX_PAGES", "20"))@@ -100,15 +74,10 @@ OCR_DPI = int(os.environ.get("RAG_OCR_DPI", "150"))OCR_TIMEOUT_S = float(os.environ.get("RAG_OCR_TIMEOUT_S", "60"))OCR_MAX_TOKENS = int(os.environ.get("RAG_OCR_MAX_TOKENS", "2048"))-# Embedder backend. "auto": sentence-transformers on a CUDA/ROCm GPU (torch fp16-# wins bulk indexing), else torch-free GGUF llama-server. Switching backends changes-# the vectors, so the index must be rebuilt.+# Switching backends changes the vectors, so the index must be rebuilt.EMBED_BACKEND = os.environ.get("RAG_EMBED_BACKEND", "auto")-# ``documents.embedding_model`` records the embedder that produced a document's-# vectors, not just the configured model name. The name alone is not the embedding-# space: llama-server ignores it and embeds through the GGUF companion, which pools-# its own way, so one name can mean two spaces on the same machine.+# The model name alone is not the embedding space: llama-server embeds through the GGUF companion and pools its own way.EMBEDDING_IDENTITY_TAGS = ("sentence-transformers", "llama-server")@@ -179,8 +148,7 @@ def _names_gguf(model: str) -> bool:return "gguf" in re.split(r"[^a-z0-9]+", model.lower())-# Suffixes unsloth puts on an unquantized re-upload; the GGUF sits on the base-# name (embeddinggemma-300m-qat-q8_0-unquantized -> embeddinggemma-300m-GGUF).+# Suffixes unsloth puts on an unquantized re-upload; the GGUF sits on the base name._QUANT_SUFFIX_RE = re.compile(r"(?:-qat)?(?:-q\d+_\d+[a-z]*)?-unquantized$", re.I)@@ -244,27 +212,20 @@ def effective_gguf_repo_for_embedding_model(model: str) -> str:from utils.embedding_model_settings import get_stored_gguf_repo, remembered_gguf_repostored = get_stored_gguf_repo(model)if stored is None:- # One stored record, so saving another model takes this one's repo- # away while a job pinned to it is still ingesting; the derived name- # would move that job's identity mid-run and split one document set- # across two tags. The memo is what this process last saw for it.+ # One stored record, so saving another model would move a pinned job's derived identity mid-run and+ # split one document set across two tags.stored = remembered_gguf_repo(model)except Exception: # noqa: BLE001 - store unavailable: fall back to the conventionstored = Nonereturn stored or gguf_repo_for_embedding_model(model)-# llama-server backend only. F16 over Q8_0: faster (no per-block dequant for this-# tiny model) and exact vs fp32, for ~30MB more on disk.+# F16 over Q8_0: faster (no per-block dequant at this size) and exact, for ~30MB more on disk.EMBED_GGUF_REPO = os.environ.get("RAG_EMBED_GGUF_REPO", "unsloth/bge-small-en-v1.5-GGUF")EMBED_GGUF_VARIANT = os.environ.get("RAG_EMBED_GGUF_VARIANT", "F16")-# Read by BOTH backends, and "auto" means something different to each, because the-# cost of a GPU is different. llama-server offloads inside its own subprocess, so-# "auto" there means GPU when there is room. sentence-transformers loads inside the-# backend process, where the first CUDA allocation pins a primary context for the life-# of the process (712 MiB on a B200, against 74 MiB of weights), so "auto" there means-# CPU. Set "gpu" to offload either one; "cpu" keeps both off the GPU.-EMBED_DEVICE = os.environ.get("RAG_EMBED_DEVICE", "auto") # "auto" | "gpu" | "cpu"+# "auto" differs per backend: llama-server offloads inside its own subprocess, while+# sentence-transformers would pin a CUDA primary context (712 MiB on a B200), so it stays on CPU.+EMBED_DEVICE = os.environ.get("RAG_EMBED_DEVICE", "auto")def embed_device_preference() -> str:@@ -297,7 +258,7 @@ def embed_device_requires_gpu() -> bool:EMBED_HOST = os.environ.get("RAG_EMBED_HOST", "127.0.0.1")-EMBED_PORT = int(os.environ.get("RAG_EMBED_PORT", "0")) # 0 = auto-pick a free port+EMBED_PORT = int(os.environ.get("RAG_EMBED_PORT", "0"))EMBED_BATCH = int(os.environ.get("RAG_EMBED_BATCH", "64"))EMBED_STARTUP_TIMEOUT_S = float(os.environ.get("RAG_EMBED_STARTUP_TIMEOUT_S", "120"))EMBED_REQUEST_TIMEOUT_S = float(os.environ.get("RAG_EMBED_REQUEST_TIMEOUT_S", "60"))diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.pyindex 9e9103711b1..1ac18958cdd 100644--- a/studio/backend/core/rag/embed_llama_server.py+++ b/studio/backend/core/rag/embed_llama_server.py@@ -96,9 +96,8 @@ class LlamaServerBackend:def __init__(self) -> None:# Lifecycle (spawn/restart/kill) is serialized; HTTP requests are not.self._lifecycle_lock = threading.Lock()- # Unload closes the client/process only after every encode/tokenize/probe- # that already entered has left, and blocks late callers from restarting- # the backend after it was unpublished.+ # Unload waits for every encode/tokenize/probe already in flight and blocks late callers from+ # restarting after unpublish.self._operation_condition = threading.Condition()self._active_operations = 0self._operation_local = threading.local()@@ -107,24 +106,20 @@ class LlamaServerBackend:self._port: int | None = Noneself._stdout_lines: list[str] = []self._stdout_thread: threading.Thread | None = None- # Belongs to whichever model the one subprocess serves, so dim() takes the- # same _serve_lock the request path does. Reentrant because- # dim() -> encode() -> _ensure_ready() re-enters it on one thread.+ # dim() takes the same _serve_lock the request path does; reentrant because dim() -> encode() ->+ # _ensure_ready() re-enters on one thread.self._dim: int | None = None- # One subprocess serves one GGUF, so readiness and the request relying on- # it must be indivisible: otherwise a swap between them lands A's POST on- # B's server, storing B's vectors under A's identity. Reentrant because- # dim() -> encode() -> _post() re-enters on one thread.+ # One subprocess serves one GGUF, so readiness and the request must be indivisible: a swap between+ # them lands A's POST on B's server.self._serve_lock = threading.RLock()self._model_path: str | None = None- # Effective GGUF repo the cached path/dim belong to; a Settings change- # makes it stale, forcing a re-resolve + respawn (see _ensure_ready).+ # A Settings change makes the cached path/dim stale, forcing a re-resolve and respawn (see _ensure_ready).self._model_repo: str | None = Noneself._binary: str | None = Noneself._binary_path_revision: int | None = None# Sticky after an auto GPU start fails: later spawns stay on CPU.self._force_cpu = False- # Pooled client (full URLs per request survive a respawn); trust_env=False skips HTTP(S)_PROXY.+ # trust_env=False skips HTTP(S)_PROXY; full URLs per request survive a respawn.self._client = httpx.Client(timeout = config.EMBED_REQUEST_TIMEOUT_S, trust_env = False)atexit.register(self._shutdown)@@ -168,8 +163,7 @@ class LlamaServerBackend:"llama-server. Install llama.cpp or set LLAMA_SERVER_PATH / ""UNSLOTH_LLAMA_CPP_PATH.")- # Before the probe, so the --help check and the spawn both run the real- # executable rather than an entrypoint that loses DYLD_* to SIP.+ # Resolve the real executable before the probe: an entrypoint loses DYLD_* to SIP.binary = _resolve_entrypoint(binary)self._assert_embedding_support(binary)self._binary = binary@@ -230,30 +224,25 @@ class LlamaServerBackend:from utils.models.model_config import _local_gguf_load_path, colocated_split_shardsfrom utils.paths import normalize_path- # Same normalization the sentence-transformers and settings probes already- # apply, so a drive-letter path is recognized here too. Without it a WSL- # user's `C:\models\...` GGUF resolved as a filename that cannot exist and- # went to the Hub, which then reported no weights in `C:\models\...-GGUF`.+ # Normalize like the sentence-transformers and settings probes, else a WSL user's C:\models\... GGUF+ # resolves as an impossible filename and goes to the Hub.p = Path(normalize_path(model)).expanduser()if p.is_file() and p.suffix.lower() == ".gguf":return None if is_appledouble_metadata(p) else str(p)if p.is_dir():files = [f for f in p.iterdir() if not is_appledouble_metadata(f)]- # Same pick as a hub listing gets, so a directory and the repo it was- # downloaded from cannot disagree about which quant the embedder opens.- # Its name tiebreak is what makes a directory of split shards resolve to- # shard 1 rather than to whichever one `iterdir` happened to yield first.+ # Same pick as a hub listing gets, and its name tiebreak resolves a directory of split shards to shard 1.remaining = list(files)while remaining:picked = LlamaServerBackend._pick_gguf(remaining, key = lambda f: f.name)if picked is None:break- # llama-server opens the siblings implicitly, so a torn family is- # not a usable answer even though shard 1 is sitting right there.+ # llama-server opens the siblings implicitly, so a torn family is not a usable answer even with+ # shard 1 present.if colocated_split_shards(picked)[1]:- # Enters the family at shard 1 and keeps a complete symlink set- # intact, the same way the chat loader resolves a local GGUF.+ # Enters the family at shard 1 and keeps a complete symlink set intact, the same way the+ # chat loader resolves a local GGUF.return str(_local_gguf_load_path(picked))remaining = [f for f in remaining if f != picked]raise RuntimeError(f"no .gguf file found in local model dir {model!r}")@@ -334,9 +323,8 @@ class LlamaServerBackend:for n in namesif name_of(n).lower().endswith(".gguf")and "mmproj" not in name_of(n).lower()- # A drafter is a companion, never a model in its own right, so it must be- # excluded everywhere mmproj is. A cache holding only the companion would- # otherwise be read as holding the embedder.+ # A drafter is a companion, so exclude it everywhere mmproj is: a cache holding only the companion+ # would read as holding the embedder.and not _is_mtp_drafter(name_of(n))]if not usable:@@ -347,8 +335,7 @@ class LlamaServerBackend:if require_variant:return Nonematch = usable- # Name breaks a length tie: a directory scan yields no guaranteed order, and among- # equal-length shard names that would otherwise decide which shard is picked.+ # Name breaks a length tie: a directory scan yields no guaranteed order.return sorted(match, key = lambda n: (len(name_of(n)), name_of(n)))[0]@staticmethod@@ -366,14 +353,14 @@ class LlamaServerBackend:from utils.hf_cache_settings import active_hf_hub_cachefrom utils.paths import resolve_cached_repo_id_case- # A repo id typed with different casing than the cache folder is still that repo;- # missing it here would re-download, or fail outright with the hub unreachable.+ # A repo id typed with different casing than the cache folder is still that repo; missing it re-+ # downloads or fails outright offline.cased = resolve_cached_repo_id_case(repo_id)repo_dir = Path(active_hf_hub_cache()) / f"models--{cased.replace('/', '--')}"try:rev = (repo_dir / "refs" / "main").read_text(encoding = "utf-8").strip()except (OSError, ValueError):- return None # not cached here, unreadable, or pinned to a commit+ return None# A ref file holds a bare commit hash; a separator would join out of the cache.if not rev or rev in (".", "..") or "/" in rev or "\\" in rev:return None@@ -393,21 +380,16 @@ class LlamaServerBackend:if snapshot is None:return Nonetry:- # Not a "*.gguf" glob: that is case-sensitive, and quant-per-directory layouts- # put the file a level down, both of which the hub listing handles.+ # Not a "*.gguf" glob: that is case-sensitive and misses quant-per-directory layouts, both of which+ # the hub listing handles.- # A complete family of sidecars is servable by every test below -- whole, quant- # matched, opens as a file -- so the retry that drops a torn real set lands on it.files = [p for p in snapshot.rglob("*") if p.is_file() and not is_appledouble_metadata(p)]except OSError:return None- # llama-server opens sibling shards implicitly, so a split set is servable only when- # it is whole. An unservable winner is dropped and the pick retried rather than- # failing the lookup, or an incomplete family would shadow a complete one simply by- # sorting ahead of it. Verified per winner, not per file: a plain file is a complete- # one-file set, and a whole-cache filter would walk siblings for every shard present.+ # llama-server opens sibling shards implicitly, so a split set is servable only when whole; drop+ # an unservable winner and retry, else an incomplete family shadows a complete one.while files:pick = LlamaServerBackend._pick_gguf(files,@@ -430,20 +412,16 @@ class LlamaServerBackend:``model_name`` is the model the caller pinned; the live setting wouldresolve B's weights for a job still tagging its vectors as A."""model = model_name or config.effective_embedding_model()- # Captured once: if the setting changes mid-download, the path must stay- # tagged with the repo it was resolved FOR, so _current() sees the new- # setting as stale and respawns instead of serving the old model.+ # Captured once: the path must stay tagged with the repo it was resolved FOR, so a mid-download+ # setting change reads as stale and respawns.desired = config.effective_gguf_repo_for_embedding_model(model)if self._model_path is not None and self._model_repo == desired:return self._model_pathlocal = self._resolve_local_gguf(model)if local is not None:return self._adopt_model_path(local, desired)- # The planned family whenever it is on disk, ahead of the variant lookup.- # Identity carries the model and repo but not the file, so a preferred- # variant arriving later would otherwise change the weights under an- # existing index without changing its tag. Consulted after completion too,- # which is what makes the record an identity, not a transfer receipt.+ # Identity carries the model and repo but not the file, so a preferred variant arriving later would+ # change the weights under an existing index without changing its tag.planned = self._planned_family_path(model, desired)if planned is not None:try:@@ -455,15 +433,12 @@ class LlamaServerBackend:# Companion repo, then the naming variants, then the model repo itself.repo = desiredcandidates = list(dict.fromkeys([repo, *config.gguf_repo_candidates(model)]))- # Only the preferred repo. A file cached under the fallback candidate must not- # pre-empt a companion repo the hub can still resolve, which is the order the- # listing follows; offline, the relaxed pass below still reaches both.+ # Only the preferred repo: a file cached under the fallback must not pre-empt a companion repo the+ # hub can still resolve.cached = self._resolve_cached_gguf(desired)if cached is not None:- # The configured variant is present in the preferred repo, which is- # what the picker advertised, so any pending marker has been answered.- # Retire it here or the model stays cache-only for the life of the- # install and a later eviction reads as "never downloaded".+ # Retire the pending marker here, or the model stays cache-only for the life of the install and a+ # later eviction reads as "never downloaded".try:from utils.embedding_model_settings import clear_stored_download_pendingclear_stored_download_pending(model)@@ -476,29 +451,22 @@ class LlamaServerBackend:except Exception: # noqa: BLE001 - old/unavailable settings storedownload_pending = Falseif download_pending:- # A published repo may not carry the configured quant; the picker- # deliberately downloaded the complete family the Hub resolver- # selected, so accept any complete cached quant here. Never fall- # through to the online loader while the explicit transfer is- # pending/cancelled.+ # A published repo may not carry the configured quant, so accept any complete cached quant; never+ # fall through to the online loader while the transfer is pending or cancelled.cached = self._resolve_cached_gguf(desired, require_variant = False)if cached is not None:- # Reaching here means the configured variant is NOT cached (the- # strict pass returned if it were), which happens two ways that- # need opposite answers.+ # Reaching here means the configured variant is NOT cached (the strict pass returned if it+ # were), which happens two ways that need opposite answers.if self._is_planned_family(model, desired, cached):- # The repo publishes no such variant, so this quant is what- # the picker advertised and fetched. Retire the marker, or the- # model stays cache-only and a later eviction refuses to index- # something that did finish downloading.+ # The repo publishes no such variant, so this quant is what the picker fetched: retire the marker+ # here too.try:from utils.embedding_model_settings import clear_stored_download_pendingclear_stored_download_pending(model)except Exception: # noqa: BLE001 - a settings write must not fail a loadpass- # Otherwise a quant left by an earlier setting, or a record too old- # to say. Serve it rather than fail the index, but leave the marker:- # retiring on a stand-in would pin this model to the wrong quant.+ # Serve a stand-in rather than fail the index, but leave the marker: retiring on it would pin this+ # model to the wrong quant.return self._adopt_model_path(cached, desired)raise RuntimeError(f"Embedding model {model!r} is not downloaded yet. "@@ -533,8 +501,7 @@ class LlamaServerBackend:if not path.is_file():return Nonepaths.append(path)- # Ordered shard 1..N by the planner, so the first is where llama-server- # enters the family.+ # Ordered shard 1..N by the planner, so the first is where llama-server enters the family.return str(paths[0]) if paths else Noneexcept Exception: # noqa: BLE001 - an unreadable record answers nothingreturn None@@ -611,8 +578,8 @@ class LlamaServerBackend:breakerrors.append(f"{candidate!r}: no .gguf files")if filename is None:- # Any cached GGUF now beats no embedder. Only where the hub failed to NAME- # one: a transfer failing on its own (no disk, bad checksum) must surface.+ # Only where the hub failed to NAME a file: a transfer failing on its own (no disk, bad checksum)+ # must surface.for candidate in candidates:cached = self._resolve_cached_gguf(candidate, require_variant = False)if cached is not None:@@ -629,9 +596,8 @@ class LlamaServerBackend:from utils.hf_cache_settings import active_hf_hub_cachecache_dir = active_hf_hub_cache()- # Fetch the whole split family, not just the shard that was picked:- # `-m shard1` makes llama-server open the siblings itself, so a- # one-file transfer fails at startup on a repo it just downloaded.+ # Fetch the whole split family, not just the picked shard: -m shard1 makes llama-server open the+ # siblings itself.downloaded = Nonefor name in family:fetched = hf_hub_download(@@ -658,7 +624,7 @@ class LlamaServerBackend:return Trueif dev == "cpu" or self._force_cpu:return False- return self._gpu_available() # auto+ return self._gpu_available()@staticmethoddef _gpu_available() -> bool:@@ -670,7 +636,7 @@ class LlamaServerBackend:from utils.hardware import is_apple_siliconif is_apple_silicon():- return True # bundled mac build offloads to Metal+ return Truefrom core.inference.llama_cpp import LlamaCppBackend# [(idx, free_mib)], honors CVD@@ -691,8 +657,8 @@ class LlamaServerBackend:return LlamaCppBackend._arch_gate_survivors(binary)def _build_cmd(self, binary: str, model_path: str, port: int, *, use_gpu: bool) -> list[str]:- # No --embd-normalize (not in every build; we normalize in Python to match- # the ST path). --fit off: don't auto-resize ctx/offload to device memory.+ # No --embd-normalize (absent in some builds; we normalize in Python to match the ST path), and+ # --fit off so ctx/offload are not auto-resized.cmd = [binary,"-m",@@ -713,46 +679,36 @@ class LlamaServerBackend:def _build_env(self, binary: str, *, use_gpu: bool) -> dict[str, str]:env = child_env_without_native_path_secret()- env["LLAMA_SET_ROWS"] = "1" # ggml set_rows fast path+ env["LLAMA_SET_ROWS"] = "1"if sys.platform == "darwin":- # _llama_lib_dir, not Path(binary).parent: the managed install puts- # an entrypoint in front of the real server, and the dylibs sit- # next to the target, not next to the wrapper. Unconditional,- # unlike the CUDA branch below: a CPU start loads the same sibling- # dylibs (#8566).+ # _llama_lib_dir, not Path(binary).parent: the managed install puts an entrypoint in front of the+ # real server and the dylibs sit next to the target (#8566).env = _with_dyld_path(env, _binary_lib_dir(binary))elif use_gpu:- # Path(binary).parent, unchanged. Resolving the entrypoint here too- # would be more correct in principle, but it moves the first- # LD_LIBRARY_PATH entry for every existing Linux GPU install whose- # llama-server is a symlink, and this change is not about them.+ # Left as Path(binary).parent: resolving the entrypoint here would move the first LD_LIBRARY_PATH+ # entry for every existing Linux GPU install whose llama-server is a symlink.self._add_linux_cuda_libs(env, str(Path(binary).parent))_pinned = self._arch_gated_gpu_ids(binary)if _pinned:from core.inference.llama_cpp import LlamaCppBackend- # prefer_rocr: a HIP-only mask still lets HSA enumerate (and die- # on) the unsupported agent; ROCR drops it at the driver layer.+ # prefer_rocr: a HIP-only mask still lets HSA enumerate, and die on, the unsupported agent.LlamaCppBackend._emit_child_gpu_visibility(env, ",".join(str(i) for i in _pinned), prefer_rocr = True)logger.info("pinning the embed server to arch-supported GPU(s) %s", _pinned)- # Not else: the darwin branch above is about dylibs, so a macOS CPU start- # still has to blank the devices, exactly as it did before that branch.+ # Not else: a macOS CPU start still has to blank the devices.if not use_gpu:# Blank devices so a CUDA build stays on CPU and reserves no VRAM.env["CUDA_VISIBLE_DEVICES"] = ""- # HIP reads CUDA_VISIBLE_DEVICES only when HIP_VISIBLE_DEVICES is unset,- # so an inherited HIP mask would keep a device (and the ~0.5 GB context it- # costs) visible to a child we just put on the CPU. Same "-1" sentinel the- # chat forced-CPU path uses. An inherited ROCR mask is left alone: it hides- # agents below HIP, so clearing it would expose MORE of them to the HSA- # enumeration that dies on an uncovered arch (#7624).+ # HIP reads CUDA_VISIBLE_DEVICES only when HIP_VISIBLE_DEVICES is unset, so an inherited HIP mask+ # keeps a device (and the ~0.5 GB context it costs) visible to a child we just put on the CPU; an+ # inherited ROCR mask is left alone, since clearing it exposes MORE agents to the HSA enumeration+ # that dies on an uncovered arch (#7624).env["HIP_VISIBLE_DEVICES"] = "-1"- # LLAMA_ARG_DEVICE / LLAMA_ARG_MAIN_GPU are the env spelling of --device /- # --main-gpu. Hiding every device while leaving that pick in place makes- # llama.cpp reject the name that no longer enumerates and exit instead of- # running on the CPU. Same clear the chat sentinel makes.+ # LLAMA_ARG_DEVICE / LLAMA_ARG_MAIN_GPU are the env spelling of --device / --main-gpu: hiding+ # every device while leaving that pick in place makes llama.cpp reject the name that no longer+ # enumerates and exit instead of running on the CPU.from core.inference.llama_cpp import LlamaCppBackendLlamaCppBackend._clear_device_placement_env(env)@@ -765,7 +721,7 @@ class LlamaServerBackend:import platformif sys.platform == "win32":- return # Windows resolves CUDA via PATH in the inherited env.+ returnarch = platform.machine()lib_dirs = [binary_dir]for pattern in (@@ -840,8 +796,8 @@ class LlamaServerBackend:**child_popen_kwargs(),)self._process = proc- # Long-lived, and child_popen_kwargs() is empty on macOS, so the crash- # record is the only thing that can reap it after a force quit.+ # child_popen_kwargs() is empty on macOS, so the crash record is the only thing that can reap it+ # after a force quit.adopt_pid(proc.pid)self._port = portself._stdout_thread = threading.Thread(@@ -946,8 +902,7 @@ class LlamaServerBackend:except Exception as e: # noqa: BLE001logger.warning("error killing llama-server embedder: %s", e)finally:- # Only once it is confirmed gone: a survivor must stay recorded so- # the next startup sweep can reap it.+ # A survivor must stay recorded so the next startup sweep can reap it.if proc.poll() is not None:forget_pid(proc.pid)self._process = None@@ -968,9 +923,7 @@ class LlamaServerBackend:self._client.close()except Exception: # noqa: BLE001pass- # Drop the interpreter-exit hook with the backend it belonged to.- # Unload/rebuild is an ordinary cycle here (Settings exposes it as a- # button), and a registry that only ever grows pins every retired+ # Unload/rebuild is an ordinary cycle here, and a registry that only grows pins every retired# backend, and its captured stdout tail, for the life of the process.try:atexit.unregister(self._shutdown)@@ -998,8 +951,7 @@ class LlamaServerBackend:last_exc: Exception | None = Nonefor attempt in range(2):with self._serve_lock:- # Held across readiness AND the request, so a swap can only land- # between whole requests.+ # Held across readiness AND the request, so a swap can only land between whole requests.self._ensure_ready(model_name)try:resp = self._client.post(f"{self._base_url}{path}", json = payload)diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex be3a409c807..7473f71958f 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -37,20 +37,17 @@ from . import configlogger = logging.getLogger(__name__)-# "false" silences the fast tokenizer's fork warning; encode() flips it to "true"-# only during a batch tokenize (rayon speedup), then restores it.+# "false" silences the fast tokenizer's fork warning; encode() flips it only during a batch tokenize.os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")_lock = threading.Lock()-# Serializes encode/tokenize (HF fast tokenizer isn't thread-safe). Separate from-# _lock so a long encode never blocks a reload.+# Serializes encode/tokenize (the HF fast tokenizer is not thread-safe); separate from _lock so a+# long encode never blocks a reload._compute_lock = threading.Lock()_model = None_name: str | None = None-# The backend that served this thread's most recent encode. The process embedder can-# swap between an encode returning and the caller asking what produced the vectors,-# and the answer has to be the backend that was actually used. See-# ``encode_with_identity``.+# The process embedder can swap between an encode returning and the caller asking, so the answer+# must be the backend actually used. See encode_with_identity._served_by = threading.local()@@ -81,8 +78,7 @@ def _device() -> str:"""if config.embed_device_preference() != "gpu":return "cpu"- # Still a table lookup, so asking for a GPU on a host without one, or on Apple- # where this backend has no torch device, lands on CPU rather than on a device+ # Still a table lookup: asking for a GPU on a host without one lands on CPU rather than on a device# string torch cannot open.return _TORCH_DEVICE.get(get_device(), "cpu")@@ -118,10 +114,8 @@ def _load_device() -> str:_torchao_stub_done = False-# Guards the stub's one-shot state and the import that must follow it. Its own-# lock, not ``_lock``: that one is held across a whole model construction, so-# borrowing it made a preflight probe wait out someone else's download. Taken-# innermost, so there is no ordering to get wrong.+# Its own lock, not _lock: that one is held across a whole model construction, so borrowing it made+# a preflight probe wait out someone else's download._stub_lock = threading.Lock()@@ -221,8 +215,8 @@ def _guard_model_security(name: str, local_only: bool = False) -> None:if local_only:load_subdirs = ()else:- # Union audio-model load roots with ST module dirs so a flagged pickle under a- # Transformer module dir blocks instead of passing as an unreferenced nested shard.+ # Union the load roots so a flagged pickle under a Transformer module dir blocks instead of passing+ # as an unreferenced nested shard.load_subdirs = tuple(dict.fromkeys((*security_load_subdirs(name, token), *_st_module_subdirs(name, token))@@ -264,18 +258,14 @@ class _CaptureLoadReport(logging.Filter):"""_SERIOUS = ("MISSING", "MISMATCH", "CONVERSION")- # The only UNEXPECTED key worth downgrading is the legacy buffer every BERT-era- # sentence-transformer ships. Any other discarded weight can genuinely change- # retrieval quality, so it stays a warning. Matched on the whole key, not as a- # substring: "encoder.position_ids_projection.weight" is a real discarded weight.+ # Only the legacy BERT-era buffer is downgraded, matched on the whole key: any other discarded+ # weight can genuinely change retrieval quality._KNOWN_BENIGN_UNEXPECTED = "embeddings.position_ids"def __init__(self) -> None:super().__init__()self.reports: list[str] = []- # These filters sit on process-global loggers, so a concurrent load on another- # thread would otherwise have its report swallowed and attributed here. Only- # capture what the thread that opened the context emits.+ # These filters sit on process-global loggers, so capture only what the thread that opened the context emits.self.thread_id = threading.get_ident()def filter(self, record: logging.LogRecord) -> bool:@@ -294,10 +284,8 @@ class _CaptureLoadReport(logging.Filter):for report in self.reports:if any(tag in report for tag in self._SERIOUS):return True- # Row by row over the table only. transformers appends a "Notes:" section- # explaining each status ("- UNEXPECTED: can be ignored when loading from- # a different task/architecture"), and reading that as a key row would make- # every unexpected report serious, including the benign one.+ # Row by row over the table only: transformers appends a "Notes:" section explaining each status+ # ("- UNEXPECTED: can be ignored ..."), whose lines would read as serious key rows.table = report.split("Notes:", 1)[0]for row in table.splitlines():if "UNEXPECTED" not in row:@@ -313,8 +301,8 @@ class _CaptureLoadReport(logging.Filter):_LOAD_REPORT_LOGGERS = ("transformers.utils.loading_report","transformers.modeling_utils",- # An adapter-backed embedding model reports through the PEFT integration's own- # logger, which is not a descendant of either of the above.+ # An adapter-backed embedding model reports through the PEFT integration's own logger, not a+ # descendant of either above."transformers.integrations.peft",)@@ -334,15 +322,10 @@ def _quiet_transformers_load():log.addFilter(capture)attached.append(log)- # The weight-load bar is transformers.utils.logging.tqdm, so disable_progress_bar()- # reaches it. The "is it on right now" probe has been spelled both ways across- # versions (is_progress_bar_enabled in 4.x/5.x, are_progress_bars_disabled in- # some builds), so accept either and skip the restore when neither exists rather- # than re-enabling a bar the caller had deliberately turned off.- # transformers' enable_progress_bar() also calls the Hub's enable_progress_bars(),- # so restoring the transformers flag would clobber a Hub-only disable that someone- # else installed (unsloth does exactly that in patch_ipykernel_hf_xet for the- # broken hf-xet/ipykernel pair). Snapshot the Hub state separately and put it back.+ # The "is it enabled" probe is spelled both ways across transformers versions, so accept either and+ # skip the restore when neither exists.+ # transformers' enable_progress_bar() also calls the Hub's, so snapshot and restore the Hub state+ # separately or a Hub-only disable is clobbered.hub_bars_off = Nonetry:from huggingface_hub.utils import are_progress_bars_disabled@@ -423,19 +406,15 @@ def _get(model_name: str | None = None):accelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU."""global _model, _namename = model_name or config.effective_embedding_model()- # Capture offline state once so the gate and the load agree (no window where the gate is- # skipped as offline but the constructor then reaches the network).+ # Capture offline state once so the gate and the load agree.try:from utils.embedding_model_settings import get_stored_download_pendingdownload_pending = get_stored_download_pending(name)except Exception: # noqa: BLE001 - old/unavailable settings storedownload_pending = Falseoffline = hf_env_offline()- # Two different questions. `local_only` says "load from cache, do not fetch",- # which a pending transfer also demands. `offline` is whether the Hub is- # reachable, which is what the security gate needs: told offline while online- # it applies the fail-closed cached-pickle rule and rejects a .bin-only repo- # the resolver just accepted and scanned, failing the first index outright.+ # local_only means "load from cache"; offline is what the security gate needs, and claiming offline+ # while online rejects a .bin-only repo the resolver just scanned.local_only = offline or download_pendingwith _lock:if _model is None or _name != name:@@ -449,82 +428,60 @@ def _get(model_name: str | None = None):st_kwargs = dict(device = device,cache_folder = active_hf_hub_cache(),- # Keyed on the device we actually load on, not on whether we degraded- # onto it. fp16 BERT on CPU is slow where it works at all, and several- # ops raise "not implemented for 'Half'" -- which _SentenceTransformers- # Backend.encode() catches and answers by swapping the whole process to- # llama-server, so getting this wrong fails as a silent backend change- # rather than as an error.+ # Keyed on the device we load on: fp16 BERT on CPU raises "not implemented for Half", which encode()+ # answers by swapping the whole process to llama-server.model_kwargs = dtype_kwargs("float32" if device == "cpu" else "float16"),)load_target = namefrom utils.paths import is_local_pathfrom utils.utils import cached_st_source, hf_cache_snapshot_dir- # The repo AND the directory that supplied the weights, together: a- # slashless name can match under sentence-transformers/ while a stale- # literal cache entry is what a second lookup would return, and ST- # weights alone are satisfied by the first finalized shard of a- # transfer still in flight.- #- # Repo ids only. A local path IS the artifact the resolver accepted,- # and the picker takes a slashless relative directory, so a folder- # named all-MiniLM-L6-v2 beside a cached sentence-transformers/ copy- # would load the Hub's weights under the local path's identity.+ # The repo AND the directory that supplied the weights, together: ST weights alone are satisfied by+ # the first finalized shard of a transfer still in flight.+ # Repo ids only: a local folder named all-MiniLM-L6-v2 would otherwise load the Hub's weights under+ # the local path's identity.st_source = None if is_local_path(name) else cached_st_source(name)if not local_only and st_source is not None:- # Settings reported this model on-device off the same predicate, so- # handing SentenceTransformer the repo id here is what lets it reach- # the Hub for a revision published since, and fetch it during the- # first index. That is the invisible transfer the picker exists to- # replace, and it changes the vectors without changing the identity- # they are tagged with. Load the snapshot that was called cached.+ # Load the snapshot that was called cached: the repo id lets ST reach the Hub for a newer revision+ # during the first index, changing the vectors without changing their identity.load_target = str(st_source[1])if local_only:- # ST-specific AND complete: a hybrid repo whose GGUF is cached, or a- # transfer that has finalized only its first shard, would otherwise- # retire the marker and be handed to SentenceTransformer.+ # ST-specific AND complete: a hybrid repo's cached GGUF, or a transfer that finalized only its first+ # shard, would otherwise retire the marker.if download_pending and st_source is None:+ # Defensive: a loadable check and snapshot lookup share no lock, so eviction between them is+ # still a pending model.raise EmbeddingModelDownloadRequiredError(f"Embedding model {name!r} is not downloaded yet. ""Finish its Settings download before indexing documents.")snapshot = st_source[1] if st_source else hf_cache_snapshot_dir(name)if snapshot is not None:- # Load from the local snapshot dir: a local path never touches the Hub, so- # this is offline-safe on ANY sentence-transformers version (even ones- # predating local_files_only).+ # A local path never touches the Hub, so this is offline-safe on ANY sentence-transformers+ # version, even ones predating local_files_only.load_target = str(snapshot)elif download_pending:- # Defensive: a loadable check and snapshot lookup share no- # lock, so eviction between them is still a pending model.raise EmbeddingModelDownloadRequiredError(f"Embedding model {name!r} is not downloaded yet. ""Finish its Settings download before indexing documents.")elif _st_accepts_local_files_only(SentenceTransformer):st_kwargs["local_files_only"] = True- # After load_target is settled, so this scans what is about to be- # deserialized: on the repo id it checked the Hub's current commit while- # the load opened an older cached one. evaluate_file_security recovers- # the repo and exact commit from a snapshot path, and fails closed- # locally when the Hub cannot answer.+ # Scan after load_target is settled: on the repo id it checked the Hub's current commit while the+ # load opened an older cached one. evaluate_file_security recovers the repo and exact commit from+ # a snapshot path._guard_model_security(load_target, offline)with _quiet_transformers_load() as report:- # The re-emit runs in finally: a load that raises after transformers- # wrote its report is exactly when a MISSING or MISMATCH line matters,- # and letting the exception skip the loop would swallow it.+ # Re-emit in finally: a load that raises after transformers wrote its report is exactly when a+ # MISSING or MISMATCH line matters.try:_model = SentenceTransformer(load_target, **st_kwargs)finally:_emit_load_reports(report)_name = nameif download_pending:- # Only once the model is actually constructed. Retiring it before- # the constructor let an unsupported architecture or a device that- # cannot initialize fall through to llama-server with no marker- # left, so the fallback was free to fetch the GGUF companion- # during the first index: the unapproved transfer this replaces.+ # Retire only once the model is constructed: retiring earlier let a failed construction fall through+ # to llama-server with no marker, freeing the fallback to fetch the GGUF companion.try:from utils.embedding_model_settings import clear_stored_download_pendingclear_stored_download_pending(name)@@ -558,9 +515,8 @@ def _st_encode("""ST encode -> (N, dim) float32. Serialized (fast-tokenizer borrow check),under inference_mode when torch is present, with rayon enabled for the call."""with _compute_lock:- # Admission and model lookup are one lease. If lookup happened first,- # unload could clear the globals and return while this call retained a- # strong local reference and had not begun inference yet.+ # Admission and model lookup are one lease: lookup first would let unload clear the globals while+ # this call still held a strong reference.model = _get(model_name)os.environ["TOKENIZERS_PARALLELISM"] = "true"try:@@ -630,10 +586,10 @@ class _SentenceTransformersBackend:try:return _st_encode(texts, model_name = model_name, normalize = normalize)except (UnsafeEmbeddingModelError, EmbeddingModelDownloadRequiredError):- raise # policy failures must hard-fail, not change backend/download+ raiseexcept Exception as st_err: # noqa: BLE001 - runtime ST/CUDA encode failure- # ST loaded but this encode blew up; swap the process to the llama-server- # embedder (so later encodes stay in one space) and retry.+ # ST loaded but this encode failed: swap the process to llama-server so later encodes stay in one+ # space, then retry.fallback = _switch_to_llama_fallback(st_err, model_name)if fallback is None:raise@@ -653,16 +609,10 @@ class _SentenceTransformersBackend:_backend_lock = threading.Lock()_backend = None_backend_key: str | None = None-# Per model: an ST runtime/load failure pins llama-server for THAT model, so the-# swap survives the next rebuild while a later safetensors-only selection can-# still build ST. Keyed by model because one (key, model) pair let a second-# failing model erase the first one's pin and send a still-running job back to ST.-# Cleared only by a reset or an explicit unload, both of which are a fresh start.-#-# Read WITHOUT _backend_lock throughout: _get_backend holds that lock across a-# whole model load, and every reader here is a probe the resolver's own budget is-# supposed to bound. dict.get is atomic, and a pin that lands mid-probe is-# answered on the next call.+# Per model, keyed by model: one (key, model) pair let a second failing model erase the first one's+# pin and send a running job back to ST.+# Read WITHOUT _backend_lock: _get_backend holds it across a whole model load; dict.get is atomic+# and a pin landing mid-probe is answered on the next call._forced_backends: dict[str, str] = {}_ST_ALIASES = frozenset({"sentence-transformers", "sentence_transformers", "st"})@@ -680,10 +630,8 @@ def _resolve_auto() -> str:is installed."""from core.inference.llama_cpp import LlamaCppBackend- # Unfiltered probe on purpose: the winner here runs under PyTorch, so the- # ROCm arch gate (which asks what the installed llama.cpp prebuilt was built- # for, #7624) must not apply. A device that prebuilt lacks kernels for is- # usually still a perfectly good sentence-transformers device.+ # Unfiltered probe: the winner runs under PyTorch, so the ROCm arch gate for the installed llama.cpp+ # prebuilt (#7624) must not apply.if LlamaCppBackend._get_gpu_free_memory():return "sentence-transformers"if LlamaCppBackend._find_llama_server_binary():@@ -722,16 +670,14 @@ def _model_names_gguf_repo(model: str | None) -> bool:try:from utils.paths import is_local_path- # A directory may be named anything: `~/models/my-gguf` holding safetensors- # is a sentence-transformers model, and only the filesystem can say so.- # _model_is_local_gguf has already answered for every local id.+ # A directory may be named anything, so only the filesystem can say that ~/models/my-gguf holding+ # safetensors is a sentence-transformers model.if is_local_path(model):return Falseexcept Exception: # noqa: BLE001 - unparseable path is not a repo id eitherreturn False- # config's predicate, not a second opinion: gguf_repo_candidates already- # counts "gguf" as a whole name segment, so owner/GGUF-model is a GGUF repo- # there and must not be a sentence-transformers one here.+ # config's predicate, not a second opinion: gguf_repo_candidates already counts "gguf" as a whole+ # name segment, so owner/GGUF-model is a GGUF repo there and must not be one here.return config._names_gguf(model.strip().rstrip("/").rsplit("/", 1)[-1])@@ -742,9 +688,8 @@ def _resolve_auto_for_model(model_name: str | None = None) -> str:records that choice; the hardware default would send it to llama-server,which has nothing to open."""model = model_name or config.effective_embedding_model()- # Ahead of the stored record: the filesystem was asked, not guessed at, and a- # stored ST record for a .gguf can only come from a force-save that failed.- # Only ``auto`` consults this, so an explicit RAG_EMBED_BACKEND still wins.+ # Ahead of the stored record, since the filesystem was asked rather than guessed at; only auto+ # consults this, so an explicit RAG_EMBED_BACKEND still wins.if _model_is_local_gguf(model):return "llama-server"try:@@ -756,13 +701,8 @@ def _resolve_auto_for_model(model_name: str | None = None) -> str:key = stored.strip().lower()if key in _ST_ALIASES or key in _LLAMA_ALIASES:return key- # Nothing validated, so read the name. _resolve_auto answers- # "sentence-transformers" on any GPU host, and a GGUF repo resolved as ST is- # rejected for publishing no safetensors, saveable only over that error --- # which leaves it pending, and _get() refuses a pending ST model before the- # llama fallback can run. Below the stored record, though, since a name is- # only a guess: a repo with a torn GGUF family and usable safetensors has a- # validated ST plan that overruling would fail as "not downloaded".+ # Below the stored record, since a name is only a guess: a repo with a torn GGUF family and usable+ # safetensors has a validated ST plan.if _model_names_gguf_repo(model):return "llama-server"return _resolve_auto()@@ -779,12 +719,8 @@ def sentence_transformers_runtime_available() -> bool:"""try:_load_device()- # Not under ``_lock``. Both the resolve GET and the PUT run this before any- # deadline-bounded Hub call, and ``_get`` holds ``_lock`` across an entire- # SentenceTransformer construction, download included: sharing it meant- # opening or saving Settings during a slow first load blocked for as long- # as that load took, past every deadline the resolver applies to itself.- # The stub keeps its ordering guarantee under its own lock.+ # Not under _lock: _get holds it across an entire SentenceTransformer construction, download+ # included, so sharing it blocked Settings for the length of a slow first load._install_torchao_stub_once()from sentence_transformers import SentenceTransformer@@ -809,10 +745,8 @@ def resolved_backend_for_model(model_name: str) -> str:forced = _forced_backends.get(model_name)key = forced or (_resolve_auto_for_model(model_name) if raw in _AUTO_ALIASES else raw)if key in _ST_ALIASES and not sentence_transformers_runtime_available():- # Match _build_st_backend_or_fallback before Settings commits an- # ST-only pending download. Without a real llama binary ST remains the- # only possible plan, and its eventual error is more useful than a- # fabricated GGUF destination.+ # Without a real llama binary ST is the only possible plan, and its eventual error is more useful+ # than a fabricated GGUF destination.if _llama_server_runtime_available():key = "llama-server"if key in _LLAMA_ALIASES:@@ -851,7 +785,7 @@ def _build_st_backend_or_fallback(model_name: str | None = None):backend.warm(model_name = model_name)return backendexcept (UnsafeEmbeddingModelError, EmbeddingModelDownloadRequiredError):- raise # policy failures must hard-fail, not change backend/download+ raiseexcept Exception as st_err: # noqa: BLE001 - any ST/torch import or load failurefallback = _try_make_llama_backend()if fallback is None:@@ -874,7 +808,7 @@ def _switch_to_llama_fallback(err, model_name: str | None = None):old = Nonewith _backend_lock:if not isinstance(_backend, _SentenceTransformersBackend):- return _backend # another thread already swapped (or was never ST)+ return _backendfallback = _try_make_llama_backend()if fallback is None:return None@@ -887,8 +821,8 @@ def _switch_to_llama_fallback(err, model_name: str | None = None):old, _backend = _backend, fallback_forced_backends[failed_model] = "llama-server"_backend_key = _backend_cache_key(_raw_backend(), "llama-server")- # The failed ST wrapper is no longer published, but its module-level model- # would otherwise survive even a later unload of the llama replacement.+ # The failed ST wrapper is no longer published, but its module-level model would survive even a+ # later unload of the llama replacement._dispose_replaced_backend(old, fallback)return fallback@@ -934,9 +868,8 @@ def _dispose_replaced_backend(old, new = None) -> None:if old is None or old is new:returnif isinstance(old, _SentenceTransformersBackend):- # Two ST wrappers share the module-level model. A replacement ST was- # already warmed against the new name, so clearing it here would discard- # the model we just selected.+ # Two ST wrappers share the module-level model and the replacement is already warmed, so clearing it+ # here would discard the model just selected.if not isinstance(new, _SentenceTransformersBackend):_release_st_model()return@@ -983,13 +916,11 @@ def _get_backend(model_name: str | None = None):)_backend = newif key in _ST_ALIASES and _is_llama_backend(new):- # The ST warm probe fell back before producing vectors. Pin that- # actual backend for this model, but let a different model retry ST.+ # Pin the backend the warm probe actually fell back to, but let a different model retry ST.key = "llama-server"_forced_backends[model] = key_backend_key = _backend_cache_key(raw, key)- # A llama shutdown can wait for an in-flight encode, so keep that wait out- # of the global publication lock. New callers already see ``new``.+ # A llama shutdown can wait for an in-flight encode, so keep that wait out of the global publication lock._dispose_replaced_backend(old, new)return new@@ -1013,16 +944,13 @@ def backend_is_loaded(model_name: str | None = None) -> bool:"""backend = _backendif backend is None:- # No published backend does not mean nothing is loaded: an ST wrapper- # retired by an unload that lands between _get_backend() and its encode- # reloads the module-level model with nothing to publish. Answering False- # stranded those weights, since release_backend returns on the same test.+ # No published backend does not mean nothing is loaded: answering False stranded module-level+ # weights, since release_backend returns on the same test.if model_name is None:return _model is not Nonereturn _model is not None and _name == model_nameif model_name is None:- # A llama backend whose process is gone is not resident, whichever model- # was asked about.+ # A llama backend whose process is gone is not resident, whichever model was asked about.if _is_llama_backend(backend):try:return bool(backend._process_alive())@@ -1033,8 +961,8 @@ def backend_is_loaded(model_name: str | None = None) -> bool:return _model is not None and _name == model_nameif _is_llama_backend(backend):try:- # The object keeps _model_repo after the subprocess exits or is- # reaped, so the repo match alone would call a dead server resident.+ # The object keeps _model_repo after the subprocess exits, so a repo match alone would call a dead+ # server resident.if not backend._process_alive():return Falsereturn backend._model_repo == config.effective_gguf_repo_for_embedding_model(model_name)@@ -1051,14 +979,13 @@ def release_backend() -> bool:retry already covers a server that went away under it."""global _backend, _backend_keywith _backend_lock:- # Unload is an explicit fresh start, so a past runtime fallback stops pinning- # the choice and the saved model picks its backend again.+ # Unload is an explicit fresh start, so a past runtime fallback stops pinning the choice and the saved+ # model picks its backend again._forced_backends.clear()backend, _backend, _backend_key = _backend, None, Noneif backend is None:- # Nothing published, but the module-level model can still be there (see- # backend_is_loaded). Freeing it here is what keeps that leak from being- # permanent; a no-op when nothing is resident.+ # Nothing published, but the module-level model can still be there (see backend_is_loaded);+ # freeing it here is what keeps that leak from being permanent.return _release_st_model()_dispose_replaced_backend(backend)return True@@ -1083,10 +1010,8 @@ def active_backend_is_llama(model_name: str | None = None) -> bool:with _backend_lock:backend = _backendif backend is not None:- # A backend exists: report what it ACTUALLY is. A concrete- # sentence-transformers backend must return False even if the- # resolver would now pick llama, so its pickle stays gated. If the- # llama import fails we cannot be llama, so fall to the safe False.+ # Report what the backend ACTUALLY is: a concrete sentence-transformers backend must return False+ # even if the resolver would now pick llama, so its pickle stays gated.try:from .embed_llama_server import LlamaServerBackendexcept Exception: # noqa: BLE001 - llama plumbing import must never block@@ -1176,7 +1101,7 @@ def encode_with_identity(vectors = encode(texts, model_name = model_name, normalize = normalize)served = getattr(_served_by, "backend", None)name = model_name or config.effective_embedding_model()- if served is None: # a stubbed encode never reached a backend+ if served is None:return vectors, embedding_identity(name)return vectors, _identity(_is_llama_backend(served), name)@@ -1241,8 +1166,7 @@ def token_counter(model_name: str | None = None) -> Callable[[str], int]:):raisewith counter_lock:- # Another counting thread may already have replaced the retired- # counter while this one was leaving it.+ # Another counting thread may already have replaced the retired counter.if state[0] is served_backend:replacement = _get_backend(model_name)if replacement is served_backend:diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.pyindex fe8d35fab53..13229b81dae 100644--- a/studio/backend/core/rag/folder_sync.py+++ b/studio/backend/core/rag/folder_sync.py@@ -575,8 +575,7 @@ def _retire_scope_rows("INSERT OR IGNORE INTO linked_folder_retired_scopes(scope, retired_at) VALUES(?, ?)",(scope, _now()),)- # the ownership check and this write cannot share a transaction across two databases,- # so a folder another process linked after that check keeps its own state+ # the ownership check and this write cannot share a transaction across two databasesbound = "" if linked_before is None else " AND created_at<=?"params = () if linked_before is None else (linked_before,)if folders_exist:@@ -739,8 +738,8 @@ def reconcile_retired_scopes(project_exists) -> dict[str, list[str]]:if not project_id:continuetry:- # under the lock create_folder and upload admission take, and rechecked inside- # it: a project recreated with the same id must not have its new folders retired+ # rechecked inside the lock create_folder takes: a project recreated with the same id must not have+ # its new folders retiredwith _scope_lock(scope):checked_at = _now()if project_exists(project_id):@@ -766,8 +765,7 @@ def reconcile_retired_scopes(project_exists) -> dict[str, list[str]]:else:continueif owner_exists:- # the id came back, so the scope has an owner again and its retirement,- # which only ever meant "ownerless", must not keep gating that owner+ # retirement only ever meant "ownerless", so a returned id must not keep gating its ownerif unretire_scope(scope):restored.append(scope)elif delete_retired_scope(scope):@@ -979,23 +977,22 @@ def _scan(continueif os.path.splitext(entry.name)[1].lower() not in config.UPLOAD_EXTS:continue- # Finder metadata carries the document's extension, and a text parser reads it- # with errors="replace", so it would be embedded and cited as a real chunk.+ # Finder metadata carries the document's extension, so a text parser would embed and cite it as a+ # real chunk.if is_appledouble_metadata(Path(full)):continuest = entry.stat(follow_symlinks = False)from_path = Falseif st.st_ino in (None, 0):- # Windows builds DirEntry from FindFirstFileW, which carries no file index, so- # DirEntry.stat leaves st_dev/st_ino at 0. os.lstat opens a handle and fills- # both; without it the stored identity is (0, 0) forever and a replacement of- # identical size and mtime never reaches the reconciliation work list.+ # Windows DirEntry.stat leaves st_dev/st_ino at 0 (FindFirstFileW carries no file index); os.lstat+ # fills both, else the stored identity is (0, 0) forever.try:st = os.lstat(full)from_path = Trueexcept OSError:- # A file that vanished mid-scan must still reach _snapshot as a failure- # rather than abort the scan, which is never authoritative for deletion.+ # A file that vanished mid-scan must reach _snapshot as a failure: a scan is never+ # authoritative for+ # deletion.passrel = os.path.relpath(full, root).replace(os.sep, "/")found[rel] = {@@ -1004,8 +1001,8 @@ def _scan("mtime_ns": st.st_mtime_ns,"device": st.st_dev,"inode": st.st_ino,- # A recovered identity is comparable to the next scan's, but not to os.fstat's:- # shared-folder and WebDAV drivers report different ids for the two call paths.+ # A recovered identity is comparable to the next scan's but not to os.fstat's: shared-folder and+ # WebDAV drivers report different ids per call path."identity_from_path": from_path,}if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:@@ -1039,10 +1036,8 @@ def _snapshot(root: str, metadata: dict) -> str:metadata["inode"],)actual = (before.st_size, before.st_mtime_ns, before.st_dev, before.st_ino)- # Only an identity os.fstat can reproduce is comparable here: os.scandir reports none at all- # on Windows, and the os.lstat _scan falls back to disagrees with os.fstat on file systems- # without stable file ids. Size and mtime still gate those, and the post-copy check below is- # fstat to fstat either way.+ # Only an identity os.fstat can reproduce is comparable: os.scandir reports none on Windows, and the+ # lstat fallback disagrees with fstat on file systems without stable ids.usable = metadata["inode"] not in (None, 0) and not metadata.get("identity_from_path")compared = 4 if usable else 2if actual[:compared] != expected[:compared]:@@ -1210,8 +1205,8 @@ def _install_mapping(raisefinally:conn.close()- # Only after the commit: a rollback here restores a live searchable document,- # unlike the delete paths where the surviving row is the retry queue.+ # Only after the commit: a rollback here restores a live searchable document, unlike the delete+ # paths where the surviving row is the retry queue.for stored_path in replaced:_remove_snapshot(stored_path)@@ -1257,8 +1252,8 @@ def _delete_mapping(folder_id: str, rel: str) -> None:raisefinally:conn.close()- # After the commit, as in _install_mapping: a rollback here restores a live- # document, and an auto_sync=0 folder may not reconcile again for a long time.+ # After the commit: a rollback restores a live document, and an auto_sync=0 folder may not reconcile+ # again for a long time._remove_snapshot(row["stored_path"])@@ -1607,9 +1602,8 @@ def _queue_successor(job_id: str) -> None:def reconcile_folder(job_id: str) -> None:"""Reconcile and persist a terminal folder state for every unexpected failure."""if _claim_job(job_id) is None:- # The caller may already hold an activated claim (the queue claims before- # dispatching); drop it here or the heartbeat renews it for the process- # lifetime and an unlink waits on a job that will never run.+ # Drop an already-activated claim here, or the heartbeat renews it for the process lifetime and an+ # unlink waits on a job that will never run.job_leases.release(job_leases.FOLDER_SYNC, job_id)returnlease_lost = False@@ -1725,8 +1719,8 @@ def _worker(stop_event: threading.Event | None = None, project_exists = None) ->try:job = _next_job()except Exception:- # Writer-lock contention must not retire the only worker: the- # initialization and periodic paths already back off and retry.+ # Writer-lock contention must not retire the only worker; the initialization and periodic paths+ # retry.logger.warning("linked-folder queue selection failed", exc_info = True)stop_event.wait(1.0)continue@@ -1818,8 +1812,8 @@ def start_auto_sync(if not rag_db.rag_available():return Falseexcept sqlite3.OperationalError:- # The worker retries initialization, so transient database contention must- # not turn a one-shot startup preflight into a process-lifetime outage.+ # The worker retries initialization, so transient database contention must not turn a startup+ # preflight into a process-lifetime outage.passwith _thread_lock:retired = _thread if _thread is not None and _thread.is_alive() else Nonediff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.pyindex cfd7c2cb337..cd846ff88d9 100644--- a/studio/backend/core/rag/ingestion.py+++ b/studio/backend/core/rag/ingestion.py@@ -21,16 +21,15 @@ from . import captioner, chunking, config, embeddings, job_leases, parsers, storlogger = logging.getLogger(__name__)-# Per-job event queues, drained by job_events; ``None`` ends the stream.+# Per-job event queues, drained by job_events; None ends the stream._jobs: dict[str, "queue.Queue"] = {}_workers: dict[str, threading.Thread] = {}_jobs_lock = threading.Lock()-_EMBED_BATCH = 64 # bounds peak memory+_EMBED_BATCH = 64_SHA256_HEX_RE = re.compile(r"^[0-9a-f]{64}$")-# Poll with a timeout so the generator wakes periodically to detect a gone-# client or a terminal job whose worker died without the None sentinel.+# Poll with a timeout so the generator notices a gone client or a worker that died without the None sentinel._SSE_POLL_SECONDS = 1.0_TERMINAL_JOB_STATUSES = {"completed", "failed", "cancelled"}@@ -240,14 +239,13 @@ def _run(caption_on = config.CAPTION_IMAGES if caption is None else caption# Skip all figure work (PDF rasterization included) without a vision model.if caption_on and is_pdf and captioner.vision_endpoint() is not None:- # Tile figure pages, transcribe+describe each tile, then merge/dedup/splice- # into the page text so small labels and every sub-figure are captured.+ # Tile figure pages, transcribe+describe each tile, then merge/dedup/splice into the page text so+ # small labels and every sub-figure are captured.try:fig_pages = parsers.pages_with_figures(stored_path,max_pages = config.CAPTION_MAX_PAGES,- # Skip only pages OCR actually transcribed (it covers them whole); a- # scanned figure page past the OCR cap or with empty OCR still tiles.+ # Skip only pages OCR actually transcribed; a scanned figure page past the OCR cap still tiles.exclude_pages = ocred,)tiles = (@@ -281,8 +279,8 @@ def _run(count = count,)if not chunks:- # An empty parse still completes the document and retires the one it replaces, so it- # needs the same guard as the chunk write below.+ # An empty parse still completes the document and retires the one it replaces, so it needs the same+ # guard as the chunk write.if _abort_if_document_deleted(conn, job_id, document_id):return# inside the write transaction the guard opened, as _progress does@@ -295,8 +293,8 @@ def _run(return_progress(conn, job_id, "embedding", 0.5)- # An ST encode failure swaps the process to llama-server, so the embedder that- # produced these vectors is only known once they exist.+ # An ST encode failure swaps the process to llama-server, so the embedder that produced these+ # vectors is only known once they exist.vectors, identity = _embed_all([c.text for c in chunks], model_name)store.set_document_embedding_model(conn, document_id, identity)@@ -314,8 +312,8 @@ def _run(if _abort_if_document_deleted(conn, job_id, document_id):returnstore.add_chunks(conn, scope, document_id, chunks, vectors, regions)- # add_chunks commits, which releases the lock taken above, so retake it before reporting- # success: a delete landing in that gap must not be recorded as a completed ingestion.+ # add_chunks commits and releases the lock, so retake it: a delete landing in that gap must not be+ # recorded as a completed ingestion.if _abort_if_document_deleted(conn, job_id, document_id):returnstore.set_document_status(conn, document_id, "completed", num_chunks = len(chunks))@@ -382,27 +380,20 @@ def start_ingestion(sha = content_hash or _sha256_file(stored_path)conn = rag_db.get_connection()try:- # Named before the transaction opens, because naming the embedder on a fresh- # process is slow work that touches no database: it searches for the- # llama-server binary, runs nvidia-smi under a ten second timeout, and on a- # host without it imports torch. Under BEGIN IMMEDIATE that is a RESERVED- # lock held for all of it, and connections wait only busy_timeout (5s) for- # one, so a concurrent ingest or job heartbeat fails with "database is- # locked" instead of queueing.+ # Name the embedder before BEGIN IMMEDIATE: it runs nvidia-smi and may import torch, and holding a+ # RESERVED lock that long fails concurrent writers with "database is locked".effective_model = model_name or config.effective_embedding_model()effective_identity = embeddings.embedding_identity(effective_model)- # Serialize admission with durable scope retirement across backend- # processes. The job lease is committed in the same transaction as the- # document, so cleanup never observes an unowned in-flight document.+ # The job lease is committed in the same transaction as the document, so cleanup never observes an+ # unowned in-flight document.conn.execute("BEGIN IMMEDIATE")if conn.execute("SELECT 1 FROM linked_folder_retired_scopes WHERE scope=?", (scope,)).fetchone():conn.rollback()raise RuntimeError("Owning scope is being deleted")- # (old_document_id, old_stored_path) replaced by this upload; deleted by- # the worker only after the replacement completes, so a failed re-index- # never destroys the still-searchable original.+ # (old_document_id, old_stored_path) replaced by this upload; deleted by the worker only after the+ # replacement completes, so a failed re-index never destroys the still-searchable original.replaces: tuple[str, str | None] | None = Noneexisting = store.document_by_hash(conn, scope, sha) if dedupe else Noneif existing is not None:@@ -410,10 +401,9 @@ def start_ingestion(empty_completed = (doc is not None and doc.get("status") == "completed" and not doc.get("num_chunks"))- # Vectors from a different embedder are stale; re-uploading must- # re-index, not dedupe. NULL (legacy rows) is assumed current. Only- # completed rows are replaceable: a pending/running duplicate has a- # live worker whose writes must not land on a deleted document.+ # Vectors from a different embedder are stale, so re-uploading must re-index; NULL (legacy rows)+ # is assumed current. Only completed rows are replaceable, since a running duplicate's writes must+ # not land on a deleted document.stale_model = (doc is not Noneand doc.get("status") == "completed"@@ -422,9 +412,7 @@ def start_ingestion())if empty_completed or stale_model:- # A prior ingest of identical bytes yielded zero chunks (e.g. a scanned- # PDF uploaded before a vision model loaded), or was embedded with a- # different model. Re-ingest, don't dedupe.+ # Zero chunks previously, or a different embedder: re-ingest rather than dedupe.replaces = (existing, doc.get("stored_path"))else:job_id = _new_job(conn, existing, scope, status = "completed", progress = 1.0)@@ -480,9 +468,8 @@ def start_ingestion(return document_id, job_idworker = threading.Thread(target = _run,- # effective_model (not the raw model_name) pins the embedder for the- # whole job: a Settings change mid-ingestion must not switch tokenizer- # or embedder between batches of one document.+ # effective_model, not the raw model_name, pins the embedder for the whole job: a Settings change+ # mid-ingestion must not switch tokenizer or embedder between batches.args = args,daemon = True,)@@ -630,11 +617,8 @@ def job_events(job_id: str):try:row = get_job_status(job_id)except Exception: # noqa: BLE001- # A transient status read (e.g. the DB momentarily locked) must- # not abort the stream: routes/rag.py would turn the raised- # exception into a terminal {type: error} frame and the UI would- # drop a document whose worker is still running. Heartbeat and- # retry on the next poll instead.+ # A transient status read must not abort the stream: routes/rag.py would turn it into a terminal+ # error frame and the UI would drop a document whose worker is still running.logger.warning("job_events status read failed for %s; continuing", job_id, exc_info = True)@@ -651,19 +635,14 @@ def job_events(job_id: str):breakyield eventfinally:- # Drop the queue once nothing more will be emitted into it: either a- # terminal exit, or a disconnect after the job already finished (the UI- # stops on the terminal event, before [DONE], so terminal is still False- # here -- _run writes the terminal DB status before emitting it). Keep it- # only while the worker is still running, so an early disconnect can- # reconnect and resume its events.+ # Keep the queue while the worker runs so an early disconnect can reconnect and resume; terminal is+ # still False at disconnect, since _run writes the DB status before emitting it.if not terminal:try:row = get_job_status(job_id)terminal = row is None or row.get("status") in _TERMINAL_JOB_STATUSESexcept Exception: # noqa: BLE001- # Can't confirm terminality (transient DB error) -- keep the queue so- # a reconnect can resume rather than orphaning a live worker's events.+ # Cannot confirm terminality, so keep the queue rather than orphan a live worker's events.terminal = Falseif terminal:with _jobs_lock:diff --git a/studio/backend/core/rag/locators.py b/studio/backend/core/rag/locators.pyindex 9331bb15ac1..d1a00dc9576 100644--- a/studio/backend/core/rag/locators.py+++ b/studio/backend/core/rag/locators.py@@ -16,8 +16,7 @@ from dataclasses import dataclassfrom pathlib import Pathfrom typing import Any-# Anchor: up to MAX interior words from the chunk's start, shrunk toward MIN-# to recover a unique match.+# Anchor: up to MAX interior words from the chunk's start, shrunk toward MIN to recover a unique match.MAX_ANCHOR_WORDS = 12MIN_ANCHOR_WORDS = 4@@ -95,7 +94,7 @@ def _rects_from_words(page_words: list, indices: list[int], pw: float, ph: floatfor j in indices:w = page_words[j]x0, y0, x1, y1 = float(w[0]), float(w[1]), float(w[2]), float(w[3])- key = (w[5], w[6]) # block, line+ key = (w[5], w[6])box = lines.get(key)if box is None:lines[key] = [x0, y0, x1, y1]diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.pyindex 0b42906b859..caf62b708b7 100644--- a/studio/backend/core/rag/parsers.py+++ b/studio/backend/core/rag/parsers.py@@ -70,11 +70,9 @@ def _html(raw: str) -> list[Page]:return [_page("\n".join(parser.out), 1)]-# pymupdf4llm rebuilds text from positioned glyphs, which mangles complex-shaping-# scripts (RTL Arabic/Hebrew emerge as shaped Presentation Forms, Indic matras drop to-# U+FFFD) and can silently drop most of a heavy-RTL page. When Markdown trips these-# signals we fall back to PyMuPDF's logical-order get_text(). Thresholds mirror the chat-# extractor guard (unslothai/unsloth#5351 review).+# pymupdf4llm rebuilds text from positioned glyphs and mangles complex-shaping scripts (RTL forms,+# Indic matras to U+FFFD), so fall back to PyMuPDF's logical-order get_text(); thresholds mirror+# unslothai/unsloth#5351._SHAPED_PRESENTATION_FORMS = re.compile("[\ufb1d-\ufdff\ufe70-\ufefc]")_PDF_FALLBACK_MIN_BAD_GLYPHS = 5_PDF_FALLBACK_BAD_GLYPH_RATIO = 0.0005@@ -130,7 +128,7 @@ def _pdf(want_images: bool,max_pages: int | None = None,) -> tuple[list[Page], list[ParsedImage], int]:- import fitz # PyMuPDF+ import fitzpages: list[Page] = []images: list[ParsedImage] = []@@ -152,9 +150,9 @@ def _pdf(page = doc[page_number]plain = page.get_text("text") or ""candidate = md[i] if md else ""- # Prefer layout-aware Markdown (keeps tables/headings legible for retrieval),- # but drop to PyMuPDF's logical-order text when Markdown is off/empty or when- # pymupdf4llm mangled it (RTL/Indic) or dropped most of the page.+ # Prefer layout-aware Markdown (keeps tables/headings legible for retrieval), but drop to PyMuPDF's+ # logical-order text when Markdown is off/empty or when pymupdf4llm mangled it (RTL/Indic) or+ # dropped most of the page.if (candidateand not _markdown_corrupted(candidate)@@ -297,7 +295,7 @@ def render_pdf_figure_tiles(wanted = [int(n) for n in page_numbers]if not wanted:return []- rows, cols = max(1, int(rows)), max(1, int(cols)) # never divide by zero+ rows, cols = max(1, int(rows)), max(1, int(cols))try:import pymupdfexcept Exception:@@ -386,19 +384,19 @@ def _docx_table_rows(table) -> list[str]:from docx.text.paragraph import Paragraphrows: list[str] = []- seen: set = set() # <w:tc> already emitted; dedups merges spanning columns or rows+ seen: set = set()for row in table.rows:cells: list[str] = [""] * getattr(row, "grid_cols_before", 0)- trailing: list[str] = [] # nested rows + any post-nested text, kept in order+ trailing: list[str] = []for cell in row.cells:- # A merged cell shares one <w:tc> across the columns and rows it spans:- # emit its text once, then placeholders, so columns and rows stay aligned.+ # A merged cell shares one <w:tc> across its span: emit its text once, then placeholders, so columns+ # and rows stay aligned.if cell._tc in seen:cells.append("")continueseen.add(cell._tc)- # Paragraph text before the first nested table is the aligned field; the- # nested table and anything after it flatten below the row, in order.+ # Paragraph text before the first nested table is the aligned field; the nested table and anything+ # after it flatten below the row.field: list[str] = []after_table = Falsefor item in cell.iter_inner_content():@@ -406,7 +404,7 @@ def _docx_table_rows(table) -> list[str]:after_table = Truetrailing.extend(_docx_table_rows(item))elif isinstance(item, Paragraph):- text = " ".join(item.text.split()) # collapse in-cell newlines+ text = " ".join(item.text.split())if text:(trailing if after_table else field).append(text)cells.append(" ".join(field)) # empty cells kept so columns line updiff --git a/studio/backend/core/rag/retrieval.py b/studio/backend/core/rag/retrieval.pyindex 505e6ecb7d2..b8210c209e9 100644--- a/studio/backend/core/rag/retrieval.py+++ b/studio/backend/core/rag/retrieval.py@@ -58,8 +58,8 @@ def retrieve_dense() -> list[Hit]:k = k or config.TOP_K_DENSEeffective = model_name or config.effective_embedding_model()- # The identity comes from the encode, so it names the backend that actually served- # this query even if a concurrent ST failure swapped the process meanwhile.+ # The identity comes from the encode, so it names the backend that served this query even if a+ # concurrent ST failure swapped the process meanwhile.vectors, identity = embeddings.encode_with_identity([query], model_name = effective, normalize = True)diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.pyindex a1360df20a2..c26e0160807 100644--- a/studio/backend/core/rag/store.py+++ b/studio/backend/core/rag/store.py@@ -64,8 +64,7 @@ def _now() -> str:_TOKEN = re.compile(r"\w+", re.UNICODE)-# Straight and curly double quotes, single quotes and backticks: how a user names a word-# instead of using it. Non-greedy and single-line so an unclosed quote spans nothing.+# Quotes mark a word being named rather than used; non-greedy and single-line so an unclosed quote spans nothing._QUOTED = re.compile(r"\"([^\"\n]+)\"|\u201c([^\u201d\n]+)\u201d|'([^'\n]+)'|`([^`\n]+)`")@@ -76,11 +75,9 @@ def _match_query(query: str) -> str:return " OR ".join(f'"{t}"' for t in toks)-# A closed list of function words, so behaviour is identical on every install. Nothing-# here can carry the subject of a question.-#-# `no` and `not` are deliberately NOT here: they carry the whole difference in "what did I-# say not to delete?", where dropping them leaves only terms BM25 floors at 1e-6.+# A closed list of function words, so behaviour is identical on every install. no and not are+# deliberately NOT here: they carry the whole difference in "what did I say not to delete?",+# where dropping them leaves only terms BM25 floors at 1e-6._ARCHIVE_STOPWORDS = frozenset("""a about all am an and any are as at be been being but by can could did do does doing@@ -90,11 +87,8 @@ was we were what when where which who why will with would you your""".split())-# Identifier-ish: a token containing a digit (ZQXVARA123, 9134) or an underscore, or one-# written in capitals and long enough not to be an "I" or an "OK". These are the tokens a-# person uses when they mean one specific thing. Containing a digit, not mixing one with a-# letter: a purely numeric subject ("the current value of 9134") otherwise has no shape at-# all once the capitals rule needs contrast.+# Identifier-ish tokens are how a person names one specific thing; a digit alone counts, since a+# purely numeric subject has no other shape._HAS_DIGIT = re.compile(r"\d", re.UNICODE)_HAS_LETTER = re.compile(r"[^\W\d_]", re.UNICODE)@@ -115,9 +109,7 @@ def _is_identifier(token: str, raw_tokens: frozenset[str]) -> bool:if "_" in token:return Trueif _HAS_DIGIT.search(token):- # A bare number needs LENGTH to be a name, the same bar the capitals rule carries.- # Without it "answer in 2 sentences" filters the archive on "2". A token mixing- # letters and digits is a name at any length.+ # A bare number needs LENGTH to be a name, else "answer in 2 sentences" filters the archive on "2".return bool(_HAS_LETTER.search(token)) or len(token) >= 3return len(token) >= 3 and token.upper() in raw_tokens@@ -158,15 +150,13 @@ def conversation_match_queries(query: str) -> list[str]:tokens = list(dict.fromkeys(_TOKEN.findall(query.lower())))if not tokens:return []- # The capitals rule needs CONTRAST: in an all-caps line every word passes it, so the- # filter ORs in "what" and "the" and filters nothing. Shape still decides, so- # ZQXVARA123 is an identifier either way; only shouted prose changes.+ # Identifier-ish: a token containing a digit (ZQXVARA123, 9134) or an underscore, or one in+ # capitals and long enough not to be an "I" or an "OK". The capitals rule needs CONTRAST: in an+ # all-caps line every word passes it and the filter filters nothing.raw_tokens = frozenset() if query == query.upper() else frozenset(_TOKEN.findall(query))identifiers = [t for t in tokens if _is_identifier(t, raw_tokens)]- # A QUOTED word is the subject whatever the stopword list thinks: `What did I say about- # "this"?` otherwise reduces to '"say"' and an archived `Use this endpoint` is- # unreachable. Quoted tokens stay out of `identifiers`, so this widens only the- # permissive pass.+ # A QUOTED word is the subject whatever the stopword list thinks; quoted tokens stay out of+ # identifiers, so this widens only the permissive pass.quoted = frozenset(tokenfor match in _QUOTED.findall(query.lower())@@ -568,25 +558,18 @@ def search_lexical(if not scopes:return []placeholders = ",".join("?" * len(scopes))- # One snapshot for the gate and the read: WAL pins it at the transaction's first- # read, so a scope retired in between cannot land rows in a result the gate already- # decided to run unfiltered. A caller's own transaction is used instead.+ # One snapshot for the gate and the read: WAL pins it at the transaction's first read, so a scope+ # retired in between cannot land rows in a result the gate decided to run unfiltered.own_read_txn = not conn.in_transaction+ # Read-only, but it has to end: an open snapshot blocks WAL checkpointing.if own_read_txn:conn.execute("BEGIN")try:- # The filtered form joins chunks and documents and runs both subqueries for every- # matched row BEFORE the LIMIT, so it costs more the commoner the query terms are.- # With nothing linked that work is provably wasted (linked_folder_rows_exist).+ # The filtered form runs both subqueries for every matched row BEFORE the LIMIT, and with nothing+ # linked that work is provably wasted (linked_folder_rows_exist).if oldest_first:- # The mirror of `newest_first`, for the same reason: rowid ordering is- # scrambled by a re-embed, which can push the oldest turn out of the window.- # NULLs first, since a row archived before the column existed is the oldest.- # Then `created_at`, then the chunk id: on a LEGACY archive every ordinal is- # NULL, so every term after the score was constant and both halves of the- # two-ended fetch returned the same arbitrary subset, which `_both_ends` then- # deduplicated -- the 256-candidate strategy reaching one end twice and the- # later revisions not at all.+ # Order by archive ordinal, not rowid, which a re-embed scrambles; NULLs first as oldest, then+ # created_at and chunk id, else on a legacy archive both halves return the same subset.sql = (f"SELECT chunks_fts.chunk_id, bm25(chunks_fts) AS s FROM chunks_fts "f"JOIN chunks c ON c.id=chunks_fts.chunk_id "@@ -596,9 +579,7 @@ def search_lexical(f"d.created_at ASC, chunks_fts.chunk_id ASC LIMIT ?")elif newest_first:- # Ordered by archive ordinal, not rowid: rowid is insertion order, and a- # re-embed reinserts a chunk, so a rowid DESC window returned ordinals- # 70, 193, 116, ... and missed the newest turn. NULLs sort last, as oldest.+ # rowid is insertion order and a re-embed reinserts a chunk, so a rowid DESC window missed the newest turn.sql = (f"SELECT chunks_fts.chunk_id, bm25(chunks_fts) AS s FROM chunks_fts "f"JOIN chunks c ON c.id=chunks_fts.chunk_id "@@ -627,7 +608,6 @@ def search_lexical()rows = conn.execute(sql, (mq, *scopes, k)).fetchall()finally:- # Read-only, but it has to end: an open snapshot blocks WAL checkpointing.if own_read_txn:conn.commit()# bm25() is negative (more negative = better); flip to higher-is-better.@@ -655,15 +635,12 @@ def search_dense(return []dim = rag_db.vec_table_dim(conn)if dim is not None and dim != len(vector):- # Embedding model switched widths and nothing re-indexed yet; the stale- # table cannot answer new-model queries (vec0 errors on the MATCH).+ # The stale table cannot answer new-model queries: vec0 errors on the MATCH.return []- # The pre-tag spelling of the same request, kept acceptable so an existing index- # keeps answering after an upgrade.+ # The pre-tag spelling of the same request, kept acceptable so an existing index keeps answering after an upgrade.untagged = config.embedding_identity_model(embedding_model) or embedding_model- # dict.fromkeys keeps the caller's order and collapses a scope named twice, which- # is now load-bearing: widening carries per-scope state, and a repeat would both- # multiply that scope's fetch twice per round and emit its hits twice into the merge.+ # dict.fromkeys collapses a scope named twice: a repeat would multiply that scope's fetch and emit+ # its hits twice into the merge.scopes = list(dict.fromkeys(s@@ -673,16 +650,10 @@ def search_dense().fetchone()))- # Over-fetch when filtering so stale-model hits don't starve the top-k. Their- # distances come from another space, so they can fill every fetched slot while- # compatible chunks sit further down the KNN list: widen until k of them survive- # the filter or the scope has nothing left to give.- #- # Per scope, not across the merge. vec0 constrains its partition key by equality,- # so each scope is its own KNN list with its own stale prefix; a project scope- # buried under another embedder's vectors would otherwise stop widening the- # moment the thread scope handed over k weak hits, and the merge would then rank- # a stronger project chunk it never fetched.+ # Stale-model hits come from another space and can fill every fetched slot, so widen until k+ # compatible ones survive the filter.+ # Per scope, not across the merge: vec0 constrains its partition key by equality, so each scope has+ # its own stale prefix.kept: dict[str, list[tuple[str, float]]] = {}fetches = dict.fromkeys(scopes, max(k * 3, k + 10))pending = list(scopes)@@ -710,8 +681,7 @@ def search_dense(return out[:k]-# Widening is bounded: past this many nearest neighbours per scope the scope is-# effectively another embedder's, and a re-upload is the answer, not a longer scan.+# Past this many nearest neighbours the scope is effectively another embedder's, and a re-upload is the answer._MAX_DENSE_FETCH = 4096# One id per bound parameter, kept under the oldest SQLITE_MAX_VARIABLE_NUMBER._ID_BATCH = 900diff --git a/studio/backend/core/rag/tool.py b/studio/backend/core/rag/tool.pyindex 671ea3847ef..ff614285780 100644--- a/studio/backend/core/rag/tool.py+++ b/studio/backend/core/rag/tool.py@@ -147,9 +147,8 @@ def format_conversation_recall(rows, hits) -> tuple[str, list[dict]]:"page": None,"text": text,"turn": int(ordinal) + 1 if ordinal is not None else None,- # Carried so a merge of two searches can reorder one long turn's pieces;- # nothing renders them. `createdAt` is the tie-breaker `_conversation_order`- # needs, since pre-ordinal rows have `turn` None and ordinals are not UNIQUE.+ # createdAt is the tie-breaker _conversation_order needs: pre-ordinal rows have turn None and+ # ordinals are not UNIQUE."chunkIndex": _row_value(r, "chunk_index"),"createdAt": _row_value(r, "created_at"),"score": round(float(h.score), 4) if h.score is not None else None,@@ -326,15 +325,15 @@ def whole_document_context(chunks, or the total exceeds ``max_tokens``."""if not scope_thread_id:return None- # A non-positive budget means "never inject" (disable whole-doc via- # RAG_THREAD_WHOLE_DOC=0), not "inject the whole corpus unbounded".+ # A non-positive budget means "never inject" (disable whole-doc via RAG_THREAD_WHOLE_DOC=0), not+ # "inject the whole corpus unbounded".if max_tokens <= 0:return Nonescope = thread_scope(scope_thread_id)conn = rag_db.get_connection()try:- # Cheap budget pre-check (SUM, no text hydration): reject an oversized attachment- # before loading the whole corpus; all_chunks_for_scope runs only once it fits.+ # Cheap SUM pre-check so an oversized attachment is rejected before the whole corpus is hydrated;+ # all_chunks_for_scope runs only once it fits.if scope_token_estimate(conn, scope) > max_tokens:return Nonerows = all_chunks_for_scope(conn, scope)diff --git a/studio/backend/core/rag/web_rank.py b/studio/backend/core/rag/web_rank.pyindex 94a2161f481..77cf4a81832 100644--- a/studio/backend/core/rag/web_rank.py+++ b/studio/backend/core/rag/web_rank.py@@ -95,10 +95,8 @@ def retrieve_web_chunks()if not chunks:continue- # The identity has to come from the encode that produced these vectors: a- # concurrent ST failure swaps the process embedder, and reading it after- # the fact would label this page with a space its vectors were never in,- # which the hybrid query below then searches instead of filtering out.+ # The identity must come from the encode that produced these vectors: a concurrent ST failure+ # swaps the process embedder, and reading it after the fact labels the page with the wrong space.vectors, identity = embeddings.encode_with_identity([chunk.text for chunk in chunks], model_name = model, normalize = True)diff --git a/studio/backend/core/research/citations.py b/studio/backend/core/research/citations.pyindex 517afa97eb3..560dac0c343 100644--- a/studio/backend/core/research/citations.py+++ b/studio/backend/core/research/citations.py@@ -15,9 +15,8 @@ import refrom core.research.redaction import _escape_link_destination-# Unrolled rather than the equivalent (?:[^\[\]]+|\[[^\[\]]*\])* : that alternation backtracks-# catastrophically on an unterminated "[Document:" (ordinary malformed model output), and this-# runs on the event loop, so one bad report would stall all of Unsloth.+# Unrolled rather than (?:[^\[\]]+|\[[^\[\]]*\])* : that alternation backtracks catastrophically on+# an unterminated "[Document:", and this runs on the event loop._DOCUMENT_CITATION = re.compile(r"\[Document:[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]")_MARKDOWN_LINK_START = re.compile(r"\[([^\]\n]+)\]\((https?://)")_SOURCES_HEADING = re.compile(@@ -195,9 +194,8 @@ def _allowed_document_citations(sources: list[dict]) -> set[str]:def _validate_report_document_sources(report: str, sources: list[dict]) -> str:allowed = _allowed_document_citations(sources)- # Tokenize valid citations first so a ``]`` inside a filename (e.g.- # ``budget [final].pdf``) does not truncate them, then strip any remaining- # (invalid) document citations and restore the valid ones.+ # Tokenize valid citations first so a "]" inside a filename ("budget [final].pdf") does not+ # truncate them, then strip the invalid ones and restore the valid.placeholders: dict[str, str] = {}for index, citation in enumerate(sorted(allowed, key = len, reverse = True)):if citation in report:diff --git a/studio/backend/core/research/parsing.py b/studio/backend/core/research/parsing.pyindex 98f994142f4..0b6a9d5a368 100644--- a/studio/backend/core/research/parsing.py+++ b/studio/backend/core/research/parsing.py@@ -21,8 +21,8 @@ from core.research.redaction import _sanitize_public_query_STREAMED_TITLE = re.compile(r'"title"\s*:\s*"((?:[^"\\]|\\.)*)"')# One more than the plan-step cap, since the plan's own title matches too._MAX_PREVIEW_LABELS = 31-# Allows the container marker of a list item or block quote before the fence ("- ```"), else the-# open is missed and a marker quoted inside is mistaken for the real boundary.+# Allow a list-item or block-quote container marker before the fence, else the open is missed and a+# marker quoted inside is taken for the real boundary._MARKDOWN_FENCE = re.compile(r"^ {0,3}(?:(?:[-*+]|\d{1,9}[.)])[ \t]+|>[ \t]?)*(`{3,}|~{3,})")@@ -126,8 +126,7 @@ def _normalize_synthesis_audit(item.get("documentCitations"),allowed_document_citations,)- # A claim is supported only when the audit maps it to web or document evidence- # gathered in this run.+ # A claim is supported only when the audit maps it to web or document evidence gathered in this run.if claim and (urls or document_citations):supported_claims.append({@@ -301,13 +300,12 @@ def _report_after_boundary(text: str, boundary: str) -> str | None:fence_char = token[0]fence_length = len(token)continue- # CommonMark measures indentation in columns with a four-column tab stop, so one tab opens- # an indented code block just as four spaces do.+ # CommonMark measures indentation in columns with a four-column tab stop, so one tab opens an+ # indented code block just as four spaces do.prefix = content[: len(content) - len(content.lstrip(" \t"))]indentation = len(prefix.expandtabs(4))- # The prompt shows the marker in backticks, so models echo it that way. splitlines- # breaks on \x0b\x0c\x1c-\x1e\x85 , which rstrip("\r\n") leaves behind,- # so strip every whitespace form rather than let a stray one hide the boundary.+ # splitlines breaks on whitespace forms rstrip("\r\n") leaves behind, so strip every one rather than+ # let a stray one hide the boundary.if indentation <= 3 and content[len(prefix) :].strip().strip("`").strip() == boundary:boundary_line = indexif boundary_line is None:diff --git a/studio/backend/core/research/redaction.py b/studio/backend/core/research/redaction.pyindex 9b235d114f2..d7c1f011c29 100644--- a/studio/backend/core/research/redaction.py+++ b/studio/backend/core/research/redaction.py@@ -15,8 +15,7 @@ import ipaddressimport re-# Wrapper delimiters used in the decision/synthesis prompts. Any occurrence inside-# untrusted evidence is escaped so gathered content cannot close a block early.+# Any occurrence inside untrusted evidence is escaped so gathered content cannot close a prompt block early._PROMPT_DELIMITER_TAGS = re.compile(r"</?\s*(?:untrusted_web_evidence|untrusted_evidence|source_catalog"r"|document_source_catalog|conversation_context_json|research_question"@@ -25,9 +24,8 @@ _PROMPT_DELIMITER_TAGS = re.compile(r"|untrusted_synthesis_audit_json|synthesis_audit_json)\s*>",re.IGNORECASE,)-# The synthesis boundary marker. Gathered pages are quoted back into the report, so an unescaped-# copy could move the boundary and truncate the report to whatever the page put after it. Spelled-# out to avoid importing prompts; the hardening test pins it against _REPORT_BOUNDARY_MARKER.+# An unescaped copy could move the synthesis boundary and truncate the report; spelled out to avoid+# importing prompts, and pinned by the hardening test._REPORT_BOUNDARY_TAG = re.compile(r"<!--\s*UNSLOTH_FINAL_REPORT\s*-->")_QUERY_CREDENTIAL = re.compile(r"""(?ix)(?<![A-Za-z0-9])(?:api[\s_-]?key|access[\s_-]?(?:key|token)@@ -55,8 +53,7 @@ _QUERY_CREDENTIAL_SUFFIXES = ("token",)_QUERY_PUBLIC_ASSIGNMENT_SUFFIXES = ("designtoken", "cancellationtoken")-# Bearer authorization tokens carry no key=value label, so the credential pattern above misses-# them; the length floor keeps ordinary prose ("bearer of bad news") from matching.+# Bearer tokens carry no key=value label; the length floor keeps ordinary prose ("bearer of bad news") from matching._QUERY_BEARER = re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{8,}")_QUERY_EMAIL = re.compile(r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b")_QUERY_PRIVATE_ID = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")@@ -67,8 +64,8 @@ _QUERY_OPAQUE_TOKEN = re.compile(r"|hf_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{20,}"r"|AKIA[A-Z0-9]{16})\b")-# International (+CC ...) or NANP-formatted phone numbers. Requires separators or a-# leading ``+`` so bare numeric research terms are not redacted.+# International (+CC) or NANP-formatted phone numbers: requires separators or a leading "+" so+# bare numeric research terms are not redacted._QUERY_PHONE = re.compile(r"(?<!\w)\+\d[\d\s().-]{7,17}\d(?!\w)|(?<!\w)\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}(?!\w)")diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.pyindex 16c4bf665cb..05ba64b36a6 100644--- a/studio/backend/core/research_runs.py+++ b/studio/backend/core/research_runs.py@@ -74,35 +74,24 @@ _MAX_CONTEXT_CHARS = 12_000_MAX_CONTEXT_MESSAGE_CHARS = 4_000_MAX_SYNTHESIS_EVIDENCE_CHARS = 32_000# The synthesis prompt must fit the loaded context or it is silently truncated and the report-# degenerates (echoes the evidence tail). The context box accepts anything from 128 up, so the-# budget adapts: the reserve covers the generated report and every trimmable section is measured-# against what the untrimmable scaffolding leaves. Unknown context keeps the full cap.+# degenerates into an echo of the evidence tail. The context box accepts anything from 128 up, so+# the budget adapts; unknown context keeps the full cap._MIN_SYNTHESIS_EVIDENCE_CHARS = 1_500-# Trimming the question or the evidence to nothing produces a confidently empty report, so each-# keeps a floor: overflow on a tiny context is recoverable, an empty prompt is not.+# Each section keeps a floor: overflow on a tiny context is recoverable, an empty prompt is not._MIN_QUESTION_CHARS = 800_SYNTHESIS_EVIDENCE_CHARS_PER_TOKEN = 3.0_SYNTHESIS_CONTEXT_RESERVE_TOKENS = 4_096-# Below this loaded context the prompt scaffolding alone fills the window and the grounded-# report degenerates, so grounding is skipped (snippet-only) for smaller loads.+# Below this loaded context the prompt scaffolding alone fills the window, so grounding is skipped._AUTO_SCRAPE_MIN_CONTEXT_TOKENS = 8_192-# Optionally ground synthesis in page text: the top results are ingested into an ephemeral RAG-# scope (deleted after, so the user's knowledge base is untouched) and hybrid-retrieved into-# <chunk> evidence. OFF by default, opt in via UNSLOTH_RESEARCH_AUTO_SCRAPE=1: benchmarking-# showed no reliable factoid-accuracy gain over snippets on a local model (snippets usually-# already carry the fact) while adding latency. Gated per run by budgets["maxAutoScrape"]-# (absent/0 means no scrape, so existing runs keep legacy behavior). Safe only with the context-# gate in _research and the adaptive budget in _synthesis_evidence_budget; without them, denser-# evidence overflows a small context.+# OFF by default (UNSLOTH_RESEARCH_AUTO_SCRAPE=1): benchmarking showed no reliable accuracy gain+# over snippets while adding latency, and it is safe only with the context gate in _research._AUTO_SCRAPE_TOP_K = 3_AUTO_SCRAPE_TOTAL_CHARS = 6_000_WEB_RAG_TOP_N = 6_WEB_RAG_MIN_SCORE = 0.30-# Poll interval while a run waits for a local model to be (re)loaded, and the detail-# routes.inference returns when nothing is loaded (its 400 is transient, not a bad request).+# routes.inference's 400 when nothing is loaded is transient, not a bad request._MODEL_WAIT_POLL_SECONDS = 2.0-# Each wait is bounded by modelTimeoutSeconds, but a model that keeps disappearing would-# otherwise re-send forever, so cap how many times one call may wait.+# A model that keeps disappearing would re-send forever, so cap how many times one call may wait._MAX_MODEL_WAITS = 3_NO_MODEL_LOADED_DETAIL = "No model loaded"# routes.inference reports the same unloaded state this way when auto-switch finds no local match.@@ -116,23 +105,20 @@ _NAMED_MODEL_WAIT_SECONDS = 60.0# Transport keepalives prevent HTTP read timeouts without proving that a model is progressing._MODEL_FIRST_OUTPUT_TIMEOUT_SECONDS = 120.0_MODEL_OUTPUT_IDLE_TIMEOUT_SECONDS = 120.0-# Cancellation is cooperative, so bound the wait for a cancelled iterator to unwind;-# otherwise a stuck one holds a timed-out call open for the rest of the wall clock.+# Cancellation is cooperative, so bound the unwind; a stuck iterator holds a timed-out call open for+# the rest of the wall clock._STREAM_CLEANUP_TIMEOUT_SECONDS = 5.0# The SSE comment routes/inference.py sends while queued, not while the backend is silent._ADMISSION_WAIT_COMMENT = ": admission-wait"_ADMISSION_DONE_COMMENT = ": admission-done"# Queue notices arrive on the configured heartbeat, so allow for a few missed ones._ADMISSION_HEARTBEAT_MISSES = 3-# Model-loading budget when the request budget is unlimited, and its ceiling for any budget:-# the poll loop stays bounded however long generation itself may run.+# Also the ceiling for any budget, so the poll loop stays bounded however long generation itself runs._DEFAULT_MODEL_TIMEOUT_SECONDS = 900.0_MAX_MODEL_WAIT_BUDGET_SECONDS = 3600.0-# Bounds a provider's Retry-After when the run has no wall clock of its own: past the hourly-# windows providers reset on, short of parking a run and its lease on one mistaken header.+# Past the hourly windows providers reset on, short of parking a run and its lease on one mistaken header._MAX_RATE_LIMIT_WAIT_SECONDS = 3600.0-# Lifetime of the internal key one model call authenticates with. Retry waits are bounded by-# what is left of it: a key that expires mid-backoff fails auth without reaching the provider.+# Retry waits measure against this: a key that expires mid-backoff fails auth without reaching the provider._MODEL_CALL_KEY_LIFETIME_SECONDS = 2 * 60 * 60# Headroom so the named stall guards expire before HTTPX's own read timeout does._STREAM_READ_TIMEOUT_MARGIN_SECONDS = 30.0@@ -185,14 +171,13 @@ def _auto_scrape_default() -> int:return 0-# Nav menus, language sidebars, and percent-encoded link lists are not evidence and derail-# retrieval; drop link-dominated and encoded-URL lines.+# Nav menus, language sidebars and percent-encoded link lists are not evidence and derail retrieval._MD_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)")_PERCENT_ESCAPE = re.compile(r"%[0-9A-Fa-f]{2}")_LIST_PREFIX = re.compile(r"^(?:[\*\-\+•]|\d+[.)])\s")_BLANK_RUN = re.compile(r"\n{3,}")-# Bare tracking/redirect URLs arrive as one unbroken token (prose never has an 80-char word);-# not evidence, and a small model will latch onto and echo it.+# Bare tracking URLs arrive as one unbroken token (prose never has an 80-char word) and a small+# model will latch onto and echo it._LONG_TOKEN = re.compile(r"\S{80,}")@@ -250,11 +235,8 @@ def _safe_error(exc: BaseException) -> str:return "Local model request timed out"if isinstance(exc, httpx.HTTPStatusError):return f"Local model request failed with HTTP {exc.response.status_code}"- # str() on a stream error is deliberately the server's own text, so that the- # token-count regex in routes/inference.py still matches it. Deep Research has no- # such regex, so reading str() here showed the raw "Context size has been exceeded."- # and dropped the Model settings hint from an oversize refusal: the very case this- # exception was introduced to explain.+ # str() must stay the server's own text so routes/inference.py's token-count regex still matches;+ # reading it here dropped the Model settings hint from an oversize refusal.friendly = getattr(exc, "friendly", None)text = friendly if isinstance(friendly, str) and friendly else str(exc)text = text.replace("\n", " ").strip()@@ -329,6 +311,7 @@ def _peek_inference_backend() -> Any:"""from core.inference import get_inference_backend+ # Native / transformers: the orchestrator the API layer reads (not the subprocess singleton).try:from core.inference.orchestrator import get_inference_backend as _realfrom core.inference.orchestrator import peek_inference_backend@@ -346,7 +329,6 @@ def _loaded_context_length() -> int | None:orchestrator) so grounding sizes evidence to the same context the API layer serves. The MLbackends live in a worker subprocess, so the core.inference.inference singleton is unpopulatedhere and importing it pulls in the ML stack; read the orchestrator the routes use instead."""- # GGUF / llama.cpp keeps context on its own backend (checked first, like the API layer).try:from routes.inference import get_llama_cpp_backendllama = get_llama_cpp_backend()@@ -356,7 +338,6 @@ def _loaded_context_length() -> int | None:return ctxexcept Exception:logger.debug("research.context_probe_llama_failed", exc_info = True)- # Native / transformers: the orchestrator the API layer reads (not the subprocess singleton).try:backend = _peek_inference_backend()name = getattr(backend, "active_model_name", None)@@ -567,8 +548,8 @@ def _rate_limit_wait(requested: float, remaining: float, headroom: float) -> floThe delay is the provider's, not a share of the model-load budget, so it is bounded by what isleft of the call minus the room the re-send needs; coming back early only spends an attempt onthe same refusal. The standing ceiling covers a run with no wall clock at all."""- # Never reserve all of what is left: a call whose wall clock is no larger than its- # first-output budget reserves the whole of it, and every wait collapses to zero.+ # Never reserve all of what is left: a call whose wall clock equals its first-output budget would+ # collapse every wait to zero.headroom = min(headroom, remaining / 2)return max(0.0, min(requested, _MAX_RATE_LIMIT_WAIT_SECONDS, remaining - headroom))@@ -732,9 +713,8 @@ def _bounded_synthesis_evidence(return "(none)"if max_chars <= 0:return ""- # Split the budget evenly across every note so a small context still keeps a slice of every- # research step. A per-note floor would let the earliest notes consume the whole budget and- # the final slice would drop later steps entirely.+ # Split evenly across notes: a per-note floor would let the earliest notes consume the whole budget+ # and drop later steps entirely.separator = "\n\n"available = max(0, max_chars - len(separator) * (len(notes) - 1))base, remainder = divmod(available, len(notes))@@ -960,12 +940,12 @@ class ResearchSupervisor:try:await self._taskexcept asyncio.CancelledError:+ # Polling is intentionally sufficient for one local process; requests never own tasks.passfinally:await asyncio.to_thread(db.release_worker_leases, self.worker_id)def wake(self) -> None:- # Polling is intentionally sufficient for one local process; requests never own tasks.passdef cancel(self, run_id: str) -> None:@@ -1054,9 +1034,7 @@ class ResearchSupervisor:)if not pages:return "", []- # Reuse Unsloth's knowledge-base RAG pipeline (ingest -> hybrid retrieve -> <chunk>- # render) over an ephemeral scope; runs off the event loop since embedding and the- # sqlite/vec index work are CPU/GPU bound.+ # Runs off the event loop, since embedding and the sqlite/vec index work are CPU/GPU bound.from core.rag import web_ranksection, _sources = await asyncio.to_thread(@@ -1128,10 +1106,8 @@ class ResearchSupervisor:except asyncio.CancelledError:raiseexcept sqlite3.OperationalError as exc:- # Losing the writer lock is normal for polling, not a fault, and produced- # six tracebacks in one session. Only that case is quietened. Neither may- # re-raise: that escapes the while loop and stops the supervisor for the- # life of the process, and the sibling `except Exception` cannot catch it.+ # Losing the writer lock is normal for polling, not a fault; neither branch may re-raise, since that+ # escapes the while loop and stops the supervisor for the life of the process.if is_sqlite_busy_error(exc):logger.warning("research.supervisor_db_busy: %s", exc)else:@@ -1171,8 +1147,8 @@ class ResearchSupervisor:model, a llama.cpp update, a name that no longer resolves) needs a user action that nowait can outlast, so surfacing the refusal beats burning the whole budget first."""loop = asyncio.get_running_loop()- # Share the model budget across the allowed waits: spending all of it on one lets the- # enclosing wall clock fire first, burying the real refusal under a timeout.+ # Share the model budget across the allowed waits: spending it all on one lets the enclosing wall+ # clock fire first and bury the real refusal.budget = _model_wait_budget(run)if max_seconds is not None:budget = min(budget, max_seconds)@@ -1194,8 +1170,8 @@ class ResearchSupervisor:"""run_id = run["id"]step = _retry_after_seconds(response) or _MODEL_SWITCH_RETRY_SECONDS- # Same budget share as _wait_for_local_model: one wait must leave room for the others and- # for the refusal, or the enclosing wall clock fires first and reports a timeout instead.+ # Same budget share as _wait_for_local_model: one wait must leave room for the others and for the+ # refusal, or the enclosing wall clock fires first and reports a timeout instead.remaining = min(step * waits, _NAMED_MODEL_WAIT_SECONDS, _model_wait_budget(run))logger.info("research.waiting_for_model_switch run_id=%s seconds=%.0f", run_id, remaining)while remaining > 0:@@ -1252,11 +1228,11 @@ class ResearchSupervisor:await asyncio.wait({task}, timeout = _STREAM_CLEANUP_TIMEOUT_SECONDS)except asyncio.CancelledError:# Must keep propagating, but the child outlives this frame, so hand it over first.+ # Bound expired but the task lives on: absorb its outcome when it cooperates.self._absorb_when_done(run_id, task, what)raiseif not task.done():logger.warning("research.%s_cleanup_timed_out run_id=%s", what, run_id)- # Bound expired but the task lives on: absorb its outcome when it cooperates.self._absorb_when_done(run_id, task, what)returntry:@@ -1301,8 +1277,8 @@ class ResearchSupervisor:discarded = Trueawait self._discard_task(run_id, line_task, "stream_iterator")await self._check_active(run_id)- # A line that arrived during the wait is earned; recomputing the- # deadline first would let an expiry in the same turn discard it.+ # A line that arrived during the wait is earned; recomputing the deadline first would let an expiry+ # in the same turn discard it.if line_task.done():breaktimeout = wait_timeout()@@ -1336,9 +1312,8 @@ class ResearchSupervisor:token, key = await asyncio.to_thread(auth_storage.create_api_key,username = run["ownerSubject"],- # The name is load-bearing, not a label: the external-provider route- # scopes its saved-credential exception to exactly this workflow, so- # the two sides must not drift apart.+ # The name is load-bearing: the external-provider route scopes its saved-credential exception to+ # exactly this workflow.name = auth_storage.DEEP_RESEARCH_WORKFLOW_KEY_NAME,expires_at = expires,internal = True,@@ -1350,21 +1325,18 @@ class ResearchSupervisor:"messages": messages,"stream": True,"stream_options": {"include_usage": True},- # Keep every model hop in this durable run on one isolated Codex- # prompt-cache session rather than sharing the transport fallback.+ # Keep every model hop in this durable run on one isolated Codex prompt-cache session rather than+ # sharing the transport fallback."thread_id": f"research:{run['id']}",- # Gathered page text lands in these prompts and research never reads tool calls- # back, so this hop must stay out of the tool loop. Both opt-outs are needed:- # --enable-tools overrides a per-request enable_tools, and an omitted+ # Both opt-outs are needed: --enable-tools overrides a per-request enable_tools, and an omitted# enabled_tools resolves to every built-in, python and terminal included."tool_choice": "none","enabled_tools": [],"temperature": inference.get("temperature", 0.2),}- # Route the hop to whichever saved connection the run was created with.- # The route's _sanitize_config already refused anything but an enabled- # saved connection of a studio-tools-capable provider type.+ # The route's _sanitize_config already refused anything but an enabled saved connection of a studio-+ # tools-capable provider type.if inference.get("providerType"):payload.update({@@ -1465,9 +1437,8 @@ class ResearchSupervisor:)if model_timeout > 0:first_output_budget = min(first_output_budget, model_timeout)- # Unlimited only drops the total wall clock; connect and headers stay bounded. This- # bound also caps the silence between queue notices, which no wall clock covers, so- # it has to clear the heartbeat those notices are paced by.+ # Unlimited only drops the total wall clock; this bound also caps the silence between queue notices,+ # so it has to clear the heartbeat they are paced by.admission_gap_budget = max(first_output_budget,_MODEL_OUTPUT_IDLE_TIMEOUT_SECONDS,@@ -1479,8 +1450,8 @@ class ResearchSupervisor:if model_timeoutelse httpx.Timeout(first_output_budget,- # Strictly looser than the guards above, so a stall is reported by name- # rather than as a message-less HTTPX ReadTimeout.+ # Strictly looser than the guards above, so a stall is reported by name rather than as a message-+ # less HTTPX ReadTimeout.read = admission_gap_budget + _STREAM_READ_TIMEOUT_MARGIN_SECONDS,))@@ -1497,8 +1468,7 @@ class ResearchSupervisor:)call_started = loop.time()- # No backoff below may outlast this call's wall clock or the key the re-send- # authenticates with, so all of them measure against whichever ends first.+ # No backoff below may outlast this call's wall clock or the key the re-send authenticates with.retry_deadline = key_minted + _MODEL_CALL_KEY_LIFETIME_SECONDSif model_timeout:retry_deadline = min(retry_deadline, call_started + model_timeout)@@ -1508,6 +1478,7 @@ class ResearchSupervisor:):response: httpx.Response | None = Nonesend_task: asyncio.Task | None = None+ # A retry builds a fresh task, so the guard starts over with it.send_discarded = Falsemodel_waits = 0attempt = 0@@ -1526,7 +1497,6 @@ class ResearchSupervisor:)try:send_task = asyncio.create_task(client.send(request, stream = True))- # A retry builds a fresh task, so the guard starts over with it.send_discarded = Falsewhile not send_task.done():await asyncio.wait({send_task}, timeout = 0.2)@@ -1539,8 +1509,7 @@ class ResearchSupervisor:response.raise_for_status()first_output_deadline = loop.time() + first_output_budgetexcept (httpx.TransportError, httpx.HTTPStatusError) as exc:- # Only reachable before a body byte is touched (the stream is consumed- # after this loop), so a re-send cannot duplicate report text.+ # Only reachable before a body byte is touched, so a re-send cannot duplicate report text.unloaded = (await _model_unloaded(exc.response)if isinstance(exc, httpx.HTTPStatusError)@@ -1568,8 +1537,8 @@ class ResearchSupervisor:if unloaded == "switching":await self._wait_for_model_switch(run, exc.response, model_waits)elif unloaded:- # Nothing loaded (restart, eject): wait for a model to come back,- # without spending a transport attempt.+ # Nothing loaded (restart, eject): wait for a model to come back, without+ # spending a transport attempt.if not await self._wait_for_local_model(run,_NAMED_MODEL_WAIT_SECONDS if unloaded == "named" else None,@@ -1578,8 +1547,7 @@ class ResearchSupervisor:else:delay = 2**attemptif rate_limited:- # This runs to minutes, so re-read the run while it- # waits; the backoff below never outlasts one poll.+ # This runs to minutes, so re-read the run while it waits.await self._wait_out_rate_limit(run,_retry_after_seconds(exc.response) or delay,@@ -1592,9 +1560,9 @@ class ResearchSupervisor:# re-check the lease and cancellation before re-sending.await self._check_active(run["id"])continue- # A proxied provider 429 arrives as a 200 whose first line is the- # refusal, so the status cannot see it. No body byte is used yet, so a- # re-send cannot duplicate report text.+ # A proxied provider 429 arrives as a 200 whose first line is the refusal, so the+ # status cannot see+ # it; no body byte is used yet.stream = self._iter_stream_lines(run["id"], response, semantic_deadline)head = await _peek_stream_head(stream)throttled = _stream_rate_limit_delay(head)@@ -1613,12 +1581,12 @@ class ResearchSupervisor:if self._cancel_event(run["id"]).is_set():await self._check_active(run["id"])if not line.startswith("data:"):- # Queueing has no timeout by design, so it is not charged: suspend- # for it, and start the budget when the slot is granted. A plain- # ": keep-alive" means a silent backend, which is what we bound.+ # Queueing has no timeout by design, so suspend for it and start the budget when the slot is+ # granted.if line.startswith(_ADMISSION_WAIT_COMMENT):- # Unlimited has no wall clock behind this, so bound the gap- # between queue notices; each notice refreshes it.+ # Unlimited has no wall clock behind this, so bound the gap between queue+ # notices; each notice+ # refreshes it.first_output_deadline = (None if model_timeout else loop.time() + admission_gap_budget)@@ -1630,13 +1598,16 @@ class ResearchSupervisor:breakif not data:continue+ # Arming research in the composer is the approval, so the plan is queued as it is stored+ # rather than parked for a second confirmation.+ # revoked before the phase event, so a cancel there cannot leak a live key.try:chunk = json.loads(data)_stream_error = stream_error_from_chunk(chunk)if _stream_error is not None:- # The server's own text names the cause and, for a context- # refusal, both token counts. Flattening it to a fixed- # string left the user with nothing to act on.+ # The server's own text names the cause and both token counts; flattening it to+ # a fixed string left+ # the user nothing to act on.raise _stream_errornormalized_usage = _normalize_completion_usage(chunk.get("usage") if isinstance(chunk, dict) else None@@ -1692,8 +1663,9 @@ class ResearchSupervisor:try:await response.aclose()except Exception:- # Closing a broken stream is best-effort and must not replace the- # generation result or the timeout/error that caused teardown.+ # Closing a broken stream is best-effort and must not replace the generation result+ # or the error+ # that caused teardown.logger.warning("research.stream_cleanup_failed run_id=%s",run["id"],@@ -1772,8 +1744,8 @@ class ResearchSupervisor:logger.debug("research.phase_event_failed run_id=%s", run_id, exc_info = True)async def _process(self, run: dict) -> None:- # The attempt this worker claimed. Everything it writes after a terminal status is- # only its to write while the run is still on that attempt.+ # Everything this worker writes after a terminal status is only its to write while the run is still+ # on that attempt.attempt = int(run.get("retryCount") or 0)cancel_event = self._cancel_event(run["id"])if await asyncio.to_thread(db.is_cancel_requested, run["id"]):@@ -1851,8 +1823,8 @@ class ResearchSupervisor:renewed = await asyncio.to_thread(db.heartbeat, run_id, self.worker_id)except Exception:logger.warning("research.heartbeat_failed run_id=%s", run_id, exc_info = True)- # A busy SQLite writer is not proof that ownership was lost.- # Retry briefly, but stop well before the 120-second lease expires.+ # A busy SQLite writer is not proof that ownership was lost; retry briefly, but stop well before the+ # 120-second lease expires.consecutive_errors += 1if consecutive_errors >= 10:self._lost_leases.add(run_id)@@ -1880,9 +1852,8 @@ class ResearchSupervisor:_planner_system_prompt(max_steps, run["config"].get("websitePolicy")),run["config"],)- # Same whole-prompt budget as the decision and synthesis paths. The question is budgeted- # before the history, but it is unbounded on its own (a pasted document arrives here- # verbatim) and would otherwise overflow before planning.+ # The question is budgeted before the history but is unbounded on its own (a pasted document arrives+ # verbatim) and would overflow before planning.planning_total = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)planning_question = question[: max(@@ -1936,8 +1907,8 @@ class ResearchSupervisor:await self._check_active(run["id"])raiserun.update(result)- # The structured inline card renders the plan; no second markdown copy below it.+ # The structured inline card renders the plan; no second markdown copy below it.async def _research(self, run: dict) -> None:resuming = run.get("claimedFromStatus") == "running"fresh = await asyncio.to_thread(db.get_run, run["id"])@@ -1952,8 +1923,7 @@ class ResearchSupervisor:tool_timeout = int(budgets["toolTimeoutSeconds"])# Absent for runs created before auto-scrape: default 0 keeps their behavior unchanged.max_auto_scrape = int(budgets.get("maxAutoScrape", 0))- # On a tiny context the prompt overhead alone fills the window and the grounded report- # degenerates, so fall back to snippet-only.+ # On a tiny context the prompt overhead alone fills the window, so fall back to snippet-only.if max_auto_scrape > 0:loaded_ctx = _loaded_context_length()if loaded_ctx is not None and loaded_ctx < _AUTO_SCRAPE_MIN_CONTEXT_TOKENS:@@ -2023,9 +1993,8 @@ class ResearchSupervisor:)for source in document_sources}- # Mirrors the live loop: evidence must hold only chunks that made it into the- # catalog, else the validator strips citations to the rest and synthesis is left- # building claims on uncataloged document text.+ # Evidence must hold only chunks that reached the catalog, else the validator strips citations to+ # the rest and synthesis builds claims on uncataloged text.accepted_rag_sources = []for source in restored_rag_sources:source_key = str(@@ -2079,9 +2048,8 @@ class ResearchSupervisor:_AGENT_SYSTEM_PROMPT + (f"\n\n{policy_prompt}" if policy_prompt else ""),run["config"],)- # Same whole-prompt budget as synthesis: a fixed 60k evidence tail is many times a- # small context, and this runs every step, so an overflow here kills the run long- # before it can synthesize what it already gathered.+ # A fixed 60k evidence tail is many times a small context and this runs every step, so an overflow+ # here kills the run before it can synthesize.decision_total = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)decision_question, decision_plan_json = _fit_decision_inputs(question,@@ -2089,8 +2057,8 @@ class ResearchSupervisor:len(decision_system),decision_total,)- # The catalog is unbounded too (maxSources entries, snippets up to 4000 chars), so it- # is fitted before the sections that depend on what it leaves.+ # The catalog is unbounded too (maxSources entries, snippets up to 4000 chars), so it is fitted+ # before the sections that depend on what it leaves.decision_catalog = _fit_source_catalog(source_catalog,_trimmable_budget(@@ -2197,9 +2165,8 @@ class ResearchSupervisor:if action is None:breakargument = action["query"]- # Persist model-derived state only after the associated action is final. Seed- # fallbacks intentionally carry no state, so rejected decisions cannot leak stale- # notes into the executed step, resume state, or synthesis.+ # Persist model-derived state only after the action is final, so rejected decisions cannot leak+ # stale notes into the executed step, resume state, or synthesis.next_state = _normalize_research_state(action.get("researchState"))if next_state:research_state = next_state@@ -2296,10 +2263,8 @@ class ResearchSupervisor:for source in accepted_rag_sources)elif rag_sources:- # Every chunk was refused by the source cap, so none has a catalog entry and the- # validator would strip any citation to it: drop the evidence rather than let- # synthesis build claims on it. Gated on rag_sources so a text-only KB reply- # ("No documents are attached to this chat.") still passes through.+ # Chunks refused by the source cap have no catalog entry and the validator would strip every+ # citation to them; gated on rag_sources so a text-only KB reply still passes through.rag_result = ""rag_sources = accepted_rag_sourcesstep_sources = []@@ -2349,8 +2314,8 @@ class ResearchSupervisor:fetched_urls.update(scraped_urls)await self._check_active(run["id"])if scraped_section:- # Additive, not replace: see _merge_scraped_evidence for why- # replacing the snippets regressed accuracy.+ # Additive, not replace: see _merge_scraped_evidence for why replacing the snippets regressed+ # accuracy.result = _merge_scraped_evidence(result, scraped_section)note = (f"### {action['title']} ({action['action']})\n"@@ -2419,8 +2384,8 @@ class ResearchSupervisor:f" Chunk ID: {source.get('chunkId') or '(unknown)'}"for index, source in enumerate(document_sources, 1))- # Budget each synthesis call as a whole. Model-derived JSON shares the evidence budget,- # and conversation history receives only the space left after the fixed prompt scaffold.+ # Model-derived JSON shares the evidence budget, and conversation history receives only what the+ # fixed scaffold leaves.total_budget = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)plan_json = json.dumps(run["plan"], ensure_ascii = False)audit_system = _system_prompt_with_instructions(@@ -2667,8 +2632,7 @@ class ResearchSupervisor:reasoning = await asyncio.to_thread(db.get_reasoning_text, run["id"])if synthesis_reasoning and synthesis_reasoning not in reasoning:reasoning += synthesis_reasoning- # Renew ownership before synchronizing the discoverable chat message.- # A restarted worker can safely overwrite this same message.+ # Renew ownership before syncing the discoverable chat message; a restarted worker can safely overwrite it.renewed = await asyncio.to_thread(db.heartbeat, run["id"], self.worker_id)if not renewed:await self._check_active(run["id"])diff --git a/studio/backend/core/tool_healing.py b/studio/backend/core/tool_healing.pyindex 59790eb6d0d..c57d9c0872a 100644--- a/studio/backend/core/tool_healing.py+++ b/studio/backend/core/tool_healing.py@@ -1,6 +1,5 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-## Bracket-tag, rehearsal, and thinking-block-strip logic adapted from forge# (https://github.com/antoinezambelli/forge), Copyright (c) 2025-2026# Antoine Zambelli, used under the MIT License.@@ -32,17 +31,15 @@ import re# One nesting level in the strip regexes; deeper may leak markup (still parsed)._BRACKETED_JSON_ONE_LEVEL = r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"-# Rehearsal ``name[ARGS]{..}`` strips; group 1 = name for tool-list gating. Closed =-# complete body, tail = truncated; ``(?<!\[CALL_ID\])`` keeps the v11 call-id from reading as a name.+# group 1 = name for tool-list gating; the (?<!\[CALL_ID\]) lookbehind keeps the v11 call-id from reading as a name._REHEARSAL_CLOSED_STRIP_RE = re.compile(r"(?<!\[CALL_ID\])\b([\w-]+)\[ARGS\]\s*" + _BRACKETED_JSON_ONE_LEVEL, re.DOTALL)_REHEARSAL_TAIL_STRIP_RE = re.compile(r"(?<!\[CALL_ID\])\b([\w-]+)\[ARGS\]\s*(?:\{.*)?$", re.DOTALL)-# Tool-XML strip patterns; hyphen in the name class covers dashed MCP names.-# Closed-pair patterns are named so _PAT_REQUIRED_TOKEN can skip a doomed lazy rescan when-# the close token is absent: an unguarded ``<tag>.*?</tag>`` rescans to EOF from every opener-# (quadratic on a stream of unclosed openers). Also reused by the quote-aware Gemma pre-pass.+# Hyphen in the name class covers dashed MCP names.+# Closed-pair patterns are named so _PAT_REQUIRED_TOKEN can skip a doomed lazy rescan: an unguarded+# <tag>.*?</tag> rescans to EOF from every opener (quadratic)._TC_JSON_CLOSED_PAT = re.compile(r"<tool_call>.*?</tool_call>", re.DOTALL)_TC_GEMMA_CLOSED_PAT = re.compile(r"<\|tool_call>.*?<tool_call\|>", re.DOTALL)_TC_FUNC_CLOSED_PAT = re.compile(r"<function=[\w-]+>.*?</function>", re.DOTALL)@@ -61,9 +58,8 @@ _TOOL_CLOSED_PATS = [# Drop the bare v11 [/TOOL_CALLS] closer the balanced scan leaves behind.re.compile(r"\[/TOOL_CALLS\]"),]-# Bare open markers strip a partial call mid-stream; the rehearsal tail needs `{` or EOF-# so prose ``foo[ARGS]`` survives. The XML open-tail forms reach EOF and are reused by-# _tool_call_markup_spans (a think tag in an unclosed call's args stays argument data).+# The rehearsal tail needs "{" or EOF so prose foo[ARGS] survives; the XML open-tail forms reach EOF+# and are reused by _tool_call_markup_spans._TOOL_OPEN_XML_TAIL_PATS = [re.compile(r"<tool_call>.*$", re.DOTALL),re.compile(r"<\|tool_call>.*$", re.DOTALL),@@ -81,21 +77,19 @@ _TOOL_ALL_PATS = (# Rehearsal strips (name in group 1); name-gated via ``enabled_tool_names``, strip-all when None._REHEARSAL_STRIP_PATS = frozenset({_REHEARSAL_CLOSED_STRIP_RE, _REHEARSAL_TAIL_STRIP_RE})-# Stripped before the quote-aware Gemma helper so a Gemma opener quoted in argument-# data cannot make the helper truncate the block and its tail.+# Stripped before the quote-aware Gemma helper so a Gemma opener quoted in argument data cannot+# truncate the block and its tail._TOOL_CLOSED_BLOCK_PATS = [_TC_JSON_CLOSED_PAT, _TC_FUNC_CLOSED_PAT]-# A lazy closed-pair pattern whose close token is absent would rescan to EOF from every-# opener; skip that doomed (quadratic) pass. Shared by both strip helpers.+# A lazy closed-pair pattern whose close token is absent would rescan to EOF from every opener; skip+# that quadratic pass._PAT_REQUIRED_TOKEN = {- # A tuple marks a required CLOSER. Besides skipping the pass when it is absent, callers- # bound the regex at the LAST one: nothing matches past it, and a tail full of openers- # would retry a lazy pattern at each one (quadratic). The bound is unconditional --- # any gap test for "are there openers after it" is itself the quadratic case.+ # A tuple marks a required CLOSER, and callers bound the regex at the LAST one unconditionally: any+ # gap test for openers past it is itself the quadratic case._TC_JSON_CLOSED_PAT: ("</tool_call>", "<tool_call>"),_TC_GEMMA_CLOSED_PAT: ("<tool_call|>", "<|tool_call>"),_TC_FUNC_CLOSED_PAT: ("</function>", "<function="),- # Literal in both rehearsal patterns: skips the sub AND its _code_spans scan on the- # common answer that has no rehearsal at all.+ # Literal in both rehearsal patterns: skips the sub AND its _code_spans scan on the common answer that has+ # no rehearsal at all._REHEARSAL_CLOSED_STRIP_RE: "[ARGS]",_REHEARSAL_TAIL_STRIP_RE: "[ARGS]",}@@ -172,8 +166,8 @@ def apply_tool_strip_patterns(elif required is not None and required not in text:continueif pat in _REHEARSAL_STRIP_PATS:- # Same two gates the parser applies in _iter_bracket_spans, so a rehearsal that- # is not promoted to a call stays visible instead of vanishing from the answer.+ # Same two gates the parser applies in _iter_bracket_spans, so an unpromoted rehearsal stays visible+ # instead of vanishing from the answer.spans = _code_spans(text)_t = texttext = pat.sub(lambda m: _rehearsal_strip(m, pat, _t, spans, enabled_tool_names), text)@@ -182,9 +176,8 @@ def apply_tool_strip_patterns(return text-# Pre-compiled patterns for tool-call XML parsing.-# <|content_invoke_tool_json|> is TML Inkling's native call marker; its JSON uses-# an ``args`` key and the block closes with <|end_message|>.+# <|content_invoke_tool_json|> is TML Inkling's native call marker: its JSON uses an "args" key and+# the block closes with <|end_message|>._TC_JSON_START_RE = re.compile(r"(?:<tool_call>|<\|content_invoke_tool_json\|>)\s*\{")_TC_GEMMA_START_RE = re.compile(r"<\|tool_call>\s*call\s*:\s*([\w.\-]+)\s*\{")_TC_FUNC_START_RE = re.compile(r"<function=([\w-]+)>\s*")@@ -197,16 +190,11 @@ _TC_PARAM_CLOSE_RE = re.compile(r"\s*</parameter>\s*$")_GEMMA_QUOTE = '<|"|>'_PARAM_CLOSE_TAG = "</parameter>"_FUNC_CLOSE_TAG = "</function>"-# A bare (unquoted) Gemma value ends at `}` or at a comma that begins the next-# `key:` pair. A comma NOT followed by a key token is part of the value (e.g.-# `location:New York, NY`), so it must not terminate the value. The key token-# must be identifier-shaped (start with a letter or underscore); a comma-# followed by digits-then-colon is value text such as a timestamp or ratio-# (`meet at 10:00, 11:00 tomorrow`), not a new key.+# A comma NOT followed by an identifier-shaped key token is part of a bare Gemma value (location:New+# York, NY; meet at 10:00, 11:00)._GEMMA_NEXT_KEY_RE = re.compile(r"\s*[A-Za-z_][\w.\-]*\s*:")-# A candidate starting inside a think block is a rehearsal (block kept so literal tags in-# real args survive); ``$`` accepts an unclosed block mid-stream.+# A candidate starting inside a think block is a rehearsal; "$" accepts an unclosed block mid-stream._THINK_TAG_RE = re.compile(r"<think>.*?(?:</think>|$)|\[THINK\].*?(?:\[/THINK\]|$)", re.DOTALL)# Bare open/close markers for prefilled-reasoning turns (template opens <think> in the prompt)._THINK_OPEN_RE = re.compile(r"<think>|\[THINK\]")@@ -215,27 +203,20 @@ _THINK_CLOSE_RE = re.compile(r"</think>|\[/THINK\]")# Mistral canonical array: [TOOL_CALLS] + JSON list of {"name","arguments"} objects._MISTRAL_ARRAY_RE = re.compile(r"\[TOOL_CALLS\]\s*(?=\[)")-# Mistral name form + v11 [ARGS]/[CALL_ID] shapes; [CALL_ID] is metadata, not the name,-# and hyphens keep dashed MCP names whole.+# [CALL_ID] is metadata, not the name, and hyphens keep dashed MCP names whole._MISTRAL_BRACKET_RE = re.compile(r"\[TOOL_CALLS\]\s*([\w-]+)(?:\[CALL_ID\][\w-]+)?(?:\[ARGS\])?\s*(?=\{)")-# Rehearsal ``name[ARGS]{json}`` (no [TOOL_CALLS]); the lookbehind keeps the v11 call-id-# from being taken as the function name.+# The lookbehind keeps the v11 call-id from being taken as the function name._REHEARSAL_RE = re.compile(r"(?<!\[CALL_ID\])\b([\w-]+)\[ARGS\]\s*(?=\{)")-# Markdown code: a fenced block (closing fence optional so a streaming block counts) or-# an inline span. Gates the markerless rehearsal form only, so quoting the syntax as-# documentation stays text instead of executing. ``>`` container prefixes are allowed so-# a fence inside a block quote counts. A backtick fence's info string cannot itself contain-# backticks, so ```` ```a``` ```` opening a line is an inline span and not a fence running-# to EOF; the closer tolerates a CR so a CRLF block still closes. Both would otherwise-# swallow the rest of the message and drop a real call after it. Inline runs are enumerated by length (double before-# single) rather than backreferenced: ``code`` must be one span and not two empty pairs-# around live markup, but a ``(`+)..\1`` form backtracks over every candidate length and-# turned 21 KB of unmatched runs into a 1.8s scan. A lone backtick is valid content inside-# a doubled span, so only a run of two closes it.+# Gates the markerless rehearsal form only, so quoting the syntax as documentation stays text+# instead of executing. A fence's info string cannot contain backticks, so a triple-backtick run+# opening a line is an inline span and not a fence running to EOF; the closer tolerates a CR so a+# CRLF block still closes. Inline runs are enumerated by length rather than backreferenced: a+# (`+)..\1 form backtracks over every candidate length and turned 21 KB of unmatched runs into a+# 1.8s stall._CODE_SPAN_RE = re.compile(r"^[ \t]*(?:>[ \t]*)*(?:```+[^`\n]*|~~~+[^\n]*)$"r".*?(?:^[ \t]*(?:>[ \t]*)*(?:```+|~~~+)[ \t\r]*$|\Z)"@@ -451,7 +432,7 @@ def _iter_bracket_spans(last_brace_close = text.rfind("}")last_array_close = max(text.rfind("]"), last_brace_close)cursor = start- code_spans: list | None = None # computed on the first rehearsal candidate only+ code_spans: list | None = Nonewhile cursor < n:for kind, rx in specs:m = nexts[kind]@@ -463,9 +444,9 @@ def _iter_bracket_spans(kind, m = min(live, key = lambda km: km[1].start())last_close = last_array_close if kind == "array" else last_brace_closeif m.end() > last_close:- # This and every later candidate of the same kind lacks its closing character.# Keep other formats live, but skip a balanced scan to EOF per doomed opener.nexts[kind] = None+ # Truncated body: skip and keep scanning; the caller's catch-all strips the tail.cursor = m.end()continueif kind == "array":@@ -473,7 +454,6 @@ def _iter_bracket_spans(else:end = _balanced_json_span(text, m.end())if end is None:- # Truncated body: skip and keep scanning; the caller's catch-all strips the tail.cursor = m.end()continueif (@@ -481,6 +461,7 @@ def _iter_bracket_spans(and enabled_tool_names is not Noneand m.group(1) not in enabled_tool_names):+ # Quoted syntax, not a call: advance past its body without yielding.# Inactive-name rehearsal is prose: advance past its body without yielding.cursor = end + 1continue@@ -488,7 +469,6 @@ def _iter_bracket_spans(if code_spans is None:code_spans = _code_spans(text)if _in_code(code_spans, m.start()):- # Quoted syntax, not a call: advance past its body without yielding.cursor = end + 1continueyield (m.start(), end + 1, kind, m)@@ -621,15 +601,14 @@ def _quote_gemma_object_keys(src: str) -> str:parts.append(src[i:colon_pos])parts.append(":")i = colon_pos + 1- # Gemma may emit bare string values ({unit:celsius}); quote them so- # json.loads succeeds. JSON scalars/objects/arrays/quoted stay as-is.+ # Gemma may emit bare string values ({unit:celsius}); quote them so json.loads succeeds.ws = iwhile i < len(src) and src[i].isspace():i += 1parts.append(src[ws:i])if i < len(src) and src[i] == "[":- # Array value: quote bare string elements (e.g. labels:[bug,ui])- # so json.loads succeeds instead of dropping the call.+ # Array value: quote bare string elements (e.g. labels:[bug,ui]) so json.loads succeeds instead+ # of dropping the call.arr_end = _balanced_bracket_end(src, i)if arr_end is None:parts.append(src[i:])@@ -639,9 +618,7 @@ def _quote_gemma_object_keys(src: str) -> str:i = arr_end + 1elif i < len(src) and src[i] not in '"{':v_start = i- # Consume the bare value up to `}` or a comma that starts the- # next key:value pair; a comma inside the value (e.g.- # `New York, NY`) does not terminate it.+ # A comma inside the value (New York, NY) does not terminate it.while i < len(src):if src[i] == "}":break@@ -653,7 +630,7 @@ def _quote_gemma_object_keys(src: str) -> str:json.loads(raw.strip())parts.append(raw)except (json.JSONDecodeError, ValueError):- # Quote bare value; empty ({k:}) becomes "" so json.loads sees {"k":""} not invalid {"k":}.+ # Empty ({k:}) becomes "" so json.loads sees {"k":""} and not invalid {"k":}.parts.append(json.dumps(raw.strip()))else:parts.append(src[key_start:i])@@ -688,9 +665,7 @@ def _inside_open_parameter(if param_start_re is None:param_start_re = _TC_PARAM_START_RElast_param_start = -1- # Both supported vocabularies begin with ``<param``, so search backwards to the nearest- # candidate and validate it with the regex instead of rescanning the whole prefix per- # marker. Unknown caller patterns keep the general finditer path.+ # Search backwards to the nearest <param candidate instead of rescanning the whole prefix per marker.reverse_param_search = (param_start_re is _TC_PARAM_START_REor param_start_re.pattern.startswith(r"<(?:parameter|param)")@@ -708,24 +683,21 @@ def _inside_open_parameter(last_param_start = match.start()if last_param_start < 0:return False- # The parameter's OWN close tag decides: if it closes after ``pos`` the position is- # argument data (even across literal function closes); an unclosed one falls back to func close.- # A closer at/before ``pos`` settles it. Bounded, so an absent alternate spelling- # does not scan the whole suffix on every later function marker.+ # The parameter's OWN close tag decides: one closing after pos makes the position argument data.+ # Bounded, so an absent alternate spelling does not scan the whole suffix.for tag in param_closers:found = content.find(tag, last_param_start, min(len(content), pos + len(tag)))if 0 <= found <= pos:return False- # Try the closer matching the opener spelling first. Cross-spelling closes are still- # accepted; the normal form just needs a shorter scan.+ # Try the closer matching the opener spelling first. Cross-spelling closes are still accepted; the normal+ # form just needs a shorter scan.if content.startswith("<param", last_param_start) and not content.startswith("<parameter", last_param_start):param_closers = tuple(reversed(param_closers))if any(content.find(tag, pos + 1) >= 0 for tag in param_closers):return True- # With no parameter close anywhere, only a function close at/before ``pos`` proves the- # candidate left the parameter. A later close and no close both mean still inside.+ # With no parameter close anywhere, a later close and no close both mean still inside.for tag in func_closers:found = content.find(tag, last_param_start, min(len(content), pos + len(tag)))if 0 <= found <= pos:@@ -773,14 +745,14 @@ def _marker_coverage(content: str, markers) -> list[tuple[int, int]]:for _s, be in brace_regions:max_end = max(max_end, be)brace_max_ends.append(max_end)- events = [] # (position, order) with order 0 = braces-done, 1 = close marker+ events = []for idx, (_start, brace_end, _kind, _m) in enumerate(markers):if brace_end >= 0:events.append((brace_end, 0, _kind, idx))for kind, close_re in (("json", _TC_END_TAG_RE), ("gemma", _TC_GEMMA_END_TAG_RE)):for cm in close_re.finditer(content):- # A close inside another call's balanced braces is quoted data; it- # must not pop an earlier close-less marker and swallow a sibling.+ # A close inside another call's balanced braces is quoted data; it must not pop an earlier close-+ # less marker and swallow a sibling.if brace_starts:region_idx = bisect.bisect_left(brace_starts, cm.start()) - 1inside_braces = region_idx >= 0 and cm.start() < brace_max_ends[region_idx]@@ -794,9 +766,9 @@ def _marker_coverage(content: str, markers) -> list[tuple[int, int]]:close_end_for: dict[int, int] = {}for _pos, order, kind, payload in events:if order == 0:- waiting[kind].append(payload) # marker index, now awaiting its close+ waiting[kind].append(payload)elif waiting[kind]:- close_end_for[waiting[kind].pop()] = payload # innermost open marker closes here+ close_end_for[waiting[kind].pop()] = payloadcoverage = []for idx, (start, brace_end, _kind, _m) in enumerate(markers):if brace_end < 0:@@ -869,8 +841,7 @@ def parse_tool_calls_from_text(in ``content`` (including its close tag when present), so a caller canremove exactly the parsed markup and keep every other byte intact."""- # Candidates starting inside a think block are rehearsals, skipped; blocks are kept, and a- # think marker opening inside a call is argument data (excluded from spans).+ # Candidates starting inside a think block are rehearsals; a think marker opening inside a call is argument data._think_spans = _think_spans_outside_tool_markup(content)_think_starts = [s for s, _e in _think_spans]@@ -881,10 +852,8 @@ def parse_tool_calls_from_text(tool_calls: list[dict] = []call_spans: list[tuple] = []- # Collect JSON/Gemma markers; _marker_coverage decides nesting so a marker inside- # another call's coverage (even one that failed to parse) is data, not executed. A- # marker opening inside a think block is a rehearsal and is skipped.- parsed_items = [] # (start, span_end, name, arguments) in document order+ # A marker inside another call's coverage, even one that failed to parse, is data and is not executed.+ parsed_items = []markers = [mk for mk in _build_markers(content) if not _in_think(mk[0])]coverage = _marker_coverage(content, markers)covered_until = -1@@ -894,7 +863,7 @@ def parse_tool_calls_from_text(if covered_by_earlier_marker:continueif brace_end < 0:- continue # unclosed: not parseable; the fallback still excludes its XML+ continueif not allow_incomplete:tail = content[brace_end + 1 :].lstrip()close_re = _TC_END_TAG_RE if kind == "json" else _TC_GEMMA_END_TAG_RE@@ -904,8 +873,8 @@ def parse_tool_calls_from_text(if kind == "json":obj = json.loads(content[m.end() - 1 : brace_end + 1])name = obj.get("name", "")- # Accept ``parameters`` alias for ``arguments`` (Llama-3.2 drift inside- # Hermes) and ``args`` (TML Inkling native calls).+ # Accept "parameters" (Llama-3.2 drift inside Hermes) and "args" (TML Inkling) as aliases for+ # "arguments".arguments = obj.get("arguments")if arguments is None:arguments = obj.get("parameters")@@ -913,9 +882,9 @@ def parse_tool_calls_from_text(arguments = obj.get("args", {})if isinstance(arguments, dict):arguments = json.dumps(arguments)- # Inkling echoes the bare tool name (and a role opener) before the- # marker: <|message_model|>NAME<|content_invoke_tool_json|>{...}.- # Fold that echo into the markup span so promotion removes it too.+ # Inkling echoes the bare tool NAME (and a role opener) before the marker:+ # <|message_model|>NAME<|content_invoke_tool_json|>{...}. Fold that echo into the markup span so+ # promotion removes it too.if name and content.startswith("<|content_invoke_tool_json|>", start):pre = content[:start]if pre.endswith(name):@@ -1019,8 +988,8 @@ def parse_tool_calls_from_text()call_spans.append((start, span_end))- # Patterns 3+4: Mistral [TOOL_CALLS] and bare rehearsal via one balanced scan in document- # order, so a Mistral call and a rehearsal in one message both parse.+ # Patterns 3+4: Mistral [TOOL_CALLS] and bare rehearsal via one balanced scan in document order, so a+ # Mistral call and a rehearsal in one message both parse.if not tool_calls:for start, end, kind, m in _iter_bracket_spans(content, enabled_tool_names = enabled_tool_names@@ -1031,13 +1000,13 @@ def parse_tool_calls_from_text(closer = re.match(r"\s*\[/TOOL_CALLS\]", content[end:])region_end = end + closer.end() if closer else endif kind == "array":- # Decode elements individually (comma-tolerant): one json.loads of the whole- # body rejects the comma-less multi-call arrays Mistral/Ollama templates emit.+ # Decode elements individually: one json.loads of the whole body rejects the comma-less multi-call+ # arrays Mistral/Ollama templates emit.payload, item_ends = _decode_array_items(content, m.end(), end)if not payload:continue- # Tile the region so every byte belongs to exactly one span; a with_spans consumer- # keeps skipped bytes visible and strips promoted markup exactly once.+ # Tile the region so every byte belongs to exactly one span and promoted markup is stripped exactly+ # once.tile_start = startlast_span_idx = -1for item_idx, item in enumerate(payload):@@ -1051,9 +1020,8 @@ def parse_tool_calls_from_text(except (json.JSONDecodeError, ValueError):passif not isinstance(args, (dict, str)):- # ``"arguments": null`` (or any non-object scalar) becomes {} like the- # <tool_call> path, not the string "null" auto-heal would mangle to- # a bogus {"query":"null"}.+ # A non-object scalar becomes {}, not the string "null" that auto-heal would mangle into+ # {"query":"null"}.args = {}tool_calls.append({@@ -1061,9 +1029,9 @@ def parse_tool_calls_from_text("type": "function","function": {"name": item.get("name", ""),- # A bare scalar string stays raw (like the <tool_call> path);- # json.dumps would double-encode it so the arg healer wraps- # "weather" with its literal quotes.+ # A bare scalar string stays raw; json.dumps would double-encode it so the arg+ # healer wraps it with+ # literal quotes."arguments": args if isinstance(args, str) else json.dumps(args),},}@@ -1121,8 +1089,7 @@ def _tool_call_markup_spans(text: str) -> list[tuple[int, int]]:stripped WITH the call, not kept as a reasoning block. Covers closed XML/bracketcalls and an unclosed XML call (run via allow_incomplete); without the open-endedspan the unclosed call's markup would leak after execution."""- # Skip a lazy closed-pair pattern whose close token is absent: its finditer would rescan- # to EOF from every opener (quadratic on a stream of unclosed openers).+ # Skip a lazy closed-pair pattern whose close token is absent: its finditer rescans to EOF from every opener.spans = []for pat in _TOOL_CLOSED_PATS:required = _PAT_REQUIRED_TOKEN.get(pat)@@ -1155,8 +1122,7 @@ def _think_spans_outside_tool_markup(text: str) -> list[tuple[int, int]]:a greedy unclosed <think> past the call is still that call's argument data."""think_spans = [m.span() for m in _THINK_TAG_RE.finditer(text)]call_spans = _tool_call_markup_spans(text)- # Prefilled reasoning: the template opens <think> in the prompt, so add a leading span- # (0..close) to skip calls rehearsed there; guarded so a stray close in a normal answer is safe.+ # Prefilled reasoning: the template opens <think> in the prompt, so a leading span skips calls rehearsed there.close = _THINK_CLOSE_RE.search(text)if close is not None:opener = _THINK_OPEN_RE.search(text)@@ -1178,8 +1144,8 @@ def strip_outside_think(text: str, strip_segment) -> str:blocks, preserving the blocks verbatim (tool-looking text inside is rehearsal).``is_last`` is True only after the final block, so trailing-tail patterns applyonly there. Shared by every strip path so they stay consistent."""- # A think marker opening inside a complete call is argument text; excluding it lets the- # stripper see the whole call. START-tested, so an unclosed match stays argument data.+ # A think marker opening inside a complete call is argument text; START-tested, so an unclosed match+ # stays argument data.think_spans = _think_spans_outside_tool_markup(text)if not think_spans:return strip_segment(text, True)@@ -1205,14 +1171,13 @@ def _strip_gemma_native_spans(text: str, *, final: bool) -> str:continuebrace_end = _balanced_brace_end(text, match.end() - 1, gemma_quotes = True)if brace_end is None:- # Unbalanced: nothing completes from here on. Drop the rest if final,- # else keep it; stop either way (rescanning would be quadratic).+ # Unbalanced: nothing completes from here on, so stop either way (rescanning would be quadratic).if final:out.append(text[cursor:start])cursor = len(text)break- # Junk between } and <tool_call|> is malformed-call markup: strip through- # the close, keep text after it. No close anywhere means stop (linear).+ # Junk between } and the close marker is malformed-call markup: strip through the close, and no+ # close anywhere means stop.close = _TC_GEMMA_END_TAG_RE.search(text, brace_end + 1)if close is None:if final:@@ -1294,9 +1259,8 @@ def _strip_markup_segment(final: bool,enabled_tool_names = None,) -> str:- # Bracket-tag calls (Mistral/rehearsal) first via balanced scan (any nesting depth,- # rehearsal name-gated); then the quote-aware Gemma-native passes so a literal- # <tool_call|> in an argument cannot truncate a block; finally the regex XML/tail sweeps.+ # Bracket-tag balanced scan first, then the quote-aware Gemma passes so a literal marker in an+ # argument cannot truncate a block, then the regex sweeps.text = _strip_bracket_tag_calls(text, enabled_tool_names = enabled_tool_names)text = _strip_closed_blocks_outside_gemma(text)text = _strip_gemma_native_spans(text, final = final)diff --git a/studio/backend/core/training/dataset_bounds.py b/studio/backend/core/training/dataset_bounds.pyindex 1a231893741..a3921e49492 100644--- a/studio/backend/core/training/dataset_bounds.py+++ b/studio/backend/core/training/dataset_bounds.py@@ -18,45 +18,34 @@ import reimport tempfilefrom typing import Any, Optional-# Deliberately loose: rows are consumed by things that never produce a step (the-# eval split carved off the train set, rows train_on_responses_only drops when the-# response template is missing). Running short only means re-reading the subset,-# but that repeats rows, and 4x is still orders of magnitude under the datasets-# this exists for.+# Deliberately loose: rows are consumed by things that never produce a step (the eval split, rows+# train_on_responses_only drops when the response template is missing), and running short only+# re-reads the subset; 4x is still orders of magnitude under the datasets this exists for.MAX_STEPS_ROW_SLACK = 4# Below this a subset is small enough to skew a run for no meaningful saving.MIN_MAX_STEPS_ROWS = 1024-# Launchers that advertise a data-parallel process count. Read as env because this-# module is torch-free and the bound is computed before any process group exists, so-# torch.distributed cannot be asked: at bound time it is almost never initialised.-# The MPI names routes/inference.py reads, plus the torchrun ones it does not, and a-# looser test: it pairs each size with its rank partner because it refuses requests;-# sizing a row bound only needs a bare size. Annotated per variable, several of which-# are widely miscited.+# Read as env because this module is torch-free and the bound is computed before any process group+# exists, so torch.distributed cannot be asked. A bare size is enough here, unlike+# routes/inference.py, which pairs each size with its rank partner because it refuses requests.WORLD_SIZE_ENV_VARS = ("WORLD_SIZE", # torchrun, accelerate launch, deepspeed. NOT set by any MPI"LOCAL_WORLD_SIZE", # torchrun's --nproc-per-node; it sets WORLD_SIZE too"MLX_WORLD_SIZE", # only mlx.launch's NCCL backend, which is CUDA-only- "OMPI_COMM_WORLD_SIZE", # Open MPI / prterun, alongside OMPI_COMM_WORLD_RANK+ "OMPI_COMM_WORLD_SIZE","PMI_SIZE", # MPICH and Intel MPI via Hydra; srun only under --mpi=pmi2"PMIX_SIZE", # nothing sets this: PMIx answers job size through PMIx_Get"MPI_WORLD_SIZE", # likewise undocumented in every MPI checked"MV2_COMM_WORLD_SIZE", # MVAPICH2, and only under its mpirun_rsh launcher)-# An mlx.launch world size that is not a number in the env: of its five backends only-# NCCL (CUDA) exports MLX_WORLD_SIZE; ring and JACCL export a path to a JSON file whose-# outer list has one entry per rank, and that length is the world size. Without these-# two an mlx.launch reads as one process, and under-counting recycles rows -- so the-# Apple path, the only one MLX training runs on, would keep the bug this fixes.+# Of mlx.launch's five backends only NCCL exports MLX_WORLD_SIZE; ring and JACCL export a path to+# a JSON file whose outer list has one entry per rank.WORLD_SIZE_ENV_FILES = ("MLX_HOSTFILE", # ring backend: a path to [["ip:port", ...], ...], one per rank"MLX_IBV_DEVICES", # jaccl backend: a path to the N x N RDMA matrix, one row per rank)-# A hostfile is a few hundred bytes per rank. Read a bounded prefix so a wrong path-# (an env var pointed at something enormous) cannot pull a file into memory; a-# truncated read fails to parse as JSON and is discarded, which is the safe answer.+# Read a bounded prefix so a wrong path cannot pull an enormous file into memory; a truncated read+# fails to parse as JSON and is discarded.MAX_WORLD_SIZE_FILE_BYTES = 1 << 20-# Written into a run's output directory at its first start; read back on resume.# Its absence is the signal that a checkpoint predates the bound.ROW_BOUND_MARKER_FILE = "unsloth_row_bound.json"# transformers writes checkpoint-<global_step> and nothing else under that prefix.@@ -111,10 +100,8 @@ def world_size_from_rank_files(environ: Any = None) -> int:if value.lstrip()[:1] in ("[", "{"):payload = json.loads(value[:MAX_WORLD_SIZE_FILE_BYTES])elif os.path.isfile(value):- # Binary, so the cap really is bytes: a text read() counts- # CHARACTERS, so 4-byte codepoints would pull 4x the cap off disk.- # json.loads takes bytes; non-UTF-8 raises UnicodeDecodeError (a- # ValueError), caught below.+ # Binary, so the cap really is bytes: a text read() counts CHARACTERS, and json.loads takes bytes+ # anyway.with open(value, "rb") as handle:payload = json.loads(handle.read(MAX_WORLD_SIZE_FILE_BYTES))else:diff --git a/studio/backend/core/training/diffusion_checkpoint.py b/studio/backend/core/training/diffusion_checkpoint.pyindex 2e540aec8c6..5326b743a47 100644--- a/studio/backend/core/training/diffusion_checkpoint.py+++ b/studio/backend/core/training/diffusion_checkpoint.py@@ -59,8 +59,8 @@ CHECKPOINT_FORMAT = "unsloth-diffusion-checkpoint"CHECKPOINT_VERSION = 1CHECKPOINT_PREFIX = "checkpoint-"-# Hidden, and does NOT start with "checkpoint-", so neither glob("checkpoint-*") nor a user's-# file browser mistakes a half-written bundle for a real one.+# Hidden, and not prefixed "checkpoint-", so neither glob("checkpoint-*") nor a file browser+# mistakes a half-written bundle for a real one._STAGING_PREFIX = ".tmp-checkpoint-"TRAINER_STATE_FILENAME = "trainer_state.json"@@ -70,8 +70,7 @@ OPTIMIZER_FILENAME = "optimizer.pt"SCHEDULER_FILENAME = "scheduler.pt"RNG_FILENAME = "rng_state.pt"-# How many checkpoint-<N> bundles to keep in a run's output dir. A LoRA bundle is a few MB, but-# a 10000-step run at save_steps=50 would otherwise leave 200 of them.+# A 10000-step run at save_steps=50 would otherwise leave 200 bundles.DEFAULT_SAVE_TOTAL_LIMIT = 2@@ -79,8 +78,8 @@ class ResumeError(ValueError):"""A resume request that cannot be honoured. The message is shown to the user."""-# ── identity ──────────────────────────────────────────────────────────────────# Human labels for the identity fields that HARD-REJECT a resume, in report order.+# ── identity ──────────────────────────────────────────────────────────────────_IDENTITY_LABELS: tuple[tuple[str, str], ...] = (("kind", "training type"),("family", "model family"),@@ -92,58 +91,44 @@ _IDENTITY_LABELS: tuple[tuple[str, str], ...] = (("lora_alpha", "LoRA alpha"),("lora_dropout", "LoRA dropout"),("cfg_dropout", "caption dropout"),- # The trajectory. Each of these is read from the INCOMING config by the trainer while the- # optimizer moments and the scheduler position come from the bundle, so changing one- # continues a run whose objective or learning-rate curve is no longer the one those moments- # were produced under -- reported as a clean continue.+ # Read from the INCOMING config while the moments and scheduler position come from the bundle, so+ # changing one continues a trajectory those moments were not produced under.("flow_shift", "timestep shift"),("weighting_scheme", "loss weighting scheme"),("snr_gamma", "min-SNR gamma"),("lr_scheduler", "learning-rate schedule"),("lr_warmup_steps", "learning-rate warmup"),- # The input stream. Both trainers build the latent cache and the crop/flip variant plan- # from these BEFORE restore_resume_state puts the RNG back, so changing one continues the- # old optimizer and sampler against a different sequence of images.+ # The latent cache and crop/flip plan are built from these BEFORE restore_resume_state puts the RNG+ # back, so changing one continues the old sampler against a different image sequence.("seed", "random seed"),("cache_latents", "latent caching"),("cache_mode", "latent cache path"),- # The mode the LOOP actually ran in, which the request does not settle:- # UNSLOTH_DIFFUSION_NO_LATENT_CACHE turns it off, and an over-budget cache falls back to- # encoding in-loop. The two paths draw their crops and flips from different streams- # (variant_rng versus the loop rng), so a restored RNG state does not reproduce the run.+ # The mode the LOOP actually ran in: UNSLOTH_DIFFUSION_NO_LATENT_CACHE turns it off and an+ # over-budget cache falls back to encoding in-loop, and the two paths draw crops and flips from+ # different streams (variant_rng versus the loop rng), so a restored RNG state does not reproduce+ # the run.("cache_variants", "cached crop variants"),("center_crop", "centre cropping"),("random_flip", "random flipping"),- # The matmul kernels. _apply_perf_flags routes CUDA matmuls through TF32 or strict fp32- # from this, so restoring moments produced under one into a run using the other continues- # the trajectory at a different numeric precision -- and off is the documented strict- # reproducibility mode, which a resume that silently defaults it back on undoes.+ # TF32 versus strict fp32 continues the trajectory at a different numeric precision, and off is the+ # documented strict-reproducibility mode.("enable_tf32", "TF32 matmuls"),- # The update shape. The batch and accumulation counts decide how many samples each- # optimizer step consumes, and the clip norm is applied to the restored moments on the- # very next update, so all three change the trajectory the checkpoint was produced on.+ # Batch, accumulation and clip norm all change the trajectory the restored moments were produced on.("train_batch_size", "batch size"),("gradient_accumulation_steps", "gradient accumulation"),("max_grad_norm", "gradient clipping"),- # Recorded from the beginning, but only for the UI until now. Both trainers resize and- # crop the incoming images to it before the restored sampler and moments see anything, so- # a checkpoint trained at 1024 could report a clean resume and finish at 768.+ # Both trainers resize and crop to this before the restored sampler and moments see anything, so a+ # bundle trained at 1024 could report a clean resume and finish at 768.("resolution", "training resolution"),("precision", "mixed precision"),("base_precision", "base precision"),- # What the frozen base was ACTUALLY converted to. fp8 and mxfp8 conversion can fail on the- # host and both fall back to bf16, so a bundle requested as fp8 records fp8 while its- # moments were produced against bf16 linears. Resuming that on a host where the conversion- # succeeds continues those moments against an fp8 base and reports a clean resume.+ # What the frozen base was ACTUALLY converted to: fp8/mxfp8 conversion can fall back to bf16, so+ # recording the request reports a clean resume across different numerics.("base_precision_effective", "resolved base precision"),- # The EMA coefficient. restore_resume_state loads the old shadow tensors and update count- # while the trainer builds LoRAEMA from the INCOMING decay, so every later update applies a- # new coefficient to an average produced under the old one and the exported EMA adapter is a- # hybrid -- reported as a faithful continuation.+ # LoRAEMA is built from the INCOMING decay while the shadow tensors are restored, so a changed+ # coefficient makes the exported EMA adapter a hybrid.("ema_decay", "EMA decay"),)-# Fields whose value can legitimately be unknown on one side (an uncached Hub repo has no-# resolvable revision; the start route knows the dataset only after its own discovery pass).# Unknown on either side means "cannot tell", which must not be reported as a mismatch._OPTIONAL_IDENTITY_FIELDS = frozenset({@@ -151,8 +136,8 @@ _OPTIONAL_IDENTITY_FIELDS = frozenset("dataset_fingerprint","lora_dropout","cfg_dropout",- # Absent only in a manifest written before these were recorded, which reads as- # "cannot tell" rather than as a mismatch.+ # Absent only in a manifest written before these were recorded, which reads as "cannot tell" rather than+ # as a mismatch."ema_decay","flow_shift","weighting_scheme",@@ -169,9 +154,8 @@ _OPTIONAL_IDENTITY_FIELDS = frozenset("train_batch_size","gradient_accumulation_steps","max_grad_norm",- # Known only inside the trainer, and only for the DiT path. Unknown on either side- # reads as "cannot tell", which is what keeps the route's pre-eviction preflight and- # the SDXL trainer unaffected.+ # Known only inside the DiT trainer; unknown reads as "cannot tell", which keeps the route preflight+ # and the SDXL trainer unaffected."base_precision_effective",})@@ -225,30 +209,24 @@ class CheckpointIdentity:base_revision: Optional[str] = Nonedataset_fingerprint: Optional[str] = Nonelora_dropout: Optional[float] = None- # Caption (classifier-free-guidance) dropout. The DiT loop asks rng.random() once per- # sample when it is above zero and swaps in the empty prompt on a hit, so changing it- # across a resume diverges the restored RNG stream on the very next step AND changes the- # objective, while the run reports a clean continue. A bundle from before this field- # existed reads None, which the optional rule skips rather than reporting as a mismatch.+ # The DiT loop draws rng.random() once per sample while this is above zero, so a change diverges the+ # restored RNG stream on the next step and changes the objective.cfg_dropout: Optional[float] = None- # Trajectory-defining knobs, all optional for the same reason: a bundle from before they- # were recorded reads unknown, not mismatched.+ # Trajectory-defining knobs, all optional for the same reason: a bundle from before they were recorded reads+ # unknown, not mismatched.flow_shift: Optional[str] = Noneweighting_scheme: Optional[str] = None- # Text, not a float: None is the documented way to DISABLE min-SNR, so a float field could- # not tell "trained with it off" from "written before the field existed" -- and the optional- # rule would skip the comparison for both. "off" is the disabled value; None is absence.+ # Text, not a float: None is the documented way to DISABLE min-SNR, so a float field could not tell+ # "trained with it off" from "written before the field existed".snr_gamma: Optional[str] = Nonelr_scheduler: Optional[str] = Nonelr_warmup_steps: Optional[int] = Noneseed: Optional[int] = None- # Booleans as text ("on"/"off") for the same reason snr_gamma is: False is a real value and- # None has to stay reserved for a manifest that predates the field.+ # Booleans as text ("on"/"off") for the same reason snr_gamma is: False is a real value and None has to stay+ # reserved for a manifest that predates the field.cache_latents: Optional[str] = None- # Resolved, not requested, and known only inside the trainer -- the start route leaves it- # None, which the optional rule reads as "cannot tell" so the pre-eviction preflight is- # unaffected. The trainer's own preflight, which runs with the real value, is where a- # mismatched cache path is caught.+ # Resolved, not requested, and left None by the start route so the pre-eviction preflight is+ # unaffected; the trainer's own preflight catches a mismatched cache path.cache_mode: Optional[str] = Nonecache_variants: Optional[int] = Nonecenter_crop: Optional[str] = None@@ -258,17 +236,14 @@ class CheckpointIdentity:gradient_accumulation_steps: Optional[int] = None# Text, so 0.0 (clipping disabled) is a value and None stays "not recorded".max_grad_norm: Optional[str] = None- # Text for the same reason: 0.0 means EMA is off, which is a real setting, and None has to- # stay reserved for a manifest that predates the field.+ # Text for the same reason: 0.0 means EMA is off, which is a real setting, and None has to stay reserved for+ # a manifest that predates the field.ema_decay: Optional[str] = None- # Post-conversion, set by the DiT trainer once it knows whether fp8/mxfp8 took. None- # everywhere else, which the optional rule reads as "cannot tell".+ # Post-conversion, set by the DiT trainer once it knows whether fp8/mxfp8 took. None everywhere else, which+ # the optional rule reads as "cannot tell".base_precision_effective: Optional[str] = None- # WHICH repo ``base_revision`` was read from, which is not always ``base_model``: a gated base- # is fetched from its byte-identical ungated mirror, and the two repos have different commit- # SHAs for the same weights. Not a mismatch reason of its own -- it only says whether the two- # revisions are comparable at all. None means a bundle written before mirrors existed, whose- # revision came from the canonical base, so it reads as ``base_model`` below.+ # WHICH repo base_revision was read from: a gated base is fetched from its byte-identical ungated+ # mirror and the two repos carry different SHAs, so this only says whether they are comparable.base_revision_repo: Optional[str] = Nonedef as_dict(self) -> dict[str, Any]:@@ -282,10 +257,8 @@ class CheckpointIdentity:"lora_target_modules": list(self.lora_target_modules),"lora_rank": int(self.lora_rank),"lora_alpha": int(self.lora_alpha),- # Both trainers pass this to the LoRA constructor, so it changes the stochastic- # forward pass the restored optimizer moments were produced against. Written as a- # rounded float: a manifest from before this field existed reads as None, which the- # optional-field rule below skips rather than reporting as a mismatch.+ # Passed to the LoRA constructor, so it changes the stochastic forward pass the restored optimizer+ # moments were produced against."lora_dropout": self.lora_dropout,"cfg_dropout": self.cfg_dropout,"flow_shift": self.flow_shift,@@ -307,10 +280,8 @@ class CheckpointIdentity:"base_precision_effective": self.base_precision_effective,"precision": self.precision,"base_precision": self.base_precision,- # A hard identity field, not just a label: both trainers resize and crop the- # incoming images to it before the restored sampler and moments see anything, so a- # bundle trained at 1024 could report a clean resume and finish at 768. Loadable- # tensors are not a faithful resume.+ # A hard identity field: both trainers crop the images to this before the restored sampler sees+ # anything, so loadable tensors are not a faithful resume."resolution": int(self.resolution),}@@ -371,19 +342,16 @@ class CheckpointIdentity:mine, theirs = self.as_dict(), other.as_dict()for field, label in _IDENTITY_LABELS:a, b = mine.get(field), theirs.get(field)- # None / "" is "cannot tell". NOT falsiness: a lora_dropout of 0.0 is a real value- # and the commonest one, so truthiness would skip exactly the comparison that- # matters -- 0.0 against 0.15.+ # None / "" is "cannot tell", NOT falsiness: a lora_dropout of 0.0 is a real value and the+ # commonest one, so truthiness would skip exactly the 0.0-against-0.15 comparison that matters.if field in _OPTIONAL_IDENTITY_FIELDS and (a in (None, "") or b in (None, "")):continueif field == "base_revision" and not (_revision_is_comparable(a) and _revision_is_comparable(b)):continue- # Two commit SHAs are only comparable when they name the same repo. A gated base is- # fetched from its byte-identical ungated mirror, so the same weights carry a- # different SHA depending on which repo the host happened to pull from, and comparing- # across the two reports a revision change that did not happen.+ # Two commit SHAs are comparable only when they name the same repo: a gated base fetched from its+ # ungated mirror carries a different SHA for identical weights.if field == "base_revision" and _revision_repo(self) != _revision_repo(other):continueif a == b:@@ -490,10 +458,8 @@ def _content_probe(path: Any) -> str:with open(path, "rb") as handle:digest.update(handle.read(_PROBE_BYTES))if size > _PROBE_BYTES:- # Every byte past the head is covered, either as the tail sample or, for a file- # under 2x the probe, in full. The old `> 2 * _PROBE_BYTES` gate left a band- # (64 KiB to 128 KiB, which is most JPEGs) reading its first 64 KiB and nothing- # else, so a same-length replacement sharing a head went unnoticed.+ # Every byte past the head is covered: the old "> 2 * _PROBE_BYTES" gate left 64-128 KiB files+ # reading only their first 64 KiB, so a same-length replacement sharing a head went unnoticed.handle.seek(max(_PROBE_BYTES, size - _PROBE_BYTES))digest.update(handle.read(_PROBE_BYTES))# ValueError: open() rejects an embedded NUL rather than raising OSError.@@ -513,9 +479,8 @@ def _resolve_lora_targets(cfg: Any) -> tuple[str, ...]:configured = tuple(cfg.lora_target_modules)if str(getattr(cfg, "resolved_family", "") or "").strip().lower() == "sdxl":return configured- # Deliberately NOT wrapped in a try/except: quietly falling back to the generic tuple would- # make the route compute a different fingerprint from the trainer, its preflight would pass,- # and the mismatch would surface in the child AFTER the GPU residents were freed.+ # Deliberately NOT wrapped: falling back to the generic tuple would make the route fingerprint+ # differ from the trainer's, and the mismatch would surface only after the residents were freed.from core.training.diffusion_dit_trainer import _SPECS, _select_lora_targetsspec = _SPECS.get(cfg.resolved_family)@@ -573,9 +538,8 @@ def with_resolved_revision(identity: "CheckpointIdentity", base_model: Any) -> "resolved = source_revision(base_model)if not _revision_is_comparable(resolved) or resolved == identity.base_revision:return identity- # The repo travels with the revision: callers pass the repo they FETCHED from, which for a- # gated base is the ungated mirror, and a SHA without the repo that produced it cannot be- # compared against one from the other repo.+ # The repo travels with the revision: a SHA without the repo that produced it cannot be compared+ # against one from the other repo.return replace(identity, base_revision = resolved, base_revision_repo = str(base_model or ""))@@ -597,8 +561,8 @@ def identity_for_config(from core.training.diffusion_train_extras import source_revisiontargets = tuple(resolved_targets) if resolved_targets else _resolve_lora_targets(cfg)- # ``base_model`` stays canonical everywhere else in the identity; only the revision pair- # below follows the repo the weights are pulled from.+ # ``base_model`` stays canonical everywhere else in the identity; only the revision pair below follows the+ # repo the weights are pulled from.fetch_base_model = str(getattr(cfg, "fetch_base_model", None) or cfg.base_model or "")return CheckpointIdentity(family = str(getattr(cfg, "resolved_family", "") or ""),@@ -608,8 +572,8 @@ def identity_for_config(lora_alpha = int(cfg.lora_alpha if cfg.lora_alpha is not None else cfg.lora_rank),lora_dropout = round(float(getattr(cfg, "lora_dropout", 0.0) or 0.0), 6),cfg_dropout = round(float(getattr(cfg, "cfg_dropout", 0.0) or 0.0), 6),- # flow_shift is float | "auto" | None, so it is recorded as text: "auto" and the number- # it resolves to are different runs, and comparing them as floats would lose that.+ # flow_shift is float | "auto" | None, so it is recorded as text: "auto" and the number it+ # resolves to are different runs, and comparing them as floats would lose that.flow_shift = str(getattr(cfg, "flow_shift", None)),weighting_scheme = str(getattr(cfg, "weighting_scheme", "") or "none"),snr_gamma = _snr_gamma_key(getattr(cfg, "snr_gamma", None)),@@ -625,20 +589,15 @@ def identity_for_config(gradient_accumulation_steps = int(getattr(cfg, "gradient_accumulation_steps", 0) or 0),max_grad_norm = f"{round(float(getattr(cfg, 'max_grad_norm', 0.0) or 0.0), 6)}",ema_decay = f"{round(float(getattr(cfg, 'ema_decay', 0.0) or 0.0), 6)}",- # The EFFECTIVE precision, not the request: a pre-Ampere card resolves bf16 to fp16, and- # recording the request let an fp16 bundle resume in bf16 on a newer card -- restored- # moments continuing under different frozen-base numerics, reported as a clean resume.+ # The EFFECTIVE precision, not the request: a pre-Ampere card resolves bf16 to fp16, so recording+ # the request let an fp16 bundle resume in bf16 on a newer card.precision = effective_mixed_precision(cfg),base_precision = str(getattr(cfg, "base_precision", "") or ""),resolution = int(cfg.resolution),kind = kind,- # The revision of the repo actually FETCHED, recorded together with which repo that was.- # Reading the canonical base instead loses the check entirely: the mirror is selected- # precisely BECAUSE the canonical repo is not cached, so it has no local ref and every- # mirror-backed bundle would record "unresolved", which mismatch_reason skips. Pairing- # the revision with its repo keeps the comparison wherever it is meaningful (same repo- # on both sides, including every pre-mirror bundle) and drops it only across a- # canonical/mirror pair, whose SHAs differ for byte-identical weights.+ # Record the revision of the repo actually FETCHED: the mirror is chosen precisely because the+ # canonical repo is not cached, so reading the canonical one records "unresolved", which+ # mismatch_reason then skips.base_revision = source_revision(fetch_base_model),base_revision_repo = fetch_base_model,dataset_fingerprint = dataset_fingerprint(dataset_pairs) if dataset_pairs else None,@@ -663,6 +622,8 @@ def capture_rng_state(streams: Optional[dict[str, Any]] = None) -> dict[str, Anytry:payload["streams"][str(name)] = _random_state_to_json(stream.getstate())except Exception: # noqa: BLE001 -- a stream we cannot read simply is not restored+ # Could not hand it back. Leaving it on disk still beats deleting the only copy of another run's+ # last resumable state.continuetry:import numpy as np@@ -686,11 +647,9 @@ def capture_rng_state(streams: Optional[dict[str, Any]] = None) -> dict[str, Anyfor i, state in enumerate(torch.cuda.get_rng_state_all()):tensors[f"torch_cuda_{i}"] = stateexcept Exception: # noqa: BLE001 -- one device erroring loses the whole capture- # ALL OR NOTHING on a CUDA host. Keeping the CPU half made the result non-empty,- # so the write-time check accepted it and the preflight (which only requires- # torch_cpu) offered it -- and the restore then left the CUDA generator at its- # freshly seeded position, silently changing every latent, noise and timestep- # draw from the resumed step on. An empty result fails the write instead.+ # ALL OR NOTHING on a CUDA host: keeping the CPU half made the result non-empty, so the preflight+ # (which only requires torch_cpu) offered it and the restore then left the CUDA generator freshly+ # seeded, silently changing every latent and timestep draw.tensors = {}except Exception: # noqa: BLE001 -- torch RNG capture is best-efforttensors = {}@@ -745,20 +704,10 @@ def restore_rng_state(if cpu is not None:torch.set_rng_state(cpu.cpu().to(torch.uint8))if torch.cuda.is_available():- # Per device, not set_rng_state_all. That call needs one state per visible- # device, so covering it with an all-or-nothing guard meant a bundle written- # with fewer devices visible than the resume sees restored NOTHING -- including- # cuda:0, the only device training touches. The whole per-step noise stream- # (randn_like for the latent and the noise, randint for the timestep) then- # restarted at the wrong offset while every other part of the resume looked- # correct, so the run finished healthy and silently wrong: measured on a real- # B200 SDXL run, 192/192 LoRA tensors differed from the uninterrupted control- # and the first step after the resume was off by 0.024. The exposure is- # ordinary -- the trainer is a spawned child inheriting CUDA_VISIBLE_DEVICES,- # which Unsloth masks elsewhere, so a mask change between the run and the Resume- # click was enough. Restoring device by device covers whatever the bundle- # holds and leaves the rest alone, and is identical to the old behaviour when- # the counts match (verified bit-exact against the same control).+ # Per device, not set_rng_state_all: that needs one state per visible device, so a bundle written+ # with fewer devices visible restored NOTHING, cuda:0 included, and the per-step noise stream+ # (randn_like for latent and noise, randint for the timestep) restarted at the wrong offset while+ # the run looked healthy.for i in range(torch.cuda.device_count()):state = tensors.get(f"torch_cuda_{i}")if state is None:@@ -834,28 +783,21 @@ def save_checkpoint(if step < 0:raise ValueError("checkpoint step must be >= 0")if not adapter_state:- # An empty safetensors file has no keys, so the bundle would fail its own validation and- # read as "no checkpoint" -- a silent un-resumable run. Fail loudly at write time instead.+ # An empty safetensors file has no keys, so the bundle would fail its own validation and read as "no+ # checkpoint"; fail loudly at write time instead.raise ValueError("refusing to write a checkpoint with no adapter tensors")root = Path(output_dir).expanduser()root.mkdir(parents = True, exist_ok = True)doomed: list[Path] = []if discard_existing:- # Recorded now, deleted AFTER the new bundle is promoted. Clearing first means a kill or- # an I/O failure mid-write leaves the directory with no checkpoint at all -- and any- # earlier run sharing the folder has irreversibly lost its own, which is the opposite of- # what this writer promises.+ # Deleted only AFTER the new bundle is promoted: clearing first leaves the directory with no+ # checkpoint at all if the write dies.doomed = list_checkpoints(root)else:- # Re-reaching a step that already has a VALID bundle happens one way SAFELY: the run- # resumed at N and stopped before N+1 completed, so its state is byte-identical to what- # is already there, and keeping it avoids the one destructive branch in _promote- # (swapping a good bundle out to make room), where a kill would leave the slot empty.- #- # It has to be THAT bundle, though. Resuming checkpoint-10 in a folder that also holds- # checkpoint-15 and stopping at 15 is the same arithmetic and a completely different- # situation: the freshly trained optimizer, scheduler, sampler and RNG would be dropped- # on the floor and checkpoint_saved would name a bundle from an earlier run.+ # Re-reaching a step that already has a valid bundle is safe only when the state is byte-identical,+ # and keeping it avoids _promote's one destructive branch.+ # It has to be THAT bundle: resuming checkpoint-10 in a folder that also holds checkpoint-15 and+ # stopping at 15 would drop the freshly trained state and name an earlier run's bundle.existing = root / f"{CHECKPOINT_PREFIX}{step}"if (source_checkpoint is not None@@ -875,15 +817,13 @@ def save_checkpoint(_save_tensors(save_file, ema_state, staging / EMA_FILENAME)files["ema"] = EMA_FILENAMEif optimizer is not None:- # torch.save, not safetensors: an optimizer state dict is a nested structure of- # tensors AND scalars (bitsandbytes AdamW8bit additionally carries uint8 moments- # plus their quantization maps), which safetensors cannot express.+ # torch.save, not safetensors: optimizer state is nested tensors AND scalars (AdamW8bit adds uint8+ # moments plus quantization maps), which safetensors cannot express._torch_save(torch, optimizer.state_dict(), staging / OPTIMIZER_FILENAME)files["optimizer"] = OPTIMIZER_FILENAMEoptimizer_class = optimizer_key(optimizer)- # `adapter_state` is trainable_state_dict(model), i.e. named_parameters() order --- # the same traversal the trainers build their param list from, so its key order IS- # the optimizer's positional order.+ # adapter_state is trainable_state_dict(model), i.e. named_parameters() order, the same traversal+ # the trainers build their param list from, so its key order IS the optimizer's positional order.optimizer_param_names = [str(name) for name in adapter_state]if lr_scheduler is not None:_torch_save(torch, lr_scheduler.state_dict(), staging / SCHEDULER_FILENAME)@@ -893,12 +833,9 @@ def save_checkpoint(rng_json = rng.get("json")rng_tensors = rng.get("tensors") or {}if not rng_tensors:- # capture_rng_state never raises -- it returns what it managed to read -- so a- # torch generator it could not snapshot produced a bundle with no rng file at- # all. save_checkpoint returned happily, the service emitted checkpoint_saved,- # and the history advertised a resumable stop that _assert_required_state then- # refuses the moment the user clicks Resume. Fail the WRITE instead, which the- # caller reports as resume_blocked_reason.+ # capture_rng_state never raises, so a generator it could not snapshot produced a bundle with no+ # rng file that _assert_required_state refuses on Resume; fail the WRITE instead, which the caller+ # reports as resume_blocked_reason.raise RuntimeError("the run's random-number state could not be captured, so this checkpoint ""would not be resumable"@@ -912,42 +849,28 @@ def save_checkpoint("kind": identity.kind,"global_step": step,"target_steps": int(target_steps),- # Checkpoints are only ever taken on an optimizer-step boundary, so the- # gradient-accumulation position is always clean. Recorded explicitly so a future- # mid-accumulation checkpoint is a value change, not a format change.+ # Checkpoints are only taken on an optimizer-step boundary; recorded explicitly so a future mid-+ # accumulation checkpoint is a value change, not a format change."micro_step": 0,"created_at": time.time(),"identity": identity.as_dict(),"sampler": sampler_state or None,"rng": rng_json,"ema_updates": int(ema_updates),- # Which optimizer wrote the moments. bitsandbytes AdamW8bit stores "state1"/"state2",- # torch AdamW stores "exp_avg"/"exp_avg_sq", and the trainers pick between them from- # the host (bnb present, fused kernel available, UNSLOTH_DIFFUSION_FP32_OPTIM). Both- # accept the other's state_dict and then KeyError on the first step, so the resume- # has to compare this instead.+ # bitsandbytes AdamW8bit stores state1/state2 and torch AdamW exp_avg/exp_avg_sq, and the trainers+ # pick between them from the host (bnb present, fused kernel available,+ # UNSLOTH_DIFFUSION_FP32_OPTIM); each loads the other's state_dict and then KeyErrors on the first+ # step, so the resume compares this."optimizer_class": optimizer_class,- # The trainable parameter names in the order the optimizer holds them. Its state is- # keyed by POSITION, so a PEFT/diffusers upgrade that changes traversal order while- # keeping the same names would rebind every Adam moment to a different tensor --- # and many LoRA projections share a shape, so it loads cleanly and silently corrupts- # the continued trajectory. Additive: an older bundle has no list and skips the- # check rather than failing it.+ # Optimizer state is keyed by POSITION, so a traversal-order change rebinds every Adam moment to a+ # different same-shaped tensor and silently corrupts the continuation. Additive, so an older+ # bundle skips the check."optimizer_param_names": optimizer_param_names,- # Loop bookkeeping that is not state to restore into an object (running loss, wall- # clock). Nested rather than merged, so a caller can never shadow a reserved key.+ # Nested rather than merged, so a caller can never shadow a reserved key."progress": dict(progress or {}),"files": files,- # Byte size per listed file, so read_checkpoint can catch a truncation the- # header parse structurally cannot. _valid_state_file walks a torch zip's pickle- # to STOP and requires one non-empty data/ member; truncating the tensor storages- # themselves leaves both intact, and torch.load then returns UNINITIALIZED memory- # -- measured at +/-1e22 and non-finite -- which a resume feeds straight into the- # optimizer as Adam moments. The run diverges on its first step while reporting a- # clean resume. This is free to write and free to check.- #- # Additive on purpose: an older bundle has no sizes and skips the check rather- # than failing it, and an older build ignores the key, so no version bump.+ # Byte sizes catch a truncation the header parse structurally cannot: torch.load then returns+ # UNINITIALIZED memory (measured +/-1e22 and non-finite) that a resume feeds in as Adam moments."file_sizes": _file_sizes(staging, files),}# LAST: the manifest is the completion marker, so nothing may be written after it.@@ -958,16 +881,12 @@ def save_checkpoint(shutil.rmtree(staging, ignore_errors = True)raisefor stale in doomed:- # `final` can be one of these when the new step reuses an existing bundle's name;# _promote already replaced it, so removing it here would delete the new checkpoint.if stale != final:shutil.rmtree(stale, ignore_errors = True)_prune_staging(root)- # The source bundle is pinned as well as the one just written. A resumed run writes into- # the directory it resumed FROM, so with keep=2 a run that resumes checkpoint-10 and saves- # 20 and 30 prunes 10 -- and "stop without saving" then removes only what this run wrote,- # leaving the ORIGINAL stopped run with no resume point at all. Its own bundle is not this- # run's to spend.+ # Pin the source bundle too: with keep=2 a run that resumes checkpoint-10 and saves 20 and 30 prunes+ # 10, leaving the original stopped run with no resume point.prune_checkpoints(root,keep = save_total_limit,@@ -1018,9 +937,8 @@ def _write_text(path: Path, text: str) -> None:_fsync_file(path)-# errno values that mean "this platform / filesystem will not flush that handle" rather than-# "the data did not make it". Windows' _commit needs write access, network and container-# filesystems return EINVAL/ENOTSUP for fsync, and neither says anything about the write.+# Windows' _commit needs write access, and network/container filesystems return EINVAL/ENOTSUP for+# fsync; neither says the data did not make it._FSYNC_UNSUPPORTED = frozenset(codefor code in (@@ -1050,8 +968,7 @@ def _fsync_file(path: Path) -> None:the durability guarantee rests on the per-file size check in read_checkpoint instead,which catches the truncation a lost writeback actually produces."""try:- # O_RDWR, not O_RDONLY: Windows' _commit maps to FlushFileBuffers, which needs write- # access on the handle. We just wrote the file, so we own it.+ # O_RDWR, not O_RDONLY: Windows' _commit maps to FlushFileBuffers, which needs write access on the handle.fd = os.open(str(path), os.O_RDWR)except OSError:return@@ -1097,19 +1014,15 @@ def _promote(staging: Path, root: Path, step: int) -> Path:if final.exists():displaced = root / f"{_STAGING_PREFIX}replaced-{step}-{uuid.uuid4().hex[:8]}"os.replace(final, displaced)- # os.replace does NOT restamp the directory: the inode keeps whatever mtime the bundle- # was written with, so an old checkpoint moved aside looked long-abandoned the instant- # it arrived and a concurrent read handed it straight back into the slot we are about to- # fill. Stamp it with the moment of the swap, which is what the grace is measuring.+ # os.replace does NOT restamp the directory, so a moved-aside bundle looked long-abandoned the+ # instant it arrived; stamp it with the moment of the swap.with contextlib.suppress(OSError):os.utime(displaced, None)try:os.replace(staging, final)except OSError:- # The slot is now empty and the only copy of the last resumable state is the one we- # moved aside, so put it back before the failure propagates. Without this, a resave of- # an occupied step (a resumed run rewriting checkpoint-15) that fails to promote leaves- # the run with no checkpoint at all -- strictly worse than the stale one it replaced.+ # Put the moved-aside copy back before the failure propagates, else a failed resave of an occupied+ # step leaves the run with no checkpoint at all.if displaced is not None:with contextlib.suppress(OSError):os.replace(displaced, final)@@ -1118,10 +1031,8 @@ def _promote(staging: Path, root: Path, step: int) -> Path:return final-# ``.tmp-checkpoint-stale-<step>-<uuid>`` / ``.tmp-checkpoint-replaced-<step>-<uuid>``: the step-# is what _promote encodes, so an orphan can be handed back to the slot it came from. ``stale``-# is a promotion killed mid-swap; ``replaced`` is a bundle a later write displaced, held so the-# slot can be restored if that write is later discarded.+# The step is encoded in the name so an orphan can be handed back to its slot: "stale" is a+# promotion killed mid-swap, "replaced" is a bundle a later write displaced._STALE_SLOT = re.compile(r"^(?:stale|replaced)-(\d+)-")_REPLACED_SLOT = re.compile(r"^replaced-(\d+)-")@@ -1144,7 +1055,6 @@ def _prune_staging(root: Path) -> None:return# Newest first, so a stacked slot gets its immediate predecessor back rather than whichever- # entry the filesystem happened to list first.def _written_at(path: Path) -> float:try:return path.stat().st_mtime@@ -1158,14 +1068,8 @@ def _prune_staging(root: Path) -> None:shutil.rmtree(entry, ignore_errors = True)-# How long a ``replaced-`` bundle must have been sitting before a READ path will hand it back.-# _promote leaves the slot empty for the microseconds between the swap-aside and the rename,-# and a history or detail read landing in that window moved the old bundle back in -- so the-# writer's own os.replace then failed against a directory that had reappeared, and the periodic-# or stop-and-save checkpoint was lost. A few seconds covers a window measured in microseconds-# while still recovering a genuinely crashed promotion on the user's next poll. Only-# ``replaced-`` entries wait: a ``stale-`` orphan is from an older build and never in flight,-# and writers pass 0 because they run after their own promotion.+# A read landing in _promote's microsecond window moved the old bundle back, so the writer's+# os.replace failed and the checkpoint was lost; only replaced- entries wait, and writers pass 0._LIVE_REPLACEMENT_GRACE_SECONDS = 5.0@@ -1211,8 +1115,7 @@ def _recover_orphaned_slot(root: Path, entry: Path) -> bool:try:os.replace(entry, slot)except OSError:- # Could not hand it back. Leaving it on disk is still better than deleting the only- # copy of the run's last resumable state.+ # Leaving it on disk still beats deleting the only copy of the run's last resumable state.passreturn True@@ -1231,11 +1134,8 @@ def _retire_replaced_slots(root: Path, *, restore: bool) -> None:except OSError:return- # NEWEST first per slot. Replacements stack: a run that crashed after displacing- # checkpoint-15 leaves its copy behind, and a later run displacing the same slot leaves- # another. Restoring whichever sorted first by NAME (a uuid, so effectively at random)- # resurrected an older branch's optimizer state instead of the predecessor that was- # actually in the slot, and later resumes then continued the wrong lineage.+ # NEWEST first per slot: replacements stack, and restoring whichever sorted first by uuid+ # resurrected an older branch's optimizer state instead of the actual predecessor.def _written_at(path: Path) -> float:try:return path.stat().st_mtime@@ -1253,8 +1153,6 @@ def _retire_replaced_slots(root: Path, *, restore: bool) -> None:restored_slots.add(slot)continueexcept OSError:- # Could not hand it back. Leaving it on disk still beats deleting the only copy- # of another run's last resumable state.continueshutil.rmtree(entry, ignore_errors = True)@@ -1300,13 +1198,8 @@ def prune_checkpoints(back. They are excluded entirely, so the limit governs only what this run wrote."""if keep <= 0:return- # Identity, not pathname, for the same reason the cleanup helpers use it: a run can save at- # a step whose directory was already there (resume checkpoint-10 in a folder that also holds- # checkpoint-15, save at 15), and the bundle occupying that path afterwards is THIS run's.- # Excluding it as pre-existing let the limit be exceeded once per overwritten slot -- an old- # checkpoint-10 with keep=2 and saves at 10, 20 and 30 retained all three -- which is real- # disk on a long run. An entry with no identity (an older caller, or a bundle that could not- # be read) keeps the pathname behaviour, which errs towards keeping.+ # Identity, not pathname: a run can save at a step whose directory already existed, and excluding+ # it as pre-existing let the limit be exceeded once per overwritten slot.kept_from_before: set[Path] = set()for entry in preexisting or ():if isinstance(entry, tuple):@@ -1414,8 +1307,8 @@ def discard_preexisting_checkpoints(else:keep[Path(entry)] = _bundle_identity(Path(entry))for stale in list_checkpoints(root):- # Identity, not pathname: a bundle this run WROTE OVER one that was here is this run's,- # and deleting it would throw away the stop checkpoint the user asked for.+ # Identity, not pathname: a bundle this run wrote OVER one that was here is this run's, and deleting+ # it would throw away the stop checkpoint the user asked for.if stale in keep and keep[stale] == _bundle_identity(stale):shutil.rmtree(stale, ignore_errors = True)_retire_replaced_slots(root, restore = False)@@ -1454,11 +1347,8 @@ def clear_own_checkpoints(output_dir: str | os.PathLike[str], preexisting: "Iterthing the user was trying not to disturb. Bundles are identified by the set captured beforethe run's first write, not by step number, because a resume writes lower numbers than theones already there."""- # Keyed by path AND by the bundle's own identity, not by path alone. A resumed run can- # periodically save at a step whose directory was already there (resume checkpoint-10 in a- # folder that also holds checkpoint-15, save at 15), which REPLACES that bundle: the- # original is gone either way, and matching on the name alone then preserved the discarded- # run's replacement as though it were the bundle it overwrote.+ # Keyed by path AND by identity: a periodic save can REPLACE a pre-existing bundle, and matching+ # on the name alone preserved the discarded run's replacement as the bundle it overwrote.keep: dict[Path, Optional[tuple]] = {}for entry in preexisting:# A bare path (an older caller) keeps the pathname-only behaviour for that entry.@@ -1471,10 +1361,8 @@ def clear_own_checkpoints(output_dir: str | os.PathLike[str], preexisting: "Iterif stale in keep and keep[stale] == _bundle_identity(stale):continueshutil.rmtree(stale, ignore_errors = True)- # A bundle this run wrote OVER a pre-existing one has just been removed, so its slot is free- # and the displaced original goes back into it. Without this the discard takes the other- # run's resume point with it: the replacement is not the bundle it overwrote, so the identity- # match above cannot keep it, and the original was already gone.+ # Put the displaced original back: the replacement is not the bundle it overwrote, so the identity+ # match cannot keep it and the original was already gone._retire_replaced_slots(Path(output_dir).expanduser(), restore = True)@@ -1489,8 +1377,8 @@ def _bundle_identity(path: Path) -> Optional[tuple]:manifest = json.loads((path / TRAINER_STATE_FILENAME).read_text(encoding = "utf-8"))except (OSError, ValueError):return None- # created_at is the moment THIS bundle's manifest was written -- the completion marker, so- # it exists on every valid bundle and differs between two writes at the same step.+ # created_at is when THIS bundle's manifest was written, the completion marker, so it exists on+ # every valid bundle and differs between two writes at the same step.return (manifest.get("created_at"), manifest.get("global_step"))@@ -1558,14 +1446,13 @@ def read_checkpoint(path: str | os.PathLike[str]) -> Optional[dict[str, Any]]:for role, name in files.items():if not isinstance(name, str) or not name or Path(name).name != name:return None- # Optimizer / scheduler state can be validly tensor-free (SGD without momentum, a- # constant LR schedule), so only the weight bundles must carry tensors.+ # Optimizer and scheduler state can validly be tensor-free (SGD without momentum, a constant LR+ # schedule), so only the weight bundles must carry tensors.if not _valid_state_file(directory / name, require_tensor = role in ("adapter", "ema")):return Noneexpected_size = sizes.get(role)if isinstance(expected_size, int) and not isinstance(expected_size, bool):- # Absent on a bundle written before sizes were recorded, which skips the check- # rather than failing it -- an old checkpoint stays resumable.+ # Absent on a bundle written before sizes were recorded, which skips the check rather than failing it.try:if (directory / name).stat().st_size != expected_size:return None@@ -1610,11 +1497,9 @@ def latest_valid_checkpoint(its own. The identity gate cannot catch that -- same family, same base, same dataset, sameLoRA shape -- so the earlier run resumes the later run's optimizer moments, LR positionand RNG under its own config, and records the wrong lineage while doing it."""- # Repair first. A promotion killed between the swap-aside and the rename leaves the run's- # only bundle under the hidden stale name, and _prune_staging fixes that -- but it runs- # after a later successful save, which a user who cannot resume will never reach. So the- # read path that decides "there is nothing to resume" hands the orphan back to its slot- # before it answers. Idempotent, and it only ever touches an EMPTY slot.+ # A promotion killed mid-swap leaves the only bundle under the hidden stale name, and+ # _prune_staging runs only after a later save a stuck user never reaches. Idempotent, and it only+ # ever touches an EMPTY slot._recover_orphaned_slots(Path(output_dir).expanduser())for candidate in list_checkpoints(output_dir):manifest = read_checkpoint(candidate)@@ -1632,8 +1517,7 @@ def latest_valid_checkpoint(created = float(manifest.get("created_at") or 0.0)except (TypeError, ValueError):created = 0.0- # A bundle with no usable created_at (0.0) is not fenced out by the upper bound:- # it predates the field, and refusing it would make an old run unresumable.+ # A bundle with created_at 0.0 predates the field, so the upper bound must not fence it out.if created and created > float(not_after):continue# An extra gate for callers whose answer PINS the bundle it names.@@ -1718,10 +1602,8 @@ def _source_checkpoint_bundle(# Not the bundle this run resumed: something replaced the slot after the fact.if not written or abs(written - float(source_created_at)) > 1e-6:return None- # Same gate the directory scan applies, for the same reason: this path is pinned back to the- # start route verbatim, and read_checkpoint is only a header scan. A source whose required- # state or real torch.load has gone since would be advertised as resumable and refused on- # the first click.+ # Same gate the directory scan applies: this path is pinned back verbatim and read_checkpoint is+ # only a header scan, so a source that lost required state would be advertised as resumable.if not _fully_loadable(candidate, manifest):return Nonereturn (candidate, manifest)@@ -1758,9 +1640,8 @@ def describe_resume_state(try:root = Path(str(output_dir)).expanduser()if not root.is_dir():- # A resume into a NEW output dir that died before its first save never created it,- # and the bundle it was validated against is still sitting in the source dir. The- # fallback below is exactly for that, so it has to be reachable from here.+ # A resume into a NEW output dir that died before its first save never created it, so the fallback+ # below has to be reachable from here.recovered = _source_checkpoint_bundle(source_checkpoint, source_created_at)if recovered is None:return {@@ -1773,18 +1654,13 @@ def describe_resume_state(root,not_before = started_at,not_after = ended_at,- # The action this answer drives sends the exact path back, which the preflight- # then treats as explicit and cannot scan past -- so the bundle named here has- # to be one that will actually load, not merely one whose header parses.+ # The exact path is sent back and treated as explicit, so the bundle named here has to be one that+ # will actually load, not merely one whose header parses.usable = _fully_loadable,)if found is None and source_checkpoint:- # This run RESUMED and then ended before writing a bundle of its own -- an OOM on- # the first restored step is the usual way. The source it was validated against is- # still on disk and still correct to continue from, but it predates started_at, so- # the fence above excludes it and the run reads as unresumable when retrying is the- # obvious thing to do. Read it directly rather than widening the fence, which exists- # to stop an unrelated earlier run's bundles being offered.+ # A run that resumed and died before its first save has a source bundle predating started_at: read+ # it directly rather than widening the fence, which exists to hide unrelated earlier runs.found = _source_checkpoint_bundle(source_checkpoint, source_created_at)except OSError:return blank@@ -1792,8 +1668,8 @@ def describe_resume_state(if not list_checkpoints(root):reason = "This run saved no resume checkpoint, so it cannot be continued."elif started_at is not None and latest_valid_checkpoint(root) is not None:- # Bundles are there, but every one predates this run: they belong to an earlier run- # that trained into the same folder. Saying "corrupt" here would be a lie.+ # Every bundle predates this run and belongs to an earlier run that trained into the same folder, so+ # "corrupt" would be a lie.reason = ("This run saved no resume checkpoint of its own; the checkpoints in its folder ""were left by an earlier run of the same adapter."@@ -1803,8 +1679,7 @@ def describe_resume_state(return {**blank, "resume_blocked_reason": reason}path, manifest = foundstep = int(manifest.get("global_step") or 0)- # The target the resume would run to. The record's own total_steps wins (it is what the UI- # replays); the manifest's copy covers a record written before that field existed.+ # The record's own total_steps wins; the manifest's copy covers a record written before that field existed.target = int(total_steps or manifest.get("target_steps") or 0)if target and step >= target:return {@@ -1840,37 +1715,35 @@ def resolve_resume_dir(path_value: str) -> Path:except ValueError as error:message = str(error)if not message.startswith("Resume checkpoint"):- # The containment resolver raises an internal "path escapes root: <abs> -> <abs>"- # that quotes server paths at the user; replace it with the same wording the LLM- # resume flow shows.+ # The containment resolver's message quotes server paths at the user; replace it with the resume+ # flow's wording.message = "Resume checkpoint must be inside Unsloth outputs."raise ResumeError(message) from error- # A name that cleans away to nothing (".", "outputs", "./.") lands on the outputs ROOT, where- # the scan would sweep checkpoint dirs across unrelated runs. Same guard the start route- # applies to output_dir.+ # A name that cleans away to nothing (".", "outputs", "./.") lands on the outputs ROOT, where the+ # scan would sweep checkpoint dirs across unrelated runs. Same guard the start route applies to+ # output_dir.if resolved.resolve(strict = False) == outputs_root().resolve(strict = False):+ # Named like a bundle, is not one, and holds no bundles either: the original message is the accurate+ # one.raise ResumeError(f"'{path_value}' is the outputs folder itself, not a training run inside it.")return resolved-# What restore_resume_state() refuses to continue without. Kept here rather than inferred from-# the manifest, because the manifest is exactly what a truncated or hand-edited bundle gets-# wrong: _assert_loadable only opens the roles the bundle happens to LIST, so a checkpoint that-# lists an adapter and nothing else passed the route preflight, the resident GPU model was-# evicted, and the child then died on "This checkpoint carries no optimizer state".-# The random.Random streams both trainers construct and hand to capture_rng_state. Named here-# because the preflight has to know what a complete bundle carries.+# Kept here rather than inferred from the manifest: _assert_loadable only opens the roles the+# bundle LISTS, so an adapter-only checkpoint passed the route preflight and the child then died+# after the resident GPU model was evicted.+# The random.Random streams both trainers hand to capture_rng_state; the preflight has to know what+# a complete bundle carries._TRAINER_RNG_STREAMS: tuple[str, ...] = ("loop", "variant")_REQUIRED_STATE: tuple[tuple[str, str], ...] = (("adapter", "the trained LoRA weights"),("optimizer", "the optimizer moments"),("scheduler", "the learning-rate schedule position"),- # Both image trainers draw the latent, the noise and the timestep from torch immediately- # after resume, so a bundle with no RNG file continues a different random stream while- # restore_rng_state leaves the generator at its fresh seed and says nothing.+ # Both image trainers draw the latent, noise and timestep from torch immediately after resume, so a+ # bundle with no RNG file continues a different stream and says nothing.("rng", "the random-number generator state"),)@@ -1884,20 +1757,14 @@ def _assert_required_state(path: Path, manifest: dict[str, Any]) -> None:for role, label in _REQUIRED_STATEif not isinstance(listed.get(role), str) or not listed.get(role)]- # The sampler's permutation lives in the manifest rather than a file. Required for an- # image bundle, where both trainers always supply a sampler and the child refuses without- # it -- after the route has evicted the resident models. Left optional for any other kind,- # whose trainer may legitimately have none.+ # Required for an image bundle, where both trainers always supply a sampler and the child refuses+ # without it; optional for any other kind.if str(manifest.get("kind") or "image") == "image" and not isinstance(manifest.get("sampler"), dict):missing.append("the dataset sampler position")- # The torch generator lives in the rng FILE; the two random.Random streams the trainers own- # (the loop's index/crop draws and the latent-cache variant picks) live in the manifest- # beside it. restore_rng_state is per-part best-effort, so a bundle that lists the file but- # has lost either stream restores torch and silently leaves those two at their fresh seeds:- # crop and flip selection diverges on the first step, and the sampler draws a different- # permutation once its saved cycle ends. Both trainers always write both.+ # restore_rng_state is per-part best-effort, so a bundle that lists the rng file but lost either+ # random.Random stream leaves crop/flip selection and the sampler permutation freshly seeded.rng_manifest = manifest.get("rng")saved_streams = rng_manifest.get("streams") if isinstance(rng_manifest, dict) else Noneif not isinstance(saved_streams, dict) or not all(@@ -1978,8 +1845,8 @@ def _assert_loadable(path: Path, manifest: dict[str, Any]) -> None:if role == "rng" and not (isinstance(state, dict) and state.get("torch_cpu") is not None):- # An rng file with no torch state restores nothing torch draws from, which- # is every latent, noise and timestep the loop asks for.+ # An rng file with no torch state restores nothing torch draws from, which is every latent, noise+ # and timestep the loop asks for.raise ResumeError(f"'{path.name}' carries no torch random-number state, so the run ""would continue on a different random stream. Resume from an earlier "@@ -2006,14 +1873,9 @@ def preflight_resume(``identity`` may leave ``dataset_fingerprint`` unset on the first (pre-discovery) pass;that comparison is then skipped and re-run once the images are known."""root = resolve_resume_dir(path_value)- # An explicit checkpoint-<N> directory resumes exactly that step; a run directory takes- # its newest complete bundle.- #- # The name alone does not settle which it is. An adapter can legitimately be called- # "checkpoint-2026", and its output directory then matches the bundle pattern while- # containing checkpoint-11 rather than a trainer_state.json of its own -- read as an- # explicit bundle it is simply rejected, with the real checkpoint sitting inside it. So the- # explicit branch is taken only when the path IS a valid bundle.+ # The name alone does not settle it: an adapter can legitimately be called "checkpoint-2026", so+ # its output directory matches the bundle pattern while holding no trainer_state.json of its own.+ # The explicit branch is taken only when the path IS a valid bundle.explicit = read_checkpoint(root) if checkpoint_step(root) >= 0 else Nonecandidates: list[tuple[Path, dict]]if explicit is not None:@@ -2021,8 +1883,6 @@ def preflight_resume(else:candidates = iter_valid_checkpoints(root)if not candidates and checkpoint_step(root) >= 0:- # Named like a bundle, is not one, and holds no bundles either: the original- # message is the accurate one.raise ResumeError(f"'{root.name}' is not a complete training checkpoint (it is missing files, ""or was left behind by an interrupted save)."@@ -2032,20 +1892,15 @@ def preflight_resume("No complete training checkpoint was found for this run, so there is nothing to ""resume from. Start a new run instead.")- # Newest first, and a failure here is not the end of the search when a DIRECTORY was given:- # the header scan that produced this list is deliberately cheap, so the newest bundle can- # fail torch.load or turn out to be missing required state while the retained older one is- # perfectly good. Stopping at the first failure made the retention policy pointless. An- # EXPLICIT bundle has no alternatives, so its first failure is still the answer.+ # Newest first, and the cheap header scan means the newest bundle can still fail torch.load while a+ # retained older one is good; stopping at the first failure made retention pointless.first_error: Optional[ResumeError] = Nonefor path, manifest in candidates:try:return _validated_resume(path, manifest, identity, target_steps)except ResumeError as exc:- # "Already at the target" is an answer, not a damaged bundle. Falling past it- # walked back to checkpoint-400 of a run that finished at 500 and retrained- # completed work -- the exact rollback the fence exists to prevent. Only a bundle- # that cannot be USED is a reason to try the retained one behind it.+ # "Already at the target" is an answer, not a damaged bundle: falling past it walked back to an+ # earlier checkpoint and retrained completed work.if getattr(exc, "terminal", False):raiseif first_error is None:@@ -2080,8 +1935,8 @@ def _validated_resume(_assert_loadable(path, manifest)step = int(manifest.get("global_step") or 0)if target_steps and step >= int(target_steps):- # Terminal: the newest bundle having nothing left to train says nothing is WRONG with- # it, so the directory scan must stop here rather than offering the retained older one.+ # Terminal: nothing is WRONG with the newest bundle, so the scan must stop rather than offer the+ # retained older one.raise _terminal(ResumeError(f"This checkpoint is already at step {step} of {int(target_steps)}, so there "diff --git a/studio/backend/core/training/diffusion_clip_formats.py b/studio/backend/core/training/diffusion_clip_formats.pyindex f87c7d6a9ab..10d44862406 100644--- a/studio/backend/core/training/diffusion_clip_formats.py+++ b/studio/backend/core/training/diffusion_clip_formats.py@@ -17,8 +17,7 @@ not count is a dataset that uploads fine and then trains on nothing.from __future__ import annotations-# Containers the video decode path (PyAV) opens. Lowercase, with the leading dot, so a-# ``Path.suffix.lower()`` can be tested against the set directly.+# Lowercase, with the leading dot, so a Path.suffix.lower() can be tested against the set directly.CLIP_EXTS = frozenset({".mp4", ".mov", ".mkv", ".webm", ".m4v", ".avi"})__all__ = ["CLIP_EXTS"]diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.pyindex 010ac0c9ab0..4a1c91f4848 100644--- a/studio/backend/core/training/diffusion_dit_trainer.py+++ b/studio/backend/core/training/diffusion_dit_trainer.py@@ -83,7 +83,8 @@ from core.training.diffusion_train_extras import (save_ema_adapter,)-# Per-family LoRA target modules (attention projections). FLUX / Qwen double-stream blocks also carry added-kv projections; Z-Image is single-stream.+# Per-family LoRA target modules (attention projections). FLUX / Qwen double-stream blocks also+# carry added-kv projections; Z-Image is single-stream._FLUX_TARGETS = ("to_q","to_k",@@ -96,7 +97,8 @@ _FLUX_TARGETS = ()_QWEN_TARGETS = _FLUX_TARGETS_ZIMAGE_TARGETS = ("to_q", "to_k", "to_v", "to_out.0")-# The Krea 2 authors' recommended defaults (their DreamBooth script): attention + SwiGLU + text-fusion projector + conditioning embedders. For long runs they suggest narrowing to attention.+# The Krea 2 authors' recommended defaults (their DreamBooth script): attention + SwiGLU + text-+# fusion projector + conditioning embedders. For long runs they suggest narrowing to attention._KREA2_TARGETS = ("img_in","final_layer.linear",@@ -138,23 +140,26 @@ class _FamilySpec:lora_targets: tuple[str, ...]# bf16 only (Z-Image overflows fp16 and its RoPE/embedder run in fp32).force_bf16: bool- # Approximate dense-bf16 transformer weight size for the family's DEFAULT base, used by- # base_precision="auto" to pick a mode that fits free VRAM. A family covering more than one+ # Used by base_precision="auto" to pick a mode that fits free VRAM; a family covering more than one# size (flux.2-klein is 4B and 9B) must not be sized off this alone -- see _dense_bf16_gb.dense_bf16_gb: float- # Phased load so the multi-GB transformer never coexists with the text encoders + VAE: ``load_conditioners`` builds the- # pipeline WITHOUT it and returns (pipe, vae); ``load_transformer`` then loads it alone once the conditioners are freed.+ # Phased load so the multi-GB transformer never coexists with the text encoders + VAE:+ # load_conditioners builds the pipeline WITHOUT it and returns (pipe, vae), then load_transformer+ # loads it alone once the conditioners are freed.load_conditioners: Callable[..., tuple[Any, Any]]load_transformer: Callable[..., Any]# Encode a list of captions -> a per-caption tuple of CPU tensors (the family's embeds).encode_prompts: Callable[..., list[tuple]]# Encode a pixel tensor [B,3,H,W] in [-1,1] -> latents (family-normalised, on device).encode_latents: Callable[..., Any]- # Encode pixels to (A, B) affine posterior params so a per-step sample is A + B * randn. B is None for a deterministic family.+ # Encode pixels to (A, B) affine posterior params so a per-step sample is A + B * randn. B is None+ # for a deterministic family.encode_latent_stats: Callable[..., tuple]- # Collate per-caption embed tuples into one batched tuple on device. ``pad_to`` pins a fixed text length for variable-length embeds (compile).+ # Collate per-caption embed tuples into one batched tuple on device. ``pad_to`` pins a fixed text+ # length for variable-length embeds (compile).collate: Callable[..., tuple]- # One transformer forward: (transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype) -> model_pred aligned with target = noise - latents.+ # One transformer forward: (transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype)+ # -> model_pred aligned with target = noise - latents.forward: Callable[..., Any]# Save the LoRA in diffusers format via the family pipeline's save_lora_weights.save: Callable[..., None]@@ -195,7 +200,8 @@ def _training_sigma_table(scheduler, flow_shift):mu = float(getattr(sc, "max_shift", None) or math.log(3.0))shifted = scheduler.time_shift(mu, 1.0, sigmas)if getattr(sc, "shift_terminal", None):- # The stretch scales off the table's own final sigma, so it must see the full descending schedule, never a batch.+ # The stretch scales off the table's own final sigma, so it must see the full descending schedule,+ # never a batch.shifted = scheduler.stretch_shift_to_terminal(shifted)return shifteds = float(flow_shift)@@ -248,7 +254,7 @@ def _encoders_to_device(pipe, device) -> None:try:enc.to(device)except (ValueError, RuntimeError, NotImplementedError):- pass # already-placed 4-bit encoder / non-movable module+ passdef _bnb_4bit_config():@@ -258,12 +264,14 @@ def _bnb_4bit_config():load_in_4bit = True,bnb_4bit_quant_type = "nf4",bnb_4bit_compute_dtype = torch.bfloat16,- # Double quantization compresses the per-block absmax scales with a second 8-bit pass: ~0.4 bits/param off the frozen base at no fidelity cost, which matters most on the 20B+ DiTs.+ # Double quantization compresses the per-block absmax scales with a second 8-bit pass: ~0.4 bits/param+ # off the frozen base at no fidelity cost, which matters most on the 20B+ DiTs.bnb_4bit_use_double_quant = True,)-# Kept as a module name for existing callers/tests; the heuristic itself moved to diffusion_train_common so config validation can use it without importing this module.+# The heuristic moved to diffusion_train_common so config validation can use it without importing+# this module; the name is kept for existing callers._repo_is_prequantized = repo_is_prequantized@@ -320,7 +328,8 @@ def _load_dit_transformer(transformer_cls, cfg, device, base_precision):transformer = transformer.to(device)return transformer- # Dense load for bf16 / fp8 / mxfp8 / int8. int8 quantizes AFTER the LoRA attaches: quantizing first makes peft dispatch TorchaoLoraLinear, whose peft-0.18 constructor rejects the torchao-0.16 config API.+ # int8 quantizes AFTER the LoRA attaches: quantizing first makes peft dispatch TorchaoLoraLinear,+ # whose peft-0.18 constructor rejects the torchao-0.16 config API.return transformer_cls.from_pretrained(cfg.base_model,subfolder = "transformer",@@ -360,9 +369,8 @@ def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family)),)- # After quantize_, as the helper requires: it reparents the Linears it wraps. Not- # best-effort -- a raise means the base is quantized but not safely runnable, which is the- # one state worse than either end.+ # After quantize_, as the helper requires (it reparents the Linears it wraps). Not best-effort: a+ # raise means the base is quantized but not safely runnable.apply_small_m_padding(transformer, "int8", family)@@ -428,7 +436,8 @@ def _mx_module_filter(mod, fqn: str) -> bool:return Falseif fqn.endswith("proj_out") or ".proj_out." in fqn:return False- # Skip biased linears: the torchao 0.17 MX path swaps the weight for a wrapper that drops the bias, changing the output the LoRA regresses against.+ # Skip biased linears: the torchao 0.17 MX path swaps the weight for a wrapper that drops the bias,+ # changing the output the LoRA regresses against.if getattr(mod, "bias", None) is not None:return Falsereturn mod.in_features % 32 == 0 and mod.out_features % 32 == 0@@ -550,22 +559,25 @@ def _resolve_base_precision(cfg, spec, device) -> str:"sm_89+, mxfp8 sm_100+) and this is a ROCm/AMD GPU. Use base_precision='nf4', ""'bf16', or 'auto'.")- # int8 has no runtime fallback, so an explicit int8 against a missing torchao (or the Windows-ROCm stub) would leave the- # transformer dense with compile disabled. The auto pick and /info gate on a FUNCTIONAL torchao; do the same here.+ # int8 has no runtime fallback, so gate on a FUNCTIONAL torchao here as the auto pick and /info do;+ # find_spec is satisfied by the Windows-ROCm stub.if mode == "int8" and not has_functional_torchao():raise ValueError("base_precision='int8' needs a functional torchao install; this host's ""torchao is missing or the non-functional Windows-ROCm stub. Use ""base_precision='nf4', 'bf16', or 'auto'.")- # The stub answers torchao.float8 / torchao.prototype.mx_formats with a no-op that reports success, so the run would report fp8 while training bf16.- # Keyed on the stub, not has_functional_torchao(): that probes int8's symbols, and a real-but-partial torchao must still reach the arch checks below.+ # The stub answers torchao.float8 / torchao.prototype.mx_formats with a no-op that reports success,+ # so the run would report fp8 while training bf16.+ # Keyed on the stub, not has_functional_torchao(): that probes int8's symbols, and a real-but-+ # partial torchao must still reach the arch checks below.if mode in ("fp8", "mxfp8") and is_stubbed("torchao"):raise ValueError(f"base_precision={mode!r} is not available on this host: torchao is the ""non-functional Windows-ROCm stub. Use base_precision='nf4', 'bf16', or 'auto'.")- # mxfp8 needs Blackwell (sm100+): its MX GEMM raises at the first training step, after a full dense load. Re-check here to fail fast for a stale client.+ # mxfp8 needs Blackwell (sm100+): its MX GEMM raises at the first training step, after a full dense+ # load. Re-check here to fail fast for a stale client.if mode == "mxfp8" and device == "cuda":try:import torch@@ -578,22 +590,22 @@ def _resolve_base_precision(cfg, spec, device) -> str:"Use base_precision='bf16', 'int8', 'nf4', or 'auto'.")return mode- # auto may only resolve to the dense modes when the run uses bf16 compute, mirroring the normalized() rule for explicit dense modes; otherwise stay on the nf4 floor.+ # auto may only resolve to the dense modes when the run uses bf16 compute, mirroring the+ # normalized() rule for explicit dense modes; otherwise stay on the nf4 floor.if getattr(cfg, "mixed_precision", "bf16") != "bf16":return "nf4"prequant = repo_is_prequantized(cfg.base_model)free_gb = Nonecapability = Nonehas_fp8 = False- # int8 has no runtime fallback, so gate the auto pick on a FUNCTIONAL torchao: find_spec("torchao") is satisfied by the Windows-ROCm stub whose quantize_ is a no-op.# Auto must not select a torchao mode on ROCm.has_torchao = has_functional_torchao() and not torch_is_rocm()if device == "cuda":try:import torch- # Windows ROCm over-reports free VRAM (#8403), which would pick a- # dense precision the card cannot actually hold.+ # Windows ROCm over-reports free VRAM (#8403), which would pick a dense precision the card cannot+ # actually hold.from utils.hardware import trusted_mem_get_infofree_gb = trusted_mem_get_info()[0] / 1e9@@ -667,14 +679,16 @@ def _flux_collate():import torch- # FLUX embeds are fixed-length, so a plain cat batches them; text_ids are shared position ids, identical across prompts.+ # FLUX embeds are fixed-length, so a plain cat batches them; text_ids are shared position ids,+ # identical across prompts.pe = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)pooled = torch.cat([e[1] for e in entries]).to(device = device, dtype = weight_dtype)text_ids = entries[0][2].to(device = device, dtype = torch.float32)return (pe, pooled, text_ids)-# Per-run cache of the step-invariant FLUX conditioning tensors (RoPE image ids + guidance vector): their shapes are fixed once resolution/batch are. Cleared at run start.+# Per-run cache of the step-invariant FLUX conditioning tensors: their shapes are fixed once+# resolution and batch are. Cleared at run start._FLUX_STATIC: dict[tuple, tuple] = {}@@ -685,7 +699,8 @@ def _flux_static_inputs(bsz, h, w, device):key = (bsz, h, w, str(device))hit = _FLUX_STATIC.get(key)if hit is None:- # Position ids drive RoPE and are indices, not activations, so keep them float32 regardless of the training dtype.+ # Position ids drive RoPE and are indices, not activations, so keep them float32 regardless of the+ # training dtype.img_ids = FluxPipeline._prepare_latent_image_ids(bsz, h // 2, w // 2, device, torch.float32)guidance = torch.full((bsz,), 1.0, device = device, dtype = torch.float32)hit = _FLUX_STATIC[key] = (img_ids, guidance)@@ -728,7 +743,8 @@ def _qwen_load_conditioners(cfg, device, weight_dtype):def _qwen_load_transformer(cfg, device, weight_dtype, base_precision):- # The prequant default ships the transformer 4-bit and loads trainable as-is under nf4; the dense modes need the 20B Qwen/Qwen-Image base.+ # The prequant default ships the transformer 4-bit and loads trainable as-is under nf4; the dense+ # modes need the 20B Qwen/Qwen-Image base.from diffusers import QwenImageTransformer2DModelreturn _load_dit_transformer(QwenImageTransformer2DModel, cfg, device, base_precision)@@ -762,10 +778,11 @@ def _qwen_latent_affine(vae, ref):def _qwen_encode_latents(vae, pixel_values):import torch- # AutoencoderKLQwenImage is a 3D (video) VAE: add a temporal dim, encode, and normalise by the per-channel latents_mean / latents_std.- px = pixel_values.to(torch.float32).unsqueeze(2) # [B,3,1,H,W]+ # AutoencoderKLQwenImage is a 3D (video) VAE: add a temporal dim, encode, and normalise by the per-+ # channel latents_mean / latents_std.+ px = pixel_values.to(torch.float32).unsqueeze(2)with torch.no_grad():- lat = vae.encode(px).latent_dist.sample() # [B,16,1,h,w]+ lat = vae.encode(px).latent_dist.sample()mean, std = _qwen_latent_affine(vae, lat)return (lat - mean) / std@@ -789,7 +806,8 @@ def _qwen_collate(import torchimport torch.nn.functional as F- # Qwen embeds are variable-length: pad to the batch max (or a pinned ``pad_to`` bucket under compile) and batch the validity mask with them.+ # Qwen embeds are variable-length: pad to the batch max (or a pinned ``pad_to`` bucket under+ # compile) and batch the validity mask with them.seqs = [e[0].shape[1] for e in entries]target = max(pad_to or 0, max(seqs))pes, masks = [], []@@ -804,7 +822,8 @@ def _qwen_collate(masks.append(mask)pe_b = torch.cat(pes).to(device = device, dtype = weight_dtype)mask_b = torch.cat(masks).to(device)- # A single unpadded sample keeps the legacy None mask (identical math; avoids a behaviour delta for existing single-image runs).+ # A single unpadded sample keeps the legacy None mask (identical math; avoids a behaviour delta for+ # existing single-image runs).if len(entries) == 1 and entries[0][1] is None and target == seqs[0]:mask_b = Nonereturn (pe_b, mask_b)@@ -816,7 +835,8 @@ def _qwen_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, devipe, mask = embeds_batchbsz, c, f, h, w = noisy.shapepacked = QwenImagePipeline._pack_latents(noisy, bsz, c, h, w)- # Each batch entry is a LIST of one (frame, h/2, w/2) tuple: the transformer indexes sample[0] / sample[1:] per entry, so a flat list breaks it.+ # Each batch entry is a LIST of one (frame, h/2, w/2) tuple: the transformer indexes sample[0] /+ # sample[1:] per entry, so a flat list breaks it.img_shapes = [[(1, h // 2, w // 2)]] * bszpred = transformer(hidden_states = packed,@@ -845,7 +865,8 @@ def _zimage_load_conditioners(cfg, device, weight_dtype):def _zimage_load_transformer(cfg, device, weight_dtype, base_precision):- # Prequant default loads 4-bit as-is under nf4; the dense modes use the bf16 Tongyi-MAI base. Z-Image is bf16 only (its RoPE/embedder run fp32; fp16 overflows).+ # Prequant default loads 4-bit as-is under nf4; the dense modes use the bf16 Tongyi-MAI base.+ # Z-Image is bf16 only (its RoPE/embedder run fp32; fp16 overflows).from diffusers import ZImageTransformer2DModelreturn _load_dit_transformer(ZImageTransformer2DModel, cfg, device, base_precision)@@ -877,7 +898,8 @@ def _zimage_encode_latents(vae, pixel_values):def _zimage_encode_latent_stats(vae, pixel_values):- # Z-Image trains from the posterior mode (deterministic), so the cached entry is the final latent: B is None and the loop skips the per-step draw.+ # Z-Image trains from the posterior mode (deterministic), so the cached entry is the final latent: B+ # is None and the loop skips the per-step draw.return _zimage_encode_latents(vae, pixel_values), None@@ -895,7 +917,8 @@ def _zimage_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, deimport torch(caps,) = embeds_batch- # List I/O: one [C,1,H,W] latent + one [seq,2560] caption per sample. The timestep convention is REVERSED ((1000 - t) / 1000) and the prediction is NEGATED.+ # List I/O: one [C,1,H,W] latent + one [seq,2560] caption per sample. The timestep convention is+ # REVERSED ((1000 - t) / 1000) and the prediction is NEGATED.x_list = list(noisy.unsqueeze(2).unbind(dim = 0))t_norm = (1000 - timesteps) / 1000out = transformer(x_list, t_norm, list(caps), return_dict = False)[0]@@ -913,7 +936,8 @@ def _zimage_save(pipe_cls, out_dir, transformer_lora_layers):# ── Krea 2 ────────────────────────────────────────────────────────────────────def _krea2_load_conditioners(cfg, device, weight_dtype):- # The krea repo ships transformers-5.x style configs the pinned 4.x line cannot parse, so the conditioning pipeline is assembled per-component (diffusion_krea2.py).+ # The krea repo ships transformers-5.x style configs the pinned 4.x line cannot parse, so the+ # conditioning pipeline is assembled per-component (diffusion_krea2.py).import torchfrom core.inference.diffusion_krea2 import load_krea2_pipeline@@ -925,7 +949,8 @@ def _krea2_load_conditioners(cfg, device, weight_dtype):def _krea2_load_transformer(cfg, device, weight_dtype, base_precision):- # The transformer subfolder is diffusers-format. No prequant repo yet, so nf4 quantizes the 12B transformer on the fly.+ # The transformer subfolder is diffusers-format. No prequant repo yet, so nf4 quantizes the 12B+ # transformer on the fly.from diffusers import Krea2Transformer2DModelreturn _load_dit_transformer(Krea2Transformer2DModel, cfg, device, base_precision)@@ -937,7 +962,8 @@ def _krea2_encode_prompts(pipe, captions, device):out = []with torch.no_grad():for cap in captions:- # encode_prompt pads/truncates to max_sequence_length, so every embed is [1, 512, num_text_layers, 2560] with a [1, 512] mask (static shapes; padding sits before the assistant suffix, as at training).+ # encode_prompt pads/truncates to max_sequence_length, so every embed is+ # [1, 512, num_text_layers, 2560] with a [1, 512] mask (static shapes, padding before the suffix).pe, mask = pipe.encode_prompt(prompt = cap,device = device,@@ -948,7 +974,8 @@ def _krea2_encode_prompts(pipe, captions, device):return out-# Krea 2 conditions on the Qwen-Image VAE with the same per-channel latents_mean / latents_std normalisation, so latent encoding is shared.+# Krea 2 conditions on the Qwen-Image VAE with the same per-channel latents_mean / latents_std normalisation, so+# latent encoding is shared._krea2_encode_latents = _qwen_encode_latents_krea2_encode_latent_stats = _qwen_encode_latent_stats@@ -971,7 +998,8 @@ def _krea2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, devfrom diffusers import Krea2Pipelinepe, mask = embeds_batch- # [B,16,1,H,W] to [B, (H/2)*(W/2), 64] 2x2 patches. Krea2Pipeline._pack_latents is an instance method (it reads self.patch_size), so the packing is inlined like the reference script.+ # Krea2Pipeline._pack_latents is an instance method (it reads self.patch_size), so the packing is+ # inlined like the reference script.bsz, c, _f, h, w = noisy.shapepacked = noisy.reshape(bsz, c, h // 2, 2, w // 2, 2)packed = packed.permute(0, 2, 4, 1, 3, 5).reshape(bsz, (h // 2) * (w // 2), c * 4)@@ -999,9 +1027,10 @@ def _krea2_save(pipe_cls, out_dir, transformer_lora_layers):)+# Both variants share Flux2Transformer2DModel and the upstream packing/forward conventions,+# differing only in the conditioning stack (dev: Mistral-3-Small; Klein: Qwen3) and size. Latents+# train patchified and batch-norm-normalised, from the posterior MODE (deterministic).# ── FLUX.2 (dev + Klein) ──────────────────────────────────────────────────────-# Both variants share Flux2Transformer2DModel and the upstream DreamBooth packing/forward conventions, differing only in the conditioning-# stack (dev: Mistral-3-Small; Klein: Qwen3) and size. Latents train patchified and batch-norm-normalised, from the posterior MODE (deterministic)._FLUX2_COMMON_TARGETS = (# Double-stream blocks: separate q/k/v plus the ModuleList out proj."to_k",@@ -1011,17 +1040,16 @@ _FLUX2_COMMON_TARGETS = (# Single-stream blocks use one fused qkv+mlp input projection."to_qkv_mlp_proj",)-# Their output projection is a plain Linear called ``to_out``. A bare suffix also matches the-# double-stream ModuleList, which PEFT cannot wrap, so the upstream trainers name each single block-# explicitly: 24 possible blocks for Klein and 48 for dev. Missing high indexes are harmless on the-# smaller Klein-4B layout, which has 20 single blocks.+# A bare "to_out" suffix also matches the double-stream ModuleList, which PEFT cannot wrap, so each+# single block is named explicitly; missing high indexes are harmless on the Klein-4B layout._FLUX2_KLEIN_TARGETS = _FLUX2_COMMON_TARGETS + tuple(f"single_transformer_blocks.{i}.attn.to_out" for i in range(24))_FLUX2_DEV_TARGETS = _FLUX2_COMMON_TARGETS + tuple(f"single_transformer_blocks.{i}.attn.to_out" for i in range(48))-# The references train dev with its guidance-distillation vector at 3.5 (Klein applies it only when the variant config carries guidance_embeds).+# The references train dev with its guidance-distillation vector at 3.5 (Klein applies it only when+# the variant config carries guidance_embeds)._FLUX2_TRAIN_GUIDANCE = 3.5@@ -1047,7 +1075,8 @@ def _flux2_encode_prompts(pipe, captions, device):out = []with torch.no_grad():for cap in captions:- # encode_prompt pads to max_sequence_length, so embeds are fixed-length and text_ids are per-caption [1, txt_len, 4]. The per-class text_encoder_out_layers defaults are left to the pipeline.+ # encode_prompt pads to max_sequence_length, so embeds are fixed-length and text_ids are per-caption+ # [1, txt_len, 4]. The per-class text_encoder_out_layers defaults are left to the pipeline.pe, text_ids = pipe.encode_prompt(prompt = cap,device = device,@@ -1072,7 +1101,8 @@ def _flux2_encode_latents(vae, pixel_values):def _flux2_encode_latent_stats(vae, pixel_values):- # FLUX.2 trains from the posterior mode (deterministic): the cached entry is the final latent itself; B is None and the loop skips the per-step sampling draw.+ # FLUX.2 trains from the posterior mode (deterministic): the cached entry is the final latent+ # itself; B is None and the loop skips the per-step sampling draw.return _flux2_encode_latents(vae, pixel_values), None@@ -1084,13 +1114,14 @@ def _flux2_collate():import torch- # Fixed-length embeds give a plain concat; text_ids are PER-SAMPLE (unlike FLUX.1's shared grid), so they concat too.+ # Fixed-length embeds give a plain concat; text_ids are PER-SAMPLE (unlike FLUX.1's shared grid), so+ # they concat too.pe = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)text_ids = torch.cat([e[1] for e in entries]).to(device = device, dtype = torch.float32)return (pe, text_ids)-# Step-invariant FLUX.2 position ids, keyed on (batch, patched h, patched w, device): the ids derive only from the latent shape, so rebuilding them every step is allocator churn.+# The ids derive only from the latent shape, so rebuilding them every step is allocator churn._FLUX2_STATIC: dict[tuple, Any] = {}@@ -1109,7 +1140,8 @@ def _flux2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, devfrom diffusers import Flux2Pipelinepe, text_ids = embeds_batch- # ``noisy`` is already in the patchified normalised space, so packing is the [B,C,H,W] to [B, H*W, C] flatten the pipeline uses.+ # ``noisy`` is already in the patchified normalised space, so packing is the [B,C,H,W] to [B, H*W, C]+ # flatten the pipeline uses.packed = Flux2Pipeline._pack_latents(noisy)img_ids = _flux2_static_img_ids(noisy, device)guidance = None@@ -1127,7 +1159,8 @@ def _flux2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, devreturn_dict = False,)[0]pred = pred[:, : packed.size(1)]- # _unpack_latents_with_ids scatters per sample; diffusers 0.39 stacks them itself (its list annotation is stale), other builds hand back the list. Same-resolution batches align either way.+ # _unpack_latents_with_ids scatters per sample; diffusers 0.39 stacks them itself (its list+ # annotation is stale), other builds hand back the list. Same-resolution batches align either way.unpacked = Flux2Pipeline._unpack_latents_with_ids(pred, img_ids)if isinstance(unpacked, (list, tuple)):unpacked = torch.stack(unpacked)@@ -1152,39 +1185,16 @@ def _flux2_klein_save(pipe_cls, out_dir, transformer_lora_layers):)+# Milestone one trains from STILL IMAGES: a 1-frame clip is valid LTX-2 input (its VAE compresses+# time by 8) and style LoRAs converge on 20-50 stills, so this spec reuses the image dataset layer.+# AUDIOVISUAL: forward REQUIRES audio arguments and diffusers has no video-only escape hatch, so+# feed a one-token audio stream with isolate_modalities=True and keep audio out of the loss and+# out of the LoRA targets. The reason _LTX2_TARGETS names its modules in full.+# Two-stage conditioning: the Gemma3-12B hidden states are ~370 MB per caption, so encode_prompts+# runs both stages and caches only the small connector output.+# Video-stream attention only, fully qualified: a bare "to_q" would also match+# audio_attn1/audio_attn2 and the cross-modality attentions.# ── LTX-2 (video) ─────────────────────────────────────────────────────────────-# The first VIDEO family. Milestone one trains from STILL IMAGES: a 1-frame clip is a valid-# LTX-2 input (its VAE compresses time by 8, so (1 - 1) // 8 + 1 = 1 latent frame), and the-# community consensus is that style / character LoRAs converge on 20-50 stills while only-# motion LoRAs need coherent clips. So this spec reuses the shared image dataset layer-# verbatim and never decodes a video; clip datasets are a follow-up.-#-# Two things make LTX-2 unlike the image DiTs:-#-# 1. It is AUDIOVISUAL. Every block carries a second, audio-side stream (audio_attn1 /-# audio_attn2 / audio_ff) plus two cross-modality attentions (audio_to_video_attn,-# video_to_audio_attn), and ``forward`` takes ``audio_hidden_states`` /-# ``audio_encoder_hidden_states`` as REQUIRED arguments. Lightricks' own trainer handles a-# video-only run by passing ``audio=None``, which their transformer short-circuits so the-# audio and cross-modality branches never execute. diffusers has no such escape hatch, so-# the equivalent here is: feed a minimal audio token stream, pass-# ``isolate_modalities=True`` (which turns off the a2v/v2a cross attention for every-# block, so the audio stream cannot influence the video prediction), keep audio out of the-# loss, and keep the audio-side modules out of the LoRA targets. The audio stream is one-# token for a still, so the wasted compute is negligible. This is a deliberate divergence-# from upstream and is the reason ``_LTX2_TARGETS`` names its modules in full.-#-# 2. Conditioning is a TWO-STAGE stack: a Gemma3-12B encoder whose hidden states are-# [batch, 1024, 3840 * num_layers] (~370 MB per caption in bf16), followed by a small-# ``connectors`` module that reduces them to the [batch, 1024, 3840] the transformer-# actually consumes. Caching the raw encoder output would be ~50x larger than caching the-# connector output for no benefit, so ``encode_prompts`` runs BOTH stages and caches only-# the connector result, then the loop frees the encoder and the connectors together.-#-# Targets: the video-stream attention projections ONLY. Fully qualified on purpose -- a bare-# "to_q" would also match audio_attn1/audio_attn2/audio_to_video_attn/video_to_audio_attn.-# This is exactly the split Lightricks document for a video-only run and ship in their-# video_inpainting / video_outpainting LoRA configs._LTX2_TARGETS = ("attn1.to_q","attn1.to_k",@@ -1196,12 +1206,8 @@ _LTX2_TARGETS = ("attn2.to_out.0",)-# Frame rate the still is placed at on the temporal RoPE axis. LTX-2's rotary temporal-# coordinate is measured in SECONDS (pixel-frame index / fps), so a 1-frame clip at the-# family's native 24 fps lands exactly where the first latent frame of a generated 24 fps-# clip lands -- the position every rendered video contains. Lightricks' preprocessor instead-# pins images to fps = 1.0, which places them at a temporal coordinate no 24 fps clip ever-# visits; we take the inference-matching choice and note the divergence.+# LTX-2's rotary temporal coordinate is in SECONDS, so a 1-frame clip at the native 24 fps lands+# where a generated clip's first latent frame lands; Lightricks instead pin images to fps=1.0._LTX2_TRAIN_FPS = 24.0@@ -1209,8 +1215,7 @@ def _ltx2_load_conditioners(cfg, device, weight_dtype):from diffusers import LTX2Pipelinepipe, vae = _load_pipe_without_transformer(LTX2Pipeline, cfg, device)- # The connectors are the second half of the conditioning stack and are NOT a text_encoder- # attribute, so ``_encoders_to_device`` never reaches them; place them explicitly.+ # The connectors are not a text_encoder attribute, so _encoders_to_device never reaches them.if getattr(pipe, "connectors", None) is not None:pipe.connectors.to(device)return pipe, vae@@ -1225,8 +1230,8 @@ def _ltx2_encode_prompts(pipe, captions, device):import torch_encoders_to_device(pipe, device)- # The Gemma3 hidden states are per-LAYER stacked ([1, 1024, 3840 * layers]); only the- # connector output reaches the transformer, so that is what gets cached.+ # The Gemma3 hidden states are per-LAYER stacked ([1, 1024, 3840 * layers]); only the connector output+ # reaches the transformer, so that is what gets cached.out = []with torch.no_grad():for cap in captions:@@ -1237,12 +1242,8 @@ def _ltx2_encode_prompts(pipe, captions, device):max_sequence_length = 1024,device = device,)- # Read AFTER encode_prompt, exactly where the pipeline reads it. encode_prompt- # sets ``tokenizer.padding_side = "left"`` itself (Gemma wants left padding for- # chat-style prompts), so a tokenizer that loaded reporting "right" would have had- # that stale value baked in here -- and the connectors build the valid-token mask- # from this, so every caption shorter than the 1024 pad length would be masked on- # the wrong end and the cached conditioning would not match what inference builds.+ # Read AFTER encode_prompt, which sets padding_side="left" itself: the connectors build the valid-+ # token mask from this, so a stale "right" would mask every short caption on the wrong end.padding_side = getattr(getattr(pipe, "tokenizer", None), "padding_side", "left")video_emb, audio_emb, conn_mask = pipe.connectors(pe, mask, padding_side = padding_side)out.append((video_emb.cpu(), audio_emb.cpu(), conn_mask.cpu()))@@ -1261,8 +1262,8 @@ def _ltx2_latent_affine(vae, ref):def _ltx2_encode_latents(vae, pixel_values):import torch- # A still is a 1-frame clip: [B,3,H,W] -> [B,3,1,H,W]. The VAE compresses 32x spatially- # and 8x temporally, so a 512px still becomes [B,128,1,16,16].+ # A still is a 1-frame clip: [B,3,H,W] -> [B,3,1,H,W]. The VAE compresses 32x spatially and 8x+ # temporally, so a 512px still becomes [B,128,1,16,16].px = pixel_values.to(torch.float32).unsqueeze(2)with torch.no_grad():lat = vae.encode(px).latent_dist.sample()@@ -1289,7 +1290,6 @@ def _ltx2_collate(import torch# encode_prompt pads to max_sequence_length, so every connector embed is [1, 1024, 3840]- # and a plain concat batches them; ``pad_to`` is moot.video = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)audio = torch.cat([e[1] for e in entries]).to(device = device, dtype = weight_dtype)mask = torch.cat([e[2] for e in entries]).to(device)@@ -1358,9 +1358,8 @@ def _ltx2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, deviaudio_hidden_states = audio_noisy,encoder_hidden_states = video_emb,audio_encoder_hidden_states = audio_emb,- # LTX-2 conditions on the UNSCALED timestep (its config carries- # timestep_scale_multiplier = 1000 and the pipeline passes scheduler timesteps- # through as-is), unlike the FLUX / Qwen families' timestep / 1000.+ # LTX-2 conditions on the UNSCALED timestep (its config carries timestep_scale_multiplier = 1000+ # and the pipeline passes scheduler timesteps through as-is), unlike FLUX / Qwen's timestep / 1000.timestep = timesteps,sigma = timesteps,encoder_attention_mask = mask,@@ -1370,8 +1369,8 @@ def _ltx2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, deviwidth = w,fps = _LTX2_TRAIN_FPS,audio_num_frames = audio_len,- # Disable the audio-to-video / video-to-audio cross attention so the placeholder- # audio stream cannot perturb the video prediction the LoRA is regressing.+ # Disable the a2v / v2a cross attention so the placeholder audio stream cannot perturb the video+ # prediction the LoRA is regressing.isolate_modalities = True,return_dict = False,)@@ -1463,7 +1462,8 @@ _SPECS: dict[str, _FamilySpec] = {family = "flux.2-dev",lora_targets = _FLUX2_DEV_TARGETS,force_bf16 = True,- # 32B DiT; the Mistral conditioning stack (~46 GB bf16) is loaded, encoded and freed BEFORE this lands on the device (the shared phased load).+ # 32B DiT; the Mistral conditioning stack (~46 GB bf16) is loaded, encoded and freed BEFORE this lands+ # on the device (the shared phased load).dense_bf16_gb = 64.5,load_conditioners = _flux2_load_conditioners,load_transformer = _flux2_load_transformer,@@ -1478,9 +1478,8 @@ _SPECS: dict[str, _FamilySpec] = {family = "ltx-2",lora_targets = _LTX2_TARGETS,force_bf16 = True,- # 19B audiovisual DiT; the transformer index reports 37.76 GB bf16. The Gemma3-12B- # conditioning stack (~24 GB resident) is loaded, encoded and freed BEFORE this lands- # on the device, via the shared phased load.+ # 19B audiovisual DiT (37.76 GB bf16); the Gemma3-12B conditioning stack is loaded, encoded and+ # freed BEFORE this lands on the device, via the shared phased load.dense_bf16_gb = 37.8,load_conditioners = _ltx2_load_conditioners,load_transformer = _ltx2_load_transformer,@@ -1494,7 +1493,8 @@ _SPECS: dict[str, _FamilySpec] = {}-# HF repos gating access behind a license acceptance: training needs a token whose account accepted it. Checked by name (no network) so a missing token fails fast with an actionable message.+# HF repos gating access behind a license acceptance: training needs a token whose account accepted+# it. Checked by name (no network) so a missing token fails fast with an actionable message._GATED_TRAIN_REPOS = frozenset({"black-forest-labs/flux.1-dev", "black-forest-labs/flux.2-dev"})@@ -1503,8 +1503,8 @@ def _assert_gated_access(base_model: str, hf_token: Optional[str]) -> None:from core.inference.diffusion_families import _is_local_pathname = str(base_model or "").strip().lower()- # A local clone named like the vendor repo is weights on disk, not a Hub fetch: no gate- # applies, and refusing it by name alone is what made that documented layout untrainable.+ # A local clone named like the vendor repo is weights on disk, not a Hub fetch: refusing it by name+ # alone is what made that documented layout untrainable.if _is_local_path(base_model):returnif name in _GATED_TRAIN_REPOS and not (hf_token and str(hf_token).strip()):@@ -1637,8 +1637,8 @@ def _build_latent_cache(passa, b = _hold(a), _hold(b)if not forced and not gated:- # Size-gate the automatic cache off the first REAL encoded variant: packed 16-channel DiT latents x variants x images- # can exhaust pinned RAM. Over budget we bail with the VAE resident. ``b`` is None for a deterministic-latent family.+ # Size-gate the automatic cache off the first REAL encoded variant: packed latents x variants x+ # images can exhaust pinned RAM.per_variant = a.numel() * a.element_size()if b is not None:per_variant += b.numel() * b.element_size()@@ -1769,13 +1769,14 @@ def _should_compile(mode = (cfg.compile_transformer or "auto").strip().lower()if device != "cuda" or mode == "off":return False- # torch.compile cannot trace the torchao int8 subclass in training (inductor rejects the aliased subclass graph outputs), so int8 always runs eager.+ # torch.compile cannot trace the torchao int8 subclass in training (inductor rejects the aliased+ # subclass graph outputs), so int8 always runs eager.if base_precision == "int8":return Falseif mode == "on":return True- # auto: regional compile is the whole point of the dense modes (2.6x measured on Z-Image bf16) but is fragile over- # bitsandbytes 4-bit modules, so it stays off for QLoRA. fp8/mxfp8 are only competitive compiled.+ # Regional compile is the whole point of the dense modes (2.6x measured on Z-Image bf16) but is+ # fragile over bitsandbytes 4-bit modules, so it stays off for QLoRA.return base_precision in ("bf16", "fp8", "mxfp8")@@ -1813,14 +1814,15 @@ def _maybe_compile_transformer(try:dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)if dynamo_cfg is not None:- # Heterogeneous-block DiTs (Z-Image: ~11 distinct block shapes) exceed dynamo's default recompile limit of 8; bump it like the inference speed layer does.+ # Heterogeneous-block DiTs (Z-Image: ~11 distinct block shapes) exceed dynamo's default recompile+ # limit of 8; bump it like the inference speed layer does.for attr in ("recompile_limit", "cache_size_limit"):if hasattr(dynamo_cfg, attr):setattr(dynamo_cfg, attr, max(getattr(dynamo_cfg, attr) or 0, 64))if hasattr(dynamo_cfg, "suppress_errors"):dynamo_cfg.suppress_errors = True- # dynamic=True matches the inference speed layer's proven default: on torch 2.10 / B200 the dynamic=False specialisation- # failed with CUBLAS_STATUS_EXECUTION_FAILED. fullgraph only on a dense base: bnb 4-bit layers graph-break by design.+ # dynamic=True matches the inference speed layer: on torch 2.10 / B200 the dynamic=False+ # specialisation failed with CUBLAS_STATUS_EXECUTION_FAILED.fn(fullgraph = not base_is_bnb, dynamic = True)return Trueexcept Exception as exc: # noqa: BLE001 -- optimisation only, never fatal@@ -1845,7 +1847,8 @@ def run_dit_lora_training(if spec is None:raise ValueError(f"No DiT trainer for family {cfg.resolved_family!r}")- # DiT families train in bf16, so an explicit fp16 request is refused, not silently upgraded. Validated before the heavy imports so a host without diffusers still sees the real error.+ # DiT families train in bf16, so an explicit fp16 request is refused, not silently upgraded.+ # Validated before the heavy imports so a host without diffusers still sees the real error.if cfg.mixed_precision == "fp16" and spec.force_bf16:raise ValueError(f"{spec.family} LoRA training requires bf16: fp16 overflows its fp32 RoPE / "@@ -1872,8 +1875,8 @@ def run_dit_lora_training(return Truedevice = "cuda" if torch.cuda.is_available() else "cpu"- # The flow-matching + 4-bit path is bf16 throughout (fp32 on a CPU-only box, to keep import/unit tests architecture-agnostic).- # Fail fast on pre-Ampere CUDA, gating on NATIVE bf16 (capability major >= 8), since is_bf16_supported() counts emulation.+ # Fail fast on pre-Ampere CUDA, gating on NATIVE bf16 (capability major >= 8), since+ # is_bf16_supported() counts emulation.if device == "cuda" and not native_bf16_supported():raise ValueError("This trainer requires a bfloat16-capable GPU (Ampere or newer); "@@ -1882,19 +1885,17 @@ def run_dit_lora_training(weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32_assert_trusted_base_model(cfg.base_model)- # The repo this run will FETCH, which is what the start route preflights. Checking the- # canonical id instead would raise here for a gated base that normalization already- # redirected to its ungated mirror -- after the route had answered 200 and freed the- # resident models, so the request fails as a dead job rather than as a fast 400.+ # Check the repo this run will FETCH: the canonical id would raise for a gated base already+ # redirected to its ungated mirror, after the route answered 200 and freed the residents._assert_gated_access(cfg.fetch_base_model or cfg.base_model, cfg.hf_token)pairs = discover_image_caption_pairs(cfg.data_dir, instance_prompt = cfg.instance_prompt, caption_column = cfg.caption_column)- # Resolve num_epochs into a concrete train_steps now the dataset size is known, and rebind cfg so every downstream read agrees.+ # Resolve num_epochs into a concrete train_steps now the dataset size is known, and rebind cfg so every+ # downstream read agrees.cfg = replace(cfg, train_steps = resolve_train_steps(cfg, len(pairs)), num_epochs = 0)- # Validate a resume request against this run's identity BEFORE the multi-GB phased load, so- # a mismatched checkpoint fails in seconds. The identity uses the RESOLVED LoRA targets,- # which for a DiT family are the spec's, not the generic default the config carries.+ # Validate a resume against this run's identity BEFORE the multi-GB phased load, using the RESOLVED+ # LoRA targets rather than the generic default the config carries.identity = identity_for_config(cfg,dataset_pairs = pairs,@@ -1914,14 +1915,14 @@ def run_dit_lora_training(lora_path = None,stopped = True,steps_run = 0,- # Same disposition the full path reports. A stop with save=false is a DISCARD- # however early it lands, and without it the resume fallback offers the source- # bundle back as though the attempt were still live.+ # A stop with save=false is a DISCARD however early it lands; without it the resume fallback offers+ # the source bundle back as though the attempt were still live.discarded = not save_on_stop,)return str(out_dir)- # TF32 / cudnn.benchmark for the run, restored on the way out (the trainer subprocess is disposable, but restoring keeps in-process callers clean).+ # TF32 / cudnn.benchmark for the run, restored on the way out (the trainer subprocess is disposable, but+ # restoring keeps in-process callers clean).perf_snap = _apply_perf_flags(cfg, device)try:return _train_dit(@@ -1953,9 +1954,8 @@ def _train_dit(from peft import LoraConfigfrom peft.utils import get_peft_model_state_dict- # Same as the SDXL trainer: diffusers is only in sys.modules from here, and it- # honours no env var, so its pipeline-loading bars can only be turned off now --- # before the conditioning cache below loads the VAE and text encoders.+ # diffusers honours no env var and is only in sys.modules from here, so its pipeline-loading bars+ # can only be turned off now.try:from loggers.config import quiet_third_party_progress_barsquiet_third_party_progress_bars()@@ -1965,15 +1965,16 @@ def _train_dit(use_lora_targets = _select_lora_targets(cfg.lora_target_modules, spec.lora_targets)out_dir = Path(cfg.output_dir).expanduser()# Load from the byte-identical public mirror selected during normalization, while keeping- # cfg.base_model canonical for the adapter sidecar, completion event, and resume identity.- # One fetch config covers the conditioning pipeline, transformer, and scheduler.+ # cfg.base_model canonical for the adapter sidecar, completion event and resume identity.fetch_cfg = replace(cfg, base_model = cfg.fetch_base_model or cfg.base_model)- # Phase 0: the persistent conditioning cache (opt-in via cond_cache_dir). When every planned latent variant AND caption embedding is on disk, the run is "warm": the VAE and text encoders never load.+ # Phase 0: the persistent conditioning cache (opt-in via cond_cache_dir). When every planned latent variant+ # AND caption embedding is on disk, the run is "warm": the VAE and text encoders never load.image_paths = [p for p, _ in pairs]captions = [c for _, c in pairs]uniq = sorted(set(captions))- # CFG dropout swaps a sample's conditioning for the empty prompt, so its embedding must be precomputed alongside the captions (the encoders are freed right after).+ # CFG dropout swaps a sample's conditioning for the empty prompt, so its embedding must be+ # precomputed alongside the captions (the encoders are freed right after).cfg_dropout = float(getattr(cfg, "cfg_dropout", 0.0) or 0.0)to_encode = uniq + ([""] if cfg_dropout > 0 and "" not in uniq else [])use_cache = cfg.cache_latents and os.environ.get(@@ -1982,13 +1983,15 @@ def _train_dit(pcache = Noneif getattr(cfg, "cond_cache_dir", None):try:- # Namespace on the CHECKPOINT, not just the family: the keys carry only caption/image content, so one cache dir reused for two checkpoints would train on the other model's embeddings and latent stats.+ # Namespace on the CHECKPOINT, not just the family: the keys carry only caption/image content, so+ # one cache dir reused for two checkpoints would train on the other model's embeddings.from .diffusion_train_extras import source_revision # noqa: PLC0415namespace = f"{spec.family}_{cfg.base_model}_{source_revision(fetch_cfg.base_model)}"pcache = PersistentConditioningCache(cfg.cond_cache_dir, namespace, cfg.resolution)except Exception as exc: # noqa: BLE001 -- the cache is an optimisation, never fatal_emit(on_event, "warning", message = f"conditioning cache disabled: {exc}")- # The crop/flip variant plan is seed-deterministic, so persistent keys are stable across runs and the warm check can run before anything loads.+ # The crop/flip variant plan is seed-deterministic, so persistent keys are stable across runs and+ # the warm check can run before anything loads.plan = _plan_cache_variants(len(image_paths), cfg.cache_variants, cfg.center_crop, cfg.random_flip, cfg.seed)@@ -1996,6 +1999,8 @@ def _train_dit(pipe = Nonevae = Nonecaption_embeds = None+ # The estimated cache exceeded the host-memory budget; keep the VAE resident and fall through to the in-loop+ # encode path (latent_cache stays None).latent_cache = Noneif pcache is not None and use_cache:caption_embeds, latent_cache = _load_warm_conditioning(@@ -2010,7 +2015,6 @@ def _train_dit(total = len(image_paths),)if caption_embeds is None:- # Phase 1 (cold): conditioning only. The pipeline loads WITHOUT its transformer, so the encoders + VAE never share VRAM with it. Captions are constant, so their embeddings are precomputed once and the encoders freed.latent_cache = Nonepipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)encoded = _encode_prompts_cached(spec, pipe, to_encode, device, pcache)@@ -2020,7 +2024,7 @@ def _train_dit(if device == "cuda":torch.cuda.empty_cache()- # Phase 2: the VAE latent cache, then free the VAE too (the cache keeps the posterior affine parameters, so per-step sampling noise is preserved).+ # The cache keeps the posterior affine parameters, so per-step sampling noise is preserved.if use_cache:latent_cache = _build_latent_cache(spec,@@ -2035,9 +2039,10 @@ def _train_dit(plan = plan,)if latent_cache is LATENT_CACHE_OVER_BUDGET:- # The estimated cache exceeded the host-memory budget; keep the VAE resident and fall through to the in-loop encode path (latent_cache stays None).+ # The estimated cache exceeded the host-memory budget; keep the VAE resident and fall through to the+ # in-loop encode path (latent_cache stays None).latent_cache = None- elif latent_cache is None: # stopped during the cache build; nothing trained yet+ elif latent_cache is None:_emit(on_event,"complete",@@ -2045,9 +2050,7 @@ def _train_dit(lora_path = None,stopped = True,steps_run = 0,- # As above: a discard is a discard however early the stop lands.- # _save_on_stop is the accessor this function is handed; the flag itself- # lives in the caller's scope.+ # A discard is a discard however early the stop lands.discarded = not _save_on_stop(),)return str(out_dir)@@ -2064,12 +2067,12 @@ def _train_dit(# Variant picks use their own stream so the training loop index/noise draws stay seed-deterministic.variant_rng = random.Random(cfg.seed + 1)- # Phase 3: only now load the transformer, in the resolved base precision (nf4 QLoRA by default; bf16 / int8 / fp8 / mxfp8 are the dense speed modes; "auto" picks from free VRAM).+ # Phase 3: only now load the transformer, in the resolved base precision (nf4 QLoRA by default; bf16 / int8+ # / fp8 / mxfp8 are the dense speed modes; "auto" picks from free VRAM).base_precision = _resolve_base_precision(cfg, spec, device)transformer = spec.load_transformer(fetch_cfg, device, weight_dtype, base_precision)base_is_bnb = base_precision == "nf4"- # Freeze the base; attach the trainable LoRA to the transformer.transformer.requires_grad_(False)transformer.add_adapter(LoraConfig(@@ -2081,7 +2084,8 @@ def _train_dit())if cfg.gradient_checkpointing:- # Non-reentrant checkpointing: reentrant recompute of a bnb 4-bit LoRA linear can trip an illegal memory access on the larger FLUX transformer, and it is the recommended mode anyway.+ # Non-reentrant checkpointing: reentrant recompute of a bnb 4-bit LoRA linear can trip an illegal+ # memory access on the larger FLUX transformer, and it is the recommended mode anyway.import functoolsimport torch.utils.checkpoint as _ckpttransformer.enable_gradient_checkpointing(@@ -2090,7 +2094,8 @@ def _train_dit(cast_training_params(transformer, dtype = torch.float32)lora_params = [p for p in transformer.parameters() if p.requires_grad]- # int8 / fp8 / mxfp8 convert the frozen base linears AFTER the LoRA attaches, so the adapter modules are excluded and stay high precision.+ # int8 / fp8 / mxfp8 convert the frozen base linears AFTER the LoRA attaches, so the adapter modules+ # are excluded and stay high precision.if base_precision == "int8":_int8_quantize_base(transformer, cfg.resolved_family)if base_precision == "fp8" and not _apply_fp8_training(transformer, on_event):@@ -2098,7 +2103,8 @@ def _train_dit(if base_precision == "mxfp8" and not _apply_mxfp8_training(transformer, on_event):base_precision = "bf16"- # LoRA EMA (opt-in via cfg.ema_decay): shadows ONLY the trainable adapter params, initialised after the precision conversions so they track the final fp32 objects. Exported as a second adapter under ema/.+ # LoRA EMA (opt-in via cfg.ema_decay): shadows ONLY the trainable adapter params, initialised after+ # the precision conversions so they track the final fp32 objects.ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else Nonecompiled = _maybe_compile_transformer(@@ -2113,7 +2119,7 @@ def _train_dit(scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token)- # Timestep-shift + loss-weighting setup (see _training_sigma_table). getattr defaults keep an un-normalized config on the historical behavior.+ # getattr defaults keep an un-normalized config on the historical behaviour.flow_shift = getattr(cfg, "flow_shift", None)if flow_shift is None:flow_shift = "auto" if spec.family in AUTO_FLOW_SHIFT_FAMILIES else 1.0@@ -2125,7 +2131,8 @@ def _train_dit(if str(getattr(cfg, "weighting_scheme", "none") or "none") == "bell"else None)- # The LR schedule advances once per optimizer update, so warmup/decay count optimizer steps; scaling by the accumulation factor would stretch warmup past the run.+ # The LR schedule advances once per optimizer update, so warmup/decay count optimizer steps; scaling+ # by the accumulation factor would stretch warmup past the run.lr_sched = get_scheduler(cfg.lr_scheduler,optimizer = optimizer,@@ -2134,30 +2141,26 @@ def _train_dit()_emit(on_event, "model_load_completed", compiled = compiled, base_precision = base_precision)- # See the SDXL trainer: the base is on disk now, so its commit can be read and written into- # every bundle this run saves. The identity built before the load says "unresolved" on the- # first run of an uncached repo, and an unresolved revision is not comparable, so a later- # resume could not tell that the repo had moved underneath it.- # The repo actually fetched, matching identity_for_config: the canonical base is not on disk- # at all when the mirror was selected, so reading it here would pin "unresolved" forever.- # with_resolved_revision records which repo the SHA came from, and mismatch_reason only- # compares two revisions that came from the same one.+ # Read the commit now the base is on disk: an identity built before the load says "unresolved",+ # which is not comparable, so a later resume could not tell the repo had moved underneath it.+ # Record the repo actually fetched: the canonical base is not on disk at all when the mirror was+ # selected, and mismatch_reason only compares two revisions that came from the same repo.identity = with_resolved_revision(identity, fetch_cfg.base_model)# See the SDXL trainer: the cache path the loop actually took, not the one requested.identity = with_cache_mode(identity, latent_cache is not None)- # ...and the precision the frozen base ended up in. base_precision is still the request at- # this point ("auto" resolves here, and a failed fp8/mxfp8 conversion has already fallen- # back to bf16), so without this a bundle records a base it was never trained against.+ # base_precision is still the request here ("auto" resolves at load, and a failed fp8/mxfp8+ # conversion has fallen back to bf16), so without this a bundle records a base it never trained on.identity = with_resolved_base_precision(identity, base_precision)transformer.train()n_images = len(image_paths)batch_size = cfg.train_batch_size- # Permutation-cycle index sampler (shared with the SDXL trainer): visits every image once per cycle, so a short run covers the whole dataset. Uses the loop's own rng to stay seed-deterministic.+ # Permutation-cycle index sampler (shared with the SDXL trainer): visits every image once per cycle,+ # so a short run covers the whole dataset.index_sampler = PermutationBatchSampler(n_images, rng)- # Restore a previous run (adapter, optimizer moments, LR position, EMA shadow, sampler cycle,- # RNG streams) before the loop, so it picks up at `resumed + 1`. None for a fresh run.+ # Restore a previous run (adapter, optimizer moments, LR position, EMA shadow, sampler cycle, RNG streams)+ # before the loop, so it picks up at `resumed + 1`. None for a fresh run.rng_streams = {"loop": rng, "variant": variant_rng}restored = restore_resume_state(cfg,@@ -2171,20 +2174,16 @@ def _train_dit(rng_streams = rng_streams,)resumed = restored.step if restored is not None else 0- # Whether this run's source bundle lives in THIS directory. Resuming from- # directory A into a reused output_dir B leaves B's existing bundles as foreign- # as they would be for a fresh run -- so the first save has to clear them, or a- # higher-numbered one survives and a later resume by B picks it over this run.+ # Resuming from directory A into a reused output_dir B leaves B's existing bundles as foreign as+ # for a fresh run, so the first save must clear them or a higher-numbered one outranks this run.resumed_here = bool(resumed) and resumed_into_this_dir(cfg, out_dir)- # The bundles that were already here when this run started, so a discard below- # removes only what this run wrote. A resumed run shares its source's directory.+ # The bundles already here when this run started, so a discard removes only what this run wrote.preexisting_checkpoints = snapshot_checkpoints(out_dir)if restored is not None:was = restored.progress.get("resolved_base_precision")if was and was != base_precision:- # Not fatal: the LoRA tensors and their optimizer moments are precision-independent,- # only the frozen base's numerics change. But base_precision="auto" picks from free- # VRAM, so this can happen silently and is worth saying out loud.+ # Not fatal, since only the frozen base's numerics change, but base_precision="auto" picks from free+ # VRAM, so this can happen silently._emit(on_event,"warning",@@ -2204,9 +2203,9 @@ def _train_dit(done = resumeddef _save_checkpoint(step: int) -> None:- # Outcome is reported by write_resume_checkpoint's own checkpoint_saved /- # checkpoint_failed events, so a run that crashes after a save is still known to be- # resumable (and one whose save failed is still known to be blocked).+ # write_resume_checkpoint reports its own checkpoint_saved / checkpoint_failed events, so a run+ # that crashes after a save is still known to be resumable and one whose save failed is still+ # known to be blocked._written, _error = write_resume_checkpoint(cfg,step = step,@@ -2218,23 +2217,18 @@ def _train_dit(ema = ema,sampler = index_sampler,rng_streams = rng_streams,- # base_precision="auto" resolves from free VRAM at load time, so the identity (which- # records the REQUESTED mode) cannot tell nf4 from bf16 across two "auto" runs.- # Recorded here so a resume can at least report the change.+ # "auto" resolves from free VRAM at load time, so the identity, which records the REQUESTED mode,+ # cannot tell nf4 from bf16 across two "auto" runs.progress = {"running_loss": running_loss, "resolved_base_precision": base_precision},- # NOT discard_existing. A fresh run does own its output dir, but deleting the- # previous run's bundles at the FIRST periodic save spends them before this run- # has produced anything: "stop without saving" then removes only what this run- # wrote, leaves the previous adapter in place, and the run it belonged to is- # unresumable -- cancelling a retrain destroyed the thing being retrained. The- # clear happens on the completion path instead, once the new adapter is saved.+ # NOT discard_existing: deleting the previous run's bundles at the FIRST periodic save spends them+ # before this run produced anything, so cancelling a retrain destroyed the thing being retrained.discard_existing = False,- # And the bundles that were here before this run: a branched resume must not- # prune the higher-numbered checkpoints it did not write.+ # A branched resume must not prune the higher-numbered checkpoints it did not write.preexisting = preexisting_checkpoints,)- # bf16 autocast around the forward + loss, matching the diffusers dreambooth scripts: it reconciles the fp32 LoRA params with the bnb 4-bit base matmuls. Without it the 4-bit backward on FLUX dies.+ # bf16 autocast around the forward + loss, matching the diffusers dreambooth scripts: it reconciles+ # the fp32 LoRA params with the bnb 4-bit base matmuls.autocast = (torch.autocast(device_type = "cuda", dtype = torch.bfloat16)if device == "cuda"@@ -2264,7 +2258,8 @@ def _train_dit(timesteps, t_indices = _sample_timesteps(scheduler, latents.shape[0], device)sigmas = _gather_sigmas(sigma_table, t_indices, device, weight_dtype, latents.ndim)if shift_active:- # The model's timestep conditioning must follow the shifted sigma (timestep = sigma * num_train_timesteps). Gather in fp32 so bf16 rounding never skews it.+ # The model's timestep conditioning must follow the shifted sigma (timestep = sigma *+ # num_train_timesteps). Gather in fp32 so bf16 rounding never skews it.timesteps = (sigma_table[t_indices].to(device = device, dtype = torch.float32).flatten()* num_train_ts@@ -2301,7 +2296,8 @@ def _train_dit(grad_norm = Noneif cfg.max_grad_norm and cfg.max_grad_norm > 0:- # clip_grad_norm_ returns the total PRE-clip norm: the health signal the UI charts (an exploding norm shows even while the clip caps the update).+ # clip_grad_norm_ returns the total PRE-clip norm: the health signal the UI charts (an exploding+ # norm shows even while the clip caps the update).grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))optimizer.step()lr_sched.step()@@ -2311,8 +2307,7 @@ def _train_dit(running_loss += step_lossdone = opt_step + 1now = time.time()- # Rates count only the steps THIS process ran, so a resumed run does not divide by- # steps it never executed.+ # Rates count only the steps THIS process ran, so a resumed run does not divide by steps it never executed.ran_here = done - resumedif ran_here == 1:# Step 1 pays the one-time costs (cudnn autotune, compile warmup), so the reported rate starts after it.@@ -2338,8 +2333,7 @@ def _train_dit(peak_memory_gb = peak_gb or None,)stop_now = _check_stop()- # Periodic resume point. Skipped on the final step (a finished run has nothing left to- # resume) and when stopping, since the stop path writes one at the exact step.+ # Skipped on the final step and when stopping, since the stop path writes one at the exact step.if (not stop_nowand cfg.save_steps@@ -2356,9 +2350,7 @@ def _train_dit(ema_path: Optional[str] = Noneif not (stopped and not _save_on_stop()):out_dir.mkdir(parents = True, exist_ok = True)- # Stopping early is exactly when the run should be resumable, so write the bundle BEFORE- # the adapter export: if that export then fails, the run still comes back. A completed run- # has nothing left to resume.+ # Write the bundle BEFORE the adapter export: if that export fails, the run still comes back.if stopped and done > 0:_save_checkpoint(done)layers = get_peft_model_state_dict(transformer)@@ -2374,23 +2366,17 @@ def _train_dit(except Exception as exc: # noqa: BLE001 -- the primary adapter is already saved_emit(on_event, "warning", message = f"EMA adapter save failed: {exc}")if not stopped:- # Same as the LoRA trainer: a completed run has nothing to resume, and the final- # iteration writes no bundle, so save_steps leaves the run's own checkpoint-400- # behind for a later resume to roll back to. An earlier run's bundles stay.+ # A completed run has nothing to resume and the final iteration writes no bundle, so save_steps+ # would leave this run's own checkpoint behind for a later resume to roll back to.retire_own_checkpoints(out_dir, preexisting_checkpoints, resumed_here = resumed_here)elif not resumed_here:- # Stopped WITH save on a fresh retrain: the stop bundle is a LOWER step than- # the leftovers of the earlier run in this directory, and resume-by-directory- # picks the newest by step -- so those would outrank the partial just saved- # and continue the wrong training. A resumed run keeps what it found.+ # A stop-with-save on a fresh retrain is a LOWER step than the earlier run's leftovers, and resume-+ # by-directory picks the newest by step, so those would outrank the partial just saved.discard_preexisting_checkpoints(out_dir, preexisting_checkpoints)else:- # Discarded: the user asked to throw this run away. Before periodic checkpoints- # existed a discard left nothing behind, because the output directory was only ever- # created inside this save gate. save_steps now writes bundles as the run goes, so- # without this a discard leaves up to save_total_limit copies of the optimizer state- # in a directory the user never got an artifact from -- invisible to every scanner,- # unresumable (the record is marked discarded), and with no delete path in the UI.+ # save_steps writes bundles as the run goes, so without this a discard leaves up to+ # save_total_limit copies of the optimizer state in a directory the user got no artifact from:+ # invisible to every scanner, unresumable, with no delete path in the UI.clear_own_checkpoints(out_dir, preexisting_checkpoints)try:out_dir.rmdir()@@ -2411,7 +2397,6 @@ def _train_dit(wall_seconds = round(time.time() - t_start, 1),resumed_from_step = resumed or None,# "Stop without saving" discards the run, so its own periodic checkpoints must not keep- # offering to continue it.discarded = bool(stopped and not _save_on_stop()),)return str(out_dir)diff --git a/studio/backend/core/training/diffusion_h3_clips.py b/studio/backend/core/training/diffusion_h3_clips.pyindex 7536d6d9ffd..779266e8252 100644--- a/studio/backend/core/training/diffusion_h3_clips.py+++ b/studio/backend/core/training/diffusion_h3_clips.py@@ -24,35 +24,28 @@ from pathlib import Pathfrom typing import Any, Callable, Optionalfrom utils.paths.path_utils import drop_appledouble_metadata+# Checkpoint contracts mirrored from diffusers.modular_pipelines.minimax_h3, which the trainer+# cannot import; asserted against the live components at load, so a move is caught.# ── the model's grid ─────────────────────────────────────────────────────────-#-# Every constant here is a checkpoint contract, mirrored from-# ``diffusers.modular_pipelines.minimax_h3`` (which the trainer cannot import without pulling-# in the whole modular stack, and which a CPU-only test host may not carry at all). The-# trainer asserts them against the live components at load, so a diffusers release that moves-# one is caught rather than silently trained through.H3_FPS = 24-# The video VAE's ``clip_length`` and ``tokens_chunk_size``: 17 pixel frames per chunk, 5 latent-# frames kept per chunk, plus a 2-frame head.+# The video VAE's clip_length and tokens_chunk_size: 17 pixel frames per chunk, 5 latent frames+# kept per chunk, plus a 2-frame head.H3_FRAMES_PER_CHUNK = 17H3_LATENTS_PER_CHUNK = 5H3_SPATIAL_COMPRESSION = 16# The transformer's (t, h, w) patch. Only the spatial half is > 1.H3_PATCH_T, H3_PATCH_H, H3_PATCH_W = 1, 2, 2-# Both axes have to survive the VAE's 16x spatial compression AND still be a whole number of-# 2x2 patch rows wide, so the canvas multiple is their product.+# Both axes have to survive the VAE's 16x spatial compression AND still be a whole number of 2x2+# patch rows, so the canvas multiple is their product.H3_CANVAS_MULTIPLE = H3_SPATIAL_COMPRESSION * H3_PATCH_WH3_AUDIO_SAMPLING_RATE = 32000H3_AUDIO_LATENTS_PER_SECOND = 40H3_AUDIO_CHANNELS = 2-# How much of a clip's audio window may be zero-padded before the clip is refused. 1% of a 5.17s-# window is ~52ms, which covers the few-millisecond tail a container routinely ends short by-# while still refusing a stream that is mostly silence.+# 1% of a 5.17s window is ~52ms, covering the tail a container routinely ends short by while still+# refusing a mostly-silent stream._MAX_AUDIO_PAD_FRACTION = 0.01-# Peak amplitude at or below which a decoded window counts as silent, on PyAV's "flt" scale where-# full scale is 1.0. About -80 dBFS: below the noise floor of any real recording, and above the-# rounding dust an encode/decode round trip can leave on a track that was authored as digital-# silence, so a genuinely muted soundtrack is caught whether or not it decodes to exact zeros.+# About -80 dBFS on PyAV's "flt" scale: below the noise floor of any real recording, above the+# rounding dust an encode/decode round trip leaves on authored digital silence._SILENT_AUDIO_PEAK = 1e-4H3_AUDIO_LATENT_CHANNELS = 32H3_VIDEO_LATENT_CHANNELS = 24@@ -67,18 +60,10 @@ H3_MAX_ASPECT_RATIO = 4H3_PIXEL_MEAN = (0.485, 0.456, 0.406)H3_PIXEL_STD = (0.229, 0.224, 0.225)-# Pixel frames one training clip covers. ONE VAE chunk, i.e. the shortest clip the video VAE-# can encode at all, which is 0.917 s at 24 fps.-#-# This is deliberately far below the 5 s floor MiniMax-H3 *generates* at, and the trade is-# explicit: the packed sequence is quadratic in its own length through the full self-attention,-# so at the released 768-short-edge canvas a 5 s clip is ~38k rows and a 22-frame clip is ~7k.-# Training at the native canvas on short clips keeps the SPATIAL statistics -- which is what a-# style LoRA learns -- exactly on distribution, and only shortens the temporal extent; training-# a 5 s clip at a canvas small enough to fit would put every spatial statistic off distribution-# instead. The temporal rotary grid of a 22-frame clip is a strict PREFIX of the grid of a-# generated one (``_temporal_position_grid`` starts at the same origin with the same spacing),-# so no row sits at a coordinate the model never visits.+# ONE VAE chunk, the shortest clip the video VAE can encode at all, which is 0.917 s at 24 fps.+# Deliberately far below the 5 s floor MiniMax-H3 generates at: the packed sequence is quadratic+# in its own length, and training short clips at the native canvas keeps the SPATIAL statistics on+# distribution. A 22-frame clip's temporal rotary grid is a strict PREFIX of a generated one's.H3_TRAIN_NUM_FRAMES = H3_FRAMES_PER_CHUNK + H3_LATENTS_PER_CHUNK_VIDEO_EXTS = {".mp4", ".mov", ".mkv", ".webm", ".m4v", ".avi"}@@ -300,8 +285,7 @@ def decode_clip()stream = container.streams.video[0]source_fps = float(stream.average_rate or stream.guessed_rate or H3_FPS) or float(H3_FPS)- # Container duration, in seconds, for the over-long note below. Best effort: an unknown- # duration simply means no note, never a failed decode.+ # Best effort: an unknown duration means no note, never a failed decode.source_duration_s = 0.0try:if stream.duration is not None and stream.time_base is not None:@@ -319,8 +303,8 @@ def decode_clip(if int(next_target * source_fps / H3_FPS) > source_index:continueimage = frame.to_image().convert("RGB")- # Before the crop, not after: the canvas is in display orientation, so cropping the- # coded frame would trim the wrong pair of edges as well as train it sideways.+ # Before the crop: the canvas is in display orientation, so cropping the coded frame would trim the+ # wrong edges as well as train it sideways.image = apply_display_rotation(image, display_rotation_degrees(frame, stream), Image)image = _cover_resize(image, width, height, Image)while (@@ -390,8 +374,7 @@ def display_rotation_degrees(frame: Any, stream: Any) -> int:except Exception: # noqa: BLE001 -- a degenerate matrix means "no rotation", not a failurereturn 0theta = int(-round(degrees)) % 360- # Only the four square turns; anything else cannot be applied without resampling, and no- # camera writes one.+ # Only the four square turns; anything else cannot be applied without resampling, and no camera writes one.return theta if theta in (90, 180, 270) else 0@@ -437,11 +420,8 @@ def _decode_clip_audio(path: Any, target_samples: int, av: Any, np: Any) -> Any:chunks = []have = 0with av.open(str(path)) as container:- # Stops at the training window, exactly as the video loop above does. An over-long source- # is accepted input here -- only the first num_frames are trained and the caller is merely- # warned -- so decoding the rest of the soundtrack would spend a whole recording's time- # and memory to build a sub-second sample, and would fail on damage in a region that is- # never used.+ # Stops at the training window: only the first num_frames are trained, so decoding the rest of the+ # soundtrack would spend a whole recording's time and fail on damage in an unused region.for frame in container.decode(audio = 0):for resampled in resampler.resample(frame):block = resampled.to_ndarray().reshape(-1, H3_AUDIO_CHANNELS)@@ -450,9 +430,8 @@ def _decode_clip_audio(path: Any, target_samples: int, av: Any, np: Any) -> Any:if have >= target_samples:breakif have < target_samples:- # Only when the stream ran out: the resampler holds a partial block back, and that- # tail is what the pad allowance below is measured against. After an early break- # there is nothing to flush for -- the window is already full.+ # Only when the stream ran out: the resampler holds a partial block back and that tail is what the+ # pad allowance measures; after an early break the window is already full.for resampled in resampler.resample(None):chunks.append(resampled.to_ndarray().reshape(-1, H3_AUDIO_CHANNELS))if not chunks:@@ -470,11 +449,7 @@ def _decode_clip_audio(path: Any, target_samples: int, av: Any, np: Any) -> Any:f"soundtrack runs its full length.")samples = np.pad(samples, ((0, missing), (0, 0)))- # A muted track runs the clip's full length, so every check above passes and the window comes- # back all zeros -- the same target the short-audio refusal exists to keep out, arriving by a- # route that refusal cannot see. Measured as peak amplitude rather than mean energy so a clip- # that is merely quiet, or silent for most of its length with one real sound in it, is kept:- # only a track with nothing above the floor anywhere is turned away.+ # A muted track passes every check above and comes back all zeros.if float(np.max(np.abs(samples))) <= _SILENT_AUDIO_PEAK:raise ValueError(f"{Path(path).name} has a soundtrack that is silent all the way through. "diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.pyindex d033e25a309..579f230d201 100644--- a/studio/backend/core/training/diffusion_h3_trainer.py+++ b/studio/backend/core/training/diffusion_h3_trainer.py@@ -92,27 +92,15 @@ from core.training.diffusion_train_common import ()from core.training.diffusion_train_extras import LoRAEMA, save_ema_adapter-# The video-stream and audio-stream projections of every block, fully qualified.-#-# There is nothing to exclude: H3 has ONE set of block weights serving all three modalities,-# so this is the whole adapter surface and the audio rows are trained through the same-# matrices as the video rows. What IS excluded is deliberate:-# - ``adaln_proj.linear`` projects the timestep embedding into the modulation table. It is-# 40% of the checkpoint's parameters, but its input is a (num_timesteps, 2688) tensor with-# two or three rows, so a rank-r adapter there sees a handful of samples per step and-# learns noise.-# - ``proj_in`` / ``audio_proj_in`` / ``proj_out`` / ``audio_proj_out`` / ``context_embedder``-# are the patch and text projections, kept in fp32 by the checkpoint's own-# ``_keep_in_fp32_modules``; adapting them mixes precisions for no benefit.-# - the ``token_refiner`` blocks carry ``attn`` and ``ff`` under the SAME leaf names-# (``token_refiner.refiner_blocks.0.attn.to_q``), so PEFT's suffix rule for a LIST of-# target names would adapt the text refiner as well as the denoiser stack.-#-# That last point is why the targets are a REGEX and not a list: PEFT globs nothing, it either-# suffix-matches a list or ``re.fullmatch``es a string, so qualifying a list entry with-# ``transformer_blocks.*.`` matches nothing at all and the adapter silently trains zero-# parameters (the trainer also refuses an empty adapter, so both halves of that mistake are-# caught).+# H3 has ONE set of block weights serving all three modalities, so this is the whole adapter+# surface; the exclusions below are deliberate. adaln_proj.linear is 40% of the checkpoint's+# parameters but its input is a (num_timesteps, 2688) tensor with two or three rows, so a rank-r+# adapter there learns noise. proj_in / audio_proj_in / proj_out / audio_proj_out /+# context_embedder are the patch and text projections, kept in fp32 by the checkpoint's own+# _keep_in_fp32_modules; adapting them mixes precisions for no benefit. The token_refiner blocks+# carry attn and ff under the SAME leaf names (token_refiner.refiner_blocks.0.attn.to_q), which is+# why the targets are a REGEX and not a list: PEFT suffix-matches a list, so it would adapt the+# text refiner as well as the denoiser stack._H3_TARGET_LEAVES = ("attn.to_q","attn.to_k",@@ -127,29 +115,24 @@ _H3_TARGETS = (+ ")")-# Modules whose weights the transformer reads a DTYPE off to align an activation with-# (``x.to(self.linear.weight.dtype)``). A bitsandbytes ``Params4bit`` reports ``uint8``, so-# quantizing one of these casts the activation to Byte and the next norm dies with-# ``"rms_norm" not implemented for 'Byte'``. Keeping them dense is the whole fix; it costs the-# nf4 saving on ``adaln_proj`` (36.6 GB resident instead of ~17 GB), which is still well under-# the 66.3 GB dense weight. torchao int8 is unaffected -- its tensor subclass reports the-# logical bfloat16 -- so ``base_precision="int8"`` quantizes everything.+# The transformer reads a DTYPE off these weights to align an activation+# (x.to(self.linear.weight.dtype)), and a bitsandbytes Params4bit reports uint8, so quantizing one+# casts the activation to Byte and the next norm dies with "rms_norm" not implemented for 'Byte'.+# Costs the nf4 saving on adaln_proj (36.6 GB instead of ~17 GB); torchao int8 is unaffected,+# since its subclass reports bfloat16, so base_precision="int8" quantizes everything._H3_NF4_SKIP_MODULES = ("context_embedder", "adaln_proj", "norm_out")# The exponential sigma shifts of the two schedules, from the released scheduler configs._H3_VIDEO_SHIFT = 12.0_H3_AUDIO_SHIFT = 3.0-# Which Qwen3-VL hidden state conditions the transformer. The last layer is post-norm and is-# not what the released weights were trained against.+# The last Qwen3-VL layer is post-norm and is not what the released weights were trained against._H3_TEXT_ENCODER_LAYER = 50-# Components the conditioning phase needs, and the one the training phase needs. Naming them-# is what keeps the 66 GB transformer off the device while the 63 GiB conditioner is on it.+# Naming the components is what keeps the 66 GB transformer off the device while the 63 GiB conditioner is on it._H3_TEXT_COMPONENTS = ("text_encoder", "tokenizer", "processor")-# The VAEs load in fp32: both carry modules diffusers refuses to cast (their encoders,-# decoders and projection heads), so a bf16 load followed by a .to(float32) both warns and-# leaves the cast half-applied.+# The VAEs load in fp32: both carry modules diffusers refuses to cast, so a bf16 load followed by+# .to(float32) warns and leaves the cast half-applied._H3_VAE_COMPONENTS = ("vae", "audio_vae")_H3_CONDITIONING_COMPONENTS = _H3_TEXT_COMPONENTS + _H3_VAE_COMPONENTS@@ -199,18 +182,14 @@ def _load_conditioners(cfg, device):from core.inference.diffusion import hub_cache_dir- # Pinned, like every loader call on the inference side. diffusers resolves an unset- # cache_dir through huggingface_hub's import-time constant, and the training subprocess is- # spawned without the cache-environment wrapper, so a cache folder changed mid-session was- # invisible here: components already sitting in the selected root were missed and roughly- # 145 GB re-downloaded into the old one.+ # Pin the cache dir: diffusers resolves an unset one through huggingface_hub's import-time+ # constant and this subprocess is spawned without the cache-environment wrapper, so components in+ # the selected root were missed and ~145 GB re-downloaded into the old one.cache_dir = hub_cache_dir()pipe = ModularPipeline.from_pretrained(cfg.base_model, token = cfg.hf_token, cache_dir = cache_dir)- # The token above opens the modular INDEX only. load_components runs a separate- # from_pretrained per component, against the repos that index names, so it has to carry the- # token again -- and it swallows a component failure as a logger.warning rather than raising,- # so an anonymous 401 leaves the attribute unset and the first use dies on None instead of- # saying the base was gated. The inference H3 loader forwards it for the same reason.+ # load_components runs a separate from_pretrained per component and swallows a failure as a+ # warning, so without the token an anonymous 401 leaves the attribute unset and the first use dies+ # on None instead of naming the gate.auth = {"token": cfg.hf_token} if cfg.hf_token else {}pipe.load_components(names = list(_H3_TEXT_COMPONENTS),@@ -306,7 +285,7 @@ def _encode_audio_latents(audio_vae, waveform, device) -> Any:cached for it."""import torch- samples = torch.from_numpy(waveform).to(device).unsqueeze(1) # (2, 1, samples)+ samples = torch.from_numpy(waveform).to(device).unsqueeze(1)with torch.no_grad():posterior = audio_vae.encode(samples, return_dict = False)[0]latents = posterior.mode()@@ -322,8 +301,8 @@ def _load_transformer(cfg, device, base_precision):from core.inference.diffusion import hub_cache_dir- # Same pin as the conditioners above: the denoiser is the 145 GB half, so an unpinned load- # here is the expensive one to get wrong.+ # Same pin as the conditioners: the denoiser is the 145 GB half, so an unpinned load here is the+ # expensive one to get wrong.cache_dir = hub_cache_dir()if base_precision == "nf4":from diffusers import BitsAndBytesConfig as DiffusersBnb@@ -421,8 +400,8 @@ def _row_timesteps(layout, num_text_tokens: int, t_video: float, t_audio: float,num_text_tokens,t_video,t_audio,- # No conditioning rows exist in this layout, so the two condition timesteps are- # unreachable; pass the generated ones so no phantom value enters ``torch.unique``.+ # No conditioning rows exist in this layout, so pass the generated timesteps and keep a phantom+ # value out of torch.unique.t_video,t_audio,)@@ -476,23 +455,14 @@ def run_h3_lora_training(cfg = config.normalized()if cfg.resolved_family != "minimax-h3":raise ValueError(f"This trainer is for minimax-h3, not {cfg.resolved_family!r}.")- # Every config-only refusal lives in the shared preflight, which the START ROUTE also calls- # BEFORE it evicts the user's resident GPU models. Raised again here so a direct call to the- # trainer is refused too, and so the two can never disagree about what H3 accepts.+ # Every config-only refusal also lives in the shared preflight the START ROUTE calls before+ # evicting resident GPU models; raised again here so a direct trainer call is refused too.reason = h3_train_unsupported_reason(cfg)if reason:raise ValueError(reason)- # The two image-LoRA augmentation knobs are fixed for a clip dataset, not honoured: every- # frame goes through the same centre cover-crop (_cover_resize), and nothing is flipped --- # a per-frame flip would tear a clip, and a per-clip one has nowhere to live, since the- # cached tensors carry no variant axis. snr_gamma is the same shape of mismatch on the loss- # side: the step is a plain unweighted MSE and the field defaults to 5.0, so every default- # request recorded min-SNR weighting that never ran (weighting_scheme, its sibling, is- # refused above because it has no default to break). All three SCHEMA defaults disagree with- # what the loop does, so they are normalised rather than refused: refusing would 422 every- # default request, and leaving them would have the run record describe a recipe that did not- # happen. Read off the shared table, which the SERVICE also applies to the config it persists- # -- normalising only here fixed the run and left the history entry lying about it.+ # The augmentation and snr_gamma defaults do not match what the clip loop does (one centre+ # cover-crop, no flips, plain unweighted MSE), so they are normalised rather than refused, which+ # would 422 every default request. Read off the shared table the SERVICE also applies.cfg = replace(cfg, **train_recipe_overrides(cfg))import torch@@ -521,11 +491,9 @@ def run_h3_lora_training()weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32- # allow_modular: this loop loads through ModularPipeline.from_pretrained, and a local- # MiniMax-H3 pipeline carries modular_model_index.json and no model_index.json, so the- # conventional shape check refused the only local layout the family has. Read off the shared- # set rather than hard-coded True, because the start route runs this same gate first and has- # to reach the same verdict about the same directory.+ # allow_modular: this loop loads through ModularPipeline.from_pretrained, and a local MiniMax-H3+ # pipeline carries modular_model_index.json and no model_index.json, so the conventional shape+ # check refused the only local layout the family has._assert_trusted_base_model(cfg.base_model,allow_modular = (cfg.resolved_family or "").strip().lower() in MODULAR_BASE_FAMILIES,@@ -570,8 +538,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savto_encode = sorted(set(captions))+ # Phase 1: the 63 GiB Qwen3-VL conditioner and both VAEs are resident here and nowhere else.# ── phase 1: conditioning. The 63 GiB Qwen3-VL conditioner and both VAEs are resident- # here and nowhere else; the 66 GB transformer has not been fetched yet.pipe = _load_conditioners(cfg, device)caption_embeds = {cap: _encode_prompt(pipe, cap, device) for cap in to_encode}_emit(on_event, "preparing", stage = "encode_prompts", done = len(to_encode), total = len(to_encode))@@ -582,9 +550,9 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savif device == "cuda":torch.cuda.empty_cache()+ # One canvas for the run, from the FIRST clip's aspect ratio: every other clip is cover-cropped+ # onto it, so a mixed-aspect dataset trains on one geometry.# ── phase 2: the clip cache. One canvas for the run, taken from the FIRST clip's aspect- # ratio: every other clip is cover-cropped onto it, so a mixed-aspect dataset trains on one- # geometry rather than reshaping the packed sequence per step.width, height = _dataset_canvas(clip_paths[0], cfg.resolution)latent_h, latent_w = height // H3_SPATIAL_COMPRESSION, width // H3_SPATIAL_COMPRESSIONcache: list[tuple[Any, Any, Any]] = []@@ -594,9 +562,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savnum_frames = num_frames,width = width,height = height,- # The window is the clip's opening and the latents are cached once for the run, so- # a longer source trains only its first seconds while its caption describes the- # whole thing. Reported rather than left silent.+ # The window is the clip's opening and the latents are cached once, so a longer source trains only+ # its first seconds while its caption describes the whole thing.on_note = lambda message: _emit(on_event, "warning", message = message),)video_a, video_b = _encode_video_stats(pipe.vae, frames, device)@@ -608,12 +575,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav)entry = (video_a, video_b, audio)if i == 0 and not _latent_cache_forced():- # Size-gate off the FIRST real entry, the way the SDXL and DiT caches do, before the- # rest of the dataset is encoded. Those two answer an over-budget estimate by dropping- # the cache and encoding per step; H3 cannot, because both VAEs are freed below to- # make room for the 66 GB transformer. So the honest answer is to say so now, with the- # numbers, instead of running the whole preparation and being OOM-killed at the end of- # it with nothing to show.+ # H3 cannot answer an over-budget estimate by encoding per step, because both VAEs are freed to+ # make room for the 66 GB transformer, so say so now with the numbers instead of being OOM-killed.from core.training import diffusion_train_common as _train_commonper_clip = int(sum(t.numel() * t.element_size() for t in entry))if _latent_cache_over_budget(per_clip, len(clip_paths)):@@ -652,9 +615,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav)transformer = _load_transformer(cfg, device, base_precision)transformer.requires_grad_(False)- # ``normalized()`` fills lora_target_modules with the generic DEFAULT_LORA_TARGETS when a- # caller does not set it, so that value means "unset" and the family's own regex wins. Any- # other explicit tuple is a deliberate override and is passed through as a list.+ # normalized() fills lora_target_modules with the generic DEFAULT_LORA_TARGETS when unset, so that+ # value means "unset" and the family's own regex wins.targets: Any = (_H3_TARGETSif tuple(cfg.lora_target_modules) == DEFAULT_LORA_TARGETS@@ -683,10 +645,10 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savf"would train nothing.")if base_precision == "int8":- # WITH the family. H3's adaln_proj is Linear(2688 -> 96768), so it clears the 512-feature- # floor and gets quantized, then runs at M = 1 and raises "self.size(0) needs to be- # greater than 16" on the first step -- after the whole 66.3 GB base has loaded. The- # family also carries the pad list (context_embedder, token_refiner) the helper applies.+ # WITH the family: H3's adaln_proj is Linear(2688 -> 96768), so it clears the 512-feature floor+ # and gets quantized, then runs at M = 1 and raises "self.size(0) needs to be greater than 16"+ # after the 66.3 GB base has loaded. The family also carries the pad list (context_embedder,+ # token_refiner) the helper applies._int8_quantize_base(transformer, cfg.resolved_family)ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None@@ -780,9 +742,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav)loss_video = F.mse_loss(pred_video[0].float(), target_video.float())loss_audio = F.mse_loss(pred_audio[0].float(), target_audio.float())- # Unweighted sum, the model's own objective: the two streams share every- # matrix the adapter touches, so down-weighting audio would not protect it, it- # would only stop the loss reporting that it had drifted.+ # Unweighted sum, the model's own objective: both streams share every matrix the adapter touches, so+ # down-weighting audio would only stop the loss reporting that it had drifted.loss = loss_video + loss_audio(loss / cfg.gradient_accumulation_steps).backward()step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps@@ -832,8 +793,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savema_path: Optional[str] = Noneif not (stopped and not _save_on_stop()):layers = get_peft_model_state_dict(transformer)- # The trained config, so the alpha survives the round trip rather than being re-derived- # as the rank. ``default`` is the adapter name add_adapter used.+ # The trained config, so the alpha survives the round trip rather than being re-derived as the+ # rank. "default" is the adapter name add_adapter used.adapter_config = dict(transformer.peft_config["default"].to_dict())_save_lora(str(out_dir), layers, adapter_config)lora_path = str(out_dir / DEFAULT_LORA_FILENAME)@@ -872,10 +833,9 @@ def _dataset_canvas(clip_path: str, short_edge: int) -> tuple[int, int]:stream = container.streams.video[0]source_w = int(stream.codec_context.width)source_h = int(stream.codec_context.height)- # The coded size is not the displayed size for a rotated clip, and decode_clip rotates- # its frames, so a portrait phone clip would otherwise get a landscape canvas and be- # cover-cropped down to it. One frame is decoded because the matrix travels with the- # frame; the container is open either way.+ # The coded size is not the displayed size for a rotated clip, so a portrait phone clip would get+ # a landscape canvas and be cropped down to it; one frame is decoded because the matrix travels+ # with the frame.theta = 0try:frame = next(container.decode(video = 0), None)diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.pyindex 7be384426cb..44b8e0f5f74 100644--- a/studio/backend/core/training/diffusion_lora_trainer.py+++ b/studio/backend/core/training/diffusion_lora_trainer.py@@ -42,7 +42,8 @@ from dataclasses import replacefrom pathlib import Pathfrom typing import Any, Optional-# Shared, family-agnostic building blocks. Re-exported so callers/tests that import them from this module keep working.+# Shared, family-agnostic building blocks. Re-exported so callers/tests that import them from this module keep+# working.from core.training.diffusion_train_common import ( # noqa: F401DEFAULT_LORA_FILENAME,DEFAULT_LORA_TARGETS,@@ -121,6 +122,7 @@ def _load_image_tensor(if random_flip and rng.random() < 0.5:img = img.transpose(Image.FLIP_LEFT_RIGHT)# A flip mirrors the crop's left origin, so report the mirrored offset (as diffusers does).+ # Mirror the crop's left origin (same as _load_image_tensor's random flip).crop_left = max(0, resized_w - resolution - left)arr = np.asarray(img, dtype = np.float32) / 255.0tensor = torch.from_numpy(arr).permute(2, 0, 1) * 2.0 - 1.0@@ -156,7 +158,6 @@ def _load_image_tensor_planned(crop_left = leftif flip:img = img.transpose(Image.FLIP_LEFT_RIGHT)- # Mirror the crop's left origin (same as _load_image_tensor's random flip).crop_left = max(0, resized_w - resolution - left)arr = np.asarray(img, dtype = np.float32) / 255.0tensor = torch.from_numpy(arr).permute(2, 0, 1) * 2.0 - 1.0@@ -235,7 +236,8 @@ def _build_sdxl_latent_cache(a = _hold(dist.mean * vae_scale)b = _hold(dist.std * vae_scale)if not forced and not gated:- # Size-gate the auto cache off the first real variant, before building the rest (it can exhaust pinned RAM). Over budget: bail with the VAE resident.+ # Size-gate the auto cache off the first real variant: it can exhaust pinned RAM. Over budget, bail+ # with the VAE resident.per_variant = a.numel() * a.element_size() + b.numel() * b.element_size()if _latent_cache_over_budget(per_variant, total_variants):_emit(@@ -311,9 +313,8 @@ def run_diffusion_lora_training(from peft import LoraConfigfrom peft.utils import get_peft_model_state_dict- # Only now is diffusers in sys.modules, and it hard-codes _tqdm_active = True and- # honours no env var, so this is the earliest point its "Loading pipeline- # components..." bar can be turned off -- and it comes before the pipeline load below.+ # diffusers hard-codes _tqdm_active = True and honours no env var, so this is the earliest point its+ # loading bar can be turned off, and it precedes the pipeline load below.try:from loggers.config import quiet_third_party_progress_barsquiet_third_party_progress_bars()@@ -341,11 +342,12 @@ def run_diffusion_lora_training(device = "cuda" if torch.cuda.is_available() else "cpu"precision = cfg.mixed_precision if device == "cuda" else "no"if precision == "bf16" and device == "cuda" and not native_bf16_supported():- # Pre-Ampere GPUs have no native bf16 and is_bf16_supported() counts emulation, so use the compute-capability probe and fall back to fp16.+ # Pre-Ampere GPUs have no native bf16 and is_bf16_supported() counts emulation, so use the compute-+ # capability probe and fall back to fp16.precision = "fp16"weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "no": torch.float32}[precision]- # TF32 / cudnn.benchmark for the run, restored on the way out. Wraps the whole body so every return restores the backend flags.+ # Wraps the whole body so every return restores the TF32 / cudnn.benchmark flags.snap = _apply_perf_flags(cfg, device)try:# Preflight the base model against the same trust gate as inference, before any fetch.@@ -354,10 +356,11 @@ def run_diffusion_lora_training(pairs = discover_image_caption_pairs(cfg.data_dir, instance_prompt = cfg.instance_prompt, caption_column = cfg.caption_column)- # Resolve num_epochs into a concrete train_steps now the dataset size is known, and rebind cfg so every downstream read agrees.+ # Resolve num_epochs into a concrete train_steps now the dataset size is known, and rebind cfg so every+ # downstream read agrees.cfg = replace(cfg, train_steps = resolve_train_steps(cfg, len(pairs)), num_epochs = 0)- # Validate a resume request against this run's identity BEFORE the multi-GB base model- # load, so a mismatched checkpoint fails in seconds instead of after the download.+ # Validate a resume request against this run's identity BEFORE the multi-GB base model load, so a+ # mismatch fails in seconds instead of after the download.identity = identity_for_config(cfg, dataset_pairs = pairs)if cfg.resume_from_checkpoint:preflight_resume(@@ -377,9 +380,8 @@ def run_diffusion_lora_training(lora_path = None,stopped = True,steps_run = 0,- # Same disposition the full path reports. A stop with save=false is a DISCARD- # however early it lands, and without it the resume fallback offers the source- # bundle back as though the attempt were still live.+ # A stop with save=false is a DISCARD however early it lands; without it the resume fallback offers+ # the source bundle back as though the attempt were still live.discarded = not save_on_stop,)return str(out_dir)@@ -414,7 +416,8 @@ def run_diffusion_lora_training(if weight_dtype != torch.float32:cast_training_params(unet, dtype = torch.float32)- # Regionally torch.compile the U-Net repeated blocks via the DiT trainer's never-fatal wrapper (failure falls back to eager).+ # Regionally torch.compile the U-Net repeated blocks via the DiT trainer's never-fatal wrapper (failure+ # falls back to eager).from core.training.diffusion_dit_trainer import _maybe_compile_transformercompiled = _maybe_compile_transformer(@@ -423,7 +426,8 @@ def run_diffusion_lora_training(lora_params = [p for p in unet.parameters() if p.requires_grad]optimizer = _make_lora_optimizer(lora_params, cfg.learning_rate)- # The scheduler advances once per optimizer update, so count warmup/decay in optimizer steps; the accumulation factor would stretch warmup.+ # The scheduler advances once per optimizer update, so count warmup/decay in optimizer steps; the+ # accumulation factor would stretch warmup.lr_sched = get_scheduler(cfg.lr_scheduler,optimizer = optimizer,@@ -434,7 +438,8 @@ def run_diffusion_lora_training(vae_scale = vae.config.scaling_factorprediction_type = noise_scheduler.config.prediction_type- # Precompute text embeddings once per unique caption, then free the ~1.5 GB CLIP encoders. Deterministic, so the math is bit-identical to in-loop encoding.+ # Deterministic, so the precomputed embeddings are bit-identical to in-loop encoding; the ~1.5 GB+ # CLIP encoders are freed afterwards.precompute = os.environ.get("UNSLOTH_DIFFUSION_NO_PRECOMPUTE", "") not in ("1", "true")caption_embeds: dict[str, tuple] = {}if precompute:@@ -448,10 +453,11 @@ def run_diffusion_lora_training(if device == "cuda":torch.cuda.empty_cache()- # Precompute the VAE latent cache, then free the VAE: it holds the posterior affine pair so per-step sampling noise is preserved.+ # The cache holds the posterior affine pair, so per-step sampling noise is preserved once the VAE is freed.use_cache = cfg.cache_latents and os.environ.get("UNSLOTH_DIFFUSION_NO_LATENT_CACHE", "") not in ("1", "true")+ # Over the host-memory budget; keep the VAE resident and encode in-loop.latent_cache = Noneif use_cache:latent_cache = _build_sdxl_latent_cache(@@ -465,9 +471,8 @@ def run_diffusion_lora_training(_check_stop,)if latent_cache is LATENT_CACHE_OVER_BUDGET:- # Over the host-memory budget; keep the VAE resident and encode in-loop.latent_cache = None- elif latent_cache is None: # stopped during the cache build; nothing trained yet+ elif latent_cache is None:out_dir = Path(cfg.output_dir).expanduser()_emit(on_event,@@ -476,7 +481,7 @@ def run_diffusion_lora_training(lora_path = None,stopped = True,steps_run = 0,- # As above: a discard is a discard however early the stop lands.+ # A discard is a discard however early the stop lands.discarded = not save_on_stop,)return str(out_dir)@@ -490,31 +495,31 @@ def run_diffusion_lora_training(gc.collect()if device == "cuda":torch.cuda.empty_cache()- # Variant picks use their own stream so the loop index/noise draws stay seed-deterministic whether or not the cache is enabled.+ # Variant picks use their own stream so the loop index/noise draws stay seed-deterministic whether+ # or not the cache is enabled.variant_rng = random.Random(cfg.seed + 1)_emit(on_event, "model_load_completed", compiled = compiled)- # The base is on disk now, so its commit can finally be read: the identity above was- # built before the load and records "unresolved" on the first run of an uncached repo,- # which no later resume could then enforce. Only ever adds the constraint.+ # The base is on disk now, so its commit can finally be read: the identity above was built before+ # the load and records "unresolved" on the first run of an uncached repo.identity = with_resolved_revision(identity, cfg.base_model)- # And the cache path the loop actually took, which the request does not settle: the- # env override and the over-budget fallback both turn it off, and the two paths draw- # crops and flips from different RNG streams.+ # The cache path the loop actually took, which the request does not settle: the env override and+ # the over-budget fallback both turn it off, and the two paths draw crops from different RNGs.identity = with_cache_mode(identity, latent_cache is not None)- # Permutation-cycle index sampler: each image is visited once per cycle before any repeat, so a short run does not leave a small dataset partly unseen.+ # Permutation-cycle index sampler: each image is visited once per cycle before any repeat, so a+ # short run does not leave a small dataset partly unseen.index_sampler = PermutationBatchSampler(len(pairs), rng)def _next_batch() -> tuple[list[int], list[str], list[str]]:- # Draw the full configured batch, not min(batch, n): the sampler refills across cycles so a small dataset still yields that many indices.+ # Draw the full configured batch, not min(batch, n): the sampler refills across cycles so a small+ # dataset still yields that many indices.idx = index_sampler.next_batch(cfg.train_batch_size)chosen = [pairs[i] for i in idx]return idx, [c[0] for c in chosen], [c[1] for c in chosen]- # Restore a previous run's state (adapter, optimizer moments, LR position, sampler cycle,- # RNG streams) BEFORE the loop, so `resumed` is the last completed step and the loop- # picks up at resumed + 1. Zero for a fresh run; a mismatch raises out to the adapter.+ # Restore a previous run's state BEFORE the loop, so resumed is the last completed step and the loop+ # picks up at resumed + 1; a mismatch raises out to the adapter.rng_streams = {"loop": rng, "variant": variant_rng}restored = restore_resume_state(cfg,@@ -527,17 +532,13 @@ def run_diffusion_lora_training(rng_streams = rng_streams,)resumed = restored.step if restored is not None else 0- # Bound HERE, not at the export below: the checkpoint scan and the periodic saves both- # need it, and the two earlier assignments live inside early-return branches -- so on- # every normal run this read came before any binding at all.+ # Bound HERE: the checkpoint scan and the periodic saves both need it, and the two earlier+ # assignments live inside early-return branches.out_dir = Path(cfg.output_dir).expanduser()- # Whether this run's source bundle lives in THIS directory. Resuming from- # directory A into a reused output_dir B leaves B's existing bundles as foreign- # as they would be for a fresh run -- so the first save has to clear them, or a- # higher-numbered one survives and a later resume by B picks it over this run.+ # Resuming from directory A into a reused output_dir B leaves B's existing bundles as foreign as+ # for a fresh run, so the first save must clear them or a higher-numbered one outranks this run.resumed_here = bool(resumed) and resumed_into_this_dir(cfg, out_dir)- # The bundles that were already here when this run started, so a discard below- # removes only what this run wrote. A resumed run shares its source's directory.+ # The bundles already here when this run started, so a discard removes only what this run wrote.preexisting_checkpoints = snapshot_checkpoints(out_dir)unet.train()@@ -552,9 +553,9 @@ def run_diffusion_lora_training(done = resumeddef _save_checkpoint(step: int) -> None:- # Outcome is reported by write_resume_checkpoint's own checkpoint_saved /- # checkpoint_failed events, so a run that crashes after a save is still known to be- # resumable (and one whose save failed is still known to be blocked).+ # write_resume_checkpoint reports its own checkpoint_saved / checkpoint_failed events, so a run+ # that crashes after a save is still known to be resumable and one whose save failed is still+ # known to be blocked._written, _error = write_resume_checkpoint(cfg,step = step,@@ -566,15 +567,10 @@ def run_diffusion_lora_training(sampler = index_sampler,rng_streams = rng_streams,progress = {"running_loss": running_loss},- # NOT discard_existing. A fresh run does own its output dir, but deleting the- # previous run's bundles at the FIRST periodic save spends them before this run- # has produced anything: "stop without saving" then removes only what this run- # wrote, leaves the previous adapter in place, and the run it belonged to is- # unresumable -- cancelling a retrain destroyed the thing being retrained. The- # clear happens on the completion path instead, once the new adapter is saved.+ # NOT discard_existing: deleting the previous run's bundles at the FIRST periodic save spends them+ # before this run produced anything, so cancelling a retrain destroyed the thing being retrained.discard_existing = False,- # And the bundles that were here before this run: a branched resume must not- # prune the higher-numbered checkpoints it did not write.+ # A branched resume must not prune the higher-numbered checkpoints it did not write.preexisting = preexisting_checkpoints,)@@ -648,7 +644,8 @@ def run_diffusion_lora_training(step_loss += float(loss.detach()) / cfg.gradient_accumulation_stepsmicro += 1- # max_grad_norm at or below 0 disables clipping (Unsloth sends 0.0); passing 0.0 to clip_grad_norm_ would zero every gradient.+ # max_grad_norm at or below 0 disables clipping (Unsloth sends 0.0); passing 0.0 to clip_grad_norm_+ # would zero every gradient.grad_norm = Noneif cfg.max_grad_norm and cfg.max_grad_norm > 0:# Returned value is the total PRE-clip norm, reported to the UI chart.@@ -659,8 +656,8 @@ def run_diffusion_lora_training(running_loss += step_lossdone = opt_step + 1now = time.time()- # Rates count only the steps THIS process ran, so a run resumed at step 11 does not- # divide by 12 steps it never executed.+ # Rates count only the steps THIS process ran, so a run resumed at step 11 does not divide by 12+ # steps it never executed.ran_here = done - resumedif ran_here == 1:# Step 1 pays the one-time costs (cudnn autotune, compile warmup), so the rate starts after it.@@ -690,8 +687,7 @@ def run_diffusion_lora_training()stop_now = _check_stop()- # Periodic resume point. Skipped on the final step (a finished run has nothing left- # to resume) and when stopping, since the stop path writes one at the exact step.+ # Skipped on the final step and when stopping, since the stop path writes one at the exact step.if (not stop_nowand cfg.save_steps@@ -708,9 +704,7 @@ def run_diffusion_lora_training(catalog_path: Optional[str] = Noneif not (stopped and not save_on_stop):out_dir.mkdir(parents = True, exist_ok = True)- # Stopping early is exactly when the run should be resumable, so write the bundle- # BEFORE the adapter export: if that export then fails, the run still comes back.- # A completed run has nothing left to resume.+ # Write the bundle BEFORE the adapter export: if that export then fails, the run still comes back.if stopped and done > 0:_save_checkpoint(done)unet_lora = convert_state_dict_to_diffusers(get_peft_model_state_dict(unet))@@ -725,27 +719,18 @@ def run_diffusion_lora_training(# ``done`` (the step reached), not cfg.train_steps: a stop at 11/500 must not advertise 500.catalog_path = _publish_to_lora_catalog(lora_path, cfg, done)if not stopped:- # A COMPLETED run has nothing to resume, and the last iteration deliberately- # writes no bundle -- so with save_steps on, the newest thing left in the- # directory is checkpoint-400 of a run that finished at 500. The preflight- # cannot see the run status, so a later resume with a raised target rolled the- # model, optimizer, scheduler, sampler and RNG back and retrained 401-500.- # Only this run's own bundles go; an earlier run's stay resumable.+ # A completed run has nothing to resume and the last iteration writes no bundle, so with+ # save_steps on the newest thing left is an earlier checkpoint, and a later resume with a raised+ # target rolled everything back and retrained. Only this run's own bundles go.retire_own_checkpoints(out_dir, preexisting_checkpoints, resumed_here = resumed_here)elif not resumed_here:- # Stopped WITH save on a fresh retrain: the stop bundle is a LOWER step than- # the leftovers of the earlier run in this directory, and resume-by-directory- # picks the newest by step -- so those would outrank the partial just saved- # and continue the wrong training. A resumed run keeps what it found.+ # A stop-with-save on a fresh retrain is a LOWER step than the earlier run's leftovers, and resume-+ # by-directory picks the newest by step, so those would outrank the partial just saved.discard_preexisting_checkpoints(out_dir, preexisting_checkpoints)else:- # Discarded: the user asked to throw this run away. Before periodic checkpoints- # existed a discard left nothing behind, because the output directory was only- # ever created inside this save gate. save_steps now writes bundles as the run- # goes, so without this a discard leaves up to save_total_limit copies of the- # optimizer state in a directory the user never got an artifact from -- invisible- # to every scanner, unresumable (the record is marked discarded), and with no- # delete path in the UI.+ # save_steps writes bundles as the run goes, so without this a discard leaves up to+ # save_total_limit copies of the optimizer state in a directory the user got no artifact from:+ # invisible to every scanner, unresumable, with no delete path in the UI.clear_own_checkpoints(out_dir, preexisting_checkpoints)try:out_dir.rmdir()@@ -763,8 +748,7 @@ def run_diffusion_lora_training(stopped = stopped,steps_run = done if cfg.train_steps else 0,resumed_from_step = resumed or None,- # "Stop without saving" discards the run, so its own periodic checkpoints must not- # keep offering to continue it.+ # A discarded run's own periodic checkpoints must not keep offering to continue it.discarded = bool(stopped and not save_on_stop),)return str(out_dir)@@ -817,7 +801,8 @@ def run_diffusion_training_process(*, event_queue: Any, stop_queue: Any, config:return got if saw else Falsetry:- # normalized() resolves + validates the family; dispatch through the registry so a DiT family runs its own trainer.+ # normalized() resolves + validates the family; dispatch through the registry so a DiT family runs+ # its own trainer.cfg = _config_from_dict(config).normalized()trainer = get_trainer(cfg.resolved_family)trainer(cfg, on_event = on_event, should_stop = should_stop)diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.pyindex ed44f1824dd..aa4f81b08bb 100644--- a/studio/backend/core/training/diffusion_train_common.py+++ b/studio/backend/core/training/diffusion_train_common.py@@ -40,15 +40,17 @@ from core.inference.diffusion_families import (from core.inference.video_families import detect_video_family, supported_video_family_namesfrom utils.paths.path_utils import drop_appledouble_metadata-# The trainers run in a spawned child that imports diffusers itself, so the inference-side install does not carry over. Both import this module first.+# The trainers run in a spawned child that imports diffusers itself, so the inference-side install+# does not carry over. Both import this module first.install_xformers_windows_rocm_stub()install_torchao_windows_rocm_stub()-# Default LoRA target modules: the attention projections common to the SDXL U-Net and the DiTs (the diffusers/kohya convention). A family may override this.+# Default LoRA target modules: the attention projections common to the SDXL U-Net and the DiTs (the+# diffusers/kohya convention). A family may override this.DEFAULT_LORA_TARGETS: tuple[str, ...] = ("to_k", "to_q", "to_v", "to_out.0")-# diffusers SchedulerType names. piecewise_constant is excluded: it alone needs a `step_rules` string the trainers-# never pass, so accepting it would pass normalized(), free the resident GPU workloads, then crash in the child.+# piecewise_constant is excluded: it alone needs a step_rules string the trainers never pass, so+# accepting it would pass normalized(), free the resident GPU workloads, then crash in the child._LR_SCHEDULERS: frozenset[str] = frozenset({"linear",@@ -60,39 +62,29 @@ _LR_SCHEDULERS: frozenset[str] = frozenset(})-# DiT families whose fp32 RoPE/embedder overflow fp16, so they train in bf16 only. Keep in sync with the DiT trainer's own specs.+# DiT families whose fp32 RoPE/embedder overflow fp16, so they train in bf16 only. Keep in sync with+# the DiT trainer's own specs._FORCE_BF16_FAMILIES: frozenset[str] = frozenset({"qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2", "minimax-h3"})-# VIDEO families (from the separate ``video_families`` registry) Unsloth can train a LoRA on.-# The video registry has no ``trainable`` flag of its own -- it exists for the inference-# picker -- so the trainable set lives here, next to the trainers, and every name in it MUST-# resolve through ``get_trainer``. Not necessarily to the DiT loop: LTX-2 is a-# ``diffusion_dit_trainer._SPECS`` family, while MiniMax-H3 has its own trainer. A video base-# outside this set is refused by name in ``resolve_trainable_family`` rather than falling-# through to a trainer that cannot run it.+# The video registry has no trainable flag, so the trainable set lives here and every name in it+# MUST resolve through get_trainer; a video base outside it is refused by name.TRAINABLE_VIDEO_FAMILIES: frozenset[str] = frozenset({"ltx-2", "minimax-h3"})-# Families whose ``flow_shift`` default is "auto" (reproduce the family's INFERENCE sigma-# distribution) rather than the historical identity 1.0. Both schedulers set-# ``use_dynamic_shifting``, so their static ``shift`` is skipped at init and ``scheduler.sigmas``-# is the unshifted uniform table; training on it would draw a distribution the model never sees-# at inference. Qwen-Image pins base_shift = max_shift = log 3 and LTX-2 evaluates its shift at-# ``max_image_seq_len`` (so mu = max_shift = 2.05 at every resolution): in both cases the-# inference mu is a constant the "auto" branch can reproduce exactly.-#-# MiniMax-H3 is here for the same reason, reached differently: its schedules carry an explicit-# exponential shift (12.0 video, 3.0 audio, from the released scheduler configs) rather than a-# dynamic one, and ``run_h3_lora_training`` applies those two whenever ``flow_shift`` is not a-# number. Without this entry an omitted flow_shift normalized to the identity 1.0, which IS a-# number, so the trainer took it and every default H3 run trained against an unshifted-# distribution the sampler never visits.+# Families whose flow_shift default is "auto" (reproduce the family's INFERENCE sigma+# distribution) rather than the identity 1.0. Both schedulers set use_dynamic_shifting, so+# scheduler.sigmas is the unshifted uniform table and training on it draws a distribution the+# model never sees at inference: Qwen-Image pins base_shift = max_shift = log 3 and LTX-2+# evaluates its shift at max_image_seq_len (mu = max_shift = 2.05 at every resolution), so the+# inference mu is a constant "auto" reproduces. MiniMax-H3 instead applies explicit exponential+# shifts (12.0 video, 3.0 audio) whenever flow_shift is not a number, so without this entry a+# default H3 run trained unshifted.AUTO_FLOW_SHIFT_FAMILIES: frozenset[str] = frozenset({"qwen-image", "ltx-2", "minimax-h3"})-# Video latents are allocated on the family's spatial compression grid, so a training-# resolution off that grid silently changes the latent geometry (or trips a reshape).-# LTX-2's VAE compresses 32x spatially, matching its ``resolution_multiple``.+# Video latents are allocated on the family's spatial compression grid, so a training resolution+# off that grid silently changes the latent geometry. LTX-2's VAE compresses 32x spatially,+# matching its resolution_multiple._VIDEO_RESOLUTION_MULTIPLE = 32_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp"}@@ -100,12 +92,14 @@ _CAPTION_EXTS = (".txt", ".caption")# diffusers' canonical single-file LoRA name, so load_lora_weights(dir) finds it.DEFAULT_LORA_FILENAME = "pytorch_lora_weights.safetensors"-# Architectures Unsloth can neither train nor load: not in the family registry but recognisable by name, so rejecting them by name gives a clear error instead of a mid-run crash.+# Architectures Unsloth can neither train nor load: not in the family registry but recognisable by+# name, so rejecting them by name gives a clear error instead of a mid-run crash._NON_TRAINABLE_RESIDUAL_TOKENS = frozenset({"sd3", "pixart", "sana", "lumina", "cogview"})_NON_TRAINABLE_RESIDUAL_PHRASES = ("stable-diffusion-3", "hunyuan-dit")EventCb = Callable[[dict[str, Any]], None]-# Returns a falsy value to keep training, or a truthy stop signal: bare True, or a dict that may carry ``save=False`` to cancel without saving a partial adapter.+# Returns a falsy value to keep training, or a truthy stop signal: bare True, or a dict that may+# carry ``save=False`` to cancel without saving a partial adapter.StopCb = Callable[[], Any]@@ -181,14 +175,8 @@ def _component_only_repos() -> dict[str, tuple[str, str, str]]:if repo:bases.add(str(repo).strip().lower())bases.update(str(r).strip().lower() for r in getattr(fam, "train_base_repos", ()) if r)- # (scheme, repo) and (base, scheme, repo). The two registries mean DIFFERENT things by- # these tables, so they cannot be read the same way. An image family's entry is a full- # quantized PIPELINE mirror, and so a base. A video family's is a hosted pre-quantized- # DENOISER -- the DiT alone, as the field's own docstring says -- which is a component in- # exactly the way a pre-cast text encoder is: no model_index.json, no VAE, no scheduler.- # Treating one as a base is what let it through the training preflight, resolve to its- # family by name, pass the unsloth/* trust gate, and fail inside from_pretrained only- # after the resident GPU workloads had been evicted.+ # An image family's entry is a full quantized PIPELINE mirror, and so a base; a video family's is a+ # hosted pre-quantized DENOISER, a component.for table in ("prequant_repos", "prequant_variant_repos"):for row in getattr(fam, table, ()) or ():if not row:@@ -290,8 +278,8 @@ def training_pipeline_import_error(resolved_family: str) -> Optional[str]:family_probe_class,)- # Either registry: a video family is invisible to detect_family, and returning None for one- # would hand the strict half of the gate back to the spawned child, after the teardown.+ # A video family is invisible to detect_family, and returning None would hand the strict half of the+ # gate to the spawned child, after the teardown.fam = _trainable_family_spec(resolved_family)if fam is None:return None@@ -323,10 +311,8 @@ def resolve_trainable_family(base_model: str, model_family: Optional[str] = None"""name = str(base_model or "").strip().lower()# GGUF weights (a ``.gguf`` file or ``*-GGUF`` repo) are inference-only: training needs the full diffusers pipeline.- # Exempt a local diffusers checkout that merely has "gguf" in its path, identified by its- # index file. ``modular_model_index.json`` counts: a MODULAR_BASE_FAMILIES checkout (MiniMax-H3)- # has that and no ``model_index.json``, so reading only the conventional name refused the one- # local form the family has, under /models/gguf/..., before model_family could even be read.+ # modular_model_index.json counts: a MODULAR_BASE_FAMILIES checkout has that and no+ # model_index.json, so reading only the conventional name refused the one local form the family has.local = Path(base_model).expanduser() if base_model else Noneis_local_diffusers = bool(local@@ -339,19 +325,17 @@ def resolve_trainable_family(base_model: str, model_family: Optional[str] = Nonef"'{base_model}' is a GGUF checkpoint/repo, which can't be a training base "f"(training needs the full diffusers model). {_trainable_hint()}")- # A component checkpoint is not a base whatever family the name matches, so this precedes- # every family branch below (including an explicit model_family override, which names the- # family but says nothing about what the repo holds).+ # A component checkpoint is not a base whatever family the name matches, so this precedes every+ # family branch, including an explicit model_family override._refuse_component_only_repo(base_model)- # Same shape as the component-only refusal: the name resolves to a real, trainable family, but- # the repo is not something the training loader can open.+ # Same shape as the component-only refusal: the name resolves to a real, trainable family, but the repo is+ # not something the training loader can open._refuse_ltx23_training_base(base_model)if model_family and str(model_family).strip():key = str(model_family).strip().lower()fam = detect_family("", override = key)if fam is None:- # Not an image family: it may still name a VIDEO family, which lives in its own- # registry. Resolve there before declaring the name unknown.+ # Not an image family: it may still name a VIDEO family, which lives in its own registry.vid = detect_video_family("", override = key)if vid is not None:if vid.name not in TRAINABLE_VIDEO_FAMILIES:@@ -375,8 +359,7 @@ def resolve_trainable_family(base_model: str, model_family: Optional[str] = None_assert_family_pipeline_available(fam)return fam.name- # Only once the image registry has declined the name: a video checkpoint. Checked here- # rather than first so an image repo the picker already claims keeps its existing route.+ # Checked here rather than first so an image repo the picker already claims keeps its existing route.vid = detect_video_family(base_model)if vid is not None:if vid.name not in TRAINABLE_VIDEO_FAMILIES:@@ -490,19 +473,19 @@ def get_trainer(family: str) -> Callable[..., str]:if key in ("flux.1", "qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2"):from core.training.diffusion_dit_trainer import run_dit_lora_trainingreturn run_dit_lora_training- # MiniMax-H3 has its own loop: it denoises video and audio jointly over one packed- # sequence on two coupled schedules, which is outside the DiT trainer's _FamilySpec seams.+ # MiniMax-H3 denoises video and audio jointly over one packed sequence on two coupled schedules,+ # which is outside the DiT trainer's _FamilySpec seams.if key == "minimax-h3":from core.training.diffusion_h3_trainer import run_h3_lora_trainingreturn run_h3_lora_trainingraise ValueError(f"No trainer is registered for family {family!r}.")-# Per-family training defaults surfaced by the Train UI: starting points, not hard limits. Families absent here fall back to the DiffusionLoraConfig defaults.+# Per-family training defaults surfaced by the Train UI: starting points, not hard limits. Families+# absent here fall back to the DiffusionLoraConfig defaults.FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {"sdxl": {"lora_rank": 16, "learning_rate": 1e-4, "resolution": 1024},- # Plain "constant" ignores lr_warmup_steps, so warmup defaults must use a- # warmup-capable scheduler.+ # Plain "constant" ignores lr_warmup_steps, so warmup defaults must use a warmup-capable scheduler."flux.1": {"lora_rank": 16,"learning_rate": 1e-4,@@ -520,7 +503,8 @@ FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {"z-image": {"lora_rank": 16, "learning_rate": 1e-4, "resolution": 768},# The Krea 2 authors' recommended starting point (their DreamBooth defaults): rank/alpha 32, lr 3e-4, 512px."krea-2": {"lora_rank": 32, "learning_rate": 3e-4, "resolution": 512},- # Upstream FLUX.2 DreamBooth references default to rank 16 / lr 1e-4; its uniform timestep draw benefits most from a warmup ramp.+ # Upstream FLUX.2 DreamBooth references default to rank 16 / lr 1e-4; its uniform timestep draw+ # benefits most from a warmup ramp."flux.2-klein": {"lora_rank": 16,"learning_rate": 1e-4,@@ -535,9 +519,8 @@ FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {"lr_scheduler": "constant_with_warmup","lr_warmup_steps": 20,},- # LTX-2, from Lightricks' own ltx-trainer LoRA configs: rank/alpha 32, lr 1e-4. The- # resolution must be a multiple of 32 (its VAE's spatial compression), and 512 keeps a- # still at 16x16x1 latents / 256 video tokens, which trains comfortably at batch 1.+ # From Lightricks' own ltx-trainer LoRA configs; the resolution must be a multiple of 32, its+ # VAE's spatial compression, and 512 keeps a still at 16x16x1 latents / 256 video tokens."ltx-2": {"lora_rank": 32,"learning_rate": 1e-4,@@ -545,11 +528,8 @@ FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {"lr_scheduler": "constant_with_warmup","lr_warmup_steps": 20,},- # MiniMax-H3. ``resolution`` is the canvas SHORT EDGE, and 768 is the one the released- # checkpoint generates on, so a training clip's spatial statistics land exactly on the- # distribution the sampler works in. rank 16 rather than LTX-2's 32: the adapter is applied- # to the audio rows as well as the video ones (one shared block stack), so a smaller- # adapter is the conservative default. Batch size is pinned to 1 by the trainer.+ # resolution is the canvas SHORT EDGE and 768 is what the released checkpoint generates on;+ # rank 16 not 32 because the adapter also serves the audio rows through one shared stack."minimax-h3": {"lora_rank": 16,"learning_rate": 1e-4,@@ -566,7 +546,8 @@ def train_defaults(family: str) -> dict[str, Any]:return dict(FAMILY_TRAIN_DEFAULTS.get((family or "").strip().lower(), {}))-# Display labels + a short VRAM/access note per trainable family, surfaced by the Train UI so users pick a base with realistic expectations.+# Display labels + a short VRAM/access note per trainable family, surfaced by the Train UI so users pick a base+# with realistic expectations._FAMILY_LABELS = {"sdxl": "SDXL","flux.1": "FLUX.1-dev",@@ -578,8 +559,7 @@ _FAMILY_LABELS = {"ltx-2": "LTX-2","minimax-h3": "MiniMax-H3",}-# Per-family training facts as fields, so the UI can chip them instead of parsing prose.-# ``params`` is the transformer size (SDXL is not quoted that way), ``note`` is the rest.+# params is the transformer size (SDXL is not quoted that way); note is the rest._FAMILY_TRAIN_SPECS: dict[str, dict[str, Any]] = {"sdxl": {"params": "", "qlora_vram_gb": 12, "gated": False, "note": "The lightest option."},"flux.1": {"params": "12B", "qlora_vram_gb": 16, "gated": True, "note": ""},@@ -598,26 +578,16 @@ _FAMILY_TRAIN_SPECS: dict[str, dict[str, Any]] = {},"flux.2-klein": {"params": "4B", "qlora_vram_gb": 10, "gated": False, "note": ""},"flux.2-dev": {"params": "32B", "qlora_vram_gb": 28, "gated": True, "note": ""},- # Video. Measured on a B200: the training LOOP peaks at 11.2 GB (nf4, rank 32, 512px- # stills, batch 1, gradient checkpointing), but the RUN peaks at 34.8 GB earlier, while- # the Gemma3-12B conditioning stack is resident and captions are encoded -- before it is- # freed and the transformer loads. The quoted figure covers the whole run, since that is- # what a card has to hold. Lightricks quote 80 GB recommended / 32 GB with their int8- # low-VRAM config for their own trainer.+ # Measured on a B200: the training LOOP peaks at 11.2 GB, but the RUN peaks at 34.8 GB while the+ # Gemma3-12B conditioning stack is resident, and the quoted figure covers the whole run."ltx-2": {"params": "19B","qlora_vram_gb": 36,"gated": False,"note": "Video: trains a style LoRA on still images.",},- # Video + audio. Measured on a B200 at the released 768 short edge, 22-frame clips,- # rank 16, batch 1, gradient checkpointing: the training LOOP peaks around 44 GB (nf4 with- # the dtype-reading modules kept dense, see _H3_NF4_SKIP_MODULES), but the RUN peaks far- # higher while the 63 GiB Qwen3-VL conditioner is resident and captions are encoded --- # before it is freed and the transformer loads. The quoted figure covers the whole run,- # since that is what a card has to hold: a real 20-step run on this branch peaked at- # 77.76 GB, so 72 was under its own measurement and sized users onto a card that OOMs- # only after the conditioner has loaded.+ # Measured on a B200: the loop peaks near 44 GB, but a 20-step run peaked at 77.76 GB with the+ # 63 GiB Qwen3-VL conditioner resident, so 72 sizes users onto a card that OOMs later."minimax-h3": {"params": "31B","qlora_vram_gb": 80,@@ -625,14 +595,13 @@ _FAMILY_TRAIN_SPECS: dict[str, dict[str, Any]] = {"note": "Video with sound: trains on clips that have a soundtrack.",},}-# Facts that differ between selectable checkpoints inside one family. Keys are canonical-# upstream ids; family_train_infos also publishes the mirror aliases so a custom mirror pick-# gets the same guidance. Values overlay the family facts in the client.+# Keys are canonical upstream ids; family_train_infos also publishes the mirror aliases, and values+# overlay the family facts in the client._BASE_TRAIN_SPECS: dict[str, dict[str, Any]] = {"black-forest-labs/flux.2-klein-base-9b": {"params": "9B",- # The auto policy measures 16.4 GB for the bf16 text encoder alone. Leave room for- # the VAE and runtime state rather than inheriting the 4B checkpoint's 10 GB floor.+ # The bf16 text encoder alone measures 16.4 GB, so leave room for the VAE and runtime state rather+ # than inheriting the 4B checkpoint's floor."qlora_vram_gb": 18,},}@@ -650,13 +619,12 @@ def _family_vram_note(name: str) -> str:return " ".join([head, *tail])-# The flow-matching DiT families (run by diffusion_dit_trainer): they expose base_precision / compile and need bf16 on CUDA. SDXL is absent (own mixed_precision path).+# The flow-matching DiT families (run by diffusion_dit_trainer): they expose base_precision /+# compile and need bf16 on CUDA. SDXL is absent (own mixed_precision path)._DIT_TRAIN_FAMILIES = frozenset({"flux.1", "qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2"})-# Families with a flow-matching trainer that is NOT diffusion_dit_trainer, but which still-# expose base_precision and need bf16 on CUDA. Kept separate so the DiT-specific levers-# (compile, the shared sigma table) do not follow.+# Kept separate so the DiT-specific levers (compile, the shared sigma table) do not follow._FLOW_TRAIN_FAMILIES = _DIT_TRAIN_FAMILIES | {"minimax-h3"}@@ -673,14 +641,9 @@ def effective_mixed_precision(cfg: Any) -> str:requested = str(getattr(cfg, "mixed_precision", "") or "")if str(getattr(cfg, "resolved_family", "") or "").strip().lower() in _FLOW_TRAIN_FAMILIES:- # No flow-matching trainer reads mixed_precision at all: weight_dtype is bf16 on CUDA and- # fp32 otherwise, in the DiT loop and in the H3 loop alike. Recording the REQUEST for those- # families put "fp16" or "no" in the identity of a run that executed in bf16, and a later- # bf16 resume of it was then rejected as a precision mismatch between two runs that ran- # identically. _FLOW_TRAIN_FAMILIES rather than _DIT_TRAIN_FAMILIES so the answer follows- # the weight dtype rather than which loop happens to own the family: H3 refuses a non-bf16- # request and writes no checkpoint today, so this is the same string either way for every- # config it accepts, but the helper stops being the odd one out the day that changes.+ # No flow-matching trainer reads mixed_precision (weight_dtype is bf16 on CUDA, fp32 otherwise),+ # so recording the REQUEST failed a later bf16 resume as a precision mismatch between identical+ # runs. Keyed on _FLOW_TRAIN_FAMILIES so the answer follows the weight dtype.return "bf16" if torch.cuda.is_available() else "no"if not torch.cuda.is_available():return "no"@@ -753,7 +716,8 @@ def dit_accelerator_missing_reason(resolved_family: str) -> Optional[str]:try:import torchdef probe(owner: Any) -> bool:- # Each accelerator is probed on its own: torch.mps.is_available() only exists from torch 2.5 while the floor is 2.4, so a shared try/except would wave a CPU-only host through.+ # Each accelerator is probed on its own: torch.mps.is_available() only exists from torch 2.5 while+ # the floor is 2.4, so a shared try/except would wave a CPU-only host through.try:fn = getattr(owner, "is_available", None)return bool(fn()) if callable(fn) else False@@ -796,7 +760,8 @@ def training_precision_preflight_error(resolved_family: str, base_precision: strfam = (resolved_family or "").strip().lower()mode = (base_precision or "").strip().lower()if fam in _FLOW_TRAIN_FAMILIES and mode in ("bf16", "int8", "fp8", "mxfp8"):- # The DiT trainer's dense precisions all require CUDA, and bf16_unsupported_reason exempts a CPU-only host, so without this a dense request would evict residents then raise in the child.+ # The DiT trainer's dense precisions all require CUDA, and bf16_unsupported_reason exempts a CPU-+ # only host, so without this a dense request would evict residents then raise in the child.try:import torchhas_cuda = torch.cuda.is_available()@@ -818,13 +783,15 @@ def training_precision_preflight_error(resolved_family: str, base_precision: str"base_precision='int8' needs a functional torchao install; this host's torchao is ""missing or the non-functional Windows-ROCm stub. Use 'nf4', 'bf16', or 'auto'.")- # Mirrors the child's fp8/mxfp8 stub guard so a doomed run is refused before eviction. Not has_functional_torchao(): that probes int8's symbols.+ # Mirrors the child's fp8/mxfp8 stub guard so a doomed run is refused before eviction. Not+ # has_functional_torchao(): that probes int8's symbols.if mode in ("fp8", "mxfp8") and is_stubbed("torchao"):return (f"base_precision={mode!r} is not available on this host: torchao is the ""non-functional Windows-ROCm stub. Use 'nf4', 'bf16', or 'auto'.")- # mxfp8 needs Blackwell (sm100+): its MX GEMM raises at the first training step, AFTER a full dense load. Re-check here so a stale client fails fast before eviction.+ # mxfp8 needs Blackwell (sm100+): its MX GEMM raises at the first training step, AFTER a full dense+ # load. Re-check here so a stale client fails fast before eviction.if mode == "mxfp8":try:import torch@@ -858,28 +825,21 @@ def family_train_infos() -> list[dict[str, Any]]:fam = _trainable_family_spec(name)if fam is None or not family_pipeline_available(fam):continue- # A video family carries no train_base_repos/deploy_base_repo: its own base repo is the- # one training base, and a video LoRA has no image catalog to deploy into.+ # A video family carries no train_base_repos/deploy_base_repo: its own base repo is the one training base.repos = list(getattr(fam, "train_base_repos", ()) or ()) or [fam.base_repo]- # base_precision applies to every flow-matching trainer, not only the shared DiT one:- # MiniMax-H3 has its own trainer but the same base_precision lever and the same bf16-on-- # CUDA requirement. Reading _DIT_TRAIN_FAMILIES here reported precision_modes = [] for it,- # which the Train panel reads as "this GPU cannot train this family" and disables Start- # with "Not supported on this GPU" -- the trainer was unreachable from Unsloth on every- # host. SDXL keeps its mixed_precision lever and is in neither set.+ # base_precision applies to every flow-matching trainer, not only the shared DiT one: reading+ # _DIT_TRAIN_FAMILIES here reported precision_modes = [] for H3, which the Train panel shows as+ # "Not supported on this GPU".is_dit = name in _FLOW_TRAIN_FAMILIES- # On a non-bf16 CUDA GPU the start preflight rejects EVERY DiT family, so advertise no precision, else /info offers an- # nf4 DiT option that always 400s. Otherwise drop any scheme this family's DiT corrupts, plus any the TRAINING bar- # holds back even though inference passes (fp8 on Qwen-Image: rendering is validated, a training run is not).+ # On a non-bf16 CUDA GPU the start preflight rejects EVERY DiT family, so advertise no precision+ # rather than an option that always 400s; also drop schemes the TRAINING bar holds back.dit_block = (bf16_unsupported_reason(name) or dit_accelerator_missing_reason(name)if is_ditelse None)- # compile is the one DiT-only lever that does NOT follow the precision one. MiniMax-H3- # refuses compile_transformer="on" -- its packed sequence changes length with every- # caption, so each step would re-trace -- so advertising the control would offer a- # selection that always 400s.+ # H3 refuses compile_transformer="on" (its packed sequence changes length with every caption, so+ # each step would re-trace), so advertising the control would offer a selection that always 400s.supports_compile = bool(not dit_block) and name in _DIT_TRAIN_FAMILIESif not is_dit or dit_block:fam_modes: list[str] = []@@ -898,8 +858,8 @@ def family_train_infos() -> list[dict[str, Any]]:base_specs[repo_mirror] = dict(base_spec)for training_repo, inference_repo in getattr(fam, "deploy_base_repos", ()):deploy_bases[training_repo] = inference_repo- # A custom base entered with the public mirror id must follow the same pairing as the- # advertised vendor id. Return the inference mirror too, so Deploy stays ungated.+ # A custom base entered with the public mirror id must follow the same pairing as the advertised+ # vendor id; return the inference mirror too so Deploy stays ungated.training_mirror = mirror_repo(training_repo)if training_mirror:deploy_bases[training_mirror] = mirror_repo(inference_repo) or inference_repo@@ -911,37 +871,29 @@ def family_train_infos() -> list[dict[str, Any]]:"base_repos": repos,"defaults": train_defaults(name),"vram_note": dit_block or _family_vram_note(name),- # Chip fields. Dropped on a dit_block, since vram_note then carries the reason.+ # Dropped on a dit_block, since vram_note then carries the reason."params": "" if dit_block else spec.get("params", ""),"qlora_vram_gb": None if dit_block else spec.get("qlora_vram_gb"),"gated": False if dit_block else bool(spec.get("gated", False)),"note": "" if dit_block else spec.get("note", ""),"precision_modes": fam_modes,"recommended_precision": "nf4" if (not is_dit or dit_block) else dit_recommended,- # compile is offered for SDXL's regional U-Net and the shared DiT trainer, except- # a family the GPU cannot train in bf16, and except a trainer that cannot compile.+ # compile is offered for SDXL's regional U-Net and the shared DiT trainer, except a family the+ # GPU cannot train in bf16, and except a trainer that cannot compile."supports_compile": supports_compile or name == "sdxl",- # Same reasoning one field up, for the other control a family can simply not- # have. save_steps is REFUSED for a checkpointless family, not ignored, so a- # panel that keeps offering "Checkpoint every" turns a nonzero value into a- # rejected Start with no way to see why from the control itself.+ # save_steps is REFUSED for a checkpointless family, not ignored, so a panel that keeps offering+ # "Checkpoint every" turns a nonzero value into a rejected Start with no way to see why."supports_checkpoints": name not in CHECKPOINTLESS_FAMILIES,- # And the third one, for the same reason. A batch > 1 is REFUSED for a family- # whose forward covers one packed sequence, so a panel that keeps offering an- # unrestricted Batch turns a perfectly reasonable 2 -- or a value carried over- # from the family the user was on a moment ago -- into a rejected Start with- # nothing on the control to say why.+ # A batch > 1 is REFUSED for a family whose forward covers one packed sequence, so leaving the+ # control unrestricted turns a reasonable 2 into a rejected Start with nothing to say why."max_train_batch_size": 1 if name in SINGLE_SEQUENCE_FAMILIES else None,# Krea trains on Raw but previews adapters on Turbo; None elsewhere (and never for a video family)."deploy_base": getattr(fam, "deploy_base_repo", None),# Families with several train/deploy pairs cannot use the scalar above."deploy_bases": deploy_bases,- # Checkpoint-specific facts overlay the family facts in the Train UI. Dropped on a- # dit_block for the same reason the family chips above are: the overlay wins in- # resolveDiffusionTrainingFacts, so a per-base entry would put the 9B / 18 GB chips- # back, and FamilyFacts renders vram_note only when there are NO chips. Leaving- # these populated therefore replaces the actionable hardware reason (no CUDA, no- # native bf16) with a size the host cannot act on.+ # Dropped on a dit_block for the same reason as the family chips: the overlay wins and+ # FamilyFacts renders vram_note only when there are no chips, so keeping these would replace the+ # actionable hardware reason with a size the host cannot act on."base_specs": {} if dit_block else base_specs,})@@ -956,67 +908,74 @@ class DiffusionLoraConfig:base_model: strdata_dir: stroutput_dir: str- # Dreambooth-style caption applied to any image without its own. Required if the dataset has no captions.jsonl / sidecars.+ # Dreambooth-style caption applied to any image without its own. Required if the dataset has no+ # captions.jsonl / sidecars.instance_prompt: Optional[str] = Noneresolution: int = 1024train_steps: int = 500- # 0 = disabled (train for train_steps). Above 0 it overrides train_steps with num_epochs full passes (see resolve_train_steps).+ # 0 = disabled (train for train_steps). Above 0 it overrides train_steps with num_epochs full passes+ # LoRA EMA decay (DiT trainer only). 0.0 disables; a positive value keeps a warmup-ramped EMA of+ # the trainable params and exports it under ema/.num_epochs: int = 0learning_rate: float = 1e-4train_batch_size: int = 1gradient_accumulation_steps: int = 1lora_rank: int = 16- lora_alpha: Optional[int] = None # defaults to lora_rank+ lora_alpha: Optional[int] = Nonelora_dropout: float = 0.0lora_target_modules: tuple[str, ...] = DEFAULT_LORA_TARGETSseed: int = 42- mixed_precision: str = "bf16" # "bf16" | "fp16" | "no"- snr_gamma: Optional[float] = 5.0 # min-SNR loss weighting; None disables+ mixed_precision: str = "bf16"+ snr_gamma: Optional[float] = 5.0gradient_checkpointing: bool = Truemax_grad_norm: float = 1.0lr_scheduler: str = "constant"lr_warmup_steps: int = 0center_crop: bool = Falserandom_flip: bool = True- caption_column: str = "text" # column in metadata.jsonl+ caption_column: str = "text"adapter_name: str = "default"hf_token: Optional[str] = None- # Derived by normalized(): the byte-identical mirror used by from_pretrained while- # base_model remains the canonical id stored in metadata and resume identity.+ # Derived by normalized(): the byte-identical mirror used by from_pretrained, while base_model stays+ # the canonical id in metadata and resume identity.fetch_base_model: Optional[str] = None- # Precompute the VAE latents once (freeing the VAE) instead of re-encoding every step. ``cache_variants`` crop/flip draws are frozen per image; the per-step VAE sampling noise is preserved.+ # cache_variants crop/flip draws are frozen per image; the per-step VAE sampling noise is preserved.cache_latents: bool = Truecache_variants: int = 4- # Persistent on-disk conditioning cache directory (DiT trainer only). None keeps the in-memory-only behavior; a- # directory persists latent stats and caption embeddings keyed by content hash, so a warm cache skips the VAE and TEs.+ # A directory persists latent stats and caption embeddings keyed by content hash, so a warm cache+ # skips the VAE and text encoders.cond_cache_dir: Optional[str] = None- # LoRA EMA decay (DiT trainer only). 0.0 disables; a positive value keeps a warmup-ramped EMA of the trainable params and exports it under ema/.+ # LoRA EMA decay (DiT trainer only). 0.0 disables; a positive value keeps a warmup-ramped EMA of+ # the trainable params and exports it under ema/.ema_decay: float = 0.0- # Regional torch.compile of the transformer blocks: "off" | "on" | "auto" (auto turns it on only for a dense, non-bitsandbytes base where it is a clean win).+ # Regional torch.compile of the transformer blocks: "off" | "on" | "auto" (auto turns it on only for+ # a dense, non-bitsandbytes base where it is a clean win).compile_transformer: str = "auto"# TF32 matmuls + cudnn autotuning for the run. Near-lossless; disable for strict bit-reproducibility A/Bs.enable_tf32: bool = True# DiT base transformer precision: "nf4" (bitsandbytes QLoRA, the memory floor and default), "bf16" (dense, fastest- # eager), "int8" (torchao weight-only), "fp8" (Ada/Hopper/Blackwell + compile) or "auto". Non-nf4 needs a dense base.base_precision: str = "nf4"- # Training-time timestep shift on the flow-matching sigma draw. None resolves per family in normalized(): "auto" for- # qwen-image, 1.0 elsewhere. A number applies s*u/(1+(s-1)*u); "auto" without dynamic shifting falls back to identity.- flow_shift: Optional[Any] = None # float | "auto" | None- # Per-sample probability of replacing the caption with the empty prompt (classifier-free-guidance dropout). 0.0 disables.+ # None resolves per family in normalized(); a number applies s*u/(1+(s-1)*u), and "auto" without+ # dynamic shifting falls back to identity.+ flow_shift: Optional[Any] = None+ # Per-sample probability of replacing the caption with the empty prompt (classifier-free-guidance dropout).+ # 0.0 disables.cfg_dropout: float = 0.0- # Per-sample loss weighting over the drawn timestep: "none" (unweighted MSE) or "bell" (Gaussian, normalized to mean 1).+ # Per-sample loss weighting over the drawn timestep: "none" (unweighted MSE) or "bell" (Gaussian, normalized+ # to mean 1).weighting_scheme: str = "none"# How often to emit a progress event (in optimizer steps).log_every: int = 1- # Periodic resume-checkpoint interval, in optimizer steps. 0 (the default) writes no periodic- # checkpoints; a stop-and-save always writes one regardless, so the Resume action stays available.+ # 0 writes no periodic checkpoints; a stop-and-save always writes one regardless, so the Resume+ # action stays available.save_steps: int = 0# How many checkpoint-<N> bundles to keep in the output dir; 0 keeps every one.save_total_limit: int = 2- # Continue a previous run: the run's output_dir, or one of its checkpoint-<N> directories.- # The start route resolves and validates it before the trainer spawns.+ # The run's output_dir, or one of its checkpoint-<N> directories; the start route resolves and+ # validates it before the trainer spawns.resume_from_checkpoint: Optional[str] = None- # Optional explicit family override; None = detect from base_model. ``resolved_family`` is filled by normalized() with the trainer family that will run.+ # Optional explicit family override; None = detect from base_model. ``resolved_family`` is filled by+ # normalized() with the trainer family that will run.model_family: Optional[str] = Noneresolved_family: str = "sdxl"@@ -1043,8 +1002,8 @@ class DiffusionLoraConfig:)if self.resolution < 64 or self.resolution % 8 != 0:raise ValueError("resolution must be a multiple of 8 and >= 64")- # A video family's VAE compresses space by 32, so an off-grid resolution changes the- # latent geometry silently. Refuse it here, before the GPU models are evicted.+ # A video family's VAE compresses space by 32, so an off-grid resolution changes the latent geometry+ # silently. Refuse it before the GPU models are evicted.if (resolved_family in TRAINABLE_VIDEO_FAMILIESand self.resolution % _VIDEO_RESOLUTION_MULTIPLE != 0@@ -1056,7 +1015,8 @@ class DiffusionLoraConfig:)if self.mixed_precision not in ("bf16", "fp16", "no"):raise ValueError("mixed_precision must be one of bf16 / fp16 / no")- # torch.manual_seed unpacks int64/uint64, so anything wider raises inside the trainer, after eviction. Catch it here.+ # torch.manual_seed unpacks int64/uint64, so anything wider raises inside the trainer, after+ # eviction. Catch it here.if not -(2**63) <= int(self.seed) <= 2**64 - 1:raise ValueError("seed must fit in torch's 64-bit range")# Refuse fp16 for a bf16-only DiT family up front, before evicting resident models.@@ -1070,8 +1030,8 @@ class DiffusionLoraConfig:f"lr_scheduler must be one of {', '.join(sorted(_LR_SCHEDULERS))}; "f"got {self.lr_scheduler!r}")- # Do not rewrite the scheduler here: it is part of checkpoint identity, so doing so would- # make legacy runs with ("constant", warmup > 0) impossible to resume.+ # Do not rewrite the scheduler: it is part of checkpoint identity, so legacy runs with ("constant",+ # warmup > 0) would become unresumable.try:lr_warmup_steps = int(self.lr_warmup_steps or 0)except (TypeError, ValueError) as exc:@@ -1082,7 +1042,6 @@ class DiffusionLoraConfig:raise ValueError("lr_warmup_steps must be >= 0")if not 1 <= int(self.cache_variants) <= 16:raise ValueError("cache_variants must be between 1 and 16")- # Checkpointing knobs. Rejected here, before the route evicts resident GPU models, rather than deep in the loop.try:save_steps = int(self.save_steps or 0)save_total_limit = int(self.save_total_limit or 0)@@ -1095,17 +1054,15 @@ class DiffusionLoraConfig:raise ValueError("save_steps must be >= 0 (0 disables periodic checkpoints)")if save_total_limit < 0:raise ValueError("save_total_limit must be >= 0 (0 keeps every checkpoint)")- # A blank resume path (the Unsloth default when the field is present but unset) means "fresh run", not the outputs root.+ # A blank resume path (the Unsloth default when the field is present but unset) means "fresh run",+ # not the outputs root.resume_from_checkpoint = (str(self.resume_from_checkpoint).strip()if self.resume_from_checkpoint is not Noneelse "") or None- # The H3 loop does not checkpoint: it neither writes a resume bundle nor restores one.- # Accepting these two silently was the dangerous part -- a caller handing over a resume- # bundle got a FRESH optimization that then overwrote the outputs it was meant to- # continue, and one asking for periodic saves got none, both discovered only after an- # expensive run. Refuse in validation, where it costs nothing, until the loop supports it.+ # The H3 loop neither writes a resume bundle nor restores one, and accepting these silently gave a+ # resume request a FRESH optimization that then overwrote the outputs it was meant to continue.if resolved_family in CHECKPOINTLESS_FAMILIES:if resume_from_checkpoint:raise ValueError(@@ -1122,7 +1079,8 @@ class DiffusionLoraConfig:ema_decay = float(self.ema_decay or 0.0)except (TypeError, ValueError) as exc:raise ValueError(f"ema_decay must be a number, got {self.ema_decay!r}") from exc- # decay = 1.0 would freeze the shadow at its init forever; the update is shadow * decay + param * (1 - decay), so valid decays live in [0, 1).+ # decay = 1.0 would freeze the shadow at its init forever; the update is shadow * decay + param * (1+ # - decay), so valid decays live in [0, 1).if not 0.0 <= ema_decay < 1.0:raise ValueError("ema_decay must be in [0, 1); 0 disables the EMA adapter")# A blank cond_cache_dir (the Unsloth default when unset) means "off", not cwd.@@ -1135,7 +1093,8 @@ class DiffusionLoraConfig:base_precision = str(self.base_precision or "nf4").strip().lower()if base_precision not in ("nf4", "bf16", "int8", "fp8", "mxfp8", "auto"):raise ValueError("base_precision must be one of nf4 / bf16 / int8 / fp8 / mxfp8 / auto")- # base_precision is a DiT-only lever, so the dense-mode gates apply only to the DiT families. The mode-name check above still runs for every family.+ # base_precision is a DiT-only lever, so the dense-mode gates apply only to the DiT families. The+ # mode-name check above still runs for every family.if resolved_family != "sdxl" and base_precision in ("bf16", "int8", "fp8", "mxfp8"):if repo_is_prequantized(self.base_model):raise ValueError(@@ -1148,12 +1107,10 @@ class DiffusionLoraConfig:f"base_precision={base_precision!r} trains in bf16 compute; set "f"mixed_precision to bf16.")- # Refuse a scheme this family's DiT is known to corrupt, and also one the training bar holds back while- # inference allows it: qwen-image fp8 now renders inside the accuracy gate, but no one has measured whether a- # LoRA converges against fp8-frozen linears, so it fails fast here rather than silently training on faith.- # MiniMax-H3 runs all three modalities through one set of linears, so the- # per-family activation range the fp8 module filter was measured against does not- # describe it. Refuse the float8 modes rather than train against a clipped forward.+ # qwen-image fp8 renders inside the accuracy gate, but no one has measured whether a LoRA converges+ # against fp8-frozen linears, so training fails fast rather than training on faith.+ # MiniMax-H3 runs all three modalities through one set of linears, so the per-family activation+ # range the fp8 module filter was measured against does not describe it.if resolved_family == "minimax-h3" and base_precision in ("fp8", "mxfp8"):raise ValueError(f"base_precision={base_precision!r} is not supported for minimax-h3: its "@@ -1161,9 +1118,8 @@ class DiffusionLoraConfig:f"so the activation range fp8 was measured against does not apply. Use "f"'nf4', 'int8', 'bf16', or 'auto'.")- # _family_train_denied, not _family_denied: it is the strict superset (every inference- # deny plus the training-only ones), so importing the narrower helper here would let a- # scheme cleared only for rendering reach a trainer.+ # _family_train_denied is the strict superset of _family_denied, so importing the narrower helper+ # here would let a scheme cleared only for rendering reach a trainer.from core.inference.diffusion_transformer_quant import _family_train_deniedif _family_train_denied(resolved_family, base_precision):@@ -1171,7 +1127,8 @@ class DiffusionLoraConfig:f"base_precision={base_precision!r} is not validated for training "f"{resolved_family}. Use 'nf4', 'int8', 'bf16', or 'auto'.")- # flow_shift: None resolves to the family default ("auto" only for qwen-image, whose scheduler skips its static shift under use_dynamic_shifting); an explicit value is validated and kept.+ # flow_shift: None resolves to the family default ("auto" only for qwen-image, whose scheduler skips its+ # static shift under use_dynamic_shifting); an explicit value is validated and kept.flow_shift = self.flow_shiftif flow_shift is None:flow_shift = "auto" if resolved_family in AUTO_FLOW_SHIFT_FAMILIES else 1.0@@ -1186,7 +1143,8 @@ class DiffusionLoraConfig:) from excif not isinstance(flow_shift, str):flow_shift = float(flow_shift)- # isfinite as well as positive: JSON accepts 1e309, which floats to inf and would poison every sampled sigma while progress looks normal.+ # isfinite as well as positive: JSON accepts 1e309, which floats to inf and would poison every+ # sampled sigma while progress looks normal.if not math.isfinite(flow_shift) or flow_shift <= 0:raise ValueError("flow_shift must be a finite number > 0 (1.0 disables the shift), or 'auto'"@@ -1200,7 +1158,8 @@ class DiffusionLoraConfig:weighting_scheme = str(self.weighting_scheme or "none").strip().lower()if weighting_scheme not in ("none", "bell"):raise ValueError("weighting_scheme must be one of none / bell")- # A zero/negative gamma would zero out (or invert) the min-SNR weight and silently train on a degenerate loss; None is the documented disable.+ # A zero/negative gamma would zero out (or invert) the min-SNR weight and silently train on a+ # degenerate loss; None is the documented disable.if self.snr_gamma is not None and float(self.snr_gamma) <= 0:raise ValueError("snr_gamma must be > 0, or null to disable min-SNR weighting")# learning_rate can arrive as a string ("1e-4") from the Unsloth config path, so coerce it before AdamW sees it.@@ -1212,7 +1171,8 @@ class DiffusionLoraConfig:raise ValueError("learning_rate must be > 0")alpha = self.lora_alpha if self.lora_alpha is not None else self.lora_ranktargets = tuple(self.lora_target_modules) or DEFAULT_LORA_TARGETS- # A blank Hub token (the Unsloth default when none is configured) must load anonymously, not as an explicit empty credential.+ # A blank Hub token (the Unsloth default when none is configured) must load anonymously, not as an+ # explicit empty credential.token = self.hf_token.strip() if isinstance(self.hf_token, str) else self.hf_tokenfrom core.inference.diffusion_families import (_is_local_path,@@ -1225,27 +1185,13 @@ class DiffusionLoraConfig:fetch_base_model = self.base_modelelse:fetch_base_model = prefer_ungated_mirror(self.base_model, token or None)- # For a GATED upstream and no token, the cache preference has to be overridden: the- # credentials this run lacks are the credentials the fetch needs, so a partial- # snapshot cannot be completed and the start route's HEAD refuses even a complete- # one. prefer_ungated_mirror's probe counts any cached weight as a hit, so a single- # leftover shard from an interrupted or previously authorized download was enough to- # keep the vendor id and hard-block the very case the mirrors exist for.- #- # Only gated, though. Most of the mirror table is ungated, mirrored to keep the- # fetch inside unsloth/*, and there the upstream is reachable anonymously: an- # override would discard a complete local cache and re-pull gigabytes, or fail- # outright offline. Those keep prefer_ungated_mirror's cache-aware answer.- #- # UNSLOTH_DIFFUSION_NO_MIRROR still wins, exactly as it does inside- # prefer_ungated_mirror: it is the documented way to pin the vendor repo, and an- # override that ignored it would make that switch a lie on the training path only.- #- # A local clone wins over both. A base can be a directory named exactly like the- # vendor id (the loaders resolve `black-forest-labs/FLUX.1-dev` on disk when it- # exists), and prefer_ungated_mirror carves that out for the same reason: rewriting- # it sends the fetch to the Hub past weights the user already has, so the run trains- # on a different repo, or fails offline.+ # For a GATED upstream with no token, override the cache preference: prefer_ungated_mirror's+ # probe counts any cached weight as a hit, so one leftover shard kept the vendor id.+ # Only gated: the rest of the mirror table is reachable anonymously, and an override there would+ # discard a complete local cache and re-pull gigabytes, or fail offline.+ # UNSLOTH_DIFFUSION_NO_MIRROR still wins, exactly as it does inside prefer_ungated_mirror.+ # A local clone wins over both: a base can be a directory named exactly like the vendor id, and+ # rewriting it sends the fetch to the Hub past weights the user already has.if (not tokenand upstream_is_gated(self.base_model)@@ -1253,10 +1199,9 @@ class DiffusionLoraConfig:and not os.environ.get("UNSLOTH_DIFFUSION_NO_MIRROR", "").strip()):fetch_base_model = mirror_repo(self.base_model) or fetch_base_model- # A blank caption_column means the default, as the start route's own preflight already- # assumes ("or 'text'"). Without this the route and the trainer resolve different captions- # from a metadata.jsonl, so their dataset fingerprints disagree and a resume that the- # route accepted is then refused in the child -- after the GPU residents are freed.+ # A blank caption_column means the default, as the start route's preflight assumes: otherwise+ # route and trainer resolve different captions from a metadata.jsonl and their fingerprints+ # disagree, so an accepted resume is refused in the child.caption_column = str(self.caption_column or "").strip() or "text"return replace(self,@@ -1324,7 +1269,8 @@ class PermutationBatchSampler:self._pos = 0def next_batch(self, k: int) -> list[int]:- # k may exceed n (batch larger than the dataset): the permutation refills across cycles so the caller always gets exactly k indices.+ # k may exceed n (batch larger than the dataset): the permutation refills across cycles so the+ # caller always gets exactly k indices.out: list[int] = []while len(out) < k:if self._pos >= len(self._order):@@ -1359,12 +1305,8 @@ class PermutationBatchSampler:restored = [int(i) for i in order]except (TypeError, ValueError):return False- # A FULL permutation, or the initial empty state. A shortened or duplicate-carrying- # order is in range and the same length as nothing in particular: the cycle then- # reshuffles early or serves the same image twice, while the RNG has already been- # restored to a point after the original draw. That is the silent reorder this- # boolean exists to prevent, arriving through a truncated or hand-edited manifest- # instead of a missing one.+ # A shortened or duplicate-carrying order is in range and plausible, so the cycle reshuffles+ # early or serves the same image twice with the RNG already past the original draw.if restored and sorted(restored) != list(range(self._n)):return Falseself._order = restored@@ -1372,10 +1314,8 @@ class PermutationBatchSampler:pos = int(state.get("pos") or 0)except (TypeError, ValueError):return False- # A position outside 0..len(order) is not a rounding error, it is a damaged manifest,- # and clamping it re-serves the permutation from the top or ends the cycle early --- # behind an RNG already restored past the draw. The same silent repeat-or-skip the- # order check above refuses, so refuse it rather than quietly correcting it.+ # A position outside 0..len(order) is a damaged manifest, and clamping it re-serves the permutation+ # from the top or ends the cycle early behind an already-restored RNG.if not 0 <= pos <= len(self._order):return Falseself._pos = pos@@ -1415,8 +1355,7 @@ def discover_image_caption_pairs(if not root.is_dir():raise FileNotFoundError(f"data_dir is not a directory: {data_dir}")- # The caption lookup resolves to the caption's own companion, which exists, so the pair- # reads as a real one.+ # The caption lookup resolves to the caption's own companion, which exists, so the pair reads as a real one.images = drop_appledouble_metadata(sorted(p for p in root.iterdir() if p.is_file() and p.suffix.lower() in _IMAGE_EXTS))@@ -1427,7 +1366,8 @@ def discover_image_caption_pairs(meta_path = root / meta_nameif not meta_path.is_file():continue- # Tolerate a bad upload (invalid UTF-8, or non-object JSON): skip the record so the instance_prompt fallback still applies.+ # Tolerate a bad upload (invalid UTF-8, or non-object JSON): skip the record so the instance_prompt+ # fallback still applies.try:meta_lines = meta_path.read_text(encoding = "utf-8").splitlines()except (OSError, UnicodeError):@@ -1452,7 +1392,8 @@ def discover_image_caption_pairs(for img in images:caption: Optional[str] = Nonesidecar_present = False- # 1. per-image sidecar caption (the user's explicit edit; wins over metadata). An EMPTY sidecar is a deliberate tombstone: it suppresses the metadata caption and leaves the image uncaptioned.+ # An EMPTY sidecar is a deliberate tombstone: it suppresses the metadata caption and leaves the+ # image uncaptioned.for ext in _CAPTION_EXTS:sidecar = img.with_suffix(ext)if sidecar.is_file():@@ -1460,10 +1401,12 @@ def discover_image_caption_pairs(try:caption = sidecar.read_text(encoding = "utf-8").strip()except (OSError, UnicodeError):- # An unreadable sidecar reads as the empty tombstone, so the instance_prompt fallback applies instead of a 500.+ # An unreadable sidecar reads as the empty tombstone, so the instance_prompt fallback applies+ # instead of a 500.caption = ""break- # 2. metadata row keyed by file name (basename or relative path, as_posix so Windows paths match). A sidecar, even empty, wins.+ # 2. metadata row keyed by file name (basename or relative path, as_posix so Windows paths+ # match). A sidecar, even empty, wins.if not sidecar_present:caption = meta_caption.get(img.name) or meta_caption.get(img.relative_to(root).as_posix()@@ -1473,7 +1416,11 @@ def discover_image_caption_pairs(caption = instance_promptif caption:if verify_images:- # Reject a corrupt/truncated image now via a cheap PIL header probe: otherwise it passes filename-only discovery, the start route frees the GPU models, and the trainer crashes in Image.open.+ # Reject a corrupt or truncated image now: otherwise it passes filename-only discovery, the start+ # route frees the GPU models, and the trainer crashes in Image.open.+ # Epoch-mode payloads carry max_steps: 0 as the "use epochs" sentinel, which the alias copies as+ # train_steps: 0. normalized() rejects train_steps < 1 before resolve_train_steps() applies+ # num_epochs, so drop a falsy value here.try:from PIL import Imagewith Image.open(img) as _probe:@@ -1493,22 +1440,18 @@ def discover_image_caption_pairs(return pairs-# Families whose trainer has no checkpoint/resume support yet. The shared DiffusionLoraConfig-# carries save_steps / resume_from_checkpoint for every family, so a loop that implements-# neither has to say so rather than ignore them.+# The shared DiffusionLoraConfig carries save_steps / resume_from_checkpoint for every family, so a+# loop that implements neither has to say so rather than ignore them.CHECKPOINTLESS_FAMILIES: frozenset[str] = frozenset({"minimax-h3"})-# Families whose forward covers ONE packed sequence, so the batch axis is a pure replication-# axis and a second clip cannot join it: the layout, the rotary grid and the row timesteps are-# set by that clip's geometry and its caption's length. Kept beside the refusal it explains.+# The batch axis is a pure replication axis for these: the layout, the rotary grid and the row+# timesteps are set by one clip's geometry and its caption's length.SINGLE_SEQUENCE_FAMILIES: frozenset[str] = frozenset({"minimax-h3"})-# Families whose trainer loads its base through ``ModularPipeline.from_pretrained``. Their local-# layout is ``modular_model_index.json`` and no ``model_index.json``, so the conventional shape-# check in ``_assert_trusted_base_model`` refuses the only local form the family HAS. The trainer-# knew this and passed allow_modular itself; the START ROUTE runs the same gate first, and-# without the same answer a local H3 pipeline 400'd there before the trainer was ever reached.-# One set, read by both, so the two cannot drift into disagreeing about the same directory.+# Families whose trainer loads its base through ModularPipeline.from_pretrained. Their local+# layout is modular_model_index.json and no model_index.json, so the conventional shape check+# refuses the only local form the family HAS. One allow_modular set, read by the trainer and by+# the START ROUTE, so the two cannot disagree about the same directory.MODULAR_BASE_FAMILIES: frozenset[str] = frozenset({"minimax-h3"})# MiniMax-H3 canvas multiple: a 16x VAE compression and a 2x patch.@@ -1549,29 +1492,23 @@ def h3_train_unsupported_reason(cfg: Any) -> Optional[str]:"audio at different sigmas in the same step, so a single weight over 'the' ""timestep is ambiguous. Use weighting_scheme='none'.")- # The batch axis of an H3 forward is a pure replication axis: the layout, the rotary grid- # and the row timesteps describe ONE packed sequence that every batch item shares. Two- # clips with different captions have different text lengths and therefore different- # layouts, so a batch > 1 cannot be formed without padding the model has no mask for.+ # Two clips with different captions have different text lengths and therefore different layouts, so+ # a batch > 1 cannot be formed without padding the model has no mask for.if cfg.train_batch_size != 1:return ("MiniMax-H3 trains at batch size 1: one forward covers one packed sequence, whose ""row layout is set by the clip's own geometry and its caption's length. Use ""gradient_accumulation_steps to raise the effective batch.")- # torch.compile is never invoked here (the packed layout changes shape with every clip's- # caption length, so each step would recompile), and the run already reports compiled=False.- # "auto" means "the trainer decides", which it does; an explicit "on" is a policy that would- # be accepted and then ignored.+ # torch.compile is never invoked here (the packed layout changes shape with every caption length),+ # so an explicit "on" would be accepted and then ignored.if str(getattr(cfg, "compile_transformer", "auto") or "auto").strip().lower() == "on":return ("MiniMax-H3 does not compile: its packed sequence changes length with every clip's ""caption, so torch.compile would re-trace each step. Use compile_transformer ""'off' or 'auto'.")- # No conditioning cache exists on this path: the conditioner is loaded and every caption and- # latent recomputed each run. Accepting the directory would promise a saving that never- # happens, so an explicit one is refused rather than silently unused.+ # No conditioning cache exists on this path, so accepting the directory would promise a saving that never happens.if str(getattr(cfg, "cond_cache_dir", "") or "").strip():return ("MiniMax-H3 has no persistent conditioning cache yet: each run loads the "@@ -1581,17 +1518,9 @@ def h3_train_unsupported_reason(cfg: Any) -> Optional[str]:return None-# The SCHEMA defaults the MiniMax-H3 loop cannot honour, and the value it actually runs with.-# Unlike the fields ``h3_train_unsupported_reason`` refuses, these have a DEFAULT that the loop-# disagrees with, so refusing them would 422 every untouched request; they are normalised-# instead. center_crop/random_flip: every frame goes through the same centre cover-crop and-# nothing is flipped (a per-frame flip tears a clip, and a per-clip one has nowhere to live ---# the cached tensors carry no variant axis). snr_gamma: the step is a plain unweighted MSE.-# cache_latents/cache_variants: the loop encodes every clip once, up front, into exactly one-# cached (video_a, video_b, audio) tuple and then frees the VAEs, with no per-step encode path-# to fall back to and nowhere to put a second draw. It never reads either field, so a request-# asking for no cache, or for the schema's four variants, got one cache of one variant anyway-# and a run record claiming otherwise.+# These have a DEFAULT the H3 loop disagrees with, so they are normalised rather than refused+# (which would 422 every untouched request): one centre cover-crop, nothing flipped, a plain+# unweighted MSE, and exactly one cached tuple._H3_FIXED_RECIPE: dict[str, Any] = {"center_crop": True,"random_flip": False,@@ -1616,8 +1545,7 @@ def train_recipe_overrides(cfg: Any) -> dict[str, Any]:return dict(_H3_FIXED_RECIPE)-# Families whose dataset is captioned video CLIPS rather than stills. LTX-2 is deliberately not-# here: it trains a style LoRA FROM still images, so it keeps the image discovery.+# LTX-2 is deliberately not here: it trains a style LoRA FROM still images, so it keeps the image discovery.CLIP_TRAINED_FAMILIES: frozenset[str] = frozenset({"minimax-h3"})@@ -1684,11 +1612,12 @@ def _plan_cache_variants(return plan-# Host-memory budget for the AUTOMATIC latent cache. It holds two fp32 posterior tensors per crop/flip variant per image,-# so a few thousand images x cache_variants can exhaust pinned RAM; over budget it falls back to per-step VAE encoding.+# Two fp32 posterior tensors per crop/flip variant per image, so a few thousand images can exhaust+# pinned RAM; over budget it falls back to per-step VAE encoding._LATENT_CACHE_BUDGET_BYTES = 4 * 1024**3 # 4 GiB-# Returned by the cache builders when the estimate exceeds budget: the caller keeps the VAE resident. Distinct from ``None`` (a stop requested mid-build).+# Returned by the cache builders when the estimate exceeds budget: the caller keeps the VAE+# resident. Distinct from ``None`` (a stop requested mid-build).LATENT_CACHE_OVER_BUDGET: Any = object()@@ -1741,13 +1670,15 @@ def _apply_perf_flags(torch.backends.cudnn.allow_tf32 = Truetorch.set_float32_matmul_precision("high")else:- # The opt-out is a strict-fp32 A/B mode, so actively clear the flags rather than inherit ambient state (cudnn TF32 defaults ON).+ # The opt-out is a strict-fp32 A/B mode, so actively clear the flags rather than inherit ambient+ # state (cudnn TF32 defaults ON).torch.backends.cuda.matmul.allow_tf32 = Falsetorch.backends.cudnn.allow_tf32 = Falsetorch.set_float32_matmul_precision("highest")if cudnn_benchmark:torch.backends.cudnn.benchmark = True- # The cuDNN SDPA TRAINING graph is broken for the FLUX attention shapes on torch 2.10 + cu130 (B200) and poisons the context, so pin flash / mem-efficient SDPA for the run (restored on exit).+ # The cuDNN SDPA TRAINING graph is broken for the FLUX attention shapes on torch 2.10 + cu130 (B200)+ # and poisons the context, so pin flash / mem-efficient SDPA for the run (restored on exit).cuda_backends = getattr(torch.backends, "cuda", None)if cuda_backends is not None and hasattr(cuda_backends, "enable_cudnn_sdp"):try:@@ -1774,7 +1705,8 @@ def _restore_perf_flags(snap: Optional[dict]) -> None:if snap.get("matmul_precision"):torch.set_float32_matmul_precision(snap["matmul_precision"])- # Restore the exact pre-run cudnn SDPA state; None means the flag was unreadable at apply time and never touched.+ # Restore the exact pre-run cudnn SDPA state; None means the flag was unreadable at apply time and+ # never touched.cuda_backends = getattr(torch.backends, "cuda", None)if (snap.get("cudnn_sdp") is not None@@ -1786,28 +1718,25 @@ def _restore_perf_flags(snap: Optional[dict]) -> None:pass-# Official safetensors-only TRAINING bases trusted in addition to the inference allowlist. The FLUX.2 bases are in that-# list too but kept here so the training gate stays independent. Exact-match lowercased; never add pickles or remote code.+# Kept here so the training gate stays independent of the inference allowlist. Exact-match+# lowercased; never add pickles or remote code._TRAIN_EXTRA_TRUSTED_REPOS = frozenset({"black-forest-labs/flux.2-dev","black-forest-labs/flux.2-klein-4b","black-forest-labs/flux.2-klein-base-4b","black-forest-labs/flux.2-klein-base-9b",- # LTX-2's official base. It is a video family, so the image-side inference allowlist- # (_is_trusted_diffusion_repo) never covered it; safetensors-only, no remote code.+ # A video family, so the image-side inference allowlist never covered it; safetensors-only, no remote code."lightricks/ltx-2",# MiniMax-H3's official base, for the same reason: safetensors-only, no remote code."minimaxai/minimax-h3",})-# LTX-2.3 repos hold SINGLE-FILE checkpoints (ltx-2.3-22b-*.safetensors) and no diffusers layout:-# no model_index.json, no transformer/ or scheduler/ subfolder. Inference assembles them with-# from_single_file plus 2.3 config overrides and components borrowed from the 2.0 base (see-# core/inference/video_ltx2.py); the trainer only knows LTX2Pipeline.from_pretrained, which cannot-# read that layout. The name still resolves to the ``ltx-2`` family, so without an explicit refusal-# the run passes preflight, evicts the user's resident models, and only then fails in the child.+# LTX-2.3 repos hold SINGLE-FILE checkpoints and no diffusers layout; inference assembles them+# with from_single_file plus 2.3 config overrides (see core/inference/video_ltx2.py), while the+# trainer only knows LTX2Pipeline.from_pretrained. The name still resolves to the ltx-2 family, so+# without an explicit refusal the run evicts residents and only then fails in the child._LTX23_TRAIN_UNSUPPORTED = ("lightricks/ltx-2.3", "lightricks/ltx-2.3-fp8")@@ -1843,13 +1772,13 @@ def _assert_trusted_base_model(base_model: str, *, allow_modular: bool = False)f"Refusing to train from untrusted base model '{base_model}'. Use a local path or "f"a trusted repo (an unsloth/* repo or an official base).")- # An existing LOCAL base is loaded as a full pipeline, which needs an index; reject a non-pipeline local dir before /diffusion/start frees the GPU models.+ # An existing LOCAL base is loaded as a full pipeline, which needs an index; reject a non-pipeline+ # local dir before /diffusion/start frees the GPU models._assert_local_base_is_pipeline(base_model, allow_modular = allow_modular)+# One writer and one reader for BOTH trainers, so an SDXL and a DiT run resume from the same bundle shape.# ── resume checkpoints ────────────────────────────────────────────────────────-# One writer and one reader for BOTH trainers, so an SDXL and a DiT run resume from the same-# bundle shape. The family-specific part (the deployable adapter export) stays in the trainers.def trainable_state_dict(model: Any) -> dict[str, Any]:"""The trainable (LoRA) parameters of ``model``, keyed by parameter name.@@ -1892,11 +1821,8 @@ def load_trainable_state_dict(model: Any, state: Optional[dict[str, Any]]) -> in)p.copy_(saved.to(device = p.device, dtype = p.dtype))restored += 1- # BOTH directions. Counting only the checkpoint's own tensors proves every saved tensor- # landed somewhere; it says nothing about a live trainable parameter the checkpoint never- # had. A truncated or hand-edited adapter file holding a strict SUBSET therefore passed,- # and the full optimizer state was then loaded on top: restored Adam moments driving- # freshly initialised weights, while the run reported a clean resume.+ # BOTH directions: counting only the checkpoint's own tensors let a truncated adapter holding a+ # strict SUBSET pass, and the optimizer state then loaded on top of freshly initialised weights.unsaved = sorted(trainable - set(state))unknown = sorted(set(state) - trainable)if unsaved or unknown:@@ -1967,17 +1893,15 @@ def write_resume_checkpoint(progress = _json_safe_progress(progress),save_total_limit = int(cfg.save_total_limit or 0),discard_existing = discard_existing,- # What this run resumed from, so the "step already written" shortcut can tell- # "we are re-saving the bundle we started from" apart from "another run left a- # bundle at this number".+ # Which bundle THIS is, so the "step already written" shortcut can tell a re-save of our own source+ # apart from another run's bundle at the same number.source_checkpoint = getattr(cfg, "resume_from_checkpoint", None),- # Bundles that predated this run are never pruned to make room for its own: a- # branched resume beside higher-numbered checkpoints used to delete them on the- # first save, irreversibly.+ # Bundles that predated this run are never pruned to make room for its own: a branched resume used+ # to delete them irreversibly on the first save.preexisting = preexisting,)- # Reported per save, not only at the end: a run that later CRASHES must still be known to- # have resumable state (and a run whose write failed must still be known to be blocked).+ # Reported per save, so a run that later crashes is still known to have resumable state and one+ # whose write failed is still known to be blocked._emit(on_event, "checkpoint_saved", checkpoint_path = path, step = step)return path, Noneexcept Exception as exc: # noqa: BLE001 -- reported, never fatal to the run@@ -2055,19 +1979,15 @@ def restore_resume_state(load_trainable_state_dict(model, ckpt.tensors("adapter"))optimizer_state = ckpt.torch_state("optimizer")if optimizer_state is not None:- # The trainers pick their optimizer from the HOST (bitsandbytes present, a fused kernel- # available, UNSLOTH_DIFFUSION_FP32_OPTIM), not from the config, so a checkpoint can- # legitimately arrive with foreign moments: AdamW8bit stores "state1"/"state2", torch- # AdamW stores "exp_avg"/"exp_avg_sq". Shapes and counts match, so load_state_dict- # accepts them and the first step dies on a bare KeyError. Refuse with a real reason.+ # The trainers pick their optimizer from the HOST (bnb present, fused kernel available,+ # UNSLOTH_DIFFUSION_FP32_OPTIM), so foreign moments arrive legitimately (state1/state2 versus+ # exp_avg/exp_avg_sq): shapes match, load_state_dict accepts them, and the first step dies on a+ # bare KeyError.saved_optimizer = ckpt.optimizer_classlive_optimizer = optimizer_key(optimizer)if not saved_optimizer:- # This writer records the class whenever it writes moments, so an optimizer file- # with no class beside it is a hand-edited or half-written bundle -- and letting it- # through is the same failure the check below exists for: foreign moments load- # cleanly (shapes and counts match) and die on the first step, in the child, after- # the route preflight has already evicted the resident models.+ # An optimizer file with no class beside it is a hand-edited or half-written bundle, and foreign+ # moments load cleanly then die on the first step, after the preflight evicted the residents.raise ResumeError("This checkpoint does not record which optimizer wrote its state, so its ""moments cannot be safely restored. Resume from an earlier checkpoint, or "@@ -2079,11 +1999,9 @@ def restore_resume_state(f"machine builds {live_optimizer}. Install the same optimizer backend (or unset "f"UNSLOTH_DIFFUSION_FP32_OPTIM) to continue this run.")- # Optimizer state is keyed by parameter POSITION, so load_state_dict rebinds the saved- # moments onto whatever order this process built. The adapter tensors above were restored- # by NAME, so a PEFT/diffusers upgrade that changes traversal order while keeping the same- # names leaves the two disagreeing: many LoRA projections share a shape, so every moment- # loads cleanly onto the wrong tensor and the continued trajectory is silently corrupt.+ # Optimizer state is keyed by parameter POSITION while the adapter was restored by NAME, so a+ # PEFT/diffusers upgrade that changes traversal order loads every moment cleanly onto a+ # same-shaped wrong tensor.saved_names = ckpt.optimizer_param_nameslive_names = list(trainable_state_dict(model))if saved_names is not None and saved_names != live_names:@@ -2092,15 +2010,12 @@ def restore_resume_state("than this build produces, so its moments cannot be matched to this run's ""tensors. Start a new run, or resume on the version that wrote it.")- # load_state_dict replaces the param groups too, so the checkpoint's learning rate wins- # over a changed cfg.learning_rate -- the same semantics as HF Trainer's resume, and the- # UI only ever replays the original run's config anyway.+ # load_state_dict replaces the param groups too, so the checkpoint's learning rate wins over a+ # changed cfg, the same semantics as HF Trainer's resume.optimizer.load_state_dict(optimizer_state)elif optimizer is not None:- # The bundle lists no optimizer at all. Every bundle this writer produces for a real run- # has one, so this is a hand-edited or half-written directory -- and continuing would- # restart Adam's moments from zero at step N while reporting a clean resume, which looks- # like a resume and trains like a fresh run with a warm LR.+ # Every bundle this writer produces has an optimizer, so continuing without one restarts Adam's+ # moments from zero at step N while reporting a clean resume.raise ResumeError("This checkpoint carries no optimizer state, so the run cannot be continued from ""it: the moments would restart from zero at the resumed step. Start a new run."@@ -2110,18 +2025,14 @@ def restore_resume_state(lr_scheduler.load_state_dict(scheduler_state)_reapply_lr_schedule(optimizer, lr_scheduler)elif lr_scheduler is not None:- # Same for the schedule: a fresh LambdaLR at step 0 would re-warm the learning rate the- # restored optimizer already moved past.+ # A fresh LambdaLR at step 0 would re-warm the learning rate the restored optimizer already moved past.raise ResumeError("This checkpoint carries no learning-rate scheduler state, so the schedule would ""restart from step 0. Start a new run.")if sampler is not None and not sampler.load_state_dict(ckpt.sampler_state):- # Every image checkpoint this format writes carries sampler state, so a bundle whose- # state is missing or malformed is not one this code wrote. Continuing would leave a- # fresh sampler behind an RNG restored to a point AFTER the saved permutation was- # drawn, so the next batch is a different order: images silently skipped or repeated,- # under a resume that reported success.+ # Every image checkpoint carries sampler state, so a missing or malformed one leaves a fresh sampler+ # behind an RNG restored past the saved permutation: images silently skipped or repeated.raise ResumeError("This checkpoint's dataset sampler state is missing or unreadable, so the image ""order cannot be continued. Start a new run."@@ -2131,11 +2042,8 @@ def restore_resume_state(if ema_state:missing = ema.missing_from(ema_state)if missing:- # A readable but INCOMPLETE shadow set. load_state_dict keeps the freshly- # initialised shadow for anything it cannot match, so continuing here would- # blend restored EMA weights with initialisation noise for the rest -- and- # report a clean resume while doing it. There is no honest way to continue- # this run's EMA, so say so instead.+ # load_state_dict keeps the freshly initialised shadow for anything it cannot match, so an+ # incomplete set blends restored EMA weights with initialisation noise under a clean resume.named = ", ".join(missing[:3]) + ("..." if len(missing) > 3 else "")raise ResumeError(f"This checkpoint's EMA state is missing or mis-shaped for {len(missing)} "@@ -2144,11 +2052,8 @@ def restore_resume_state()ema.load_state_dict(ema_state, updates = ckpt.ema_updates)else:- # EMA is not part of the validated identity, so a resume may turn it on for a run- # that had it off. The EMA object is built BEFORE the adapter is restored, so its- # shadow holds freshly initialised LoRA weights and there is nothing saved to- # replace them with -- every later update, and the exported EMA adapter, would- # blend the restored weights with that initialisation. Start from what was restored.+ # EMA is not part of the validated identity, so a resume may turn it on: the object is built+ # BEFORE the adapter is restored, so its shadow holds freshly initialised weights.ema.reseed_from(model)restore_rng_state(ckpt.rng_json, ckpt.torch_state("rng"), rng_streams)_emit(@@ -2157,9 +2062,8 @@ def restore_resume_state(checkpoint_path = str(path),step = step,total_steps = cfg.train_steps,- # Which bundle THIS is, not just where it sat. Another run can write its own- # checkpoint-<N> over the same slot, and the pathname alone would then offer that- # replacement back as this run's lineage.+ # Which bundle THIS is, not just where it sat: another run can write its own checkpoint over the+ # same slot, and the pathname alone would offer that replacement back as this run's lineage.source_created_at = ckpt.manifest.get("created_at"),)return ckpt@@ -2243,11 +2147,13 @@ def _write_lora_sidecar(sidecar_path.write_text(json.dumps(meta, indent = 2), encoding = "utf-8")-# Aliases from the generic Unsloth training payload onto DiffusionLoraConfig fields, so the shared request shape can also drive this trainer.+# Aliases from the generic Unsloth training payload onto DiffusionLoraConfig fields, so the shared+# request shape can also drive this trainer._CONFIG_ALIASES = {"model_name": "base_model","max_steps": "train_steps",- # num_epochs already matches the diffusion field name, but list it so the epochs override is threaded through explicitly.+ # num_epochs already matches the diffusion field name, but list it so the epochs override is+ # threaded through explicitly."num_epochs": "num_epochs","batch_size": "train_batch_size","lora_r": "lora_rank",@@ -2287,8 +2193,8 @@ def _config_from_dict(config: dict) -> DiffusionLoraConfig:for k, v in config.items():if k in valid:kwargs[k] = v- # Epoch-mode payloads carry max_steps: 0 as the "use epochs" sentinel, which the alias copies as train_steps: 0.- # normalized() rejects train_steps < 1 before resolve_train_steps() applies num_epochs, so drop a falsy value here.+ # Epoch-mode payloads carry max_steps: 0 as the "use epochs" sentinel, and normalized() rejects+ # train_steps < 1 before resolve_train_steps() applies num_epochs.try:_num_epochs = int(kwargs.get("num_epochs") or 0)except (TypeError, ValueError):diff --git a/studio/backend/core/training/diffusion_train_extras.py b/studio/backend/core/training/diffusion_train_extras.pyindex be1ab7a24c8..89f6ab1aaff 100644--- a/studio/backend/core/training/diffusion_train_extras.py+++ b/studio/backend/core/training/diffusion_train_extras.py@@ -40,10 +40,11 @@ import refrom pathlib import Pathfrom typing import Any, Iterable, Optional++# Effective decay is min(decay, (1 + updates) / (WARMUP_OFFSET + updates)); at offset 10 step 1+# averages aggressively (~0.18) and the ramp reaches 0.99 after ~1000 updates.# ── LoRA EMA ──────────────────────────────────────────────────────────────────-# Warmup horizon for the EMA decay ramp: effective decay is min(decay, (1 + updates) / (WARMUP_OFFSET + updates)).-# With the offset at 10, step 1 averages aggressively (~0.18) and the ramp reaches 0.99 after ~1000 updates._EMA_WARMUP_OFFSET = 10.0@@ -234,7 +235,7 @@ def _hub_cache_roots() -> list[str]:longer changed this key and a warm run silently reused the old embeddings and latents. Theconstant stays as a fallback, since the trainer subprocess may run without Unsloth's settingsmodule importable."""- import os # noqa: PLC0415 — keep the module import list light for the subprocess+ import os # noqa: PLC0415 - keep the module import list light for the subprocessroots: list[str] = []try:@@ -256,12 +257,9 @@ def _hub_cache_roots() -> list[str]:return roots-# Pipeline subdirectories whose weights decide what the conditioning cache holds, so an in-place-# edit of any of them must change the fingerprint. text_encoder*/tokenizer* produce the embeddings-# and vae* the latents, for every family. "connectors" is LTX-2's: the Gemma3 hidden states are-# per-layer stacked and only the connector output reaches the transformer, so that projection ---# not the raw encoder state -- is what gets cached. No other supported family runs a module between-# encode_prompt and the DiT; the rest go straight from the text encoders to the cached tensors.+# An in-place edit of any of these must change the fingerprint: text_encoder*/tokenizer* produce+# the embeddings and vae* the latents. "connectors" is LTX-2's, the only family running a module+# between encode_prompt and the DiT: its connector output, not the raw Gemma3 state, is cached._CACHE_SOURCE_SUBDIRS = ("text_encoder", "tokenizer", "vae", "connectors")@@ -275,7 +273,7 @@ def source_revision(ref: Any) -> str:never touches the encoders, since the point of the cache is that a warm run doesnot load them."""- import os # noqa: PLC0415 — keep the module import list light for the subprocess+ import os # noqa: PLC0415 - keep the module import list light for the subprocesstry:name = str(ref or "").strip()if not name:@@ -313,7 +311,7 @@ def source_revision(ref: Any) -> str:if len(names) == 1:return f"rev-{names[0][:16]}"return "unresolved"- except Exception: # noqa: BLE001 — best-effort, never block a run+ except Exception: # noqa: BLE001 - best-effort, never block a runreturn "unresolved"@@ -332,7 +330,6 @@ class PersistentConditioningCache:self.resolution = int(resolution)self.root.mkdir(parents = True, exist_ok = True)- # -- keys --def latent_key(self,image_path: str,@@ -355,7 +352,6 @@ class PersistentConditioningCache:def has(self, key: str) -> bool:return self.path_for(key).is_file()- # -- IO --def put(self, key: str, tensors: Iterable[Any]) -> None:"""Store an ordered tuple of tensors (None entries allowed: their slotindices are recorded in the metadata so ``get`` restores them)."""@@ -397,12 +393,14 @@ class PersistentConditioningCache:return None+# Buckets snap to 64 pixels: the DiT families divide by 8 in the VAE and 2 again in latent patching,+# and regional torch.compile prefers few distinct shapes.# ── aspect-ratio bucketing ────────────────────────────────────────────────────-# Pixel-dimension divisor for bucket shapes: the DiT families divide by 8 in the VAE and 2 again in latent patching, and regional torch.compile prefers few distinct shapes, so buckets snap to 64 pixels.BUCKET_DIVISOR = 64-# Widest aspect ratio a bucket may take; anything more extreme clamps to it (matching the common practice of capping panoramas).+# Widest aspect ratio a bucket may take; anything more extreme clamps to it (matching the common+# practice of capping panoramas).MAX_BUCKET_RATIO = 2.0diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.pyindex 2202a1d696a..afdef557d60 100644--- a/studio/backend/core/training/diffusion_training_service.py+++ b/studio/backend/core/training/diffusion_training_service.py@@ -52,8 +52,8 @@ def _finite_or_none(value: Any) -> Optional[float]:def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) -> None:- # Fresh spawned interpreter: re-apply the OS-trust-store injection, inside- # the secret scrub and before the trainer imports diffusers.+ # Fresh spawned interpreter: re-apply the OS-trust-store injection, inside the secret scrub and+ # before the trainer imports diffusers.from utils.native_tls import activate_native_tlsactivate_native_tls()@@ -61,9 +61,8 @@ def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) -># Imported lazily so this module (and the route layer) stays torch-free at import.from .diffusion_lora_trainer import run_diffusion_training_process- # This child never runs LogConfig.setup_logging, so it installs the Hub default- # here; the diffusers half is done inside the two trainer entrypoints, which are- # the first points where diffusers is actually imported.+ # This child never runs LogConfig.setup_logging, so it installs the Hub default here; the diffusers+ # half is done inside the trainer entrypoints.try:from loggers.config import quiet_third_party_progress_barsquiet_third_party_progress_bars()@@ -74,14 +73,16 @@ def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) ->def _default_target(*, event_queue: Any, stop_queue: Any, config: dict) -> None:- # First thing in the child (before torch): self-bind to parent death and scrub the native path secret, like the other workers.+ # First thing in the child (before torch): self-bind to parent death and scrub the native path+ # secret, like the other workers.from utils.native_path_leases import run_without_native_path_secretrun_without_native_path_secret(_run_diffusion_child, event_queue = event_queue, stop_queue = stop_queue, config = config)-# Cap on retained metric points; over it, arrays are decimated so a long run stays bounded while the chart keeps its shape.+# Cap on retained metric points; over it, arrays are decimated so a long run stays bounded while the+# chart keeps its shape._METRIC_CAP = 4000@@ -103,8 +104,8 @@ def _llm_training_active() -> bool:return False+# One JSON file per terminal run, not the LLM sqlite, so diffusion runs stay off the LLM Runs page.# ── persisted run history ──────────────────────────────────────────────────────-# Every terminal run is recorded as one JSON file (summary + scrubbed config + metric logs) for the Train tab's history. JSON, not the LLM sqlite, so diffusion runs stay off the LLM Runs page.def _runs_dir() -> Path:from utils.paths.storage_roots import studio_root@@ -159,9 +160,8 @@ def _resume_fields(return {"can_resume": can_resume,"checkpoint_step": state.get("checkpoint_step"),- # The EXACT bundle this run would continue, so the client resumes the one it was shown- # rather than "whatever is newest in that folder" (which, in a folder two runs share,- # can be a different run's).+ # The EXACT bundle this run would continue, so the client resumes the one it was shown rather than+ # whatever is newest in a folder two runs share."checkpoint_path": state.get("checkpoint_path") if can_resume else None,"resume_blocked_reason": state.get("resume_blocked_reason"),}@@ -185,21 +185,18 @@ def _refresh_resume_state(rec: dict) -> dict:status = rec.get("status"),started_at = rec.get("started_at"),ended_at = rec.get("ended_at"),- # Same rule as the write side, and via the same helper: a run that died during- # model loading never got a "resumed" event to seed total_steps, and zero here- # sends the calculation back to the checkpoint manifest's OLDER target. In EPOCH- # mode train_steps is the request model's unused default, so recomputing from it- # here undid the persisted answer on every read -- a 600-step checkpoint of a run- # resolved to 1000 read as 600/500 and Resume was disabled.+ # A run that died during model loading never got a "resumed" event to seed total_steps, and zero+ # here sends the calculation back to the manifest's OLDER target, so a 600-step checkpoint read as+ # 600/500 and Resume was disabled.total_steps = _resolved_total_steps(rec, config if isinstance(config, dict) else {}),write_error = rec.get("checkpoint_write_error"),- # What this run itself resumed from, so a resumed run that died before its first- # save can still offer the bundle it was validated against.+ # What this run itself resumed from, so a resumed run that died before its first save can still+ # offer the bundle it was validated against.source_checkpoint = (config.get("resume_from_checkpoint") if isinstance(config, dict) else None),- # And WHICH bundle that was, so a slot another run has since rewritten is not- # offered back as this run's own lineage.+ # And WHICH bundle that was, so a slot another run has since rewritten is not offered back as this+ # run's own lineage.source_created_at = rec.get("resumed_source_created_at"),))@@ -222,15 +219,14 @@ def list_diffusion_runs(limit: int = 20) -> list[dict]:rec = json.loads(p.read_text(encoding = "utf-8"))except Exception: # noqa: BLE001 -- a corrupt record never breaks the listingcontinue- # Skip a wrong-shape record so one bad file cannot blow up the route's DiffusionTrainingRunSummary(**r) or the whole panel.+ # Skip a wrong-shape record so one bad file cannot blow up the route's+ # DiffusionTrainingRunSummary(**r) or the whole panel.if not isinstance(rec, dict):continueif not (isinstance(rec.get("job_id"), str) and isinstance(rec.get("status"), str)):continue- # Resume state comes from the checkpoints on disk NOW, before the config is dropped.- # Per record, because the refresh reads counters straight out of the file: one- # hand-edited or older record with a nonnumeric total_steps took the whole endpoint- # down, where every other kind of corruption here is skipped.+ # Resume state comes from the checkpoints on disk NOW, before the config is dropped. Per record,+ # because one hand-edited nonnumeric total_steps otherwise took the whole endpoint down.try:_refresh_resume_state(rec)_restate_live_job(rec)@@ -282,10 +278,8 @@ def _restate_live_job(rec: dict) -> dict:if rec.get("job_id") == live.get("job_id") and live.get("status") == "running":rec["status"] = "running"rec["message"] = live.get("message") or ""- # And nothing resumable, which is what the interim record's error status made- # _refresh_resume_state derive a moment ago. _UNRESUMABLE_STATUS rejects a running job- # for a reason: its output directory is being written right now, so the only thing a- # Resume action there can do is race the live run and 409.+ # _UNRESUMABLE_STATUS rejects a running job because its output directory is being written right now,+ # so a Resume there can only race the live run and 409.rec["can_resume"] = Falserec["checkpoint_path"] = Nonereturn rec@@ -317,16 +311,14 @@ def _idle_state() -> dict[str, Any]:"base_model": None,"samples_per_second": None,"peak_memory_gb": None,- # Resume state for this job: where its newest checkpoint-<N> bundle is, the step it holds,- # why one could not be written (surfaced as the Resume action's disabled tooltip), and the- # step a resumed run picked up from.+ # Where the newest checkpoint-<N> bundle is, the step it holds, why one could not be written (the+ # Resume action's disabled tooltip), and the step a resumed run picked up from."checkpoint_path": None,"checkpoint_step": None,"resume_blocked_reason": None,"resumed_from_step": None,- # The manifest timestamp of the bundle this run resumed FROM. A pathname is not an- # identity: another run can write over the same checkpoint-<N> slot, and the fallback- # would then offer that replacement back under this run's lineage.+ # A pathname is not an identity: another run can write over the same checkpoint-<N> slot, and the+ # fallback would offer that replacement back under this run's lineage."resumed_source_created_at": None,"started_at": None,"updated_at": None,@@ -335,16 +327,16 @@ def _idle_state() -> dict[str, Any]:"metric_loss": [],"metric_lr": [],"metric_grad_norm": [],- # The two halves of MiniMax-H3's joint objective. Null on every other family, and null- # on H3 steps that reported only the combined loss, so the series stay index-aligned.+ # Null on every other family, and on H3 steps that reported only the combined loss, so the series+ # stay index-aligned."metric_video_loss": [],"metric_audio_loss": [],}-# The value series, in append order, paired index-for-index with ``metric_steps``. One list so a-# new series cannot be added to the appends and forgotten in the decimation below, which would-# leave the curves silently misaligned rather than failing.+# The value series, in append order, paired index-for-index with metric_steps. One list so a new+# series cannot be added to the appends and forgotten in the decimation below, which would+# silently misalign the curves._METRIC_SERIES: tuple[str, ...] = ("metric_loss","metric_lr",@@ -380,8 +372,7 @@ def _append_metric(floss = _finite_or_none(loss)if floss is None: # non-numeric or non-finite: skip, keep the curve JSON-safereturn- # Every series but loss may be None or non-finite; non-finite is nulled (not dropped) to- # stay index-aligned.+ # Non-finite is nulled rather than dropped, to stay index-aligned.values = {"metric_loss": floss,"metric_lr": _finite_or_none(lr),@@ -397,8 +388,8 @@ def _append_metric(steps = state["metric_steps"]steps.append(istep)for key in _METRIC_SERIES:- # setdefault, not [key]: a run resumed from a record written before a series existed- # carries a state dict without it, and losing the whole point is worse than a short tail.+ # setdefault, not [key]: a run resumed from a record written before a series existed carries a state+ # dict without it.state.setdefault(key, []).append(values[key])@@ -431,29 +422,28 @@ class DiffusionTrainingService:self._ctx = ctx if ctx is not None else _CTXself._target = target if target is not None else _default_targetself._lock = threading.Lock()- # Set by reserve() while a start is in flight (before the route frees GPU models) so the load guards refuse a concurrent load. Cleared by unreserve().+ # Set by reserve() while a start is in flight (before the route frees GPU models) so the load guards+ # refuse a concurrent load. Cleared by unreserve().self._reserved = False- # Dataset mutations in flight. A start refuses while any is open and a mutation refuses once a start is reserved, both under _lock, so neither slips through the other's check-then-act window.+ # Dataset mutations in flight. A start refuses while any is open and a mutation refuses once a+ # start is reserved, both under _lock, so neither slips through the other's window.self._dataset_mutations = 0- # "Stop without saving" for the CURRENT job, remembered in the parent. The trainer- # reports it on its completion event, but a child that is killed or OOMs after the- # request never emits one -- and the unexpected-exit path then recorded a resumable- # error run, re-offering Resume from the very checkpoint the user asked to discard.+ # The trainer reports a no-save stop on its completion event, but a child killed or OOMed after+ # the request never emits one, and the unexpected-exit path then re-offered Resume from the+ # discarded checkpoint.self._discard_requested = False- # Bundle paths THIS job reported writing, so a discard the child could not carry out- # itself removes exactly those and nothing that predated the run.+ # Bundle paths THIS job reported writing, so a discard the child could not carry out removes exactly+ # those and nothing that predated the run.self._own_checkpoints: list[str] = []- # Set when the child reported a discarded completion, which it emits only AFTER running- # its own clear_own_checkpoints. That helper hands a displaced slot back, so the path- # this run wrote to can now hold ANOTHER run's restored bundle -- and the parent-side- # cleanup, which knows only pathnames, would delete it.+ # The child emits a discarded completion only after its own clear_own_checkpoints, which hands a+ # displaced slot back, so the parent's pathname-only cleanup would delete another run's bundle.self._child_cleared_own = False# Set by a checkpoint_saved event, cleared once the pump has written the record.self._persist_interim = False- # Whether this job has already had a stop signalled. Only the first one reaches the- # child, so only the first one may set the parent's disposition.+ # Only the first stop reaches the child, so only the first may set the parent's disposition.self._stop_signalled = False- # GPU load admissions in flight (a load between its training guard and its arbiter registration). Same two-sided rule as the dataset mutations.+ # GPU load admissions in flight (a load between its training guard and its arbiter registration).+ # Same two-sided rule as the dataset mutations.self._gpu_admissions = 0self._proc: Any = Noneself._stop_queue: Any = None@@ -490,14 +480,15 @@ class DiffusionTrainingService:"then start the run.")if self._gpu_admissions:- # A load already passed its training guard and is about to take the GPU. Reserving now would free residents it has not- # registered yet. Refusing is safe: the admission is held only across the registration, not the load itself.+ # A load past its training guard is about to take the GPU, and reserving now would free residents+ # it has not registered yet; refusing is safe, since admission is held only across registration.raise RuntimeError("A model is being loaded onto the GPU right now. Wait for that to finish, ""then start the run.")- # The LLM trainer under the SAME lock, not just at the route's earlier check: several network-bound preflights separate the- # two, so an LLM start could spawn in between. The LLM route holds gpu_load_admission() across its own spawn, so one of the two always raises.+ # Under the SAME lock as the LLM trainer, not just at the route's earlier check: several+ # network-bound preflights separate the two, and the LLM route holds gpu_load_admission() across+ # its spawn, so one of the two always raises.if _llm_training_active():raise RuntimeError("An LLM training job is already running. "@@ -569,15 +560,15 @@ class DiffusionTrainingService:Raises ValueError for an unusable config (before any spawn) and RuntimeError if ajob is already running. Returns the new job id."""- # Validate cheaply BEFORE spawning so a bad request fails fast with a clear error. The- # normalised config is kept: it carries the resolved family the recipe overrides below- # are keyed on, which the raw request dict does not have to name.+ # Validate before spawning, and keep the normalised config: it carries the resolved family the+ # recipe overrides are keyed on, which the raw request dict need not name.from .diffusion_lora_trainer import _config_from_dictfrom .diffusion_train_common import train_recipe_overridesnormalized_cfg = _config_from_dict(config).normalized()- # Join a finished job's pump OUTSIDE the lock: its final state writes take this lock, so joining under it would stall the start and let the stale pump overwrite the new state.+ # Join a finished job's pump OUTSIDE the lock: its final state writes take this lock, so joining+ # under it would stall the start and let the stale pump overwrite the new state.with self._lock:if self._proc is not None and self._proc.is_alive():raise RuntimeError("A diffusion training job is already running.")@@ -613,7 +604,7 @@ class DiffusionTrainingService:self._proc.start()try:from utils.process_lifetime import adopt_pid- adopt_pid(self._proc.pid) # bind to parent lifetime (no zombie on exit)+ adopt_pid(self._proc.pid)except Exception: # noqa: BLE001 -- lifetime binding is best-effortpass@@ -628,18 +619,12 @@ class DiffusionTrainingService:started_at = now,updated_at = now,)- # AFTER the reset above, which replaces the whole state dict. The route has already- # validated and PINNED a source bundle, so its identity is known before the child- # runs: recording it here means a resume that dies during the model load -- before- # the trainer can emit "resumed" -- still has the timestamp its fallback is checked- # against, instead of trusting the pathname and offering back whatever later- # occupies that slot.+ # AFTER the reset above: the route has already pinned a source bundle, so recording its identity+ # here means a resume that dies during the model load still has its timestamp to check against.self._seed_source_identity(config)- # Keep the config (minus secrets) for the persisted run record, with the fields this- # family's loop REPLACES rather than honours set to what it will actually run. The- # trainer applies the same table in the child, which the record is not written from,- # so without this Previous runs described a recipe (cropping, flipping, min-SNR- # weighting) that no step of the run ever used.+ # Record the config with the fields this family's loop REPLACES set to what it will actually run:+ # the trainer applies the same table in the child, so without this Previous runs described a+ # recipe no step ever used.self._config = {k: v for k, v in dict(config).items() if k != "hf_token"}self._config.update(train_recipe_overrides(normalized_cfg))self._pump = threading.Thread(@@ -657,12 +642,9 @@ class DiffusionTrainingService:if self._proc is None or not self._proc.is_alive() or self._stop_queue is None:return Falseif self._stop_signalled:- # The child consumes the FIRST signal and acts on it. A second one cannot- # un-save an adapter it has already exported, so honouring a later- # stop-without-saving set a parent discard the child never carried out: the- # run was marked discarded and its checkpoints deleted while the adapter and- # the catalog entry it published stayed on disk. The disposition is whichever- # one the child actually got.+ # The child consumes the FIRST signal, so honouring a later stop-without-saving set a parent+ # discard the child never carried out: the run was marked discarded and its checkpoints deleted+ # while the adapter and catalog entry stayed on disk.return Truetry:# Bare True = older wire format; the dict form carries the no-save cancel flag.@@ -671,8 +653,8 @@ class DiffusionTrainingService:return Falseself._stop_signalled = Trueif not save:- # Remembered here so a child that dies before its discarded completion still- # blocks the resume the user threw away.+ # Remembered here so a child that dies before its discarded completion still blocks the resume the+ # user threw away.self._discard_requested = Trueself._state["message"] = ("Stop requested; finishing the current step and saving a partial adapter..."@@ -726,9 +708,8 @@ class DiffusionTrainingService:self._persist_interim = Falseself._persist_run_record(interim = True)if ev.get("type") in _TERMINAL:- # An exception raised on the current step is a terminal `error`, and the pump- # returns here rather than through the dead-process branch -- so the discard the- # user asked for has to be applied on this path too.+ # An exception raised on the current step is a terminal error and returns here rather than through+ # the dead-process branch, so the discard has to be applied on this path too.with self._lock:discarding = self._discard_requested and self._proc is procchild_cleared = self._child_cleared_own@@ -826,10 +807,8 @@ class DiffusionTrainingService:"ema_path": s.get("ema_path"),"catalog_path": s.get("catalog_path"),"saved": bool(s.get("lora_path")),- # Resume lineage + state. output_dir is what a Resume replays, so it is recorded on- # the run rather than only inside the config. can_resume / checkpoint_step are a- # snapshot taken now and RE-DERIVED from disk on read, so deleting the checkpoints- # later takes the action away instead of leaving a button that 400s.+ # output_dir is what a Resume replays, so it is recorded on the run; can_resume / checkpoint_step+ # are re-derived from disk, so deleting the checkpoints takes the action away."output_dir": str(adapter) if adapter else None,"resumed_from_job_id": cfg.get("resumed_from_job_id") or None,"resumed_from_step": s.get("resumed_from_step"),@@ -839,18 +818,11 @@ class DiffusionTrainingService:str(adapter) if adapter else None,status = s.get("status"),started_at = s.get("started_at"),- # The REQUESTED target, falling back to the config the start preflight- # accepted: total_steps is seeded by the "resumed" event, and a run that- # dies during model loading never gets one. Zero there sent the resume- # calculation back to the checkpoint manifest's older target, which then- # reported "nothing left to train" for a run whose whole point was a- # raised target.- #- # NOT in epoch mode, though: num_epochs overrides train_steps, which then- # still carries the Pydantic default of 500. A run resolved to 1000 that- # died before its "resumed" event would report a 600-step checkpoint as- # 600/500 and refuse the resume. Zero there is honest, and the manifest's- # own target (written with the resolved count) is the right answer.+ # total_steps is seeded by the "resumed" event and a run that dies during model loading never gets+ # one; zero there sent the resume calculation back to the manifest's older target, reporting+ # "nothing left to train" for a run whose point was a raised target.+ # NOT in epoch mode: num_epochs overrides train_steps, which keeps its Pydantic default of 500, so a+ # run resolved to 1000 reported a 600-step checkpoint as 600/500 and refused the resume.total_steps = _resolved_total_steps(s, cfg),write_error = s.get("resume_blocked_reason"),source_checkpoint = cfg.get("resume_from_checkpoint"),@@ -893,7 +865,8 @@ class DiffusionTrainingService:elif etype == "model_load_completed":s.update(in_model_load = False, message = "Training...")elif etype == "preparing":- # A long precompute phase (e.g. the VAE latent cache) before the first step; surfaced so the UI shows progress instead of a silent stall.+ # A long precompute phase (e.g. the VAE latent cache) before the first step; surfaced so the UI+ # shows progress instead of a silent stall.done, total = ev.get("done"), ev.get("total")stage = str(ev.get("stage", "prepare")).replace("_", " ")s.update(@@ -909,27 +882,21 @@ class DiffusionTrainingService:# Non-fatal trainer notes; keep training state, surface the text.s["message"] = str(ev.get("message", "warning"))elif etype == "resumed":- # The trainer restored a checkpoint and is continuing from it. Recorded so the run- # history shows lineage instead of an unexplained run that starts mid-way. This is- # the step resumed FROM, not a checkpoint this run wrote, so it does not touch- # checkpoint_step (which tracks this run's own newest bundle).+ # This is the step resumed FROM, not a checkpoint this run wrote, so it does not touch checkpoint_step.s.update(resumed_from_step = ev.get("step"),resumed_source_created_at = ev.get("source_created_at"),message = f"Resuming from step {ev.get('step')}...",)- # Seed the LIVE counters from the same event. They are what the UI renders and- # what an errored run persists, and until the first post-resume progress event- # they still read 0/0 -- so a resume of step 400 of 500 shows "0/0", and an OOM- # on the first step records a failed run at step 0 with a target of 0.- # checkpoint_step stays untouched: this bundle is not one this run wrote.+ # Seed the LIVE counters from the same event: until the first post-resume progress they read 0/0, so+ # a resume of step 400 of 500 showed "0/0" and an OOM recorded a failed run at step 0 of 0.if ev.get("step") is not None:s["step"] = int(ev["step"])if ev.get("total_steps") is not None:s["total_steps"] = int(ev["total_steps"])elif etype == "checkpoint_saved":- # Folded as each bundle lands, not only at the end, so a run that later crashes is- # still reported as resumable. A good write also clears an earlier write's failure.+ # Folded as each bundle lands, not only at the end, so a run that later crashes is still reported as+ # resumable; a good write also clears an earlier failure.s.update(checkpoint_path = ev.get("checkpoint_path"),checkpoint_step = ev.get("step"),@@ -938,23 +905,15 @@ class DiffusionTrainingService:written = ev.get("checkpoint_path")if written and written not in self._own_checkpoints:self._own_checkpoints.append(str(written))- # The bundle is on disk; the run JSON is not, and only a terminal event writes- # one. Unsloth being killed or shut down after a periodic save therefore left a- # resumable checkpoint that Previous runs -- which reads those JSONs and- # nothing else -- had no entry for, so the Resume action did not exist. Written- # as an interrupted run because that is what it IS until the run ends: the- # terminal persist overwrites the same file, and the live job is reported as- # running by the reader below.+ # Only a terminal event writes the run JSON, so being killed after a periodic save left a resumable+ # checkpoint Previous runs had no entry for.self._persist_interim = Trueelif etype == "checkpoint_failed":- # Sticky: whatever older bundle is on disk predates the work this run did, so- # resuming from it would silently lose steps. Mirrors the MLX resume_blocked flag.+ # Sticky: an older bundle on disk predates the work this run did, so resuming from it would+ # silently lose steps. Mirrors the MLX resume_blocked flag.s["resume_blocked_reason"] = str(ev.get("message", "checkpoint write failed"))- # Persisted for the same reason a successful write is. In memory only, an Unsloth- # exit after this left the last record advertising the OLDER checkpoint as- # resumable -- the one this service has just decided is stale -- and resuming it- # rolls the run back past everything after it. A later checkpoint_saved clears- # the reason and replaces the record.+ # Persisted because, in memory only, an exit after this left the last record advertising the stale+ # older checkpoint as resumable.self._persist_interim = Trueelif etype == "progress":# Null any non-finite float so the JSON stays strict-parseable; a missing key keeps the last value.@@ -968,9 +927,8 @@ class DiffusionTrainingService:grad_norm = (_finite_or_none(ev["grad_norm"]) if "grad_norm" in ev else s["grad_norm"])- # The two halves of a joint objective, when the trainer reports them. Folded- # like the rest rather than dropped: on MiniMax-H3 the combined loss can hold- # steady while one modality degrades, and these are the only signal that says so.+ # Folded like the rest rather than dropped: on MiniMax-H3 the combined loss can hold steady while+ # one modality degrades, and these are the only signal that says so.video_loss = (_finite_or_none(ev["video_loss"]) if "video_loss" in ev else s["video_loss"])@@ -1005,7 +963,8 @@ class DiffusionTrainingService:ev.get("audio_loss"),)elif etype == "complete":- # Reset in_model_load: a stop during model load emits complete with no preceding model_load_completed, leaving a stale indicator.+ # Reset in_model_load: a stop during model load emits complete with no preceding+ # model_load_completed, leaving a stale indicator.s.update(active = False,in_model_load = False,@@ -1027,17 +986,13 @@ class DiffusionTrainingService:s["family"] = ev.get("family")if ev.get("base_model") is not None:s["base_model"] = ev.get("base_model")- # Checkpoint state is folded from the per-save events above; only the lineage is- # (re)confirmed here, for a pump that missed the earlier ``resumed`` event.+ # Only the lineage is re-confirmed here, for a pump that missed the earlier "resumed" event.if ev.get("resumed_from_step") is not None:s["resumed_from_step"] = ev.get("resumed_from_step")if ev.get("discarded"):- # Cancelled with "stop without saving": the user threw this run away, so its- # own periodic checkpoints must not keep offering to continue it.- #- # The child ran its own cleanup before emitting this, so the parent must not- # repeat it by pathname: a slot this run overwrote has been handed back to- # the bundle it displaced, and that one belongs to another run.+ # A discarded run's own periodic checkpoints must not keep offering to continue it.+ # The child ran its own cleanup before emitting this, so the parent must not repeat it by pathname:+ # a slot this run overwrote has been handed back to another run's bundle.self._child_cleared_own = Trues["resume_blocked_reason"] = ("This run was stopped without saving, so it was discarded."diff --git a/studio/backend/core/training/provenance.py b/studio/backend/core/training/provenance.pyindex 1b6480ef74c..b29444c28e7 100644--- a/studio/backend/core/training/provenance.py+++ b/studio/backend/core/training/provenance.py@@ -131,8 +131,8 @@ def exact_resume_requires_current_4bit(config: dict[str, Any]) -> bool:return Falseexcept Exception:return False- # The same disjunction effective_training_load_in_4bit tests: routes/training.py- # fills require_exact_model_resource from exact_resume_resource_requirements and+ # The same disjunction effective_training_load_in_4bit tests: routes/training.py fills+ # require_exact_model_resource from exact_resume_resource_requirements and# require_exact_resume_resources from resource_provenance_is_complete.return bool(requires_exact_model or resource_provenance_is_complete(config))diff --git a/studio/backend/core/training/resume.py b/studio/backend/core/training/resume.pyindex c018084abbd..681b398d332 100644--- a/studio/backend/core/training/resume.py+++ b/studio/backend/core/training/resume.py@@ -26,6 +26,7 @@ def _is_under_outputs(path: Path) -> bool:resolved.relative_to(root)return Trueexcept (OSError, RuntimeError, ValueError):+ # Unrecognized state-file formats are not usable resume state.return False@@ -88,7 +89,6 @@ def _valid_state_file(path: Path, require_tensor: bool = True) -> bool:and info.file_size > 0for info in infos)- # Unrecognized state-file formats are not usable resume state.return Falseexcept (OSError, ValueError, zipfile.BadZipFile):return False@@ -287,7 +287,8 @@ def can_resume_run(run: dict, *, resource_cache: Optional[dict[str, bool]] = Nonstatus = run.get("status")if status == "error":- # A save-time crash can report final_step == total_steps with no artifacts; checkpoint state alone decides resumability.+ # A save-time crash can report final_step == total_steps with no artifacts; checkpoint state alone+ # decides resumability.resume_state_available = has_resume_state(run.get("output_dir"))else:final_step = run.get("final_step")diff --git a/studio/backend/core/training/s3_dataset.py b/studio/backend/core/training/s3_dataset.pyindex e25b1479419..d1c7e92e2d9 100644--- a/studio/backend/core/training/s3_dataset.py+++ b/studio/backend/core/training/s3_dataset.py@@ -68,7 +68,7 @@ def _build_s3_client(s3_config: dict):Uses explicit access keys when provided, otherwise falls back to thedefault credential chain (IAM role / instance profile / env / shared creds)."""- import boto3 # lazy: optional dependency+ import boto3region = s3_config.get("region") or "us-east-1"use_iam_role = bool(s3_config.get("use_iam_role"))@@ -98,14 +98,14 @@ def _list_dataset_keys(client, bucket: str, prefix: Optional[str]) -> list[str]:for obj in page.get("Contents", []):key = obj["Key"]if key.endswith("/"):- continue # directory placeholder+ continueif os.path.basename(key).lower() in _IGNORED_METADATA_FILENAMES:continueif key.lower().endswith(SUPPORTED_EXTENSIONS):keys.append(key)- # A sync from a Mac uploads Finder metadata too, under the shard's own extension, and- # _validate_single_extension_family cannot see it. Nothing to read here, so a key is- # dropped only when the object it would describe is in the same listing.+ # A Mac sync uploads Finder metadata under the shard's own extension and+ # _validate_single_extension_family cannot see it, so a key is dropped only when the object it+ # would describe is in the same listing.return drop_shadowed_appledouble_names(keys)diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex 6fbc241a61a..be63f934464 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -11,13 +11,12 @@ import osimport sysimport types-# Off on Linux so datasets' forked map() workers can't deadlock. On spawn platforms-# map() runs in-process, so keep the fast tokenizer's Rust threads on.+# Off on Linux so datasets' forked map() workers cannot deadlock; on spawn platforms map() runs in-+# process, so the fast tokenizer's Rust threads stay on.os.environ["TOKENIZERS_PARALLELISM"] = "true" if sys.platform in ("win32", "darwin") else "false"-# Make compiled cache modules importable by any subprocess: spawned dataset.map()-# workers re-import top-level modules whose cache files import torch + unsloth_zoo.-# Do NOT import unsloth_zoo.compiler here -- it pulls in heavy torch/triton imports.+# Spawned dataset.map() workers re-import top-level modules whose cache files import torch ++# unsloth_zoo. Do NOT import unsloth_zoo.compiler here: it pulls in heavy torch/triton imports.if sys.platform in ("win32", "darwin"):_compile_cache = os.environ.get("UNSLOTH_COMPILE_LOCATION", "unsloth_compiled_cache")if not os.path.isabs(_compile_cache):@@ -94,8 +93,7 @@ from .dataset_bounds import bound_dataset_rows, world_size_env_report, world_sizlogger = get_logger(__name__)-# Streaming eval has no __len__, so an unbounded eval would iterate the whole-# source on every eval step. Cap it so each evaluation terminates.+# Streaming eval has no __len__, so cap it or an unbounded eval iterates the whole source on every eval step.STREAMING_EVAL_MAX_SAMPLES = 500@@ -186,9 +184,8 @@ def _spark_tts_tokenizer_kwargs(audio_type: Optional[str], lookup_name: str) ->name = str(lookup_name).replace("\\", "/").rstrip("/")if name.endswith("/LLM"):return {}- # An LLM/ child is the canonical layout, and a cache-pinned or offline snapshot root has- # one too. Treating that as "already at the tokenizer" sent AutoTokenizer at the root,- # which holds no tokenizer, and failed the very case this helper exists for.+ # A cache-pinned or offline snapshot root also has an LLM/ child, and treating that as "already at+ # the tokenizer" sent AutoTokenizer at a root that holds none.if os.path.isdir(os.path.join(lookup_name, "LLM")):return {"subfolder": "LLM"}# A local checkpoint that carries its own tokenizer is already the right directory.@@ -197,12 +194,11 @@ def _spark_tts_tokenizer_kwargs(audio_type: Optional[str], lookup_name: str) ->return {"subfolder": "LLM"}-# Enough rows to see past a leading null or malformed value, few enough to stay cheap on a-# streamed dataset, where each row is pulled over the network.+# Enough rows to see past a leading null or malformed value, few enough to stay cheap on a streamed dataset._AUDIO_SNIFF_ROWS = 16-# Kept in step with detect_multimodal_dataset, so the veto answers False only about the very-# columns that set the flag it is vetoing.+# Kept in step with detect_multimodal_dataset, so the veto answers False only about the very columns+# that set the flag it is vetoing._AUDIO_COLUMN_KEYWORDS = ("audio", "speech", "wav", "waveform", "sound")@@ -237,14 +233,9 @@ def _dataset_has_audio_column(dataset) -> Optional[bool]:except Exception: # noqa: BLE001 - a mapping that is not a features dictreturn None- # No Audio feature, so nothing decodes here and reading rows cannot hit the- # missing-FFmpeg path. Several rows, not one: a JSON/CSV audio dataset can carry a null- # or malformed first value and real paths after it, and the preprocessors skip such a- # row rather than fail, so row 0 alone would call it textual.- #- # Audio-like values count anywhere, but only an audio-NAMED column can answer False: a- # transcript is populated in every row of a path-backed audio dataset, so counting it- # would report "no audio here" on the strength of the text. No evidence stays None.+ # Several rows, not one: a JSON/CSV audio dataset can carry a null or malformed first value and real paths after it.+ # Only an audio-NAMED column can answer False: a transcript is populated in every row of a path-+ # backed audio dataset. No evidence stays None.try:from utils.datasets.format_detection import (_AUDIO_EXTENSIONS,@@ -297,17 +288,16 @@ class UnslothTrainer:self.save_on_stop = Trueself.load_in_4bit = True- self.is_cpt = False # Continued Pretraining+ self.is_cpt = Falseself.is_vlm = Falseself.is_audio = False- self.is_audio_vlm = False # e.g. Gemma 3N trained on audio- self._audio_type = None # 'csm', 'whisper', 'snac', 'bicodec', 'dac'- # True until a probe says otherwise, so a path that never probes cannot trip the- # inconclusive-detection guard.+ self.is_audio_vlm = False+ self._audio_type = None+ # True until a probe says otherwise, so a path that never probes cannot trip the inconclusive-detection guard.self._audio_type_known = Trueself._is_dataset_audio = False- self._cuda_audio_used = False # Set once after audio CUDA preprocessing; never cleared- self._spark_tts_repo_dir = None # Spark-TTS repo path, for BiCodecTokenizer+ self._cuda_audio_used = False+ self._spark_tts_repo_dir = Noneself._spark_tts_code_dir = Noneself._outetts_code_dir = Noneself.model_name = None@@ -350,10 +340,8 @@ class UnslothTrainer:if hf_token:os.environ["HF_TOKEN"] = hf_token- # Detect audio type (config.json only, no VRAM). Checked, because an unreadable- # tokenizer_config.json (gated repo, offline, or a cache holding weights but not the- # tokenizer files) otherwise reads as "not an audio model" and sends a TTS run down- # the text path.+ # Checked, because an unreadable tokenizer_config.json (gated, offline, or a weights-only cache)+ # otherwise reads as "not an audio model" and sends a TTS run down the text path.self._audio_type, self._audio_type_known = detect_audio_type_checked(lookup_name,hf_token,@@ -392,8 +380,8 @@ class UnslothTrainer:self.is_vlm,)- # Load tokenizer/processor (CPU only, no VRAM)- # Whisper needs AutoProcessor; others AutoTokenizer (CSM loads its own inline).+ # Load tokenizer/processor (CPU only, no VRAM) Whisper needs AutoProcessor; others AutoTokenizer (CSM+ # loads its own inline).def _load_tokenizer_for_detected_type():if self._audio_type == "whisper":from transformers import AutoProcessor@@ -412,17 +400,14 @@ class UnslothTrainer:token = hf_token,local_files_only = local_files_only,revision = model_revision,- # Spark-TTS keeps its tokenizer under LLM/, the subfolder _load_model- # loads weights from. Reading the root finds no vocab and fails blaming- # a missing sentencepiece that is installed and irrelevant.+ # Spark-TTS keeps its tokenizer under LLM/; reading the root finds no vocab and fails blaming a+ # missing sentencepiece that is installed and irrelevant.**_spark_tts_tokenizer_kwargs(self._audio_type, lookup_name),)- # The probe and the tokenizer load are separate reads of the same repo, so a transient- # timeout or 5xx can leave the probe inconclusive while the read after it succeeds.- # Ask again rather than refuse the run later claiming tokenizer_config.json could not- # be read when it just was. detect_audio_type_checked caches only definitive results,- # so this costs a cache hit on the common path.+ # The probe and the tokenizer load are separate reads of the same repo, so a transient 5xx can+ # leave the probe inconclusive while the read after it succeeds; detect_audio_type_checked caches+ # only definitive results.def _retry_audio_detection() -> bool:"""Re-probe an inconclusive type. True if the answer changed."""if self._audio_type_known:@@ -437,6 +422,7 @@ class UnslothTrainer:if not retried_known:return Falseself._audio_type, self._audio_type_known = retried_type, True+ # audio_vlm is detected as an audio_type now; handle separatelyif self._audio_type == "audio_vlm":self.is_audio = Falseself.is_audio_vlm = is_dataset_audio@@ -451,10 +437,8 @@ class UnslothTrainer:)return self._audio_type != previous_type- # An inconclusive probe leaves _audio_type None and the kwargs are chosen from it, so- # a Spark-TTS repo is read at its tokenizer-less root and raises here. Re-probe before- # giving up, or the retry below is unreachable for exactly the models whose layout the- # type decides.+ # An inconclusive probe leaves _audio_type None, so a Spark-TTS repo is read at its tokenizer-less+ # root; re-probe, or the retry below is unreachable for exactly those models.try:_load_tokenizer_for_detected_type()except Exception:@@ -469,9 +453,8 @@ class UnslothTrainer:logger.info("Pre-loaded tokenizer for %s", model_name)- # The loader and its kwargs are chosen from the type, so a retry that changes the- # answer has left the wrong object in self.tokenizer: whisper needs an AutoProcessor- # (_preprocess_whisper_dataset reads .feature_extractor and .tokenizer off it), and+ # A retry that changes the answer leaves the wrong object in self.tokenizer: whisper needs an+ # AutoProcessor (_preprocess_whisper_dataset reads .feature_extractor and .tokenizer off it) and# Spark-TTS its LLM/ subfolder.if _retry_audio_detection():_load_tokenizer_for_detected_type()@@ -520,10 +503,8 @@ class UnslothTrainer:trainer_ref = selfclass _ProgressCallback(TrainerCallback):- # Per-batch evaluation progress used to exist only as HF's tqdm bar, which- # this PR drops. A long eval would otherwise look stalled between the last- # step log and the eval result, so it is republished, throttled, as status- # and one structured line.+ # Republished, throttled: without HF's tqdm bar a long eval looks stalled between the last step log+ # and the eval result._eval_seen = 0_eval_last_report = 0.0@@ -538,8 +519,8 @@ class UnslothTrainer:had_status = cls._eval_last_report > 0.0cls._eval_seen = 0cls._eval_last_report = 0.0- # An empty status is ignored downstream on purpose, so the UI would- # sit on "Evaluating..." for the rest of the run.+ # An empty status is ignored downstream on purpose, so the UI would sit on "Evaluating..." for the+ # rest of the run.if had_status and not trainer_ref.should_stop:trainer_ref._update_progress(status_message = "Training in progress...")@@ -556,7 +537,7 @@ class UnslothTrainer:total = Nonetry:total = len(eval_dataloader) if eval_dataloader is not None else None- except TypeError: # an iterable-style loader has no length+ except TypeError:total = Nonenow = time.time()if cls._eval_last_report and (now - cls._eval_last_report) < 15.0:@@ -576,11 +557,9 @@ class UnslothTrainer:):if not logs:return- # Trainer's end-of-run and end-of-eval summaries carry numbers that- # appear nowhere else in Unsloth (train_samples_per_second,- # train_steps_per_second, total_flos, memory, eval runtime). They used- # to reach the log only through PrinterCallback's raw stdout dict, so- # with that callback gone they are re-published here, structured, once.+ # Trainer's end-of-run and end-of-eval summaries carry numbers that appear nowhere else+ # (train_samples_per_second, train_steps_per_second, total_flos, memory, eval runtime) and used to+ # reach the log only through PrinterCallback's raw stdout.if any(k in logs for k in _TRAINER_SUMMARY_KEYS):logger.info("trainer summary",@@ -590,12 +569,8 @@ class UnslothTrainer:if isinstance(k, str) and k not in _RESERVED_LOG_KEYS},)- # HF logs the end-of-run summary as {"train_runtime": ..., "train_loss": ...}- # with no "loss" key, and `train_loss` is the MEAN loss over the whole run,- # not a step loss. Falling back to it reported the average as the final- # step's value: it landed on the chart at the same global_step as the real- # last step and became `final_loss` on /api/train/runs, disagreeing with the- # checkpoint. Keep reporting it, as the summary it is.+ # train_loss is the MEAN loss over the whole run, so falling back to it put the average on the chart+ # at the last step and into final_loss, disagreeing with the checkpoint.loss_value = logs.get("loss")is_run_summary = loss_value is None and (logs.get("train_loss") is not None or logs.get("train_runtime") is not None@@ -692,8 +667,7 @@ class UnslothTrainer:"seed": random_seed,"output_dir": output_dir,"report_to": _build_report_targets(training_args),- # The subprocess stdout is teed into the server log, so HF's bar is- # noise there; --verbose restores it. See _hf_stdout_progress_disabled.+ # The subprocess stdout is teed into the server log, so HF's bar is noise there; --verbose restores it."disable_tqdm": _hf_stdout_progress_disabled(),}@@ -770,7 +744,6 @@ class UnslothTrainer:sys.path.remove(path)removed_paths.append(path)- # Remove stale audio modules from sys.modulesprefixes = ("snac", "whisper", "sparktts", "outetts")removed_modules = [key for key in sys.modules if key.startswith(prefixes)]for key in removed_modules:@@ -813,7 +786,6 @@ class UnslothTrainer:"speaker_col": speaker_col,}- # Hardcoded fallbackaudio_col = next((c for c in cols if c.lower() in ("audio", "speech")), None)text_col = next((c for c in cols if c.lower() in ("text", "sentence", "transcript", "transcription")),@@ -849,8 +821,8 @@ class UnslothTrainer:model_revision: Optional[str] = None,) -> bool:"""Load model for training (supports both text and vision models)"""- self.load_in_4bit = load_in_4bit # For training_meta.json- self.trust_remote_code = trust_remote_code # For AutoProcessor etc. used during training+ self.load_in_4bit = load_in_4bit+ self.trust_remote_code = trust_remote_codelookup_name = model_load_name or model_nameself.model_load_error = Nonetry:@@ -868,8 +840,8 @@ class UnslothTrainer:# Clear audio sys.path/sys.modules state; stale entries deadlock forked map() workersself._cleanup_audio_artifacts()- # Reload Unsloth-patched modeling modules before clearing the cache: __UNSLOTH_PATCHED__- # blocks re-compilation, so clearing the disk cache alone would leave files missing.+ # Reload the Unsloth-patched modeling modules first: __UNSLOTH_PATCHED__ blocks re-compilation, so+ # clearing the disk cache alone would leave files missing.import importlibfor _key, _mod in list(sys.modules.items()):@@ -878,15 +850,13 @@ class UnslothTrainer:try:importlib.reload(_mod)except Exception:- pass # Non-critical — Unsloth handles stale modules+ pass- # Remove stale compiled cache so the new model gets a fresh onefrom utils.cache_cleanup import clear_unsloth_compiled_cache_preserve = ["Unsloth*Trainer.py"] if sys.platform in ("win32", "darwin") else Noneclear_unsloth_compiled_cache(preserve_patterns = _preserve)- # Detect audio type (config.json + tokenizer). Checked: this reassigns- # _audio_type, so an unchecked answer would leave the flag describing the+ # Checked: this reassigns _audio_type, so an unchecked answer would leave the flag describing the# previous probe.self._audio_type, self._audio_type_known = detect_audio_type_checked(lookup_name,@@ -895,10 +865,9 @@ class UnslothTrainer:revision = model_revision,)self._is_dataset_audio = is_dataset_audio- # audio_vlm is detected as an audio_type now; handle separatelyif self._audio_type == "audio_vlm":self.is_audio = False- self.is_audio_vlm = is_dataset_audio # Only use audio VLM path if dataset has audio+ self.is_audio_vlm = is_dataset_audioself._audio_type = Noneelse:self.is_audio = self._audio_type is not None@@ -907,7 +876,6 @@ class UnslothTrainer:if not self.is_audio and not self.is_audio_vlm:self._cuda_audio_used = False- # VLM: vision model + image dataset (mutually exclusive with audio)vision = (is_vision_model(lookup_name,@@ -987,16 +955,15 @@ class UnslothTrainer:f"Using device_map='{device_map}' ({get_visible_gpu_count()} GPU(s) visible)")- # ROCm without native bf16 (e.g. RDNA2/gfx103x) dies with an LLVM error on the first- # bf16 kernel when dtype=None picks bf16, so force float16. NVIDIA keeps None so- # unsloth's auto-detect is honored -- T4/V100 must NOT be coerced to float16.+ # ROCm without native bf16 (e.g. RDNA2/gfx103x) dies with an LLVM error on the first bf16 kernel+ # when dtype=None picks bf16, so force float16; NVIDIA keeps None so T4/V100 are not coerced._is_rocm = (bool(getattr(torch.version, "hip", None)) or "rocm" in torch.__version__.lower())_auto_dtype = torch.float16 if (_is_rocm and not is_bfloat16_supported()) else Noneif self._audio_type == "csm":- # CSM: FastModel, auto_model=CsmForConditionalGeneration, load_in_4bit=False+ # Whisper: FastModel, auto_model=WhisperForConditionalGeneration, load_in_4bit=Falsefrom unsloth import FastModelfrom transformers import CsmForConditionalGeneration@@ -1016,7 +983,6 @@ class UnslothTrainer:logger.info("Loaded CSM audio model")elif self._audio_type == "whisper":- # Whisper: FastModel, auto_model=WhisperForConditionalGeneration, load_in_4bit=Falsefrom unsloth import FastModelfrom transformers import WhisperForConditionalGeneration@@ -1034,7 +1000,6 @@ class UnslothTrainer:revision = model_revision,use_exact_model_name = model_revision is not None,)- # Generation settings (notebook lines 100-105)self.model.generation_config.language = "<|en|>"self.model.generation_config.task = "transcribe"self.model.config.suppress_tokens = []@@ -1058,13 +1023,17 @@ class UnslothTrainer:logger.info(f"Loaded {self._audio_type} audio model (FastLanguageModel)")elif self._audio_type == "bicodec":- # Spark-TTS: download the full repo (sparktts + BiCodec weights), load only the LLM- # subfolder. model_name is "Spark-TTS-0.5B/LLM" (YAML mapping) or "unsloth/Spark-TTS-0.5B".+ # Download the full repo (sparktts + BiCodec weights) but load only the LLM subfolder; model_name is+ # "Spark-TTS-0.5B/LLM" or "unsloth/Spark-TTS-0.5B".+ # Use FastModel.get_peft_model (codec audio + audio VLM)+ # Audio VLM (e.g. Gemma 3N): FastModel returns (model, processor).+ # OuteTTS: uses FastModel (not FastLanguageModel) with load_in_4bit=False+ # Spark-TTS: download the full repo (sparktts + BiCodec weights), load only the LLM subfolder.+ # model_name is "Spark-TTS-0.5B/LLM" (YAML mapping) or "unsloth/Spark-TTS-0.5B".from unsloth import FastModelfrom huggingface_hub import snapshot_downloadif model_name.endswith("/LLM"):- # "Spark-TTS-0.5B/LLM" → repo "unsloth/Spark-TTS-0.5B"hf_repo = f"unsloth/{model_name.rsplit('/', 1)[0]}"else:hf_repo = model_name@@ -1076,7 +1045,7 @@ class UnslothTrainer:hf_repo,revision = model_revision,)- self._spark_tts_repo_dir = os.path.abspath(repo_path) # Absolute for sys.path+ self._spark_tts_repo_dir = os.path.abspath(repo_path)llm_path = os.path.join(self._spark_tts_repo_dir, "LLM")self.model, self.tokenizer = FastModel.from_pretrained(@@ -1093,7 +1062,6 @@ class UnslothTrainer:logger.info("Loaded Spark-TTS (bicodec) model")elif self._audio_type == "dac":- # OuteTTS: uses FastModel (not FastLanguageModel) with load_in_4bit=Falsefrom unsloth import FastModelself.model, self.tokenizer = FastModel.from_pretrained(lookup_name,@@ -1109,7 +1077,6 @@ class UnslothTrainer:logger.info("Loaded OuteTTS (dac) model (FastModel)")elif self.is_audio_vlm:- # Audio VLM (e.g. Gemma 3N): FastModel returns (model, processor).from unsloth import FastModelself.model, self.tokenizer = FastModel.from_pretrained(model_name = lookup_name,@@ -1140,7 +1107,6 @@ class UnslothTrainer:)logger.info("Loaded vision model")- # Did FastVisionModel return a Processor or a raw tokenizer?from transformers import ProcessorMixintok = self.tokenizer@@ -1184,7 +1150,6 @@ class UnslothTrainer:return Falseif full_finetuning:- # All params trainable (else frozen)self.model.for_training()self._update_progress(status_message = "Model loaded successfully")@@ -1196,8 +1161,8 @@ class UnslothTrainer:if "could not get source code" in str(e) and not getattr(self, "_source_code_retried", False):- # Unsloth patching can leave stale state that breaks inspect.getsource() when- # switching model families (e.g. gemma3 -> gemma3n); the first failure clears it.+ # Unsloth patching can leave stale state that breaks inspect.getsource() when switching model+ # families; the first failure clears it.self._source_code_retried = Truelogger.info(f"\n'could not get source code' — retrying once...\n")return self.load_model(@@ -1238,7 +1203,6 @@ class UnslothTrainer:except Exception as e:self.model_load_error = eerror_msg = str(e)- # Surface a friendly message for gated/auth errorserror_lower = error_msg.lower()if any(k in error_lower@@ -1264,7 +1228,6 @@ class UnslothTrainer:def prepare_model_for_training(self,use_lora: bool = True,- # Vision-specific LoRA parameters (only used if is_vlm=True)finetune_vision_layers: bool = True,finetune_language_layers: bool = True,finetune_attention_modules: bool = True,@@ -1286,7 +1249,6 @@ class UnslothTrainer:if self.model is None:raise ValueError("Model not loaded. Call load_model() first.")- # Full finetuning: skip PEFTif not use_lora:self._update_progress(status_message = "Full finetuning mode - no LoRA adapters")logger.info("Full finetuning mode - training all parameters\n")@@ -1311,7 +1273,6 @@ class UnslothTrainer:"down_proj",]- # Normalize gradient_checkpointing to True, False, or "unsloth"if isinstance(use_gradient_checkpointing, str):use_gradient_checkpointing = use_gradient_checkpointing.strip().lower()if use_gradient_checkpointing == "" or use_gradient_checkpointing == "unsloth":@@ -1351,12 +1312,12 @@ class UnslothTrainer:)if self._audio_type in ("csm", "bicodec", "dac") or self.is_audio_vlm:- # Use FastModel.get_peft_model (codec audio + audio VLM)from unsloth import FastModellabel = self._audio_type or "audio_vlm"logger.info(f"{label} LoRA configuration:")logger.info(f" - Target modules: {target_modules}")+ # Audio VLM models support VLM-style layer selectionif self.is_audio_vlm:logger.info(f" - Finetune vision layers: {finetune_vision_layers}")logger.info(f" - Finetune language layers: {finetune_language_layers}")@@ -1377,7 +1338,6 @@ class UnslothTrainer:use_dora = use_dora,loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,)- # Audio VLM models support VLM-style layer selectionif self.is_audio_vlm:peft_kwargs.update(finetune_vision_layers = finetune_vision_layers,@@ -1514,7 +1474,7 @@ class UnslothTrainer:CsmOutputWithPast,)- base_csm = self.model.base_model.model # CsmForConditionalGeneration+ base_csm = self.model.base_model.model# Original forward (@can_return_tuple wrapped version)_original_forward = CsmForConditionalGeneration.forward@@ -1681,8 +1641,8 @@ class UnslothTrainer:trust_remote_code = getattr(self, "trust_remote_code", False),)- # Some fine-tuned models save pad_to_multiple_of in tokenizer_config.json and- # _merge_kwargs leaks it into audio_kwargs, where EncodecFeatureExtractor rejects it.+ # Some fine-tuned models save pad_to_multiple_of in tokenizer_config.json and _merge_kwargs leaks it+ # into audio_kwargs, where EncodecFeatureExtractor rejects it.processor.tokenizer.init_kwargs.pop("pad_to_multiple_of", None)resolved = self._resolve_audio_columns(dataset, custom_format_mapping)@@ -1802,10 +1762,8 @@ class UnslothTrainer:f"Audio VLM dataset needs 'audio' and 'text' columns, got: {dataset.column_names}")- # Needed by the collator closureself._audio_vlm_audio_col = audio_col- # Cast audio to 16kHz (standard for speech models)dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 16000))def format_messages(samples):@@ -1867,7 +1825,6 @@ class UnslothTrainer:max_length = self.max_seq_length or 2048tokenizer = self.tokenizer- # Orpheus special token IDs (hardcoded in tokenizer vocabulary)START_OF_HUMAN = 128259END_OF_HUMAN = 128260START_OF_AI = 128261@@ -1907,7 +1864,6 @@ class UnslothTrainer:snac_model = SNAC.from_pretrained(SNAC_MODEL_NAME)snac_model = snac_model.to(device).eval()- # Resample transform (created once)resample_transform = (T.Resample(orig_freq = ds_sample_rate, new_freq = SNAC_SAMPLE_RATE)if ds_sample_rate != SNAC_SAMPLE_RATE@@ -1939,7 +1895,6 @@ class UnslothTrainer:skipped += 1continue- # Encode audio with SNAC (notebook 122-142)waveform = (torch.from_numpy(audio_data["array"]).unsqueeze(0).to(dtype = torch.float32))@@ -1950,7 +1905,6 @@ class UnslothTrainer:with torch.inference_mode():codes = snac_model.encode(waveform)- # Interleave 7 codes per frame with layer offsets (notebook 134-142)all_codes = []for i in range(codes[0].shape[1]):all_codes.append(codes[0][0][i].item() + AUDIO_OFFSET)@@ -1965,14 +1919,12 @@ class UnslothTrainer:skipped += 1continue- # Dedup consecutive frames with same first code (notebook 185-207)deduped = all_codes[:7]for i in range(7, len(all_codes), 7):if all_codes[i] != deduped[-7]:deduped.extend(all_codes[i : i + 7])all_codes = deduped- # Build text tokens (notebook 217-224)text_prompt = (f"{example[speaker_col]}: {text}"if has_source and example.get(speaker_col)@@ -1981,7 +1933,6 @@ class UnslothTrainer:text_ids = tokenizer.encode(text_prompt, add_special_tokens = True)text_ids.append(END_OF_TEXT)- # Build full input_ids (notebook 225-234)input_ids = ([START_OF_HUMAN]+ text_ids@@ -2073,8 +2024,8 @@ class UnslothTrainer:f"BiCodec dataset needs 'audio' and 'text' columns, got: {dataset.column_names}")- # Cast so datasets 4.x AudioDecoder objects decode to dicts. No resample here --- # BiCodec's target_sr may differ; the loop does it.+ # Cast so datasets 4.x AudioDecoder objects decode to dicts; no resample here, since BiCodec's+ # target_sr may differ.from datasets import Audiodataset = dataset.cast_column(audio_col, Audio())@@ -2278,7 +2229,6 @@ class UnslothTrainer:f"DAC dataset needs 'audio' and 'text' columns, got: {dataset.column_names}")- # Cast audio to 24kHz (notebook: cast_column("audio", Audio(sampling_rate=24000)))from datasets import Audiodataset = dataset.cast_column(audio_col, Audio(sampling_rate = 24000))@@ -2330,13 +2280,11 @@ class UnslothTrainer:audio_array = np.array(audio_data["array"], dtype = np.float32)sampling_rate = audio_data.get("sampling_rate", 24000)- # Convert to WAV bytes (Whisper needs a file path)buf = io.BytesIO()sf.write(buf, audio_array, sampling_rate, format = "WAV", subtype = "FLOAT")buf.seek(0)audio_bytes = buf.getvalue()- # 1. Get word timings from Whisperwith tempfile.NamedTemporaryFile(suffix = ".wav",delete = False,@@ -2369,7 +2317,6 @@ class UnslothTrainer:skipped += 1continue- # 2. Create speaker representation with AudioProcessorspeaker_data_dict = {"audio": {"bytes": audio_bytes},"text": normalized_transcript,@@ -2380,7 +2327,6 @@ class UnslothTrainer:skipped += 1continue- # 3. Get training prompt from PromptProcessorprompt = prompt_processor.get_training_prompt(speaker)if prompt:processed_examples.append({"text": prompt})@@ -2395,7 +2341,6 @@ class UnslothTrainer:status_message = f"Preprocessing audio with OuteTTS... {idx + 1}/{len(dataset)}")- # Free Whisper from GPU (notebook: whisper_model.to('cpu'))logger.info("Moving Whisper model to CPU...\n")whisper_model.to("cpu")del whisper_model@@ -2442,10 +2387,8 @@ class UnslothTrainer:f"Whisper dataset needs 'audio' and 'text' columns, got: {dataset.column_names}")- # Cast audio to 16kHz (Whisper's expected sample rate)dataset = dataset.cast_column(audio_col, Audio(sampling_rate = WHISPER_SAMPLE_RATE))- # Train/eval split (notebook does dataset.train_test_split)eval_dataset_raw = Noneif eval_split:splits = dataset.train_test_split(test_size = 0.06, seed = 42)@@ -2474,11 +2417,9 @@ class UnslothTrainer:skipped += 1continue- # Extract audio features (notebook 112-115)features = self.tokenizer.feature_extractor(audio_data["array"], sampling_rate = audio_data["sampling_rate"])- # Tokenize text (notebook 116)tokenized_text = self.tokenizer.tokenizer(text)processed.append(@@ -2580,22 +2521,16 @@ class UnslothTrainer:"""from core.training.s3_dataset import S3DownloadCancelled- # `datasets` exposes no env var, so its bars can only be quieted through the- # class, and only once the module is in. It is (module-level import above), and- # this is the first point before any load: the local-file and Hub branches below- # both call load_dataset(), whose "Generating train split" / "Downloading data"- # bars would otherwise write into the worker's structured stream. It also covers- # the map/filter bars of every branch further down.+ # datasets exposes no env var, so its bars can only be quieted through the class, and this is the+ # first point before any load_dataset() in either branch.try:from loggers.config import quiet_third_party_progress_barsquiet_third_party_progress_bars()except Exception: # noqa: BLE001 - never let log tidying stop a runpass- # An Audio column decodes inside load_dataset() below, before the audio branches- # that already call this. This worker starts without the shim the API process- # installs, so the load raised `datasets`' own "please install 'torchcodec'".- # A False here is still reported by those branches, naming FFmpeg.+ # An Audio column decodes inside load_dataset() below, and this worker starts without the shim the+ # API process installs, so the load raised datasets' own "please install torchcodec".try:ensure_audio_decoding()except Exception: # noqa: BLE001 - never let a broken librosa stop a text run@@ -2608,7 +2543,7 @@ class UnslothTrainer:dataset = Noneeval_dataset = Nonedataset_attestation_source = None- has_separate_eval_source = False # True if eval comes from a separate HF split+ has_separate_eval_source = Falseeval_enabled = eval_steps is not None and eval_steps > 0raw_text_mode = is_cpt or format_type == "raw"dataset_loaded_from_cache = False@@ -2672,8 +2607,8 @@ class UnslothTrainer:logger.info(f"Downloaded {len(local_datasets)} file(s) from S3\n")if local_datasets:- # load_dataset() gives an Arrow-backed result; in-memory Dataset.from_list() has no- # cache and forces num_proc=1 during tokenization/map (sharding needs Arrow files).+ # load_dataset() is Arrow-backed; an in-memory Dataset.from_list() has no cache and forces+ # num_proc=1 during tokenization.all_files = self._resolve_local_files(local_datasets)if all_files:@@ -2744,11 +2679,11 @@ class UnslothTrainer:)self._update_progress(status_message = f"Streaming {dataset_source}")else:- # Non-streaming: with a slice end, stream only the needed rows and materialize them- # (avoids downloading everything); the eager [start, end] trim happens below.+ # With a slice end, stream only the needed rows and materialize them rather than downloading+ # everything._slice_start = dataset_slice_start or 0- # streaming=True rejects HF slice syntax (e.g. "train[:50%]") with "Bad split", so- # fall back to the regular download path when train_split already carries a slice.+ # streaming=True rejects HF slice syntax with "Bad split", so fall back to the regular download path+ # when train_split already carries a slice._split_has_slice = (train_split or "").find("[") != -1dataset = Noneif dataset_local_files_only:@@ -2843,7 +2778,6 @@ class UnslothTrainer:logger.info("Stopped during dataset loading\n")return None- # Resolve eval split from a separate HF split (explicit or auto)if eval_enabled:effective_train = train_split or "train"if eval_split and eval_split != effective_train:@@ -2855,8 +2789,9 @@ class UnslothTrainer:eval_load_kwargs["revision"] = dataset_revisionif dataset_streaming:- # Probe splits first: load_dataset(streaming=True) returns an IterableDataset without- # validating the name, so a typo would only surface on the first eval batch.+ # load_dataset(streaming=True) returns an IterableDataset without validating the+ # name, so a typo+ # would only surface on the first eval batch.from datasets import get_dataset_split_namesprobe_kwargs = {"path": dataset_source}@@ -2880,8 +2815,9 @@ class UnslothTrainer:f"{available_splits}")eval_dataset = load_dataset(**eval_load_kwargs, streaming = True)- # Streaming eval has no __len__; bound it so each evaluation terminates. .take()- # stays lazy and survives the later format/raw-text .map() passes.+ # Streaming eval has no __len__; .take() stays lazy and survives the later format+ # and raw-text map()+ # passes.if not hasattr(eval_dataset, "__len__"):eval_dataset = eval_dataset.take(STREAMING_EVAL_MAX_SAMPLES)logger.info(@@ -2938,7 +2874,6 @@ class UnslothTrainer:"Streaming mode does not support using the same split for both train and eval. ""Please provide a separate eval split or set eval_steps to 0.")- # Same split as training — split 80/20 after formattinglogger.info(f"Eval split '{eval_split}' is the same as train split — will split 80/20\n")@@ -2947,7 +2882,6 @@ class UnslothTrainer:raise ValueError("Streaming mode currently requires an explicit eval split when evaluation is enabled.")- # Auto-detect eval split from HF (separate dataset or None)if dataset_loaded_from_cache:split_info = getattr(getattr(dataset, "info", None), "splits", None)available_splits = list(split_info or ())@@ -2993,8 +2927,8 @@ class UnslothTrainer:if dataset is None:raise ValueError("No dataset provided")- # Eager-only index slicing (inclusive both ends). Streaming already sliced lazily via- # skip()/take(); the non-streaming path fetched end+1 rows and is trimmed here.+ # Streaming already sliced lazily via skip()/take(); the non-streaming path fetched end+1 rows and+ # is trimmed here.if (not dataset_streaming) and (dataset_slice_start is not None or dataset_slice_end is not None):@@ -3011,10 +2945,9 @@ class UnslothTrainer:status_message = f"Sliced dataset to {len(dataset)} rows (indices {start}-{end})")- # Bound before the formatting/template/tokenization passes, which map over- # every row. Skipped when streaming (bounded lazily above) or when the user- # named an explicit range, including a bracketed split like train[1000:2000]:- # those are already the rows they asked for, not a corpus to sample from.+ # Bound before the formatting and tokenization passes, and skipped when streaming or when the user+ # named an explicit range, including a bracketed split like train[1000:2000]: those rows are+ # already what they asked for.if ((not dataset_streaming)and dataset_slice_start is None@@ -3040,19 +2973,13 @@ class UnslothTrainer:if self.should_stop:logger.info("Stopped before applying chat template\n")+ # No separate HF eval split - caller handles programmatic splittingreturn None# ========== AUDIO MODELS: custom preprocessing ==========- # An inconclusive probe must not read as "not an audio model": falling through- # with an audio dataset lands on the text path, which fails much later with- # "Could not auto-detect format mapping", a column-mapping complaint that says- # nothing about the repo read that actually failed.- # getattr, because this method is also driven against lightweight stand-ins; both- # defaults leave the guard closed.- # _is_dataset_audio arrives from the client and is true on a column-NAME match- # alone, so _dataset_has_audio_column is the tiebreaker: only a positive "no audio- # here" reopens the text path, unknown still refuses.- # raw/CPT is exempt: it reads the text column and ignores any audio one by design.+ # An inconclusive probe must not read as "not an audio model": falling through lands on the text+ # path, which fails later with a column-mapping complaint. _dataset_has_audio_column is the+ # tiebreaker because _is_dataset_audio is true on a column-NAME match alone; raw/CPT is exempt.if (not self._audio_typeand not getattr(self, "_audio_type_known", True)@@ -3101,6 +3028,10 @@ class UnslothTrainer:# ========== RAW TEXT BYPASS ==========if raw_text_mode:+ # Say which variable said so: a stale size from an earlier mpirun or an HPC container image+ # reads as a multi-rank launch on a one-process machine, and the only symptom is this run being+ # told it makes several passes. The row bound already trusts the same variables; what was+ # missing was a way to see them.logger.info(f"{_raw_mode_label().capitalize()} mode: bypassing chat template, ""using raw text\n"@@ -3127,8 +3058,7 @@ class UnslothTrainer:f"({eval_rows}) kept as raw text\n")elif eval_enabled and not has_separate_eval_source and not dataset_streaming:- # _resolve_eval_split_from_dataset does a train_test_split (needs len/random access).- # Streaming always has a separate eval split (route-enforced), so non-streaming only.+ # _resolve_eval_split_from_dataset does a train_test_split, which needs len and random access.split_result = self._resolve_eval_split_from_dataset(dataset)if split_result is not None:train_portion, eval_dataset = split_result@@ -3142,8 +3072,8 @@ class UnslothTrainer:)logger.info(f"Raw-text dataset ready ({n_display} samples)\n")- # Streaming datasets can report column_names as None, which would make the `in` check- # raise; resolve_column_names falls back to features/first-row probing.+ # Streaming datasets can report column_names as None, which would make the `in` check raise;+ # resolve_column_names falls back to features/first-row probing.train_columns = resolve_column_names(train_dataset)if "text" not in train_columns:raise ValueError(f"Raw-text dataset missing 'text' column: {train_columns}")@@ -3188,7 +3118,6 @@ class UnslothTrainer:)logger.info(f"Dataset formatted successfully ({final_n} samples, {detected})\n")- # ========== THEN SPLIT ==========if has_separate_eval_source and eval_dataset is not None:eval_n = len(eval_dataset) if hasattr(eval_dataset, "__len__") else "?"logger.info(f"Formatting eval dataset ({eval_n} rows)...\n")@@ -3284,7 +3213,6 @@ class UnslothTrainer:f"from the training data when enough rows are available: {e}")- # No separate HF eval split — caller handles programmatic splittingreturn Nonedef _resolve_eval_split_from_dataset(self, dataset) -> Optional[tuple]:@@ -3352,9 +3280,9 @@ class UnslothTrainer:self._update_progress(error = "Model not loaded")return False- # Pre-import heavy transformers modules on the main thread: Unsloth's patched_import- # isn't thread-safe with importlib's cache (KeyError: 'size') from a worker thread.- import transformers # noqa: F401 – ensures submodules are cached+ # Pre-import heavy transformers modules on the main thread: Unsloth's patched_import is not thread-+ # safe with importlib's cache.+ import transformers # noqa: F401 - ensures submodules are cachedfrom transformers import ( # noqa: F401Trainer as _HFTrainer,TrainingArguments as _TrainingArguments,@@ -3458,35 +3386,22 @@ class UnslothTrainer:self._online_eval_dataset = eval_datasettrain_dataset = dataset["dataset"] if isinstance(dataset, dict) else dataset- # A step-capped run says nothing about passes, but Unsloth knows the rows- # and microbatch size, so resolve it here. World size scales rows consumed- # per step; omitting it would understate the passes.+ # A step-capped run says nothing about passes, and world size scales the rows consumed per step.resolved_epochs = Nonemax_steps = int(config_args.get("max_steps") or 0)if max_steps > 0:try:rows = len(train_dataset)- # Every launcher variable, not WORLD_SIZE alone: no MPI sets it (Open- # MPI uses OMPI_COMM_WORLD_SIZE, Hydra/Intel MPI PMI_SIZE, mlx.launch's- # CUDA-only NCCL backend MLX_WORLD_SIZE), LOCAL_WORLD_SIZE is defensive- # since torchrun sets both and the max prefers the global count. Reading- # one variable calls an mpirun launch single-process and engages a view- # that re-tokenizes every extra pass; it also raises on junk like- # int("auto"), leaving resolved_epochs None so the gate reads inf and- # kills the feature. Env-only, deliberately NOT worker.py's- # _data_parallel_world_size, which also counts visible CUDA devices:- # that bounds a row subset where over-counting is free, this feeds a- # veto where both directions are wrong. Unsloth's multi-GPU load is a- # sharding device_map, model-parallel to transformers with _n_gpu = 1,- # so a sharded 4-GPU run draws one GPU's rows per step and counting- # devices would report 4x the passes, vetoing a qualifying run.+ # Every launcher variable, not WORLD_SIZE alone: Open MPI sets OMPI_COMM_WORLD_SIZE, Hydra/Intel+ # MPI PMI_SIZE, mlx.launch's CUDA-only NCCL backend MLX_WORLD_SIZE, and LOCAL_WORLD_SIZE is+ # defensive since torchrun sets both. Reading one variable calls an mpirun launch single-process+ # or raises on junk like int("auto"), leaving resolved_epochs None. Env-only, deliberately NOT+ # worker.py's _data_parallel_world_size, which also counts visible CUDA devices: Unsloth's+ # multi-GPU load is a sharding device_map, model-parallel to transformers.world_size = world_size_from_env()if world_size > 1:- # Say which variable said so: a stale size from an earlier mpirun- # or an HPC container image reads as a multi-rank launch on a- # one-process machine, and the only symptom is this run being told- # it makes several passes. The row bound already trusts the same- # variables; what was missing was a way to see them.+ # Say which variable said so: a stale size from an earlier mpirun reads as a multi-rank launch, and+ # this veto is the only symptom.logger.info(f"Launcher environment reports {world_size} data-parallel "f"processes ({world_size_env_report()}); a step-capped run "@@ -3537,10 +3452,9 @@ class UnslothTrainer:try:text_field = config_args.get("dataset_text_field", "text") or "text"max_length = int(config_args.get("max_seq_length") or 2048)- # The generated `__init__` clamps `max_seq_length` to the model's cap- # and derives `max_length` from it, but runs after this. Apply the same- # cap here or the transform truncates wider than the eager map it- # replaces, and the attestation claims a width nothing enforced.+ # The generated __init__ clamps max_seq_length to the model's cap and derives max_length from it,+ # but runs after this: apply the same cap here or the transform truncates wider than the eager map+ # it replaces.model_cap = getattr(self.model, "max_seq_length", None)try:if model_cap is not None and 0 < int(model_cap) < max_length:@@ -3566,9 +3480,8 @@ class UnslothTrainer:if isinstance(dataset, dict):dataset["dataset"] = viewif eval_dataset is not None:- # Probe the eval split's own first row: TRL calls _prepare_dataset- # once per split, so reusing the train answer would tokenize eval- # differently whenever the splits disagree about a leading BOS.+ # Probe the eval split's own first row: TRL calls _prepare_dataset once per split, so reusing the+ # train answer tokenizes eval differently whenever the splits disagree about a leading BOS.eval_add_special_tokens = resolve_add_special_tokens(self.tokenizer, first_sample_text(eval_dataset, text_field))@@ -3615,8 +3528,8 @@ class UnslothTrainer:except Exception as exc: # noqa: BLE001 - cleanup must never fail a finished runlogger.warning(f"Online tokenization worker shutdown failed: {exc}")return- # `_online_prewarm_batches` is left alone: it records how the run was- # configured and the A/B harness reads it back. A second call is a no-op.+ # _online_prewarm_batches records how the run was configured and the A/B harness reads it back; a+ # second call is a no-op.if released:logger.info(f"Online tokenization: shut down {released} DataLoader workers\n")@@ -3638,8 +3551,7 @@ class UnslothTrainer:if prewarm:from utils.datasets.online_tokenization import memoize_train_dataloader- # Hold the loader this barrier fills, or train() forks fresh workers- # and everything drained here is wasted.+ # Hold the loader this barrier fills, or train() forks fresh workers and everything drained here is wasted.memoize_train_dataloader(self.trainer)try:loader = self.trainer.get_train_dataloader()@@ -3649,9 +3561,9 @@ class UnslothTrainer:try:next(iterator)except StopIteration:- break # a short split simply prewarms fewer- # Drop the local names: on the eager path that tears the loader down;- # on the online path the memo still holds it, so the workers survive.+ break+ # Drop the local names: on the eager path that tears the loader down, while on the online path the+ # memo still holds it so the workers survive.del iterator, loaderexcept StopIteration:return (@@ -3671,7 +3583,7 @@ class UnslothTrainer:except Exception:input_ids = getattr(batch, "input_ids", None)if input_ids is None:- return None # some collators omit input_ids+ return Noneseq_len = input_ids.shape[-1] if input_ids.ndim > 0 else 0if not (input_ids.is_floating_point() or input_ids.numel() == 0 or seq_len == 0):@@ -3708,8 +3620,8 @@ class UnslothTrainer:never passed as a bare ``IterableDataset``."""try:- # On spawn platforms, put compiled-cache dirs on sys.path/PYTHONPATH before any- # dataset.map() so spawned workers can import e.g. UnslothSFTTrainer.+ # On spawn platforms, put the compiled-cache dirs on sys.path/PYTHONPATH before any dataset.map()+ # so spawned workers can import e.g. UnslothSFTTrainer.if sys.platform in ("win32", "darwin"):from utils.cache_cleanup import register_compiled_cache_on_pathregister_compiled_cache_on_path()@@ -3730,7 +3642,6 @@ class UnslothTrainer:output_dir = str(resolve_output_dir(training_args.get("output_dir")))ensure_dir(Path(output_dir))- # ========== AUDIO TRAINER BRANCH ==========if self._audio_type == "csm":# CSM uses plain HF Trainer with remove_unused_columns=False for the depth decoder.from transformers import Trainer as HFTrainer, TrainingArguments@@ -3750,8 +3661,10 @@ class UnslothTrainer:args = TrainingArguments(**config),)self.trainer.add_callback(self._create_progress_callback())- # Unsloth publishes progress itself, so HF's stdout callbacks are pure- # duplication in a log that has no terminal. --verbose keeps them.+ # Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that has+ # no terminal; --verbose keeps them.+ # Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that+ # has no terminal. --verbose keeps them._drop_hf_stdout_callbacks(self.trainer)batch_size = training_args.get("batch_size", 2)@@ -3771,8 +3684,7 @@ class UnslothTrainer:returnelif self._audio_type == "snac":- # Orpheus: LM with SNAC codec tokens -- plain HF Trainer. DataCollatorForSeq2Seq- # pads variable-length sequences per batch and pads labels with -100.+ # DataCollatorForSeq2Seq pads variable-length sequences per batch and pads labels with -100.from transformers import (Trainer as HFTrainer,TrainingArguments,@@ -3791,8 +3703,6 @@ class UnslothTrainer:),)self.trainer.add_callback(self._create_progress_callback())- # Unsloth publishes progress itself, so HF's stdout callbacks are pure- # duplication in a log that has no terminal. --verbose keeps them._drop_hf_stdout_callbacks(self.trainer)batch_size = training_args.get("batch_size", 2)@@ -3812,7 +3722,6 @@ class UnslothTrainer:returnelif self._audio_type == "whisper":- # Whisper: Seq2SeqTrainer with custom speech collatorfrom transformers import Seq2SeqTrainer, Seq2SeqTrainingArgumentsfrom utils.datasets import DataCollatorSpeechSeq2SeqWithPadding@@ -3838,8 +3747,6 @@ class UnslothTrainer:self.trainer = Seq2SeqTrainer(**trainer_kwargs)self.trainer.add_callback(self._create_progress_callback())- # Unsloth publishes progress itself, so HF's stdout callbacks are pure- # duplication in a log that has no terminal. --verbose keeps them._drop_hf_stdout_callbacks(self.trainer)batch_size = training_args.get("batch_size", 2)@@ -3882,7 +3789,6 @@ class UnslothTrainer:data_collator = Noneif is_deepseek_ocr:- # DeepSeek OCR collator - auto-install if neededlogger.info("Detected DeepSeek OCR model\n")if not _ensure_deepseek_ocr_installed():error_msg = (@@ -3900,8 +3806,8 @@ class UnslothTrainer:logger.info("Configuring DeepSeek OCR data collator...\n")FastVisionModel.for_training(self.model)- # (image_size, base_size, crop_mode) is a coupled preset; changing image_size alone- # desyncs the per-crop grid from num_queries. Use the Gundam preset.+ # (image_size, base_size, crop_mode) is a coupled preset: changing image_size alone desyncs the per-+ # crop grid from num_queries.if training_args.get("vision_image_size") is not None:logger.info("Vision image resize ignored for DeepSeek OCR "@@ -3926,7 +3832,7 @@ class UnslothTrainer:elif self.is_audio_vlm and not raw_text_mode:# Audio VLM collator (e.g. Gemma 3N), mirrors the Gemma3N_(4B)-Audio notebook.logger.info("Configuring audio VLM data collator...\n")- processor = self.tokenizer # FastModel returns processor as tokenizer+ processor = self.tokenizeraudio_col_name = getattr(self, "_audio_vlm_audio_col", "audio")@@ -3944,7 +3850,6 @@ class UnslothTrainer:batch = processor(text = texts, audio = audios, return_tensors = "pt", padding = True)- # Labels = input_ids with special tokens maskedlabels = batch["input_ids"].clone()labels[labels == processor.tokenizer.pad_token_id] = -100for attr in (@@ -4003,11 +3908,9 @@ class UnslothTrainer:"report_to": _build_report_targets(training_args),"disable_tqdm": _hf_stdout_progress_disabled(),"include_num_input_tokens_seen": True,- # serial_as_none = False: this is a config boundary, not a map() call site. The audio- # paths ask for 1 to keep dataset workers off a process holding audio/CUDA state, and- # a config None reads as "auto-size me" downstream. Pass None (not a count) otherwise:- # the shared policy sizes an auto request from CPU affinity + cgroup quota, while an- # integer computed here comes from host os.cpu_count() and skips that.+ # serial_as_none = False: this is a config boundary, not a map() call site. The audio paths ask+ # for 1 to keep dataset workers off a process holding audio/CUDA state; pass None otherwise, so+ # the shared policy sizes it from CPU affinity and cgroup quota rather than host os.cpu_count()."dataset_num_proc": dataset_map_num_proc(1 if (self.is_audio or self.is_audio_vlm or self._cuda_audio_used) else None,serial_as_none = False,@@ -4051,7 +3954,6 @@ class UnslothTrainer:else:logger.info(f"Training for {config_args['num_train_epochs']} epochs\n")- # ========== EVAL CONFIGURATION ==========eval_dataset = training_args.get("eval_dataset", None)eval_steps_val = training_args.get("eval_steps", 0.00)if eval_dataset is not None:@@ -4076,13 +3978,12 @@ class UnslothTrainer:else:logger.info("No eval dataset — evaluation disabled\n")- # Use training_args optim/lr_scheduler_type if given, else defaultsoptim_value = training_args.get("optim", "adamw_8bit")lr_scheduler_type_value = training_args.get("lr_scheduler_type", "linear")if (self.is_vlm or self.is_audio_vlm) and not raw_text_mode:- # Vision / audio VLM both need skip_prepare_dataset + remove_unused_columns;- # raw-text VLM goes to the text path below.+ # Vision and audio VLM both need skip_prepare_dataset + remove_unused_columns; raw-text VLM goes to+ # the text path below.label = "audio VLM" if self.is_audio_vlm else "vision"logger.info(f"Configuring {label} model training parameters\n")optim_value = training_args.get("optim", "adamw_torch_fused")@@ -4131,7 +4032,7 @@ class UnslothTrainer:f"Sequence packing: {'enabled' if packing_enabled else 'disabled'}\n")- # Audio codec overrides — BiCodec/DAC use the text SFTTrainer path+ # Audio codec overrides - BiCodec/DAC use the text SFTTrainer pathif self._audio_type == "bicodec":config_args["packing"] = Falselogger.info("Applied BiCodec overrides: packing=False\n")@@ -4139,9 +4040,8 @@ class UnslothTrainer:config_args["packing"] = Falselogger.info("Applied DAC overrides: packing=False\n")- # ========== ONLINE (OVERLAPPED) TOKENIZATION ==========- # Plain-text single-pass runs tokenize in the DataLoader workers- # instead of a blocking .map(); everything else stays eager.+ # Plain-text single-pass runs tokenize in the DataLoader workers instead of a blocking .map();+ # everything else stays eager.self._online_prewarm_batches = 0online_decision = self._configure_online_tokenization(config_args = config_args,@@ -4160,8 +4060,9 @@ class UnslothTrainer:logger.info("Training configuration prepared\n")# ========== TRAINER INITIALIZATION ==========if self.is_audio_vlm and not raw_text_mode:- # Audio VLM (e.g. Gemma 3N + audio): raw Dataset from _format_audio_vlm_dataset.- # Notebook uses processing_class=processor.tokenizer; raw-text runs use the text path.+ # Image VLM: dict wrapper from format_and_template_dataset (raw-text uses the text path).+ # Audio VLM (e.g. Gemma 3N + audio): raw Dataset from _format_audio_vlm_dataset. Notebook uses+ # processing_class=processor.tokenizer; raw-text runs use the text path.train_dataset = dataset["dataset"] if isinstance(dataset, dict) else datasetprocessing_class = (self.tokenizer.tokenizer@@ -4179,7 +4080,6 @@ class UnslothTrainer:trainer_kwargs["eval_dataset"] = eval_datasetself.trainer = SFTTrainer(**trainer_kwargs)elif self.is_vlm and not raw_text_mode:- # Image VLM: dict wrapper from format_and_template_dataset (raw-text uses the text path).train_dataset = dataset["dataset"] if isinstance(dataset, dict) else datasettrainer_kwargs = {"model": self.model,@@ -4192,8 +4092,10 @@ class UnslothTrainer:trainer_kwargs["eval_dataset"] = eval_datasetself.trainer = SFTTrainer(**trainer_kwargs)else:- # For text-only, unwrap a Processor (Gemma-3 returns ProcessorMixin even for text) to- # the raw tokenizer; else SFTTrainer sets _is_vlm, skips _prepare_dataset, no input_ids.+ # Unwrap a Processor for text-only (Gemma-3 returns ProcessorMixin even for text), else SFTTrainer+ # sets _is_vlm, skips _prepare_dataset and produces no input_ids.+ # For text-only, unwrap a Processor (Gemma-3 returns ProcessorMixin even for text) to the raw+ # tokenizer; else SFTTrainer sets _is_vlm, skips _prepare_dataset, no input_ids.from transformers import ProcessorMixinsft_tokenizer = self.tokenizer@@ -4254,8 +4156,7 @@ class UnslothTrainer:self.trainer.processing_class = self.tokenizerlogger.info("Trainer initialized\n")- # ========== TRAIN ON RESPONSES ONLY ==========- # Raw-text datasets always train on all tokens.+ # ========== TRAIN ON RESPONSES ONLY ========== Raw-text datasets always train on all tokens.is_cpt = training_args.get("is_cpt", False)train_on_responses_enabled = (False@@ -4270,10 +4171,8 @@ class UnslothTrainer:"Raw-text mode: skipping train_on_responses_only — training on all tokens\n")- # DeepSeek OCR handles this internally in its collator, so skip- # Audio VLM handles label masking in its collator, so skip- # Markers auto-detected from the chat template first, manual table as fallback;- # gpt-oss stays on manual markers. See apply_completion_masking.+ # Markers auto-detected from the chat template first, with the manual table as fallback; gpt-oss+ # stays on manual markers. See apply_completion_masking.if (train_on_responses_enabledand not self.is_audio_vlm@@ -4290,8 +4189,8 @@ class UnslothTrainer:else:logger.info(f"{message}\n")- # No try/except: the helper handles detection failures itself, so an exception here is- # a real masking failure that must fail the run, not silently train full sequences.+ # No try/except: the helper handles detection failures itself, so an exception here is a real+ # masking failure that must fail the run, not silently train full sequences.self.trainer, masking_applied = apply_completion_masking(self.trainer,self.model_name,@@ -4306,10 +4205,9 @@ class UnslothTrainer:if masking_applied:try:- # Safety net: train_on_responses_only masks non-response tokens with -100, and a row- # becomes all -100 (Unsloth drops it) when the response template is missing from the- # formatted text -- usually a dataset/template mismatch, sometimes max_seq_length- # truncation. len()-based, so skip for streaming.+ # Safety net: train_on_responses_only masks non-response tokens with -100, and a row becomes all+ # -100 (Unsloth drops it) when the response template is missing from the formatted text;+ # len()-based, so skipped for streaming.if detect_streaming_dataset(self.trainer.train_dataset):logger.info("Skipping post-filter length check for streaming dataset\n")else:@@ -4359,10 +4257,11 @@ class UnslothTrainer:else:logger.info("Training on full sequences (including prompts)\n")- # ========== PROGRESS TRACKING ==========self.trainer.add_callback(self._create_progress_callback())- # Unsloth publishes progress itself, so HF's stdout callbacks are pure- # duplication in a log that has no terminal. --verbose keeps them.+ # Unsloth publishes progress itself, so HF's stdout callbacks duplicate a log that has no terminal;+ # --verbose keeps them.+ # Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that has+ # no terminal. --verbose keeps them._drop_hf_stdout_callbacks(self.trainer)train_dataset_obj = dataset["dataset"] if isinstance(dataset, dict) else dataset@@ -4398,8 +4297,8 @@ class UnslothTrainer:)self._update_progress(total_steps = total_steps)- # ========== START TRAINING ==========# Fail fast on an invalid first batch (empty/float input_ids) vs a step-1 crash.+ # ========== START TRAINING ==========preflight_error = self._preflight_first_batch()if preflight_error:logger.error(preflight_error)@@ -4413,9 +4312,8 @@ class UnslothTrainer:resume_from_checkpoint = training_args.get("resume_from_checkpoint"))finally:- # Before _finalize_training, not after: merging and exporting are- # where the memory goes, and the workers still hold a fork of this- # process.+ # Before _finalize_training: merging and exporting are where the memory goes, and the workers still+ # hold a fork of this process.self._release_online_dataloader()# ========== SAVE MODEL ==========@@ -4429,8 +4327,8 @@ class UnslothTrainer:self._update_progress(is_training = False, error = str(e))finally:- # Backstop for returns that never reach train(), notably a preflight- # error: that path has already forked the workers.+ # Backstop for returns that never reach train(), notably a preflight error, which has already forked+ # the workers.self._release_online_dataloader()self.is_training = False@@ -4520,9 +4418,8 @@ def _ensure_deepseek_ocr_installed():import osscript_dir = os.path.dirname(os.path.abspath(__file__))- parent_dir = os.path.dirname(script_dir) # project root+ parent_dir = os.path.dirname(script_dir)- # Download to project root as 'deepseek_ocr' folderlocal_dir = os.path.join(parent_dir, "deepseek_ocr")snapshot_download("unsloth/DeepSeek-OCR", local_dir = local_dir, local_dir_use_symlinks = False)diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.pyindex 6ba57916778..fec2390b36c 100644--- a/studio/backend/core/training/training.py+++ b/studio/backend/core/training/training.py@@ -49,13 +49,12 @@ def _env_int(name: str, default: int) -> int:return default-# Stop-watchdog escalation timeouts. Primary trigger: a short grace once "complete" (save-# done). The absolute cap is a backstop: long for save=True, shorter for a cancel.+# Primary trigger is a short grace once "complete" (save done); the absolute cap is a backstop, long+# for save=True and shorter for a cancel._STOP_GRACE_S = _env_int("UNSLOTH_STUDIO_TRAINING_STOP_GRACE_S", 15)_STOP_TIMEOUT_S = _env_int("UNSLOTH_STUDIO_TRAINING_STOP_TIMEOUT_S", 600)_CANCEL_TIMEOUT_S = _env_int("UNSLOTH_STUDIO_TRAINING_CANCEL_TIMEOUT_S", 120)-# Backstop to reclaim the GPU from a worker wedged in teardown. Generous: is_run_finished-# already unwedges the UI, and a post-run wandb sync can legitimately take a while.+# Generous: is_run_finished already unwedges the UI, and a post-run wandb sync can legitimately take a while._COMPLETE_EXIT_GRACE_S = _env_int("UNSLOTH_STUDIO_TRAINING_COMPLETE_EXIT_GRACE_S", 120)# A few short retries so a transient SQLite lock doesn't lose the terminal state.@@ -105,7 +104,7 @@ def _load_pyplot():try:import matplotlib- matplotlib.use("Agg") # headless backend+ matplotlib.use("Agg")import matplotlib.pyplot as plt_pyplot = plt@@ -309,9 +308,9 @@ def _sanitize_db_config(config: dict[str, Any]) -> dict[str, Any]:_MODEL_SNAPSHOT_METADATA = ("config.json", "adapter_config.json")-# refs/main can point at a revision that only ever fetched metadata, so prefer a snapshot-# that actually carries weights. Keep in step with _MODEL_WEIGHT_CANDIDATES in-# routes/training.py: selecting a snapshot the start route rejects reproduces the 400.+# refs/main can point at a revision that only ever fetched metadata, so prefer a snapshot that+# carries weights. Keep in step with _MODEL_WEIGHT_CANDIDATES in routes/training.py: selecting a+# snapshot the start route rejects reproduces the 400._MODEL_SNAPSHOT_WEIGHTS = ("model.safetensors","model.safetensors.index.json",@@ -344,8 +343,8 @@ def _resolve_model_snapshot(model_name: str, local_path: Optional[str]) -> Optiorepo_id = canonical_model_repo_id(model_name)metadata_names = _with_load_subdirs(model_name, _MODEL_SNAPSHOT_METADATA)weight_names = _with_load_subdirs(model_name, _MODEL_SNAPSHOT_WEIGHTS)- # Pass 1 demands metadata AND weights, so neither a metadata-only refs/main nor a- # weights-only fetch displaces a complete sibling. Pass 2 keeps the old metadata-only path.+ # Pass 1 demands metadata AND weights, so neither a metadata-only refs/main nor a weights-only fetch+ # displaces a complete sibling; pass 2 keeps the old metadata-only path.passes: tuple[dict[str, Any], ...] = ({"required_groups": (metadata_names, weight_names)},{"metadata_filenames": metadata_names},@@ -626,9 +625,8 @@ class TrainingProgress:eval_loss: Optional[float] = Nonepeak_memory_gb: Optional[float] = Noneoutput_dir: Optional[str] = None- # Set on the end-of-run record HF emits after leaving the training loop. It has no- # step loss, so the progress filter would drop it, and with it the only elapsed- # time that includes the final evaluation, checkpoint save and best-model reload.+ # The end-of-run record has no step loss, so the progress filter would drop it, and with it the only+ # elapsed time that includes the final evaluation, checkpoint save and best-model reload.is_run_summary: bool = False@@ -800,10 +798,8 @@ class _MLXTrainerAdapter:max_train_rows: Optional[int] = None,max_train_rows_seed: int = 3407,) -> Optional[tuple]:- # Signature must match UnslothTrainer, which hands back this adapter on an- # MLX host. The MLX worker loads its own data and derives the row bound from- # its config, so the two bound arguments are accepted and deliberately not- # forwarded: a copy here would be a second source of truth.+ # Signature must match UnslothTrainer: the MLX worker loads its own data and derives the row bound+ # from its config, so the two bound arguments are accepted and deliberately not forwarded.self._dataset_config = {"hf_dataset": dataset_source or "","local_datasets": local_datasets,@@ -1097,8 +1093,7 @@ class TrainingBackend:self._provenance_lock = threading.Lock()self._run_intent_lock = threading.RLock()- # Stop watchdog: escalates to force_terminate() if the worker doesn't exit in time. The- # watched proc is tracked so a new run always gets its own watcher.+ # The watched proc is tracked so a new run always gets its own watchdog.self._stop_watchdog: Optional[threading.Thread] = Noneself._stop_watchdog_proc: Optional[mp.Process] = Noneself._complete_seen = threading.Event()@@ -1112,8 +1107,8 @@ class TrainingBackend:# Throttled training-status logging to the server log (not one line/step).self._last_progress_log_ts: float = 0.0self._last_progress_log_step: int = -1- # (elapsed_seconds, num_tokens) at the previous logged line, so the next one- # can report throughput over the interval between them.+ # (elapsed_seconds, num_tokens) at the previous logged line, so the next one reports throughput over+ # the interval between them.self._last_progress_log_elapsed: Optional[float] = Noneself._last_progress_log_tokens: Optional[int] = None@@ -1155,8 +1150,6 @@ class TrainingBackend:logger.info("TrainingBackend initialized (subprocess mode)")- # --- Public API (called by routes/training.py) ----def reserve_start_request(self, start_request_id: str, job_id: str) -> tuple[str, TrainingStartRequestRecord]:@@ -1299,8 +1292,8 @@ class TrainingBackend:return "cancelled", existingif existing.state == "pending" and not owns_current:- # Not the active run, so it does not get the owner's over-cap slot: start- # plus cancel could otherwise be repeated to grow the table without bound.+ # Not the active run, so it does not get the owner's over-cap slot: start plus cancel could+ # otherwise be repeated to grow the table without bound.self._reserve_start_cancel_tombstone_locked(start_request_id)cancelled = replace(existing,@@ -1499,10 +1492,8 @@ class TrainingBackend:raise TrainingStartCancellationCapacityError("Too many training start cancellations are pending")- # Everything left is unexpired (pruned above), so evicting one would forget a- # live cancellation and let its delayed /start spawn the job we just cancelled.- # Overshoot instead: only the owner of the active start reaches this, and there- # is at most one of those, so the table stays bounded.+ # Everything left is unexpired, so evicting one would forget a live cancellation and let its delayed+ # /start spawn the job just cancelled.self._start_cancel_tombstone_reservations[start_request_id] = 1return True@@ -1543,8 +1534,8 @@ class TrainingBackend:start_request_id: Optional[str] = None,**kwargs,) -> bool:- # Reserve before lifecycle locking and synchronous validation: routes call start_training- # from worker threads, so this compare-and-set stops two requests reaching the spawn.+ # Reserve before lifecycle locking and validation: routes call start_training from worker threads,+ # so this compare-and-set stops two requests reaching the spawn.with self._new_job_spawn_reservation(job_id) as spawn_reserved:if not spawn_reserved:logger.warning("Training subprocess already running")@@ -1606,7 +1597,7 @@ class TrainingBackend:logger.warning("Training subprocess already running")return False- # Join prior pump thread — refuse to start if it won't die+ # Wait for pump thread to finish DB finalization (8s covers SQLite's 5s lock timeout).if self._pump_thread is not None and self._pump_thread.is_alive():self._pump_thread.join(timeout = 5.0)if self._pump_thread.is_alive():@@ -1623,11 +1614,9 @@ class TrainingBackend:initialize_resource_provenance(config)- # Split GPU validation from placement around the VRAM hook:- # * Explicit gpu_ids are validated here (raises -> the route 400s before any teardown)- # and their placement is VRAM-independent, so it survives the hook freeing memory.- # * Auto-selection ranks GPUs by *free* VRAM, so it is deferred until after the hook,- # else it could pin training onto a GPU the hook is about to clear.+ # Explicit gpu_ids are validated here, so the route 400s before any teardown and their placement+ # survives the VRAM hook; auto-selection ranks GPUs by FREE VRAM, so it is deferred until after+ # the hook, else it pins training onto a GPU the hook is about to clear.from utils.hardware import hardware as _hwgpu_ids = kwargs.get("gpu_ids")@@ -1690,8 +1679,7 @@ class TrainingBackend:start_request_id,)return False- # Synchronous validation passed -> free VRAM (export + chat) before auto-selection and the- # spawn. After the handshake, so a lost race can't tear down chat for a run that never spawns.+ # Free VRAM after the handshake, so a lost race cannot tear down chat for a run that never spawns.if before_spawn is not None:try:before_spawn()@@ -1747,7 +1735,7 @@ class TrainingBackend:self.current_job_id = job_idself.current_start_request_id = start_request_idtry:- adopt_pid(proc.pid) # bind to parent lifetime (Windows job / sweep)+ adopt_pid(proc.pid)except Exception:logger.error("Failed to adopt training subprocess; terminating it",@@ -1813,7 +1801,7 @@ class TrainingBackend:self._metric_buffer.clear()self._run_finalized = Falseself._db_run_created = False- self._db_create_in_progress = False # a stale watchdog create can't block this run+ self._db_create_in_progress = Falseself._db_total_steps_set = Falseself._db_config = _sanitize_db_config(config)self._db_started_at = datetime.now(timezone.utc).isoformat()@@ -1824,8 +1812,8 @@ class TrainingBackend:self._xet_fallback_used = Falseself._needs_xet_respawn = False- # Create the DB run row before the pump consumes events, so it appears in history during- # model loading and a fast terminal worker can't race the pump. From here the pump finalizes.+ # Create the DB run row before the pump consumes events, so it appears in history during model+ # loading and a fast terminal worker cannot race the pump.self._ensure_db_run_created()if resume_source_run_id and not self._db_run_created:if proc.is_alive():@@ -1846,6 +1834,7 @@ class TrainingBackend:self._stop_queue = stop_queueself._proc = procself._pump_thread = new_pump+ # Start under the lock so a concurrent _ensure_pump_alive can't spawn yet another pump.new_pump.start()if self._new_job_spawn_id == job_id:self._spawn_in_progress = False@@ -1915,9 +1904,8 @@ class TrainingBackend:with self._lock:if self.current_job_id != run_id:return False- # The pump can finish the run between the route's terminal check and this- # lock, so re-test: latching _should_stop after the fact would report a- # saved run as stopped for good.+ # The pump can finish the run between the route's terminal check and this lock, so re-test: latching+ # _should_stop after the fact would report a saved run as stopped for good.if save and self._run_finished_locked():return Falseif save or not run_id:@@ -1955,11 +1943,9 @@ class TrainingBackend:expected_job_id is not None and self.current_job_id != expected_job_id):return "superseded"- # An unscoped reset cannot prove it means THIS run, so it never force-- # terminates: _cancel_requested is cleared after current_job_id is set, so a- # bodyless reset landing in that window would kill the run that just started.- # A stop already asked for means the pre-rework cancel-then-dismiss flow, so- # let it clear its UI; otherwise this is a stale reset of a live run (409).+ # An unscoped reset cannot prove it means THIS run, so it never force-terminates: _cancel_requested+ # is cleared after current_job_id is set, so a bodyless reset landing in that window would kill the+ # run that just started. Otherwise this is a stale reset of a live run (409).if expected_job_id is None and is_active:return "superseded" if self._cancel_requested else "active"cancel_requested = self._cancel_requested@@ -2115,10 +2101,10 @@ class TrainingBackend:re-guarded on target_proc so a run that did replace the worker keeps its handle."""with self._lock:if target_proc is not None and self._proc is not target_proc:- return # a new run replaced the worker; never touch its state+ returnif watched_job_id is not None and self.current_job_id != watched_job_id:- return # a new run is already starting up; leave its state alone- run_id = self.current_job_id # == watched_job_id+ return+ run_id = self.current_job_idself._progress.is_training = Falseterminal_payload = self._terminal_finalize_kwargs()status = terminal_payload["status"]@@ -2134,8 +2120,8 @@ class TrainingBackend:elif status != "completed":self._progress.status_message = "Training stopped."# A completed run keeps its message; reaping a wedged worker must not relabel it.- # Create the row if a start-time create failed (no-op otherwise; skips when the pump- # is mid-create, in which case its create-then-finalize records the run instead).+ # Create the row if a start-time create failed; skipped while the pump is mid-create, whose create-+ # then-finalize records the run instead.self._ensure_db_run_created()with self._provenance_lock:with self._lock:@@ -2151,7 +2137,7 @@ class TrainingBackend:if clear_output_dir:self._output_dir = self._progress.output_dir = Noneif claim:- self._run_finalized = True # claim this run's finalize+ self._run_finalized = Truebatch = list(self._metric_buffer)del self._metric_buffer[: len(batch)]final_step = self._progress.step@@ -2253,7 +2239,7 @@ class TrainingBackend:with self._lock:proc = self._procif target_proc is not None and proc is not target_proc:- return # superseded by a new run; do not touch the new worker+ returnif proc is not None and proc.is_alive():logger.info("Force-terminating training subprocess (pid=%s)", proc.pid)proc.terminate()@@ -2309,10 +2295,8 @@ class TrainingBackend:if proc is not None and proc.is_alive():proc.terminate()if not recover:- # terminate() is only a request: arm the same backstop as the other terminal- # paths so a worker that ignores it cannot hold the GPU for good. Signal first,- # since arming no-ops when a watchdog already watches this proc and only this- # drops it to the grace.+ # terminate() is only a request, so arm the same backstop as the other terminal paths. Signal first,+ # since arming no-ops when a watchdog already watches this proc.self._complete_seen.set()self._start_stop_watchdog(cancel = False,@@ -2433,7 +2417,7 @@ class TrainingBackend:new_proc.start()from utils.process_lifetime import adopt_pid- adopt_pid(new_proc.pid) # bind to parent lifetime (Windows job / sweep)+ adopt_pid(new_proc.pid)except Exception:logger.error("Failed to respawn training subprocess", exc_info = True)self._spawn_in_progress = False@@ -2490,7 +2474,6 @@ class TrainingBackend:)new_pump = threading.Thread(target = self._pump_loop, daemon = True)self._pump_thread = new_pump- # Start under the lock so a concurrent _ensure_pump_alive can't spawn yet another pump.new_pump.start()return True@@ -2509,7 +2492,7 @@ class TrainingBackend:Status and progress read this so a finished run reports terminal at once; the GPUadmission guards keep using is_training_active(), since a lingering worker holds VRAM."""if getattr(self, "_spawn_in_progress", False):- return False # a new run is spawning; _progress is still the old run's+ return Falsewith self._lock:return self._run_finished_locked()@@ -2599,8 +2582,6 @@ class TrainingBackend:return self._create_loss_plot(progress, self.current_theme)return None- # --- Compatibility shims: routes/training.py accesses these ----class _TrainerShim:"""Minimal shim so routes that access backend.trainer.* still work."""@@ -2630,8 +2611,6 @@ class TrainingBackend:"""Compatibility shim for routes that access backend.trainer.*"""return self._TrainerShim(self)- # --- Event pump (background thread) ----def _safe_handle_event(self, event: dict) -> None:"""Apply one event, swallowing any handler error.@@ -2673,8 +2652,8 @@ class TrainingBackend:self._safe_handle_event(event)continue- # Snapshot: the watchdog drops _proc last, so a re-read can hit None and kill- # this thread. A dropped handle means it already finalized.+ # Snapshot: the watchdog drops _proc last, so a re-read can hit None and kill this thread; a dropped+ # handle means it already finalized.proc = self._procif proc is None:self._pump_running = False@@ -2687,8 +2666,8 @@ class TrainingBackend:for e in self._drain_queue(self._event_queue):self._safe_handle_event(e)- # Model-load stall: respawn over HTTP instead of finalizing as failure. Starts a fresh pump- # on this thread (no self-join); it takes over _pump_running, so this exit leaves it set.+ # Model-load stall: respawn over HTTP instead of finalizing as failure. The fresh pump takes over+ # _pump_running, so this exit leaves it set.with self._lock:needs_xet_respawn = self._needs_xet_respawnself._needs_xet_respawn = False@@ -2698,7 +2677,6 @@ class TrainingBackend:):return- # Mark done if no explicit complete/error was received.with self._lock:if self._progress.is_training:if self._should_stop:@@ -2725,8 +2703,8 @@ class TrainingBackend:returndef _has_current_resume_checkpoint(self, output_dir, step) -> bool:- # A valid checkpoint at the current step means the stop-and-save landed on- # disk even if the worker died before confirming it.+ # A valid checkpoint at the current step means the stop-and-save landed on disk even if the worker+ # died before confirming it.if not output_dir or not isinstance(step, int) or step <= 0:return Falsefrom core.training.resume import get_resume_checkpoint_path@@ -2846,8 +2824,8 @@ class TrainingBackend:_safe_loss = None_loss_is_nonfinite = _safe_loss is not None and not math.isfinite(_safe_loss)if _loss_is_nonfinite:- # Drop the value rather than laundering it back to the last finite loss; clients see- # loss=None at this step so the NaN is not hidden. Training continues.+ # Drop the value rather than laundering it back to the last finite loss: clients see loss=None at+ # this step so the NaN is not hidden._safe_loss = Noneif not getattr(self._progress, "_nonfinite_loss_warned", False):self._progress._nonfinite_loss_warned = True@@ -2890,9 +2868,8 @@ class TrainingBackend:step = event.get("step", 0)loss = _safe_losslr = _safe_lr- # Only ever move forward. HF can log more than one record at the same- # global_step around the end of a run, and each one used to add another- # point, so a 30-step run charted 33 with the last few stacked on step 30.+ # Only ever move forward: HF can log more than one record at the same global_step around the end of+ # a run, so a 30-step run charted 33 points._last_step = self.step_history[-1] if self.step_history else Noneif step > 0 and loss is not None and (_last_step is None or step > _last_step):self.loss_history.append(loss)@@ -3021,8 +2998,7 @@ class TrainingBackend:elif etype == "error":self._progress.is_training = Falseself._progress.error = event.get("error", "Unknown error")- # Nothing left to save: drop an in-flight watchdog to its grace, not the- # save backstop.+ # Nothing left to save: drop an in-flight watchdog to its grace, not the save backstop.self._complete_seen.set()if self._cancel_requested:self._output_dir = self._progress.output_dir = None@@ -3055,7 +3031,6 @@ class TrainingBackend:}self._terminal_finalize_payload = dict(db_action_kwargs)- # --- DB I/O outside the lock ---if db_action == "create_run":self._ensure_db_run_created()if self._db_run_created:@@ -3079,8 +3054,8 @@ class TrainingBackend:elif db_action == "finalize":self._finalize_run_in_db(**db_action_kwargs)- # Bound how long a worker that will not exit can hold the UI at 100%. No-ops on a- # prompt exit. Outside the lock: _start_stop_watchdog takes it, not reentrant.+ # Bound how long a worker that will not exit can hold the UI at 100%. Outside the lock:+ # _start_stop_watchdog takes it and it is not reentrant.if etype in ("complete", "error"):self._start_stop_watchdog(cancel = False,@@ -3093,6 +3068,7 @@ class TrainingBackend:self._log_training_progress()def _persist_output_dir(self) -> None:+ # Re-queue the claimed batch at the front so it retries on the next flush.with self._lock:if (not self._output_dir@@ -3124,11 +3100,9 @@ class TrainingBackend:now = time.monotonic()if prev >= 0 and step > prev and not is_final and (now - self._last_progress_log_ts) < 30.0:return- # Throughput over the interval since the previous logged line. Trainer speed is- # the number people watch a training run for, and the only place it used to- # appear was HF's own tqdm bar ("1.84s/it") and its per-step print- # ("train_tokens_per_second"), both of which are raw stdout rather than- # structured. Carry it here instead, so the structured line is not a downgrade.+ # Throughput over the interval since the previous logged line: it used to appear only in HF's own+ # tqdm bar ("1.84s/it") and its per-step train_tokens_per_second print, both raw stdout rather+ # than structured.elapsed = p.elapsed_secondstokens = p.num_tokenss_per_step = tok_per_s = None@@ -3141,11 +3115,8 @@ class TrainingBackend:s_per_step = round(d_time / d_steps, 3)if tokens is not None and prev_tokens is not None and tokens > prev_tokens:tok_per_s = round((tokens - prev_tokens) / d_time, 1)- # The first logged line reports no throughput on purpose: elapsed_seconds is- # wall time since the worker started, which includes imports, the model- # download and load and the dataset build, and on a resumed run the step and- # token counters predate this process entirely. Dividing by it would report a- # number nobody wants. The next line has a real in-training interval.+ # The first logged line reports no throughput on purpose: elapsed_seconds is wall time since the+ # worker started (imports, download, load, dataset build), and a resumed run's counters are older.self._last_progress_log_ts = nowself._last_progress_log_step = step@@ -3178,7 +3149,7 @@ class TrainingBackend:):self._run_intent_lock.release()return- self._db_create_in_progress = True # only one caller creates+ self._db_create_in_progress = Truejob_id = self.current_job_iddb_config = self._db_configstarted_at = self._db_started_at or datetime.now(timezone.utc).isoformat()@@ -3215,9 +3186,8 @@ class TrainingBackend:logger.warning("Failed to create DB run record for early failure", exc_info = True)finally:with self._lock:- # Publish the flags only if this is still the current run. A killed worker lets a new /start- # proceed mid-create, and these flags are backend-wide, so a stale create must not satisfy- # the new run's DB state (the row was created by id; the new run creates its own).+ # Publish the flags only if this is still the current run: they are backend-wide, and a killed+ # worker lets a new /start proceed mid-create.if self.current_job_id == job_id:if created:self._db_run_created = True # publish only after the insert commits@@ -3351,8 +3321,6 @@ class TrainingBackend:)return events- # --- Plot generation ----def _create_loss_plot(self,progress: TrainingProgress,diff --git a/studio/backend/core/youtube_transcript.py b/studio/backend/core/youtube_transcript.pyindex 9ec5cd27d44..0bec641ba86 100644--- a/studio/backend/core/youtube_transcript.py+++ b/studio/backend/core/youtube_transcript.py@@ -41,8 +41,7 @@ _TIMEOUT = httpx.Timeout(20.0)# Captions are text; a 4 MB track is already an outlier for a very long video._MAX_CAPTION_BYTES = 4 * 1024 * 1024_MAX_PLAYER_BYTES = 4 * 1024 * 1024-# Roughly 25k tokens. A three hour video's captions run past 200k characters, which-# would swallow the model's context window on its own.+# Roughly 25k tokens: a three hour video's captions would swallow the model's context window on their own._MAX_TRANSCRIPT_CHARS = 100_000# Timedtext normally answers 200, but a hop is re-validated rather than refused._MAX_CAPTION_REDIRECTS = 3@@ -201,8 +200,8 @@ def _select_track(base = wanted.split("-")[0]for want_generated in (False, True):candidates = [t for t in tracks if (t.get("kind") == "asr") is want_generated]- # exact locale before the base-language fallback: a pt-BR request must not- # take a pt-PT track just because it is listed first+ # exact locale before the base-language fallback: a pt-BR request must not take a pt-PT track just+ # because it is listed firstfor matches_wanted in (lambda code: code == wanted,lambda code: code.split("-")[0] == base,@@ -257,8 +256,8 @@ async def _fetch_track_text(client: httpx.AsyncClient, base_url: str) -> str:body = b""for _ in range(_MAX_CAPTION_REDIRECTS + 1):- # Redirects are followed by hand so the host allowlist covers every hop, not- # just the URL the player response handed us.+ # Redirects are followed by hand so the host allowlist covers every hop, not just the URL the player+ # response handed us.async with client.stream("GET", url, headers = {"User-Agent": _USER_AGENT}, follow_redirects = False) as response:diff --git a/studio/backend/hub/routes/inventory.py b/studio/backend/hub/routes/inventory.pyindex 1a779cd918b..bd2fd37eaa3 100644--- a/studio/backend/hub/routes/inventory.py+++ b/studio/backend/hub/routes/inventory.py@@ -59,8 +59,8 @@ async def list_local_models(return await local_inventory.list_local_models_response(models_dir)-# Plain `def` (not async): synchronous SQLite + filesystem work runs in-# FastAPI's thread-pool instead of blocking the event loop.+# Plain def, not async: synchronous SQLite and filesystem work runs in FastAPI's thread pool instead+# of blocking the event loop.@router.get("/scan-folders", response_model = ScanFoldersResponse)def get_scan_folders(current_subject: str = Depends(get_current_subject)):return local_inventory.get_scan_folders_response()diff --git a/studio/backend/hub/schemas/downloads.py b/studio/backend/hub/schemas/downloads.pyindex 9397bdc9196..93d4ae105d7 100644--- a/studio/backend/hub/schemas/downloads.py+++ b/studio/backend/hub/schemas/downloads.py@@ -77,16 +77,15 @@ class DownloadStartResponse(BaseModel):job_key: strstate: straccepted: bool- # True when this start attached to a job already running rather than- # starting one. Accepted either way.+ # True when this start attached to a job already running rather than starting one.attached: bool = Falsegeneration: int- # The transport the job is really on: the one the backend resolved for a- # fresh start, or the running job's own when this start adopted one.- # Either can differ from what the client asked for.+ # The transport the job is really on, which can differ from the one the client asked for.transport: Optional[str] = None- # Set only when an adopted job had fallen back from Xet to HTTP: stopping- # it still writes the original marker, so it is a restart, not a resume.+ # Set only when an adopted job had fallen back from Xet to HTTP: stopping it still writes the+ # original marker, so it is a restart, not a resume.+ # Set only on a job that fell back from Xet to HTTP mid-flight: cancelling it still writes the original+ # transport's marker, so the partial is restart-only even though the worker is on resumable HTTP.cancel_transport: Optional[str] = None@@ -101,9 +100,6 @@ class ActiveDownload(BaseModel):repo_id: Optional[str] = Nonevariant: Optional[str] = Nonetransport: Optional[str] = None- # Set only on a job that fell back from Xet to HTTP mid-flight: cancelling- # it still writes the original transport's marker, so the partial is- # restart-only even though the worker is on resumable HTTP.cancel_transport: Optional[str] = Nonestate: strfiles: Optional[List[str]] = Field(@@ -135,8 +131,8 @@ class TransportCapabilities(BaseModel):xet: TransportCapabilityauto_resolves_to: Literal["xet", "http"] = "xet"auto_reason: Optional[str] = None- # False when the installed huggingface_hub refetches an interrupted file from zero, so no- # transport can offer a byte-resume and the UI must not label one.+ # False when the installed huggingface_hub refetches an interrupted file from zero, so no transport+ # can offer a byte-resume and the UI must not label one.partials_resumable: bool = True@@ -148,8 +144,8 @@ class TransportStatusResponse(BaseModel):class DownloadProgressResponse(BaseModel):downloaded_bytes: int- # Finalized-blob bytes only (no ``.incomplete``). Registry-loss completion- # fallbacks key off this so a partial isn't mistaken for a finished download.+ # Finalized-blob bytes only: registry-loss completion fallbacks key off this, so a partial is not+ # mistaken for a finished download.completed_bytes: int = 0complete_on_disk: bool = Field(False,diff --git a/studio/backend/hub/schemas/inventory.py b/studio/backend/hub/schemas/inventory.pyindex de83251bedb..c7e31077f00 100644--- a/studio/backend/hub/schemas/inventory.py+++ b/studio/backend/hub/schemas/inventory.py@@ -241,8 +241,8 @@ class CachedRepoBase(BaseModel):runtime: ModelRuntime = "unknown"format_variant: Optional[str] = Nonecapabilities: LocalModelCapabilities = Field(default_factory = LocalModelCapabilities)- # Inferred pipeline task ("text-to-image" / "text-to-video" / a chat task / None). The task-scoped pickers filter On- # Device rows on it and the chat picker routes a diffusion pick by it, so a row without one is dropped from those lists.+ # The task-scoped pickers filter On Device rows on the inferred task and the chat picker routes a+ # diffusion pick by it, so a row without one is dropped from those lists.task: Optional[str] = Noneaudio_type: Optional[str] = None@@ -270,17 +270,16 @@ class CachedModelRepo(CachedRepoBase):pipeline_tag: Optional[str] = Nonelibrary_name: Optional[str] = Nonetags: Optional[List[str]] = None- # True for a diffusion-tagged repo with NO top-level model_index.json: a single-file checkpoint needing from_single_file- # + a filename. Pickers must not offer it as a pipeline load unless the catalog carries a curated artifact for it.+ # True for a diffusion-tagged repo with NO top-level model_index.json: a single-file checkpoint+ # needing from_single_file plus a filename. Pickers must not offer it as a pipeline load unless+ # the catalog carries a curated artifact.single_file: bool = False- # True for an sd.cpp companion mirror: a VAE / text-encoder repo with no denoiser, so it is- # never a pick on ANY page. It still gets a row, because these run to tens of GB and the row- # is how they are seen and deleted; the pickers filter on this instead.+ # An sd.cpp companion mirror is never a pick on any page, but still gets a row, because these run to+ # tens of GB and the row is how they are seen and deleted.companion: bool = False- # True when the SELECTED revision is a diffusers pipeline. An unrecognised one carries no- # task and no root config for can_chat, so this flag is all that keeps it out of a chat- # picker. Declared because response_model drops undeclared keys, which left the CLI- # (reading the dict in-process) and the frontend disagreeing about the same row.+ # An unrecognised pipeline carries no task and no root config for can_chat, so this flag is all+ # that keeps it out of a chat picker. Declared because response_model drops undeclared keys, which+ # left the CLI and the frontend disagreeing about the same row.diffusers: bool = Falsediff --git a/studio/backend/hub/services/datasets/cache_inventory.py b/studio/backend/hub/services/datasets/cache_inventory.pyindex b70bad1c04b..f627563d2e6 100644--- a/studio/backend/hub/services/datasets/cache_inventory.py+++ b/studio/backend/hub/services/datasets/cache_inventory.py@@ -201,21 +201,21 @@ _DATASET_NON_PAYLOAD_FILENAMES = ()-# suffixes no loader can turn into rows: none is in `datasets`' extension map, and a script also-# needs `trust_remote_code`, which no load path here passes and `datasets>=4` dropped.+# suffixes no loader can turn into rows; a script also needs trust_remote_code, which no load path+# here passes and datasets>=4 dropped._DATASET_NON_PAYLOAD_SUFFIXES = frozenset({".cff", ".md", ".py", ".pyc"})def _is_payload_dir(name: str) -> bool:- # the only rule `datasets` applies to a directory, which also covers a Mac zip's `__MACOSX`.- # the metadata FILE names must not be applied here: `license/train.parquet` loads fine, and- # pruning that subtree hid the dataset from On Device.+ # the only rule datasets applies to a directory, which also covers a Mac zip's __MACOSX. the+ # metadata FILE names must not be applied here: license/train.parquet loads fine, and pruning that+ # subtree hid the dataset from On Device.return not name.startswith(".") and not name.startswith("__")def _is_payload_name(name: str) -> bool:# as above, plus the metadata names: AppleDouble sidecars, every dotfile the list does not- # enumerate, and the cards and licences a cancelled download leaves behind.+ # enumerate, and the cards a cancelled download leaves behind.if not _is_payload_dir(name):return Falsereturn name.lower() not in _DATASET_NON_PAYLOAD_FILENAMES@@ -224,21 +224,20 @@ def _is_payload_name(name: str) -> bool:def _is_payload_file(name: str) -> bool:if not _is_payload_name(name):return False- # the resolver's suffix rule, which walks the whole chain and drops a trailing compression- # suffix: `train.parquet.backup` is still parquet, `data.py.gz` is still a script.+ # the resolver's suffix rule drops a trailing compression suffix: train.parquet.backup is still+ # parquet, data.py.gz is still a script.suffix = local_options._data_suffix(name)return suffix is None or suffix.lower() not in _DATASET_NON_PAYLOAD_SUFFIXESdef _is_present_payload_file(path: Path) -> bool:- # existence is not enough. `_empty_payload` covers both shapes that look like payload and- # are not: a zero-byte file, and a `blobs/` link whose blob was pruned.+ # existence is not enough: a zero-byte file, and a blobs/ link whose blob was pruned, both look like+ # payload and are not.if not _is_payload_file(path.name):return Falseif local_options._empty_payload(path):return False- # bytes are not rows: `_rowless` probes a header-only csv or a `[]` json, reading a short- # head for those two builders only. it drops the file, not the snapshot, as datasets does.+ # bytes are not rows: a header-only csv or a [] json drops the file, not the snapshot, as datasets does.module = local_options._file_module(path.name)return module is None or not local_options._rowless(path, path.name, module)@@ -255,9 +254,8 @@ def _snapshot_holds_payload(snapshot: Path) -> Optional[bool]:nonlocal unreadableunreadable = True- # roots still to walk, and every directory already walked or queued by resolved path. a- # junction pointing at its own ancestor resolves back inside the snapshot, so containment- # alone leaves `data/loop/loop/...` descending until the path length gives out.+ # a junction pointing at its own ancestor resolves back inside the snapshot, so containment alone+ # leaves data/loop/loop/... descending until the path length gives out.seen: set[Path] = set()pending: list[Path] = []@@ -286,30 +284,28 @@ def _snapshot_holds_payload(snapshot: Path) -> Optional[bool]:base = Path(directory)kept = []for name in dirnames:- # nothing under a hidden or `__`-prefixed dir can supply rows, so- # `.hidden/notes.txt` must not clear `partial`.+ # nothing under a hidden or __-prefixed dir can supply rows, so .hidden/notes.txt must not clear+ # partial.if not _is_payload_dir(name):continueentry = base / name- # containment, not a link-type test: `is_symlink()` is false for a Windows- # junction and `is_junction()` postdates 3.12, which this still supports, so- # only comparing resolved paths catches every redirect on every runtime.+ # containment, not a link-type test: is_symlink() is false for a Windows junction and is_junction()+ # postdates 3.12, so only comparing resolved paths catches every redirect.try:redirected = not entry.resolve(strict = True).is_relative_to(root)linked = entry.is_symlink()except (OSError, RuntimeError, ValueError):unreadable = Truecontinue- # walked as a root of its own, not taken as proof: migrated caches keep- # their data behind a redirect, and a stale one holds nothing.+ # walked as a root of its own, not taken as proof: migrated caches keep their data behind a+ # redirect, and a stale one holds nothing.if redirected:target = _book(entry)if target is not None:pending.append(target)continue- # a symlink back inside this root is skipped: the walk never descends one,- # and booking its target would prune the real directory whenever `alias` is- # listed before `data`. a junction reports False here, hence the visited set.+ # a symlink back inside this root is skipped, since booking its target would prune the real+ # directory whenever alias is listed before data; a junction reports False here.if linked:continueif _book(entry) is None:@@ -701,7 +697,7 @@ def _delete_cached_dataset_blocking(repo_id: str, cache_path: Optional[str] = Notarget_root = resolve_delete_target_root("dataset", repo_id, cache_path, owners.keys())# A processed-only dataset row sends its Arrow cache path, which is not a Hub datasets-- dir, so- # resolve_delete_target_root returns None. Accept it and fall through to the processed delete.+ # resolve_delete_target_root returns None.if target_root is None and not (cache_pathand (_is_processed_dataset_cache_path(repo_id, cache_path) or app_entry is not None)@@ -733,9 +729,8 @@ def _delete_cached_dataset_blocking(repo_id: str, cache_path: Optional[str] = Noexc_info = True,)- # Restrict the processed Arrow-cache delete to the selected cache's datasets root. A processed- # cache_path scopes to its own root; a Hub target to the datasets root sharing its cache home;- # an unspecified cache_path stays global (legacy).+ # A processed cache_path scopes to its own root, a Hub target to the datasets root sharing its cache+ # home, and an unspecified one stays global.processed_roots: Optional[set[Path]]if not cache_path:processed_roots = None@@ -773,7 +768,7 @@ def _delete_cached_dataset_blocking(repo_id: str, cache_path: Optional[str] = No),)- # ``scan_cache_dir()`` skips blob-only/corrupt repos the revision delete can't touch, yet the+ # scan_cache_dir() skips blob-only or corrupt repos the revision delete cannot touch, yet the# fallback scanner shows them, so purge the whole dir. Hub cache targets only.cache_purged = partial_purged = state_purged = Falseif target_root is not None:diff --git a/studio/backend/hub/services/datasets/downloads.py b/studio/backend/hub/services/datasets/downloads.pyindex f84ea4084c4..75299a25c28 100644--- a/studio/backend/hub/services/datasets/downloads.py+++ b/studio/backend/hub/services/datasets/downloads.py@@ -70,8 +70,7 @@ def get_dataset_snapshot_metadata_cached(size, hashes, restricted, cached_fp, ts = cachedif (time.monotonic() - ts) >= _DATASET_SIZE_POS_TTL:del _dataset_size_cache[repo_id]- # A gated/private repo's metadata is only served back to the token- # that fetched it; another token may have no access at all.+ # A gated or private repo's metadata is only served back to the token that fetched it.elif not restricted or cached_fp == token_fp:_dataset_size_cache.move_to_end(repo_id)return size, hashes@@ -165,7 +164,7 @@ async def download_dataset_response(key = _download_job_key(repo_id)# Off the event loop: resolving "auto" can run the Xet reachability probe, and a blackholed DNS- # makes that outlast its 3s budget while every other Unsloth request waits behind it.+ # makes that outlast its 3s budget while every other request waits behind it.use_xet, transport_reason = await asyncio.to_thread(download_lifecycle.resolve_requested_use_xet,getattr(body, "transport_mode", None),@@ -188,9 +187,8 @@ async def download_dataset_response()generation = _registry.current_generation(key)if not claimed:- # Pollable when rejected by this repo's own in-flight job, and that is- # also the only case that attached to anything; an in-progress delete- # leaves no job, so both come from ``adoptable``.+ # Both come from adoptable: an in-progress delete leaves no job, and only an in-flight job of this+ # repo attached to anything.adoptable = _registry.adoptable(key)return {"repo_id": repo_id,@@ -198,11 +196,11 @@ async def download_dataset_response("accepted": adoptable,"attached": adoptable,"generation": generation,- # An adopted job keeps the transport it started on, so report it- # rather than let the caller assume the one it asked for.+ # An adopted job keeps the transport it started on, so report it rather than let the caller assume+ # the one it asked for."transport": _registry.job_transport(key),- # And its cancel marker: a run that fell back from Xet to HTTP- # still cancels into a restart-only partial.+ # And its cancel marker: a run that fell back from Xet to HTTP still cancels into a restart-only+ # partial."cancel_transport": _registry.job_cancel_transport(key),}download_manifest.clear_cancel_marker(@@ -239,8 +237,7 @@ async def download_dataset_response("accepted": True,"attached": False,"generation": generation,- # See models: the resolved transport, which a downgrade can make- # different from the one requested.+ # See models: the resolved transport, which a downgrade can make different from the one requested."transport": transport,}diff --git a/studio/backend/hub/services/datasets/formatting.py b/studio/backend/hub/services/datasets/formatting.pyindex a4944d2f3de..7881947b6b4 100644--- a/studio/backend/hub/services/datasets/formatting.py+++ b/studio/backend/hub/services/datasets/formatting.py@@ -160,11 +160,8 @@ def _serialize_preview_value(value):value.keys() - {"bytes", "path"}):return _serialize_binary_value(raw)- # A decoded Audio cell is {"array", "sampling_rate", "path"}. torchcodec hands back- # an AudioDecoder, which falls through to str() below, but the soundfile fallback- # returns the waveform and the dataset formatter turns it into a plain list -- one- # float per sample, so ten preview rows of a few seconds each is tens of MB of JSON- # and the client dies rendering it.+ # A decoded Audio cell becomes one float per sample under the soundfile fallback, so ten preview+ # rows of a few seconds each are tens of MB of JSON and the client dies rendering it.if "sampling_rate" in value and isinstance(value.get("array"), (list, tuple)):return _serialize_decoded_audio(value)return {str(key): _serialize_preview_value(item) for key, item in value.items()}@@ -441,7 +438,7 @@ def check_format_response()if preview_slice is None:- # Tier 2: full streaming (resolves all files — slow for large repos)+ # Tier 2: full streaming (resolves all files - slow for large repos)logger.info("Tier 2: falling back to full streaming load_dataset")try:load_kwargs = {diff --git a/studio/backend/hub/services/datasets/local.py b/studio/backend/hub/services/datasets/local.pyindex 4360128a122..28655b26fb4 100644--- a/studio/backend/hub/services/datasets/local.py+++ b/studio/backend/hub/services/datasets/local.py@@ -26,8 +26,8 @@ from utils.paths.path_utils import (is_appledouble_metadata,)-# Tabular formats are preferred over archives for Tier 1 preview: archives (e.g. images.zip)-# load as ImageFolder with synthetic columns that don't match the real schema.+# Archives load as ImageFolder with synthetic columns that do not match the real schema, so tabular+# formats are preferred for Tier 1 preview._TABULAR_EXTS = (".parquet", ".json", ".jsonl", ".csv", ".tsv", ".arrow")_ARCHIVE_EXTS = (".tar", ".tar.gz", ".tgz", ".gz", ".zst", ".zip", ".txt")DATA_EXTS = _TABULAR_EXTS + _ARCHIVE_EXTSdiff --git a/studio/backend/hub/services/datasets/local_options.py b/studio/backend/hub/services/datasets/local_options.pyindex fa9ff724fe2..ee49a7fbac7 100644--- a/studio/backend/hub/services/datasets/local_options.py+++ b/studio/backend/hub/services/datasets/local_options.py@@ -220,8 +220,7 @@ def _snapshot_metadata_file(snapshot: Path, name: str) -> Optional[Path]:return path if 0 < size <= _MAX_METADATA_BYTES else None-# _read_card_metadata returns this when a card has front matter datasets cannot parse.-# DatasetCard.load raises on it, so nothing in the snapshot is loadable.+# DatasetCard.load raises on front matter datasets cannot parse, so nothing in the snapshot is loadable._UNPARSABLE_METADATA = object()@@ -238,8 +237,7 @@ def _read_card_metadata(path: Path) -> Any:except YAMLError:return _UNPARSABLE_METADATAif payload is None:- # Empty, comment-only or null front matter, which RepoCard reads as an empty- # card rather than refusing.+ # Empty, comment-only or null front matter, which RepoCard reads as an empty card rather than refusing.return {}except StopIteration:# Front matter that never closes is not front matter at all.@@ -252,24 +250,17 @@ def _read_card_metadata(path: Path) -> Any:return payload if isinstance(payload, dict) else _UNPARSABLE_METADATA+# Mirrors datasets' get_data_patterns (sharded data/{split}-NNNNN files, then directory keywords,+# then filename keywords, then one train split) without importing datasets, which+# dataset_cache.py forbids on cache paths.+# An offered split has to be trainable, and this is deliberately tighter in two places: a file+# whose symlink leaves the repository is refused, and only trainable extensions are offered.++# Past this the result would depend on traversal order, so nothing is offered.# --- Split inference for cached datasets whose card declares nothing -------------------#-# datasets resolves a metadata-free directory with get_data_patterns: it tries sharded-# data/{split}-NNNNN-of-NNNNN files, then directory-name keywords, then filename keywords,-# and falls back to one train split holding everything. The first stage that names any-# split wins, and one builder is chosen for all of them. This mirrors that without-# importing datasets, which dataset_cache.py forbids on cache paths.-#-# The contract is that an offered split has to be trainable: anything looser is offered and-# then 422s, anything tighter hides a usable option. Deliberately tighter in two places,-# both to keep training inside the cache: a file whose symlink leaves the repository is-# refused, and only the extensions Unsloth can train on are offered.--# Only names are read during the scan, so this can sit well above any real snapshot. Past-# it the result would depend on traversal order, so nothing is offered._MAX_SNAPSHOT_DATA_FILES = 200_000-# datasets drops these by basename before it infers anything, so a metadata-only cache is-# empty rather than a bogus train split.+# datasets drops these by basename before it infers anything, so a metadata-only cache is empty+# rather than a bogus train split._IGNORED_DATA_FILENAMES = frozenset({"README.md",@@ -283,11 +274,11 @@ _IGNORED_DATA_FILENAMES = frozenset(# What fsspec can decompress in an Unsloth install, as suffixes that sit after the real one._COMPRESSION_EXTENSIONS = frozenset({".gz", ".gzip", ".bz2", ".xz", ".zip"})# Named by datasets but needing codecs an Unsloth install does not ship, so they raise-# "Compression type not supported" and offering them would put a dead split in the picker.-# .lzma is the legacy alone-format: datasets registers .xz for its filter, not this.+# "Compression type not supported" and would put a dead split in the picker; .lzma is the legacy+# alone-format, for whose filter datasets registers .xz._UNREADABLE_COMPRESSION = frozenset({".zst", ".zstd", ".lz4", ".lzma"})-# datasets picks one builder for the whole dataset, from the extensions it finds. Splits-# that disagree make it raise, so a snapshot mixing formats is not offerable at all.+# datasets picks one builder for the whole dataset, and splits that disagree make it raise, so a+# snapshot mixing formats is not offerable at all._MODULE_EXTENSIONS = {"arrow": ".arrow","csv": ".csv",@@ -300,19 +291,15 @@ _MODULE_EXTENSIONS = {"xml": ".xml","hdf5": ".h5 .hdf5",}-# The folder builders. datasets registers only these in both letter cases, so only their-# extensions match case-insensitively; everything else is matched by a case-sensitive glob.+# datasets registers only the folder builders in both letter cases, so only their extensions match case-insensitively._FOLDER_EXTENSIONS = frozenset(- # imagefolder".apng .blp .bmp .bufr .bw .cur .dcx .dds .dib .emf .eps .fit .fits .flc .fli .ftc .ftu "".gbr .gif .grib .icb .icns .ico .iim .im .j2c .j2k .jfif .jp2 .jpc .jpe .jpeg .jpf "".jpg .jpx .msp .pbm .pcd .pcx .pgm .png .pnm .ppm .ps .psd .pxr .ras .rgb .rgba .sgi "".tga .tif .tiff .vda .vst .webp .wmf .xbm .xpm "- # audiofolder".3g2 .3gp .aiff .asf .au .avr .caf .f4v .flac .flv .htk .ircam .m4v .mat4 .mat5 .mp3 "".mpc2k .mpg .mxf .nist .nut .ogg .ogm .opus .paf .pvf .raw .rf64 .sd2 .sds .svx .voc "".w64 .wav .wavex .webm .wma .wmv .wve .xi "- # videofolder and pdffolder".avi .mkv .mov .mp4 .mpeg .pdf".split())_EXTENSION_MODULES = {@@ -321,8 +308,8 @@ _EXTENSION_MODULES = {# datasets infers a split's module from its first 200 files, in resolved (sorted) order._MAX_MODULE_INFERENCE_FILES = 200# datasets' tie-break once the counts are level, then the extension string itself.-# TRAINING_DATA_EXTS only ever resolves to these, so any other builder means the-# snapshot holds nothing trainable and no file needs opening to find that out.+# TRAINING_DATA_EXTS only ever resolves to these, so any other builder means the snapshot holds+# nothing trainable._TRAINABLE_MODULES = frozenset({"csv", "json", "parquet"})_ROW_PROBE_BYTES = 8192_EXTENSION_PRIORITY = (".parquet", ".jsonl", ".json", ".csv")@@ -348,8 +335,8 @@ _DIR_NAME_KEYWORD_PATTERNS = ()# data/{split}-NNNNN-of-NNNNN*.*, where a * matches nothing as happily as something._SHARDED_DATA_RE = re.compile(r"^data/(?P<split>[^/]*)-[0-9]{5}-of-[0-9]{5}[^/]*\.[^/]*$")-# datasets' own split grammar, which is stricter than the one the picker accepts. A shard-# named outside it makes the whole snapshot unloadable rather than falling through.+# datasets' own split grammar is stricter than the one the picker accepts: a shard named outside it+# makes the whole snapshot unloadable rather than falling through._SHARD_SPLIT_RE = re.compile(r"^\w+(\.\w+)*$")@@ -423,15 +410,15 @@ def _snapshot_data_files(snapshot: Path) -> Optional[list[PurePosixPath]]:for filename in filenames:if filename in _IGNORED_DATA_FILENAMES or filename.startswith("."):continue- # resolve_pattern keeps a link only when its target is a file, so a dangling- # one is not a file datasets sees, let alone one that condemns its split.+ # resolve_pattern keeps a link only when its target is a file, so a dangling one is not a file+ # datasets sees.if not (base / filename).is_file():continue- # Files with no builder of their own are kept: they cannot win the vote, but a- # split holding nothing else is one datasets refuses to build.+ # Files with no builder of their own are kept: they cannot win the vote, but a split holding nothing+ # else is one datasets refuses to build.if len(found) >= _MAX_SNAPSHOT_DATA_FILES:- # Past the cap this is a traversal-order prefix, which cannot be compared- # with what the loader would resolve.+ # Past the cap this is a traversal-order prefix, which cannot be compared with what the loader would+ # resolve.return Nonefound.append(PurePosixPath((relative / filename).as_posix()))found.sort(key = lambda path: path.as_posix())@@ -530,8 +517,8 @@ def _offerable(entries: list[PurePosixPath], snapshot: Path, module: str) -> Optresolved = resolved_dataset_snapshot_file(snapshot, path.as_posix())if resolved is None or _blocked_by_compression(path.name, module):return None- # Once a builder is chosen, datasets reads only what that builder claims, so a- # training file left behind by a folder builder is not data this split offers.+ # Once a builder is chosen datasets reads only what that builder claims, so a training file left+ # behind by a folder builder is not data this split offers.if _file_module(path.name) != module or not _trainable_name(path.name):continueif _empty_payload(resolved):@@ -540,8 +527,8 @@ def _offerable(entries: list[PurePosixPath], snapshot: Path, module: str) -> Optif _rowless(resolved, path.name, module):continuetrainable = True- # Every builder but json fails outright on a file with no rows, and datasets prepares- # every split before handing one back, so such a file condemns its siblings too.+ # Every builder but json fails outright on a file with no rows, and datasets prepares every split+ # before handing one back, so such a file condemns its siblings too.if empty and module != "json":return Nonereturn trainable@@ -642,8 +629,8 @@ def _declares_configs(snapshot: Path, name: str) -> bool:if not isinstance(payload, dict):# DatasetCardData updates a dict from it, which raises on anything else.return bool(payload)- # Only configs count: 4.3.0 builds no config from dataset_info declared here, so a- # feature schema in this file leaves the loader inferring the files by pattern.+ # Only configs count: 4.3.0 builds no config from dataset_info declared here, so a feature schema+ # leaves the loader inferring the files by pattern.return bool(payload.get("configs"))@@ -662,9 +649,8 @@ def _malformed_info(payload: Any) -> bool:def _snapshot_options(snapshot: Path) -> set[tuple[str, str]]:options: set[tuple[str, str]] = set()- # Metadata we cannot read still names the loader's configs, so inference must not- # step in beside it: a card too large or too unsafe to open, or the standalone yaml- # file, which datasets reads as a card and we do not.+ # Metadata we cannot read still names the loader's configs, so inference must not step in beside it:+ # a card too large or unsafe to open, or the standalone yaml file.declared = _unreadable_metadata(snapshot, "README.md") or _declares_configs(snapshot, ".huggingface.yaml")@@ -675,15 +661,15 @@ def _snapshot_options(snapshot: Path) -> set[tuple[str, str]]:# datasets raises out of DatasetCard.load, so no option here would ever start.return optionsif isinstance(card_data, dict):- # datasets merges the standalone YAML into the card, so a README that- # declares nothing does not undo a declaration made there.+ # datasets merges the standalone YAML into the card, so a README that declares nothing does not undo+ # a declaration made there.declared = declared or bool(card_data.get("configs"))_add_config_options(options, card_data.get("configs"))named = len(options)info = card_data.get("dataset_info")_add_dataset_info_options(options, info)- # dataset_info carrying only a feature schema names no config, so datasets- # still resolves the files by pattern and inference has to run.+ # dataset_info carrying only a feature schema names no config, so datasets still resolves the files+ # by pattern and inference has to run.declared = (declared or len(options) > named or _malformed_info(info) or _declares_splits(info))@@ -691,7 +677,7 @@ def _snapshot_options(snapshot: Path) -> set[tuple[str, str]]:for filename in ("dataset_infos.json", "dataset_info.json"):metadata = _snapshot_metadata_file(snapshot, filename)if metadata is None:- # datasets json.loads dataset_infos.json unconditionally, so one it cannot+ # datasets json.loads dataset_infos.json unconditionally while resolving configs, so one it cannot# read raises before any split exists, inferred or not.declared = declared or (filename == "dataset_infos.json" and _metadata_present(snapshot, filename)@@ -699,17 +685,16 @@ def _snapshot_options(snapshot: Path) -> set[tuple[str, str]]:continuepayload = _safe_json_file(metadata, snapshot, allow_snapshot_symlink = True)if filename == "dataset_infos.json":- # datasets json.loads this one while resolving configs, so a file it cannot- # parse raises before any split exists, inferred or not.+ # datasets json.loads this one while resolving configs, so a file it cannot parse raises before any+ # split exists, inferred or not.declared = declared or payload is None_add_dataset_info_options(options, payload)else:_add_info_options(options, payload)if not options and not declared:- # Nothing was declared, which is the case #8140 reports: the loader still resolves- # the files by pattern, so the picker infers the same splits it would. A card that- # did declare something names its own configs, which inference cannot reproduce.+ # Nothing was declared, the case #8140 reports: the loader still resolves the files by pattern, so+ # the picker infers the same splits.options.update(_inferred_snapshot_options(snapshot))return optionsdiff --git a/studio/backend/hub/services/download_lifecycle.py b/studio/backend/hub/services/download_lifecycle.pyindex 787916ffec3..50997ca75da 100644--- a/studio/backend/hub/services/download_lifecycle.py+++ b/studio/backend/hub/services/download_lifecycle.py@@ -82,16 +82,11 @@ def resolve_auto_use_xet() -> tuple[bool, str]:if health is not None and not health.use_xet:return (False, reason)if _health_is_forced(health):- # ``UNSLOTH_FORCE_XET=1``. The zoo's off switches (``UNSLOTH_DISABLE_XET``,- # ``UNSLOTH_STABLE_DOWNLOADS``, ``HF_HUB_DISABLE_XET``) already win above, so the on switch- # has to win here or the pair is asymmetric and the only escape from the RAM rule is- # editing the request. Buffers are still clamped to free RAM; only the transport choice is- # left to the operator, the same stand-down the zoo makes for a user-set- # HF_XET_HIGH_PERFORMANCE.+ # UNSLOTH_FORCE_XET=1 has to win here, since the zoo's off switches already win above and the pair+ # would otherwise be asymmetric; buffers are still clamped to free RAM.return (True, reason)- # Health said Xet, or had no opinion at all. Free RAM is separate evidence, read from a- # different zoo module, so it still gets a say: a missing health module says nothing about- # whether this machine can afford Xet right now.+ # Free RAM is separate evidence, read from a different zoo module: a missing health module says+ # nothing about whether this machine can afford Xet right now.pressure = _memory_pressure_reason()if pressure is not None:return (False, pressure)@@ -188,31 +183,28 @@ def spawn_worker(env["HF_HUB_DISABLE_TELEMETRY"] = "1"env["HF_HUB_DISABLE_XET"] = "0" if use_xet else "1"if use_xet:- # hf_xet sizes its buffers from constants, not from the machine, and reads its config- # natively at import, so the worker's env has to be sized here. unsloth_zoo decides, so- # there is one rule and not two: it sizes from RAM/cores/disk and honours a user-set- # high-performance flag (standing its own sizing down, since xet-core voids it anyway).- # UNSLOTH_XET_FORCE_CAPS=1 bounds the machine regardless. Unsloth hand-rolled this, and the- # copies drifted: on a 2TB host the worker got a 24GB laptop's buffer and ran 3.4x slower.+ # hf_xet sizes its buffers from constants and reads its config natively at import, so the worker's+ # env is sized here, and by unsloth_zoo so there is one rule and not two (UNSLOTH_XET_FORCE_CAPS=1+ # bounds the machine regardless): the hand-rolled copies drifted, and on a 2TB host the worker got+ # a laptop's 24GB buffer and ran 3.4x slower.from utils import hf_xet_fallback- # The worker's own cache, which the sizing measures: this backend's env may still name the- # one it started with, since moving the cache in Settings does not rewrite the live process.+ # The worker's own cache, which the sizing measures: this backend's env may still name the one it+ # started with, since moving the cache in Settings does not rewrite the live process.sized = hf_xet_fallback.apply_xet_env(env, env.get("HF_HUB_CACHE"))if sized is None and not _allow_high_performance():- # No tuning module: that unsloth_zoo is also the one setting HF_XET_HIGH_PERFORMANCE=1- # at import, and `env` is seeded from the parent, so that inherited "1" would hand the- # worker a 64GB ceiling (xet-core applies the preset AFTER reading the environment).+ # No tuning module: that unsloth_zoo is also the one setting HF_XET_HIGH_PERFORMANCE=1 at import,+ # and the inherited "1" would hand the worker a 64GB ceiling, since xet-core applies the preset+ # AFTER reading the environment.for key in ("HF_XET_HIGH_PERFORMANCE", "HF_XET_HP"):env[key] = "0"- # No token in Unsloth settings: fall back to the backend's own HF_TOKEN so- # private repos stay downloadable (needed while inkling repos are private).- # Not for a repo an API caller named: that would lend them the owner's identity.+ # Fall back to the backend's own HF_TOKEN so private repos stay downloadable, but never for a repo+ # an API caller named: that would lend them the owner's identity.if not hf_token and allow_ambient_token:hf_token = os.environ.get("HF_TOKEN") or Noneenv["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "0" if hf_token else "1"- # hf_transfer's parallel Range chunks can leave sparse partials even in- # "http" mode; disable so the worker's writer is always sequential.+ # hf_transfer's parallel Range chunks can leave sparse partials even in "http" mode, so disable it+ # and keep the worker's writer sequential.env["HF_HUB_ENABLE_HF_TRANSFER"] = "0"for token_key in ("HF_TOKEN",@@ -248,9 +240,8 @@ def spawn_worker(return procfinally:if use_xet:- # Tie the sizing's RAM reservation to the worker, so it frees when the worker exits and- # a sibling starting in this window sizes against the remainder. A spawn that raised- # passes None, which drops the reservation instead of leaking it.+ # Tie the sizing's RAM reservation to the worker so it frees when the worker exits and a sibling+ # sizes against the remainder; a spawn that raised passes None, dropping the reservation.from utils import hf_xet_fallbackhf_xet_fallback.bind_worker_budget(proc.pid if proc is not None else None)@@ -375,10 +366,8 @@ def finalize_worker_exit(if state == "complete":hf_cache_scan.invalidate_hf_cache_scans()registry.set_job(key, "complete")- # Where /v1 learns a new model exists: its resolver answers from a cached scan- # with no watcher, so it would report the model absent and serve whatever is- # resident. Models only: noting a dataset id as a local model would refuse a- # bare request naming it instead of letting a foreign id fall through.+ # Where /v1 learns a new model exists: its resolver answers from a cached scan with no watcher.+ # Models only, since noting a dataset id as a local model would refuse a bare request naming it.if repo_type == "model":try:from core.inference.local_model_resolver import (@@ -389,8 +378,7 @@ def finalize_worker_exit(note_downloaded(repo_id)invalidate_index(additions_only = True)- # Rebuild here, not on the first request, to keep the scan off the- # request path.+ # Rebuild here, not on the first request, to keep the scan off the request path.warm_index_soon()except Exception:pass@@ -404,8 +392,8 @@ def finalize_worker_exit(else:logger.info(f"{log_prefix} worker diagnostics for {label}: {stderr_text}")logger.info(f"{log_prefix} complete: {label}")- # Defensive cleanup: the canonical clear is at download-start; this- # catches the rare case where that failed but the download succeeded.+ # Defensive cleanup: the canonical clear is at download-start; this catches the rare case where that+ # failed but the download succeeded.if repo_type and repo_id:try:download_manifest.clear_cancel_marker(@@ -417,8 +405,8 @@ def finalize_worker_exit(except Exception as exc:logger.debug(f"clear_cancel_marker failed for {repo_id} (rc=0): {exc}")elif state == "cancelled":- # Read metadata before the terminal set_job so a concurrent eviction- # can't drop it; the job key is the fallback variant label.+ # Read metadata before the terminal set_job so a concurrent eviction cannot drop it; the job key is+ # the fallback variant label.registry.set_job(key, "cancelled")logger.info(f"{log_prefix} cancelled: {label} (rc={rc})")download_registry.persist_cancel_marker(@@ -616,9 +604,8 @@ def _try_transport_retry(claimed, conflict_state = registry.claim(key,- # A XET retry re-claims the SAME transport, the permissive case in claim(): only a- # cross-transport claim serializes against a sibling, since only that can mix an HTTP- # resume with a XET rewrite over one shared blob.+ # A XET retry re-claims the SAME transport, the permissive case in claim(): only a cross-transport+ # claim can mix an HTTP resume with a XET rewrite over one shared blob.retry_transport,repo_type = repo_type,repo_id = repo_id,@@ -631,14 +618,14 @@ def _try_transport_retry(cancel_marker_transport = original_metadata.transport,hub_cache = original_metadata.hub_cache,xet_cache = original_metadata.xet_cache,- # Carry the scoped file list across the reclaim: the record it overwrites is what a later start for this scope slot is- # compared against, so dropping it makes an identical start read as a different file set and 409 instead of adopting.+ # Carry the scoped file list across the reclaim: the record it overwrites is what a later start is+ # compared against, so dropping it makes an identical start 409 instead of adopting.scoped_files = original_metadata.scoped_files or None,)if claimed:break- # An STT owner is not an active job, so mark_pending_cancel cannot reach- # this loop; waiting it out here would be an uninterruptible spin.+ # An STT owner is not an active job, so mark_pending_cancel cannot reach this loop and waiting it+ # out would be an uninterruptible spin.if conflict_state in ("deleting", "repository_owned"):logger.debug("%s XET retry claim rejected for %s; blocked by %s",@@ -671,7 +658,8 @@ def _try_transport_retry(args.append("--dataset")elif variant:args.extend(["--variant", variant])- # A scoped job must retry as the SAME scoped download; without its file list the recovery worker would fall through to a full snapshot.+ # A scoped job must retry as the SAME scoped download; without its file list the recovery worker+ # would fall through to a full snapshot.if original_metadata.scoped_files:args.extend(["--files-json", write_files_manifest(original_metadata.scoped_files)])@@ -724,10 +712,8 @@ def _try_transport_retry()registry.update_job_transport(key, original_metadata.transport)if retry_over_xet:- # The EXTRA worker could not be started (fd exhaustion, a broken interpreter path): a- # local failure, not a verdict on the download, so drop to the rung this job would have- # taken without the retry instead of stranding it in "error". Bounded: the HTTP- # direction never re-enters this branch.+ # A failed EXTRA worker spawn is a local failure, not a verdict on the download, so drop to the+ # rung this job would have taken without the retry instead of stranding it in "error".logger.warning("%s XET retry could not be spawned for %s; falling back to HTTP",log_prefix,@@ -936,11 +922,11 @@ def _start_stall_watchdog()on_stall(message)try:- # Kill only: the _watch thread is already reaping this process, so a wait() here would- # race it for the exit status.+ # Kill only: the _watch thread is already reaping this process, so a wait() here would race it for+ # the exit status.proc.kill()except ProcessLookupError:- pass # already exited between the stall verdict and the kill+ passexcept Exception:logger.exception("%s failed to kill stalled worker for %s", log_prefix, label)@@ -951,18 +937,14 @@ def _start_stall_watchdog(cache_dir = cache_dir,on_stall = _on_stall,child_pid = proc.pid,- # Scope the measurement to partials this worker holds open; otherwise the shared helper- # stays repo-wide, child_pid does nothing, and two concurrent same-transport GGUF- # variants of one repo reset each other's stall timer. This scopes the DATA clock only:- # before its first byte a variant is still covered by the shared peer-progress check,- # repo-wide by design so a lock wait behind a live sibling is not read as a hang.+ # Scope the measurement to partials this worker holds open; otherwise the shared helper stays+ # repo-wide, child_pid does nothing, and two concurrent same-transport GGUF variants of one repo+ # reset each other's stall timer. The DATA clock only: before its first byte a variant is still+ # covered by the repo-wide peer-progress check.watch_new_partials_only = True,- # The shared 90s zero-byte default assumes a single-file download whose pre-byte phase- # is one HEAD. This worker calls snapshot_download(max_workers=1), so its pre-byte phase- # is a model_info lookup with retries plus one sequential HEAD per file, which for an- # already-cached repo is the ENTIRE job with no byte written. A few hundred files on a- # slow link legitimately exceeds 90s, so a healthy worker would be killed before it- # started.+ # The shared 90s zero-byte default assumes a single-file download whose pre-byte phase is one+ # HEAD; snapshot_download(max_workers=1) does a model_info lookup plus one sequential HEAD per+ # file, which for an already-cached repo is the entire job with no byte written.connect_timeout = 600.0,xet_disabled = False,)@@ -1007,23 +989,22 @@ def register_worker(_get_metadata = getattr(registry, "get_job_metadata", None)_metadata = _get_metadata(key) if callable(_get_metadata) else None_cache_dir = getattr(_metadata, "hub_cache", None) if _metadata is not None else None- # The variant's own blobs, so a sibling variant writing into the same repo is not counted as- # this job's progress. None for job shapes that cannot have a concurrent same-repo sibling.+ # The variant's own blobs, so a sibling variant writing into the same repo is not counted as this job's progress._own_blob_hashes = (getattr(_metadata, "blob_hashes", frozenset())if getattr(_metadata, "variant", None)else None)- # Companions (a shared mmproj) live only in the progress set, and this worker was writing- # one when it died just as much as it was writing the main quant.+ # Companions such as a shared mmproj live only in the progress set, and this worker was writing one+ # as much as it was writing the main quant._owned_for_sweep = (getattr(_metadata, "progress_blob_hashes", None) or _own_blob_hashesif _own_blob_hashes is not Noneelse None)- # Sampled before the worker can write, so "did this job move bytes over Xet" is answerable on- # exit. launch_worker samples BEFORE spawn(); sampling here would race a fast child that already- # finalized its blobs, making a real transfer look like a no-op and leaving the streak uncleared.+ # Sampled before the worker can write, as launch_worker samples BEFORE spawn(): sampling later+ # would race a fast child that already finalized its blobs, making a real transfer look like a+ # no-op and leaving the streak uncleared._bytes_before = (_job_bytes_on_disk(repo_type, repo_id, _cache_dir, _own_blob_hashes)if bytes_before is _UNSAMPLED@@ -1041,10 +1022,8 @@ def register_worker()is None)- # This path had NO stall detection: it relied on the worker's exit code, and a Xet- # transfer that hangs with no progress and no error never produces one (the frozen- # progress bar on the model-hub page). Watch the cache for byte-level progress and kill- # a hung worker; the SIGKILL surfaces as "error", which triggers the HTTP retry below.+ # A Xet transfer that hangs with no progress and no error never produces an exit code, so watch+ # the cache for byte-level progress and kill it; the SIGKILL surfaces as "error".if can_retry_http:watchdog_stop = _start_stall_watchdog(registry,@@ -1072,13 +1051,11 @@ def register_worker(defer_error = can_retry_http,)if watchdog_stop is not None:- # Stop measuring once the worker is reaped: post-download work (symlinking,- # verification) makes no byte-level progress and must not read as a stall.+ # Stop measuring once the worker is reaped: post-download symlinking and verification make no byte-+ # level progress and must not read as a stall.watchdog_stop.set()- # A hung Xet transfer usually clears on a fresh process, so spend one more XET worker- # before changing transport. Only a DATA-phase verdict qualifies: retrying a pre-byte- # trip would buy a second full 600s connect window before HTTP ever starts, and that- # trip is as likely slow metadata as a broken Xet.+ # Spend one more XET worker only on a DATA-phase verdict: retrying a pre-byte trip would buy a+ # second full 600s connect window, and that trip is as likely slow metadata as a broken Xet.retry_xet = (can_retry_httpand state == "error"@@ -1086,22 +1063,14 @@ def register_worker(and _is_data_phase_stall(stalled[0])and xet_attempt < _xet_attempt_budget())- # Evidence for the WHOLE Xet phase: what earlier attempts held back, updated with this- # worker's verdict. Reported once, when the phase ends.+ # Evidence for the WHOLE Xet phase, reported once when the phase ends.xet_failure = pending_xet_failureif stalled:- # Exclude only the PRE-BYTE trip: "did not start" means no byte ever arrived, as- # likely slow metadata, a queue of HEADs or a cache lock as a broken Xet, and two- # recorded failures pin this machine to HTTP for 24h. Everything else is evidence.- # "did not resume" fires only after bytes HAVE flowed and is the shape this worker- # hangs in most often (snapshot_download owns no partial between files); an earlier- # allow-list keyed on "no progress" silently dropped it. Excluding one wording- # rather than allowing one also fails cheaply if the shared wording changes.- #- # `state == "error"` is the other half: the watchdog appends its verdict before the- # kill lands, so a worker that completed or was cancelled in that instant would be- # charged a failure it did not earn -- and on the completed path that also skips the- # success-clearing below, costing two streak steps the wrong way.+ # Exclude only the PRE-BYTE trip: "did not start" is as likely slow metadata, a queue of HEADs or a+ # cache lock as a broken Xet, and two recorded failures pin this machine to HTTP for 24h.+ # state == "error" is the other half: the watchdog appends its verdict before the kill lands, so a+ # worker that completed or was cancelled in that instant would be charged a failure it did not+ # earn.if state == "error" and _is_data_phase_stall(stalled[0]):xet_failure = stalled[0]else:@@ -1114,23 +1083,17 @@ def register_worker(if state == "cancelled" or (state == "complete" and transport == download_registry.TRANSPORT_XET):- # A XET completion proved Xet works here, and a cancel was never evidence against- # it: either way an earlier stall is dropped. An HTTP completion proves nothing- # about Xet, so a verdict carried onto that rung is still charged below.+ # A XET completion proved Xet works here and a cancel was never evidence against it; an HTTP+ # completion proves nothing about Xet, so a verdict carried onto that rung is still charged.xet_failure = Noneif xet_failure is not None and not retry_xet:# Xet phase over (HTTP next, or nothing left to try): report it, once._record_xet_failure(xet_failure, logger)xet_failure = Noneif not stalled and transport == download_registry.TRANSPORT_XET and state == "complete":- # Clear the streak so "two failures in a row" means in a row: otherwise a stall- # today and another next week count as consecutive despite every download between- # them succeeding, pinning Auto to HTTP for no reason. Only a job that actually- # moved bytes says anything about Xet's health though: a fully cached repo (the UI's- # re-download on an up-to-date model) exits 0 without touching the network, and- # clearing an earned demotion on that puts a bad machine back on Xet. Unmeasurable- # means do not clear: a missed clear costs one streak entry, a wrong clear undoes- # the demotion.+ # Clear the streak so "two failures in a row" means in a row, but only for a job that moved bytes:+ # a fully cached repo exits 0 without touching the network, and clearing an earned demotion on+ # that puts a bad machine back on Xet.bytes_after = _job_bytes_on_disk(repo_type, repo_id, _cache_dir, _own_blob_hashes)if (_bytes_before is not None@@ -1138,9 +1101,8 @@ def register_worker(and bytes_after > _bytes_before):_record_xet_success(logger)- # Recovery: spend another XET worker if the stall looked recoverable and the budget- # allows, else fall back to HTTP. One guard per direction: `transport == TRANSPORT_XET`- # (inside can_retry_http) makes the HTTP rung terminal, `xet_attempt` bounds the XET one.+ # One guard per direction: transport == TRANSPORT_XET makes the HTTP rung terminal, xet_attempt+ # bounds the XET one.if can_retry_http and state == "error":_try_transport_retry(registry,@@ -1159,20 +1121,19 @@ def register_worker(),xet_attempt = xet_attempt + 1 if retry_xet else xet_attempt,pending_xet_failure = xet_failure,- # The ORIGINAL pre-Xet baseline: resampling would fold the killed worker's- # partial writes in, so a recovered attempt would read as a cached no-op.+ # The ORIGINAL pre-Xet baseline: resampling would fold the killed worker's partial writes in, so a+ # recovered attempt would read as a cached no-op.bytes_before = _bytes_before,allow_ambient_token = allow_ambient_token,)except Exception:if watchdog_stop is not None:watchdog_stop.set()- # finalize_worker_exit is the only thing that clears running/cancelling;- # if it raises, force a terminal state so claim() isn't blocked until restart.+ # finalize_worker_exit is the only thing that clears running/cancelling, so if it raises, force a+ # terminal state or claim() is blocked until restart.logger.exception("download watcher crashed for %s", key)- # finalize may have raised before reaping the worker; terminate the- # still-registered Popen first, else the terminal set_job clears the- # repo guard and a live worker would race a retry on the same repo.+ # finalize may have raised before reaping the worker, so terminate the still-registered Popen+ # first: the terminal set_job clears the repo guard and a live worker would race a retry.try:kill_and_reap_process(proc, label = label, logger = logger)except Exception:@@ -1196,15 +1157,10 @@ def register_worker(except Exception:logger.exception("post-finalize marker cleanup failed for %s", key)try:- # The second look for anything prepare_cache_for_transport spared as too- # recently written. A download runs for long enough that a partial orphaned- # before it started is well past the grace by now, and the peer set still- # shields a same-repo variant writing a shared companion right now.- # This job's own blobs skip the abandonment wait, but ONLY once the job is- # genuinely finished. A cancelled worker's partial was written seconds ago, so- # the wait would strand it for the rest of the session; and its writer has just- # been reaped, which is the very thing the wait is there to guess at. A retry- # relaunched above leaves the job active, and then it is not ours to assume.+ # The second look for anything prepare_cache_for_transport spared as too recently written. This+ # job's own blobs skip the abandonment wait, but ONLY once the job is genuinely finished: a+ # cancelled worker's partial was written seconds ago, and a retry relaunched above leaves the job+ # active, so it is not ours to assume.terminal = registry.get_job(key).state not in ("running", "cancelling")_owned, _owns_all = (_sweep_ownership(@@ -1217,13 +1173,12 @@ def register_worker(repo_type,repo_id,protected_blob_hashes = registry.peer_blob_hashes(key),- # A companion a sibling is writing right now is still held back by- # peer_blob_hashes above, whatever this job believes it owns.+ # A companion a sibling is writing right now is still held back by peer_blob_hashes above, whatever+ # this job believes it owns.owned_blob_hashes = _owned,owns_all_blobs = _owns_all,- # The cache this worker actually wrote to. Resolving the live one instead- # would miss the orphan whenever the download location changed mid-run,- # and sweep a cache this job never touched.+ # The cache this worker actually wrote to: resolving the live one would miss the orphan whenever the+ # download location changed mid-run, and sweep a cache this job never touched.root = _cache_dir,)if swept:@@ -1254,13 +1209,12 @@ def launch_worker(watch_name: str,allow_ambient_token: bool = True,) -> str:- # Only the Xet success-recording consumes this, and sampling lazy-loads unsloth_zoo (so torch- # and transformers) on the request path. An HTTP start skips it entirely.+ # Only the Xet success-recording consumes this, and sampling lazy-loads unsloth_zoo, so torch and+ # transformers, on the request path._baseline: Optional[int] = Noneif transport == download_registry.TRANSPORT_XET:# Before spawn(), deliberately: a small download can finalize its blobs while we are still- # registering the process, and a later baseline would show no growth for a transfer that- # really happened, leaving the streak uncleared.+ # registering the process, and a later baseline would show no growth for a real transfer._get_metadata = getattr(registry, "get_job_metadata", None)_metadata = _get_metadata(key) if callable(_get_metadata) else None_baseline = _job_bytes_on_disk(@@ -1313,8 +1267,8 @@ def cancel_worker(logger,) -> str:proc = registry.get_process(key)- # No worker process yet: arm a pending cancel so register_process kills it on- # arrival during the claim-to-register window.+ # No worker process yet: arm a pending cancel so register_process kills it on arrival during the+ # claim-to-register window.if proc is None:if registry.mark_pending_cancel(key, generation):return "cancelling"@@ -1337,8 +1291,8 @@ def cancel_worker(if not registry.request_cancel(key, proc, generation):return registry.get_job(key).state- # No eager marker: finalize_worker_exit writes it on a "cancelled" exit.- # Persisting before the kill races a clean completion and strands a stale marker.+ # No eager marker: finalize_worker_exit writes it on a "cancelled" exit, and persisting before the+ # kill races a clean completion and strands a stale marker.try:proc.kill()except ProcessLookupError:@@ -1388,8 +1342,8 @@ def active_download_refs(else:ref_repo_id = metadata.repo_id if metadata is not None else ref.keyvariant = None- # Scoped jobs share one slot per repo, so publish the file list an adopting client needs to recognise its own transfer.- # Absent metadata (a job hydrated before the registry knew it) reports null, which the client reads as unprovable.+ # Scoped jobs share one slot per repo, so publish the file list an adopting client needs; absent+ # metadata reports null, which the client reads as unprovable.scoped_files = list(metadata.scoped_files) if metadata is not None else []downloads.append(ActiveDownload(diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.pyindex b1633572d63..67607cbe618 100644--- a/studio/backend/hub/services/models/cache_inventory.py+++ b/studio/backend/hub/services/models/cache_inventory.py@@ -41,11 +41,9 @@ from hub.services.models.common import (_runtime_for_format,)-# Imported at module scope (not inside the per-repo scan loop) so a broken-# import surfaces at startup instead of silently emptying the inventory: the-# scan loop swallows per-repo exceptions and would drop every repo. Lives under-# ``utils`` (not ``utils.models``) to avoid the eager model-config/checkpoint-# imports in ``utils/models/__init__.py``.+# Imported at module scope so a broken import surfaces at startup instead of silently emptying the+# inventory: the scan loop swallows per-repo exceptions. Lives under utils, not utils.models, to+# avoid the eager model-config imports in utils/models/__init__.py.from utils.paths.path_utils import is_appledouble_metadatafrom utils.audio_tokens import detect_local_tts_audio_typefrom utils.hidden_models import (@@ -76,14 +74,14 @@ _cached_inventory_flights: dict[tuple[asyncio.AbstractEventLoop, tuple[str, int]], asyncio.Task[list[dict]]] = {}-# Retrying a superseded scan is only worth it while invalidations are occasional;-# past this the endpoint has to answer instead of restarting the walk forever.+# Retrying a superseded scan is only worth it while invalidations are occasional; past this the+# endpoint has to answer instead of restarting the walk forever._INVENTORY_SCAN_MAX_ATTEMPTS = 8# Last scan per inventory name that confirmed its epoch, served when the cap is hit._last_confirmed_inventory: dict[str, list[dict]] = {}-# Identity for a cached file with no HF blob (Windows without Developer Mode: hf-# moves the blob into snapshots/ and leaves blobs/ empty).+# Identity for a cached file with no HF blob: on Windows without Developer Mode hf moves the blob+# into snapshots/ and leaves blobs/ empty._LOCAL_SIZE_IDENTITY_PREFIX = "size:"@@ -187,9 +185,8 @@ def _repo_gguf_last_modified(repo_info) -> float:def _repo_has_mmproj(repo_info) -> bool:- # An mmproj file only makes a repo vision-capable when it is an actual GGUF- # projector; a non-GGUF sidecar (e.g. mmproj_config.json) does not, and the- # runtime's projector detection is GGUF-only.+ # Only an actual GGUF projector makes a repo vision-capable; a non-GGUF sidecar+ # (e.g. mmproj_config.json) does not, and the runtime's projector detection is GGUF-only.return any(_is_gguf_filename(f.file_name) and _is_mmproj_filename(f.file_name)for revision in repo_info.revisions@@ -217,8 +214,8 @@ def cached_repo_files(revision) -> list:Every classification below reads these by name, and a "._" companion carries the describedfile's own name, so it answers each one the way the real file does."""- # The "._" name is on the snapshot entry, the bytes are in the content-addressed blob whose- # own name carries no prefix -- but the entry is a symlink to it, so one open reads both.+ # The "._" name is on the snapshot entry while the bytes are in the content-addressed blob, but the+ # entry is a symlink to it, so one open reads both.return [ffor f in getattr(revision, "files", ()) or ()@@ -409,8 +406,8 @@ def _cache_inventory_fields(active_hub_cache = active_hub_cache,payload_snapshots = payload_snapshots,)- # The directory this row loads from: the non-GGUF scan passes only- # *identity*, so snapshot_path alone classified nothing.+ # The directory this row loads from: the non-GGUF scan passes only identity, so snapshot_path alone+ # classified nothing.classify_snapshot = identity.load_snapshot or snapshot_pathcan_chat_override = Noneif classify_snapshot is not None:@@ -433,27 +430,19 @@ def _cache_inventory_fields(else repo_info is not None and _repo_has_mmproj(repo_info)):capabilities["supports_vision"] = True- # Qwen3-ASR's required mmproj is an audio projector, not a vision- # projector. Curated STT rows are visible only to task-scoped consumers and- # must never become eligible for chat auto-load.- # ``stt_only`` covers any repo whose config sniffs as Whisper, curated or not: a- # third-party checkpoint or a user's own fine-tune is just as unchattable, and can_chat- # is what auto-load and the chat picker filter on, neither of which looks at the task.+ # Qwen3-ASR's required mmproj is an audio projector, not a vision one, and stt_only covers any+ # repo whose config sniffs as Whisper: can_chat is what auto-load and the chat picker filter on.if stt_only or is_curated_stt_repo_id(repo_id):capabilities["supports_vision"] = Falsecapabilities["can_chat"] = False- # Same rule for the speech-emitting half of the Audio page. The codec probe in- # _local_transformers_can_chat covers uncurated safetensors copies; native audio- # architectures are also passed explicitly. A GGUF repo ships no tokenizer_config- # to probe, so the curated ids answer for those.+ # The codec probe covers uncurated safetensors copies and native audio architectures are passed+ # explicitly; a GGUF repo ships no tokenizer_config to probe, so the curated ids answer for those.if tts_only or is_curated_tts_repo_id(repo_id):capabilities["can_chat"] = Falseif hidden_infra:capabilities["can_chat"] = False- # A VAE / text-encoder mirror holds no language model, so it cannot chat whatever its weight- # format says. Set HERE rather than left to the row's companion flag alone: startup auto-load- # filters on capabilities.can_chat (isChattableCachedRepo), not on that flag, so a row that- # only carried the flag was still auto-loadable as a chat model.+ # A VAE / text-encoder mirror holds no language model. Set HERE rather than left to the row's+ # companion flag alone: startup auto-load filters on capabilities.can_chat, not on that flag.if companion:capabilities["can_chat"] = Falsereturn {@@ -502,9 +491,8 @@ def _cached_row_companion(repo_id: str, snapshot: Optional[Path] = None) -> boolif normalized in native_companions:return True- # MOSS Local may name a different compatible tokenizer in its processor- # config. Classify the codec architecture too, so that dynamically- # resolved companion cannot surface as a chat checkpoint after download.+ # MOSS Local may name a different compatible tokenizer, so classify the codec architecture too or+ # that dynamically resolved companion surfaces as a chat checkpoint.config = _read_json_object(snapshot / "config.json") if snapshot is not None else {}model_type = str(config.get("model_type") or "").strip().lower()if model_type in {@@ -536,8 +524,8 @@ def _cached_row_task(pick by it, so a row that arrives without one is dropped from those lists entirely."""try:- # Module-qualified: rebinding these names here re-points a load that resolved to- # routes.models before the move, which verify_import_hoist.py blocks.+ # Module-qualified: rebinding these names re-points a load that resolved to routes.models before the+ # move, which verify_import_hoist.py blocks.from hub.services.models import catalog_classificationreturn (catalog_classification._repo_gguf_task(repo_info, selected)@@ -581,14 +569,8 @@ def _scan_cached_gguf(active_hub_cache = active_hub_cache,)except Exception as e:- # Symmetric with _scan_cached_models below. The index is built once for the- # whole scan and outside the per-repository try, so one malformed repo- # identity anywhere in the cache took the endpoint with it: a cache- # directory name holding a byte the filesystem encoding cannot decode- # arrives as a lone surrogate, hashing it for the repo key raises, and- # /api/hub/cached-gguf answers 500 with every valid row hidden. Falling- # back to the per-repo reads this replaced costs speed on a broken cache- # and nothing else.+ # The index is built once for the whole scan and outside the per-repository try, so one undecodable+ # cache directory name, hashed for the repo key, answered 500 with every valid row hidden.logger.warning("Could not build shared cached-GGUF state index: %s", e)variant_states = None@@ -622,20 +604,16 @@ def _scan_cached_gguf(str(snapshot_path) if snapshot_path is not None else None,)is_curated_stt = is_curated_stt_repo_id(repo_id)- # Hide infra repos unless the user downloaded a variant via- # the Hub; variant state only exists for user downloads. Exact- # curated STT repos are still emitted as management rows and- # the frontend's task allowlist keeps them out of Chat.+ # Hide infra repos unless the user downloaded a variant: variant state only exists for user+ # downloads, and curated STT repos are still emitted as management rows.if is_hidden_infra and not is_curated_stt and not has_variant_state:continueif total_size == 0 and not has_variant_state:continue# Must run after the skips above and before the partial walk it scopes.gguf_snapshot, gguf_payload_snapshots = _repo_gguf_payload_snapshots(repo_info)- # Resolved before the row is classified, not inside _cache_inventory_fields- # below: a load_id left as the repo id resolves through refs/main, which can- # name an older revision than the newest payload snapshot. Classifying the- # newest then described a revision the load never reads.+ # Resolved before the row is classified: a load_id left as the repo id resolves through refs/main,+ # which can name an older revision than the newest payload snapshot.gguf_identity = _resolve_load_identity(repo_id,repo_path = repo_path,@@ -677,8 +655,8 @@ def _scan_cached_gguf("partial": partial,# A marker-only sibling moves neither size nor mtime."has_variant_state": has_variant_state,- # GGUF row-level transport is ambiguous (variants may differ);- # per-variant detail lives on GgufVariantDetail.+ # GGUF row-level transport is ambiguous, since variants may differ; per-variant detail lives on+ # GgufVariantDetail."partial_transport": None,"partial_resumable": False,}@@ -729,9 +707,8 @@ def _scan_cached_inventory_snapshot(scanner, expected_epoch: int) -> list[dict]:if hf_cache_scan.hf_cache_scans_epoch() != expected_epoch:raise _CacheSourceChangedrows = scanner(cache_scans = cache_scans, active_hub_cache = active_hub_cache)- # The walk itself takes seconds, so a delete or a finished download landing- # during it supersedes these rows just as surely as one landing before it:- # without this a repo deleted mid-scan is still listed in the response.+ # The walk itself takes seconds, so a delete or finished download landing during it supersedes these+ # rows as surely as one landing before it.if hf_cache_scan.hf_cache_scans_epoch() != expected_epoch:raise _CacheSourceChangedreturn rows@@ -752,9 +729,8 @@ async def _shared_cached_inventory_scan(name: str, scanner) -> _CachedInventoryScontinue_last_confirmed_inventory[name] = rowsreturn _CachedInventoryScan(rows, True)- # Invalidations are arriving faster than the walk completes, so no scan will- # confirm as current. Answer with the last one that did rather than spin a- # full cache walk per epoch forever.+ # Invalidations are arriving faster than the walk completes, so answer with the last scan that+ # confirmed rather than spin a full cache walk per epoch forever.logger.warning("Cached %s inventory kept racing cache invalidations; serving the last confirmed scan",name,@@ -907,15 +883,12 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload:if snapshot is not None:for config_name, key in (("config.json", "has_config"),- # A diffusers pipeline's root manifest, which plays exactly the role config.json- # plays for transformers: from_pretrained reads it at the snapshot root to learn- # the component layout. Without it here no pure pipeline snapshot could classify- # (real repos ship model_index.json and per-component configs, never a root- # config.json), payload_snapshots came back empty, and every cached diffusion base- # pipeline was force-flagged partial and dropped from the On Device lists.+ # model_index.json plays for a diffusers pipeline the role config.json plays for transformers:+ # without it no pure pipeline snapshot could classify, and every cached diffusion base was+ # force-flagged partial and dropped from On Device.("model_index.json", "has_config"),- # Modular Diffusers pipelines use this root manifest instead. Saved or custom- # modular snapshots may omit both conventional root manifests.+ # Modular Diffusers pipelines use this root manifest instead, and saved or custom modular snapshots+ # may omit both.("modular_model_index.json", "has_config"),("adapter_config.json", "has_adapter_config"),):@@ -934,9 +907,8 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload:)def _revisions_of(fmt: str) -> tuple[list, list]:- # Weights pool across revisions, so the pinned snapshot must classify alone; trusted=False- # because from_pretrained needs config.json in that one directory. Filename classification- # also matches torn revisions, so the whole ones are tracked separately.+ # Weights pool across revisions, so the pinned snapshot must classify alone; trusted=False because+ # from_pretrained needs config.json in that one directory.snapshots = [snapshotfor snapshot, flags in revision_flags@@ -948,9 +920,9 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload:payload_snapshots, complete = _revisions_of(model_format)if not complete:- # The repo-wide flags are OR-ed across revisions, so they can name a format whose every- # revision is torn while another format has a whole one sitting right there. Load what- # loads: an interrupted safetensors attempt must not hide a complete checkpoint.+ # The repo-wide flags are OR-ed across revisions, so they can name a format whose every revision+ # is torn while another format has a whole one: an interrupted safetensors attempt must not hide+ # a complete checkpoint.for candidate in ("safetensors", "checkpoint", "adapter"):if candidate == model_format:continue@@ -1154,14 +1126,12 @@ def _scan_cached_models(else None),)- # A companion-only prefetch (pipeline manifest + VAE / text-encoder but no transformer shards, left by a GGUF load) passes- # the download check yet cannot from_pretrained, so mark it partial and the pickers stop advertising it as on-device.- # Scoped to the row's own snapshot, like the download check above it: the repo-wide twin picks the newest revision for- # itself, so a newer companion-only snapshot beside the complete one refs/main resolves to marked this row partial.+ # A companion-only prefetch passes the download check yet cannot from_pretrained, so mark it+ # partial.companion_only = hf_cache_scan.snapshot_pipeline_missing_denoiser(load_snapshot)snapshot_partial = download_partial or companion_only- # Flags are OR-ed over revisions, so no payload snapshot means no directory- # serves the row and it would reach for the Hub.+ # Flags are OR-ed over revisions, so no payload snapshot means no directory serves the row and it+ # would reach for the Hub.if not payload.payload_snapshots:snapshot_partial = Truetry:@@ -1176,8 +1146,9 @@ def _scan_cached_models(if is_whisper_sttelse ("text-to-speech"- # The probe answers for a repo whose card says nothing, so the- # Audio page, which selects by task, still lists it.+ # The probe answers for a repo whose card says nothing, so the Audio page, which+ # selects by task,+ # still lists it.if is_output_audio or local_metadata.get("pipeline_tag") == "text-to-speech"else _cached_row_task(repo_info, gguf = False, selected = load_snapshot))@@ -1202,7 +1173,9 @@ def _scan_cached_models(repo_id,repo_cache_dir = repo_path,)- # Only a genuine download partial has a transport; a companion-only snapshot arrived intact and has no Resume story.+ # Only a genuine download partial has a transport; a companion-only snapshot arrived+ # intact and has+ # no Resume story.if download_partialelse None),@@ -1215,15 +1188,14 @@ def _scan_cached_models(if download_partialelse False),- # Diffusion repos with no pipeline index load only via from_single_file, so the task pickers must not offer them as- # pipeline loads. Without this the picker's single_file gate sees undefined on every hub-sourced row. Read from the- # row's snapshot: the manifest a sibling revision carries is not the one this row's load_id opens.+ # Diffusion repos with no pipeline index load only via from_single_file, so the task pickers must+ # not offer them as pipeline loads."single_file": bool(row_task is not Noneand not hf_cache_scan.snapshot_has_pipeline_index(load_snapshot)),- # Listed so tens of GB of companion weights stay visible and deletable, but- # flagged so no picker offers a denoiser-less repo as a load.+ # Listed so tens of GB of companion weights stay visible and deletable, but flagged so no picker+ # offers a denoiser-less repo as a load."companion": _cached_row_companion(repo_id, load_snapshot),"diffusers": _cached_row_is_diffusers(repo_info, load_snapshot),**local_metadata,@@ -1246,9 +1218,8 @@ def _scan_cached_models(tts_only = is_output_audio,))- # Native backend selection reads the load identity itself. A custom native- # fork addressed only by repo id is indistinguishable from an ordinary LLM,- # even though this inventory row detected its architecture from the snapshot.+ # Native backend selection reads the load identity itself, so a custom native fork addressed only by+ # repo id is indistinguishable from an ordinary LLM.if native_audio_type and load_snapshot is not None:row["load_id"] = str(load_snapshot)if _prefer_cache_row(row, existing):diff --git a/studio/backend/hub/services/models/catalog_classification.py b/studio/backend/hub/services/models/catalog_classification.pyindex deef60252bd..22c6c5a3445 100644--- a/studio/backend/hub/services/models/catalog_classification.py+++ b/studio/backend/hub/services/models/catalog_classification.py@@ -84,11 +84,9 @@ def _is_h3_bundle_gguf_hint(hint: Optional[str]) -> bool:def _gguf_architecture(path: str) -> Optional[str]:- # Every read inventory classification makes goes through here, so this is where "never open- # a cloud placeholder" holds: opening one recalls its whole payload. The task probes below- # ask the same question again because an unhydrated file needs a different ROUTE, not just a- # skipped read; the audio-type probe has no route to pick and relies on this alone. Load-time- # inspection calls read_gguf_architecture directly and still hydrates, on purpose.+ # Every read inventory classification makes goes through here, so this is where "never open a+ # cloud placeholder" holds: opening one recalls its whole payload. Load-time inspection calls+ # read_gguf_architecture directly and still hydrates, on purpose.if not file_contents_available_locally(path):return Nonefrom utils.models.gguf_metadata import read_gguf_architecture@@ -165,11 +163,8 @@ def _unhydrated_gguf_task(name_hints: tuple[Optional[str], ...]) -> Optional[str"""if any(_is_h3_bundle_gguf_hint(hint) for hint in name_hints):return _VIDEO_GEN_TASK- # Leaves only. A filesystem row hands over its whole path, and family detection matches a- # keyword in ANY segment of it, so a chat GGUF under .../FLUX.1-dev-GGUF/extra/ read as- # text-to-image. With an architecture that mismatch only picks the wrong family; here the- # name is the entire case, so an ancestor directory -- the user's shelf, not this model --- # must not decide it.+ # Leaves only: family detection matches a keyword in ANY path segment, so a chat GGUF under+ # .../FLUX.1-dev-GGUF/extra/ read as text-to-image.return _name_hint_media_task(tuple(_hint_leaf(hint) for hint in name_hints if hint), None)@@ -274,8 +269,8 @@ def _gguf_folder_task(fallback: Optional[str] = Nonetry:scored: list[tuple[tuple[str, str], Path]] = []- # Recorded as the tail is dropped, never inferred from `scored` afterwards: trimming cuts- # back to the cap, so an overflowing folder is indistinguishable from one that fit.+ # Recorded as the tail is dropped, never inferred from scored afterwards: trimming cuts back to the+ # cap, so an overflowing folder would be indistinguishable from one that fit.overflowed = Falsefor path in _iter_gguf_paths(root, deadline):name = path.name@@ -288,8 +283,7 @@ def _gguf_folder_task(overflowed = Truescored.sort(key = lambda item: item[0])paths = [path for _, path in scored[:_MAX_TASK_CLASSIFY_GGUFS]]- # Whether every candidate made it into `paths`: the walk gives up at its own deadline and- # the cap drops the tail, so either can leave a sibling unseen.+ # The walk gives up at its own deadline and the cap drops the tail, so either can leave a sibling unseen.complete = (not overflowedand len(scored) <= _MAX_TASK_CLASSIFY_GGUFS@@ -309,8 +303,8 @@ def _gguf_folder_task(if file_contents_available_locally(path):task = _arch_to_task(_gguf_architecture(str(path)), name_hints = hints)else:- # Its header stays unread, so a name that says nothing leaves the candidate- # unclassified rather than voting text-generation for the whole folder.+ # Its header stays unread, so a name that says nothing leaves the candidate unclassified rather than+ # voting text-generation for the whole folder.task = _unhydrated_gguf_task(hints)except Exception:# Unread, so unranked: this file might have been the runnable sibling.@@ -322,8 +316,8 @@ def _gguf_folder_task(continueif task in _LOADABLE_MEDIA_GGUF_TASKS:return task- # Speech is last resort: nothing here runs a llama-csm GGUF, so answering speech while a- # sibling is loadable hides that sibling. A speech-only folder still tags speech.+ # Speech is last resort: nothing here runs a llama-csm GGUF, so answering speech while a sibling is+ # loadable hides that sibling.if task == _SPEECH_TASK:if speech is None:speech = task@@ -541,10 +535,9 @@ def _repo_is_diffusers(repo_info, selected: Optional[Path] = None) -> bool:return Trueexcept Exception:pass- # Video too, as _local_is_diffusers already asks. _cached_repo_task returns None for an- # unbuildable video repo, so a single-file video checkpoint with no pipeline index carried- # no task and no diffusers flag, and an inconclusive config leaves can_chat set: every gate- # the chat picker has passes and the video weights reach the text loader.+ # _cached_repo_task returns None for an unbuildable video repo, so a single-file video checkpoint+ # with no pipeline index carried no task and no diffusers flag, and an inconclusive config leaves+ # can_chat set: the video weights reach the text loader.try:from core.inference.video_families import detect_video_familyreturn detect_video_family(repo_id) is not Nonediff --git a/studio/backend/hub/services/models/common.py b/studio/backend/hub/services/models/common.pyindex d50a5644515..3b7f9bc9d87 100644--- a/studio/backend/hub/services/models/common.py+++ b/studio/backend/hub/services/models/common.py@@ -34,8 +34,8 @@ LocalModelSource = Literal["models_dir", "hf_cache", "lmstudio", "ollama", "custdef _safe_is_dir(path) -> bool:- # Py >= 3.12 propagates PermissionError (EACCES) from is_dir(); folder scans- # probe root-owned system dirs, so treat un-stat-able paths as not-a-dir.+ # Py >= 3.12 propagates PermissionError (EACCES) from is_dir(), and folder scans probe root-owned+ # system dirs, so treat un-stat-able paths as not-a-dir.try:return Path(path).is_dir()except OSError:@@ -165,8 +165,7 @@ _NON_GENERATIVE_ARCHITECTURE_SUFFIXES = ("ForVideoClassification","ForZeroShotImageClassification",)-# Generative, but not of a chat reply: they match a generative suffix below yet-# cannot answer a text turn, and they are small enough to be tried first.+# Generative, but not of a chat reply: they match a generative suffix below yet cannot answer a text turn._NON_CHAT_GENERATIVE_MODEL_TYPES = frozenset({"blip",@@ -250,8 +249,8 @@ _ENCODER_ONLY_MODEL_TYPES = frozenset("squeezebert","vision-text-dual-encoder","xlm-roberta",- # Vision and audio backbones: their bare ``*Model`` names carry no task- # suffix, so only the model type identifies them.+ # Vision and audio backbones: their bare *Model names carry no task suffix, so only the model type+ # identifies them."beit","convnext","convnextv2",@@ -292,8 +291,8 @@ def _read_local_json_object(path: Path) -> dict:return {}data = json.loads(path.read_text(encoding = "utf-8"))return data if isinstance(data, dict) else {}- # ValueError covers JSONDecodeError and UnicodeDecodeError; deeply nested- # JSON raises RecursionError, which is neither.+ # ValueError covers JSONDecodeError and UnicodeDecodeError; deeply nested JSON raises+ # RecursionError, which is neither.except (ValueError, OSError, RecursionError):return {}@@ -309,14 +308,12 @@ def _local_transformers_can_chat(path: Path) -> Optional[bool]:if not _safe_is_dir(path):return None- # Before every architecture test below: a TTS model is an ordinary causal LM wearing- # a codec vocabulary (Orpheus is LlamaForCausalLM), so the suffix rules answer True- # and auto-load, which prefers the smallest, then picks it.+ # Before every architecture test below: a TTS model is an ordinary causal LM wearing a codec+ # vocabulary (Orpheus is LlamaForCausalLM), so the suffix rules answer True and auto-load picks it.if detect_local_tts_audio_type(path) is not None:return False- # SentenceTransformers exports carry this even when the config names a- # broadly reusable encoder class.+ # SentenceTransformers exports carry this even when the config names a broadly reusable encoder class.try:if (path / "modules.json").is_file():return False@@ -341,8 +338,7 @@ def _local_transformers_can_chat(path: Path) -> Optional[bool]:)model_type_raw = config.get("model_type")normalized_type = model_type_raw.strip().lower() if isinstance(model_type_raw, str) else ""- # Before the generative suffix: Whisper and friends end in- # ForConditionalGeneration but cannot answer a text turn.+ # Before the generative suffix: Whisper and friends end in ForConditionalGeneration but cannot answer a text turn.if normalized_type in _NON_CHAT_GENERATIVE_MODEL_TYPES or any(name in _NON_CHAT_GENERATIVE_ARCHITECTURES for name in names):@@ -351,16 +347,13 @@ def _local_transformers_can_chat(path: Path) -> Optional[bool]:return Trueif names and all(name.endswith(_NON_GENERATIVE_ARCHITECTURE_SUFFIXES) for name in names):return False- # AutoModel.save_pretrained on a chat family writes the backbone name, and a- # backbone has no LM head. Listed explicitly, not shape-matched, so an- # unfamiliar FooModel still fails open.+ # AutoModel.save_pretrained on a chat family writes the backbone name, which has no LM head. Listed+ # explicitly, not shape-matched, so an unfamiliar FooModel still fails open.if names and all(name in _BARE_TEXT_BACKBONE_ARCHITECTURES for name in names):return False- # The type alone decides: requiring the name shape too kept rows chat-capable- # when it did not fit, e.g. google/siglip2-* omits architectures entirely and- # CLIPTextModelWithProjection ends in neither Model nor a known suffix.- # Anything generative returned True above, so no chat row reaches here.+ # The type alone decides: requiring the name shape too kept rows chat-capable when it did not fit,+ # e.g.if normalized_type in _ENCODER_ONLY_MODEL_TYPES:return Falsereturn None@@ -393,18 +386,16 @@ def _base_transformers_can_chat(except (OSError, RuntimeError, ValueError):return None- # The adapter's own root first, then the active root, then the configured ones. The scan- # covers legacy and previously configured roots, so an adapter can be listed from an- # inactive root with its base cached beside it; the active root alone answered None there,- # and None is inconclusive, which left a Whisper or encoder LoRA in the chat picker.+ # The scan covers legacy and previously configured roots, so an adapter can be listed from an+ # inactive root with its base cached beside it; the active root alone answered None, which is+ # inconclusive and left encoder LoRAs in the chat picker.try:from huggingface_hub import try_to_load_from_cacheexcept Exception:return None- # Each source collected independently: under one try, a failure enumerating the OPTIONAL- # extra roots discarded the adapter's own root too and answered None, the same fail-open- # this function exists to close.+ # Each source collected independently: under one try, a failure enumerating the OPTIONAL extra roots+ # discarded the adapter's own root too and answered None.roots: list[Path] = []def _add(root: Optional[Path]) -> None:@@ -438,7 +429,6 @@ def _base_transformers_can_chat(except Exception:continue# A non-str is _CACHED_NO_EXIST ("we know it is absent here") or None ("unknown"),- # and neither rules the base out of a different root.if isinstance(found, str):config_path = foundbreak@@ -553,8 +543,8 @@ def _apply_format_aware_partial(if not target:rewritten.append(row)continue- # GGUF row-level transport is ambiguous (variants may differ); per-variant- # detail lives on GgufVariantDetail.partial_transport via the variants endpoint.+ # GGUF row-level transport is ambiguous, since variants may differ; per-variant detail lives on+ # GgufVariantDetail.partial_transport.partial_transport = None if row.model_format == "gguf" else snapshot_partial_transportrewritten.append(row.model_copy(diff --git a/studio/backend/hub/services/models/companion_cleanup.py b/studio/backend/hub/services/models/companion_cleanup.pyindex 3c251c236e6..856d5bac61f 100644--- a/studio/backend/hub/services/models/companion_cleanup.py+++ b/studio/backend/hub/services/models/companion_cleanup.py@@ -115,8 +115,8 @@ def _remaining_main_gguf_variants(repo_info, *, excluding: Optional[str] = None)passif not _is_main_gguf_filename(name):continue- # The delete this previews ignores proven metadata, so counting it here reports a- # checkpoint as surviving that the deletion itself does not see.+ # The delete this previews ignores proven metadata, so counting it here would report a checkpoint as+ # surviving that the deletion itself does not see.if path and is_appledouble_metadata(Path(path)):continuekey = gguf_variant_key(name).lower()@@ -157,7 +157,6 @@ def _delete_impact_blocking(repo_id: str, variant: Optional[str]) -> dict:reclaimed += _variant_bytes(repo_info, variant) if variant else _repo_blob_bytes(repo_info)# Would this delete leave the repo with no runnable checkpoint? Only then can its companions- # become reclaimable; while a sibling quant survives they stay in use.removes_last_checkpoint = Trueif variant:for repo_info in repos:@@ -185,18 +184,13 @@ def _delete_impact_blocking(repo_id: str, variant: Optional[str]) -> dict:entry = {"repo_id": display, "size_bytes": base_bytes, "needed_by": holders}if holders:retained.append(entry)- # The SAME offerability test orphan_companions_response applies, because this row is a- # pointer at that list. A borrowed chat GGUF repo (the native Qwen-Image text encoder) is a- # curated companion id but holds a denoiser, so the orphan endpoint skips it; advertising- # it here sent the user to Free up space to remove a row that is never there. Per copy,- # like the listing: one pipeline copy in another cache root must not hide a companion-only- # copy that Free up space really will offer.+ # The SAME offerability test orphan_companions_response applies, since this row points at that+ # list: a borrowed chat GGUF repo is a curated companion id but holds a denoiser, so advertising+ # it sent the user to Free up space to remove a row that is never there.elif base_key in offerable and any(not _repo_holds_denoiser(r) for r in base_repos):freeable.append(entry)- # Else: a base only a recorded link names. It is unheld, but the orphan endpoint is- # table-only by design (a mis-recorded link must never turn an unrelated repo into a- # delete candidate), so advertising it here pointed the user at a Free up space list it- # will never appear in.+ # A base only a recorded link names: the orphan endpoint is table-only by design, so advertising it+ # here pointed the user at a Free up space list it will never appear in.return {"repo_id": repo_id,@@ -204,10 +198,9 @@ def _delete_impact_blocking(repo_id: str, variant: Optional[str]) -> dict:"reclaimed_bytes": reclaimed,"retained_companions": retained,"freeable_companions": freeable,- # Same predicate the destructive path uses, so the dialog says what the delete will do.- # A variant delete usually cannot strand anything, but the native Qwen-Image encoder is a- # named quant inside a chat GGUF repo, and previewing only whole-repo deletes left Delete- # enabled on it and the refusal to arrive after the user confirmed.+ # Same predicate the destructive path uses: the native Qwen-Image encoder is a named quant inside+ # a chat GGUF repo, so previewing only whole-repo deletes left Delete enabled and the refusal+ # arriving after the user confirmed."blocked_by": (companion_dependents(repo_id, scans, ignore_repo_ids = [repo_id])if companion_assets.is_companion_base(repo_id)@@ -245,27 +238,20 @@ def _orphan_companions_blocking() -> dict:if required.get(base_key):continuerepos = by_id[base_key]- # A repo that holds a runnable denoiser is a model the user installed, not a leftover.- # Several curated bases are perfectly good pipelines in their own right, so this is the- # difference between the two ways the same repo id reaches the cache: a companion fetch- # takes everything BUT the denoiser folder (``_base_file_downloaded`` skips- # ``transformer/``), while a pipeline pick takes it. Its presence is therefore the- # derived answer to "did the user ask for this repo, or did a GGUF drag it in".- # Per COPY, not pooled: the loop below emits one row per cache root because a delete is- # scoped to one, so a full pipeline copy in one root must not suppress an orphaned- # companion-only copy in another that nothing can otherwise reclaim.+ # A repo holding a runnable denoiser is a model the user installed: a companion fetch takes+ # everything BUT transformer/, while a pipeline pick takes it, so its presence answers whether the+ # user asked for this repo. Per COPY, since a delete is scoped to one cache root.repos = [r for r in repos if not _repo_holds_denoiser(r)]if not repos:continue- # One row per cache root. A delete is scoped to a single cache, so pooling copies from- # several would promise bytes one removal cannot deliver.+ # One row per cache root: a delete is scoped to a single cache, so pooling copies from several would+ # promise bytes one removal cannot deliver.for repo in repos:size = _repo_blob_bytes(repo)if size <= 0:continue- # The repo dir itself, not its parent: ``scoped_delete_root`` resolves the owning- # cache by walking up to the ``models--`` component, so a bare root resolves to- # nothing and the delete comes back "Invalid cache_path".+ # The repo dir itself, not its parent: scoped_delete_root walks up to the models-- component, so a+ # bare root resolves to nothing and the delete comes back "Invalid cache_path".try:cache_path = str(Path(getattr(repo, "repo_path")))except (TypeError, OSError):diff --git a/studio/backend/hub/services/models/deletion.py b/studio/backend/hub/services/models/deletion.pyindex 726c728cd30..eead89e4d5c 100644--- a/studio/backend/hub/services/models/deletion.py+++ b/studio/backend/hub/services/models/deletion.py@@ -126,8 +126,7 @@ def _repo_file_matches(target_repo, predicate) -> list[tuple[Path, Optional[Pathcontinueif not file_path:continue- # Every predicate here keys on the name, which a sidecar answers exactly as its- # neighbour does, so it would be counted as a deleted model in its own right.+ # Every predicate here keys on the name, which a sidecar answers exactly as its neighbour does.# Proven metadata only: anything else carrying this key is a file to delete.if is_appledouble_metadata(Path(file_path)):continue@@ -156,10 +155,9 @@ def _remove_empty_variant_dirs(target_repos: list, variant: str) -> tuple[int, l"""Remove now-empty ``snapshots/<rev>/<quant>/`` folders for *variant* (thequant label names the folder); only empty dirs go, so siblings are safe.Returns (count removed, removal failures other than a concurrent refill)."""- # A qualified key names its own folder; its quant token belongs to sibling checkpoints too,- # so it must not reach for a <quant>/ dir it does not own. Qualified means a path- # (``distilled/...-Q6_K``), an H3 root stem, or a bpw modifier (``IQ4_XS-3.53bpw``, whose- # token-only ``IQ4_XS/`` folder is a different build's).+ # A qualified key names its own folder and must not reach for a <quant>/ dir it does not own;+ # qualified means a path (distilled/...-Q6_K), an H3 root stem, or a bpw modifier+ # (IQ4_XS-3.53bpw, whose token-only IQ4_XS/ folder is a different build's).qualified = (is_qualified_gguf_variant_key(variant)or (quant_token_with_bpw(variant) or "").lower() == variant.lower()@@ -201,8 +199,8 @@ def _remove_empty_variant_dirs(target_repos: list, variant: str) -> tuple[int, lsub.rmdir()removed += 1except OSError as e:- # A concurrent download refilling the dir (ENOTEMPTY) is not a- # failure; a read-only cache or locked dir is, so surface it.+ # A concurrent download refilling the dir (ENOTEMPTY) is not a failure; a read-only cache or locked+ # dir is, so surface it.if e.errno != errno.ENOTEMPTY:failures.append(f"{sub.name}: {e}")return removed, failures@@ -255,8 +253,8 @@ def _variant_keys_to_delete(target_repo, variant: str) -> set[str]:}if wanted in keys:return {wanted}- # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant- # names both partitions, so it must not delete either.+ # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant names+ # both partitions, so it must not delete either.aliased = {key for key in keys if "/" in key and bare_quant_alias(key).lower() == wanted}return aliased if len(aliased) == 1 else {wanted}@@ -295,11 +293,7 @@ def _delete_gguf_variant_from_repos(if matched and not sibling_active and not _has_remaining_main_gguf(target_repo):companion_matches = _repo_file_matches(target_repo,- # Companions: mmproj and the drafters Unsloth downloads (MTP with- # every variant, DSpark on opt-in). No main GGUF is left, so they- # cannot be launched; reclaim them with the last variant. An imatrix- # joins them: no longer offered as a variant, so a copy an older build- # fetched as one would be unreachable from the UI.+ # No main GGUF is left, so the companions cannot be launched; reclaim them with the last variant.lambda name: _is_gguf_filename(name)and (_is_mmproj_filename(name)@@ -510,8 +504,8 @@ def reclaim_replaced_gguf_variant(and gguf_variant_key(name).lower() == variant_key,)for snap, blob, name in matches:- # Prune only a file we can identify as a real, stale cache blob. A- # no-symlink snapshot file has no identifiable blob hash, so keep it.+ # Prune only a file we can identify as a real, stale cache blob: a no-symlink snapshot file has no+ # identifiable blob hash, so keep it.blob_hash = (_blob_hash_from_path(blob)if cache_inventory._is_real_cache_blob(blob, repo_dir)@@ -691,7 +685,8 @@ def _diffusion_blocks_delete(repo_id: str) -> Optional[str]:if status.get("loaded") and status.get("repo_id"):if _loaded_id_matches_repo(str(status["repo_id"]), repo_id):return "Unload the model before deleting"- # sd.cpp re-reads companion VAE / text-encoder files every generation and status().repo_id covers only the main GGUF, so refuse the companions too.+ # sd.cpp re-reads companion VAE / text-encoder files every generation and status().repo_id covers+ # only the main GGUF, so refuse the companions too.for lid in getattr(engine, "loaded_repo_ids", tuple)():if _loaded_id_matches_repo(str(lid), repo_id):return "Unload the model before deleting"@@ -716,13 +711,14 @@ def _video_blocks_delete(repo_id: str) -> Optional[str]:return Nonestatus = backend.status()if status.get("loaded"):- # repo_id names the checkpoint; for a GGUF / single-file load the companion base supplies the VAE and text encoders, so refuse it too.+ # repo_id names the checkpoint; for a GGUF / single-file load the companion base supplies the VAE+ # and text encoders, so refuse it too.for key in ("repo_id", "base_repo"):held = status.get(key)if held and _loaded_id_matches_repo(str(held), repo_id):return "Unload the model before deleting"# The native H3 runtime re-reads its Qwen encoder and both VAEs from companion repos that are- # neither of the two ids above, so refuse those as well, exactly as the Images guard does.+ # neither of the two ids above, so refuse those as well.for lid in getattr(backend, "loaded_repo_ids", tuple)():if _loaded_id_matches_repo(str(lid), repo_id):return "Unload the model before deleting"@@ -807,10 +803,7 @@ async def delete_cached_model_response(detail = f"Invalid gguf_variant: {variant!r}",)- # Guard fails closed: if a live backend's load state can't be read, abort- # with 503 rather than risk unlinking weights under a running process.- # Every guard is sync and the chat ones reach get_inference_backend(), whose cold build waits- # on hardware detection. One worker keeps the `or` short-circuit and keeps the event loop free.+ # Fail closed with 503 rather than unlink weights under a running process.def _load_state_blocks_delete() -> Optional[str]:if _llama_cpp_blocks_delete(repo_id, variant) or (_inference_backend_blocks_delete(repo_id)@@ -842,10 +835,9 @@ async def delete_cached_model_response()raise HTTPException(status_code = 400, detail = detail)try:- # Re-derived now the scope is reserved, as only_if_orphan re-derives its own answer- # below. The first read ran before the reservation existed, so a load starting in- # between published its claim too late to be seen, and begin_delete misses it too:- # image and video loads download directly rather than through a registry claim.+ # Re-derived now the scope is reserved, as only_if_orphan re-derives its own answer below: the+ # first read ran before the reservation existed, so a load starting in between published its claim+ # too late, and begin_delete misses it too.try:blocks_detail = await asyncio.to_thread(_load_state_blocks_delete)except Exception as e:@@ -880,12 +872,8 @@ def _delete_cached_model_blocking(*,only_if_orphan: bool = False,) -> dict:- # Free up space asks for this: the row it is removing was an orphan when the list was built,- # and the list can be minutes old. A download of that same repo finishing in the background- # turns it into an installed checkpoint, and neither guard below catches that -- begin_delete- # only refuses a download still in flight, and the companion guard deliberately ignores the- # target as its own dependent. Re-derived here, after begin_delete has closed the repo to new- # downloads, so the answer cannot go stale between the check and the unlink.+ # Free up space's list can be minutes old, and a background download finishing turns that orphan+ # into an installed checkpoint neither guard below catches.if only_if_orphan:from hub.services.models import companion_cleanupfrom hub.utils import companion_assets@@ -894,9 +882,8 @@ def _delete_cached_model_blocking(copies = companion_cleanup._repos_by_id(cache_inventory.all_hf_cache_scans()).get(repo_id.strip().lower(), [])- # Only the copy being removed. The orphan listing emits one row per cache root- # precisely because a delete is scoped to one, so a full-pipeline copy in another- # remembered cache must not veto removing the companion-only copy that was listed.+ # Only the copy being removed: the orphan listing emits one row per cache root, so a full-pipeline+ # copy in another remembered cache must not veto removing the companion-only copy listed.if cache_path:wanted = Path(cache_path)copies = [@@ -905,11 +892,8 @@ def _delete_cached_model_blocking(if getattr(r, "repo_path", None) and Path(getattr(r, "repo_path")) == wanted]if not copies:- # No fallback to the other copies. An empty match means the target root is- # not in this scan (its scan failed, or the copy is gone), and the delete- # below can still purge that directory by path -- so concluding "orphan" from- # copies we did not look at is exactly the fail-open this precondition exists- # to prevent. Raising here lands in the fail-closed 503 below.+ # An empty match means the target root is not in this scan, and concluding "orphan" from copies we+ # did not look at is the fail-open this precondition prevents; raising lands in the 503.raise RuntimeError(f"cache root not present in the scan: {cache_path}")still_orphan = not any(companion_assets.repo_holds_denoiser(repo) for repo in copies)except Exception as e:@@ -928,26 +912,18 @@ def _delete_cached_model_blocking()# A companion base repo carries the text encoders, VAE and tokenizer for every quant of its- # family, so removing it while one is installed leaves that quant unloadable with nothing on- # screen to say why. Derived from what is installed right now, never from a stored count, and- # only for a WHOLE-repo delete: a variant delete cannot touch another repo. Deleting the- # dependants first makes the base an orphan, which Free up space then offers.- #- # Here rather than in the async caller so it shares this function's cache walk and its stubs:- # the check IS part of the destructive stage, and a caller that replaces that stage should not- # end up with half of it still running.- # A variant delete normally cannot touch another repo, so the guard is a whole-repo check.- # The exception is a companion whose asset IS a named GGUF variant: native Qwen-Image opens- # exactly Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf inside a chat GGUF repo, so removing that one- # quant strands the image checkpoint however many siblings remain, and none of them is a- # substitute for a fixed filename. A FLAG, never a rewrite of `variant`: that name is the- # destructive scope, and widening it here would delete every revision and manifest the user- # did not ask for, and purge a sibling quant out from under an in-flight download.+ # family, so removing it while one is installed leaves that quant unloadable. Derived from what is+ # installed right now, and only for a WHOLE-repo delete.+ # Here rather than in the async caller so it shares this function's cache walk and stubs: the check+ # IS part of the destructive stage.+ # The exception is a companion whose asset IS a named GGUF variant: native Qwen-Image opens one+ # fixed filename inside a chat GGUF repo, so removing that quant strands the image checkpoint. A+ # FLAG, never a rewrite of `variant`: widening the scope would delete revisions the user did not+ # ask for.guard_this_delete = variant is None or _variant_is_a_required_companion_asset(repo_id, variant)if guard_this_delete and _is_companion_base_repo(repo_id):- # Fails CLOSED, and only here: the lookup above already established this repo IS a- # companion base, so an unreadable cache means the dependants cannot be enumerated, not- # that there are none. Every other repo skips the check entirely and is unaffected.+ # Fails CLOSED, and only here: the lookup above already established this repo IS a companion base,+ # so an unreadable cache means the dependants cannot be enumerated, not that there are none.try:shared_detail = _companion_share_blocks_delete(repo_id)except Exception as e:@@ -963,17 +939,16 @@ def _delete_cached_model_blocking(raise HTTPException(status_code = 400, detail = shared_detail)try:- # If a sibling quant is downloading concurrently, restrict this delete to- # the variant's own files and leave the shared mmproj companion for it.+ # If a sibling quant is downloading concurrently, restrict this delete to the variant's own files+ # and leave the shared mmproj companion for it.sibling_active = bool(variant and downloads.registry.has_active_peer_variant(repo_id, variant))cache_scans = cache_inventory.all_hf_cache_scans()- # A repo can live in several remembered caches. Group its copies by the- # cache root that owns each, then target exactly one cache so a delete- # never removes copies in other, previously selected caches.+ # A repo can live in several remembered caches, so target exactly one or a delete removes copies in+ # other, previously selected caches.owners: dict = {}for hf_cache in cache_scans:for repo_info in hf_cache.repos:diff --git a/studio/backend/hub/services/models/downloads.py b/studio/backend/hub/services/models/downloads.pyindex 7bca8476f77..4704466408b 100644--- a/studio/backend/hub/services/models/downloads.py+++ b/studio/backend/hub/services/models/downloads.py@@ -47,7 +47,8 @@ def _download_job_key(repo_id: str, variant: Optional[str]) -> str:)-# A scope rides the variant slot as "@name". No GGUF quant label starts with "@", so a scoped job never collides with a real variant or the full snapshot.+# A scope rides the variant slot as "@name". No GGUF quant label starts with "@", so a scoped job+# never collides with a real variant or the full snapshot._SCOPE_PREFIX = "@"@@ -217,7 +218,7 @@ async def download_model_response(variant = scope_variantkey = _download_job_key(repo_id, variant)# Off the event loop: resolving "auto" can run the Xet reachability probe, and a blackholed DNS- # makes that outlast its 3s budget while every other Unsloth request waits behind it.+ # makes that outlast its 3s budget while every other request waits behind it.use_xet, transport_reason = await asyncio.to_thread(download_lifecycle.resolve_requested_use_xet,getattr(body, "transport_mode", None),@@ -303,9 +304,9 @@ async def download_model_response("this one."),)- # claim_state is the blocking job's state. Attaching and accepting are- # one verdict: only this key's own in-flight job can be joined, and a- # cross-variant conflict or in-progress delete joined nothing.+ # claim_state is the blocking job's state. Attaching and accepting are one verdict: only this+ # key's own in-flight job can be joined, and a cross-variant conflict or in-progress delete joined+ # nothing.adoptable = _registry.adoptable(key)return {"job_key": key,@@ -313,11 +314,11 @@ async def download_model_response("accepted": adoptable,"attached": adoptable,"generation": generation,- # An adopted job keeps the transport it started on, so report it- # rather than let the caller assume the one it asked for.+ # An adopted job keeps the transport it started on, so report it rather than let the caller assume+ # the one it asked for."transport": _registry.job_transport(key),- # And its cancel marker: a run that fell back from Xet to HTTP- # still cancels into a restart-only partial.+ # And its cancel marker: a run that fell back from Xet to HTTP still cancels into a restart-only+ # partial."cancel_transport": _registry.job_cancel_transport(key),}download_manifest.clear_cancel_marker(@@ -326,8 +327,8 @@ async def download_model_response(variant,hub_cache = cache_paths.hub_cache,)- # Blobs a concurrent same-repo variant is already writing (e.g. a shared- # mmproj). The worker must not purge these during cache preparation.+ # Blobs a concurrent same-repo variant is already writing, such as a shared mmproj: the worker must+ # not purge these during cache preparation.protected_blob_hashes = _registry.peer_blob_hashes(key) if variant else frozenset()label = f"{repo_id}{f' [{variant}]' if variant else ''}"@@ -361,9 +362,8 @@ async def download_model_response("accepted": True,"attached": False,"generation": generation,- # The transport that was actually resolved: an explicit "xet" is- # downgraded to HTTP where hf_xet is unavailable, and a client that- # assumed its request stood would offer the wrong stop control.+ # The transport that was actually resolved: an explicit "xet" is downgraded to HTTP where hf_xet is+ # unavailable, and a client that assumed its request stood would offer the wrong stop control."transport": transport,}@@ -456,9 +456,8 @@ def _variant_transport_status(repo_id: str, variant: str, hf_token: Optional[strrepo_id,variant,)- # A partial only counts toward "resumable" while a writer that reopens it is installed,- # since the UI turns the flag into "Resume with HTTP to keep the progress you already- # have" and the next download start sweeps whatever cannot be reopened.+ # A partial counts toward "resumable" only while a writer that reopens it is installed, since the+ # next download start sweeps whatever cannot be reopened.resumable_hashes = download_registry.incomplete_blob_hashes("model",repo_id,@@ -540,36 +539,29 @@ def _variant_manifest_decision(caller's catalog-hinted total. Active cache first, so the common case is onelookup; every candidate found has to agree before one is returned."""- # The active cache's manifest is a candidate like any other, NOT an early return. Its repo- # dir can be gone while its scoped state still holds an old manifest, and idle progress goes- # on scanning the remembered caches -- so returning it unexamined applies a stale revision's- # hashes to a remembered cache that has the complete variant, and filters every blob of it- # out. That is the same wrong answer as two remembered caches disagreeing.+ # The active cache's manifest is a candidate like any other, NOT an early return: its repo dir can+ # be gone while its scoped state holds an old manifest, and applying those hashes to a remembered+ # cache that has the complete variant filters out every blob of it.found: list[download_manifest.Manifest] = []- # ``active_root`` is the root the job records, which is the one snapshot_progress scans; it- # is not necessarily the configured default (a cache moved mid-download), and reading the- # default there would be another cache's answer again.+ # active_root is the root the job records, which is the one snapshot_progress scans and not+ # necessarily the configured default.active_manifest = download_manifest.read_manifest("model", repo_id, variant, hub_cache = active_root)if active_manifest is not None:found.append(active_manifest)- # The active cache was just probed by the call above and a state-dir miss is not free, so- # skip the entry that repeats it. In the common case preferred_repo_cache_dirs returns only- # that entry and this loop does no work at all.+ # The active cache was just probed by the call above and a state-dir miss is not free, so skip the+ # entry that repeats it; in the common case preferred_repo_cache_dirs returns only that entry.active = download_manifest._canonical_hub_cache(active_root)- # The SAME cache dirs snapshot_progress will scan. A running or cancelling job writes into- # the active root and is read only from there, so a remembered cache's manifest for the same- # variant is not merely a second opinion -- its hashes would be applied to the active root's- # blobs and filter out every byte the live download has written, leaving the card at 0 B- # until Hub metadata comes back.+ # The SAME cache dirs snapshot_progress will scan: a remembered cache's manifest for the same+ # variant would have its hashes applied to the active root's blobs, leaving the card at 0 B.for entry in preferred_repo_cache_dirs("model", repo_id, force_active = force_active, active_root = active_root):if active is not None and download_manifest._canonical_hub_cache(entry.parent) == active:if active_manifest is None:- # The cache snapshot_progress will scan, with no manifest of its own. Anything- # returned here would be another cache's answer applied to its blobs.+ # Anything returned for a cache with no manifest of its own would be another cache's answer applied+ # to its blobs.return ("refused", None)continuemanifest = download_manifest.read_manifest(@@ -579,23 +571,18 @@ def _variant_manifest_decision(hub_cache = entry.parent,)if manifest is None:- # A scanned cache that contributed NOTHING. Its snapshot may be the complete one --- # a manifest can be deleted, or never written by an older build -- and returning- # some other cache's hashes filters every blob of it out AND disables the per-entry- # name-based fallback that would still have counted them. Refuse instead.+ # A scanned cache that contributed NOTHING may hold the complete snapshot, since a manifest can be+ # deleted or never written, and another cache's hashes would filter out every blob AND disable the+ # name-based fallback.return ("refused", None)found.append(manifest)if not found:return ("absent", None)# One answer, or several that agree: safe to apply to every scanned entry, which is what- # snapshot_progress does with the hash set this produces.- #- # Several that DISAGREE is the case worth refusing. snapshot_progress picks its reading by- # bytes, across all the preferred cache dirs, but the hashes come from this single lookup --- # so handing it the first cache's older revision filters out every blob of a LATER cache that- # holds the complete variant, and reports 0 or partial for a finished download. That is the- # exact failure this fallback exists to prevent, just sourced from the wrong cache. Returning- # None instead degrades to the name-based fallback, which stays attributable per entry.+ # snapshot_progress does with this hash set.+ # Several that DISAGREE must be refused: snapshot_progress picks its reading by bytes across all+ # preferred cache dirs while the hashes come from one lookup, so the first cache's older revision+ # filters out every blob of a later complete one. None degrades to the name-based fallback.first = _manifest_hashes(found[0])if any(_manifest_hashes(m) != first for m in found[1:]):return ("refused", None)@@ -640,8 +627,7 @@ async def get_gguf_download_progress_response(return requirement.download_size_bytes, requirement.required_hashesjob_key = _download_job_key(resolved_repo_id, progress_variant)job = _registry.get_job(job_key)- # getattr, the same way snapshot_progress reads it: a registry without the accessor- # simply has no recorded root, which is the "use the configured one" case.+ # getattr, the same way snapshot_progress reads it: a registry without the accessor simply has no recorded root.get_job_metadata = getattr(_registry, "get_job_metadata", None)job_metadata = get_job_metadata(job_key) if callable(get_job_metadata) else Nonehub_cache = getattr(job_metadata, "hub_cache", None)@@ -658,12 +644,9 @@ async def get_gguf_download_progress_response(frozenset(file.sha256 for file in manifest.expected_files if file.sha256),)if verdict == "refused":- # A refusal is not a miss. The lookup above found manifests and ruled that none of- # them may be applied across the caches snapshot_progress scans; the blob-hash- # helper reads the DEFAULT cache's manifest with none of that scoping, so falling- # through here reinstates the very hashes just rejected and filters out every blob- # of whichever cache actually holds the variant. An empty set degrades to the- # per-entry name-based fallback, which stays attributable to the entry it counted.+ # A refusal is not a miss: the blob-hash helper reads the DEFAULT cache's manifest with none of+ # this scoping, so falling through reinstates the hashes just rejected. An empty set degrades to+ # the per-entry name-based fallback.return (expected_total, frozenset())return (expected_total,@@ -697,15 +680,10 @@ async def get_gguf_download_progress_response(return requirement.expected_files if requirement is not None else ()def _variant_file_matcher(path: str, *, companions: bool = True) -> bool:- # Which snapshot files a quant owns, for the reading snapshot_progress- # falls back to when the blob hashes cannot be resolved. Main shards are- # matched by quant label; mmproj and the MTP drafter are downloaded with- # every variant, so they belong to whichever one is being polled.- #- # ``companions`` False asks the narrower question -- does this path prove the quant- # ITSELF is here -- which the caller uses first: shared companions belong to every- # quant in the repo, so counting them for a variant whose main shard was deleted- # reported bytes for a file that is gone.+ # Main shards are matched by quant label; mmproj and the MTP drafter are downloaded with every+ # variant, so they belong to whichever one is being polled.+ # companions=False asks the narrower question, whether this path proves the quant ITSELF is here:+ # shared companions belong to every quant, so counting them reported bytes for a deleted file.if progress_variant is None:return Falseif gguf_plan.is_main_gguf_variant_path(path, progress_variant):diff --git a/studio/backend/hub/services/models/folder_browser.py b/studio/backend/hub/services/models/folder_browser.pyindex d6d0d50586e..45fe758a73a 100644--- a/studio/backend/hub/services/models/folder_browser.py+++ b/studio/backend/hub/services/models/folder_browser.py@@ -130,13 +130,12 @@ def _is_path_inside_allowlist(target: Path, allowed_roots: list[Path]) -> bool:return Truedrive, tail = os.path.splitdrive(root_real)if os.path.dirname(root_real) == root_real and not drive:- # Bare POSIX filesystem root ("/"): equality above is the only- # match; do not let it authorize arbitrary descendants.+ # Bare POSIX filesystem root: the equality above is the only match, so it must not authorize+ # arbitrary descendants.continueif drive.startswith(("\\\\", "//")) and not tail:- # Bare UNC share root (\\server\share): os.path.commonpath raises- # "can't mix absolute and relative" on it, so authorize its- # descendants with a boundary-safe prefix test (normcase applied).+ # os.path.commonpath raises "can't mix absolute and relative" on a bare UNC share root, so authorize+ # its descendants with a boundary-safe prefix test.if target_real.startswith(root_real.rstrip("\\/") + os.sep):return Truecontinuediff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.pyindex 36905580584..d5731ac38e6 100644--- a/studio/backend/hub/services/models/gguf_variants.py+++ b/studio/backend/hub/services/models/gguf_variants.py@@ -72,14 +72,12 @@ _VARIANT_HASH_CACHE: "OrderedDict[tuple[str, str, str, bool], tuple[frozenset[st_VARIANT_REQUIREMENT_CACHE: "OrderedDict[tuple[str, str, str], tuple[_GgufVariantRequirement, float]]" = OrderedDict()_VARIANT_REQUIREMENT_NEG_CACHE: "OrderedDict[tuple[str, str], float]" = OrderedDict()_VARIANT_HASH_MAX = 512-# Blob hashes are derived from the same mutable remote revision metadata as-# variant requirements, so they must not outlive that freshness window.+# Blob hashes are derived from the same mutable remote revision metadata as variant requirements, so+# they must not outlive that freshness window._VARIANT_HASH_POS_TTL = 60.0# Refresh resolved variant requirements so a moved repo revision is picked up-# within the session instead of being pinned for the backend's lifetime._VARIANT_REQUIREMENT_POS_TTL = 60.0# Suppress retries on a metadata-fetch failure so a slow/flaky link doesn't-# re-hammer the API on every page refresh._VARIANT_REQUIREMENT_NEG_TTL = 60.0# Fail fast on a slow link so the variant render isn't blocked for seconds._GGUF_METADATA_TIMEOUT_SECONDS = 5.0@@ -119,6 +117,7 @@ def _variant_requirement_neg_cache_active(key: tuple[str, str]) -> bool:return Falseif (time.monotonic() - cached_at) < _VARIANT_REQUIREMENT_NEG_TTL:_VARIANT_REQUIREMENT_NEG_CACHE.move_to_end(key)+ # Split-named link on an ordinary target: loads as-is, nothing missing.return True_VARIANT_REQUIREMENT_NEG_CACHE.pop(key, None)return False@@ -381,8 +380,8 @@ def _variant_dependency_key(repo_id: str, filename: str) -> Optional[str]:unknown = hashlib.sha256(filename.lower().encode("utf-8")).hexdigest()[:16]return f"{fam.name}:unknown:{unknown}"encoders = sd_cpp_text_encoders_for(fam, repo_id, filename, inner_dim = inner_dim)- # Hashed, not joined raw: the encoder table is long, and the key is opaque- # to the client, which only ever compares it for equality.+ # Hashed, not joined raw: the encoder table is long and the key is opaque to the client, which only+ # ever compares it for equality.digest = hashlib.sha256("\n".join("/".join(str(part) for part in entry) for entry in encoders).encode("utf-8")).hexdigest()[:16]@@ -519,8 +518,7 @@ def _local_main_gguf_blobs_by_quant(if not hashes:continueif _is_imatrix_filename(normalized):- # Not a companion either: nothing fetches an imatrix, so a stale copy- # on disk must not vouch for a variant's blobs.+ # Nothing fetches an imatrix, so a stale copy on disk must not vouch for a variant's blobs.continueif _is_mmproj_filename(normalized) or _is_mtp_drafter_path(normalized):companion_blobs.setdefault(normalized, set()).update(@@ -528,9 +526,8 @@ def _local_main_gguf_blobs_by_quant()continuequant = gguf_variant_key(normalized).lower()- # The endian predicate reads a quant TOKEN so it can tell a parent-only quant- # from a big-endian build; the qualified key makes it misread the path and drop- # the blob, which leaves update detection with no local main files to compare.+ # The endian predicate reads a quant TOKEN, so a qualified key makes it misread the path and drop+ # the blob, leaving update detection with no local main files to compare.if is_big_endian_gguf_path(normalized, extract_quant_label(normalized)):continuebucket = result.setdefault(quant, {}).setdefault(normalized, set())@@ -595,8 +592,8 @@ def delete_variant_incomplete_blobs_result(companions: bool = True,root: Optional[Path] = None,) -> VariantIncompleteDeleteResult:- # With a sibling still downloading, ``companions=False`` keeps a shared mmproj- # from being unlinked out from under it; the repo's last delete reclaims it.+ # With a sibling still downloading, companions=False keeps a shared mmproj from being unlinked out+ # from under it; the repo's last delete reclaims it.target_hashes = (gguf_variant_blob_hashes(repo_id, variant, hf_token, include_companions = companions)| extra_hashes@@ -614,9 +611,8 @@ def delete_variant_incomplete_blobs_result(unresolved = has_variant_partial_state and has_repo_partials,)deleted = 0- # Destructive iterator: only the exact-case match (or abort if ambiguous),- # so a case-variant sibling repo's partials are never unlinked. ``root`` scopes- # the purge to one cache so a delete never touches another cache's partials.+ # Destructive iterator: only the exact-case match, or abort if ambiguous, so a case-variant sibling+ # repo's partials are never unlinked; root scopes the purge to one cache.for entry in iter_destructive_repo_cache_dirs("model", repo_id, root = root):blobs_dir = entry / "blobs"if not blobs_dir.is_dir():@@ -736,8 +732,8 @@ def _direct_gguf_loads(path: Path) -> bool:)-# llama.cpp's split grammar (model_config._GGUF_SPLIT_FILE_RE): five digits exactly, since a-# shorter name loads as an ordinary file, not the cache scan's looser -\d{3,}- resume form.+# llama.cpp's split grammar (model_config._GGUF_SPLIT_FILE_RE) wants five digits exactly: a+# shorter name loads as an ordinary file, unlike the cache scan's looser resume form._DIRECT_SPLIT_RE = re.compile(r"^(?P<stem>.+)-(?P<index>\d{5})-of-(?P<total>\d{5})$", re.IGNORECASE)@@ -780,7 +776,6 @@ def _direct_gguf_split_is_whole(path: Path) -> bool:"""m = _DIRECT_SPLIT_RE.match(target.name.rsplit(".", 1)[0])if m is None:- # Split-named link on an ordinary target: loads as-is, nothing missing.return Truetarget_total = int(m.group("total"))pattern = re.compile(@@ -798,12 +793,11 @@ def _direct_gguf_split_is_whole(path: Path) -> bool:found = _indexes_beside(path, sibling)if not found >= set(range(1, total + 1)) and path.is_symlink():# _local_gguf_load_path resolves siblings from the TARGET, so a renamed alias- # still loads the target's real set -- and a torn target stays torn.return _target_set_is_whole(path.resolve())except OSError:return True- # Declared indexes, not a count: an over-indexed stray must not stand in for a missing- # shard, and a zero-byte sibling is an interrupted copy.+ # Declared indexes, not a count: an over-indexed stray must not stand in for a missing shard, and a+ # zero-byte sibling is an interrupted copy.return found >= set(range(1, total + 1))@@ -836,10 +830,8 @@ def _will_serve(resolved: Optional[str]) -> bool:return Falsetry:path = Path(resolved)- # The resolver answers for nonexistent paths, so absence is caught here: "no such- # file" is definite, a read error (Windows lock window) stays unknown and serves.- # stat(), not exists(), which swallows every OSError on 3.14 and would read a- # sharing violation as absence.+ # The resolver answers for nonexistent paths, so absence is caught here. stat(), not exists(), which+ # swallows every OSError on 3.14 and would read a sharing violation as absence.try:size = path.stat().st_sizeexcept (FileNotFoundError, NotADirectoryError):@@ -859,10 +851,9 @@ def _loadable_variants(identifier: str, variants):"""from utils.models.model_config import _find_local_gguf_by_variant- # from_identifier only consults the variant for a DIRECTORY; a direct file loads itself- # regardless, so leave it unanswered rather than be stricter than the load. stat(), not- # is_file(): a locked file must stay unanswered (an empty list is authoritative at the- # gate), which is_file() cannot express -- it raises here, and answers False from 3.14.+ # from_identifier only consults the variant for a DIRECTORY, so leave a direct file unanswered+ # rather than be stricter than the load. stat(), not is_file(): a locked file must stay unanswered,+ # which is_file() cannot express, since it raises here and answers False from 3.14.import stat as _stattry:@@ -873,9 +864,9 @@ def _loadable_variants(identifier: str, variants):except OSError:return None- # The resolver walks the tree per call, so spellings are deduped against `seen` first:- # ~2 calls per row (36 for 18 quants, 179ms over 144 files), not one per spelling. Each- # alias is still confirmed, so a token binding a different file is never advertised.+ # The resolver walks the tree per call, so spellings are deduped against `seen` first (~2 calls+ # per row); each alias is still confirmed, so a token binding a different file is never+ # advertised.accepted: list = []seen = set()for variant in variants:@@ -892,10 +883,9 @@ def _loadable_variants(identifier: str, variants):if key not in seen:seen.add(key)accepted.append(quant)- # The resolver also takes the snapshot-relative stem (BF16/model) and the basename's- # own tokens; derive those and let it confirm each. It returns an absolute path, so a- # relative identifier must be resolved the same way or its alias is lost. Unresolved- # first: a symlink out of the tree still answers BF16/model there, but not resolved.+ # The resolver also takes the snapshot-relative stem and the basename's own tokens, and returns an+ # absolute path, so a relative identifier must be resolved the same way or its alias is lost.+ # Unresolved first: a symlink out of the tree still answers there, but not resolved.relative = Path(bound).namefor base_raw, bound_path in ((Path(identifier).expanduser(), Path(bound)),@@ -978,11 +968,8 @@ def _complete_with_servable(snapshot: str, complete, variants):return repaired-# One scan per identical request in flight. Aborting the HTTP request cannot stop the scan-# already running in its thread, so the picker's Retry, and reopening a row, would each start-# another against a filesystem that is not answering: measured 23 retries filling all 20-# default-executor workers, starving unrelated offloaded work and holding up exit. Joining the-# running scan costs at most its own duration in staleness, well inside the client's cache TTL.+# One scan per identical request in flight: aborting the HTTP request cannot stop the scan already+# running, and 23 retries filled all 20 default-executor workers._VARIANTS_INFLIGHT: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary()@@ -1001,7 +988,7 @@ async def _shared_variants_scan(key: tuple, compute):def _release(finished: asyncio.Future) -> None:inflight.pop(key, None)if not finished.cancelled():- finished.exception() # retrieved, so a caller-less failure stays quiet+ finished.exception()inflight[key] = tasktask.add_done_callback(_release)@@ -1046,12 +1033,11 @@ async def get_gguf_variants_answer(with file sizes, whether the model supports vision, and the recommendeddefault variant."""- # Set by whichever branch answers, and returned with the listing: the HF cache answers- # before local_path, so a caller cannot infer the copy from the request alone.+ # Returned with the listing because the HF cache answers before local_path, so a caller cannot infer+ # the copy from the request alone.answered_from: list[Optional[str]] = [None]- # Set by the existence-first local branch below: a repo-shaped id resolving to a directory- # is answered by that directory alone, not the HF cache of the same-named repo -- else a- # GGUF-less directory could evict the resident model via a cleanable row from the cache.+ # A repo-shaped id resolving to a directory is answered by that directory alone, not the HF cache+ # of the same-named repo, else a GGUF-less directory could evict the resident model.answered_locally = [False]def _compute() -> GgufVariantsResponse:@@ -1212,13 +1198,11 @@ async def get_gguf_variants_answer(return responsereturn response.model_copy(update = {"variants": [*response.variants, *extra]})- # Local directory path (e.g. LM Studio models) — scan filesystem. Load-path parity:- # from_identifier resolves existence-first, so a marker-less relative name that exists- # here is a local model, not a Hub id, and a direct .gguf file loads without the- # metadata siblings the directory scan requires. It also normalizes first, so every- # question below must ask about the same path: under WSL "C:\\models\\qwen" maps to- # /mnt/c/models/qwen, and probing the raw spelling would report a working model- # unloadable.+ # Load-path parity: from_identifier resolves existence-first, so a marker-less relative name that+ # exists here is a local model, not a Hub id, and a direct .gguf file loads without the metadata+ # siblings. It normalizes first, so every question below asks about the same path: under WSL+ # C:\models\qwen maps to /mnt/c/models/qwen, and probing the raw spelling would report a working+ # model unloadable.local_id = _loader_normalize_path(repo_id) if is_local_path(repo_id) else repo_idlocal_target = Nonetry:@@ -1229,8 +1213,7 @@ async def get_gguf_variants_answer(local_target = Noneif local_target is not None:variants, has_vision = list_local_gguf_variants(local_id)- # The load id is this path, so a scan-torn quant is still ready when the file the- # resolver binds opens fine.+ # The load id is this path, so a scan-torn quant is still ready when the file the resolver binds opens fine.complete = _complete_with_servable(local_id, _complete_quants_under(local_id), variants)if (not variants@@ -1238,14 +1221,14 @@ async def get_gguf_variants_answer(and local_target.suffix.lower() == ".gguf"and _direct_gguf_loads(local_target)):- # An unmarked-parent .gguf is skipped by the directory scan but detect_gguf_model- # still loads it; falling back only here keeps a marked parent's siblings.+ # An unmarked-parent .gguf is skipped by the directory scan but detect_gguf_model still loads it;+ # falling back only here keeps a marked parent's siblings.try:size = local_target.stat().st_sizeexcept OSError:size = 0- # The load resolver's own extractor over the context it reads, so the quant is- # what the echoed load resolves (the hub one differs on F16-checkpoint-Q4_K_M).+ # The load resolver's own extractor over the context it reads, so the quant is what the echoed load+ # resolves; the hub one differs on F16-checkpoint-Q4_K_M.from utils.models.model_config import _extract_quant_labelfrom utils.models.model_config import colocated_split_shards@@ -1266,15 +1249,13 @@ async def get_gguf_variants_answer(shard_count = len(shards) if split_complete and len(shards) > 1 else 0,)]- # The shard scan resolves a file to its marked parent, so an unmarked one walks- # a bare file and misreports the row. Ask the file itself: whole unless it is a- # split missing a sibling or a zero-byte interrupted copy.+ # The shard scan resolves a file to its marked parent, so an unmarked one walks a bare file and+ # misreports the row; ask the file itself.complete = None if size > 0 and _direct_gguf_split_is_whole(local_target) else set()answered_from[0] = repo_idanswered_locally[0] = True- # Surface the resolution so the CLI gate matches on the local resolver's exact- # labels rather than the id's shape, and answers its real question -- would this- # load resolve -- with the loader itself, so no client mirrors its grammar.+ # Surface the resolution so the CLI gate matches the local resolver's exact labels rather than the+ # id's shape, and no client mirrors its grammar.return _local_response(repo_id, variants, has_vision, complete).model_copy(update = {"resolved_locally": True,@@ -1283,8 +1264,8 @@ async def get_gguf_variants_answer(})- # Reject invalid remote repo_ids up front (like download/delete) so a- # malformed id returns 400 instead of a 500 from the HF client.+ # Reject invalid remote repo_ids up front (like download/delete) so a malformed id returns 400 instead+ # of a 500 from the HF client.if not _is_valid_repo_id(repo_id):raise HTTPException(status_code = 400, detail = f"Invalid repo_id: {repo_id!r}")@@ -1321,8 +1302,8 @@ async def get_gguf_variants_answer(variants, has_vision, complete, snapshot = _merge_when_the_repo_id_loads(repo_id, cached, hub_cache)- # Name the answering snapshot: a repo-wide walk could read a different cache's- # context length, and a repo-dir walk a sibling revision's.+ # Name the answering snapshot: a repo-wide walk could read a different cache's context length, and a+ # repo-dir walk a sibling revision's.answered_from[0] = str(snapshot)# The lister leaves torn quants in: they stay listed for management, but not ready.return _with_state_partials(@@ -1372,8 +1353,8 @@ async def get_gguf_variants_answer(repo_id, cached, hub_cache)answered_from[0] = str(snapshot)- # Same reason as the local_only branch above, state partials included:- # an unreachable Hub is exactly when a resume has nowhere else to surface.+ # Same reason as the local_only branch above: an unreachable Hub is exactly when a resume has+ # nowhere else to surface, so state partials are included.return _with_state_partials(_local_response(repo_id, variants, has_vision, complete))@@ -1392,8 +1373,8 @@ async def get_gguf_variants_answer(return fallbackraise- # siblings is None only when the lister answered from its own repo-wide cache. Falling- # through is deliberate: readiness below still counts against this request's own cache.+ # siblings is None only when the lister answered from its own repo-wide cache; falling through is+ # deliberate, since readiness still counts against this request's own cache.if siblings is None:if not cache_reads_authorized:# `variants` is already the cache's answer, so declining a second one is@@ -1409,10 +1390,8 @@ async def get_gguf_variants_answer(best = pick_best_gguf(_default_variant_candidates(variants))default_variant = gguf_variant_key(best) if best else None- # Per-snapshot accounting: a variant counts as present only when one- # snapshot holds all its files (split GGUFs need every shard together),- # sizes are max across snapshots so shared blobs aren't double-counted,- # and keys are lowercased since cache dir casing can differ from repo_id.+ # Per-snapshot accounting: split GGUFs need every shard together, sizes are max across snapshots+ # so shared blobs are not double-counted, and keys are lowercased since cache casing can differ.cached_filenames_by_snapshot: list[dict[str, int]] = []cached_quant_bytes_by_snapshot: list[dict[str, int]] = []if _is_valid_repo_id(repo_id):@@ -1524,8 +1503,8 @@ async def get_gguf_variants_answer()):return True- # Byte fallback so a present quant isn't demoted by a filename mismatch;- # vision repos still need an mmproj cached (any precision).+ # Byte fallback so a present quant is not demoted by a filename mismatch; vision repos still need an+ # mmproj cached, at any precision.if not _quant_bytes_present(quant, variant.size_bytes):return Falseif (@@ -1557,8 +1536,8 @@ async def get_gguf_variants_answer(repo_signal_applies = hf_cache_scan.repo_signal_applies_to_snapshot(repo_cache_dir, scan_snapshot_dir)- # The excuse is that this snapshot holds the quant whole, so a quant it lacks stays the- # cancelled download and keeps its resume and delete affordances.+ # The excuse is that this snapshot holds the quant whole, so a quant it lacks stays the cancelled+ # download and keeps its resume and delete affordances.excused_quants = (frozenset()if repo_signal_applies or scan_snapshot_dir is None@@ -1570,8 +1549,8 @@ async def get_gguf_variants_answer(def _repo_signals_apply_to(quant: str) -> bool:return repo_signal_applies or quant.lower() not in excused_quants- # Manifest + marker + main incomplete-blob check: catches variants whose- # download was cancelled or whose expected shards are missing/undersized.+ # Manifest + marker + main incomplete-blob check: catches variants whose download was cancelled or whose+ # expected shards are missing/undersized.for variant in variants:try:requirement = requirements_by_quant.get(variant.quant.lower())@@ -1610,7 +1589,6 @@ async def get_gguf_variants_answer(if requirement is None or not _repo_signals_apply_to(variant.quant):continue# companion_hashes adds the MTP drafter (mmproj_hashes covers- # every mmproj precision in the repo, not just the planned one).if ((requirement.mmproj_hashes | requirement.companion_hashes) & incomplete_hashes) and _filenames_cached(@@ -1671,8 +1649,8 @@ async def get_gguf_variants_answer()def _compute_with_cleanables() -> VariantsAnswer:- # Returned with the answer, not read from the closure afterwards: coalesced callers- # share one computation and must all see the copy it actually answered from.+ # Returned with the answer, not read from the closure afterwards: coalesced callers share one+ # computation and must all see the copy it answered from.return VariantsAnswer(_compute_response(), answered_from[0])def _compute_response() -> GgufVariantsResponse:@@ -1680,9 +1658,7 @@ async def get_gguf_variants_answer(try:response = _compute()except Exception:- # Offline / metadata fetch failed with only an empty leftover- # <quant>/ folder cached: still surface it so the UI can delete it,- # otherwise re-raise the original error.+ # Surface an empty leftover <quant>/ folder so the UI can delete it; otherwise re-raise the original error.if skip:raiseenriched = _mark_empty_dir_cleanables(@@ -1709,8 +1685,7 @@ async def get_gguf_variants_answer(bool(offline),local_path or "",hf_cache_scan.token_fingerprint(hf_token),- # Switching cache storage must start a fresh scan rather than join one that is- # stuck on the old volume.+ # Switching cache storage must start a fresh scan rather than join one that is stuck on the old volume.configured_cache_key(),)try:diff --git a/studio/backend/hub/services/models/local_inventory.py b/studio/backend/hub/services/models/local_inventory.pyindex 5f2e5a00021..7e77d61b4a0 100644--- a/studio/backend/hub/services/models/local_inventory.py+++ b/studio/backend/hub/services/models/local_inventory.py@@ -65,16 +65,15 @@ _local_inventory_flights: dict[] = {}-# Retrying a superseded scan is only worth it while invalidations are occasional;-# past this the endpoint must answer instead of restarting the walk forever.+# Retrying a superseded scan is only worth it while invalidations are occasional; past this the endpoint must answer._LOCAL_INVENTORY_MAX_ATTEMPTS = 8class _LocalCacheChanged(RuntimeError):def __init__(self, response: LocalModelListResponse) -> None:super().__init__("local inventory sources changed during the scan")- # Carried so the attempt cap can serve the freshest scan it has instead- # of looping forever or answering with nothing.+ # Carried so the attempt cap can serve the freshest scan it has instead of looping forever or+ # answering with nothing.self.response = response@@ -121,6 +120,8 @@ def _has_immediate_model_weight(if entry.is_file() and _is_immediate_model_weight_file(entry):return Trueexcept OSError:+ # Skip individual children that are unreadable (permissions, broken symlinks, etc.) rather than+ # failing the entire scan.continueexcept OSError:return False@@ -218,8 +219,6 @@ def _scan_models_dir(continuehas_model_files = is_gguf_file or _has_immediate_model_signal(child)except OSError:- # Skip individual children that are unreadable (permissions, broken- # symlinks, etc.) rather than failing the entire scan.continueif not has_model_files:continue@@ -311,9 +310,8 @@ def _scan_hf_cache(if not discovered:return []if variant_states is None:- # Reached precisely when the caller's own guarded build already failed, so- # leaving this one bare handed the same exception straight back and undid- # that guard. Degrade to the per-repo reads instead, as the callers do.+ # Reached precisely when the caller's own guarded build already failed, so leaving this bare handed+ # the same exception back and undid that guard; degrade to the per-repo reads instead.try:variant_states = download_manifest.build_variant_state_index([("model", model_id, cache_dir) for _repo, model_id, _updated in discovered],@@ -364,8 +362,8 @@ def _scan_hf_cache(resolved = hf_cache_scan.resolve_hf_cache_realpath(repo_dir)scan_path = Path(resolved) if resolved else repo_dirload_path = repo_dir if active_cache else scan_path- # partial=False here; _apply_format_aware_partial below rewrites per-row- # so a hybrid repo's gguf row doesn't taint its safetensors row.+ # _apply_format_aware_partial below rewrites per row, so a hybrid repo's gguf row does not taint its+ # safetensors row.rows = _classify_local_path(scan_path,"hf_cache",@@ -394,8 +392,7 @@ def _scan_hf_cache()]else:- # Fallback row's model_format is "unknown"; either signal- # applies because we can't dispatch to a specific predicate.+ # The fallback row's model_format is "unknown", so either signal applies.rows = [_local_model_info(scan_path = repo_dir,@@ -445,8 +442,8 @@ def _scan_lmstudio_dir(lm_dir: Path, *, entry_limit: int | None = None) -> List[if not lm_dir.exists() or not lm_dir.is_dir():return []- # If the dir is itself a model dir (config + weights, or a diffusers pipeline root), it's not- # an LM Studio publisher structure -- return it as a single entry rather than descend.+ # A directory that is itself a model dir is not an LM Studio publisher structure, so return it as a+ # single entry rather than descend.if _is_model_directory(lm_dir) or _is_diffusers_pipeline_dir(lm_dir):try:updated_at = lm_dir.stat().st_mtime@@ -494,8 +491,8 @@ def _scan_lmstudio_dir(lm_dir: Path, *, entry_limit: int | None = None) -> List[)continue- # Child is itself a model dir: surface it directly, not as a publisher. A diffusers- # pipeline counts, or its component subdirs are walked as if they were models.+ # A child that is itself a model dir is surfaced directly, not as a publisher; a diffusers pipeline+ # counts, or its component subdirs are walked as models.if _is_model_directory(child) or _is_diffusers_pipeline_dir(child):try:updated_at = child.stat().st_mtime@@ -579,8 +576,7 @@ def _inventory_path_identity(raw_path: str) -> str:normalized = normalize_path(raw)return os.path.normcase(os.path.realpath(os.path.expanduser(normalized)))except (OSError, UnicodeError, ValueError):- # Keep malformed sources distinct until the worker's existing request or- # per-folder error boundary turns them into a 403/skip.+ # Keep malformed sources distinct until the worker's error boundary turns them into a 403 or a skip.return os.path.normcase(raw)@@ -696,8 +692,8 @@ async def _collect_models_from_default_sources(lambda path: _discover_hf_cache(path, entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES),folder_path,)- # Carry the registered path: the status registry is keyed on the row, not- # on the normalized Path this scan walks.+ # Carry the registered path: the status registry is keyed on the row, not on the normalized Path+ # this scan walks.custom_sources.append((folder_path, discovered, str(folder["path"])))state_repositories.extend(("model", model_id, folder_path) for _repo, model_id, _updated in discovered@@ -745,8 +741,8 @@ async def _collect_models_from_default_sources(if isinstance(e, OSError):record_scan_failure(row_path, e)continue- # Off the loop, like the scan above it: the probe opens directories, and on a- # stalled network mount scandir sits in the kernel with nothing to yield to.+ # Off the loop, like the scan above it: the probe opens directories, and on a stalled network mount+ # scandir sits in the kernel with nothing to yield to.await asyncio.to_thread(note_scan_folder_scanned, row_path, found = bool(custom_models))local_models.extend(_promote_to_custom_source(model) for model in custom_models)@@ -765,9 +761,8 @@ def _scan_custom_folder(supported_formats: set[ModelFormat] = {"gguf", "safetensors", "adapter"}def _is_supported(m: LocalModelInfo) -> bool:- # A diffusers pipeline keeps its weights in component subdirs, so the root has no loose- # weight file to classify and lands as "unknown". It is exactly what the Images and Video- # loaders take, so judge it on its shape rather than on a format the layout cannot report.+ # A diffusers pipeline keeps its weights in component subdirs, so its root lands as "unknown"; judge+ # it on its shape rather than on a format the layout cannot report.if m.model_format in supported_formats:return Truereturn _is_diffusers_pipeline_dir(Path(m.path))@@ -834,9 +829,8 @@ def _promote_to_custom_source(model: LocalModelInfo) -> LocalModelInfo:"custom",partial = model.partial,requires_variant = model.capabilities.requires_variant,- # Rebuilding from the format alone restored can_chat on rows the- # classifier had ruled out. The format is unchanged here, so- # carrying the old verdict through is idempotent.+ # Rebuilding from the format alone restored can_chat on rows the classifier had ruled out; the+ # format is unchanged here, so carrying the old verdict through is idempotent.can_chat_override = model.capabilities.can_chat,),}@@ -967,10 +961,8 @@ async def list_local_models_response(models_dir: str = "./models") -> LocalModel"""Coalesce overlapping local inventory requests for the same models root."""def classify(response: LocalModelListResponse) -> LocalModelListResponse:- # These rows feed the same pickers as /api/models/local. Classified inside the- # shared worker so retrying waiters do not repeat GGUF metadata reads, and only- # for a response that is actually about to be served.- # Classification reads GGUF headers, so keep it off the event loop too.+ # Classified inside the shared worker so retrying waiters do not repeat GGUF metadata reads, and off+ # the event loop, since classification reads GGUF headers.try:from hub.services.models import catalog_classification@@ -1005,8 +997,8 @@ async def list_local_models_response(models_dir: str = "./models") -> LocalModel# Discard obsolete results and retry their waiters against the current cache epoch.superseded: Optional[LocalModelListResponse] = Nonefor _attempt in range(_LOCAL_INVENTORY_MAX_ATTEMPTS):- # Epoch first: the sources and folders below are read after it, so any- # change to them lands in a later epoch and the post-scan check sees it.+ # Epoch first: the sources and folders below are read after it, so any change to them lands in a+ # later epoch and the post-scan check sees it.epoch = hf_cache_scan.hf_cache_scans_epoch()custom_folders = await _load_custom_folders()sources = _local_inventory_sources()@@ -1029,9 +1021,7 @@ async def list_local_models_response(models_dir: str = "./models") -> LocalModelexcept _LocalCacheChanged as changed:superseded = changed.responsecontinue- # Invalidations are outpacing the walk, so no scan will ever confirm as- # current. Answer with the freshest one (the loop only reaches here through- # the retry path, so there is always one) instead of rescanning forever.+ # Invalidations are outpacing the walk, so answer with the freshest scan instead of rescanning forever.logger.warning("Local inventory kept racing cache invalidations; serving the last scan")return await asyncio.to_thread(classify, superseded)@@ -1043,9 +1033,8 @@ def get_models_folder_response() -> dict:the desktop app reveals it in the OS file manager."""path = _resolve_hf_cache_dir()- # Create it if missing so "Open folder" works before the first download:- # HF builds the cache lazily, and studio only pre-creates the *default*- # dir, not a user's explicit HF_HOME / HF_HUB_CACHE.+ # Create it if missing so "Open folder" works before the first download: HF builds the cache+ # lazily, and studio pre-creates only the default dir, not a user's explicit HF_HOME/HF_HUB_CACHE.try:path.mkdir(parents = True, exist_ok = True)except OSError as e:diff --git a/studio/backend/hub/services/models/ollama.py b/studio/backend/hub/services/models/ollama.pyindex 62bf3c7b86e..7edccb20fdc 100644--- a/studio/backend/hub/services/models/ollama.py+++ b/studio/backend/hub/services/models/ollama.py@@ -48,8 +48,7 @@ _OLLAMA_LOADABLE_LAYER_MEDIA_TYPES = frozenset({"application/vnd.ollama.image.model","application/vnd.ollama.image.projector",- # License text does not affect model behavior and does not need to be- # carried into llama.cpp.+ # License text does not affect model behavior and does not need to be carried into llama.cpp."application/vnd.ollama.image.license",})@@ -149,8 +148,8 @@ def _ollama_links_dir(ollama_dir: Path) -> Optional[Path]:if _ensure_writable_dir(primary) is not None:return primary- # Namespace by a hash of the ollama_dir so two different Ollama roots- # don't collide. This is a cache path, not a security boundary.+ # Namespace by a hash of the ollama_dir so two different Ollama roots do not collide. A cache path,+ # not a security boundary.try:digest = hashlib.sha256(str(ollama_dir.resolve()).encode()).hexdigest()[:12]except (OSError, RuntimeError):@@ -192,8 +191,7 @@ def _make_ollama_blob_link(link_dir: Path, link_name: str, target: Path) -> Optilogger.debug("Could not resolve Ollama blob %s: %s", target, e)return None- # Skip if the link already points at the same blob. Use samefile, not size:- # `ollama pull` can swap a tag to a same-sized blob, leaving a stale link.+ # samefile, not size: `ollama pull` can swap a tag to a same-sized blob, leaving a stale link.try:if link_path.exists() and os.path.samefile(str(link_path), str(resolved)):return str(link_path)diff --git a/studio/backend/hub/services/snapshot_progress.py b/studio/backend/hub/services/snapshot_progress.pyindex 449ed185c29..3f42de8bc92 100644--- a/studio/backend/hub/services/snapshot_progress.py+++ b/studio/backend/hub/services/snapshot_progress.py@@ -37,17 +37,15 @@ logger = get_logger(__name__)# (repo_id, hf_token) -> (expected_total_bytes, expected_blob_hashes)SnapshotMetadataResolver = Callable[[str, Optional[str]], "tuple[int, frozenset[str]]"]-# (repo_id, hf_token) -> the files HF says this target should contain. Optional,-# and the only thing that lets a materialized snapshot with no manifest settle.+# The files HF says this target should contain: optional, and the only thing that lets a+# materialized snapshot with no manifest settle.SnapshotExpectedFilesResolver = Callable[[str, Optional[str]], Sequence["download_manifest.ExpectedFile"]]-# repo-relative snapshot path -> whether it belongs to the variant being polled.# Supplied per repo kind, so this module keeps knowing nothing about quant labels.VariantFileMatcher = Callable[[str], bool]# One progress log per 10% step per job, so an active download reports progress-# without emitting a line on every poll._progress_step_lock = threading.Lock()_last_progress_step: dict[str, int] = {}@@ -146,9 +144,8 @@ def _variant_bytes_on_disk(if not download_manifest.expected_path_is_safe(expected.path):continueif expected.sha256 and expected.sha256 in active_partial_hashes:- # A force/retry can leave the previous materialized file in the snapshot while- # a replacement for the same logical blob is being written. Count the current- # partial, not both generations of the file.+ # A force or retry can leave the previous materialized file beside a replacement for the same+ # logical blob; count the current partial, not both generations.continuetry:total += (snapshot_dir / expected.path).stat().st_size@@ -206,9 +203,8 @@ def _variant_main_shard_present("""if snapshot_dir is None or variant_file_matcher is None:return None- # An entry we could not read may BE the main shard -- a transient network-filesystem hiccup- # or a Windows ACL denial on one directory is not evidence the variant is gone. A positive- # match elsewhere still settles it; otherwise the reading stays unknown.+ # An entry we could not read may BE the main shard, so a transient failure is not evidence the+ # variant is gone: a positive match elsewhere still settles it, otherwise the reading is unknown.entries, complete = _walk_files(snapshot_dir)for path in entries:relative = path.relative_to(snapshot_dir).as_posix()@@ -267,24 +263,21 @@ def _materialized_bytes(snapshot_dir: Path, variant_file_matcher: "VariantFileManever linked contributes nothing, and the Windows copy layout is read as is."""try:- entries = list(snapshot_dir.rglob("*")) # unordered: the result is a sum+ entries = list(snapshot_dir.rglob("*"))except OSError:return 0# Same false "still active" as the companion clause below, reached by a stranded sidecar.entries = [path for path in entries if not is_appledouble_metadata(path)]def _accepts(relative: str, *, companions: bool) -> bool:- # A matcher that understands the distinction gets asked for it; an older one is- # answered as before. Only the GGUF matcher takes the keyword today.+ # A matcher that understands the distinction gets asked for it; only the GGUF matcher takes the keyword today.try:return bool(variant_file_matcher(relative, companions = companions))except TypeError:return bool(variant_file_matcher(relative))- # Companions (mmproj, the MTP drafter) are shared by every quant in the repo, so on their- # own they are not evidence that THIS one is here. Deleting a quant while a sibling keeps- # its companion left a positive reading for the deleted variant, which hydration reads as- # active: it re-adopts the stale job and blocks a fresh download of the same quant.+ # Companions are shared by every quant in the repo, so alone they are not evidence THIS one is+ # here: a stranded companion left a positive reading that hydration re-adopts.owns_a_main = Falsefor path in entries:try:@@ -343,21 +336,11 @@ def _snapshot_complete_on_disk(return Falsemanifest = entry_manifest.get()if manifest is None:- # No manifest, but HF metadata describes the same thing a manifest does:- # the exact files this download was supposed to fetch, at their declared- # sizes. Verify against that rather than refusing outright, because a- # manifest that was never written (metadata was unreachable when the run- # ended), was deleted, or sits under a cache scope this reader cannot- # name is not evidence of an unfinished download -- and refusing left a- # materialized snapshot partial forever.- #- # Nothing weaker will do here. The caller's expected_bytes is a catalog- # hint, and a byte total assembled from the shared blobs/ dir cannot- # tell this variant's bytes from a sibling's or a stray companion's, so- # neither is grounds for calling a download complete. Checking the named- # files against the snapshot dir also catches the case a blob-level- # tally structurally cannot: HF writes a blob and then links it, so a- # run killed in between leaves bytes that nothing points at.+ # HF metadata names the same files a manifest does, so verify against it: a manifest never+ # written, deleted, or under an unnameable cache scope is not evidence of an unfinished download,+ # and refusing left a materialized snapshot partial forever.+ # Nothing weaker will do: expected_bytes is a catalog hint, and a blobs/ tally cannot tell this+ # variant's bytes from a sibling's.metadata_expected = metadata_files.get()if not metadata_expected:return False@@ -368,15 +351,11 @@ def _snapshot_complete_on_disk(started_at = "",expected_files = metadata_expected,)- # ANY retained snapshot: the variant can be complete in an older revision while the newest- # holds only a sibling, and checking the newest alone left that download at 99% forever.- #- # But an older revision can carry the same FILENAMES at the same sizes and different- # content, and verify_against_disk does not read sha256 -- so with the resolved hashes in- # hand, require the snapshot's entries to actually resolve to them. Without that, a blob- # finalized but not yet linked (a crash between the two) let a stale revision satisfy the- # check and the job settled on files the app would not load. No resolved hashes means no- # such claim is possible, and the filename check stands alone as before.+ # ANY retained snapshot: the variant can be complete in an older revision while the newest holds+ # only a sibling, and checking the newest alone left that download at 99% forever.+ # An older revision can carry the same FILENAMES at the same sizes with different content and+ # verify_against_disk does not read sha256, so require the entries to resolve to known hashes;+ # with none resolved the filename check stands alone.for snap in snapshots:if not download_manifest.verify_against_disk(manifest, snap).ok:continue@@ -489,26 +468,20 @@ def compute_snapshot_progress(active_root = Path(metadata_hub_cache) if metadata_hub_cache else Noneexpected_total = max(expected_bytes, 0)- # Always resolve the revision's blob hashes so stale blobs from a superseded- # revision can't inflate the count; hashes degrade to empty (count-all) only- # when metadata is unavailable (e.g. offline). Take the larger total so a low- # caller hint can't cap the bar below the revision's real size.+ # Always resolve the revision's blob hashes so stale blobs from a superseded revision cannot+ # inflate the count; they degrade to empty (count-all) only when metadata is unavailable (e.g.+ # offline). Take the larger total so a low caller hint cannot cap the bar.meta_total, expected_hashes = metadata_resolver(repo_id, hf_token)meta_total = max(0, meta_total)expected_total = max(expected_total, meta_total)- # Without resolved hashes, a variant must not count unscoped blobs: sibling- # quants share one blobs/ dir, so a sibling's bytes (or .incomplete) would be- # misattributed and make the bar jump backward. A no-variant snapshot owns- # the whole dir, so it counts unscoped.+ # Without resolved hashes a variant must not count unscoped blobs, since sibling quants share one+ # blobs/ dir; a no-variant snapshot owns the whole dir and counts unscoped.count_unscoped = variant is None- # An empty hash set is "the expected file set could not be determined", not- # "this variant has no bytes" -- model_info failing (offline, or a 401 on a- # gated repo) is negatively cached, so one failed lookup keeps the set empty- # for the whole TTL. Filtering every blob out then reports a finished 33 GB- # variant as "0 B of 33 GB" and, because completion is never observed, leaves- # Retry/Resume on the card. Fall back to the variant's own files in the- # snapshot dir, which stay attributable when the blob hashes do not.+ # An empty hash set means the expected file set could not be determined, not that the variant has+ # no bytes: model_info failing (offline, or a 401 on a gated repo) is negatively cached, so one+ # failed lookup would report a finished 33 GB variant as "0 B of 33 GB" for the whole TTL. Fall+ # back to the snapshot dir's own files.variant_file_set_unknown = variant is not None and not expected_hashes# Resolved at most once, and only if a reading gets far enough to need it.metadata_files: "_Lazy[tuple[download_manifest.ExpectedFile, ...]]" = _Lazy(@@ -520,9 +493,8 @@ def compute_snapshot_progress()readings: list[tuple[int, int, Optional[str], bool, Optional[bool]]] = []- # The enumeration suppresses OSError per root, so an unreadable cache root came back as- # "no dirs" -- indistinguishable from a wiped cache, and hydration retires the job on- # that. Collected so the empty answer below can say unknown instead of absent.+ # The enumeration suppresses OSError per root, so an unreadable cache root came back as "no dirs",+ # indistinguishable from a wiped cache, which hydration retires the job on.scan_errors: list = []cache_dirs = (preferred_repo_cache_dirs(@@ -539,21 +511,19 @@ def compute_snapshot_progress()for entry in cache_dirs:completed_bytes = 0- # Keyed by logical blob: a broken advisory lock leaves several process-unique writers- # racing on one etag, and each downloads the WHOLE file, so summing them overshoots.+ # Keyed by logical blob: a broken advisory lock leaves several writers racing on one etag, each+ # downloading the WHOLE file, so summing them overshoots.partial_bytes: dict[str, int] = {}completed_hashes: set[str] = set()- # A partial this reading could not attribute to any target. It is not evidence FOR this- # variant -- it may be a sibling quant's -- but with the hashes unresolved it is not- # evidence against it either, and the by-name scan cannot see it because a partial is- # not linked into a snapshot yet.+ # A partial attributable to no target is not evidence for this variant, nor against it while the+ # hashes are unresolved, and the by-name scan cannot see it.unattributable_partial = Falsecache_path = hf_cache_scan.resolve_hf_cache_realpath(entry)blobs_dir = entry / "blobs"+ # Skip a blob that vanished mid-poll rather than zeroing the reading.try:- # os.stat, not Path.is_dir(): is_dir() swallows the OSError and answers False, so a- # Windows ACL or network-filesystem failure on the directory ITSELF read as "no- # blobs here" -- a MEASURED absence, which hydration retires the job on.+ # os.stat, not Path.is_dir(): is_dir() swallows the OSError and answers False, turning a Windows ACL+ # or network-filesystem failure into a MEASURED absence hydration retires on.blobs_present = stat_module.S_ISDIR(os.stat(blobs_dir).st_mode)except FileNotFoundError:blobs_present = False@@ -564,13 +534,11 @@ def compute_snapshot_progress(try:blob_entries = list(blobs_dir.iterdir())except OSError as exc:- # An unreadable blobs dir is not an empty one. Swallowing it produced a- # measured zero -- target_present false, cache_measured true -- and hydration- # retires a job whose cache was never actually read.+ # An unreadable blobs dir is not an empty one: swallowing it produced a measured zero+ # (target_present false, cache_measured true) and retired a job whose cache was never read.scan_errors.append(exc)blob_entries = []for f in blob_entries:- # Skip a blob that vanished mid-poll rather than zeroing the reading.try:if not f.is_file():continue@@ -594,29 +562,19 @@ def compute_snapshot_progress(completed_hashes.add(f.name)completed_bytes += f.stat().st_sizeexcept OSError as exc:- # A blob we could not inspect is not a blob that is not there. Swallowing it- # produced a MEASURED absence, which hydration reads as "gone" and retires a- # persisted download whose target may be intact behind the error.+ # A blob we could not inspect is not a blob that is not there: swallowing it produced a MEASURED+ # absence, which hydration reads as gone.scan_errors.append(exc)continue- # A finalized blobs/<hash> supersedes every partial for the same logical blob: the racer- # that installed the file settled it, so a leftover writer's bytes are a duplicate, not- # progress. Counting both overshot the expected total and, because completion requires- # no bytes in flight, pinned a fully downloaded variant at 0.99 until the orphan was- # swept -- and an orphan outlives the process that would have unlinked it on the way out.+ # A finalized blobs/<hash> supersedes every partial for the same logical blob, so counting both+ # overshot the expected total and pinned a downloaded variant at 0.99 until the orphan was swept.for blob_hash in completed_hashes:partial_bytes.pop(blob_hash, None)- # Largest wins, deliberately, even though a killed attempt's partial can sit beside its- # replacement's under the same etag. Preferring the freshest mtime reads better against- # a corpse but oscillates between two GENUINELY live writers, which is what a broken- # advisory lock produces: each write makes a different one newest, so the bar would- # jump between the leader and the straggler.- #- # A corpse is not this reading's problem to solve, because it should not outlive the- # job that made it: a terminal job sweeps its own blobs without waiting out the- # abandonment grace, and a backend that died before it could is caught at boot. If one- # survives both (an unrelated client holding the blob lock over it) the bar over-reads- # until the next sweep, which is a smaller wrong than a reading that will not sit still.+ # Largest wins deliberately: preferring the freshest mtime reads better against a corpse but+ # oscillates between two genuinely live writers, which is what a broken advisory lock produces.+ # A corpse should not outlive the job that made it (a terminal job sweeps its own blobs, and a+ # backend that died first is caught at boot); if one survives both, over-reading until the next+ # sweep is a smaller wrong than a reading that will not sit still.in_progress_bytes = sum(partial_bytes.values())snapshot_dirs: "_Lazy[list[Path]]" = _Lazy(lambda entry = entry: _retained_snapshot_dirs(entry)@@ -630,12 +588,9 @@ def compute_snapshot_progress())if variant is not None:- # The best reading across every retained snapshot, for the same reason presence is- # established across all of them: the variant can live in an older revision while- # the newest holds a sibling, and reading only the newest reported 0 bytes for a- # complete cached quant. Do this even when hashes resolved: huggingface_hub 1.18's- # Windows copy layout can move a completed file directly into the snapshot and- # leave no finalized entry in blobs/, so a blob-only tally stays at zero.+ # The best reading across every retained snapshot, since the variant can live in an older+ # revision, and because huggingface_hub 1.18's Windows copy layout can move a completed file+ # straight into the snapshot and leave a blob-only tally at zero.manifest = entry_manifest.get()on_disk = max((@@ -650,50 +605,34 @@ def compute_snapshot_progress(),default = 0,)- # Clamped, because the matcher behind the no-manifest half of that reading accepts- # every companion in the repo and so can overshoot.+ # Clamped, because the matcher behind the no-manifest half accepts every companion in the repo and+ # so can overshoot.if expected_total > 0:on_disk = min(on_disk, expected_total)completed_bytes = max(completed_bytes, on_disk)- # Does THIS target have anything here, as opposed to the repo dir existing at all?- # Sibling quants share one repo cache dir, so deleting a variant's files leaves the- # dir standing and the reading came back "zero bytes, cache_path names a directory"- # -- which hydration reads as resumable and adopts as a phantom, blocking a fresh- # download of that same variant until the idle grace expires. False only on positive- # evidence of absence: a named variant, a file set we could resolve, no manifest of- # its own and nothing counted. Anything less certain stays None.+ # Sibling quants share one repo cache dir, so deleting a variant's files leaves the dir standing+ # and the reading came back "zero bytes, cache_path names a directory", which hydration adopts as+ # a phantom. False only on positive evidence of absence; anything less certain stays None.target_present: Optional[bool] = Noneif variant is not None and not variant_file_set_unknown:- # The MATERIALIZED file, not the blob tally. These counters come from the shared- # blobs/ dir: deleting a variant's snapshot symlink leaves its finalized blob- # behind, and a companion blob shared with a sibling keeps the count positive on its- # own -- so a quant that is gone read as present and hydration re-adopted the- # phantom. The by-name scan the unknown-hash branch already uses answers this- # properly; bytes only stand in when it cannot (nothing readable to look at), where- # in-progress bytes are still real evidence that this download is live.+ # The MATERIALIZED file, not the blob tally: deleting a snapshot symlink leaves the finalized blob+ # behind and a shared companion keeps the count positive, so a quant that is gone read as present.+ # Bytes only stand in when there is nothing readable to scan.scanned = _variant_present_in_any_snapshot(entry, variant_file_matcher)if scanned is not None:target_present = scanned or bool(in_progress_bytes)else:target_present = bool(completed_bytes or in_progress_bytes)elif variant is not None:- # An unresolvable file set does not have to mean unknown. The byte reading above- # already walked the snapshot dir, whose entries are named per file, so it can- # answer the narrower question the hash filter could not: is a main shard of THIS- # quant here? Without this, a repo dir kept alive by a sibling read as "zero bytes,- # cache_path names a directory", which hydration adopts as a resumable phantom and- # which then blocks a fresh download of the deleted quant until the idle grace runs- # out -- the same trap as above, on the metadata-unavailable path.- # ...across EVERY snapshot the entry retains, not only the newest by mtime. A cache- # can hold several revisions, and the requested quant living in an older one while- # the newest holds a sibling read as absent -- and hydration retires a job whose- # target is still usable.+ # The byte reading already walked the snapshot dir, whose entries are named per file, so it can+ # answer whether a main shard of THIS quant is here; without it a repo dir kept alive by a sibling+ # read as "zero bytes, cache_path names a directory" and was adopted as a resumable phantom.+ # Across EVERY snapshot the entry retains, not only the newest: a quant living in an older+ # revision read as absent and hydration retired a job whose target is still usable.scanned = _variant_present_in_any_snapshot(entry, variant_file_matcher)if scanned is not None:- # ...unless the shared blobs/ dir holds a partial nothing here can attribute.- # An idle or restarted download whose hashes were refused has its bytes in an- # .incomplete blob that is not linked into any snapshot yet, so the by-name scan- # answers a confident "absent" -- and hydration retires a job that is resumable.+ # Unless the shared blobs/ dir holds an unattributable partial: an idle or restarted download+ # whose hashes were refused has its bytes in an .incomplete blob no snapshot links.target_present = None if (not scanned and unattributable_partial) else scannedreadings.append((@@ -719,52 +658,39 @@ def compute_snapshot_progress(selected = max(readings,- # complete_on_disk last-but-one: two remembered caches can clamp to the SAME byte total- # while only one has a manifest that verifies against disk, and root order then carried- # the unverified reading -- the response stayed capped at 99% and kept offering Retry.+ # complete_on_disk last-but-one: two remembered caches can clamp to the SAME byte total while only+ # one has a manifest that verifies against disk, and root order then capped the response at 99%.key = lambda item: (item[0] + item[1], bool(item[3]), item[0]),default = None,)if selected is None:- # Nothing measured AND a root that could not be listed: the cache may be entirely- # intact behind that error, so this is unknown, not gone.+ # Nothing measured AND a root that could not be listed: the cache may be entirely intact behind that+ # error, so this is unknown, not gone.if scan_errors:return _empty_progress(expected_bytes, measured = False)return emptycompleted_bytes, in_progress_bytes, cache_path, complete_on_disk, target_present = selected- # Presence is a property of the SET of caches, not of the one that happened to hold the most- # bytes. A sibling-only repo dir and a cache that still holds this variant's manifest both- # read as zero bytes, so root order alone could pick the False and retire a job whose target- # another scanned cache proves is there. A positive reading anywhere wins.+ # Presence is a property of the SET of caches: a sibling-only repo dir and a cache still holding+ # this variant's manifest both read as zero bytes, so a positive reading anywhere wins.presence = [reading[4] for reading in readings]if any(verdict is True for verdict in presence):target_present = Trueelif any(verdict is None for verdict in presence):- # And absence needs EVERY scanned cache to say so. One unknown reading -- a cache with- # no readable snapshot to identify the variant from, whose shared blobs dir may still- # hold an unattributable partial -- is not evidence the target is gone.+ # Absence needs EVERY scanned cache to say so: one unknown reading is not evidence the target is gone.target_present = Nonedownloaded_bytes = completed_bytes + in_progress_bytes- # A reading taken while some root could not be listed is only ever a LOWER bound. The- # active root raising EACCES/EIO while a remembered cache still holds the repo dir (with a- # sibling quant in it) gives a non-null selection carrying target_present False and zero- # bytes -- and hydration reads that as "the variant was deleted" and retires the job,- # though the inaccessible root may hold every byte of it. Absence claims need a complete- # scan behind them, so downgrade them to unknown; a positive reading is unaffected,- # because bytes we did see are bytes that are there.+ # A reading taken while some root could not be listed is only ever a LOWER bound: the active root+ # raising EACCES/EIO while a remembered cache holds the repo dir gives target_present False and+ # zero bytes, which hydration reads as "deleted". Downgrade absence claims to unknown; a positive+ # reading is unaffected.scan_incomplete = bool(scan_errors)if scan_incomplete:if not target_present:target_present = None- # Subtract the companion baseline only while still counted in completed_bytes- # and the variant is not yet verified complete, else genuine progress reads as- # 0-byte. A baseline that covers the whole expected total is never subtracted- # either: a variant that was already on disk when the job was claimed carries- # exactly that, and netting it out leaves "0 B of 0 B" -- nothing for the bar- # to draw and, to the frontend, a job to evict. complete_on_disk covered that- # only while completion could be verified, which is the one thing an- # unresolvable file set takes away.+ # Subtract the companion baseline only while it is still counted in completed_bytes and the+ # variant is unverified, and never when it covers the whole expected total: that leaves+ # "0 B of 0 B", which the frontend evicts as a dead job.effective_baseline_bytes = (completed_baseline_bytesif (@@ -801,8 +727,8 @@ def compute_snapshot_progress("cache_measured": not scan_incomplete,}- # Cap at 0.99 until the manifest-backed disk check verifies completion: on- # resume, completed bytes can sit above the threshold while files still download.+ # Cap at 0.99 until the manifest-backed disk check verifies completion: on resume, completed bytes+ # can sit above the threshold while files still download.progress = (1.0if complete_on_diskdiff --git a/studio/backend/hub/storage/scan_folders.py b/studio/backend/hub/storage/scan_folders.pyindex c19f43fac9a..2fa3a57b994 100644--- a/studio/backend/hub/storage/scan_folders.py+++ b/studio/backend/hub/storage/scan_folders.py@@ -32,8 +32,8 @@ def _denied_path_prefixes() -> list[str]:if system == "Linux":return ["/proc", "/sys", "/dev", "/etc", "/boot", "/run"]if system == "Darwin":- # realpath() resolves /etc -> /private/etc, /tmp -> /private/tmp on macOS,- # so include the /private variants to avoid bypasses.+ # realpath() resolves /etc -> /private/etc and /tmp -> /private/tmp on macOS, so include the+ # /private variants to avoid bypasses.return ["/System","/Library",@@ -127,8 +127,8 @@ def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:if not os.path.isdir(normalized):raise ValueError("Path must be a directory, not a file")if is_local_filesystem_root(normalized):- # A local fs root ("/", "C:\\") would expose denied system dirs via browse;- # a UNC share root (\\server\share) has none under it and stays registerable.+ # A local fs root would expose denied system dirs via browse; a UNC share root has none under it and+ # stays registerable.raise ValueError("The filesystem root cannot be registered")if _contains_sensitive_path_component(normalized):raise ValueError("Credential or configuration directories are not allowed")diff --git a/studio/backend/hub/tests/test_companion_assets.py b/studio/backend/hub/tests/test_companion_assets.pyindex 5827faef4bb..15dc2152f57 100644--- a/studio/backend/hub/tests/test_companion_assets.py+++ b/studio/backend/hub/tests/test_companion_assets.py@@ -92,8 +92,6 @@ def _install(monkeypatch, *repos):# ---------------------------------------------------------------------------------------------# 1. Compatible quants reuse ONE cached copy of the companion assets.-# --------------------------------------------------------------------------------------------def test_two_quants_of_one_family_resolve_to_a_single_companion_base():@@ -105,8 +103,6 @@ def test_two_quants_of_one_family_resolve_to_a_single_companion_base():# ---------------------------------------------------------------------------------------------# 2. Deletion shows what is reclaimed and what remains.-# --------------------------------------------------------------------------------------------def test_deleting_one_of_two_quants_retains_the_companions(monkeypatch):@@ -140,8 +136,6 @@ def test_whole_repo_delete_reclaims_every_quant(monkeypatch):# ---------------------------------------------------------------------------------------------# 3. Shared assets are not removed while another installed model needs them. ADVERSARIAL.-# --------------------------------------------------------------------------------------------def test_shared_base_cannot_be_deleted_while_a_quant_is_installed(monkeypatch):@@ -184,8 +178,8 @@ def test_a_base_reached_only_through_a_card_tag_is_still_protected(monkeypatch):gguf = _repo("unsloth/FLUX.2-klein-9B-GGUF", [("flux-2-klein-9b-Q4_K_M.gguf", Q4_K_M_BYTES)])other_base = "black-forest-labs/FLUX.2-klein-9B"_install(monkeypatch, gguf, _base_repo(other_base))- # Before any load has been recorded: the id says klein-9B, so the curated klein-9B base is- # derived even though the family default is klein-4B.+ # Before any load has been recorded: the id says klein-9B, so the curated klein-9B base is derived+ # even though the family default is klein-4B.assert companion_cleanup.companion_dependents(other_base) == ["unsloth/FLUX.2-klein-9B-GGUF"]companion_assets.record_companion_link("unsloth/FLUX.2-klein-9B-GGUF", other_base)assert companion_cleanup.companion_dependents(other_base) == ["unsloth/FLUX.2-klein-9B-GGUF"]@@ -221,8 +215,6 @@ def test_the_guard_leaves_ordinary_repos_alone(monkeypatch):# ---------------------------------------------------------------------------------------------# 4. Orphaned companion assets can be found and removed without hand-editing the HF cache.-# --------------------------------------------------------------------------------------------def test_orphan_listing_is_empty_while_a_quant_is_installed(monkeypatch):@@ -421,8 +413,8 @@ def test_reusing_an_existing_link_still_refreshes_its_recency(monkeypatch):monkeypatch.setattr(companion_assets, "_MAX_LINKS", 2)assert companion_assets.record_companion_link("unsloth/old-GGUF", BASE_REPO) is Trueassert companion_assets.record_companion_link("unsloth/new-GGUF", BASE_REPO) is True- # The old one resolves again to the SAME base: nothing new to record, but it is now the- # freshest link, so the next checkpoint displaces the other one.+ # The old one resolves again to the SAME base: nothing new to record, but it is now the freshest+ # link, so the next checkpoint displaces the other one.assert companion_assets.record_companion_link("unsloth/old-GGUF", BASE_REPO) is Falseassert companion_assets.read_companion_links()["unsloth/old-gguf"] == [BASE_REPO]assert companion_assets.record_companion_link("unsloth/third-GGUF", BASE_REPO) is Truediff --git a/studio/backend/hub/tests/test_dataset_local_options.py b/studio/backend/hub/tests/test_dataset_local_options.pyindex a7549c3da41..c650eac8b8b 100644--- a/studio/backend/hub/tests/test_dataset_local_options.py+++ b/studio/backend/hub/tests/test_dataset_local_options.py@@ -233,6 +233,7 @@ def test_snapshot_options_infer_one_train_split_from_unlabelled_files(tmp_path):snapshot = _snapshot(tmp_path)_rows(snapshot, "records.jsonl")+ # A card with no bytes declares nothing, so it blocks nothing.assert local_options._snapshot_options(snapshot) == {("default", "train")}@@ -255,6 +256,11 @@ def test_snapshot_options_reject_a_shard_name_the_loader_refuses(tmp_path):_rows(snapshot, "data/train-clean-00000-of-00001.parquet")# datasets raises on the name rather than falling through to the keyword stages.+ # datasets calls .get on each entry, so a list of scalars raises.+ # DatasetCard.load reads the card as utf-8 and raises before any data file.+ # The inner .jsonl still votes, so datasets picks json and dies on the .zst.+ # DatasetCardData updates a dict from it and raises on a scalar.+ # imagefolder wins the vote and drops the jsonl, so there is nothing to train on.assert local_options._snapshot_options(snapshot) == set()@@ -290,6 +296,9 @@ def test_snapshot_options_ignore_a_metadata_file_when_it_steers_a_split(tmp_path(snapshot / "test").mkdir()(snapshot / "test" / "dataset_info.json").write_text("{}", encoding = "utf-8")+ # The header alone yields no row, and datasets still builds train around it.+ # resolve_pattern keeps a link only when its target is a file, so the loader never sees this one and the+ # surviving split still loads.assert local_options._snapshot_options(snapshot) == {("default", "train")}@@ -298,6 +307,7 @@ def test_snapshot_options_match_extensions_case_sensitively(tmp_path):_rows(snapshot, "TRAIN.JSONL")# A POSIX glob never matches .JSONL, so datasets finds no data files here.+ # datasets treats the empty declaration as authoritative and exposes no config.assert local_options._snapshot_options(snapshot) == set()@@ -379,7 +389,6 @@ def test_snapshot_options_read_a_blob_symlink_the_cache_owns(tmp_path):snapshot.mkdir(parents = True)(snapshot / "records.jsonl").symlink_to(blobs / "abc123")- # The normal cache layout: every file is a link into blobs.assert local_options._snapshot_options(snapshot) == {("default", "train")}@@ -388,8 +397,8 @@ def test_snapshot_options_leave_a_declared_card_to_its_own_configs(tmp_path):_card(snapshot, "configs:\n- config_name: foo\n data_dir: foo\n")_rows(snapshot, "foo/records.jsonl", "test.jsonl")- # The card names the loader's configs, and inference cannot reproduce them, so the- # picker does not invent a default one beside them.+ # The card names the loader's configs, and inference cannot reproduce them, so the picker does not+ # invent a default one beside them.assert local_options._snapshot_options(snapshot) == set()@@ -422,7 +431,6 @@ def test_snapshot_options_stay_empty_for_a_readme_only_cache(tmp_path):snapshot = _snapshot(tmp_path)(snapshot / "README.md").write_text("# Alpaca\n", encoding = "utf-8")- # The first #8140 negative: nothing to train on, so nothing is offered.assert local_options._snapshot_options(snapshot) == set()@@ -431,7 +439,6 @@ def test_snapshot_options_stay_empty_for_a_licence_only_subdirectory(tmp_path):(snapshot / "legal").mkdir()(snapshot / "legal" / "LICENSE").write_text("MIT\n", encoding = "utf-8")- # The second #8140 negative.assert local_options._snapshot_options(snapshot) == set()@@ -669,7 +676,6 @@ def test_snapshot_options_reject_a_split_whose_module_needs_a_missing_codec(tmp_(snapshot / "train.csv").write_text("text\nrow\n", encoding = "utf-8")(snapshot / "train2.jsonl.zst").write_bytes(b"\x28\xb5\x2f\xfd\x00\x00")- # The inner .jsonl still votes, so datasets picks json and dies on the .zst.assert local_options._snapshot_options(snapshot) == set()@@ -686,7 +692,6 @@ def test_snapshot_options_stand_down_beside_a_card_that_cannot_be_decoded(tmp_pa(snapshot / "README.md").write_bytes(b"\xff\xfe---\nconfigs: x\n---\n")_rows(snapshot, "train.jsonl")- # DatasetCard.load reads the card as utf-8 and raises before any data file.assert local_options._snapshot_options(snapshot) == set()@@ -695,7 +700,6 @@ def test_snapshot_options_stand_down_beside_a_malformed_dataset_info_list(tmp_pa_card(snapshot, "dataset_info: [foo]\n")_rows(snapshot, "train.jsonl")- # datasets calls .get on each entry, so a list of scalars raises.assert local_options._snapshot_options(snapshot) == set()@@ -704,8 +708,6 @@ def test_snapshot_options_ignore_a_dangling_link_beside_a_good_file(tmp_path):_rows(snapshot, "train.jsonl")(snapshot / "train-missing.jsonl").symlink_to("../../blobs/gone")- # resolve_pattern keeps a link only when its target is a file, so the loader- # never sees this one and the surviving split still loads.assert local_options._snapshot_options(snapshot) == {("default", "train")}@@ -714,7 +716,6 @@ def test_snapshot_options_drop_a_header_only_csv_split(tmp_path):(snapshot / "train.csv").write_text("text\nrow\n", encoding = "utf-8")(snapshot / "test.csv").write_text("text\n", encoding = "utf-8")- # The header alone yields no row, and datasets still builds train around it.assert local_options._snapshot_options(snapshot) == {("default", "train")}@@ -730,7 +731,6 @@ def test_snapshot_options_stand_down_beside_an_empty_split_declaration(tmp_path)_card(snapshot, "dataset_info:\n splits: {}\n")_rows(snapshot, "train.jsonl")- # datasets treats the empty declaration as authoritative and exposes no config.assert local_options._snapshot_options(snapshot) == set()@@ -754,6 +754,7 @@ def test_snapshot_options_keep_a_compressed_csv_split(tmp_path):snapshot = _snapshot(tmp_path)(snapshot / "train.csv.gz").write_bytes(gzip.compress(b"text\nrow\n"))+ # `[]` parses fine and yields no row, so datasets builds train around it.# Compressed bytes say nothing about the rows inside, so the split still stands.assert local_options._snapshot_options(snapshot) == {("default", "train")}@@ -763,7 +764,6 @@ def test_snapshot_options_drop_a_split_holding_an_empty_json_container(tmp_path)_rows(snapshot, "train.jsonl")(snapshot / "test.json").write_text("[]", encoding = "utf-8")- # `[]` parses fine and yields no row, so datasets builds train around it.assert local_options._snapshot_options(snapshot) == {("default", "train")}diff --git a/studio/backend/hub/tests/test_dataset_services.py b/studio/backend/hub/tests/test_dataset_services.pyindex 816748a0cdb..d78b502e602 100644--- a/studio/backend/hub/tests/test_dataset_services.py+++ b/studio/backend/hub/tests/test_dataset_services.py@@ -254,8 +254,8 @@ def test_raw_dataset_cache_has_data_does_not_loop_on_a_link_to_an_ancestor(monkerepo_root = _dataset_snapshot(monkeypatch, tmp_path, ("README.md", "data/notes.md"))data = next((repo_root / "snapshots").iterdir()) / "data"(data / "loop").symlink_to(data, target_is_directory = True)- # a junction resolves like a link while reporting `is_symlink()` False, and Linux cannot- # create one, so this is a symlink that answers the way a junction does.+ # a junction resolves like a link while reporting is_symlink() False, and Linux cannot create one,+ # so this is a symlink that answers the way a junction doesreal_is_symlink = Path.is_symlinkmonkeypatch.setattr(Path,@@ -442,8 +442,8 @@ def test_raw_dataset_cache_has_data_never_walks_outside_the_snapshot(monkeypatch(outside / "deep").mkdir(parents = True)(outside / "deep" / "huge.parquet").write_bytes(b"PAR1")repo_root = _dataset_snapshot(monkeypatch, tmp_path, ("README.md",))- # Named like clutter, so it is not payload evidence either, and the tree behind it is- # never entered -- which is what stops a scan from stalling on someone else's disk.+ # Named like clutter, so it is not payload evidence either, and the tree behind it is never entered,+ # which is what stops a scan from stalling on someone else's disk.(repo_root / "snapshots" / "abc" / ".hidden_link").symlink_to(outside, target_is_directory = True)assert cache_inventory._raw_dataset_cache_has_data("Org/Data", repo_root) is False@@ -704,8 +704,8 @@ def test_delete_processed_dataset_scopes_to_selected_root(monkeypatch, tmp_path))assert result == {"status": "deleted", "repo_id": "Org/Data"}- assert not (selected_root / "Org___Data").exists() # the selected copy is deleted- assert (other_root / "Org___Data").exists() # the other cache home is untouched+ assert not (selected_root / "Org___Data").exists()+ assert (other_root / "Org___Data").exists()def _app_cache_entry(monkeypatch, hub_cache: Path, repo_id: str, commit_hash: str):diff --git a/studio/backend/hub/tests/test_download_lifecycle.py b/studio/backend/hub/tests/test_download_lifecycle.pyindex 241b47354fd..6e8a1b53940 100644--- a/studio/backend/hub/tests/test_download_lifecycle.py+++ b/studio/backend/hub/tests/test_download_lifecycle.py@@ -97,9 +97,9 @@ def test_completion_invalidates_inventory_before_publishing_state(monkeypatch):def test_xet_failure_retries_over_http_for_model_and_dataset(monkeypatch, tmp_path):monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")monkeypatch.setattr(download_lifecycle.threading, "Thread", _ImmediateThread)- # _ImmediateThread mutates the stdlib threading module, which the shared Zoo watchdog also- # imports, so it would run INLINE and block in Event.wait() before finalize_worker_exit could- # set its stop flag. Stub the seam instead.+ # _ImmediateThread mutates the stdlib threading module the shared Zoo watchdog also imports, so it+ # would run INLINE and block in Event.wait() before finalize_worker_exit could stop it. Stub the+ # seam instead.monkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", lambda *a, **k: None)register_worker = download_lifecycle.register_worker@@ -303,7 +303,7 @@ def test_a_verdict_carried_onto_the_http_rung_is_still_charged(monkeypatch, tmp_spawned.append(use_xet)if use_xet:raise OSError("cannot fork")- return _Proc(0) # the HTTP worker starts and completes+ return _Proc(0)monkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", _start)monkeypatch.setattr(download_lifecycle, "spawn_worker", flaky_spawn)@@ -334,9 +334,8 @@ def test_a_verdict_carried_onto_the_http_rung_is_still_charged(monkeypatch, tmp_transport = download_registry.TRANSPORT_XET,watch_name = "model-watch",)- # Before the verdict: a double whose signature has fallen behind spawn_worker raises- # TypeError at the call, which reads as a spawn failure and still records the verdict via- # _give_up() -- green while this path never ran. Assert the HTTP worker actually started.+ # Before the verdict: a double whose signature has fallen behind spawn_worker raises TypeError,+ # which reads as a spawn failure and still records the verdict, staying green.assert spawned == [True, False], "the HTTP rung never ran, so the verdict proves nothing"assert recorded == [verdict], "a real Xet stall was dropped when HTTP finished the download"@@ -344,9 +343,8 @@ def test_a_verdict_carried_onto_the_http_rung_is_still_charged(monkeypatch, tmp_def test_http_failure_remains_terminal(monkeypatch, tmp_path):monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")monkeypatch.setattr(download_lifecycle.threading, "Thread", _ImmediateThread)- # _ImmediateThread mutates the stdlib threading module, which the shared Zoo watchdog also- # imports, so it would run INLINE and block in Event.wait() before finalize_worker_exit could- # set its stop flag. Stub the seam instead.+ # _ImmediateThread mutates the stdlib threading module the shared Zoo watchdog also imports, so stub+ # the seam instead.monkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", lambda *a, **k: None)register_worker = download_lifecycle.register_workerregistry = download_registry.DownloadRegistry()@@ -515,7 +513,7 @@ def _trip_xet_worker(monkeypatch.setattr(download_lifecycle.threading, "Thread", _ImmediateThread)def _start(registry, key, proc, *, on_stall, **kwargs):- on_stall(message) # the watchdog tripped+ on_stall(message)return Nonemonkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", _start)@@ -688,7 +686,7 @@ def test_the_xet_baseline_is_sampled_before_the_worker_spawns(monkeypatch, tmp_pmonkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", lambda *a, **k: None)order = []- sizes = iter([0, 5_000]) # nothing before the spawn, bytes present after it+ sizes = iter([0, 5_000])monkeypatch.setattr(download_lifecycle,@@ -743,7 +741,7 @@ def test_a_stall_verdict_racing_a_completed_worker_is_not_recorded(monkeypatch,monkeypatch,tmp_path,"Download appears stalled (xet transport) -- no progress for 30s",- rc = 0, # the worker actually finished+ rc = 0,)== []), "a completed worker was charged a stall it raced"diff --git a/studio/backend/hub/tests/test_download_manifest_scoping.py b/studio/backend/hub/tests/test_download_manifest_scoping.pyindex 7cbd137fdda..691702824f1 100644--- a/studio/backend/hub/tests/test_download_manifest_scoping.py+++ b/studio/backend/hub/tests/test_download_manifest_scoping.py@@ -77,8 +77,8 @@ def test_purge_state_preserves_active_legacy_when_deleting_inactive_cache(monkeyremoved = download_manifest.purge_state("model", "Org/Model", hub_cache = str(previous))assert removed is True- assert not scoped.is_file() # the inactive cache's copy is gone- assert legacy.is_file() # the active cache's legacy state survives+ assert not scoped.is_file()+ assert legacy.is_file()def test_purge_state_removes_legacy_owned_by_the_deleted_cache(monkeypatch, tmp_path):@@ -100,7 +100,7 @@ def test_purge_state_removes_legacy_owned_by_the_deleted_cache(monkeypatch, tmp_removed = download_manifest.purge_state("model", "Org/Model", hub_cache = str(previous))assert removed is True- assert not legacy.is_file() # owned by the deleted cache -> purged+ assert not legacy.is_file()def test_scope_digest_is_shared_with_the_ownership_canonicalization(monkeypatch, tmp_path):@@ -302,7 +302,7 @@ def test_windows_shaped_copy_cache_scope_survives_a_restart(monkeypatch, tmp_pathub_cache = tmp_path / "Hub"snapshot = hub_cache / "models--Org--Model" / "snapshots" / "rev0"snapshot.mkdir(parents = True)- (snapshot / "model.gguf").write_bytes(b"x" * 16) # a copy, not a symlink+ (snapshot / "model.gguf").write_bytes(b"x" * 16)assert not (snapshot / "model.gguf").is_symlink()monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")monkeypatch.setattr(@@ -741,14 +741,14 @@ def test_an_unreadable_cache_root_is_unknown_rather_than_absent(monkeypatch, tmpmetadata_resolver = lambda *a, **k: (33_000_000_000, frozenset()),)assert "cache_path" not in reading- # And it survives DownloadProgressResponse, which defaults cache_path to None and would- # otherwise reinstate the omission as an explicit "absent" before the frontend saw it.+ # And it survives DownloadProgressResponse, which defaults cache_path to None and would otherwise reinstate+ # the omission as an explicit "absent" before the frontend saw it.assert reading["cache_measured"] is Falsefrom hub.schemas.downloads import DownloadProgressResponse- # DECLARED on the response model, or FastAPI drops it before the frontend sees it: these- # readings are serialized through DownloadProgressResponse, whose cache_path defaults to- # None, so the omission alone was reinstated as an explicit "absent" on the wire.+ # DECLARED on the response model, or FastAPI drops it before the frontend sees it: these readings+ # serialize through DownloadProgressResponse, whose cache_path defaults to None, so the omission+ # alone was reinstated as an explicit "absent".assert "cache_measured" in DownloadProgressResponse.__annotations__assert reading["downloaded_bytes"] == 0@@ -772,7 +772,7 @@ def test_a_scope_whose_payload_is_lost_reads_back_as_a_digest(monkeypatch, tmp_p((variant, path),) = download_manifest.iter_variant_markers("model", "Org/Model", hub_cache = str(hub_cache))- assert variant == "@diffusion" # intact while the payload is readable+ assert variant == "@diffusion"assert "--variant--@sha256-" in path.namepayload = json.loads(path.read_text(encoding = "utf-8"))@@ -804,7 +804,7 @@ def test_a_variant_with_nothing_of_its_own_says_so(monkeypatch, tmp_path):entry = tmp_path / "hub" / "models--unsloth--Model-GGUF"(entry / "blobs").mkdir(parents = True)- (entry / "blobs" / "sibling").write_bytes(b"x" * 32) # Q8_0's, not ours+ (entry / "blobs" / "sibling").write_bytes(b"x" * 32)monkeypatch.setattr(snapshot_progress, "preferred_repo_cache_dirs", lambda *a, **k: [entry])diff --git a/studio/backend/hub/tests/test_empty_variant_folder.py b/studio/backend/hub/tests/test_empty_variant_folder.pyindex dc4872b0709..cb9d312d837 100644--- a/studio/backend/hub/tests/test_empty_variant_folder.py+++ b/studio/backend/hub/tests/test_empty_variant_folder.py@@ -17,7 +17,7 @@ def _make_snapshot(root: Path) -> Path:(snap / "UD-IQ1_M").mkdir(parents = True)(snap / "UD-IQ1_M" / "GLM-UD-IQ1_M-00001-of-00002.gguf").write_bytes(b"x")(snap / "UD-IQ1_M" / "GLM-UD-IQ1_M-00002-of-00002.gguf").write_bytes(b"y")- (snap / "UD-IQ1_S").mkdir(parents = True) # empty leftover+ (snap / "UD-IQ1_S").mkdir(parents = True)return snap@@ -29,17 +29,17 @@ def test_list_empty_gguf_variant_dirs_finds_empty_leftover(tmp_path, monkeypatchdef test_list_empty_excludes_quant_with_files_in_another_snapshot(tmp_path, monkeypatch):snap1 = tmp_path / "s1" / "snapshots" / "rev"- (snap1 / "UD-IQ1_S").mkdir(parents = True) # empty here+ (snap1 / "UD-IQ1_S").mkdir(parents = True)snap2 = tmp_path / "s2" / "snapshots" / "rev"(snap2 / "UD-IQ1_S").mkdir(parents = True)- (snap2 / "UD-IQ1_S" / "m-UD-IQ1_S-00001-of-00001.gguf").write_bytes(b"z") # has shards+ (snap2 / "UD-IQ1_S" / "m-UD-IQ1_S-00001-of-00001.gguf").write_bytes(b"z")monkeypatch.setattr(gguf, "iter_hf_cache_snapshots", lambda repo_id: iter([snap1, snap2]))assert gguf.list_empty_gguf_variant_dirs("org/Repo-GGUF") == set()def test_list_empty_ignores_non_quant_dirs(tmp_path, monkeypatch):snap = tmp_path / "snapshots" / "rev"- (snap / "not-a-quant").mkdir(parents = True) # empty but not a quant label+ (snap / "not-a-quant").mkdir(parents = True)monkeypatch.setattr(gguf, "iter_hf_cache_snapshots", lambda repo_id: iter([snap]))assert gguf.list_empty_gguf_variant_dirs("org/Repo-GGUF") == set()@@ -143,8 +143,8 @@ def test_mark_empty_dir_cleanables_flips_listed_variant(monkeypatch):def _force_compute_to_raise(monkeypatch):- # Drive _compute() down its remote path, fail metadata, and have both cache- # fallbacks miss so the original error re-raises.+ # Drive _compute() down its remote path, fail metadata, and have both cache fallbacks miss so the+ # original error re-raises.def _boom(*a, **k):raise RuntimeError("offline")@@ -164,8 +164,8 @@ def _force_compute_to_raise(monkeypatch):def test_get_variants_surfaces_cleanable_when_metadata_fails(monkeypatch):- # Offline / model_info fails and only an empty leftover folder is cached:- # the cleanable must still be returned instead of the error propagating.+ # Offline / model_info fails and only an empty leftover folder is cached: the cleanable must still+ # be returned instead of the error propagating.import asyncio_force_compute_to_raise(monkeypatch)diff --git a/studio/backend/hub/tests/test_model_services.py b/studio/backend/hub/tests/test_model_services.pyindex ded44508c2a..32146b25899 100644--- a/studio/backend/hub/tests/test_model_services.py+++ b/studio/backend/hub/tests/test_model_services.py@@ -232,7 +232,8 @@ def test_custom_inventory_filters_mtp_companions_at_registered_root(tmp_path, modef test_custom_inventory_filters_dspark_companions_at_registered_root(tmp_path):- # Registering dspark/ as the scan root strips it from every relative path, so the basename prefix carries the exclusion.+ # Registering dspark/ as the scan root strips it from every relative path, so the basename prefix+ # carries the exclusion.root = tmp_path / "dspark"root.mkdir()for name in (@@ -887,7 +888,7 @@ def test_cached_inventory_discards_a_scan_that_raced_an_invalidation(def scan(**_kwargs):scans.append(1)if len(scans) == 1:- epoch[0] += 1 # a deletion completes while this walk runs+ epoch[0] += 1return [{"repo_id": "Org/Deleted"}]return [{"repo_id": "Org/Kept"}]@@ -913,7 +914,7 @@ def test_cached_inventory_scan_stops_retrying_under_constant_invalidation(monkeydef scan(**_kwargs):scans.append(1)- epoch[0] += 1 # every walk is superseded before it returns+ epoch[0] += 1return [{"repo_id": "Org/Model"}]monkeypatch.setattr(cache_inventory, "all_hf_cache_scans", lambda: [])@@ -962,7 +963,7 @@ def test_local_inventory_scan_stops_retrying_under_constant_invalidation(monkeypasync def fake_scan(models_dir, custom_folders, sources):scans.append(1)- epoch[0] += 1 # every walk is superseded before it returns+ epoch[0] += 1return SimpleNamespace(models = [], model_copy = lambda update = None: SimpleNamespace(models = []))async def no_folders():@@ -1010,11 +1011,8 @@ def test_local_inventory_filters_and_dedupes_off_event_loop(monkeypatch):@pytest.mark.parametrize("change_kind", ["folders", "epoch"])def test_local_inventory_requests_share_scan(monkeypatch, change_kind):- # What the flight key needs from the helper is that it is total and stable on- # hostile input, not that it echoes it back. POSIX realpath() rejects an- # embedded NUL with ValueError so the raw string is normcased through; Windows- # non-strict realpath falls back to abspath and joins the cwd instead. Assert- # the property, not one platform's spelling of it.+ # Assert the property, not one platform's spelling: POSIX realpath() rejects an embedded NUL with+ # ValueError while Windows non-strict realpath falls back to abspath and joins the cwd.for hostile in ("\0", "\ud800"):identity = local_inventory._inventory_path_identity(hostile)assert identity == local_inventory._inventory_path_identity(hostile)@@ -1390,9 +1388,9 @@ def test_legacy_unscoped_download_state_falls_back_only_for_selected_cache(monke@pytest.mark.parametrize("variant",[- "unknown/variant with spaces", # unsafe characters force the hashed filename- "double--hyphen", # aliases the --variant-- delimiter, also hashed- "Q4_K_M", # spelled literally, the control+ "unknown/variant with spaces",+ "double--hyphen",+ "Q4_K_M",],)def test_index_finds_an_unreadable_marker_stored_under_a_hashed_filename(@@ -1447,11 +1445,9 @@ def test_cached_gguf_scan_degrades_when_the_shared_index_cannot_be_built(monkeyp"utils.hf_cache_settings.get_hf_cache_paths",lambda: SimpleNamespace(hub_cache = hub_cache),)- # An undecodable byte in a cache directory name reaches us as a lone surrogate,- # and the repo key cannot be hashed from it. Spelled directly rather than via- # os.fsdecode(b"...\xff...") because Windows decodes filenames as UTF-8 with- # surrogatepass, where a bare 0xff raises instead of producing a surrogate.- # Nothing here touches the filesystem, so the string alone is the whole case.+ # An undecodable byte in a cache directory name reaches us as a lone surrogate, and the repo key+ # cannot be hashed from it. Spelled directly rather than via os.fsdecode(b"...\xff...") because+ # Windows decodes filenames as UTF-8 with surrogatepass, where a bare 0xff raises.bad_repo = "Org/Re\udcffpo"repos = [SimpleNamespace(@@ -1509,9 +1505,8 @@ def test_state_scan_survives_a_state_filename_with_an_undecodable_byte(monkeypatwith open(corrupt, "wb") as handle:handle.write(json.dumps({"version": 2, "repo_type": "model"}).encode("utf-8"))except (OSError, UnicodeError) as e:- # Only POSIX filesystems that accept arbitrary bytes can hold this name, so- # only there is the bug reachable. macOS rejects it with EILSEQ and Windows- # has no byte-level filename API at all.+ # Only POSIX filesystems that accept arbitrary bytes can hold this name: macOS rejects it with+ # EILSEQ and Windows has no byte-level filename API at all.pytest.skip(f"filesystem will not hold an undecodable filename: {e}")index = download_manifest.build_variant_state_index(@@ -1938,8 +1933,8 @@ def _diffusion_scan(monkeypatch.setattr(cache_inventory, "all_hf_cache_scans", lambda: [SimpleNamespace(repos = [repo])])- # The real signature takes snapshot_dir; a double that omits it raises TypeError, which the- # per-repo except swallows into an empty row list, silently skipping every assertion below.+ # The real signature takes snapshot_dir; a double that omits it raises TypeError, which the per-repo+ # except swallows into an empty row list, silently skipping every assertion below.monkeypatch.setattr(cache_inventory.hf_cache_scan,"is_snapshot_partial",@@ -1994,10 +1989,9 @@ def test_cached_models_scan_keeps_a_complete_pipeline_loadable(monkeypatch, tmp__file("model_index.json", 900),_file("vae/diffusion_pytorch_model.safetensors", 300_000_000),_file("text_encoder/model.safetensors", 900_000_000),- # Unsharded, because this fixture stands for a COMPLETE pipeline. It used to name a- # lone "-00001-of-00002" shard with no index, which is not loadable at all: with no- # index diffusers reads the component as unsharded and asks for the plain name, never- # the numbered file. The scan now calls that partial, as it always should have.+ # Unsharded, because this fixture stands for a COMPLETE pipeline: it used to name a lone+ # "-00001-of-00002" shard with no index, and with no index diffusers reads the component as+ # unsharded and asks for the plain name, so a lone shard is not loadable._file("transformer/diffusion_pytorch_model.safetensors", 4_000_000_000),],task = "text-to-image",@@ -2065,8 +2059,8 @@ def test_a_companion_mirror_carries_the_flag_on_the_hub_row(monkeypatch, tmp_pat)assert row["companion"] is True- # Startup auto-load filters on capabilities.can_chat (isChattableCachedRepo), never on the- # flag, so a row carrying only the flag was still auto-loadable as a chat model.+ # Startup auto-load filters on capabilities.can_chat, never on the flag, so a row carrying only the+ # flag was still auto-loadable as a chat model.assert row["capabilities"]["can_chat"] is False@@ -2268,8 +2262,7 @@ def test_cached_scans_hide_embedders_configured_by_snapshot_path(monkeypatch, tmdef test_cached_models_scan_keeps_unrelated_repo_with_custom_generic_embedder(monkeypatch, tmp_path):- # A custom embedder with a generic basename ("org/model") must be hidden by EXACT repo-id- # match only: substring basename matching used to drop real chat models from the inventory.+ # EXACT repo-id match only: substring basename matching used to drop real chat models from the inventory.from core.rag import config as rag_configmonkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/model")@@ -3006,11 +2999,8 @@ def test_gguf_download_progress_fallback_logs_warning(monkeypatch):))- # cache_path is ABSENT, not null. Null means "no cache dir for this repo exists", which- # hydration acts on by retiring the persisted job; a scan that threw is not evidence of- # that, and dropping a job whose partial cache is still on disk costs the user the resume.- # cache_measured carries the same distinction through DownloadProgressResponse, which- # defaults cache_path to None and would otherwise turn the omission back into an "absent".+ # cache_path is ABSENT, not null: null means no cache dir exists, which hydration acts on by+ # retiring the persisted job, and cache_measured carries the same distinction.assert result == {"downloaded_bytes": 0,"completed_bytes": 0,@@ -3289,8 +3279,8 @@ def test_gguf_progress_shows_main_when_companion_left_the_count(monkeypatch, tmpdef test_gguf_progress_complete_on_disk_ignores_full_baseline(monkeypatch, tmp_path):- # A variant already complete on disk carries a baseline equal to its full size; subtracting it- # would report 0/0 for a finished variant, which the frontend evicts as gone.+ # A variant already complete on disk carries a baseline equal to its full size; subtracting it would+ # report 0/0, which the frontend evicts as gone.entry = tmp_path / "models--Org--Model-GGUF"snap = entry / "snapshots" / "rev0"blobs = entry / "blobs"@@ -3386,13 +3376,13 @@ def test_gguf_progress_complete_on_disk_ignores_full_baseline(monkeypatch, tmp_pdef test_gguf_progress_scoped_hashes_exclude_sibling_quant(monkeypatch, tmp_path):- # The "instant ~900 MB" bug: with this variant's hashes resolved, progress counts ONLY its- # in-progress blob, never a sibling quant's finalized bytes in the shared blobs/ dir.+ # The "instant ~900 MB" bug: with this variant's hashes resolved, progress counts ONLY its in-+ # progress blob, never a sibling quant's finalized bytes in the shared blobs/ dir.entry = tmp_path / "models--Org--Model-GGUF"blobs = entry / "blobs"blobs.mkdir(parents = True)- (blobs / "siblinghash").write_bytes(b"z" * 900) # other quant, complete- (blobs / "mainhash.incomplete").write_bytes(b"x" * 5) # this variant, started+ (blobs / "siblinghash").write_bytes(b"z" * 900)+ (blobs / "mainhash.incomplete").write_bytes(b"x" * 5)requirement = gguf_variants._GgufVariantRequirement(main_filenames = frozenset({"model-Q4_K_M.gguf"}),@@ -3451,10 +3441,8 @@ def test_gguf_progress_scoped_hashes_exclude_sibling_quant(monkeypatch, tmp_pathdef test_gguf_progress_unknown_hashes_does_not_count_foreign_blobs(monkeypatch, tmp_path):- # With a variant's hashes unresolved (metadata flaked, no manifest), the- # shared blobs/ dir's FINALIZED blobs must NOT be counted wholesale: a cached- # sibling quant (``siblinghash``) alongside is the "instant ~900 MB" bug.- # With no .incomplete present, downloaded must be 0.+ # With hashes unresolved, the shared blobs/ dir's FINALIZED blobs must not be counted wholesale: a+ # cached sibling quant alongside is the "instant ~900 MB" bug.entry = tmp_path / "models--Org--Model-GGUF"snap = entry / "snapshots" / "rev0"blobs = entry / "blobs"@@ -3503,16 +3491,13 @@ def test_gguf_progress_unknown_hashes_does_not_count_foreign_blobs(monkeypatch,def test_gguf_progress_unknown_hashes_drops_unscoped_incomplete_blob(monkeypatch, tmp_path):- # With hashes unresolved, an .incomplete in the shared blobs/ dir can't be- # attributed to this variant (it may be a concurrent sibling's active write),- # so it is dropped, mirroring the finalized-blob guard. In production the- # worker writes the manifest before any .incomplete exists, so hashes resolve- # via the manifest backstop and this window never suppresses real progress.+ # With hashes unresolved an .incomplete cannot be attributed to this variant, so it is dropped; in+ # production the worker writes the manifest before any .incomplete exists.entry = tmp_path / "models--Org--Model-GGUF"blobs = entry / "blobs"blobs.mkdir(parents = True)- (blobs / "activehash.incomplete").write_bytes(b"x" * 50) # unattributable- (blobs / "siblinghash").write_bytes(b"z" * 900) # finalized sibling+ (blobs / "activehash.incomplete").write_bytes(b"x" * 50)+ (blobs / "siblinghash").write_bytes(b"z" * 900)async def _run_inline(fn, *args, **kwargs):return fn(*args, **kwargs)@@ -3547,22 +3532,20 @@ def test_gguf_progress_unknown_hashes_drops_unscoped_incomplete_blob(monkeypatch))- assert result["downloaded_bytes"] == 0 # unscoped .incomplete not leaked- assert result["completed_bytes"] == 0 # finalized sibling still ignored+ assert result["downloaded_bytes"] == 0+ assert result["completed_bytes"] == 0def test_gguf_progress_unknown_hashes_no_backward_dip_when_variant_finalizes(monkeypatch, tmp_path):- # Regression for the two-variant dip: with hashes unresolved, the first quant- # finalizes while the sibling still writes its .incomplete. The sibling's- # bytes used to leak into this numerator, dipping the bar ~99% -> ~78% for- # one poll. The unscoped .incomplete must be dropped so the reading stays 0.+ # Regression for the two-variant dip: the sibling's .incomplete bytes used to leak into this+ # numerator, dipping the bar to ~78% for one poll.entry = tmp_path / "models--unsloth--SmolLM2-360M-Instruct-GGUF"blobs = entry / "blobs"snap = entry / "snapshots" / "rev0"blobs.mkdir(parents = True)snap.mkdir(parents = True)- own_total = 218_673_760 # Q2_K finished blob size (denominator)- sibling_total = 234_686_560 # Q3_K_M total+ own_total = 218_673_760+ sibling_total = 234_686_560def _sparse_file(path: Path, size: int) -> None:with path.open("wb") as handle:@@ -3607,8 +3590,8 @@ def test_gguf_progress_unknown_hashes_no_backward_dip_when_variant_finalizes(mon))- assert result["downloaded_bytes"] == 0 # sibling .incomplete did not leak- assert result["progress"] == 0 # no ~0.78 backward dip+ assert result["downloaded_bytes"] == 0+ assert result["progress"] == 0def _unresolvable_variant_metadata(@@ -3672,7 +3655,7 @@ def test_gguf_progress_unknown_hashes_reports_the_variant_files_on_disk(monkeypablobs.mkdir(parents = True)(snap / "model-Q4_K_M.gguf").write_bytes(b"x" * 100)(snap / "mmproj-F16.gguf").write_bytes(b"y" * 30)- (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900) # sibling quant+ (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900)(blobs / "mainhash").write_bytes(b"x" * 100)(blobs / "siblinghash").write_bytes(b"z" * 900)monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")@@ -3686,9 +3669,9 @@ def test_gguf_progress_unknown_hashes_reports_the_variant_files_on_disk(monkeypa))- assert result["completed_bytes"] == 130 # main quant + shared companion+ assert result["completed_bytes"] == 130assert result["downloaded_bytes"] == 130- assert result["progress"] > 0 # not the "0 B of 130" card+ assert result["progress"] > 0def test_gguf_progress_unknown_hashes_keeps_a_total_under_a_full_baseline(monkeypatch, tmp_path):@@ -3761,7 +3744,7 @@ def test_gguf_progress_unknown_hashes_prefers_the_manifest_file_set(monkeypatch,)assert result["completed_bytes"] == 100- assert result["complete_on_disk"] is True # manifest verifies against disk+ assert result["complete_on_disk"] is Trueassert result["progress"] == 1.0@@ -3942,7 +3925,7 @@ def test_gguf_progress_without_a_manifest_needs_every_expected_blob(monkeypatch,snap.mkdir(parents = True)blobs.mkdir(parents = True)(snap / "model-Q4_K_M-00001-of-00002.gguf").write_bytes(b"x" * 400)- (blobs / "shard1").write_bytes(b"x" * 400) # shard2 never arrived+ (blobs / "shard1").write_bytes(b"x" * 400)monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")async def _run_inline(fn, *args, **kwargs):@@ -3989,7 +3972,7 @@ def test_gguf_progress_without_a_manifest_needs_every_expected_blob(monkeypatch,)assert result["complete_on_disk"] is False- assert result["progress"] == 0.99 # capped, not settled+ assert result["progress"] == 0.99def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypatch, tmp_path):@@ -4017,8 +4000,8 @@ def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypatblobs = entry / "blobs"snap.mkdir(parents = True)blobs.mkdir(parents = True)- # Copy layout: the snapshot holds real files, and blobs/ was never populated- # with anything this reading could have matched by hash.+ # Copy layout: the snapshot holds real files, and blobs/ was never populated with anything this reading+ # could have matched by hash.(snap / "model-Q4_K_M.gguf").write_bytes(b"x" * 33_000)assert not (snap / "model-Q4_K_M.gguf").is_symlink()@@ -4027,9 +4010,8 @@ def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypat"utils.hf_cache_settings.get_hf_cache_paths",lambda: SimpleNamespace(hub_cache = str(hub_cache)),)- # A manifest as an older build filed it: hashed from the unresolved spelling,- # and with no sha256 because HF metadata was already unreachable when the- # worker recorded it from the finished snapshot.+ # A manifest as an older build filed it: hashed from the unresolved spelling, and with no sha256+ # because HF metadata was already unreachable when the worker recorded it.legacy = state_dir.manifest_path("model","Org/Model-GGUF",@@ -4064,10 +4046,10 @@ def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypat))- assert result["downloaded_bytes"] == 33_000 # not "0 B"- assert result["completed_bytes"] == 33_000 # what settles the job terminal+ assert result["downloaded_bytes"] == 33_000+ assert result["completed_bytes"] == 33_000assert result["expected_bytes"] == 33_000- assert result["complete_on_disk"] is True # so Retry/Resume goes away+ assert result["complete_on_disk"] is Trueassert result["progress"] == 1.0@@ -4090,8 +4072,8 @@ def test_gguf_progress_without_a_manifest_needs_the_snapshot_materialized(monkeyblobs = entry / "blobs"snap.mkdir(parents = True)blobs.mkdir(parents = True)- (blobs / "mainhash").write_bytes(b"x" * 100) # never linked into rev0- (snap / "mmproj-F32.gguf").write_bytes(b"y" * 5_000) # not in this plan+ (blobs / "mainhash").write_bytes(b"x" * 100)+ (snap / "mmproj-F32.gguf").write_bytes(b"y" * 5_000)monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")async def _run_inline(fn, *args, **kwargs):@@ -4147,7 +4129,7 @@ def test_running_job_never_reads_progress_from_a_remembered_cache(monkeypatch, tcomplete = previous / "models--Org--Model" / "blobs"complete.mkdir(parents = True)(complete / "mainhash").write_bytes(b"x" * 100)- active = tmp_path / "active" # not created yet+ active = tmp_path / "active"monkeypatch.setattr(hf_cache_state,"hf_cache_roots",@@ -4476,9 +4458,8 @@ def test_completed_gguf_split_variant_requires_all_shards(tmp_path):def test_completed_gguf_variants_ignores_big_endian_by_the_loader_label(tmp_path):- # The scan reads Q4_K_M here, the loader reads F16 and refuses the file as big-endian. By- # the scan's label it would vouch for Q4_K_M, and _complete_with_servable skips complete- # quants, so the torn split below would be reported downloaded.+ # The scan reads Q4_K_M while the loader reads F16 and refuses the file as big-endian, so by the+ # scan's label it would vouch for Q4_K_M and _complete_with_servable would skip the torn split.snapshot = tmp_path / "snapshot"snapshot.mkdir()(snapshot / "0-F16-be-checkpoint-Q4_K_M.gguf").write_bytes(b"GGUF")@@ -4658,9 +4639,8 @@ def test_download_registry_repo_keys_are_case_insensitive():repo_id = "Org/Repo",variant = "Q8_0",)- # The same variant under a different-cased repo id resolves to the same- # job, so the second claim attaches to the running one instead of starting- # a duplicate.+ # The same variant under a different-cased repo id resolves to the same job, so the second claim+ # attaches to the running one instead of starting a duplicate.duplicate_claimed, duplicate_state = registry.claim("org/repo::Q8_0",download_registry.TRANSPORT_HTTP,@@ -4709,9 +4689,9 @@ def test_download_registry_allows_disjoint_gguf_variant_downloads():def test_download_registry_allows_overlapping_same_transport_variant_downloads():- # Two variants sharing one mmproj blob still download together on one- # transport: huggingface_hub's per-blob lock serializes the shared write and- # prepare_cache_for_transport never purges a blob a peer is writing.+ # Two variants sharing one mmproj blob still download together on one transport:+ # huggingface_hub's per-blob lock serializes the shared write and prepare_cache_for_transport+ # never purges a blob a peer is writing.registry = download_registry.DownloadRegistry()claimed, state = registry.claim(@@ -4740,9 +4720,8 @@ def test_download_registry_allows_overlapping_same_transport_variant_downloads()def test_download_registry_variant_delete_does_not_block_sibling_download():- # Deleting one quant's partial must be allowed while a different quant of the- # same repo is downloading, and must protect every blob the live sibling is- # writing (including a shared mmproj companion).+ # Deleting one quant's partial must be allowed while a different quant of the same repo is+ # downloading, and must protect every blob the live sibling is writing.registry = download_registry.DownloadRegistry()registry.claim("Org/Repo::Q8_0",@@ -4764,8 +4743,8 @@ def test_download_registry_variant_delete_does_not_block_sibling_download():assert registry.has_active_peer_variant("Org/Repo", "Q4_K_M") is Trueassert registry.has_active_peer_variant("Org/Repo", "Q8_0") is False- # While Q4_K_M is being deleted, re-downloading it is blocked but an- # untouched third variant may still start.+ # While Q4_K_M is being deleted, re-downloading it is blocked but an untouched third variant may still+ # start.blocked, blocked_state = registry.claim("Org/Repo::Q4_K_M",download_registry.TRANSPORT_HTTP,@@ -4790,8 +4769,9 @@ def test_download_registry_variant_delete_does_not_block_sibling_download():def test_partial_gguf_reconstruction_dedupes_variant_casing(monkeypatch):- # The manifest keeps original casing while the marker is lowercased; offline- # reconstruction must collapse them to ONE entry (manifest's casing), not two.+ # The manifest keeps original casing while the marker is lowercased; offline reconstruction must+ # collapse them to ONE entry, in the manifest's casing.+ # Per-variant blob hashes (distinct main shard, shared mmproj companion).monkeypatch.setattr(download_manifest,"iter_variant_manifests",@@ -4812,16 +4792,13 @@ def test_partial_gguf_reconstruction_dedupes_variant_casing(monkeypatch):def test_partial_gguf_reconstruction_drops_a_variant_read_off_the_filename(monkeypatch):- # An unreadable payload leaves the reader the filename, whose digest fragment- # names nothing and would re-key to a further hash on resume. A variant genuinely- # called sha256-<32 hex> reads the same but is stored under the hash of itself, so- # the file it came from tells them apart where the spelling cannot.+ # An unreadable payload leaves only the filename, whose digest fragment names nothing; a variant+ # genuinely called sha256-<32 hex> reads the same but is stored under the hash of itself.digest = "sha256-" + "0" * 32entries = [- # variant, the file it was recovered from- (f"@{digest}", Path(f"repo--variant--@{digest}.json")), # scope, payload lost- (digest, Path(f"repo--variant--{digest}.json")), # ditto, older tag- (digest, Path("repo--variant--@sha256-" + "c" * 32 + ".json")), # real, payload intact+ (f"@{digest}", Path(f"repo--variant--@{digest}.json")),+ (digest, Path(f"repo--variant--{digest}.json")),+ (digest, Path("repo--variant--@sha256-" + "c" * 32 + ".json")),("Q4_K_M", Path("repo--variant--q4_k_m.json")),]monkeypatch.setattr(@@ -4838,8 +4815,8 @@ def test_partial_gguf_reconstruction_drops_a_variant_read_off_the_filename(monkedef test_download_registry_serializes_cross_transport_variant_downloads():- # An HTTP append-resume and an XET rewrite of the same shared blob would- # corrupt each other, so different-transport variants are serialized.+ # An HTTP append-resume and an XET rewrite of the same shared blob would corrupt each other, so+ # different-transport variants are serialized.registry = download_registry.DownloadRegistry()claimed, state = registry.claim(@@ -4868,10 +4845,9 @@ def test_download_registry_serializes_cross_transport_variant_downloads():def test_download_registry_allows_unknown_hash_gguf_variant_downloads():- # Resolved blob hashes are NOT required to run two same-transport variants- # concurrently: on-disk safety comes from each worker purging only its own- # main-quant blobs plus huggingface_hub's per-etag lock. Requiring them here- # used to reject the second variant whenever a metadata fetch flaked.+ # Resolved blob hashes are NOT required to run two same-transport variants concurrently: safety+ # comes from each worker purging only its own blobs plus huggingface_hub's per-etag lock, and+ # requiring them rejected the second variant whenever a metadata fetch flaked.registry = download_registry.DownloadRegistry()claimed, state = registry.claim(@@ -4902,9 +4878,8 @@ def test_download_registry_allows_unknown_hash_gguf_variant_downloads():def test_finalize_worker_exit_never_kills_a_healthy_worker(monkeypatch, tmp_path):- # finalize_worker_exit relies solely on the worker's exit code and never kills- # a live process: huggingface_hub already bounds reads with timeouts, so a- # liveness kill could only false-cancel a healthy download.+ # finalize_worker_exit relies solely on the worker's exit code and never kills a live process:+ # huggingface_hub already bounds reads with timeouts, so a liveness kill could only false-cancel.import inspectimport ioimport logging@@ -5022,9 +4997,8 @@ def test_prepare_cache_for_transport_purges_cross_transport_companion(monkeypatcblobs = _vision_cache_root(monkeypatch, tmp_path)companion = frozenset({"shared-mmproj"})- # An interrupted XET download stamps the companion marker "xet" and leaves a- # sparse partial. A later HTTP download of a different variant must purge it,- # else the HTTP resumer appends to the sparse bytes and corrupts the blob.+ # An interrupted XET download leaves a sparse partial, so a later HTTP download of a different+ # variant must purge it, else the HTTP resumer appends to the sparse bytes and corrupts the blob.download_registry.prepare_cache_for_transport("model","Org/Vision",@@ -5050,8 +5024,7 @@ def test_prepare_cache_for_transport_purges_cross_transport_companion(monkeypatcdef test_prepare_cache_for_transport_preserves_same_transport_companion(monkeypatch, tmp_path):"""Only a hub that can still append to the partial earns the same-transport reprieve."""- # The purge asks partial_is_resumable, so patching the hub-version helper it wraps would- # be a no-op here.+ # The purge asks partial_is_resumable, so patching the hub-version helper it wraps would be a no-op here.monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: True)blobs = _vision_cache_root(monkeypatch, tmp_path)companion = frozenset({"shared-mmproj"})@@ -5569,10 +5542,8 @@ def test_model_download_watcher_invalidates_hf_cache_scan(monkeypatch):def test_two_concurrent_same_repo_variants_both_complete(monkeypatch, tmp_path):- # End-to-end proof that two GGUF variants of ONE repo download concurrently- # without cancelling each other, with real registry/finalize/subprocess/watch- # threads exercising the claim gate, register, finalize funnel, and- # classify_exit under true concurrency.+ # End-to-end proof that two GGUF variants of one repo download concurrently without cancelling each+ # other, with real registry, finalize, subprocess and watch threads under true concurrency.import subprocessimport time@@ -5592,7 +5563,6 @@ def test_two_concurrent_same_repo_variants_both_complete(monkeypatch, tmp_path):"download_transport_unavailable_reason",lambda _transport: None,)- # Per-variant blob hashes (distinct main shard, shared mmproj companion).monkeypatch.setattr(downloads.gguf_variants,"gguf_variant_blob_hashes",@@ -5718,8 +5688,8 @@ def _patch_variant_delete_side_effects(monkeypatch, hub_cache = None):"purge_state",lambda *_args, **_kwargs: False,)- # The repo under test lives in this cache; make it the active one so the- # delete scopes to it (default target root is the active hub cache).+ # The repo under test lives in this cache; make it the active one so the delete scopes to it (default target+ # root is the active hub cache).if hub_cache is not None:monkeypatch.setattr("utils.hf_cache_settings.get_hf_cache_paths",@@ -6000,8 +5970,8 @@ def test_a_partial_scan_cannot_report_the_target_as_gone(monkeypatch, tmp_path):def test_a_complete_scan_still_reports_the_target_as_gone(monkeypatch, tmp_path):- # The same reading with no scan error keeps the positive-evidence verdict, or the fix- # above would simply disable the phantom-adoption guard it is protecting.+ # The same reading with no scan error keeps the positive-evidence verdict, or the fix above would+ # simply disable the phantom-adoption guard it is protecting.entry = tmp_path / "models--Org--Model-GGUF"(entry / "blobs").mkdir(parents = True)(entry / "snapshots" / "rev0").mkdir(parents = True)@@ -6058,7 +6028,7 @@ def test_delete_variant_keeps_blob_shared_with_other_snapshot(monkeypatch, tmp_passert not (repo_dir / "snapshots" / "rev1" / "model-Q4_K_M.gguf").exists()assert (repo_dir / "blobs" / "sharedblob").exists()extra = repo_dir / "snapshots" / "rev1" / "extra-copy.gguf"- assert extra.is_symlink() and extra.exists() # not dangling+ assert extra.is_symlink() and extra.exists()def test_delete_variant_unlinks_unshared_blob(monkeypatch, tmp_path):@@ -6785,8 +6755,8 @@ def test_gguf_progress_unknown_hashes_calls_a_sibling_only_dir_absent(monkeypatcsnap = entry / "snapshots" / "rev0"snap.mkdir(parents = True)(entry / "blobs").mkdir(parents = True)- (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900) # the sibling that keeps the dir alive- (snap / "mmproj-F16.gguf").write_bytes(b"y" * 30) # shared by every quant in the repo+ (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900)+ (snap / "mmproj-F16.gguf").write_bytes(b"y" * 30)monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")_unresolvable_variant_metadata(monkeypatch, entry)@@ -6816,8 +6786,7 @@ def test_gguf_progress_target_presence_is_aggregated_across_caches(monkeypatch,holder = tmp_path / "b" / "models--Org--Model-GGUF"(holder / "snapshots" / "rev0").mkdir(parents = True)(holder / "blobs").mkdir(parents = True)- # The variant's own file, named but empty: zero bytes keeps the tie with the sibling-only- # reading, and the name is what proves the target is in this cache.+ # Zero bytes keeps the tie with the sibling-only reading, and the name is what proves the target is in this cache.(holder / "snapshots" / "rev0" / "model-Q4_K_M.gguf").write_bytes(b"")monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")assert download_manifest.write_manifest(@@ -6992,7 +6961,7 @@ def test_one_unknown_cache_keeps_absence_unknown(monkeypatch, tmp_path):(sibling / "blobs").mkdir(parents = True)(sibling / "snapshots" / "rev0" / "model-Q2_K.gguf").write_bytes(b"z" * 900)unknown = tmp_path / "b" / "models--Org--Model-GGUF"- (unknown / "blobs").mkdir(parents = True) # no snapshot dir at all: nothing to identify+ (unknown / "blobs").mkdir(parents = True)monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")_unresolvable_variant_metadata(monkeypatch, sibling, state = "idle")monkeypatch.setattr(@@ -7050,7 +7019,7 @@ def test_a_subtree_that_cannot_be_scanned_keeps_presence_unknown(monkeypatch, tmsnap = entry / "snapshots" / "rev0"snap.mkdir(parents = True)(entry / "blobs").mkdir(parents = True)- (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900) # a sibling keeps the dir alive+ (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900)denied = snap / "split"denied.mkdir()(denied / "model-Q4_K_M.gguf").write_bytes(b"x" * 100)@@ -7281,7 +7250,7 @@ def test_a_deleted_snapshot_link_is_absent_even_with_its_blob_left_behind(monkeyblobs.mkdir(parents = True)# The finalized blob survives; the snapshot entry that named it does not.(blobs / "mainhash").write_bytes(b"x" * 100)- (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900) # a sibling keeps the repo dir alive+ (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900)monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")async def _run_inline(fn, *args, **kwargs):@@ -7326,7 +7295,7 @@ def test_a_stale_revisions_filenames_do_not_settle_the_resolved_one(monkeypatch,stale.mkdir(parents = True)blobs.mkdir(parents = True)(blobs / "oldhash").write_bytes(b"y" * 100)- (blobs / "newhash").write_bytes(b"x" * 100) # finalized, never linked+ (blobs / "newhash").write_bytes(b"x" * 100)os.symlink(blobs / "oldhash", stale / "model-Q4_K_M.gguf")monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")assert download_manifest.write_manifest(@@ -7451,7 +7420,7 @@ def test_local_inventory_classifies_a_superseded_result_off_the_event_loop(monkeresponse.model_copy = lambda update: SimpleNamespace(models = update["models"])async def always_superseded(*_args):- epoch[0] += 1 # every walk is invalidated before it returns+ epoch[0] += 1return responseasync def no_folders():@@ -7668,8 +7637,8 @@ def test_every_row_key_the_scanner_emits_survives_the_response_schema():}return set()- # Each scanner against ITS OWN schema. A union would let a key emitted on a model row pass- # because the GGUF schema happens to declare it, which is not what response_model does.+ # Each scanner against ITS OWN schema: a union would let a key emitted on a model row pass because+ # the GGUF schema happens to declare it, which is not what response_model does.emitted = literal_keys("_cache_inventory_fields") | literal_keys("_scan_cached_models")watched = ("diffusers", "companion", "single_file", "partial", "load_id", "task")for flag in watched:@@ -7679,8 +7648,8 @@ def test_every_row_key_the_scanner_emits_survives_the_response_schema():f"FastAPI's response_model strips it before the frontend sees the row.")- # And prove it end to end rather than by field name alone: a declared-but-mistyped field is- # dropped or 500s at serialization time, which a model_fields check cannot see.+ # Prove it end to end rather than by field name alone: a declared-but-mistyped field is dropped or+ # 500s at serialization time, which a model_fields check cannot see.row = {"repo_id": "Org/Pipeline","size_on_disk": 4096,diff --git a/studio/backend/hub/tests/test_process_unique_incomplete_repro.py b/studio/backend/hub/tests/test_process_unique_incomplete_repro.pyindex 6a7aa36f560..f2138a4d258 100644--- a/studio/backend/hub/tests/test_process_unique_incomplete_repro.py+++ b/studio/backend/hub/tests/test_process_unique_incomplete_repro.py@@ -48,8 +48,8 @@ def test_registry_groups_duplicate_process_unique_writers_by_blob(monkeypatch, t)assert download_registry.incomplete_blob_hashes("model", "Org/Model") == {_BLOB_HASH}- # Nonce partials are refetched rather than resumed, so none of those bytes are bytes the- # next attempt gets to skip. Their grouping is still asserted, one blob not two.+ # Nonce partials are refetched rather than resumed, so none of those bytes are bytes the next+ # attempt skips; their grouping is still asserted, one blob not two.assert (download_registry.existing_blob_bytes("model",diff --git a/studio/backend/hub/tests/test_resumable_partials.py b/studio/backend/hub/tests/test_resumable_partials.pyindex 3fb0a3ee59c..e6ae51b5f92 100644--- a/studio/backend/hub/tests/test_resumable_partials.py+++ b/studio/backend/hub/tests/test_resumable_partials.py@@ -71,8 +71,6 @@ def _patched_writer(module):# ----------------------------------------------------------------------------------------------# When it engages-# ---------------------------------------------------------------------------------------------@pytest.mark.parametrize(@@ -207,6 +205,7 @@ def test_each_cache_root_is_judged_on_its_own_filesystem(tmp_path, monkeypatch):local_root.mkdir()network_root.mkdir()# Past the version gate, so the filesystem is what is left to decide.+ # And out through the public entry point, past the version gate.monkeypatch.setattr("huggingface_hub.__version__", "1.28.0", raising = False)monkeypatch.setattr(rp, "_hub_is_patchable", lambda: True)monkeypatch.setattr(@@ -354,7 +353,6 @@ def test_an_unrunnable_probe_travels_up_rather_than_answering(tmp_path, monkeypawith pytest.raises(rp._ProbeUnavailable):rp._exclusion_is_provable()- # And out through the public entry point, past the version gate.monkeypatch.setattr("huggingface_hub.__version__", "1.28.0", raising = False)monkeypatch.setattr(rp, "_hub_is_patchable", lambda: True)with pytest.raises(rp._ProbeUnavailable):@@ -390,15 +388,13 @@ def test_changing_the_cache_home_invalidates_the_verdict(monkeypatch):assert hf_cache_state.hf_partials_are_resumable() is Truemonkeypatch.setattr(rp, "can_restore_partials", lambda _c = None: False)- # No cache at this layer any more: the verdict follows the filesystem, and a result kept- # against the path alone outlives a remount at the same name.+ # No cache at this layer any more: the verdict follows the filesystem, and a result kept against the+ # path alone outlives a remount at the same name.assert hf_cache_state.hf_partials_are_resumable() is Falsehf_cache_state.invalidate_partial_resumability()# ----------------------------------------------------------------------------------------------# What it does once it has-# ---------------------------------------------------------------------------------------------def test_it_appends_to_the_stable_name_and_says_how_far_it_got(monkeypatch, tmp_path):@@ -466,10 +462,8 @@ def test_a_partial_left_by_another_user_is_not_built_on(monkeypatch, tmp_path):partial = tmp_path / "abc.incomplete"partial.write_bytes(b"poison" * 100)- # Only the planted file reads as somebody else's: owning one for real needs root, and the- # fresh partial that replaces it has to still be ours or the test proves nothing about the- # retry. Keyed on which open it is rather than on the inode, since a filesystem is free to- # hand the replacement the inode the unlinked file just released.+ # Only the planted file reads as somebody else's, and the fresh partial replacing it has to still+ # be ours; keyed on which open it is, since a filesystem may reuse the released inode.real_fstat = os.fstatopens: list[int] = []@@ -819,8 +813,6 @@ def test_patching_twice_keeps_one_layer(monkeypatch):# ----------------------------------------------------------------------------------------------# The capability the UI reads-# ---------------------------------------------------------------------------------------------def test_the_ui_is_told_partials_are_resumable_again(monkeypatch):diff --git a/studio/backend/hub/tests/test_unresumable_partial_purge.py b/studio/backend/hub/tests/test_unresumable_partial_purge.pyindex b5c1e228347..6346fab0e00 100644--- a/studio/backend/hub/tests/test_unresumable_partial_purge.py+++ b/studio/backend/hub/tests/test_unresumable_partial_purge.py@@ -120,7 +120,7 @@ def test_a_nonce_partial_is_unresumable_even_under_a_legacy_writer(monkeypatch):def test_unresumable_partial_is_purged_despite_a_matching_marker(monkeypatch, blobs):"""The marker vouches for provenance, which is worth nothing with no resumer left."""monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)- _prepare() # writes the http marker+ _prepare()partial = blobs / _NONCE_PARTIALpartial.write_bytes(b"x" * 25)_abandon(partial)@@ -137,6 +137,7 @@ def test_resumable_partial_survives_a_matching_marker(monkeypatch, blobs):partial.write_bytes(b"x" * 25)_abandon(partial)+ # Too fresh at download start, so prepare leaves it alone.assert _prepare() == 0assert partial.exists()@@ -146,7 +147,7 @@ def test_a_partial_still_being_written_is_left_alone(monkeypatch, blobs):monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)_prepare()partial = blobs / _NONCE_PARTIAL- partial.write_bytes(b"x" * 25) # mtime is now, as a live writer's would be+ partial.write_bytes(b"x" * 25)assert _prepare() == 0assert partial.exists()@@ -217,7 +218,6 @@ def test_a_skipped_partial_is_swept_once_it_ages_out(monkeypatch, blobs):partial = blobs / _NONCE_PARTIALpartial.write_bytes(b"x" * 25)- # Too fresh at download start, so prepare leaves it alone.assert _prepare() == 0assert partial.exists()@@ -313,7 +313,7 @@ def test_a_finalized_blob_still_counts_against_the_disk_check(monkeypatch, blobsdef test_startup_sweep_does_not_depend_on_a_breadcrumb(monkeypatch, tmp_path, blobs):"""finalize_worker_exit drops the breadcrumb, so the boot sweep cannot be driven off one."""workers = tmp_path / "workers"- workers.mkdir() # deliberately empty, as it is once drop_process has run+ workers.mkdir()partial = blobs / _NONCE_PARTIALpartial.write_bytes(b"x" * 25)_abandon(partial)@@ -354,7 +354,7 @@ def test_a_reaped_job_does_not_wait_out_the_grace_on_its_own_blobs(monkeypatch,"""Cancelling writes the partial seconds before the sweep, so waiting strands it."""monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)partial = blobs / _NONCE_PARTIAL- partial.write_bytes(b"x" * 25) # freshly written, as a just-cancelled download's would be+ partial.write_bytes(b"x" * 25)monkeypatch.setattr(download_registry,"iter_destructive_repo_cache_dirs",@@ -434,8 +434,8 @@ def test_the_sweep_accepts_the_string_root_the_metadata_holds(monkeypatch, tmp_ppartial.write_bytes(b"x" * 25)_abandon(partial)- # A Path-only signature raised AttributeError here, and the caller's broad except- # swallowed it, so the terminal sweep silently did nothing for every real download.+ # A Path-only signature raised AttributeError here and the caller's broad except swallowed it, so+ # the terminal sweep silently did nothing for every real download.swept = download_registry.sweep_abandoned_partials("model","Org/Model",@@ -450,7 +450,7 @@ def test_a_job_owning_its_whole_repo_needs_no_hash_list(monkeypatch, blobs):"""A download with no variant resolves no blob hashes, and claim() gives it the repo."""monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)partial = blobs / _NONCE_PARTIAL- partial.write_bytes(b"x" * 25) # fresh, as a just-cancelled snapshot download's would be+ partial.write_bytes(b"x" * 25)monkeypatch.setattr(download_registry,"iter_destructive_repo_cache_dirs",@@ -504,7 +504,7 @@ def test_the_boot_sweep_runs_after_the_orphan_is_killed(monkeypatch, tmp_path, bdef _kill(_pid):order.append("kill")- locked["held"] = False # the lock dies with the process+ locked["held"] = Falsereturn Truemonkeypatch.setattr(download_registry, "_kill_orphan", _kill)@@ -525,7 +525,7 @@ def test_a_companion_the_dead_worker_was_writing_is_owned_too(monkeypatch, blobs"""A shared mmproj lives in progress_blob_hashes, never in the main blob_hashes set."""monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)companion = blobs / f"{_PEER}.feedface{hf_cache_state.INCOMPLETE_SUFFIX}"- companion.write_bytes(b"x" * 25) # fresh, as a just-cancelled worker's companion would be+ companion.write_bytes(b"x" * 25)monkeypatch.setattr(download_registry,"iter_destructive_repo_cache_dirs",@@ -567,7 +567,7 @@ def test_the_reaper_waits_for_the_worker_to_actually_die(monkeypatch):download_registry._kill_orphan(4242)- assert alive["n"] == 0 # returned only once the process was gone, not straight after kill+ assert alive["n"] == 0def test_a_worker_that_will_not_die_keeps_its_breadcrumb_and_its_partial(@@ -597,14 +597,14 @@ def test_a_worker_that_will_not_die_keeps_its_breadcrumb_and_its_partial(monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)monkeypatch.setattr(download_registry, "_process_alive", lambda _pid: True)monkeypatch.setattr(download_registry, "_is_our_worker", lambda *_a: True)- monkeypatch.setattr(download_registry, "_kill_orphan", lambda _pid: False) # would not die+ monkeypatch.setattr(download_registry, "_kill_orphan", lambda _pid: False)monkeypatch.setattr(download_registry, "hf_cache_roots", lambda *_a, **_k: [tmp_path / "none"])download_registry.reap_orphan_workers()_join_background_sweep()assert partial.exists()- assert crumb.exists() # still tracked, so the next boot tries again+ assert crumb.exists()def test_a_locked_peer_partial_still_counts_against_the_disk_check(monkeypatch, blobs):@@ -673,7 +673,7 @@ def test_ownership_is_recovered_from_the_manifest_when_hashes_never_resolved(monmetadata, frozenset(), frozenset(), "model", "Org/Model")- assert owns_all is False # a variant job never owns its siblings' blobs+ assert owns_all is Falseassert owned == frozenset({_MAIN})@@ -702,8 +702,8 @@ def test_a_filesystem_without_flock_does_not_escape_the_probe(monkeypatch, tmp_p# No lock file: nobody has locked this blob, whatever the filesystem supports.assert hf_cache_state.blob_download_lock_held(entry, _MAIN) is False- # With one, the answer is "held" rather than an exception -- which is also what a- # SoftFileLock would say, since its file IS the lock and that file is present.+ # With one, the answer is "held" rather than an exception, which is also what a SoftFileLock would+ # say, since its file IS the lock.lock_path.touch()assert hf_cache_state.blob_download_lock_held(entry, _MAIN) is True@@ -759,7 +759,7 @@ def test_unreadable_breadcrumbs_do_not_cancel_the_cache_sweep(monkeypatch, tmp_pdef test_an_owned_partial_that_is_still_growing_is_spared(monkeypatch, blobs):"""Ownership proves OUR writer died, never that no other process shares the cache."""monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)- monkeypatch.setattr(download_registry, "blob_download_lock_held", lambda *_a: False) # lies+ monkeypatch.setattr(download_registry, "blob_download_lock_held", lambda *_a: False)monkeypatch.setattr(download_registry, "_STILLNESS_PROBE_SECONDS", 0.05)monkeypatch.setattr(download_registry,@@ -774,7 +774,7 @@ def test_an_owned_partial_that_is_still_growing_is_spared(monkeypatch, blobs):def _write_while_we_watch(_seconds):real_sleep(_seconds)with partial.open("ab") as handle:- handle.write(b"y" * 10) # an external writer, mid-transfer+ handle.write(b"y" * 10)monkeypatch.setattr(download_registry.time, "sleep", _write_while_we_watch)@@ -798,7 +798,7 @@ def test_an_owned_partial_that_never_moves_is_swept_without_the_full_grace(monkelambda *_a, **_k: [blobs.parent],)partial = blobs / _NONCE_PARTIAL- partial.write_bytes(b"x" * 25) # written seconds ago, far inside the abandonment grace+ partial.write_bytes(b"x" * 25)swept = download_registry.sweep_abandoned_partials("model",@@ -828,11 +828,11 @@ def test_a_breadcrumb_whose_worker_already_exited_is_claimed(monkeypatch, tmp_paencoding = "utf-8",)partial = blobs / _NONCE_PARTIAL- partial.write_bytes(b"x" * 25) # fresh, so only an ownership claim reaches it+ partial.write_bytes(b"x" * 25)monkeypatch.setattr(download_registry.state_dir, "workers_dir", lambda: workers)monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)- monkeypatch.setattr(download_registry, "_process_alive", lambda _pid: False) # already dead+ monkeypatch.setattr(download_registry, "_process_alive", lambda _pid: False)monkeypatch.setattr(download_registry, "_STILLNESS_PROBE_SECONDS", 0.05)monkeypatch.setattr(download_registry, "_settle_orphaned_download", lambda *_a, **_k: None)monkeypatch.setattr(download_registry, "hf_cache_roots", lambda *_a, **_k: [tmp_path / "none"])diff --git a/studio/backend/hub/utils/companion_assets.py b/studio/backend/hub/utils/companion_assets.pyindex a674c7531d0..c96968bb6f3 100644--- a/studio/backend/hub/utils/companion_assets.py+++ b/studio/backend/hub/utils/companion_assets.py@@ -51,8 +51,8 @@ _LINKS_VERSION = 1_MAX_LINKS = 512# ... and per checkpoint, for the same reason: one key must not grow the file on its own._MAX_BASES_PER_CHECKPOINT = 16-# Serialises the read-modify-write below. Losing a link to a lost update is the one direction-# that matters: an unrecorded base can look orphaned and be offered for removal.+# Serialises the read-modify-write below: losing a link to a lost update is the one direction that+# matters, since an unrecorded base can look orphaned and be offered for removal._WRITE_LOCK = threading.Lock()@@ -105,10 +105,7 @@ def _write_companion_links(links: dict[str, list[str]]) -> bool:payload = {"version": _LINKS_VERSION, "links": links}tmp = path.with_name(f".{path.name}.tmp-{uuid.uuid4().hex[:8]}")try:- # NOT sort_keys: json.loads keeps document order, so the file IS the recency record the- # trim above reads. Sorting it made the next read alphabetical, and the trim then evicted- # the lexicographically smallest link rather than the oldest -- so a link recorded minutes- # ago could go, and its base become deletable while the checkpoint was still installed.+ # NOT sort_keys: json.loads keeps document order, so the file IS the recency record the trim reads.tmp.write_text(json.dumps(payload, indent = 2), encoding = "utf-8")os.replace(tmp, path)return True@@ -147,18 +144,11 @@ def record_companion_link(checkpoint_repo_id: str, base_repo_id: str) -> bool:key = _normalise(checkpoint)existing = links.get(key, [])known = any(_normalise(b) == _normalise(base) for b in existing)- # Re-inserted at the end, not updated in place: recording is the only recency signal the- # trim has, so a checkpoint that just resolved must not keep an old position and be- # evicted ahead of one nothing has touched since. That applies to a REPEAT resolution too- # -- a checkpoint reloaded every day but first recorded long ago is the most-used link- # there is, and returning early on it let the cap throw it away.+ # Re-inserted at the end, not updated in place: recording is the only recency signal the trim has,+ # and returning early on a REPEAT resolution let the cap throw away the most-used link.links.pop(key, None)- # Recency inside the list too, and capped: an explicit base_repo (or a card tag that- # changes) makes one checkpoint resolve a new base each load, and appending forever grew- # the file that _MAX_LINKS is supposed to bound -- it counts checkpoints, not their bases,- # and every dependency check parses and mirror-expands the whole list. A checkpoint has- # one or two bases in practice, so the cap is only ever reached by the runaway writer it- # is here for, and dropping the least recently resolved degrades to derivation.+ # Recency inside the list too, and capped: an explicit base_repo or a changing card tag makes one+ # checkpoint resolve a new base each load, and appending forever grew the file _MAX_LINKS bounds.fresh = [b for b in existing if _normalise(b) != _normalise(base)]links[key] = [*fresh, base][-_MAX_BASES_PER_CHECKPOINT:]wrote = _write_companion_links(links)@@ -337,16 +327,14 @@ def _family_bases(repo_id: str, gguf_filename: Optional[str] = None) -> set[str]except Exception: # noqa: BLE001return set()if fam is None:- # The loader resolves a base with this same call. No family means no load, so there is- # no dependent here to protect -- not a gap, the two fail together.+ # The loader resolves a base with this same call, so no family means no load and there is no+ # dependent here to protect: the two fail together.return set()bases = {resolve_base_repo(fam, None)}bases |= _curated_variant_bases(fam, repo_id, gguf_filename)- # The NATIVE engine's companions as well. It never reads the diffusers base: it fetches a- # single-file VAE and text encoder from their own repos, and those repos are offerable for- # deletion, so a pre-existing native GGUF with no recorded link would have had its encoder- # listed as an unused asset and removed underneath it. The recorded links cover a load that- # has happened since; this covers the install that predates them.+ # The NATIVE engine never reads the diffusers base: it fetches a single-file VAE and text encoder+ # from their own repos, so a pre-existing native GGUF with no recorded link would have had its+ # encoder listed as unused and removed underneath it.bases |= {repo for repo, _file in _sd_cpp_component_specs(fam, repo_id, gguf_filename)}return _with_mirrors(bases)@@ -413,10 +401,8 @@ def _sd_cpp_component_specs(vae = getattr(fam, "sd_cpp_vae", None)if vae and vae[0]:specs.add((vae[0], vae[1]))- # EVERY set the family could pick, not the one the string fallback guesses. There is no- # header to read here, and a renamed FLUX.2-klein 9B file carries no size token either,- # so guessing answers 4B and leaves the 9B encoder the load actually fetched unprotected.- # Naming both costs a delete that is refused; guessing costs an installed model.+ # EVERY set the family could pick: there is no header to read and a renamed FLUX.2-klein 9B file+ # carries no size token, so guessing answers 4B and leaves the 9B encoder unprotected.for encoder in sd_cpp_text_encoder_candidates(fam) or ():if encoder and encoder[0]:specs.add((encoder[0], encoder[1]))@@ -435,8 +421,7 @@ def required_companion_asset_files(cache_scans) -> dict[str, set[str]]:pick_names = _cached_checkpoint_pick_names(cache_scans)files: dict[str, set[str]] = {}for repo_id in _cached_model_repo_ids(cache_scans):- # Every cached GGUF, because one repo can hold checkpoints of two families and each opens- # its own components.+ # One repo can hold checkpoints of two families, and each opens its own components.for name in dict.fromkeys([None, *pick_names.get(_normalise(repo_id), [])]):fam = _detect_family(repo_id, name)if fam is None:@@ -497,17 +482,12 @@ def known_companion_base_ids() -> set[str]:Deliberately NOT "anything a link ever named": orphan cleanup offers only repos this setrecognises, so a mis-recorded link can never turn an unrelated repo into a delete candidate."""- # Family bases and their gated/mirror variants. The pairs are companion bases by construction- # -- they exist only because a GGUF pick of that family needs an ungated copy of them -- and- # they cover the variants one family entry serves through a card tag (klein-9B under the- # klein-4B family, and so on).+ # The pairs are companion bases by construction, existing only because a GGUF pick of that family+ # needs an ungated copy, and they cover the variants one family entry serves through a card tag.bases = _curated_base_ids()- # The native engine's component-only repos (a single-file VAE, a text encoder) are curated- # table entries too, and for an sd.cpp pick they ARE the companions -- the largest half of the- # footprint. Leaving them out made them link-only strangers: unlisted by Free up space and- # dropped from the delete preview, so the assets this cleanup exists for stayed invisible.- # Safe against the one hazard that table warns about, a chat model borrowed as a text encoder:- # the orphan listing skips any repo holding a GGUF, so it is never offered as a leftover.+ # The native engine's component-only repos ARE the companions for an sd.cpp pick, and the largest+ # half of the footprint; leaving them out made them link-only strangers. Safe against a chat model+ # borrowed as a text encoder, since the orphan listing skips any repo holding a GGUF.try:from core.inference.diffusion_families import sd_cpp_companion_only_repo_idsbases |= set(sd_cpp_companion_only_repo_ids())@@ -517,12 +497,9 @@ def known_companion_base_ids() -> set[str]:def _mirror_pair_ids() -> set[str]:- # The WHOLE table, gated and ungated. Gating decides whether a fetch may override a user's- # cache; it says nothing about whether a base can strand companions, and most of the table- # is ungated (Klein 4B and base-4B, HiDream Dev / Fast, SDXL Turbo, ...). Reading only the- # gated half would drop those from the curated ids, so before a companion link is recorded- # the cleanup would not recognise them and could offer an installed checkpoint's companions- # for deletion.+ # The WHOLE table, gated and ungated: gating decides whether a fetch may override a user's cache,+ # not whether a base can strand companions, so reading only the gated half would offer an+ # installed checkpoint's companions for deletion.try:from core.inference.diffusion_families import _MIRROR_PAIRSexcept Exception: # noqa: BLE001@@ -542,10 +519,8 @@ def is_companion_base(repo_id: str) -> bool:key = _normalise(repo_id)if key in known_companion_base_ids():return True- # Through the same identity expansion required_companion_bases uses, so the two agree on- # WHICH copy is protected. A link recorded against the unsloth mirror is satisfied on an- # upgraded install by the community repack the fetch fell back to, and comparing the literal- # id alone left that copy deletable while the checkpoint holding it was still installed.+ # Through the same identity expansion required_companion_bases uses, so the two agree WHICH copy+ # is protected: comparing the literal id left an upgraded install's repack copy deletable.return any(key in {_normalise(rid) for rid in _with_mirrors([base])}for bases in read_companion_links().values()@@ -595,10 +570,8 @@ def required_companion_bases(recorded = links.get(key)if recorded:bases |= _with_mirrors(recorded)- # Canonical, not literal: a cached MIRROR of a base resolves its own family back to the- # UPSTREAM id, which would make each identity a dependent of the other and leave a cache- # holding both unable to delete either. They are copies of one repo, not a pair that- # needs each other.+ # Canonical, not literal: a cached MIRROR resolves its own family back to the UPSTREAM id, which+ # would make each identity a dependent of the other and leave a cache holding both stuck.self_keys = {key, _normalise(_canonical(repo_id))}for base in bases:base_key = _normalise(base)diff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.pyindex f6d92ea581e..310367f4ed8 100644--- a/studio/backend/hub/utils/dataset_format.py+++ b/studio/backend/hub/utils/dataset_format.py@@ -403,13 +403,11 @@ def detect_multimodal_dataset(dataset):detected_text_col = Noneif audio_columns:- # Two passes, not one list: a set carrying both an instruction-like "prompt" and a- # real "transcript" would otherwise be mapped by schema order, and training an ASR- # set against its instructions instead of its ground truth fails silently.+ # Two passes, not one list: a set carrying both an instruction-like "prompt" and a real+ # "transcript" would be mapped by schema order, training an ASR set against its instructions.transcript_names = ("text", "sentence", "transcript", "transcription", "label")- # TTS corpora name the line to speak rather than a transcript of it: every- # svjack/SparkTTS_* set uses "prompt", LJSpeech derivatives use "normalized_text".- # Without these the set needs a manual mapping it cannot satisfy.+ # TTS corpora name the line to speak rather than a transcript of it: SparkTTS sets use "prompt",+ # LJSpeech derivatives "normalized_text".fallback_names = ("prompt", "normalized_text")for candidates in (transcript_names, fallback_names):for col_name in column_names:diff --git a/studio/backend/hub/utils/download_manifest.py b/studio/backend/hub/utils/download_manifest.pyindex e70c076c5fb..5ac8fe3cdd0 100644--- a/studio/backend/hub/utils/download_manifest.py+++ b/studio/backend/hub/utils/download_manifest.py@@ -119,14 +119,9 @@ class VariantState:def has_marker(self, variant: str) -> bool:if variant.lower() in self._markers:return True- # An unreadable marker keeps its entry fail-closed but loses the payload,- # so its only remaining identity is the filename. For a variant that has to- # be stored hashed (slashes, spaces, `--`, an existing hash prefix) that- # identity is the digest, not the variant, and a plain lookup misses it --- # leaving a cancelled variant advertised as complete even though- # has_cancel_marker still finds the file by rebuilding the same digest.- # Match the spellings the variant could have been written under, as that- # per-variant lookup does.+ # An unreadable marker loses its payload, so its only identity is the filename, which for a+ # variant stored hashed is the digest rather than the variant: a plain lookup missed it and left a+ # cancelled variant advertised as complete.try:fragments = variant_key_fragments(variant)except (UnicodeError, ValueError):@@ -176,9 +171,8 @@ def _hub_cache_spellings(hub_cache = get_hf_cache_paths().hub_cacheexcept Exception:return None, None- # Shared with state_dir.cache_scope_name so the ownership string recorded in- # a payload and the cache-<digest> directory it is filed under can never be- # derived from two different normalizations of the same directory.+ # Shared with state_dir.cache_scope_name so the ownership string recorded in a payload and the+ # cache-<digest> directory it is filed under can never come from two different normalizations.return normalize_hub_cache(hub_cache), hub_cache@@ -225,9 +219,8 @@ def _scope_spellings(hub_cache: str | Path) -> tuple[str, ...]:def _read_state_payload(path: Path) -> Optional[dict]:try:data = json.loads(path.read_text(encoding = "utf-8"))- # The decoder raises RecursionError for adversarially deep JSON. One corrupt- # state file must not abort the shared one-pass index: returning None lets- # manifests fail open and cancellation markers retain their fail-closed path.+ # The decoder raises RecursionError for adversarially deep JSON, and one corrupt state file must not+ # abort the shared one-pass index.except (OSError, ValueError, RecursionError) as exc:logger.debug("Could not read Hub state %s: %s", path, exc)return None@@ -339,8 +332,8 @@ def _state_read_path(def applies(path: Path) -> bool:payload = _read_state_payload(path)plausible = _state_payload_identity_matches_entry(path, payload, repo_type)- # Cancellation remains fail-closed when a marker cannot identify its- # owner. Parseable state from another repository is never borrowed.+ # Cancellation stays fail-closed when a marker cannot identify its owner; parseable state from+ # another repository is never borrowed.if fail_closed and not plausible:return Trueif plausible and variant is not None:@@ -402,14 +395,10 @@ def _state_paths(collapses to one entry whenever the two spellings agree, which is every paththat resolves to itself."""- # _scope_spellings, not cache_scope_names, for the reason the repo-wide purge already- # uses it: the single-variant delete route hands us a root that resolve_delete_target_root- # has ALREADY resolved, so the raw spelling reproduces the canonical digest and the- # pre-resolve scope is never probed. The read paths do probe it (they are fed the raw- # configured setting), so an exact-variant delete left a legacy-scoped manifest or cancel- # marker behind for the next read to resurrect the variant as partial or cancelled.- # _scope_spellings adds the configured cache's own spelling only when it names the same- # directory, so deleting out of a non-active cache still cannot touch the active one.+ # _scope_spellings, not cache_scope_names: the single-variant delete route hands over a root+ # resolve_delete_target_root has ALREADY resolved, so the raw spelling reproduces the canonical+ # digest and the pre-resolve scope is never probed, leaving state for the next read to resurrect.+ # It adds the configured cache's own spelling only when it names the same directory.scopes = ((None,)if hub_cache is None@@ -827,18 +816,11 @@ def verify_against_disk(manifest: Manifest, snapshot_dir: Path) -> VerifyResult:missing: list[str] = []mismatched: list[str] = []for expected in manifest.expected_files:- # Counted missing rather than skipped, so an unverifiable entry can- # never read as verified. A Windows-separator path lands here, and- # cannot be folded onto its posix spelling in expected_path_is_safe:- # that guard also fronts resolved_dataset_snapshot_file, which splits on- # PurePosixPath, where "a\b" is one component and accepting it would- # hand a traversal straight through on the platform that disagrees. No- # writer in this package produces one -- HF rfilenames are posix and- # expected_files_from_snapshot_dir calls as_posix -- and one that- # reached a payload would already have failed _manifest_from_payload's- # identical check, so this stays defence in depth for a Manifest built- # some other way. Should such a writer ever appear, normalize it in- # _payload_file_path, not here.+ # Counted missing rather than skipped, so an unverifiable entry can never read as verified. A+ # Windows-separator path cannot be folded onto its posix spelling in expected_path_is_safe: that+ # guard also fronts resolved_dataset_snapshot_file, which splits on PurePosixPath, where "a\b" is+ # one component and accepting it would hand a traversal through. No writer here produces one (HF+ # rfilenames are posix and expected_files_from_snapshot_dir calls as_posix).if not expected_path_is_safe(expected.path):missing.append(expected.path)continue@@ -871,8 +853,8 @@ def expected_files_from_snapshot_dir(snapshot_dir: Path) -> list[ExpectedFile]:"""out: list[ExpectedFile] = []try:- # Baking a companion into the completion contract makes the download read as partial- # forever once macOS cleans it up.+ # Baking a companion into the completion contract makes the download read as partial forever once+ # macOS cleans it up.entries = drop_appledouble_metadata(sorted(snapshot_dir.rglob("*")))except OSError:return out@@ -1130,10 +1112,9 @@ def purge_state(else:requested, raw = _hub_cache_spellings(hub_cache)candidates: list[Path] = []- # Legacy unscoped state is shared: an unowned file belongs to the active- # cache (per _legacy_state_applies), so only purge it when it belongs to- # the cache being deleted -- else deleting an inactive cache would erase- # the active cache's resume/cancel state.+ # Legacy unscoped state is shared: an unowned file belongs to the active cache, so purge it only+ # when it belongs to the cache being deleted, else deleting an inactive cache erases the active+ # cache's state.for path_factory, fail_closed in ((manifest_path, False),(marker_path, True),@@ -1213,13 +1194,11 @@ def purge_all_state_for_repo(]else:# This cache's scoped dirs plus the legacy unscoped base; glob (not rglob) so other caches are untouched.- # Both scope spellings, else a repo delete leaves the pre-resolve copy- # behind for a later read to resurrect as state for a cache that is gone.+ # Both scope spellings, else a repo delete leaves the pre-resolve copy behind for a later read to+ # resurrect as state for a cache that is gone.search = []- # _scope_spellings, not cache_scope_names: every production caller of this- # function resolves its target root first, so the raw-spelling probe would- # be inert here while the read path still has it -- a purged variant that- # the next read brings back.+ # _scope_spellings, not cache_scope_names: every production caller resolves its target root first,+ # so the raw-spelling probe would be inert here while the read path still has it.scopes = _scope_spellings(hub_cache)for path_factory, base, cancel_markers in ((manifest_path, manifests_dir(create = False), False),@@ -1457,14 +1436,10 @@ def build_variant_state_index(repo_keys.setdefault(prefix[: -len("--variant--")], set()).add((repo_type, normalized_repo))- # Both spellings of the scope dir, so state filed under the pre-resolve- # digest is indexed rather than silently read as "no manifest". Derived- # from the caller's own spelling, not the canonical one, which resolves- # to itself and would yield the canonical digest twice. Normally one key.- # _scope_spellings for the same reason as the delete: the inventory callers- # reach here with a directory derived from huggingface_hub.scan_cache_dir,- # which has already resolved it, so the raw probe alone finds nothing and- # the cached-model views would disagree with the progress endpoint.+ # Both spellings of the scope dir, derived from the caller's own rather than the canonical one, so+ # state filed under the pre-resolve digest is indexed instead of read as "no manifest": the+ # inventory callers arrive with a directory huggingface_hub.scan_cache_dir already resolved, so+ # the raw probe alone finds nothing.for scope in _scope_spellings(hub_cache):caches_by_scope.setdefault(scope, set()).add(canonical_cache)@@ -1505,15 +1480,10 @@ def build_variant_state_index(try:canonical = marker_path(repo_type, repo_id, variant, create = False)except (UnicodeError, ValueError, OSError):- # A state filename can carry a byte the filesystem encoding cannot- # decode, which iterdir() surfaces as a lone surrogate; hashing that- # for the canonical spelling raises UnicodeEncodeError. Per-repo- # reads used to contain the damage to the one repo that owned the- # file. This index is built once for the whole scan and outside the- # per-repo try, so letting it escape turns one corrupt filename into- # a 500 that hides every cached model. `canonical` only decides the- # "is this the canonical filename" half of `priority`, so treat the- # entry as non-canonical and keep indexing.+ # A state filename can carry a byte the filesystem encoding cannot decode, which iterdir()+ # surfaces as a lone surrogate, and hashing it raises UnicodeEncodeError. This index is built once+ # for the whole scan and outside the per-repo try, so letting it escape turns one corrupt filename+ # into a 500 that hides every cached model; treat the entry as non-canonical and keep indexing.canonical = Noneadd_entry(cache,@@ -1597,14 +1567,10 @@ def _iter_variant_state_files(returnpath_factory = marker_path if cancel_markers else manifest_pathrequested, raw = _hub_cache_spellings(hub_cache)- # Every scope this cache's state can sit in, so a variant filed under the- # pre-resolve digest is enumerated instead of reading as "no variant state".- #- # _scope_spellings, not cache_scope_names, for the same reason the delete and index paths- # use it: cache_scope_names recovers the legacy digest only from an UNRESOLVED path, and a- # variant request carrying local_path arrives here already resolved by- # _repo_cache_dir_for_request. On a cache reached through a symlink or a junction that left- # the offline listing unable to see its own partial download, or its resume control.+ # Every scope this cache's state can sit in, so a variant filed under the pre-resolve digest is+ # enumerated instead of reading as "no variant state".+ # _scope_spellings, since cache_scope_names recovers the legacy digest only from an UNRESOLVED path,+ # while a variant request carrying local_path arrives here already resolved.scopes = _scope_spellings(raw) if requested is not None and raw is not None else (None,)scoped_dirs: list[Path] = []for scope in scopes:@@ -1679,10 +1645,8 @@ def _iter_variant_state_files(variant_payload = Nonevariant = _variant_from_state_payload(variant_payload, fallback)try:- # Unscoped: only the basename is compared below, and that is- # the same under every scope. Asking for a scoped path would- # re-derive the digest -- and pay its ``resolve`` -- once per- # candidate file, for a directory component never read.+ # Unscoped: only the basename is compared below, and a scoped path would re-derive the digest, and+ # pay its resolve, once per candidate file.canonical = path_factory(repo_type,repo_id,@@ -1691,10 +1655,7 @@ def _iter_variant_state_files(create = False,)except (UnicodeError, ValueError, OSError):- # Same undecodable-filename case guarded in add_path: a lone- # surrogate from iterdir() cannot be hashed for the canonical- # spelling. Before, this loop yielded such an entry as-is;- # letting the hash escape would instead abort the whole+ # Same undecodable-filename case as add_path: letting the hash escape would abort the whole# iteration and lose the repo's valid state alongside it.canonical = Nonepriority = (not legacy, canonical is not None and canonical.name == entry.name)diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.pyindex 296df2d4d91..1c9f3843954 100644--- a/studio/backend/hub/utils/download_registry.py+++ b/studio/backend/hub/utils/download_registry.py@@ -55,9 +55,8 @@ from typing import Callable, Iterator, Literal, NamedTuple, Optional, Sequencefrom loggers import get_logger-# One floor, one name, shared. Written out by hand in each module it was needed-# in at first, and the site that got missed was missed because "who enforces it"-# was a question you answered by reading rather than by grepping.+# One floor, one name, shared: hand-written copies meant the site that got missed was missed because+# "who enforces it" had to be read rather than grepped.from utils.process_lifetime import is_signalable_pidfrom hub.utils import state_dir@@ -99,12 +98,11 @@ class DownloadTransportCapability:class DownloadTransportCapabilities:http: DownloadTransportCapabilityxet: DownloadTransportCapability- # What "auto" would pick right now, and why, so the picker can say- # "Auto (HTTP -- Xet stalled twice on this machine)" instead of just "Auto".+ # What "auto" would pick right now, and why, so the picker can say "Auto (HTTP -- Xet stalled twice+ # on this machine)" instead of just "Auto".auto_resolves_to: str = TRANSPORT_XETauto_reason: Optional[str] = None- # Whether an interrupted HTTP transfer leaves bytes the next attempt can append to. False on- # huggingface_hub >= 1.18, so the UI stops offering a byte-resume no writer can honour.+ # False on huggingface_hub >= 1.18, so the UI stops offering a byte-resume no writer can honour.partials_resumable: bool = True@@ -126,22 +124,16 @@ def get_download_transport_capabilities(try:from utils.hf_xet_fallback import cached_xet_health, xet_health- # Ordinary UI polls are read-only and must not load Zoo. `probe=True` is different:- # the frontend sends it only while resolving Auto for a download, then submits the- # concrete answer as xet/http, so this is the actual first-download decision.- # `ram_gate` loads it too, without probing: an empty cache reads as the optimistic- # Xet, so the row promised Xet while the next download loaded an unhealthy verdict- # and chose HTTP. Still `probe=probe`, so only a download start probes live.+ # Ordinary UI polls are read-only and must not load Zoo; probe=True is the actual first-download+ # decision, and ram_gate loads it too without probing: an empty cache reads as the optimistic Xet,+ # so the row promised Xet while the next download chose HTTP.health_fn = xet_health if (probe or ram_gate) else cached_xet_healthhealth = health_fn(probe = probe)if health is not None:auto_transport = TRANSPORT_XET if health.use_xet else TRANSPORT_HTTPauto_reason = str(health.reason)- # UNSLOTH_FORCE_XET=1: an operator override, not a measurement, so the free-RAM- # gate below stands down exactly as `resolve_auto_use_xet` does. Same helper, so- # the probe and the API "auto" path cannot disagree about what "forced" means.- # Imported separately and guarded: an older or stubbed shim that lacks it must not- # cost us the health verdict just recorded above.+ # UNSLOTH_FORCE_XET=1 is an operator override, not a measurement, so the free-RAM gate below stands+ # down exactly as resolve_auto_use_xet does.try:from utils.hf_xet_fallback import xet_health_is_forcedauto_forced = bool(xet_health_is_forced(health))@@ -156,10 +148,9 @@ def get_download_transport_capabilities(and auto_transport == TRANSPORT_XETand not auto_forced):- # Free RAM belongs in the same verdict: this IS the Auto decision, since the UI submits the- # answer as an explicit xet/http. Read outside the health try, because a health module that- # is missing or raising says nothing about RAM. Never on an ordinary poll, which stays- # read-only and still does not load Zoo -- only for a probe or an explicit ram_gate.+ # Free RAM belongs in the same verdict, since the UI submits the answer as an explicit xet/http.+ # Read outside the health try, because a missing health module says nothing about RAM, and never+ # on an ordinary poll: only for a probe or an explicit ram_gate.try:from utils.hf_xet_fallback import free_ram_pressure_reasonpressure = free_ram_pressure_reason()@@ -305,8 +296,8 @@ def _is_our_worker(pid: int, repo_id: Optional[str]) -> bool:return Falseif "hub.workers.hf_download" not in cmdline:return False- # Exact --repo-id match: a substring match would let a stale breadcrumb for- # Org/Model reap a live worker for Org/Model-v2.+ # Exact --repo-id match: a substring match would let a stale breadcrumb for Org/Model reap a live+ # worker for Org/Model-v2.if isinstance(repo_id, str) and repo_id:return _cmdline_repo_id(cmdline) == repo_idreturn True@@ -321,9 +312,8 @@ def _kill_orphan(pid: int) -> bool:reason to hold up startup -- and answering False there matters: a survivor must keep itsbreadcrumb and must not have its live partial claimed as ours to delete."""- # Its caller floors the pid too. Repeated here because this one sends the- # signal, and a helper that kills should not depend on every future caller- # having checked first.+ # Repeated here because this one sends the signal, and a helper that kills should not depend on+ # every future caller having checked first.if not is_signalable_pid(pid):return Falsetry:@@ -421,8 +411,8 @@ def reap_orphan_workers() -> None:try:entries = list(parent.iterdir())except OSError:- # Unreadable breadcrumbs means no worker can be claimed as reaped, not that the caches- # go unswept: they are a separate tree and may well be readable.+ # Unreadable breadcrumbs means no worker can be claimed as reaped, not that the caches go unswept:+ # they are a separate tree._boot_sweep(reaped)returnfor entry in entries:@@ -435,26 +425,22 @@ def reap_orphan_workers() -> None:continuepid = data.get("pid") if isinstance(data, dict) else Nonerepo_id = data.get("repo_id") if isinstance(data, dict) else None- # pid 1 as well as 0 and negatives. The cmdline check below is what- # actually keeps this honest and init cannot pass it, but the same was- # true of the reaper's start-time check until a record naming pid 1- # slipped through it, so the cheap bound goes in front of the clever one.+ # pid 1 as well as 0 and negatives: the cmdline check below is what keeps this honest, but a+ # record naming pid 1 once slipped through the reaper's start-time check.signalable = is_signalable_pid(pid)try:if not signalable:- # Nothing is signalled on this record's word, but its repo fields- # are still readable, and the settle below is what preserves the- # partial's resume marker. Unlinking straight from here would cost- # the user a restarted download to pay for a bug that is ours.+ # Its repo fields are still readable, and the settle below preserves the partial's resume marker:+ # unlinking from here would cost the user a restarted download to pay for a bug that is ours.passelif not _process_alive(pid):- # Already gone, which is better proof than killing it ourselves. Its partial- # is ours to sweep even though this invocation reaped nothing.+ # Already gone, which is better proof than killing it ourselves; its partial is ours to sweep even+ # though this invocation reaped nothing.reaped.append((data.get("repo_type") or "model", repo_id, data.get("hub_cache")))elif _is_our_worker(pid, repo_id):if not _kill_orphan(pid):- # Still running. Keeping the breadcrumb keeps it tracked for the next boot,- # and claiming no ownership keeps its live partial out of the sweep.+ # Still running: keeping the breadcrumb keeps it tracked for the next boot, and claiming no+ # ownership keeps its live partial out of the sweep.logger.warning("Could not reap download worker pid=%s repo=%s; leaving its ""breadcrumb and partial in place.",@@ -462,9 +448,8 @@ def reap_orphan_workers() -> None:repo_id,)continue- # Its partial is unreadable now and its writer is gone, but only as of this- # line. The sweep has to come after the kill, not before, or it reads the- # still-held blob lock and spares a file nothing will ever finish.+ # The sweep has to come after the kill, not before, or it reads the still-held blob lock and spares+ # a file nothing will ever finish.reaped.append((data.get("repo_type") or "model", repo_id, data.get("hub_cache")))logger.warning("Reaped orphaned download worker pid=%s repo=%s from a "@@ -533,8 +518,8 @@ class _PurgeOutcome(NamedTuple):failed: int-# Only the unresumable sweep waits out ABANDONED_PARTIAL_SECONDS: it reclaims disk, while a-# marker-mismatch purge exists to stop a corrupt append and cannot defer.+# Only the unresumable sweep waits out ABANDONED_PARTIAL_SECONDS: it reclaims disk, while a marker-+# mismatch purge exists to stop a corrupt append and cannot defer.def _purge_incomplete_blobs(@@ -590,10 +575,8 @@ def _purge_incomplete_blobs(if unresumable_only:if partial_is_resumable(blob.name, entry.parent):continue- # Two independent ways to spot a live writer, because neither is sufficient- # alone: the lock is the precise signal but upstream calls it best-effort and- # some filesystems grant it to everyone, while mtime cannot tell a dead writer- # from one stalled on a slow network for longer than the grace.+ # Neither signal is sufficient alone: the lock is precise but upstream calls it best-effort and+ # some filesystems grant it to everyone, while mtime cannot tell a dead writer from a stalled one.if blob_download_lock_held(entry, blob_hash):continueowned = owns_all_blobs or bool(owned_hashes and blob_hash in owned_hashes)@@ -607,8 +590,8 @@ def _purge_incomplete_blobs(blob.unlink()removed += 1except FileNotFoundError:- # A peer finalized or removed it after enumeration. The requested- # end state was reached, so this is not a failed purge.+ # A peer finalized or removed it after enumeration, so the requested end state was reached and this+ # is not a failed purge.continueexcept OSError:failed += 1@@ -617,8 +600,7 @@ def _purge_incomplete_blobs(return _PurgeOutcome(removed + watched_outcome.removed, failed + watched_outcome.failed)-# One shared pause is enough to tell a frozen corpse from a writer mid-transfer: hf writes a-# partial continuously, so anything untouched across it is not being written by anyone.+# One shared pause is enough to tell a frozen corpse from a writer mid-transfer: hf writes a partial continuously._STILLNESS_PROBE_SECONDS = 2.0@@ -638,7 +620,7 @@ def _purge_still_partials(watched: "Sequence[tuple[Path, str, int, float]]") ->try:stat = blob.stat()if stat.st_size != size or stat.st_mtime != mtime:- continue # it moved, so somebody owns it after all+ continueblob.unlink()removed += 1except FileNotFoundError:@@ -713,9 +695,8 @@ def _marker_path(entry: Path, variant: Optional[str] = None) -> Path:def _is_transport_marker_file(path: Path) -> bool:- # Matches ".transport", its tmps, and variant-scoped ".transport.gguf-*".- # Real HF cache entries (blobs/refs/snapshots/.no_exist) never start with- # ".transport.".+ # Matches ".transport", its tmps and variant-scoped ".transport.gguf-*"; real HF cache entries+ # (blobs/refs/snapshots/.no_exist) never start with ".transport.".return path.name == TRANSPORT_MARKER_NAME or path.name.startswith(f"{TRANSPORT_MARKER_NAME}.")@@ -729,22 +710,21 @@ def _read_marker_value(marker: Path) -> Optional[str]:return Nonevalue = marker.read_text(encoding = "utf-8").strip()except (OSError, UnicodeDecodeError):- # UnicodeDecodeError is a ValueError, so it would escape and abort- # prepare_cache_for_transport. An unknown value just purges and restarts.+ # UnicodeDecodeError is a ValueError, so it would escape and abort prepare_cache_for_transport; an+ # unknown value just purges and restarts.return Nonereturn value if value in VALID_TRANSPORTS else Nonedef _write_marker_value(marker: Path, mode: str) -> None:try:- # tmp + rename so a SIGKILL mid-write can't leave a half-written marker.- # The tmp name is per-process so concurrent writers don't clobber tmps.+ # tmp + rename so a SIGKILL mid-write cannot leave a half-written marker, with a per-process tmp+ # name so concurrent writers do not clobber tmps.tmp = marker.with_name(f"{marker.name}.tmp-{os.getpid()}")tmp.write_text(mode, encoding = "utf-8")os.replace(tmp, marker)except OSError:- # Best-effort: a missing marker next run purges the partial defensively,- # the safe failure mode.+ # Best-effort: a missing marker next run purges the partial defensively, which is the safe failure mode.pass@@ -814,8 +794,8 @@ def prepare_cache_for_transport("""if mode not in VALID_TRANSPORTS:if mode == TRANSPORT_AUTO:- # "auto" is a request preference, not a cache writer: it must be resolved to xet/http- # before reaching this layer. Naming it turns "invalid transport" into the actual bug.+ # "auto" is a request preference, not a cache writer: naming it turns "invalid transport" into the+ # actual bug.raise ValueError(f"{TRANSPORT_AUTO!r} must be resolved to a concrete transport before preparing the "f"cache; expected one of {sorted(VALID_TRANSPORTS)}"@@ -833,9 +813,8 @@ def prepare_cache_for_transport(except OSError:return 0if not entries:- # First download: pre-create the repo dir so the marker lands before the- # worker writes any bytes. Otherwise a SIGKILL mid-download leaves a- # partial with no marker that the resume then purges.+ # Pre-create the repo dir so the marker lands before the worker writes any bytes; otherwise a+ # SIGKILL mid-download leaves a partial with no marker that the resume then purges.canonical = repo_cache_dir_name(repo_type, repo_id)new_entry = root / canonicaltry:@@ -852,11 +831,7 @@ def prepare_cache_for_transport(if mode == TRANSPORT_XET:main_purge = _purge_incomplete_blobs(entry, only_blob_hashes, protected)else:- # A matching marker vouches for provenance, which is only worth something if- # something can still append to the partial it vouches for. When nothing can, what- # survives is dead weight that holds the disk the refetch needs and, carrying the- # etag of the blob being refetched, pins the bar to its own stale high-water mark- # until the new attempt overtakes it. Sweep it, but only once abandoned.+ # A matching marker only matters while something can still append to the partial it vouches for.if _read_marker(entry, variant) != mode:main_purge = _purge_incomplete_blobs(entry, only_blob_hashes, protected)else:@@ -981,14 +956,13 @@ def sweep_abandoned_partials(this when a download reaches a terminal state and every file skipped then gets a secondlook, by which point the grace has long since elapsed."""- # DownloadMetadata.hub_cache is a str, and every caller hands its captured root straight- # through, so normalize here rather than trusting each one to remember.+ # DownloadMetadata.hub_cache is a str and every caller hands its captured root straight through, so+ # normalize here rather than trusting each one.if isinstance(root, str):root = Path(root) if root else Noneremoved = 0- # The destructive iterator, not the active one: this deletes, and on a case-insensitive- # collision the active iterator yields every spelling while this one resolves to the exact- # directory or refuses. A job owning "its" repo does not own a differently-cased neighbour.+ # The destructive iterator, not the active one: on a case-insensitive collision the active iterator+ # yields every spelling while this one resolves to the exact directory or refuses.for entry in iter_destructive_repo_cache_dirs(repo_type, repo_id, root = root):outcome = _purge_incomplete_blobs(entry,@@ -1195,11 +1169,9 @@ def existing_blob_bytes(download still needs to write before the run starts."""if not blob_hashes:return 0- # One tally for ALL the repo dirs the root holds, not one per dir. The Hub resolves repo ids- # case-insensitively while huggingface_hub keeps the caller's casing in the folder name, so- # a case-sensitive filesystem really does end up with models--Org--Model beside- # models--org--model, each holding its own copy of the same blob. Summing the dirs counted- # that shard twice, and the caller's max(0, total - have) then clamped to zero.+ # One tally for ALL the repo dirs the root holds: the Hub resolves repo ids case-insensitively+ # while huggingface_hub keeps the caller's casing, so a case-sensitive filesystem holds two copies+ # of one blob and summing the dirs counted that shard twice.present = {blob_hash: 0 for blob_hash in blob_hashes}for entry in iter_active_repo_cache_dirs(repo_type, repo_id, root = root):blobs_dir = entry / "blobs"@@ -1222,28 +1194,22 @@ def existing_blob_bytes(and not partial_is_resumable(blob.name, entry.parent)and not blob_download_lock_held(entry, blob_hash)):- # Callers spend this on "bytes we will not have to fetch again", and- # _preflight_disk_space subtracts it from the space a download needs. An- # unresumable partial is refetched in full into a new path, so counting it- # would clear a download for a disk that cannot hold it. A LOCKED one is- # different: a live peer is finishing it and snapshot_download will block- # on that lock and reuse the result, so those bytes are not ours to find- # room for. Two GGUF variants sharing an mmproj hit this every time.+ # An unresumable partial is refetched in full into a new path, so counting it would clear a+ # download for a disk that cannot hold it. A LOCKED one is different: a live peer is finishing it+ # and snapshot_download blocks on that lock and reuses the result, so those bytes are not ours to+ # find room for.continue- # A partial is measured by the bytes actually ON DISK, not by its logical- # length: hf_transfer's parallel Range writer leaves a sparse file whose- # st_size runs ahead of what has been written -- observed at 1.2 GB reported- # against 112 MB present, and once st_size reached the declared size the- # remainder read as zero for a file barely started. A finalized blob is whole- # by construction, so it keeps st_size: st_blocks is smaller than the file on- # a compressing filesystem.+ # Measured by the bytes actually ON DISK: hf_transfer's parallel Range writer leaves a sparse file+ # whose st_size runs ahead of what was written, observed at 1.2 GB reported against 112 MB. A+ # finalized blob is whole by construction, so it keeps st_size: st_blocks is smaller than the file+ # on a compressing filesystem.bytes_here = (blob_bytes_present(blob)if partial_hash is not Noneelse max(0, int(blob.stat().st_size)))- # Broken advisory locks can leave several process-unique writers for one etag.- # They are duplicate attempts, not additive completion, so keep the largest.+ # Broken advisory locks can leave several process-unique writers for one etag: duplicate attempts,+ # not additive completion, so keep the largest.present[blob_hash] = max(present[blob_hash], max(0, int(bytes_here)))except OSError:continue@@ -1269,14 +1235,11 @@ class DownloadMetadata:variant: Optional[str]transport: Optional[str]cancel_marker_transport: Optional[str] = None- # GGUF variant main/writable hashes, identifying the variant-specific shards- # for concurrency decisions.+ # GGUF variant main/writable hashes, identifying the variant-specific shards for concurrency decisions.blob_hashes: frozenset[str] = field(default_factory = frozenset)- # Full required hash set for progress/completion (includes the shared mmproj- # companion for vision GGUF repos).+ # Includes the shared mmproj companion for vision GGUF repos.progress_blob_hashes: frozenset[str] = field(default_factory = frozenset)# Bytes already complete before this job started; not counted as this run's- # progress.completed_baseline_bytes: int = 0hub_cache: Optional[str] = Nonexet_cache: Optional[str] = None@@ -1380,8 +1343,8 @@ class DownloadRegistry:self._cancel_marker_transports: dict[str, str] = {}self._pending_cancel: dict[str, Optional[int]] = {}self._generations: dict[str, int] = {}- # Monotonic across keys so an evicted then re-claimed key never reuses a- # prior generation (which would let a stale cancel match a new run).+ # Monotonic across keys so an evicted then re-claimed key never reuses a prior generation, which+ # would let a stale cancel match a new run.self._generation_seq = 0self._deleting: dict[str, set[Optional[str]]] = {}# Publish external cache owners under the same lock as Model Hub jobs.@@ -1633,11 +1596,8 @@ class DownloadRegistry:with self._lock:if repo in self._repository_owners:return False, "repository_owned"- # Run the final external admission check while the registry lock is- # held, immediately before inspecting and publishing active state.- # The GGUF load path establishes its marker before calling- # its active-job probe, so either this claim observes that marker- # or the load's later probe observes this claim.+ # Run the final admission check under the registry lock: the GGUF load path establishes its marker+ # before its active-job probe, so either this claim sees that marker or the load sees this claim.if admission_check is not None and not admission_check():return False, "admission_blocked"deleting_scopes = self._deleting.get(repo)@@ -1656,11 +1616,9 @@ class DownloadRegistry:stale_keys.append(other_key)continueother_metadata = self._metadata.get(other_key)- # Same-transport variants of one model run concurrently: each- # worker purges only its own re-resolved main blobs and the- # shared companion is guarded by its marker. Cross-transport- # stays serialized so an HTTP resume and an XET rewrite never- # write one shared blob at once.+ # Same-transport variants of one model run concurrently, since each worker purges only its own+ # re-resolved main blobs and the shared companion is guarded by its marker; cross-transport stays+ # serialized so an HTTP resume and an XET rewrite never write one blob at once.concurrent_gguf_variants = (repo_type == "model"and bool(variant)@@ -1679,8 +1637,8 @@ class DownloadRegistry:return False, conflict_statecurrent = self._jobs.get(key, DownloadState("idle")).stateif current in _ACTIVE_STATES and not replace_active:- # A scope slot is shared by every file set that rides it (two quants of one repo both key as "@diffusion"), so adopting- # the live job would let the caller wait on files it never asked for. Reject instead, under the lock.+ # A scope slot is shared by every file set that rides it, so adopting the live job would let the+ # caller wait on files it never asked for.live = self._metadata.get(key)if (scoped_files is not None@@ -1840,9 +1798,8 @@ class DownloadRegistry:candidate_keys = list(self._repo_active.get(repo_key, set()))else:candidate_keys = [key for active in self._repo_active.values() for key in active]- # An XET->HTTP retry handoff briefly drops its key from _repo_active- # while its job stays active; include those released-but-active jobs- # so the waiting retry still lists and can be adopted or cancelled.+ # An XET->HTTP retry handoff briefly drops its key from _repo_active while its job stays active, so+ # include those released-but-active jobs.seen = set(candidate_keys)for key, job in self._jobs.items():if key in seen or job.state not in _ACTIVE_STATES:@@ -1937,11 +1894,8 @@ class DownloadRegistry:continueif self._active_job_variant_locked(key) != target:return True- # An XET->HTTP retry peer between release_active_slot() and its reclaim- # is briefly absent from _repo_active while its job stays active and- # still owns the shared companion; mirror the released-but-active scan- # used by _delete_blocked_by_active_locked so it still blocks companion- # deletion of a different variant.+ # A retry peer between release_active_slot() and its reclaim is briefly absent from _repo_active+ # while it still owns the shared companion, so mirror the released-but-active scan.for key, job in self._jobs.items():if key in active_keys or _repo_of_key(key) != repo_id:continue@@ -1981,18 +1935,12 @@ class DownloadRegistry:]live_keys = {key for key, _proc, _metadata in live}# Flag as an intentional stop so the watcher's exit classification- # reports them cancelled rather than an OOM/crash once SIGKILL lands.for key, _proc, _metadata in live:if self._jobs.get(key, DownloadState("idle")).state == "running":self._jobs[key] = DownloadState("cancelling")- # Settle active jobs without a live worker too. Two cases: an- # XET->HTTP retry parked in the reclaim wait loop has dropped its- # worker and slot guard, so it is absent from `live`; and a- # registered worker that already exited with an error but whose- # watcher has not yet run would otherwise stay `running` and spawn an- # HTTP retry after this shutdown snapshot. Skip a registered worker- # that exited cleanly (rc == 0): it completed and the watcher will- # mark it done, so marking it cancelling would strand a stale marker.+ # Settle active jobs without a live worker: a retry parked in the reclaim wait has dropped its+ # worker, and a registered worker that errored before its watcher ran would stay running and spawn+ # an HTTP retry. Skip one that exited cleanly, which would strand a stale marker.for key, job in list(self._jobs.items()):if job.state not in _ACTIVE_STATES or key in live_keys:continue@@ -2000,22 +1948,16 @@ class DownloadRegistry:if proc is not None:if proc.poll() == 0:continue- # A registered worker that exited nonzero on its own over HTTP- # is a genuine terminal download failure, not a shutdown cancel- # and not retry-capable: leave its error status intact rather- # than persisting a cancel marker that would read as- # cancelled/resumable after restart. Only an exited XET worker- # could still spawn a post-shutdown HTTP retry, so only that- # needs settling here.+ # A registered worker that exited nonzero over HTTP is a genuine terminal failure, not a shutdown+ # cancel: only an exited XET worker could still spawn a post-shutdown HTTP retry.metadata = self._metadata.get(key)if metadata is not None and metadata.transport == TRANSPORT_HTTP:continueself._pending_cancel[key] = self._generations.get(key)self._jobs[key] = DownloadState("cancelling")settled_no_proc.append(self._metadata.get(key))- # Persist a cancel marker for each settled no-live-worker job outside the- # lock (mirroring the reaped path) so shutdown records resumable/cancelled- # state even if it returns before the daemon watcher wakes to do so.+ # Persist the cancel marker outside the lock so shutdown records resumable state even if it returns+ # before the daemon watcher wakes.for metadata in settled_no_proc:if metadata is not None:persist_cancel_marker(@@ -2051,8 +1993,7 @@ class DownloadRegistry:logger.warning(f"shutdown: {kind} worker for {key} did not exit after kill")except Exception:pass- # Mark only genuinely interrupted workers (rc != 0, or None on wait- # timeout); persisting before the exit is known would strand a stale+ # Mark only genuinely interrupted workers: persisting before the exit is known would strand a stale# marker on a worker that completed cleanly during shutdown.if metadata is not None and proc.poll() != 0:persist_cancel_marker(diff --git a/studio/backend/hub/utils/gguf.py b/studio/backend/hub/utils/gguf.pyindex 181504eaa78..0986e154420 100644--- a/studio/backend/hub/utils/gguf.py+++ b/studio/backend/hub/utils/gguf.py@@ -95,10 +95,9 @@ def is_mmproj_filename(filename: str) -> bool:return "mmproj" in filename.lower()-# Anchored at an END of the stem, never a substring, for the reason-# is_mtp_drafter_path documents: a name that merely contains the word-# (``Qwen3-Imatrix-Tuned-Q4_K_M.gguf``) is a real model, while every published-# imatrix leads or closes with it.+# Anchored at an END of the stem, never a substring, for the reason is_mtp_drafter_path documents:+# a name that merely contains the word (Qwen3-Imatrix-Tuned-Q4_K_M.gguf) is a real model, while+# every published imatrix leads or closes with it._IMATRIX_TOKEN_RE = re.compile(r"^imatrix(?:[._\-]|$)|[._\-]imatrix$", re.IGNORECASE)@@ -122,12 +121,11 @@ def is_imatrix_filename(path: str) -> bool:return bool(_IMATRIX_TOKEN_RE.search(stem)) or name.lower().endswith(".imatrix")-# Separate-file drafter kinds. dspark and dflash are the same DeepSeek V4 Flash-# drafter: the folder it ships in and the architecture it reports.+# dspark and dflash are the same DeepSeek V4 Flash drafter: the folder it ships in and the architecture it reports._DRAFTER_KINDS = ("mtp", "dspark", "dflash")-# Directories only: mtp/ and dspark/ are always a publisher's companion folder,-# while dflash/ is a family name a user picks for real weights.+# Directories only: mtp/ and dspark/ are always a publisher's companion folder, while dflash/ is a+# family name a user picks for real weights._DRAFTER_DIR_KINDS = ("mtp", "dspark")@@ -226,10 +224,9 @@ def is_gguf_filename(filename: str) -> bool:return filename.lower().endswith(".gguf")-# Every repo that bundles H3's denoisers together with its companion models. The Unsloth mirror-# carries the Qwen3-VL text-encoder quants beside the denoisers, so it needs the same filtering as-# the community repack it replaced; listing only one of them would let the encoder GGUFs be-# aggregated as if they were selectable transformer quants.+# Every repo that bundles H3's denoisers with its companion models: the Unsloth mirror carries the+# Qwen3-VL encoder quants beside the denoisers, so listing one repo would aggregate encoder GGUFs+# as selectable transformer quants._H3_BUNDLE_REPOS = frozenset({"leejet/minimax-h3-gguf", "unsloth/minimax-h3-gguf"})@@ -237,10 +234,9 @@ def is_h3_bundle_repo(repo_id: str) -> bool:return repo_id.strip().lower() in _H3_BUNDLE_REPOS-# Both released denoiser partitions are valid picks -- which one is picked IS the task. Kept in+# Both released denoiser partitions are valid picks and which one is picked IS the task. Kept in# step with validate_h3_transformer_filename in core/inference/video_minimax_h3.py, which the load-# enforces; listing only FL2VA hid every published Ref2VA quant from the picker even though the-# loader routes it and the reference UI depends on it.+# enforces: listing only FL2VA hid every published Ref2VA quant from the picker._H3_DENOISER_PARTITIONS = ("minimax_h3_fl2va", "minimax_h3_ref2va")@@ -394,20 +390,17 @@ def extract_quant_token(filename: str) -> Optional[str]:return None-# A bits-per-weight modifier trailing the quant token. Two builds of one base quant at different-# bpw are two checkpoints (byteshape ships IQ4_XS at 3.53, 3.97 and 4.19), and the loader's own-# label keeps the modifier for exactly that reason. The variant KEY has to keep it too: without it-# the lister advertises IQ4_XS-3.53bpw while the plan, the download map and the delete predicate-# all say IQ4_XS, so the advertised name 404s and the collapsed one unlinks every build.-#-# Applied with ``match`` against the text that follows the token, never ``search``: only a-# modifier IMMEDIATELY after the quant qualifies it, so ``flux1-dev-Q8_0-fp32-08.577bpw`` keeps-# the bare ``Q8_0`` rather than borrowing a number that describes something else in the name.+# Two builds of one base quant at different bpw are two checkpoints (byteshape ships IQ4_XS at+# 3.53, 3.97 and 4.19), so the variant KEY has to keep the modifier: without it the advertised+# name 404s.+# match, never search: only a modifier IMMEDIATELY after the quant qualifies it, so+# flux1-dev-Q8_0-fp32-08.577bpw keeps the bare Q8_0 rather than borrowing a number describing+# something else._GGUF_BPW_SUFFIX_RE = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw", re.IGNORECASE)-# The same modifier ending a name of its own, with or without the extension still on it. Used-# only for the basename under a quant DIRECTORY (``Q6_K/model-3.5bpw.gguf``), where the token is-# in the parent and the number is all the file has to say which build it is.+# The same modifier ending a name of its own, used only for the basename under a quant DIRECTORY+# (Q6_K/model-3.5bpw.gguf), where the token is in the parent and the number is all the file has to+# identify the build._GGUF_BPW_TRAILING_RE = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw(?=\.[A-Za-z0-9]+$|$)", re.IGNORECASE)@@ -834,9 +827,8 @@ def bare_quant_alias(key: str) -> str:token = quant_token_with_bpw(basename)if token is not None:return token- # Nothing in the name is a quant, so the alias is the unknown-variant spelling. That one- # still re-stems, and a key arrives already extension-stripped, so hand it an extension to- # strip rather than let it cut at the dot in "ltx-2.3".+ # Nothing in the name is a quant, so the alias is the unknown-variant spelling; hand it an extension+ # to strip rather than let it cut at the dot in "ltx-2.3".return extract_quant_label(f"{basename}.gguf")@@ -892,9 +884,8 @@ def gguf_variant_key(filename: str) -> str:quant = quant_token_with_bpw(path)if quant is None:return _unknown_gguf_variant_key(path)- # MiniMax H3 bundles two denoiser partitions, and may publish both full and pruned- # builds at one quant. Each file is a different loadable checkpoint, so the bare- # quant cannot identify the row, download state, or file the loader should open.+ # MiniMax H3 bundles two denoiser partitions and may publish full and pruned builds at one quant,+ # so the bare quant cannot identify the row, its download state, or the file to open.if path.rsplit("/", 1)[-1].lower().startswith(_H3_DENOISER_PARTITIONS):return _unknown_gguf_variant_key(path)parents = path.rpartition("/")[0]@@ -924,9 +915,8 @@ def _apply_gguf_display_labels(variants: list[GgufVariantInfo]) -> None:]ambiguous = len(unknown_variants) > 1- # The bpw modifier is part of the key but reads perfectly well on its own- # ("IQ4_XS-3.53bpw"), so a key that is only the token plus its bpw suffix is NOT a- # path-qualified one and needs no scope label.+ # The bpw modifier reads perfectly well on its own, so a key that is only the token plus its bpw+ # suffix is NOT path-qualified and needs no scope label.def _plain_key(variant) -> Optional[str]:return quant_token_with_bpw(variant.filename)@@ -1009,8 +999,8 @@ def iter_hf_cache_snapshots(repo_id: str, root: Optional[Path] = None):)for repo_dir in repo_dirs:snapshots_dir = repo_dir / "snapshots"- # is_dir() ignores only ENOENT/ENOTDIR/EBADF/ELOOP, so an unreadable root raised- # EACCES up to 3.13 (3.14 returns False, gh-101357). Skip it instead of 500ing.+ # is_dir() ignores only ENOENT/ENOTDIR/EBADF/ELOOP, so an unreadable root raised EACCES up to 3.13+ # (3.14 returns False, gh-101357); skip it instead of 500ing.try:if not snapshots_dir.is_dir():continue@@ -1143,8 +1133,7 @@ def merge_sibling_snapshot_variants(merged_complete.add(variant.quant)whole.add(key)if changed:- # Labels disambiguate within a revision, so a merged set can hold names only the- # merge brings together.+ # Labels disambiguate within a revision, so a merged set can hold names only the merge brings together._apply_gguf_display_labels(merged)return merged, has_vision or any(merged_vision.values()), merged_complete, snapshot@@ -1189,9 +1178,8 @@ def list_partial_gguf_variants_from_state("""from hub.utils import download_manifest- # Variant identity on disk is case-insensitive (_entry_key lowercases it), so- # dedupe on the lowercased key. Manifests are read first to keep their- # original-casing label over a lowercased cancel marker for the same variant.+ # Variant identity on disk is case-insensitive, so dedupe on the lowercased key; manifests are read+ # first to keep their original-casing label over a lowercased cancel marker.seen: set[str] = set()ordered: list[str] = []sources = (@@ -1247,14 +1235,13 @@ def list_partial_gguf_variants_from_state(if not is_gguf_filename(expected.path):continueif is_imatrix_filename(expected.path):- # A manifest predating this filtering can still name one; it is- # neither weights nor a companion, so it counts towards neither size.+ # A manifest predating this filtering can still name one: it is neither weights nor a companion, so+ # it counts towards neither size.imatrix_only = Truecontinueif is_mtp_drafter_path(expected.path):- # Downloaded with every variant (like mmproj) but not a- # selectable quant; count it so the shown download size- # matches what is fetched.+ # Downloaded with every variant like mmproj but not a selectable quant; counted so the shown+ # download size matches what is fetched.companion_bytes += max(0, int(expected.size or 0))continueif is_mmproj_filename(expected.path):@@ -1266,11 +1253,9 @@ def list_partial_gguf_variants_from_state(main_filenames.append(expected.path)size_bytes += max(0, int(expected.size or 0))if main_filename is None:- # An older build could download the imatrix as a variant of its own, so its- # interrupted state is still on disk. Naming the synthetic file after the- # variant would put that row back in the menu, at zero bytes, on exactly the- # offline path this listing serves. Only when NOTHING eligible was found:- # a marker-only quant keeps its synthetic row so it can be resumed or deleted.+ # An older build could download the imatrix as a variant of its own, so naming the synthetic file+ # after the variant would put that interrupted row back in the menu at zero bytes. Only when+ # NOTHING eligible was found.if imatrix_only or is_imatrix_filename(variant):continuemain_filename = f"{variant}.gguf"@@ -1384,8 +1369,10 @@ def list_gguf_variants(if is_mmproj_filename(filename):has_vision = Truecontinue- # The two extractors disagree on F16-be-checkpoint-Q4_K_M shapes; judge with the- # loader's label so no row is advertised for a file the remote detector refuses.+ # The two extractors disagree on F16-be-checkpoint-Q4_K_M shapes; judge with the loader's label so+ # no row is advertised for a file the detector refuses.+ # The two extractors disagree on F16-be-checkpoint-Q4_K_M shapes; judge with the loader's label so no+ # row is listed for a file the local detector refuses.from utils.models.model_config import _extract_quant_label as _loader_quantif is_big_endian_gguf_path(filename, _loader_quant(filename)):@@ -1440,8 +1427,8 @@ def list_local_gguf_variants(main_files: list[tuple[str, int]] = []has_vision = False- # Match the cache dir of ANY H3 bundle repo, not just one of them: the same aggregation runs- # over whichever mirror the user actually downloaded.+ # Match the cache dir of ANY H3 bundle repo: the aggregation runs over whichever mirror the user+ # actually downloaded.root_key = root.as_posix().lower()h3_bundle_repo = next((r for r in _H3_BUNDLE_REPOS if f"models--{r.replace('/', '--')}" in root_key), None@@ -1453,9 +1440,8 @@ def list_local_gguf_variants(if is_imatrix_filename(file.name):continueif is_mmproj_filename(file.name):- # Header metadata distinguishes vision projectors from audio-only ones. Read it- # only when Windows reports the file fully present; opening a cloud placeholder- # would recall the projector during discovery.+ # Header metadata distinguishes vision projectors from audio-only ones, read only when Windows+ # reports the file fully present: opening a cloud placeholder would recall it during discovery.try:info = file.stat()has_vision = has_vision or (@@ -1473,8 +1459,6 @@ def list_local_gguf_variants(rel = file.relative_to(root).as_posix()if _is_local_mtp_drafter(file, custom_root, rel):continue- # The two extractors disagree on F16-be-checkpoint-Q4_K_M shapes; judge with the- # loader's label so no row is listed for a file the local detector refuses.from utils.models.model_config import _extract_quant_label as _loader_quantif is_big_endian_gguf_path(rel, _loader_quant(rel)):diff --git a/studio/backend/hub/utils/gguf_plan.py b/studio/backend/hub/utils/gguf_plan.pyindex 4e1f22f558b..154f2b29268 100644--- a/studio/backend/hub/utils/gguf_plan.py+++ b/studio/backend/hub/utils/gguf_plan.py@@ -84,9 +84,8 @@ def is_main_gguf_variant_path(path: str, variant: str) -> bool:and not is_mmproj_filename(path)and not is_mtp_drafter_path(path)and not is_imatrix_filename(path)- # The endian predicate reads a quant TOKEN, so it gets the label: handed the qualified key- # it cannot see a parent-only quant and drops the file, leaving the plan with no main- # files at all and an interrupted download with no hashes to resume against.+ # The endian predicate reads a quant TOKEN, so hand it the label: given the qualified key it+ # cannot see a parent-only quant and drops the file, leaving the plan with no main files.and not is_big_endian_gguf_path(path, extract_quant_label(path))and gguf_variant_key(path).lower() == variant.lower())@@ -218,9 +217,9 @@ def dflash_plan_files(def build_gguf_variant_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]:- # Family grouping keeps the family holding the lexicographically first name, which is the- # "._" one, so the plan fetched the sidecar and marked the variant complete -- leaving local- # discovery, which judges by header, no main GGUF to load.+ # Family grouping keeps the family holding the lexicographically first name, which is the "._"+ # one, so the plan fetched the sidecar and marked the variant complete, leaving header-based local+ # discovery no main GGUF to load.siblings = drop_shadowed_appledouble_siblings(list(siblings))main: dict[str, list] = {}all_mmproj = mmproj_siblings(siblings)@@ -242,24 +241,22 @@ def build_gguf_variant_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]:name = _gguf_rfilename(sibling)if name is None:continue- # Companions are folded into every plan below; keep them out of the- # quant grouping so a drafter never lands in a variant's main files- # (the root mtp-*.gguf carries a quant label, e.g. Q8_0).- # An imatrix leaves entirely rather than joining companions_expected: no- # variant needs llama-quantize's calibration data downloaded.+ # Keep companions out of the quant grouping so a drafter never lands in a variant's main files: the+ # root mtp-*.gguf carries a quant label.+ # An imatrix leaves entirely rather than joining companions_expected: no variant needs llama-+ # quantize's calibration data downloaded.if is_mmproj_filename(name) or is_mtp_drafter_path(name) or is_imatrix_filename(name):continuequant = gguf_variant_key(name).lower()- # The endian predicate reads a quant TOKEN -- it decides whether the quant came from the- # parent directory only -- so a qualified key would make it misread the path and drop the- # file from every plan.+ # The endian predicate reads a quant TOKEN, so a qualified key would make it misread the path and+ # drop the file from every plan.if is_big_endian_gguf_path(name, extract_quant_label(name)):continuemain.setdefault(quant, []).append(sibling)plans: dict[str, GgufVariantPlan] = {}- # Every weight in the listing, so the ranking can tell a sidecar naming a- # neighbouring family from one naming this variant's.+ # Every weight in the listing, so the ranking can tell a sidecar naming a neighbouring family from+ # one naming this variant's.all_weight_names = [name.rsplit("/", 1)[-1]for quant_siblings in main.values()@@ -272,10 +269,9 @@ def build_gguf_variant_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]:for sibling in target_main_siblingsif (file := expected_file_from_sibling(sibling)) is not None)- # Per variant, unlike mmproj and the MTP drafter: ranked against the weight- # being fetched, so a multi-family repo does not hand B the drafter naming A.- # Against the family plan_from_expected_files KEEPS, not the listing's first,- # or a two-family variant key pairs the discarded one's sidecar.+ # Per variant, unlike mmproj and the MTP drafter: ranked against the weight being fetched, and+ # against the family plan_from_expected_files KEEPS, or a two-family variant key pairs the wrong+ # sidecar.kept_main = _one_shard_family(main_expected)target_weight_name = (min(file.path for file in kept_main).rsplit("/", 1)[-1] if kept_main else None@@ -316,8 +312,8 @@ def plan_for_variant(plans: dict[str, GgufVariantPlan], variant: str) -> Optionaexact = plans.get(wanted)if exact is not None:return exact- # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant- # names both partitions, and picking either would load a different task.+ # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant names+ # both partitions, and picking either would load a different task.matches = [key for key in plans if "/" in key and bare_quant_alias(key).lower() == wanted]return plans[matches[0]] if len(matches) == 1 else None@@ -354,15 +350,13 @@ def plan_from_expected_files(expected = tuple(expected_files)all_main = tuple(file for file in expected if is_main_gguf_variant_path(file.path, variant))main_files = _one_shard_family(all_main)- # A discarded family has to leave the plan ENTIRELY, not just its main files. It is- # target_filenames, required_hashes and download_size_bytes that the worker fetches and the- # manifest checks against; leaving the copy there downloaded it, then reclaim deleted it as- # not-ours (it is absent from main_hashes) and the job reported partial and fetched it again.+ # A discarded family has to leave the plan ENTIRELY: target_filenames, required_hashes and+ # download_size_bytes are what the worker fetches, so leaving the copy there downloaded it, then+ # reclaim deleted it as not-ours (absent from main_hashes) and the job fetched it again.kept = {file.path for file in main_files}expected = tuple(file for file in expected if file not in all_main or file.path in kept)companion_files = tuple(file for file in expected if is_companion_gguf_path(file.path))- # Manifest-resume fallback for the mmproj fields below: companion_files- # also holds the MTP drafter, so keep an mmproj-only view.+ # companion_files also holds the MTP drafter, so keep an mmproj-only view for the manifest-resume fallback.mmproj_files = tuple(file for file in companion_files if is_mmproj_filename(file.path))main_hashes = frozenset(file.sha256 for file in main_files if file.sha256)companion_hashes = frozenset(file.sha256 for file in companion_files if file.sha256)diff --git a/studio/backend/hub/utils/hf_cache_state.py b/studio/backend/hub/utils/hf_cache_state.pyindex 2f3cf101810..0042b9cb961 100644--- a/studio/backend/hub/utils/hf_cache_state.py+++ b/studio/backend/hub/utils/hf_cache_state.py@@ -18,9 +18,8 @@ EXIT_CANCELLED = 130TRANSPORT_HTTP = "http"TRANSPORT_XET = "xet"VALID_TRANSPORTS = frozenset({TRANSPORT_HTTP, TRANSPORT_XET})-# A *request* preference, deliberately NOT in VALID_TRANSPORTS: "auto" is resolved to a real-# transport before anything is spawned, and the on-disk .transport marker must keep naming the-# writer that produced a partial, or a resume picks the wrong strategy.+# A request preference, deliberately NOT in VALID_TRANSPORTS: the on-disk .transport marker must+# keep naming the writer that produced a partial, or a resume picks the wrong strategy.TRANSPORT_AUTO = "auto"VALID_TRANSPORT_MODES = frozenset({TRANSPORT_HTTP, TRANSPORT_XET, TRANSPORT_AUTO})TRANSPORT_MARKER_NAME = ".transport"@@ -51,10 +50,8 @@ def incomplete_blob_hash(name: str) -> Optional[str]:# The last huggingface_hub line whose partials a later attempt can append to._LAST_RESUMABLE_PARTIAL_VERSION = (1, 17)-# How long a partial must sit untouched before it reads as abandoned rather than in flight.-# huggingface_hub writes to one continuously, so anything still advancing has a live writer ---# possibly a client in another process that no registry here can see. Shared by the sweep that-# deletes abandoned partials and the progress scan that must not report one as current.+# huggingface_hub writes to a partial continuously, so anything still advancing has a live writer,+# possibly a client in another process no registry here can see.ABANDONED_PARTIAL_SECONDS = 120@@ -146,8 +143,8 @@ def blob_download_lock_held(entry: Path, blob_hash: str) -> bool:"""lock_path = entry.parent / ".locks" / entry.name / f"{blob_hash}.lock"if not lock_path.exists():- # hf creates the lock file before taking the lock, so no file means no writer. It is- # also the answer for a SoftFileLock, whose file IS the lock (see below).+ # hf creates the lock file before taking the lock, so no file means no writer; it is also the answer+ # for a SoftFileLock, whose file IS the lock.return Falsetry:from filelock import FileLock, Timeout@@ -159,14 +156,10 @@ def blob_download_lock_held(entry: Path, blob_hash: str) -> bool:except Timeout:return Trueexcept Exception: # noqa: BLE001 - deliberately broad, see below- # A filesystem without flock raises NotImplementedError here, and upstream's- # WeakFileLock answers it by retrying as a SoftFileLock (huggingface_hub- # utils/_fixes.py). Retrying is pointless for a PROBE: a soft lock is its file, and we- # only reach this line because that file exists, so the soft answer is "held" too.- # What matters is that the exception does not escape -- it used to travel out through- # the purge and fail the download on every retry. Any other unprobeable error answers- # the same way, because the caller's remaining guard is a staleness check that an- # ownership claim may skip, and a wrong "free" there deletes a live writer's file.+ # A filesystem without flock raises NotImplementedError, which upstream answers by retrying as a+ # SoftFileLock; retrying is pointless for a PROBE, since we only reach here because the file+ # exists. What matters is that the exception does not escape: a wrong "free" deletes a live+ # writer's file.return True@@ -248,10 +241,9 @@ def hf_cache_roots(scan_errors: Optional[list] = None) -> list[Path]:try:key = str(path.resolve())except OSError as exc:- # A root that stats but will not resolve -- an intermittent network mount, a- # Windows reparse point -- is a root we could not read, not a root that is gone.- # Dropping it silently let the progress scan answer "measured, no cache", which- # hydration acts on by retiring a download whose files may be entirely intact.+ # A root that stats but will not resolve is a root we could not read, not one that is gone:+ # dropping it silently let the progress scan answer "measured, no cache", which retires a download+ # whose files may be intact.if scan_errors is not None:scan_errors.append(exc)return@@ -305,11 +297,9 @@ def blob_bytes_present(path: Path) -> int:blocks = getattr(st, "st_blocks", None)if blocks is not None and blocks > 0:return min(blocks * 512, st.st_size)- # A present zero is not a missing field. A parallel writer that sets the partial to its- # final length before its first chunk lands sits exactly here, and reading st_size then- # says "0 B left" on a download that has transferred nothing. The zero alone is not enough- # to act on -- a mount that never populates st_blocks looks the same -- so confirm the- # emptiness directly before believing it.+ # A present zero is not a missing field: a parallel writer that sets the partial to its final+ # length before its first chunk lands sits exactly here, and a mount that never populates+ # st_blocks looks the same, so confirm the emptiness directly before believing st_size.if blocks == 0 and _holds_no_data(path):return 0if sys.platform == "win32":@@ -481,8 +471,7 @@ def latest_snapshot_from_cache_path(try:def has_metadata(path: Path) -> bool:- # required_groups is an AND of ORs: the snapshot must carry at least one file from every- # group. That is what "loadable" means: metadata alone or weights alone is not enough.+ # required_groups is an AND of ORs: "loadable" means metadata alone or weights alone is not enough.for group in required_groups:if not any((path / name).is_file() for name in group):return False@@ -619,7 +608,7 @@ def iter_active_repo_cache_dirs(for entry in root.iterdir():if entry.name.lower() == target:yield entry- except OSError as exc: # see iter_repo_cache_dirs on why the caller may want this+ except OSError as exc:if scan_errors is not None:scan_errors.append(exc)return@@ -633,22 +622,17 @@ def preferred_repo_cache_dirs(active_root: Optional[Path] = None,scan_errors: Optional[list] = None,) -> list[Path]:- # iter_active_repo_cache_dirs already matches case-insensitively, so a repo- # dir the active root does hold is found whatever its casing; the canonical- # name below is only ever a placeholder for one that is not there yet.+ # iter_active_repo_cache_dirs already matches case-insensitively, so the canonical name below is+ # only ever a placeholder for a repo dir that is not there yet.active_entries = list(iter_active_repo_cache_dirs(repo_type, repo_id, root = active_root, scan_errors = scan_errors))if active_entries:return active_entriesif force_active:- # A running or cancelling job writes into the active root and nowhere- # else, so its progress may only ever be read from there. hf_cache_root- # returns None for a root that is not a directory yet -- the first- # download into a freshly configured cache creates it -- and falling- # through from there would read a previous cache's completed copy as- # this run's progress and finalize a job that has not written a byte.- # Name the directory this run will create instead.+ # A running or cancelling job writes into the active root and nowhere else, so its progress may+ # only be read from there: hf_cache_root returns None for a root not yet created, and falling+ # through would read a previous cache's completed copy as this run's progress.root = hf_cache_root(root = active_root) or active_root or _configured_hub_cache()if root is not None:canonical = repo_cache_dir_name(repo_type, repo_id)@@ -657,9 +641,8 @@ def preferred_repo_cache_dirs(def _configured_hub_cache() -> Optional[Path]:- # Path()-wrapped: the setting is typed Path, but a caller (or a test) that- # hands back a str would otherwise turn `root / name` above into a TypeError- # that surfaces as an empty progress reading -- the very card being fixed.+ # Path()-wrapped: the setting is typed Path, but a caller handing back a str would turn `root /+ # name` into a TypeError that surfaces as an empty progress reading.try:from utils.hf_cache_settings import get_hf_cache_pathsconfigured = get_hf_cache_paths().hub_cache@@ -833,9 +816,8 @@ def with_load_subdirs(model_name: str, names: tuple[str, ...]) -> tuple[str, ...from utils.security import security_load_subdirssubdirs = security_load_subdirs(model_name, local_files_only = True)except Exception:- # Degrading to root-only is fail-closed at every caller, so nothing is wrongly accepted. But a- # real cache permission or corruption fault then reaches the user as "your cached model isn't- # cached" with no clue why, and four sites now share this handler.+ # Degrading to root-only is fail-closed at every caller, but a real cache permission or corruption+ # fault then reaches the user as "your cached model isn't cached" with no clue why.from loggers import get_loggerget_logger(__name__).debug("Load-subdir detection failed for %s; using root only.",diff --git a/studio/backend/hub/utils/inventory_scan.py b/studio/backend/hub/utils/inventory_scan.pyindex e0fca5a9a1b..4eb6ab87239 100644--- a/studio/backend/hub/utils/inventory_scan.py+++ b/studio/backend/hub/utils/inventory_scan.py@@ -49,9 +49,8 @@ from hub.utils.hf_cache_state import ()from utils.paths.path_utils import drop_appledouble_metadata, is_appledouble_metadata-# Inventory is invalidated explicitly on every app-driven cache mutation, so-# this TTL only bounds staleness from out-of-band edits while skipping re-walks-# on rapid UI navigation.+# Inventory is invalidated explicitly on every app-driven cache mutation, so this TTL only bounds+# staleness from out-of-band edits._HF_CACHE_SCANS_TTL_SECONDS = 15.0_GGUF_SPLIT_RE = re.compile(r"-(\d{3,})-of-(\d{3,})(?=\.gguf$)", re.IGNORECASE)# transformers shard naming: each shard names the set's total.@@ -70,9 +69,8 @@ class _HfCacheScanFlight:_hf_cache_scans_flight: Optional[_HfCacheScanFlight] = None_hf_cache_scans_result: Optional[list] = None_hf_cache_scans_cached_at: float = 0.0-# Bumped on every invalidation. A scan tags itself with the epoch it began-# under; an invalidation mid-scan changes the epoch so the in-flight result is-# neither cached nor served to callers that arrived after the mutation.+# A scan tags itself with the epoch it began under, so an invalidation mid-scan makes the in-flight+# result neither cached nor served to callers that arrived after the mutation._hf_cache_scans_epoch: int = 0_T = TypeVar("_T")@@ -123,9 +121,8 @@ def all_hf_cache_scans() -> list:return list(_hf_cache_scans_result)start_epoch = _hf_cache_scans_epochflight = _hf_cache_scans_flight- # Only coalesce onto an in-flight scan from the current epoch; one that- # began before an intervening invalidation is superseded so- # post-mutation callers never receive pre-mutation data.+ # Only coalesce onto an in-flight scan from the current epoch, so post-mutation callers never+ # receive pre-mutation data.if flight is None or flight.epoch != start_epoch:flight = _HfCacheScanFlight(event = threading.Event(), epoch = start_epoch)_hf_cache_scans_flight = flight@@ -183,7 +180,7 @@ _HF_REPO_TYPES = frozenset({"model", "dataset", "space"})# Mirrors huggingface_hub's Cached{File,Revision,Repo}Info field-for-field; frozen because-# HFCacheInfo.delete_revisions() set-diffs ``revisions``.+# HFCacheInfo.delete_revisions() set-diffs revisions.@dataclass(frozen = True)class _RecoveredFileInfo:file_name: str@@ -723,8 +720,8 @@ def _repo_cache_dir_has_snapshot_legacy_partial() -> bool:if _repo_cache_dir_has_non_gguf_broken_snapshot_symlinks(repo_cache_dir, snapshot_dir):return True- # ``.incomplete`` blobs carry no revision, so they need attributing. Judged on this row's- # weights alone: a torn quant beside them is another row's payload, not this one's.+ # ".incomplete" blobs carry no revision, so they need attributing; judged on this row's weights+ # alone, since a torn quant beside them is another row's payload.if snapshot_dir is not None and not _repo_signal_applies_to_snapshot(repo_cache_dir, snapshot_dir, quants = False):@@ -797,15 +794,13 @@ def _completed_gguf_variants(snapshot_dir: Optional[Path]) -> set[str]:or is_imatrix_filename(rel)):continue- # Metadata vouching for a quant marks a torn snapshot ready: a set whose sidecars are- # all present but whose weights are not answers the shard count exactly as the real- # files would, in its own family and under their quant.+ # Metadata vouching for a quant marks a torn snapshot ready: a set whose sidecars are all present+ # answers the shard count exactly as the real files would.if is_appledouble_metadata(path):continuequant = gguf_variant_key(rel)- # Mirror the lister: a big-endian build is never offered, so it cannot vouch for the- # quant. Judged with the loader's label, since the two extractors disagree on- # F16-be-checkpoint-Q4_K_M and this file must not mark Q4_K_M complete.+ # A big-endian build is never offered, so it cannot vouch for the quant; judged with the loader's+ # label, since the two extractors disagree on F16-be-checkpoint-Q4_K_M.from utils.models.model_config import _extract_quant_label as _loader_quantif is_big_endian_gguf_path(rel, _loader_quant(rel)):@@ -1090,12 +1085,12 @@ def _snapshot_payload(snapshot_dir: Path) -> Optional[_SnapshotPayload]:if _required_config_is_unreadable(snapshot_dir / config_name, config_empty):unreadable.update(formats)model_format = _classify_non_gguf_model_format(**flags, trusted_hf_cache_repo = False)- # from_pretrained never globs, so shards with no index are invisible and neither serve nor veto.- # An unusable index is picked and failed on instead.+ # from_pretrained never globs, so shards with no index are invisible and neither serve nor veto; an+ # unusable index is picked and failed on instead.unloadable: set = set()invisible: set = set()- # Selected by its own name, so it counts even when the walk grouped no shard of it, and it- # carries its own paths: every weight_map entry resolves against the index.+ # Selected by its own name, so it counts even when the walk grouped no shard of it, and every+ # weight_map entry resolves against the index.root_indexes: set[str] = set()unusable_root_indexes: set[str] = set()for suffix, canonical in _LOADER_WEIGHT_NAMES["base"].items():@@ -1153,9 +1148,8 @@ def _index_cannot_serve_its_shards(index_path: Path, family_files: set[str]) ->with index_path.open(encoding = "utf-8") as handle:index = json.load(handle)except (OSError, UnicodeDecodeError, ValueError, RecursionError):- # RecursionError (deeply nested json) escapes every caller's fail-open guard. The loader- # parses this index with the same json module, so one too deep to parse there cannot serve- # its shards here either.+ # RecursionError escapes every caller's fail-open guard, and the loader parses this index with the+ # same json module, so one too deep to parse there cannot serve its shards here either.return Trueweight_map = index.get("weight_map") if isinstance(index, dict) else Noneif not isinstance(weight_map, dict) or not weight_map:@@ -1166,16 +1160,15 @@ def _index_cannot_serve_its_shards(index_path: Path, family_files: set[str]) ->if not isinstance(shard, str) or not shard:return Trueparts = PurePosixPath(shard.replace("\\", "/"))- # is_absolute() is per flavour: PurePosixPath reads "C:/weights/x.safetensors" as a relative- # subdirectory called "C:", but the join below is a platform Path, so on Windows that name- # replaces the index directory outright. A drive is never part of a shard name.+ # is_absolute() is per flavour: PurePosixPath reads "C:/weights/x.safetensors" as a relative "C:"+ # subdirectory, but the join below is a platform Path, so on Windows that name replaces the+ # index directory outright.windows = PureWindowsPath(shard)if parts.is_absolute() or ".." in parts.parts or windows.is_absolute() or windows.drive:return Trueshards.add(parts)named = {shard.name for shard in shards}- # Coverage matters only for the family this index describes: one it names nothing of is stale- # content beside it, which the loader never reads because it opens weight_map and nothing else.+ # Coverage matters only for the family this index describes: the loader opens weight_map and nothing else.if not family_files <= named and not named.isdisjoint(family_files):return Truefor shard in shards:@@ -1187,7 +1180,7 @@ def _index_cannot_serve_its_shards(index_path: Path, family_files: set[str]) ->except (OSError, ValueError):return True# A shard names its own total, so an index listing one of a set has to list the whole set: the- # loader opens exactly what is mapped and silently drops whatever the map leaves out.+ # loader silently drops whatever the map leaves out.declared: dict[tuple[str, str, int], set[int]] = {}for shard in shards:match = _WEIGHT_SHARD_RE.search(shard.name)@@ -1225,15 +1218,14 @@ def _snapshot_lacks_a_complete_weight_family(snapshot_dir: Path) -> bool:# The loader opens this name first and finds it empty. Same exemption as below.return kind == wanted or wanted not in payload.ungroupedif canonical_empty is False:- # Only the row's own kind proves it loads, and it vetoes nothing once that kind's- # payload is here but names no family.+ # Only the row's own kind proves it loads, and it vetoes nothing once that kind's payload is here+ # but names no family.return kind != wanted and wanted not in payload.ungrouped- # from_pretrained reads the snapshot root, so only families named there are judged; a- # subdirectory layout is carried by ungrouped instead.+ # from_pretrained reads the snapshot root, so only families named there are judged; a subdirectory+ # layout is carried by ungrouped instead.if kind == "base" and suffix in payload.root_indexes:- # Selected and loaded for exactly what it names, wherever those paths point, so- # judge its contents rather than this walk's families: a numbered file it names- # nothing of is never read. The next name is never tried.+ # Selected and loaded for exactly what it names, wherever those paths point, so judge its contents+ # rather than this walk's families; the next name is never tried.if suffix in payload.unusable_root_indexes:return kind == wanted or wanted not in payload.ungroupedreturn kind != wanted and wanted not in payload.ungrouped@@ -1586,16 +1578,16 @@ def _current_revisions(repo_info):return revisions-# One definition of "the denoiser is on disk", shared by the repo-wide and the snapshot-scoped-# check so the two cannot drift into disagreeing about the same directory.+# One definition of "the denoiser is on disk", shared by the repo-wide and snapshot-scoped checks so+# the two cannot drift into disagreeing about the same directory._DENOISER_DIRS = ("transformer", "unet")_DENOISER_WEIGHT_SUFFIXES = (".safetensors", ".bin")-# The two names a default load can end up opening at the component root: the safetensors one-# _get_model_file is asked for first, and the .bin the pickle fallback under it drops to.+# The two names a default load can open at the component root: the safetensors one _get_model_file+# asks for first, and the .bin its pickle fallback drops to._DEFAULT_DENOISER_WEIGHTS = frozenset(f"diffusion_pytorch_model{suffix}" for suffix in _DENOISER_WEIGHT_SUFFIXES)-# The one sharded index a default load resolves: use_safetensors unset coerces to True, and+# The one sharded index a default load resolves: use_safetensors unset coerces to True and# _fetch_index_file then builds only _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant), so with# variant unset this exact name. Our load path passes neither (core/inference/{diffusion,video}.py)._SELECTED_DENOISER_INDEX = "diffusion_pytorch_model.safetensors.index.json"@@ -1649,9 +1641,9 @@ def _manifest_denoiser_components(snapshot: Path) -> Optional[tuple[str, ...]]:name = key.lower()if name != "unet" and "transformer" not in name:continue- # A component is a [library, class] pair keyed by its directory; [null, null] means- # deliberately absent (Wan 2.2's 5B transformer_2). Anything else names no directory to- # infer (ACE-STEP maps "transformer" to a config dict), so leave it to the loader.+ # A component is a [library, class] pair keyed by its directory; [null, null] means deliberately+ # absent (Wan 2.2's 5B transformer_2), and anything else names no directory to infer (ACE-STEP+ # maps "transformer" to a config dict).if not isinstance(value, (list, tuple)) or not any(v for v in value):continuefound.append(key)@@ -1668,7 +1660,6 @@ def _denoiser_index_shards(index: Path) -> Optional[set[str]]:with index.open("r", encoding = "utf-8") as fh:weight_map = json.load(fh).get("weight_map")except (OSError, ValueError, AttributeError, RecursionError):- # RecursionError (deeply nested json) would escape the caller's fail-open guard.return Noneif not isinstance(weight_map, dict):return None@@ -1704,28 +1695,25 @@ def _component_weights_complete(component: Path) -> bool:So with no selected index there are exactly two names left, the pair ``_get_model_file`` ishanded, and every other weight in the directory is one ``from_pretrained`` never resolves."""- # iterdir() raises on an unreadable dir, reaching the caller's fail-open guard; glob() would- # swallow that OSError and read as "no weights".+ # iterdir() raises on an unreadable dir, reaching the caller's fail-open guard; glob() would swallow+ # that OSError and read as "no weights".next(component.iterdir(), None)- # Existence alone makes the component sharded (``is_sharded`` comes from ``is_file()``), so an- # index we cannot read is not no-evidence here, it IS the failure. ``is_file()`` is also the- # loader's own test, so a dangling snapshot symlink is absent to both.+ # Existence alone makes the component sharded (is_sharded comes from is_file()), so an index we+ # cannot read IS the failure; is_file() is the loader's own test too.selected = component / _SELECTED_DENOISER_INDEXif selected.is_file():if _index_cannot_serve_its_shards(selected, set()):return False- # The loader opens exactly what ``weight_map`` lists and reads each one as a checkpoint,- # so a map naming something that is not a weight file -- a ``config.json`` a corrupt fetch- # left behind -- fails at load however present that file is.+ # The loader opens exactly what weight_map lists and reads each one as a checkpoint, so a map naming+ # a config.json a corrupt fetch left behind fails at load however present that file is.shards = _denoiser_index_shards(selected)return bool(shards) and all(name.lower().endswith(_DENOISER_WEIGHT_SUFFIXES) for name in shards)- # No index, so the component is not sharded to the loader either, and ``_get_model_file`` is- # asked for the safetensors default and then the ``.bin`` under it and opens nothing else. A- # weight under any other name is one ``from_pretrained`` never resolves: a numbered shard is- # reachable only THROUGH an index, a dtype twin only under a matching ``variant``, and a- # ``model.safetensors`` or adapter sidecar never at all.+ # No index means the component is not sharded to the loader either, and _get_model_file opens only+ # the safetensors default and the .bin under it: a numbered shard is reachable only THROUGH an+ # index, a dtype twin only under a matching variant, and a model.safetensors or adapter sidecar+ # never at all.return any((component / name).is_file() for name in _DEFAULT_DENOISER_WEIGHTS)@@ -1749,14 +1737,14 @@ def snapshot_pipeline_missing_denoiser(snapshot: Optional[Path]) -> bool:root = Path(snapshot)declared = _manifest_denoiser_components(root)if declared is not None:- # all(()) is True: a manifest declaring no denoiser has none to prove absent, so it- # reads complete rather than being hunted for one it never had.+ # all(()) is True: a manifest declaring no denoiser has none to prove absent, so it reads complete+ # rather than being hunted for one it never had.return not all((root / name).is_dir() and _component_weights_complete(root / name)for name in declared)- # Unreadable manifest: either fixed name will do, since a UNet pipeline has no- # transformer/ and a DiT one no unet/.+ # Unreadable manifest: either fixed name will do, since a UNet pipeline has no transformer/ and a+ # DiT one no unet/.return not any((root / name).is_dir() and _component_weights_complete(root / name)for name in _DENOISER_DIRS@@ -2041,9 +2029,8 @@ def partial_resume_available(if partial_transport_for(repo_type, repo_id, variant, repo_cache_dir) != "http":return False- # Same root the transport was read from. A row can be displayed from a remembered, legacy- # or custom cache, and both halves have to answer about THAT directory: the active root- # neither holds its partials nor shares its manifest scope.+ # Same root the transport was read from: a row can be displayed from a remembered, legacy or custom+ # cache, and the active root neither holds its partials nor shares its manifest scope.return download_registry.is_resumable_partial(repo_type,repo_id,diff --git a/studio/backend/hub/utils/paths.py b/studio/backend/hub/utils/paths.pyindex d36a0e8468a..dd887575092 100644--- a/studio/backend/hub/utils/paths.py+++ b/studio/backend/hub/utils/paths.py@@ -18,8 +18,8 @@ from loggers import get_loggerfrom utils.paths import path_utils as _path_utilsfrom utils.paths.path_utils import wsl_automount_root-# One policy, defined in utils.paths.storage_roots. The copy that used to live here-# drifted: a BOM'd settings.json was honoured by one side and dropped by the other (#9748).+# One policy, defined in utils.paths.storage_roots: the copy that used to live here drifted, and a+# BOM'd settings.json was honoured by one side and dropped by the other (#9748).from utils.paths.storage_roots import (lmstudio_model_dirs,ollama_model_dirs,@@ -111,8 +111,7 @@ def hf_default_cache_dir() -> Path:return Path.home() / ".cache" / "huggingface" / "hub"-# normalize_path reads these at call time and tests set them-# (test_gguf_variants_local_resolution), so they stay attributes of this module.+# normalize_path reads these at call time and tests set them, so they stay attributes of this module._IS_WSL = _path_utils._IS_WSL_WSL_AUTOMOUNT_ROOT = wsl_automount_root()@@ -165,8 +164,8 @@ def is_valid_repo_id(repo_id: str) -> bool:segments = repo_id.split("/")if len(segments) not in (1, 2):return False- # Match huggingface_hub.validate_repo_id: the 96-char limit applies per segment (repo name /- # namespace), not to the whole "namespace/repo_name" string.+ # Match huggingface_hub.validate_repo_id: the 96-char limit applies per segment, not to the whole+ # "namespace/repo_name" string.return all(segment not in ("", ".", "..")and len(segment) <= _MAX_REPO_ID_LENGTH@@ -196,7 +195,7 @@ def is_valid_gguf_variant(variant: str) -> bool:return all(segment not in ("", ".", "..") for segment in normalized.split("/"))-# Per-process memo for resolve_cached_repo_id_case. Bounded LRU so a long-lived process can't+# Per-process memo for resolve_cached_repo_id_case. Bounded LRU so a long-lived process cannot# grow it without limit; evicted cold entries simply recompute._CACHE_CASE_RESOLUTION_MEMO_MAX = 512_CACHE_CASE_RESOLUTION_MEMO: "OrderedDict[tuple[str, str], str]" = OrderedDict()@@ -288,8 +287,8 @@ def resolve_dataset_path(path_value: str) -> Path:raw = str(path_value or "").strip()if "\x00" in raw:raise ValueError("dataset path may not contain null bytes")- # Normalize first so Windows/UNC and backslash paths resolve like the rest of the Hub path- # layer, and a backslashed '..' is caught by the traversal guard below.+ # Normalize first so Windows/UNC and backslash paths resolve like the rest of the Hub path layer,+ # and a backslashed ".." is caught by the traversal guard below.normalized = normalize_path(raw)path = Path(normalized).expanduser()if ".." in path.parts:@@ -375,8 +374,8 @@ def resolve_cached_repo_id_case(continueif entry.name.lower() != expected_lower:continue- # The lowercased full-name match already proves the prefix matches; a case-sensitive- # startswith would reject a mixed-case imported dir such as Models--Org--Repo.+ # The lowercased full-name match already proves the prefix matches; a case-sensitive startswith+ # would reject a mixed-case imported dir such as Models--Org--Repo.repo_part = entry.name[len(prefix) :]if not repo_part:continuediff --git a/studio/backend/hub/utils/resumable_partials.py b/studio/backend/hub/utils/resumable_partials.pyindex f7a62c6de40..52a790bd3bf 100644--- a/studio/backend/hub/utils/resumable_partials.py+++ b/studio/backend/hub/utils/resumable_partials.py@@ -58,16 +58,14 @@ LAST_STOCK_RESUMABLE_VERSION = (1, 17)# The newest major whose internals this has been read against. A 2.x is not assumed to look alike.MAX_SUPPORTED_MAJOR = 1-# What flock reports when another holder has the lock, and nothing else. EACCES belongs to fcntl,-# not flock; ENOLCK and EOPNOTSUPP mean this filesystem cannot lock at all.+# What flock reports when another holder has the lock, and nothing else: EACCES belongs to fcntl,+# and ENOLCK / EOPNOTSUPP mean this filesystem cannot lock at all._CONTENDED = frozenset({errno.EWOULDBLOCK, errno.EAGAIN})-# Filesystems backed by storage attached to this host, so a lock taken here is the only lock.# An allowlist rather than a list of network types: a FUSE mount reports whatever name its daemon-# chose, and unless it negotiates FUSE_FLOCK_LOCKS the kernel answers flock locally (libfuse-# fuse_lowlevel.h), so fuse.rclone or fuse.s3fs over shared object storage passes the probe while-# excluding nobody. Naming the ones we know instead means an unrecognised or blank type keeps the-# stock writer rather than silently re-enabling a shared one.+# chose and, unless it negotiates FUSE_FLOCK_LOCKS, the kernel answers flock locally, so+# fuse.rclone or fuse.s3fs over shared object storage would pass a network-name test. An+# unrecognised or blank type keeps the stock writer._LOCAL_FSTYPES = frozenset({# Linux@@ -156,8 +154,8 @@ def _probe_dir(hub_cache: Optional[Path | str] = None) -> Optional[Path]:logger.debug("resumable partials: no hub cache to probe (%s)", exc)return Noneif hub_cache is not None:- # Asked about a named root, so only report on one that is there. Creating it would- # resurrect a cache the user detached, and an absent root holds no partials to judge.+ # Asked about a named root, so only report on one that is there: creating it would resurrect a cache+ # the user detached, and an absent root holds no partials to judge.return root if root.is_dir() else Nonetry:root.mkdir(parents = True, exist_ok = True)@@ -205,7 +203,7 @@ def _filesystem_is_local_on(directory: str, device: int) -> bool:"""path = Path(directory).resolve()if str(path).startswith("\\\\") or str(path).startswith("//"):- return False # UNC share+ return Falsetry:table = _mounts()except Exception as exc: # noqa: BLE001 - unreadable now does not mean unreadable next time@@ -234,8 +232,8 @@ def _lock_is_honoured_at(directory: str) -> bool:return _lock_is_honoured_on(directory, _device_at(directory))-# Keyed on the directory and the device, so moving the cache, or swapping the mount under it,-# re-probes instead of reusing a verdict about a filesystem that is no longer there.+# Keyed on the directory and the device, so moving the cache or swapping the mount under it re-+# probes instead of reusing a verdict about a filesystem that is gone.@lru_cache(maxsize = 8)def _lock_is_honoured_on(directory: str, device: int) -> bool:"""Take the lock twice and require the second to be refused.@@ -247,8 +245,8 @@ def _lock_is_honoured_on(directory: str, device: int) -> bool:"""import fcntl- # A random, exclusively created file: the cache can be shared, and a predictable name there- # lets another user pre-place a symlink that an unguarded open would follow and truncate.+ # A random, exclusively created file: the cache can be shared, and a predictable name lets another+ # user pre-place a symlink an unguarded open would follow and truncate.try:handle, name = tempfile.mkstemp(dir = directory, prefix = ".unsloth-flock-probe.")except Exception as exc: # noqa: BLE001 - nowhere to probe now is not nowhere to probe later@@ -300,16 +298,13 @@ def _exclusion_is_provable(hub_cache: Optional[Path | str] = None) -> bool:try:import fcntl # noqa: F401except ImportError:- # No fcntl on Windows, where huggingface_hub locks via msvcrt: mandatory rather than- # advisory. A network share still cannot be spoken for, which the locality check catches.- # Windows has no way to establish who owns a partial: os.stat reports st_uid 0 for every- # file, and reading an ACL needs pywin32, which is not a dependency here. Without an owner- # check, another account on a shared NTFS cache could leave a partial with a chosen prefix- # and have the remaining range appended to it, which the size-only check downstream would- # pass. So the shared name stays off here and Windows keeps the stock writer.+ # No fcntl on Windows, where huggingface_hub locks via msvcrt, and Windows has no way to establish+ # who owns a partial (os.stat reports st_uid 0 for every file and reading an ACL needs pywin32),+ # so another account on a shared NTFS cache could leave a partial with a chosen prefix and have+ # the remaining range appended to it, which the size-only check would pass.return False- # _ProbeUnavailable is deliberately not caught: a probe that could not run is not an answer,- # and letting it out keeps any caller from caching one. Callers turn it into "not proven".+ # _ProbeUnavailable is deliberately not caught: a probe that could not run is not an answer, and+ # letting it out keeps any caller from caching one.return _filesystem_is_local(str(directory)) and _lock_is_honoured_at(str(directory))@@ -356,8 +351,7 @@ def _objection_to(descriptor: int) -> Optional[str]:return "hard linked from elsewhere"euid = getattr(os, "geteuid", None)if euid is None:- # No owner to compare against, so nothing here can be vouched for. Reachable only if the- # writer is ever enabled without geteuid; _exclusion_is_provable refuses that today.+ # No owner to compare against, so nothing here can be vouched for.return "on a platform where ownership cannot be established"if info.st_uid != euid():return "owned by another user"@@ -419,10 +413,8 @@ def _open_stable_partial(path: Path) -> Optional[Any]:if exc.errno in (errno.ELOOP, errno.EMLINK):objection = "a symlink"else:- # Anything else is a partial this account cannot use and must not judge: a- # 0600 one left by another user answers EACCES, a directory in its place- # EISDIR. Raising would fail every attempt at the blob for good, while the- # stock writer creates a file of its own and does not need this one at all.+ # Anything else is a partial this account cannot use and must not judge (another user's 0600 file+ # answers EACCES, a directory EISDIR); raising would fail every attempt at the blob for good.logger.warning("Cannot open the download partial at %s (%s); leaving it alone and ""letting the stock writer fetch the file.",@@ -451,8 +443,8 @@ def restore_resumable_partials() -> bool:try:permitted = can_restore_partials()except _ProbeUnavailable as exc:- # The worker calls this at import, so an escaping exception would take the whole download- # process down instead of leaving it with the stock writer it can always fall back on.+ # The worker calls this at import, so an escaping exception would take the whole download process+ # down instead of leaving it the stock writer.logger.debug("resumable partials: %s", exc)return Falseif not permitted:@@ -478,8 +470,8 @@ def restore_resumable_partials() -> bool:if destination_path.exists() and not force_download:return- # A XET-backed repo still comes down over HTTP when hf_xet is absent or disabled, so what- # matters is whether XET will run, not whether its metadata exists. XET writes its own way.+ # A XET-backed repo still comes down over HTTP when hf_xet is absent or disabled, so what matters is+ # whether XET will run, not whether its metadata exists.uses_xet = xet_file_data is not None and file_download.is_xet_available()if force_download or uses_xet:return stock(@@ -512,10 +504,8 @@ def restore_resumable_partials() -> bool:with opened as handle:resume_size = handle.tell()if expected_size is not None and resume_size > expected_size:- # Longer than the file is supposed to be, so there is nothing to resume from: a- # Range starting past the end answers 416 and would do so on every retry, where- # the stock writer's fresh name recovers. Truncating through the handle we already- # validated, rather than unlinking, keeps the name from being swapped underneath.+ # Longer than the file is supposed to be, so there is nothing to resume from: a Range starting past+ # the end answers 416 on every retry.logger.warning("Restarting '%s': the partial holds %s bytes but the file is %s.",filename,@@ -543,9 +533,8 @@ def restore_resumable_partials() -> bool:expected_size = expected_size,tqdm_class = kwargs.get("tqdm_class"),)- # _chmod_and_move resolves the name again, so publish only if the name still holds the- # file that was actually written. Otherwise another account could swap something in- # after the last write and have it installed as the blob under a verified descriptor.+ # _chmod_and_move resolves the name again, so publish only if the name still holds the file that was+ # actually written; otherwise another account could swap something in after the last write.if not _still_the_written_file(incomplete_path, written):logger.warning("Not publishing '%s': the partial at %s was replaced while it was being written.",diff --git a/studio/backend/hub/utils/snapshot_filters.py b/studio/backend/hub/utils/snapshot_filters.pyindex 67f5b47bdcb..65ea85552f6 100644--- a/studio/backend/hub/utils/snapshot_filters.py+++ b/studio/backend/hub/utils/snapshot_filters.py@@ -44,8 +44,8 @@ def _size(sibling) -> int:def repo_ships_transformers_weights(filenames: Iterable[str]) -> bool:- # A "._consolidated.safetensors" does not start with "consolidated", so it would answer yes- # here and then have every consolidated* file stripped from the download.+ # A "._consolidated.safetensors" does not start with "consolidated", so it would answer yes here and+ # then have every consolidated* file stripped from the download.for name in drop_shadowed_appledouble_names(list(filenames)):base = name.rsplit("/", 1)[-1].lower()if base.startswith("consolidated"):@@ -94,9 +94,8 @@ def total_size_for_siblings(siblings: Iterable) -> int:def blob_hashes_for_siblings(siblings: Iterable) -> frozenset[str]:- # Blob filename == file etag (LFS sha256, else git blob id). Collecting both- # lets progress count exactly this revision's files without summing stale- # blobs from other revisions.+ # Blob filename == file etag (LFS sha256, else git blob id), so collecting both lets progress count+ # exactly this revision's files without summing stale blobs from other revisions.hashes: set[str] = set()for sibling in siblings:sha = getattr(getattr(sibling, "lfs", None), "sha256", None)diff --git a/studio/backend/hub/utils/state_dir.py b/studio/backend/hub/utils/state_dir.pyindex 45cf9950048..35ad21e918f 100644--- a/studio/backend/hub/utils/state_dir.py+++ b/studio/backend/hub/utils/state_dir.py@@ -57,8 +57,7 @@ _MAX_VARIANT_FRAGMENT_LENGTH = 64_CACHE_SCOPE_DIGEST_LENGTH = 32_HASH_PREFIXES = ("sha256-", "@sha256-")_LEGACY_HASH_FRAGMENT = re.compile(r"sha256-[0-9a-f]{32}")-# Either tag _variant_fragment stamps, so a reader can spot a digest handed to it-# in place of a variant name.+# Either tag _variant_fragment stamps, so a reader can spot a digest handed to it in place of a variant name._HASHED_FRAGMENT = re.compile(r"@?sha256-[0-9a-f]{32}")@@ -91,9 +90,8 @@ def _subdir(name: str, *, create: bool = False) -> Optional[Path]:def repo_cache_basename(repo_type: RepoType, repo_id: str) -> str:- # Reject a bad repo_type at runtime: a wrong value would silently produce a- # wrong filename and a misclassified scanner row (the Literal only guards- # statically; dynamic/JSON-sourced values slip past it).+ # Reject a bad repo_type at runtime: the Literal only guards statically and dynamic/JSON-sourced+ # values slip past it, silently producing a wrong filename and a misclassified scanner row.if repo_type not in _VALID_REPO_TYPES:raise ValueError(f"repo_type must be one of {_VALID_REPO_TYPES}, got {repo_type!r}")return f"{repo_type}s--{repo_id.replace('/', '--')}".lower()@@ -187,8 +185,8 @@ def _variant_fragment(legacy_hash_key: bool = False,) -> str:normalized_variant = variant.strip().lower()- # Double-hyphen variants can alias a repository component plus the- # ``--variant--`` delimiter, so give them an injective hashed fragment.+ # Double-hyphen variants can alias a repository component plus the --variant-- delimiter, so give+ # them an injective hashed fragment.if _SAFE_VARIANT_FRAGMENT.fullmatch(normalized_variant) and (legacy_variant_keyor ("--" not in normalized_variant and not normalized_variant.startswith(_HASH_PREFIXES))@@ -271,12 +269,9 @@ def normalize_hub_cache(hub_cache: str | Path) -> str:try:resolved = str(Path(hub_cache).expanduser().resolve(strict = False))except (OSError, RuntimeError, ValueError):- # Windows can refuse to resolve a path it can still open (OneDrive- # placeholders, a locked junction). Degrade to the expanded spelling- # rather than dropping the scope. It has to be exactly what- # legacy_cache_scope_name below builds, or that read-side counterpart- # would not recover the state this branch writes -- so expanduser here- # too, and fall back again if even that is unavailable.+ # Windows can refuse to resolve a path it can still open (OneDrive placeholders, a locked+ # junction), so degrade to the expanded spelling; it has to be exactly what+ # legacy_cache_scope_name builds, or the read side cannot recover this state.try:resolved = str(Path(hub_cache).expanduser())except (OSError, RuntimeError, ValueError):@@ -307,9 +302,8 @@ def legacy_cache_scope_name(hub_cache: str | Path) -> str:try:normalized = os.path.normcase(str(Path(hub_cache).expanduser()))except (OSError, RuntimeError, ValueError):- # Guarded like normalize_hub_cache, and for the same reason: this is now- # fed the caller's raw spelling, so a homeless "~" that expanduser- # refuses would otherwise escape a plain read as a RuntimeError.+ # Guarded like normalize_hub_cache: this is fed the caller's raw spelling, so a homeless "~" that+ # expanduser refuses would otherwise escape a plain read as a RuntimeError.normalized = os.path.normcase(str(hub_cache))return _cache_scope_digest(normalized)@@ -337,9 +331,7 @@ def _cache_scope() -> Optional[Path]:if hub_cache is None:return parent- # A precomputed scope skips re-deriving the digest -- and with it the- # ``resolve`` inside it -- once per probe when a caller fans one cache path- # out across the filename-migration spellings.+ # A precomputed scope skips re-deriving the digest, and the resolve inside it, once per probe.scoped = parent / (cache_scope or cache_scope_name(hub_cache))if not create:return scopeddiff --git a/studio/backend/hub/workers/hf_download.py b/studio/backend/hub/workers/hf_download.pyindex b66c1e2249f..be44372e096 100644--- a/studio/backend/hub/workers/hf_download.py+++ b/studio/backend/hub/workers/hf_download.py@@ -60,16 +60,16 @@ from hub.utils.gguf_plan import (from hub.utils.state_dir import RepoTypefrom hub.utils.resumable_partials import restore_resumable_partials-# Put huggingface_hub's 1.17 HTTP writer back where it is safe to. The SIGKILL then restart loop-# documented above reads ``.incomplete`` for its resume offset, and 1.18+ leaves nothing to read.+# Put huggingface_hub's 1.17 HTTP writer back: the SIGKILL then restart loop reads .incomplete for+# its resume offset, and 1.18+ leaves nothing to read._PARTIALS_RESUMABLE = restore_resumable_partials()# typing.Union, not `str | bool | None`: an alias is evaluated on import and PEP 604 raises below 3.10.HfTokenArg = Union[str, bool, None]-# Bound the metadata fetch so a stalled connection fails the worker (exit 1) instead of hanging-# at 0%. The file download itself is governed by huggingface_hub's own timeout.+# Bound the metadata fetch so a stalled connection fails the worker instead of hanging at 0%; the+# file download itself is governed by huggingface_hub's own timeout._METADATA_REQUEST_TIMEOUT = 10.0_METADATA_RETRY_TIMEOUT = 30.0_METADATA_RETRY_DELAY = 1.0@@ -154,10 +154,8 @@ def _parent_is_alive(parent_pid: int) -> bool:def _terminate_orphaned_self() -> None:- # Hard exit from the watchdog thread: a self-SIGTERM would be deferred while the main thread is- # GIL-blocked in a C socket read. The partial .incomplete resumes byte-exact and marker/manifest- # writes are atomic, so code 130 is safe. The diagnostic is best-effort: a dead parent's closed- # stderr pipe can raise BrokenPipeError, which must never preempt the exit.+ # Hard exit from the watchdog thread: a self-SIGTERM would be deferred while the main thread is GIL-+ # blocked in a C socket read, and the partial resumes byte-exact with atomic marker writes.try:print("Parent process exited; stopping orphaned download worker.",@@ -239,8 +237,8 @@ def _dataset_info_with_retry(repo_id: str, hf_token: str | None):)-# Tied to drain_stderr_excerpt's 500-byte head/tail window in the parent: listing every expected-# file would blow past it and lose the diagnostic, so cap the preview.+# Tied to drain_stderr_excerpt's 500-byte head/tail window: listing every expected file would blow+# past it and lose the diagnostic._VERIFY_PATH_LIST_CAP = 10@@ -554,9 +552,8 @@ def _download_snapshot(repo_id: str, hf_token: str | None, mode: str) -> None:from hub.utils.download_registry import prepare_cache_for_transportfrom hub.utils import download_manifest- # One metadata fetch powers both the ignore-pattern decision (drop consolidated.* when- # transformers weights exist) and the manifest's expected_files. A failure is non-fatal: fall- # back to the legacy ignore set and skip the manifest, losing verification but not the download.+ # One metadata fetch powers both the ignore-pattern decision and the manifest's expected_files; a+ # failure is non-fatal and falls back to the legacy ignore set, losing verification only.try:info = _model_info_with_retry(repo_id, hf_token)except Exception as e:@@ -570,9 +567,8 @@ def _download_snapshot(repo_id: str, hf_token: str | None, mode: str) -> None:download_manifest.clear_cancel_marker("model", repo_id, None)if info is not None:ignore_patterns, expected_files = _snapshot_download_plan(info)- # Written for every transport. The manifest verifies the finalized files under snapshots/, which- # both transports produce identically (XET also renames a full, correctly-sized blob into place).- # XET's block-level dedup lives only in the chunk-cache, so per-file size verification is valid.+ # The manifest verifies the finalized files under snapshots/, which both transports produce+ # identically; XET's block-level dedup lives only in the chunk cache.download_manifest.write_manifest("model", repo_id, None, expected_files, mode)else:ignore_patterns = list(SNAPSHOT_IGNORE_PATTERNS)@@ -624,9 +620,9 @@ def _gguf_variant_target_plan(raise RuntimeError(f"Metadata unavailable while resolving GGUF variant '{variant}' " f"for {repo_id}") from e- # plan_for_variant, not .get: a repo that files every variant under one shared container- # qualifies every key, and a stored pin or an explicit repo:Q4_K_M then missed the map and- # the worker exited with "No GGUF shards matching variant".+ # plan_for_variant, not .get: a repo filing every variant under one shared container qualifies+ # every key, so a stored pin or an explicit repo:Q4_K_M missed the map and the worker exited with+ # "No GGUF shards matching variant".return plan_for_variant(build_gguf_variant_plans(list(info.siblings)), variant)@@ -662,8 +658,8 @@ def _download_gguf_variant(repo_id: str, variant: str, hf_token: str | None, modmode,)else:- # Metadata unreachable (offline / gated / private). Resume the exact shards the original attempt- # recorded so snapshot_download can range over the surviving .incomplete blobs.+ # Metadata unreachable: resume the exact shards the original attempt recorded so snapshot_download+ # can range over the surviving .incomplete blobs.manifest = download_manifest.read_manifest("model", repo_id, variant)if manifest is None or not manifest.expected_files:print(@@ -707,8 +703,8 @@ def _download_gguf_variant(repo_id: str, variant: str, hf_token: str | None, mod"hashes; starting without partial cache reuse.",file = sys.stderr,)- # Main quant blobs are owned by this variant (variant-scoped marker). The shared vision companion- # (mmproj) has its own marker and is never purged while a concurrent peer is writing it.+ # Main quant blobs are owned by this variant; the shared mmproj companion has its own marker and is+ # never purged while a concurrent peer is writing it.purged = prepare_cache_for_transport("model",repo_id,@@ -830,8 +826,8 @@ def _download_scoped_snapshot(max_workers = 1,)if info is None:- # With no metadata there is no manifest, so verification is a no-op, and snapshot_download RETURNS- # AN EXISTING SNAPSHOT FOLDER when repo_info also fails -- flipping the job to complete with no weights.+ # With no metadata there is no manifest, and snapshot_download RETURNS AN EXISTING SNAPSHOT FOLDER+ # when repo_info also fails, flipping the job to complete with no weights.root = Path(snapshot_path)absent = tuple(f for f in files if not (root / f).exists())if absent:@@ -1002,9 +998,9 @@ def main() -> None:except SystemExit:raiseexcept Exception as e:- # Surface a precise message so the UI doesn't show a generic "worker exited with code 1".- # huggingface_hub recommends force_download=True to recover, which our "Restart" UI maps to a- # fresh start by purging the partial via prepare_cache_for_transport.+ # Surface a precise message rather than a generic "worker exited with code 1": huggingface_hub+ # recommends force_download=True to recover, which our Restart maps to purging the partial via+ # prepare_cache_for_transport.print(f"{type(e).__name__}: {e}", file = sys.stderr)sys.exit(1)diff --git a/studio/backend/lan_access.py b/studio/backend/lan_access.pyindex 11bb319f65e..acc3e41719f 100644--- a/studio/backend/lan_access.py+++ b/studio/backend/lan_access.py@@ -44,8 +44,8 @@ _START_TIMEOUT = 10.0# kept under the ~5s Windows console-close budget run.py's shutdown path works to_STOP_TIMEOUT = 3.0-# a LAN request already accepted can run for minutes, and it stays a remote caller-# for all of them; on expiry the trust flag is left active rather than downgraded+# a LAN request already accepted can run for minutes and stays a remote caller throughout; on expiry+# the trust flag is left active rather than downgraded_DRAIN_TIMEOUT = 300.0# networking mode rarely changes, but a failed wslinfo probe must eventually recover@@ -62,8 +62,8 @@ _serve_loop: Any = None_sockets: tuple[socket.socket, ...] = ()_port: Optional[int] = None_error: Optional[str] = None-# stopped listeners whose accepted requests are still running; they remain remote-# callers, so the trust flag stays up until every one of them has drained+# stopped listeners whose accepted requests are still running remain remote callers, so the trust+# flag stays up until every one has drained_pending_drains = 0# rebound whole, never mutated: request_on_lan_listener reads it without the lock_bound_addresses: tuple[str, ...] = ()@@ -77,10 +77,8 @@ def detect_lan_addresses(ip_version: int = 4) -> list[str]:-- a cloud VM binding its own public IP is the same operation as a laptopbinding its Wi-Fi address, and the caller decides whether that is wanted."""- # WSL's NAT-side address belongs to a private Hyper-V network, not the- # physical LAN. A second device cannot open it directly. Mirrored mode is- # different: WSL participates in the host's network and its addresses can be- # reached subject to the host firewall.+ # WSL's NAT-side address belongs to a private Hyper-V network a second device cannot open; mirrored+ # mode is different, since WSL joins the host's networkif _wsl_networking_mode() not in (None, "mirrored"):return []@@ -123,8 +121,8 @@ def detect_lan_addresses(ip_version: int = 4) -> list[str]:if probe is not None:probe.close()- # every other adapter that is up: a route probe picks one source address, and- # an isolated LAN has no route at all, so neither it nor the hostname enumerates them+ # a route probe picks one source address and an isolated LAN has no route at all, so neither it nor+ # the hostname enumerates the other adaptersfor address in _interface_addresses(ip_version):_add(address)return addresses@@ -322,8 +320,8 @@ def start_lan_listener(logger.info("LAN access skipped unbindable addresses: %s", "; ".join(failures))server = uvicorn.Server(_listener_config(app, bound[0], port))- # published before the socket can accept: a request served in between would- # still read the loopback-only trust defaults+ # published before the socket can accept: a request served in between would still read the loopback-+ # only trust defaultsset_lan_connector_active(True)serving = server.serve(sockets = sockets)try:@@ -434,8 +432,8 @@ def stop_lan_listener() -> bool:"""global _server, _serve_loop, _sockets, _bound_addresses, _port, _error- # a start holds _lock while waiting for this loop to run serve(), so a stop that- # arrives on the loop itself must not block on it or the two wait each other out+ # a start holds _lock while waiting for this loop to run serve(), so a stop arriving on the loop+ # itself must not block on itif not _lock.acquire(blocking = not _running_on_event_loop()):logger.info("LAN access stop deferred: a listener change is in flight")return False@@ -451,9 +449,8 @@ def stop_lan_listener() -> bool:return Trueserver.should_exit = Trueif _running_on_event_loop():- # /api/shutdown tears down from a task on this very loop; waiting would deadlock.- # ownership is kept because uvicorn cannot close the sockets until the loop is- # free again, and _graceful_shutdown blocks it for seconds stopping subprocesses+ # /api/shutdown tears down from a task on this very loop, so waiting would deadlock; ownership is+ # kept because uvicorn cannot close the sockets until the loop is free againlogger.info("LAN access stopping")return Trueif loop is None or loop.is_closed() or not loop.is_running():@@ -468,8 +465,8 @@ def stop_lan_listener() -> bool:_release_listener_state()logger.info("LAN access stopped")return True- # ownership is kept so a retry waits on these same sockets, and so a second- # stop cannot report success while the port may still be accepting+ # ownership is kept so a retry waits on these same sockets, and so a second stop cannot report+ # success while the port may still be accepting_error = "stop_timed_out"logger.warning("LAN access did not release port %s within %ss", port, _STOP_TIMEOUT)return Falsediff --git a/studio/backend/loggers/config.py b/studio/backend/loggers/config.pyindex 3e322af4d3d..22376597d06 100644--- a/studio/backend/loggers/config.py+++ b/studio/backend/loggers/config.py@@ -34,16 +34,11 @@ def _env_int(name: str, default: int) -> int:return default-# Cap on the rendered traceback in a single log record. A traceback is normally a few-# KB, but an exception whose message embeds a request body is not: a binary upload-# rejected by request validation produced one 2.2 MB line. Keep the head (where the-# raising frame is) and the tail (where the actual exception type and message are),-# and say how much was dropped. 0 disables the cap.+# An exception whose message embeds a request body is not a few KB: a rejected binary upload+# produced one 2.2 MB line._MAX_EXC_CHARS = _env_int("UNSLOTH_STUDIO_MAX_EXCEPTION_CHARS", 16384)_EXC_TAIL_CHARS = 2048-# The middleware logs the same exception twice: once rendered as a traceback under-# "exception" and once as str(exc) under "error". Capping only the first still lets an-# exception whose message embeds the request body through, so bound both.+# The middleware logs the same exception twice, rendered as a traceback and as str(exc), so bound both._MAX_ERROR_CHARS = 2048@@ -77,14 +72,13 @@ def truncate_exception(event_dict: dict) -> dict:if isinstance(error, str):event_dict["error"] = _truncate_middle(error, message_cap, _EXC_TAIL_CHARS)# f-string call sites interpolate the exception straight into the message- # (routes/inference.py: logger.error(f"...: {e}", exc_info = True)), so the event- # itself is a third copy that can carry the whole payload.+ # (routes/inference.py: logger.error(f"...: {e}", exc_info = True)), so the event itself is a+ # third copy that can carry the whole payload.event = event_dict.get("event")if isinstance(event, str):event_dict["event"] = _truncate_middle(event, message_cap, _EXC_TAIL_CHARS)- # logger.error("stream error: %s", exc) keeps the exception under positional_args,- # and the chain has no PositionalArgumentsFormatter, so the renderer stringifies it- # untouched. Render and cap it here instead.+ # logger.error("...: %s", exc) keeps the exception under positional_args and the chain has no+ # PositionalArgumentsFormatter, so render and cap it here instead.args = event_dict.get("positional_args")if isinstance(args, (list, tuple)) and args:event_dict["positional_args"] = [@@ -110,25 +104,17 @@ def _plain_tracebacks_enabled() -> bool:)-# Prefix on every echoed line. NOT whitespace: RFC 8259 lets a parser skip leading-# space/tab, so json.loads(' {"event": ...}') SUCCEEDS and a request-derived exception-# message could forge a record (CWE-117). "| " cannot begin a JSON value, so no echoed-# line parses as one, and a records-only reader can drop them on the prefix alone.+# NOT whitespace: RFC 8259 lets a parser skip leading space/tab, so json.loads(' {"event": ...}')+# SUCCEEDS and a request-derived exception message could forge a record (CWE-117); "| " cannot+# begin a JSON value._TRACEBACK_ECHO_PREFIX = "| "-# Unicode's Bidi_Control set (PropList.txt), exactly what the UAX #9 algorithm acts on and-# what UTR #36 / Trojan Source (CVE-2021-42574) name. json.dumps already escapes these-# (ensure_ascii), so only the echo would emit them raw.-#-# Deliberately NOT all of category Cf: U+200B-200D, U+00AD and U+FEFF occur in ordinary-# text (ZWNJ in Persian/Arabic, ZWJ in emoji) and reorder nothing.-_BIDI_CONTROLS = frozenset(- "" # ARABIC LETTER MARK- "" # LEFT-TO-RIGHT / RIGHT-TO-LEFT MARK- "" # LRE, RLE, PDF, LRO, RLO- "" # LRI, RLI, FSI, PDI-)+# Unicode's Bidi_Control set (PropList.txt), exactly what UAX #9 acts on and what UTR #36 /+# Trojan Source (CVE-2021-42574) name; json.dumps already escapes these (ensure_ascii), so only+# the echo would emit them raw. Deliberately NOT all of category Cf: U+200B-200D, U+00AD and+# U+FEFF occur in ordinary text (ZWNJ in Persian/Arabic, ZWJ in emoji) and reorder nothing.+_BIDI_CONTROLS = frozenset("")def _escape_unprintable(text: str) -> str:@@ -207,8 +193,8 @@ def _cap_echoed_lines(lines: list[str], limit: int) -> str:used = 0for line in reversed(lines[len(head) :]):if used + len(line) + 1 > tail_budget:- # Cut the boundary line rather than drop it: losing a traceback's last line- # leaves the reader every frame and no reason.+ # Cut the boundary line rather than drop it: losing a traceback's last line leaves the reader every+ # frame and no reason.room = tail_budget - used - 1if room > 0:tail.append(line[:room])@@ -265,17 +251,16 @@ def with_readable_traceback(renderer):return _render-# Set alongside HF_HUB_DISABLE_PROGRESS_BARS when the value is Unsloth's default rather-# than the operator's, so allow_progress_bars() can tell them apart.+# Set alongside HF_HUB_DISABLE_PROGRESS_BARS when the value is Unsloth's default rather than the+# operator's, so allow_progress_bars() can tell them apart._PROGRESS_BARS_DEFAULTED = "UNSLOTH_STUDIO_PROGRESS_BARS_DEFAULTED"-# huggingface_hub's own spelling of truth (utils/_runtime.py ENV_VARS_TRUE_VALUES),-# so "off" and "no" mean "keep the bars" here exactly as they do there.+# huggingface_hub's own spelling of truth (utils/_runtime.py ENV_VARS_TRUE_VALUES), so "off" and+# "no" mean "keep the bars" here exactly as they do there._ENV_TRUE = frozenset({"1", "on", "yes", "true"})-# Set once this process has deliberately taken its bars back (the export worker draws-# them, the training worker reads them). quiet_third_party_progress_bars() then stops-# being a switch a later call can flip the other way.+# Set once this process has deliberately taken its bars back, so quiet_third_party_progress_bars()+# stops being a switch a later call can flip the other way._BARS_RESTORED = False@@ -323,6 +308,7 @@ def _silence_datasets_bar_output() -> None:stream keeps the counter (and the status) alive while the log stays clean."""if "datasets" not in sys.modules:+ # Operator asked to keep them; leave every library alone.returntry:from datasets.utils.tqdm import tqdm as bar_cls@@ -386,7 +372,6 @@ def keep_progress_bars_countable() -> None:"""value = os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS")if value is None or not _env_is_true(value):- # Nothing quieted them here: --verbose, or an operator who asked to keep them.returnif not os.environ.get(_PROGRESS_BARS_DEFAULTED):# The operator turned them off; that is not ours to undo.@@ -452,42 +437,35 @@ def quiet_third_party_progress_bars() -> None:about to replace with its transformers sidecar. `--verbose` skips it entirely."""if _BARS_RESTORED:- # This process took its bars back on purpose: the export worker shows them, the- # training worker reads them out of tqdm._instances (where a disabled bar is- # never registered) and has already redirected their output.+ # This process took its bars back on purpose: the training worker reads them out of tqdm._instances,+ # where a disabled bar is never registered, and has already redirected their output.returnif _verbose_logging_requested() and os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS") is None:- # --verbose promises everything back, so it must not install this default- # either; the flag is inherited by the workers, which would stay quiet.+ # --verbose promises everything back, so it must not install this default either; the flag is+ # inherited by the workers.returnif os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS") is None:os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"- # Marks the value as ours rather than the operator's, so a process that needs- # bars back (the export worker streams Hub upload progress into the export- # dialog) can tell the difference. Inherited by every child process.+ # Marks the value as ours rather than the operator's, so a process that needs bars back can tell the+ # difference. Inherited by every child process.os.environ[_PROGRESS_BARS_DEFAULTED] = "1"elif not _env_is_true(os.environ["HF_HUB_DISABLE_PROGRESS_BARS"]):# Operator asked to keep them; leave every library alone.return- # Only touch Hub if something already imported it. Importing it here would cache- # the base environment's copy before a subprocess prepends its transformers- # sidecar to sys.path, leaving that process on an incompatible Hub.+ # Only touch Hub if something already imported it: importing it here would cache the base+ # environment's copy before a subprocess prepends its transformers sidecar to sys.path.if "huggingface_hub" in sys.modules:try:from huggingface_hub.utils import disable_progress_barsdisable_progress_bars()- except Exception: # noqa: BLE001 — quieting logs must never break startup+ except Exception: # noqa: BLE001 - quieting logs must never break startuppass- # transformers derives its own _tqdm_active from the hub flag at import time,- # so a module imported BEFORE this ran still needs the explicit call.- #- # datasets is handled separately (see _silence_datasets_bar_output): its `Map:` and- # `Standardizing chat format (num_proc=8):` bars from dataset preparation were the- # ones actually landing inside JSON records, but the UI reads their counter, so- # only the output goes. datasets is imported long after logging setup, which is why- # this function is safe to call again once a library is in.+ # transformers derives its own _tqdm_active from the hub flag at import time, so a module imported+ # BEFORE this ran still needs the explicit call.+ # datasets is handled separately: the UI reads its bar counters, so only the output goes, and it is+ # imported long after logging setup, which is why this function is safe to call again.for _mod in ("transformers", "diffusers"):module = sys.modules.get(_mod)if module is None:@@ -519,8 +497,8 @@ class LogConfig:log_level_name = os.getenv("LOG_LEVEL", "INFO").upper()log_level = getattr(logging, log_level_name, logging.INFO)- # Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows,- # LANG=C), so key off the stream, not the platform.+ # Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows, LANG=C), so key off the+ # stream, not the platform.for stream in (sys.stdout, sys.stderr):if getattr(stream, "encoding", "") and not str(stream.encoding).lower().replace("-", ""@@ -534,21 +512,20 @@ class LogConfig:structlog.configure(processors = [# Ordered to control output field order.- structlog.processors.TimeStamper(fmt = "iso"), # timestamp first- structlog.processors.add_log_level, # level second+ structlog.processors.TimeStamper(fmt = "iso"),+ structlog.processors.add_log_level,structlog.contextvars.merge_contextvars,structlog.processors.format_exc_info,filter_sensitive_data,- # After redaction, not before: redact_native_paths replaces exact- # strings, so cutting the middle out of a traceback first could leave- # half a path behind for it to miss.+ # After redaction, not before: redact_native_paths replaces exact strings, so cutting the middle out+ # of a traceback first could leave half a path behind for it to miss._truncate_exception_processor,# Flatten the extra field into the main dict.lambda logger, method_name, event_dict: {"timestamp": event_dict.get("timestamp"),"level": event_dict.get("level"),"event": event_dict.get("event"),- **(event_dict.get("extra", {})), # Flatten extra into main dict+ **(event_dict.get("extra", {})),**{k: vfor k, v in event_dict.items()diff --git a/studio/backend/loggers/handlers.py b/studio/backend/loggers/handlers.pyindex 7b7a8226c85..f57f012f39c 100644--- a/studio/backend/loggers/handlers.py+++ b/studio/backend/loggers/handlers.py@@ -18,8 +18,7 @@ from typing import TYPE_CHECKINGimport structlog-# Annotations only: a runtime import makes the ASGI stack a hard dependency of-# every CLI command.+# Annotations only: a runtime import makes the ASGI stack a hard dependency of every CLI command.if TYPE_CHECKING:from starlette.types import ASGIApp, Message, Receive, Scope, Send@@ -36,34 +35,29 @@ def _env_int(name: str, default: int) -> int:return default-# Collapse identical GET/2xx logs within the window (the SPA fans one invalidation-# into many list fetches). Mutations and errors always log. 0 = off.+# Collapse identical GET/2xx logs within the window, since the SPA fans one invalidation into many+# list fetches; mutations and errors always log._ACCESS_LOG_DEDUP_MS = _env_int("UNSLOTH_STUDIO_ACCESS_LOG_DEDUP_MS", 300)# Liveness/UI polls whose line means only "still polling"; collapse to a longer-# heartbeat. First hit and errors still log. 0 = off._QUIET_POLL_DEDUP_MS = _env_int("UNSLOTH_STUDIO_ACCESS_LOG_POLL_DEDUP_MS", 10000)-# The desktop watchdog probe is slower than every other poll here: 15s between rounds-# (HEALTH_WATCHDOG_INTERVAL, src-tauri/src/commands.rs) plus up to a 10s probe budget,-# ~19s in the sample below. A 10s window that stamps only on emit can never close over two-# of those, so it would collapse nothing. Its own window, wide enough to span a round.-# 0 = off.+# The desktop watchdog probe rounds are ~19s apart, so a 10s window that stamps only on emit would+# collapse nothing; it gets its own, wider window._WATCHDOG_POLL_DEDUP_MS = _env_int("UNSLOTH_STUDIO_ACCESS_LOG_WATCHDOG_DEDUP_MS", 60000)-# Both windows off is what --verbose sets; the drop-the-2xx suppressor below has no-# window of its own, so it must read the same signal to honour --verbose.+# Both windows off is what --verbose sets, and the drop-the-2xx suppressor below has no window of+# its own, so it reads the same signal._VERBOSE_ACCESS_LOG = _ACCESS_LOG_DEDUP_MS <= 0 and _QUIET_POLL_DEDUP_MS <= 0_QUIET_POLL_PATHS = {"/api/health","/api/auth/status","/api/inference/status","/api/inference/monitor",- # The loaded-models indicator polls all four runtimes every 5s for as long- # as the app is open, and on the desktop every line is mirrored into- # tauri.log. /api/inference/status is already above; these are its siblings.+ # The loaded-models indicator polls all four runtimes every 5s for as long as the app is open, and+ # on the desktop every line is mirrored into tauri.log."/api/inference/images/status","/api/inference/video/status","/api/inference/audio/stt/status",- # Re-read whenever the settings dialog or the remote-access section is open, and it is- # a plain read of a toggle: 116 lines in the 4h sample.+ # Re-read whenever the settings dialog or the remote-access section is open, and it is a plain read+ # of a toggle: 116 lines in the 4h sample."/api/settings/remote-access",# List polls the tabs refetch on a timer and on every tab switch."/api/train/runs",@@ -79,30 +73,20 @@ _QUIET_POLL_PATHS = {"/api/inference/video/generate-progress",# Polled every 1.5s while the train UI is open."/api/train/diffusion/status",- # Unlike /api/inference/load-progress, these handlers log nothing and- # diffusion.loaded / video.loaded are terminal, so a minutes-long load would- # otherwise emit nothing at all.+ # These handlers log nothing and diffusion.loaded / video.loaded are terminal, so a minutes-long+ # load would otherwise emit nothing at all."/api/inference/images/load-progress","/api/inference/video/load-progress",# Templated; matched through normalize_poll_path. See _TEMPLATED_POLL_PATHS."/api/chat/threads/{id}","/api/chat/threads/{id}/forks",}-# The pure-liveness subset of _QUIET_POLL_PATHS. Every one of these answers the same-# question ("the server is up and answering"), and the SPA fires them together in one-# burst, so heartbeating them independently emits one line per path per window instead-# of one line per window. They share a single bucket: the first of the burst logs with-# its real path, the rest of that window is dropped. Measured over four Unsloth sessions-# these were 39-69% of the access log and the shared bucket removed 25-47% of it.-#-# Only this group is shared. The other _QUIET_POLL_PATHS entries (/api/train/runs,-# /api/models/checkpoints, /api/models/local, /api/rag/knowledge-bases, the download-# polls) each report on a different subsystem, so they keep their own heartbeat.-# Two paths are deliberately NOT here because their latency is worth seeing on its-# own rather than being stamped out by a cheap sibling: /api/health (main.py waits up-# to a second for hardware detection, and the desktop preflight has a two-second-# deadline) and /api/inference/status (its handler reads llama.cpp capabilities and-# runs a release-freshness check in an executor).+# The pure-liveness subset: the SPA fires them together in one burst, so they share a single bucket,+# the first logging with its real path and the rest of the window dropping.+# Only this group is shared: the other quiet paths each report on a different subsystem, so they+# keep their own heartbeat.+# Two paths are deliberately NOT here because their latency is worth seeing: /api/health waits on+# hardware detection, and /api/inference/status reads llama.cpp capabilities in an executor._LIVENESS_POLL_PATHS = frozenset({"/api/auth/status",@@ -112,14 +96,10 @@ _LIVENESS_POLL_PATHS = frozenset("/api/inference/audio/stt/status",})-# The desktop shell's own watchdog probe. /api/health was already quiet but its sibling was-# in no suppressor at all: 760 lines on an idle 4h session, 14% of tauri.log, to say the-# process is still up. Out of _QUIET_POLL_PATHS because that window is narrower than the-# poll interval; see _WATCHDOG_POLL_DEDUP_MS. Start/stop transitions reach the phase log-# either way.+# The desktop shell's watchdog probe was in no suppressor at all: 760 lines on an idle 4h session.+# Out of _QUIET_POLL_PATHS because that window is narrower than the poll interval._WATCHDOG_POLL_PATHS = {"/api/liveness"}-# Bucket key for the group above. Not a real (method, path, query, status), so it can-# never collide with one.+# Bucket key for the group above; not a real (method, path, query, status), so it can never collide with one._LIVENESS_DEDUP_KEY = ("GET", "\x00liveness", b"", 200)_DEDUP_MAP_MAX = 4096_NATIVE_PATH_LEASE_RE = re.compile(@@ -141,9 +121,8 @@ _EXCLUDED_SUFFIXES = (".woff2",".ttf",)-# GET polls whose 2xx line carries no signal (their progress/phase events and the UI-# do), so drop it entirely; non-2xx still logs. Only /api/hub download polls emit-# events; the legacy /api/models and /api/datasets ones heartbeat via _QUIET_POLL_PATHS.+# GET polls whose 2xx line carries no signal, so drop it entirely; non-2xx still logs. Only /api/hub+# download polls emit events._QUIET_SUCCESS_PATHS = {"/api/inference/load-progress","/api/llama/update-status",@@ -159,39 +138,33 @@ _QUIET_SUCCESS_PATHS = {"/api/hub/datasets/download-progress","/api/hub/datasets/active-downloads","/api/hub/datasets/transport-status",- # Boot-burst catalog reads. The SPA refetches these on every auth change, store- # rehydration and settings/provider dialog open, and each one is a plain read whose- # outcome is already visible as the populated (or empty) list in the UI.- # Provider registry + configs: syncExternalProvidersFromBackend fetches the pair in- # one Promise.all, so they always arrive as two lines saying the same thing.+ # Boot-burst catalog reads: the SPA refetches these on every auth change and dialog open, and each+ # outcome is already visible in the UI.+ # syncExternalProvidersFromBackend fetches the pair in one Promise.all, so they always arrive as two+ # lines saying the same thing."/api/providers/registry","/api/providers/",- # Fetched in the same Promise.all as /api/models/list and /api/inference/status, both- # of which keep their access line, so the resync is still traceable without it.+ # Fetched in the same Promise.all as two routes that keep their access line, so the resync is still+ # traceable without it."/api/models/loras",- # Re-read once per auth generation at boot. Suppression is GET-only, so the PUT that- # actually writes a profile change still logs.+ # Suppression is GET-only, so the PUT that actually writes a profile change still logs."/api/settings/personalization",}-# The token-refresh route. Its first 2xx means the client has obtained a valid-# session, so from then on chat 401s are real failures and must stay visible.+# The token-refresh route: its first 2xx means the client has a valid session, so from then on chat+# 401s are real failures and must stay visible._AUTH_REFRESH_PATH = "/api/auth/refresh"-# High-frequency chat list polls; their 2xx is covered by generation/tool-call/stats-# events. Exact paths only, so detail/message reads (/threads/{id}, .../messages,-# /projects/{id}) keep their logs. The pre-auth 401 race also fires on these polls.+# Exact paths only, so detail and message reads keep their logs; the pre-auth 401 race also fires on these polls._CHAT_LIST_PATHS = {"/api/chat/threads","/api/chat/projects",}-# Templated paths. The sets above match EXACTLY, which #7087 chose so detail reads kept-# their access line; what changed is that streaming now drives these two on a loop (25 and-# 21 lines, 34% of the access log, over a 20s four-tab session). So they get a heartbeat-# rather than silence: the first of each window logs with its real path, the rest drop.+# The sets above match EXACTLY so detail reads keep their access line, but streaming drives these+# two on a loop (34% of the access log over a 20s session), so they get a heartbeat instead._CHAT_THREAD_DETAIL = "/api/chat/threads/{id}"_CHAT_THREAD_FORKS = "/api/chat/threads/{id}/forks"_TEMPLATED_POLL_PATHS = frozenset({_CHAT_THREAD_DETAIL, _CHAT_THREAD_FORKS})-# One id segment, no slashes: a deeper path such as /threads/{id}/messages/{mid} must NOT-# collapse into the detail bucket, since a message read is a different question.+# One id segment, no slashes: a deeper path such as /threads/{id}/messages/{mid} must NOT collapse+# into the detail bucket._CHAT_THREAD_PATH_RE = re.compile(r"^/api/chat/threads/(?!$)[^/]+(/forks)?$")@@ -208,13 +181,10 @@ def normalize_poll_path(path: str) -> str:return _CHAT_THREAD_FORKS if m.group(1) else _CHAT_THREAD_DETAIL-# The log viewer polls these while reading the very file this middleware writes,-# so unsuppressed each poll appends a record the next poll reads back and the log-# grows forever. _is_redundant_repeat does not cover it: it keys on the query-# string, and every poll carries a fresh cursor.-# Separate from _QUIET_SUCCESS_PATHS because --verbose must NOT lift this one:-# --verbose is what someone debugging turns on, and here the extra noise (~390-# bytes/s at the 1 Hz Live poll rate) buries the failure they opened the viewer for.+# The log viewer polls these while reading the very file this middleware writes, so unsuppressed+# each poll appends a record the next reads back; _is_redundant_repeat cannot cover it.+# Separate from _QUIET_SUCCESS_PATHS because --verbose must NOT lift this one: the extra noise+# buries the failure the viewer was opened for._SELF_READ_PATHS = {"/api/settings/debug/logs","/api/settings/debug/logs/sources",@@ -238,17 +208,14 @@ def _is_quiet_success(method: str, path: str, status_code: int, pre_auth: bool)return pre_auth and status_code == 401 and path in _CHAT_LIST_PATHS-# An unhandled request exception is logged twice: once here as a structured-# request_failed event whose "exception" field already carries the whole traceback-# (format_exc_info renders it, see loggers/config.py), and then again by uvicorn on-# stderr as "Exception in ASGI application" after the re-raise. The desktop shell-# mirrors every stderr line separately into tauri.log, so the second copy alone costs-# ~90 lines per failure. Keep the structured copy and drop uvicorn's.+# An unhandled request exception is logged twice: here as a structured request_failed event whose+# "exception" field carries the whole traceback (format_exc_info renders it, see+# loggers/config.py), and again by uvicorn on stderr, which the desktop shell mirrors into+# tauri.log at ~90 lines per failure. Keep the structured copy._UVICORN_ASGI_EXC_MSG = "Exception in ASGI application"-# Set on the exception instance itself rather than tracked in a side table: the object-# is what uvicorn hands us, so the match cannot go stale or collide with a recycled id,-# and an exception raised above this middleware (CORS, remote-access, the protocol-# layer) carries no marker and keeps uvicorn's traceback.+# Set on the exception instance itself rather than a side table: the object is what uvicorn hands+# us, so the match cannot go stale or collide with a recycled id, and an exception raised above+# this middleware (CORS, remote-access, the protocol layer) keeps uvicorn's traceback._LOGGED_EXC_ATTR = "_unsloth_request_failed_logged"@@ -294,8 +261,8 @@ class LoggingMiddleware:self.app = app# (method, path, query, status_code) -> monotonic ts of the last EMITTED log.self._last_log: dict[tuple[str, str, bytes, int], float] = {}- # Flips True after the first successful /api/auth/refresh; before that, chat- # list-poll 401s are the transient bootstrap race and are suppressed.+ # Before the first successful /api/auth/refresh, chat list-poll 401s are the transient bootstrap+ # race and are suppressed.self._auth_refreshed = Falsedef _is_redundant_repeat(@@ -306,9 +273,8 @@ class LoggingMiddleware:heartbeat. Stamps only on emit, so steady polls still log."""if method != "GET" or not (200 <= status_code < 300):return False- # A query makes the request something other than the background poll:- # /api/inference/audio/stt/status?model=... extends the downloaded check to a- # custom repo, so it keeps its own identity rather than joining the bucket.+ # A query makes the request something other than the background poll, so it keeps its own identity+ # rather than joining the bucket.is_liveness = path in _LIVENESS_POLL_PATHS and not query# Bucketed by template, so four tabs polling four threads share one heartbeat.norm = normalize_poll_path(path) if not query else path@@ -321,9 +287,8 @@ class LoggingMiddleware:window_ms = _ACCESS_LOG_DEDUP_MSif window_ms <= 0:return False- # The liveness group shares one bucket, so a burst of them logs once, not once- # per path. Only the query-less form joins it, so a parameterized call still- # gets its own status and latency line.+ # The liveness group shares one bucket, so a burst logs once rather than once per path; only the+ # query-less form joins it.key = _LIVENESS_DEDUP_KEY if is_liveness else (method, norm, query, status_code)last = self._last_log.get(key)if last is not None and (now - last) * 1000.0 < window_ms:diff --git a/studio/backend/mcp_server.py b/studio/backend/mcp_server.pyindex e950842fcf5..d2820180ed7 100644--- a/studio/backend/mcp_server.py+++ b/studio/backend/mcp_server.py@@ -27,8 +27,8 @@ class BearerTokenMiddleware:# A non-ASCII token cannot be sent in an HTTP header; reject it here.raise ValueError("Unsloth MCP bearer token must contain ASCII characters only")self.app = app- # Compare on raw header bytes: str hmac.compare_digest raises on non-ASCII- # input, which would surface as a 500 instead of a clean 401.+ # Compare on raw header bytes: str hmac.compare_digest raises on non-ASCII input, which would+ # surface as a 500 instead of a clean 401.self.expected = token.encode("utf-8")async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:@@ -168,9 +168,8 @@ def create_studio_mcp() -> FastMCP:from models.data_recipe import RecipePayloadfrom routes.data_recipe.validate import validate- # Direct call, so the ViaApiKey dependency never runs and its `= False`- # default would read as a UI session. This surface is a remote static- # bearer, so say so explicitly.+ # Direct call, so the ViaApiKey dependency never runs and its `= False` default would read as a UI+ # session; this surface is a remote static bearer.return _dump(validate(RecipePayload(recipe = recipe), via_api_key = True))@mcp.tooldiff --git a/studio/backend/models/auth.py b/studio/backend/models/auth.pyindex 9eabc6d4ee8..e0251a6c0d8 100644--- a/studio/backend/models/auth.py+++ b/studio/backend/models/auth.py@@ -76,8 +76,6 @@ class ChangePasswordRequest(BaseModel):# ----------------------------------------------------------------------------# API key schemas-# ---------------------------------------------------------------------------class CreateApiKeyRequest(BaseModel):diff --git a/studio/backend/models/data_recipe.py b/studio/backend/models/data_recipe.pyindex e6f27e64dfa..b9b89ca82bb 100644--- a/studio/backend/models/data_recipe.py+++ b/studio/backend/models/data_recipe.py@@ -81,7 +81,6 @@ class SeedInspectUploadRequest(BaseModel):block_id: str | None = Nonefile_ids: list[str] | None = Nonefile_names: list[str] | None = None- # Shared fieldspreview_size: int = Field(default = 10, ge = 1, le = 50)seed_source_type: str | None = Noneunstructured_chunk_size: int | None = Field(default = None, ge = 1, le = 20000)@@ -122,7 +121,7 @@ class UnstructuredFileUploadResponse(BaseModel):file_id: strfilename: strsize_bytes: int- status: str # "ok" or "error"+ status: strerror: str | None = Nonediff --git a/studio/backend/models/export.py b/studio/backend/models/export.pyindex 3be2d1e3af6..49eb7b049fb 100644--- a/studio/backend/models/export.py+++ b/studio/backend/models/export.py@@ -107,9 +107,8 @@ class ExportStatusResponse(BaseModel):False,description = "True while a load / export / cleanup operation is running",)- # Recovery fields: when a blocking export POST is cut off by a Cloudflare tunnel- # timeout (524 at ~100s), the client polls this endpoint to learn the real- # outcome of the operation that kept running on the backend.+ # Recovery fields: when a blocking export POST is cut off by a Cloudflare tunnel timeout (524 at+ # ~100s), the client polls this endpoint to learn the real outcome.active_op_kind: Optional[str] = Field(None,description = "Kind of the currently running op (load_checkpoint / export_* / cleanup)",@@ -209,8 +208,6 @@ class ExportMergedModelRequest(ExportCommonOptions):class ExportBaseModelRequest(ExportCommonOptions):"""Request for exporting a non-PEFT (base) model."""- # Uses fields from ExportCommonOptions only-class ExportGGUFRequest(BaseModel):"""Request for exporting the current model to GGUF format."""diff --git a/studio/backend/models/mcp_servers.py b/studio/backend/models/mcp_servers.pyindex 354356164ae..6beb116823c 100644--- a/studio/backend/models/mcp_servers.py+++ b/studio/backend/models/mcp_servers.py@@ -66,5 +66,5 @@ class McpServerImportRequest(BaseModel):class McpServerImportResult(BaseModel):created: list[McpServerResponse] = Field(default_factory = list)- skipped: list[str] = Field(default_factory = list) # display names skipped as duplicates+ skipped: list[str] = Field(default_factory = list)errors: list[str] = Field(default_factory = list)diff --git a/studio/backend/models/models.py b/studio/backend/models/models.pyindex 33468d70550..cd9bb7f0c24 100644--- a/studio/backend/models/models.py+++ b/studio/backend/models/models.py@@ -153,9 +153,9 @@ class GgufVariantDetail(BaseModel):filename: str = Field(..., description = "GGUF filename (e.g., 'gemma-3-4b-it-Q4_K_M.gguf')")quant: str = Field(..., description = "Quantization label or internal GGUF variant key")- # Mirrors hub.schemas.inventory.GgufVariantDetail. The route builds THIS model, so a field- # that exists only on the hub twin is dropped by pydantic without a word, and a qualified- # row falls back to rendering its whole relative path.+ # Mirrors hub.schemas.inventory.GgufVariantDetail. The route builds THIS model, so a field that+ # exists only on the hub twin is dropped by pydantic without a word and a qualified row falls back+ # to rendering its whole relative path.display_label: Optional[str] = Field(None, description = "Optional user-facing label when quant is an internal key")diff --git a/studio/backend/models/responses.py b/studio/backend/models/responses.pyindex 59130dc6732..862b70b402f 100644--- a/studio/backend/models/responses.py+++ b/studio/backend/models/responses.py@@ -8,9 +8,6 @@ from pydantic import BaseModel, Fieldfrom typing import Optional, List-# --- Training route response models -----class TrainingStopResponse(BaseModel):"""Response for stopping a training job"""@@ -34,9 +31,6 @@ class TrainingMetricsResponse(BaseModel):current_step: Optional[int] = Field(None, description = "Most recent step number")-# --- Model management route response models -----class LoRABaseModelResponse(BaseModel):"""Response for getting a LoRA's base model"""diff --git a/studio/backend/models/training.py b/studio/backend/models/training.pyindex d7af4ca4b69..6ec7d81f818 100644--- a/studio/backend/models/training.py+++ b/studio/backend/models/training.py@@ -38,8 +38,7 @@ _MAX_LORA_ALPHA = 32_768_MIN_VISION_IMAGE_SIZE = 256# 2048 is the highest most llms stay stable at_MAX_VISION_IMAGE_SIZE = 2048-# Upper bound for dataset slice indices, capping `.skip(n)` on streaming datasets so an absurd-# index can't iterate effectively forever. 1e9 is far beyond any realistic dataset row count.+# Caps .skip(n) on streaming datasets so an absurd index cannot iterate effectively forever._MAX_DATASET_SLICE_INDEX = 1_000_000_000@@ -81,6 +80,7 @@ class S3Config(BaseModel):def _parse_lr(v: Any) -> float:"""Parse learning_rate as a positive float strictly below _MAX_LR_VALUE."""+ # mode="before" sees True/False as bool (not 1/0) for a precise error.if v is None:raise ValueError("learning_rate is required")if isinstance(v, bool):@@ -206,8 +206,8 @@ class TrainingStartRequest(BaseModel):def _normalize_project_name(cls, value: Optional[str]) -> Optional[str]:return normalize_project_name(value)- # NOTE: pydantic runs all `mode="after"` validators in definition order, and- # `_check_steps_or_epochs` is lower in this class; keep these checks order-independent.+ # pydantic runs all mode="after" validators in definition order and _check_steps_or_epochs is lower+ # in this class, so keep these checks order-independent.@model_validator(mode = "after")def _validate_dataset_slice(self) -> "TrainingStartRequest":# start == end is intentionally allowed (a deliberate single-row slice); the trainer warns.@@ -316,7 +316,7 @@ class TrainingStartRequest(BaseModel):@field_validator("num_epochs")@classmethoddef _check_num_epochs(cls, v: int) -> int:- # 0 is a sentinel for "use max_steps instead" (frontend toggle).+ # 0 is the frontend's sentinel for "use max_steps instead".if v is None:return 1if v < 0 or v > _MAX_EPOCHS:@@ -466,9 +466,8 @@ class TrainingStartRequest(BaseModel):max_steps: Optional[int] = Field(None, description = "Maximum training steps")save_steps: int = Field(100, description = "Steps between checkpoints")weight_decay: float = Field(0.001, description = "Weight decay")- # All three clip knobs are finite as well as non-negative: JSON 1e309 (and- # FastAPI's Infinity literal) floats to inf, which clears ge=0 but never- # binds, so the run would train unclipped while reporting a threshold.+ # Finite as well as non-negative: JSON 1e309 floats to inf, which clears ge=0 but never binds, so+ # the run would train unclipped while reporting a threshold.max_grad_norm: Optional[float] = Field(None,ge = 0,@@ -536,7 +535,6 @@ class TrainingStartRequest(BaseModel):use_dora: bool = Field(False, description = "Use DoRA")train_on_completions: bool = Field(False, description = "Train on completions only")- # Vision-specific LoRA parametersfinetune_vision_layers: bool = Field(False, description = "Finetune vision layers")finetune_language_layers: bool = Field(True, description = "Finetune language layers")finetune_attention_modules: bool = Field(True, description = "Finetune attention modules")@@ -582,8 +580,8 @@ class TrainingStartRequest(BaseModel):@model_validator(mode = "after")def _validate_streaming_splits(self) -> "TrainingStartRequest":- # Streaming load_dataset does not accept HF slice syntax (probe-confirmed: ValueError: Bad- # split). Reject early with a clear message so the user knows to use a plain split name.+ # Streaming load_dataset does not accept HF slice syntax (probe-confirmed "Bad split"), so reject+ # early with a clear message.if self.dataset_streaming:for field_name, split_val in (("train_split", self.train_split),@@ -605,9 +603,8 @@ class TrainingStartRequest(BaseModel):@model_validator(mode = "after")def _validate_lora_variant_flags(self) -> "TrainingStartRequest":- # The frontend only ever sends one of these and never under Full Finetuning, but a direct- # API/YAML/CLI caller can bypass that. Nothing downstream breaks, but reject early for a- # clear error instead of a silently-ignored flag.+ # A direct API, YAML or CLI caller can bypass the frontend; nothing downstream breaks, but reject+ # early instead of silently ignoring the flag.active = [namefor name, enabled in (@@ -622,8 +619,8 @@ class TrainingStartRequest(BaseModel):f"Only one LoRA variant may be enabled at a time; got {active}. ""use_rslora, use_loftq, and use_dora are mutually exclusive.")- # getattr, not self.training_type: model_construct() (used by single-field tests) leaves- # required fields unset, and this mode="after" validator still runs on that partial instance.+ # getattr, not self.training_type: model_construct() leaves required fields unset and this+ # mode="after" validator still runs on that partial instance.if getattr(self, "training_type", None) == "Full Finetuning" and active:raise ValueError(f"{active[0]} requires an adapter method (LoRA/QLoRA or "@@ -735,8 +732,7 @@ class TrainingRunSummary(BaseModel):error_message: Optional[str] = Noneloss_sparkline: Optional[List[float]] = Nonecan_resume: bool = False- # Why resume is unavailable when the reason is the recorded resource provenance rather than- # the checkpoint. None when resumable or when the checkpoint itself is the problem.+ # Why resume is unavailable when the reason is the recorded resource provenance rather than the checkpoint itself.resume_blocked_reason: Optional[str] = Noneresumed_later: bool = Falseartifacts_available: bool = False@@ -827,35 +823,36 @@ class DiffusionTrainingStartRequest(BaseModel):"ceil(N / (batch x grad_accum)) optimizer steps over the N-image dataset"),)- # Upper bound as well as positive, matching the LLM schema: JSON accepts 1e309, which floats to inf and satisfies a- # gt-only constraint, so the route would evict residents and start AdamW at an infinite rate. Values >= 1.0 diverge.+ # Upper bound as well as positive: JSON 1e309 floats to inf and satisfies a gt-only constraint, so+ # the route would evict residents and start AdamW at an infinite rate; values >= 1.0 diverge.learning_rate: float = Field(1e-4, gt = 0, lt = 1.0)train_batch_size: int = Field(1, ge = 1, le = 64)gradient_accumulation_steps: int = Field(1, ge = 1, le = 256)lora_rank: int = Field(16, ge = 1, le = 320)lora_alpha: Optional[int] = Field(None, ge = 1, le = 640, description = "Defaults to lora_rank")- # Strictly below 1: PEFT turns lora_dropout into nn.Dropout(p=...), so 1.0 zeroes the LoRA branch and the run saves an- # untrained adapter while reporting normal progress. Matches TrainingStartRequest's validator.+ # Strictly below 1: PEFT turns lora_dropout into nn.Dropout(p=...), so 1.0 zeroes the LoRA branch+ # and the run saves an untrained adapter while reporting normal progress.lora_dropout: float = Field(0.0, ge = 0.0, lt = 1.0)- # Mirror the remaining training-affecting knobs of DiffusionLoraConfig so a client that sets them is not silently trained with defaults. Defaults to the SDXL projections.+ # Mirror the remaining training-affecting knobs of DiffusionLoraConfig so a client that sets them is+ # not silently trained with defaults. Defaults to the SDXL projections.lora_target_modules: List[str] = Field(default_factory = lambda: ["to_k", "to_q", "to_v", "to_out.0"],description = "U-Net modules to attach LoRA to",)- # Finite as well as non-negative: JSON accepts 1e309 (and FastAPI's parser the Infinity literal), which floats to inf and satisfies a ge-only- # constraint. clip_grad_norm_ then clamps its coefficient to 1.0, so the run trains completely unclipped while the config reports clipping.+ # Finite as well as non-negative: an inf max_grad_norm makes clip_grad_norm_ clamp its coefficient+ # to 1.0, so the run trains completely unclipped while the config reports clipping.max_grad_norm: float = Field(1.0,ge = 0,allow_inf_nan = False,description = "Gradient clipping max-norm; 0 disables clipping",)- # Bounded to what torch.manual_seed unpacks (int64 low / uint64 high): an out-of-range value otherwise passes every- # preflight, evicts the resident models, spawns the trainer, and only then dies unpacking long long.+ # Bounded to what torch.manual_seed unpacks: an out-of-range value otherwise passes every preflight,+ # evicts the resident models, spawns the trainer, and only then dies unpacking long long.seed: int = Field(42, ge = -(2**63), le = 2**64 - 1)mixed_precision: Literal["bf16", "fp16", "no"] = Field("bf16")- # Finite for the same reason as max_grad_norm: an inf gamma passes gt here and in normalized(), then collapses every- # min-SNR weight to 1.0, silently training on plain unweighted MSE. null is the documented disable.+ # Finite for the same reason as max_grad_norm: an inf gamma collapses every min-SNR weight to 1.0,+ # silently training on plain unweighted MSE.snr_gamma: Optional[float] = Field(5.0, gt = 0, allow_inf_nan = False, description = "Min-SNR loss weighting; null disables")@@ -904,7 +901,8 @@ class DiffusionTrainingStartRequest(BaseModel):"or auto (pick by free VRAM + GPU class). Dense modes need a non-prequant base."),)- # DiT-only levers the trainer implements. Undeclared, they were silently dropped by model_dump(); the defaults match DiffusionLoraConfig.+ # DiT-only levers the trainer implements. Undeclared, they were silently dropped by model_dump();+ # the defaults match DiffusionLoraConfig.ema_decay: float = Field(0.0, ge = 0.0, lt = 1.0, description = "EMA of the LoRA weights; 0 disables it")@@ -979,8 +977,8 @@ class DiffusionMetricHistory(BaseModel):loss: List[float] = Field(default_factory = list)lr: List[Optional[float]] = Field(default_factory = list)grad_norm: List[Optional[float]] = Field(default_factory = list)- # The two halves of a joint video+audio objective (MiniMax-H3). All-null on every other- # family, which is what lets the chart decide whether to draw the split curves at all.+ # All-null on every family but MiniMax-H3, which is what lets the chart decide whether to draw the+ # split curves at all.video_loss: List[Optional[float]] = Field(default_factory = list)audio_loss: List[Optional[float]] = Field(default_factory = list)@@ -999,8 +997,7 @@ class DiffusionTrainingStatusResponse(BaseModel):learning_rate: Optional[float] = None# Total pre-clip gradient norm from the last optimizer step (health signal the UI charts).grad_norm: Optional[float] = None- # The last per-modality losses of a joint video+audio run (MiniMax-H3), None elsewhere. The- # combined loss can hold steady while one modality degrades, so these are reported apart.+ # The combined loss can hold steady while one modality degrades, so these are reported apart.video_loss: Optional[float] = Noneaudio_loss: Optional[float] = Nonenum_images: Optional[int] = None@@ -1009,15 +1006,16 @@ class DiffusionTrainingStatusResponse(BaseModel):lora_path: Optional[str] = None# The second, EMA-averaged adapter written in the run's ema subdir when ema_decay was enabled.ema_path: Optional[str] = None- # Where the adapter was mirrored into the Unsloth LoRA catalog, and what family / base it trained from, so the UI can deploy it.+ # Where the adapter was mirrored into the Unsloth LoRA catalog, and what family / base it trained+ # from, so the UI can deploy it.catalog_path: Optional[str] = Nonefamily: Optional[str] = Nonebase_model: Optional[str] = None# Live throughput + peak VRAM (from the trainer's progress events).samples_per_second: Optional[float] = Nonepeak_memory_gb: Optional[float] = None- # Resume state for this job: the newest checkpoint bundle written, the step it holds, why one- # could not be written (the Resume action's disabled tooltip), and where a resumed run started.+ # The newest checkpoint bundle written, the step it holds, why one could not be written (the Resume+ # action's disabled tooltip), and where a resumed run started.checkpoint_path: Optional[str] = Nonecheckpoint_step: Optional[int] = Noneresume_blocked_reason: Optional[str] = None@@ -1049,13 +1047,13 @@ class DiffusionTrainingRunSummary(BaseModel):ended_at: Optional[float] = None# The run's adapter directory, which is also what a Resume replays as resume_from_checkpoint.output_dir: Optional[str] = None- # Whether this run can be continued, re-derived from the checkpoints on disk at read time (not- # the value frozen when the run ended), with the step the newest bundle holds. When it cannot,- # resume_blocked_reason says why, and the UI shows that as the disabled action's tooltip.+ # Re-derived from the checkpoints on disk at read time, not the value frozen when the run ended,+ # so deleting them takes the action away. When it cannot resume, resume_blocked_reason says why+ # and the UI shows it as the disabled action's tooltip.can_resume: bool = Falsecheckpoint_step: Optional[int] = None- # The exact bundle a resume would continue. Clients send it back as resume_from_checkpoint so- # they get the one they were shown, not whatever is newest in a folder two runs may share.+ # The exact bundle a resume would continue, sent back so clients get the one they were shown rather+ # than whatever is newest in a folder two runs may share.checkpoint_path: Optional[str] = Noneresume_blocked_reason: Optional[str] = None# Lineage: the run this one continued, and the step it picked up from.@@ -1086,8 +1084,11 @@ class DiffusionDatasetSummary(BaseModel):name: strpath: strimage_count: int- # Clips, for the families that train from video. Defaults to 0 so an older backend's- # payload (and every image-only caller) stays valid.+ # Defaults to 0 so an older backend's payload, and every image-only caller, stays valid.+ # Clips, for the families that train from video. Defaults to 0 so an older backend's payload (and every+ # image-only caller) stays valid.+ # Clips, for the families that train from video. Defaults to 0 so an older backend's payload (and every+ # image-only caller) stays valid.clip_count: int = 0caption_count: int@@ -1106,22 +1107,20 @@ class DiffusionTrainableFamily(BaseModel):qlora_vram_gb: Optional[int] = Nonegated: bool = Falsenote: str = ""- # base_precision modes this machine supports for the family (empty = no selector, e.g. SDXL), the recommended pick, and- # whether regional torch.compile applies. Defaults keep older backends' payloads valid.+ # base_precision modes this machine supports for the family (empty = no selector, e.g. SDXL), the+ # recommended pick, and whether regional torch.compile applies. Defaults keep older backends'+ # payloads valid.precision_modes: List[str] = Field(default_factory = list)recommended_precision: str = "nf4"supports_compile: bool = False- # Whether this family's loop writes checkpoint bundles. False makes the panel drop the- # "Checkpoint every" control: save_steps is refused, not ignored, for such a family, so- # offering the control means offering a value that rejects Start. Defaults True so an older- # backend's payload, which has no such families in it, keeps the control.+ # save_steps is refused, not ignored, for a checkpointless family, so offering the control means+ # offering a value that rejects Start; defaults True so an older backend's payload keeps it.supports_checkpoints: bool = True- # The largest batch this family's loop can form, or None for unrestricted. 1 for a family- # whose forward covers one packed sequence: the panel then drops the Batch control, since a- # value above the cap is refused rather than clamped. Declared here or Pydantic silently- # drops it from the response and the panel's clamp never receives its input.+ # 1 for a family whose forward covers one packed sequence: a value above the cap is refused rather+ # than clamped, and declaring it here is what stops Pydantic dropping it from the response.max_train_batch_size: Optional[int] = None- # When set, a LoRA trained on this family previews on this repo instead of the training base (Krea trains on Raw, runs on Turbo).+ # When set, a LoRA trained on this family previews on this repo instead of the training base (Krea+ # trains on Raw, runs on Turbo).deploy_base: Optional[str] = None# Variant-specific training-base to inference-base pairs, including public mirror ids.deploy_bases: Dict[str, str] = Field(default_factory = dict)@@ -1146,8 +1145,6 @@ class DiffusionDatasetUploadResponse(BaseModel):name: strpath: strimage_count: int- # Clips, for the families that train from video. Defaults to 0 so an older backend's- # payload (and every image-only caller) stays valid.clip_count: int = 0caption_count: intuploaded: int@@ -1162,9 +1159,8 @@ class DiffusionDatasetImageRecord(BaseModel):filename: strcaption: Optional[str] = Nonecaption_source: Literal["sidecar", "metadata", "none"] = "none"- # Which kind of item this is. Clips are listed so a caller can see everything the folder- # holds under a stem, but they carry no pixel dimensions and the labeling grid skips them.- # Defaults to "image" so an older backend's payload still validates.+ # Clips are listed so a caller can see everything the folder holds under a stem, but they carry no+ # pixel dimensions and the labeling grid skips them; defaults to "image" for older payloads.kind: Literal["image", "clip"] = "image"width: intheight: int@@ -1220,8 +1216,6 @@ class DiffusionDatasetImportResponse(BaseModel):name: strpath: strimage_count: int- # Clips, for the families that train from video. Defaults to 0 so an older backend's- # payload (and every image-only caller) stays valid.clip_count: int = 0caption_count: intimported: intdiff --git a/studio/backend/picker/routes/templates.py b/studio/backend/picker/routes/templates.pyindex 8ca20e23798..f410fc104e2 100644--- a/studio/backend/picker/routes/templates.py+++ b/studio/backend/picker/routes/templates.py@@ -38,8 +38,7 @@ async def get_default_chat_template_route(hf_token: HfTokenArg = Depends(get_request_hf_token),current_subject: str = Depends(get_current_subject),) -> ModelTemplateResponse:- # Cached repos resolve from disk, but a cache miss falls through to the hub and- # offline that costs one retry backoff per candidate template file.+ # A cache miss falls through to the hub, and offline that costs one retry backoff per candidate template file.from core.inference.llama_cpp import _hf_offline_if_unreachable_fordef _read():diff --git a/studio/backend/picker/schemas.py b/studio/backend/picker/schemas.pyindex c43d769817d..c32cce3065e 100644--- a/studio/backend/picker/schemas.py+++ b/studio/backend/picker/schemas.py@@ -5,9 +5,8 @@ from typing import Optionalfrom pydantic import BaseModel, Field, field_validator-# Mirror the frontend's 64 KiB chat-template contract (per-model-config.ts) at-# the API boundary so a direct caller cannot make Jinja parse an oversized-# template. MaxBodyMiddleware only caps the whole request body, not this field.+# Mirror the frontend's 64 KiB chat-template contract (per-model-config.ts) at the API boundary:+# MaxBodyMiddleware only caps the whole request body, not this field.MAX_CHAT_TEMPLATE_BYTES = 65_536diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.pyindex d4d46da370f..8f0739b9b68 100644--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py@@ -1,6 +1,5 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-# Intentionally empty. Data-designer loads submodules lazily via qualified names-# in plugin.py, so importing this package must not touch data_designer.engine.*-# during Unsloth bootstrap (circular import).+# Intentionally empty: data-designer loads submodules lazily by qualified name in plugin.py, so+# importing this package must not touch data_designer.engine.* during bootstrap (circular import).diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.pyindex 330e917f3f0..97411c57f73 100644--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py@@ -16,10 +16,8 @@ from .config import GitHubRepoSeedSourcefrom .scraper import ScrapeConfig, materialize_to_jsonl-# In-process cache: config signature -> JSONL path. A recipe job reads the seed-# multiple times (validation, preview, sampling); memoize to avoid re-scraping-# GitHub on each call. Key uses a short SHA-256 of the token (never the raw value)-# so token never hits memory twice and rotation invalidates cleanly.+# Memoize seed materialization so a recipe job does not re-scrape GitHub on every call; the key+# uses a short SHA-256 of the token, so the raw value never hits memory twice._SCRAPE_CACHE: dict[tuple, str] = {}_SCRAPE_CACHE_LOCK = threading.Lock()diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.pyindex 0ba3394ca3f..2d90b58e61a 100644--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py@@ -133,6 +133,7 @@ class GitHubClient:)def _check_rate_and_wait(self, kind: str) -> None:+ # Reset remaining so we don't spinif kind == "graphql":remaining = self.graphql_remainingreset = self.graphql_reset@@ -144,7 +145,6 @@ class GitHubClient:if remaining is not None and remaining < min_remaining:if reset:self._sleep_until(reset)- # Reset remaining so we don't spinif kind == "graphql":self.graphql_remaining = Noneelse:diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.pyindex a7ddaef5fef..7664510eb25 100644--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py@@ -49,15 +49,12 @@ class RepoScraper:self.base_dir = base_dirself.client = clientself.trial_limits = trial_limits or {}- # light=True uses trimmed GraphQL queries (no reviewThreads/reviews/- # commits/timelineItems/files) so PR pages can be larger without- # hitting GitHub's node-count ceiling.+ # light=True uses trimmed GraphQL queries so PR pages can be larger without hitting GitHub's node-count ceiling.self.light = lightself.repo_dir = base_dir / f"{owner}__{name}"self.repo_dir.mkdir(parents = True, exist_ok = True)self.state = StateStore(base_dir / "state" / f"{owner}__{name}.json")- # Writersself.writers: Dict[str, JsonlWriter] = {}for key in ("issues",@@ -115,8 +112,8 @@ class RepoScraper:return 0total_new = 0page = 0- # Light query skips heavy nested fields; safe at 50/page. Clamp by- # trial_limit so limit=1 asks for first:1, not a full 50-item page.+ # The light query skips heavy nested fields, so 50/page is safe; clamp by trial_limit so limit=1+ # does not fetch a full page.page_cap = 50 if self.light else 15trial_cap = self.trial_limits.get(key)per_page = min(page_cap, trial_cap) if trial_cap and trial_cap > 0 else page_cap@@ -222,9 +219,8 @@ class RepoScraper:return 0total_new = 0page = 0- # Heavy nested PR query caps at 3/page (GitHub node-count ceiling);- # light query skips nested fields and goes to 25/page. Clamp by- # trial_limit so limit=1 does not fetch a whole 25-item page.+ # The heavy nested PR query caps at 3/page against GitHub's node-count ceiling and the light one at+ # 25; clamp by trial_limit so limit=1 does not fetch a whole page.page_cap = 25 if self.light else 3trial_cap = self.trial_limits.get(key)per_page = min(page_cap, trial_cap) if trial_cap and trial_cap > 0 else page_capdiff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.pyindex 1a580c45b35..330afdd9ca1 100644--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py@@ -73,9 +73,8 @@ def _read_line(raw: bytes, codepage: str) -> _Reading:a JSON backslash, so the record fails to parse and its id is forgotten."""as_utf8 = _parse(raw, "utf-8")- # A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a- # 76 MB shard, and these reach gigabytes. Only a dict, since key lookup falls- # through to the codepage when UTF-8 yields none.+ # A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a 76 MB shard, and+ # these reach gigabytes.if isinstance(as_utf8, dict):return _Reading(as_utf8, None)for encoding in (codepage, "latin-1", *_DOUBLE_BYTE_ENCODINGS):@@ -90,11 +89,11 @@ def _read_line(raw: bytes, codepage: str) -> _Reading:class _Scan(NamedTuple):"""What a pass over an existing shard established about it."""- legacy: bool # enough evidence to trust the codepage reading's keys+ legacy: boolreadable: bool- saw_non_ascii: bool # some line's meaning depends on the encoding- utf8_keys: set # keys from lines UTF-8 could read- legacy_keys: set # keys only the codepage reading yields+ saw_non_ascii: bool+ utf8_keys: set+ legacy_keys: setclass StateStore:@@ -103,12 +102,10 @@ class StateStore:self.path.parent.mkdir(parents = True, exist_ok = True)self._lock = threading.Lock()self._data: Dict[str, Any] = {}- # Read whole, and UTF-8 only unlike the shards below: a checkpoint holds- # nothing but base64 cursors and booleans, so a codepage retry could only ever- # add non-ASCII. That would resume on a mojibaked cursor, which GitHub rejects- # with INVALID_CURSOR_ARGUMENTS, and the empty page it returns marks the stream- # done and skips the rest for good. Dropping a damaged checkpoint re-scrapes- # from the first page, which the writers dedup.+ # Read whole, and UTF-8 only unlike the shards below: a checkpoint holds nothing but base64+ # cursors and booleans, so a codepage retry could only resume on a mojibaked cursor GitHub rejects+ # with INVALID_CURSOR_ARGUMENTS, whose empty page marks the stream done. Dropping a damaged+ # checkpoint re-scrapes from page one, which the writers dedup.if self.path.exists():try:raw = self.path.read_bytes()@@ -165,9 +162,8 @@ class JsonlWriter:if scan.legacy:self._count_seen_keys |= scan.legacy_keysif scan.saw_non_ascii or not scan.readable:- # Never convert: the writing encoding is unrecoverable and guessing- # mojibakes the records. Pure ASCII appends store identically under- # every codepage, and json.loads turns the \uXXXX escapes back.+ # Never convert: the writing encoding is unrecoverable and guessing mojibakes the records. Pure+ # ASCII appends store identically under every codepage, and json.loads turns the escapes back.encoding = "ascii"self._ensure_ascii = Trueself._fh = self.path.open("a", buffering = 1, encoding = encoding, errors = "strict")diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.pyindex 47dc5db140c..2ba60b3ac92 100644--- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py+++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py@@ -146,10 +146,7 @@ def materialize_unstructured_seed_dataset(def materialize_multi_file_unstructured_seed(- *,- file_entries: list[tuple[Path, str]], # (extracted_txt_path, original_filename)- chunk_size: int,- chunk_overlap: int,+ *, file_entries: list[tuple[Path, str]], chunk_size: int, chunk_overlap: int) -> tuple[Path, list[dict[str, str]]]:"""Chunk multiple files into one parquet dataset with a source_file column."""chunk_size, chunk_overlap = resolve_chunking(chunk_size, chunk_overlap)diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.pyindex 825b050e070..f5393292299 100644--- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py+++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py@@ -30,8 +30,8 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]):meta = json_mod.loads(meta_path.read_text(encoding = "utf-8"))orig_name = meta.get("original_filename", path_obj.name)except (json_mod.JSONDecodeError, OSError, UnicodeDecodeError):- # Undecodable metadata is as malformed as invalid JSON, so- # fall back to the file's own name rather than abort the seed.+ # Undecodable metadata is as malformed as invalid JSON, so fall back to the file's own name rather+ # than abort the seed.passfile_entries.append((path_obj, orig_name))diff --git a/studio/backend/startup_banner.py b/studio/backend/startup_banner.pyindex 145c7f6dcf5..b8bc8360524 100644--- a/studio/backend/startup_banner.py+++ b/studio/backend/startup_banner.py@@ -114,12 +114,11 @@ def print_studio_access_banner(else:external_url = f"http://{display_host}:{port}"- # The exact aliases the canned loopback_url below is valid for; any other bind- # (e.g. a specific LAN IP) must show its real address, not http://127.0.0.1.+ # The exact aliases the canned loopback_url is valid for; any other bind (e.g. a specific LAN IP)+ # must show its real address, not http://127.0.0.1.loopback_bind = bind_host in ("127.0.0.1", "localhost", "::1")- # Use the loopback URL only when reachable on loopback; otherwise show- # the actual bound address.+ # Use the loopback URL only when reachable on loopback; otherwise show the actual bound address.primary_url = loopback_url if listen_all or loopback_bind else external_urlapi_base = primary_urldiff --git a/studio/backend/state/active_generations.py b/studio/backend/state/active_generations.pyindex 0c0bb584cc3..9b17627d25b 100644--- a/studio/backend/state/active_generations.py+++ b/studio/backend/state/active_generations.py@@ -23,8 +23,8 @@ import timeimport uuidfrom typing import Any, Optional-# handle id -> entry. Keyed by handle, not thread_id: a tool continuation can register-# before the previous leg unregisters, and one key would drop the other.+# Keyed by handle, not thread_id: a tool continuation can register before the previous leg+# unregisters, and one key would drop the other._ACTIVE: dict[str, dict[str, Any]] = {}_LOCK = threading.Lock()@@ -64,9 +64,8 @@ class ActiveGeneration:def __enter__(self) -> "ActiveGeneration":with _LOCK:- # A durable supervisor registers before model loading starts. The- # route later enters its normal tracker with that exact event/run;- # borrow the outer registration so lifecycle counts stay truthful.+ # A durable supervisor registers before model loading starts and the route later enters its normal+ # tracker with that same event and run, so borrow the outer registration.if self.run_id:for entry in _ACTIVE.values():if entry["run_id"] != self.run_id or entry["event"] is not self.cancel_event:diff --git a/studio/backend/state/tool_approvals.py b/studio/backend/state/tool_approvals.pyindex f66226b61d5..059f960022b 100644--- a/studio/backend/state/tool_approvals.py+++ b/studio/backend/state/tool_approvals.py@@ -24,12 +24,11 @@ import secretsimport threadingfrom typing import Optional-# Generous ceiling so a user can deliberate; cancellation (stop button /-# disconnect) still breaks the wait early via ``cancel_event``.+# Generous ceiling so a user can deliberate; the stop button or a disconnect still breaks the wait+# early via cancel_event._DECISION_TIMEOUT = 3600.0-# Fed to the model as the tool result when the user denies a call, so it-# can adapt and keep responding instead of the turn ending abruptly.+# Fed to the model as the tool result when the user denies a call, so it can adapt instead of the turn ending abruptly.TOOL_REJECTED_MESSAGE = "The user declined to run this tool call."_lock = threading.Lock()diff --git a/studio/backend/storage/api_usage_db.py b/studio/backend/storage/api_usage_db.pyindex d022929248c..ba36fd759e9 100644--- a/studio/backend/storage/api_usage_db.py+++ b/studio/backend/storage/api_usage_db.py@@ -18,9 +18,8 @@ from typing import Callable, Optionalfrom storage.studio_db import get_connection, is_sqlite_busy_error-# Kept aligned with the API monitor's defensive upper bound. The storage layer-# validates independently because callers can invoke it directly in tests or-# future integrations.+# Kept aligned with the API monitor's defensive upper bound; the storage layer validates+# independently because callers can invoke it directly.MAX_TOKEN_COUNT = 1 << 40MAX_RECEIPT_ID_CHARS = 128MAX_SUBJECT_CHARS = 256@@ -164,9 +163,8 @@ def record_api_usage(receipt: ApiUsageReceipt) -> bool:except sqlite3.OperationalError as exc:if not _is_busy_error(exc) or attempt + 1 == _WRITE_RETRIES:raise- # The worker is the only production writer of these receipts. A- # short bounded backoff lets unrelated Unsloth transactions finish- # without ever holding up the inference/streaming caller.+ # The worker is the only production writer of these receipts, so a short bounded backoff lets+ # unrelated transactions finish without holding up the streaming caller._sleep_after_busy(min(0.01 * (2**attempt), _WORKER_BUSY_RETRY_SECONDS))if inserted:@@ -214,7 +212,6 @@ class ApiUsageWriter:self._stopped = Trueself._queue.put_nowait(_STOP)# Production calls this through asyncio.to_thread so even the bounded- # wait cannot pause inference or the event loop.self._thread.join(timeout = max(0.0, timeout))drained = not self._thread.is_alive()if not drained:@@ -241,11 +238,9 @@ class ApiUsageWriter:if not _is_busy_error(exc):logger.warning("api usage receipt persistence failed", exc_info = True)break- # record_api_usage already made its bounded fast retries.- # Retain this accepted item at the head of the single- # writer until a normal long Unsloth transaction releases- # SQLite. The stop sentinel remains behind it, so final- # shutdown drains rather than silently dropping usage.+ # record_api_usage already made its bounded fast retries: retain this accepted item at+ # the head of+ # the single writer until a long transaction releases SQLite, with the stop sentinel behind it.busy_failures += 1if busy_failures == 1 or busy_failures % 20 == 0:logger.warning(diff --git a/studio/backend/storage/chat_generation_runs_db.py b/studio/backend/storage/chat_generation_runs_db.pyindex 8720dc92cc6..fcb097d30bd 100644--- a/studio/backend/storage/chat_generation_runs_db.py+++ b/studio/backend/storage/chat_generation_runs_db.py@@ -22,9 +22,8 @@ _EVENTS_CHANGED = threading.Condition()_RUN_TOMBSTONE_PREFIX = "chat-generation-run-tombstone:"ChatGenerationEventInput = Union[tuple[str, dict[str, Any]], tuple[str, dict[str, Any], int]]-# Progress lease columns, added here rather than in _ensure_schema so the base table-# definition stays owned by studio_db. Named _schema_ready to match the flag the test-# harness resets on every storage module when it swaps UNSLOTH_STUDIO_HOME.+# Progress lease columns live here rather than in _ensure_schema so the base table stays owned by+# studio_db; named _schema_ready to match the flag the test harness resets._schema_ready = False_schema_lock = threading.Lock()@@ -65,9 +64,8 @@ def _connect() -> sqlite3.Connection:conn.commit()_schema_ready = Trueexcept sqlite3.OperationalError:- # A writer holds the database. The columns are additive, so let this call through- # and migrate later rather than turning contention into a failed history read;- # _without_lease_columns below covers the paths that then cannot assume them.+ # A writer holds the database, and the columns are additive, so let this call through and migrate+ # later rather than turning contention into a failed history read.conn.rollback()except Exception:conn.close()@@ -208,8 +206,8 @@ def _touch_progress_locked(conn: sqlite3.Connection, run_id: str, tokens: int) -(now, now, max(0, int(tokens)), run_id),)except sqlite3.OperationalError as exc:- # The migration has not landed yet. The run keeps streaming and simply ages out- # on started_at/created_at, which the sweep already falls back to.+ # The migration has not landed yet, so the run ages out on started_at/created_at, which the sweep+ # already falls back to.if not _missing_lease_columns(exc):raise@@ -548,8 +546,8 @@ def append_events(conn.commit()return []sequences = _append_events_locked(conn, run_id, batch)- # The producer's only regular write, so it is also the lease renewal: output- # reaching the database is the definition of progress this sweep reaps on.+ # The producer's only regular write, so it is also the lease renewal: output reaching the database+ # is the definition of progress this sweep reaps on._touch_progress_locked(conn, run_id, sum(1 for event in batch if event[0] == "chunk"))_commit(conn, notify = True)return sequences@@ -769,8 +767,8 @@ def reconcile_runs(WHERE status IN ('queued','running','cancelling')"""args: tuple[Any, ...] = ()if stale_after_ms is not None:- # started_at/created_at carry a run that has not streamed anything yet,- # so a producer wedged before its first token still ages out.+ # started_at/created_at carry a run that has not streamed anything yet, so a producer wedged before+ # its first token still ages out.sql += " AND COALESCE(progress_at, started_at, created_at) <= ?"args = (completed - int(stale_after_ms),)try:@@ -778,11 +776,9 @@ def reconcile_runs(except sqlite3.OperationalError as exc:if not _missing_lease_columns(exc):raise- # Contention blocked the migration, so no run can persist progress. Falling- # back to started_at/created_at is not conservative but the opposite: those- # stamps are older by the whole life of the run, so one that streamed a chunk- # moments ago is reaped once its total AGE passes the timeout. Boot reconcile- # passes no stale_after_ms and is unaffected.+ # Contention blocked the migration, so falling back to started_at/created_at is the opposite of+ # conservative: those stamps are older by the whole life of the run, so one that streamed moments+ # ago is reaped once its total AGE passes the timeout. Boot reconcile passes no stale_after_ms.if stale_after_ms is not None:conn.rollback()return []@@ -793,9 +789,8 @@ def reconcile_runs().fetchall()for row in rows:run_id = row["id"]- # A Stop that was already recorded outlives the restart. Reporting it as a- # backend failure would tell the user Studio broke when they stopped it, and- # finish_run settles this same case as cancelled.+ # A Stop that was already recorded outlives the restart, and reporting it as a backend failure+ # would tell the user Studio broke when they stopped it; finish_run settles this case as cancelled.if str(row["status"]) == "cancelling" or bool(row["cancel_requested"]):status, finish_reason, message = "cancelled", "cancelled", Noneterminal = ("run.cancelled", {"status": status, "finishReason": finish_reason})@@ -809,8 +804,8 @@ def reconcile_runs(updated_at=?, completed_at=? WHERE id=?""",(status, finish_reason, message, completed, completed, run_id),)- # Stamps incomplete: {reason: "interrupted"} on the assistant message, which- # is what releases the frontend's "generating" state and restores Send.+ # Stamps incomplete on the assistant message, which is what releases the frontend's "generating"+ # state and restores Send._sync_assistant_status_locked(conn, run_id, status)settled.append(str(run_id))_commit(conn, notify = bool(settled))diff --git a/studio/backend/storage/profile_stats_db.py b/studio/backend/storage/profile_stats_db.pyindex 477b4ee9edd..ba109cac443 100644--- a/studio/backend/storage/profile_stats_db.py+++ b/studio/backend/storage/profile_stats_db.py@@ -33,8 +33,8 @@ from storage.studio_db import count_chat_message_attachments, get_connectionlogger = get_logger(__name__)-# Gaps longer than this end a "sitting at the keyboard" stretch: without the cap-# a thread reopened a week later would report a week-long chat.+# Gaps longer than this end a sitting-at-the-keyboard stretch: without the cap a thread reopened a+# week later would report a week-long chat.SESSION_GAP_SECONDS = 30 * 60# Cap on the daily activity series handed to the UI (the heatmap draws a year).MAX_DAILY_DAYS = 366@@ -43,8 +43,8 @@ MAX_TZ_OFFSET_MINUTES = 14 * 60# Top-N lists returned to the client.TOP_MODELS = 8RECENT_RUNS = 5-# Serve a memoised payload for this long even if the fingerprint is unchanged,-# so a chat that is mid-stream still refreshes reasonably promptly.+# Serve a memoised payload for this long even when the fingerprint is unchanged, so a chat that is+# mid-stream still refreshes promptly.CACHE_TTL_SECONDS = 20.0_cache_lock = threading.Lock()@@ -227,7 +227,6 @@ def _fold_api_usage(conn, zone, subject: str) -> _ApiUsageFold:fold.prompt_tokens += prompt_tokensfold.completion_tokens += completion_tokens# Preserve the provider's authoritative total even when it differs- # from the input/output sum.fold.total_tokens += total_tokensfold.requests += 1@@ -242,8 +241,7 @@ def _fold_api_usage(conn, zone, subject: str) -> _ApiUsageFold:model_id,{"id": model_id, "label": _model_label(model_id), "messages": 0, "tokens": 0},)- # One terminal API request represents one model response in the- # combined leaderboard.+ # One terminal API request represents one model response in the combined leaderboard.model["messages"] += 1model["tokens"] += total_tokensreturn fold@@ -362,20 +360,15 @@ def _fold_messages(conn, zone) -> _MessageFold:thread_messages = 0previous_created = None- # Compare mode stores one thread per pane under a shared pair_id, and- # the sidebar shows them as a single conversation. Count them that way- # too, or one comparison inflates the chat total and drags the average- # tokens per chat down.+ # Compare mode stores one thread per pane under a shared pair_id and the sidebar shows them as a+ # single conversation, so counting them twice inflates the chat total.conversation_id = row["pair_id"] or thread_id# A fork is its own visible conversation, so it counts towards the chat- # total from the moment it exists, before any new turn is added.fold.threads.add(conversation_id)- # Forking clones the whole ancestry into the new thread, keeping each- # copy's original timestamp. Skip a clone while the row it was taken- # from is still there to be counted; once that original is gone, only- # the elected fork stands in for it, so nothing is lost or doubled.+ # Forking clones the whole ancestry keeping each copy's timestamp, so skip a clone while the row+ # it came from is still countable; once that original is gone the elected fork stands in.source_id = row["forked_from_thread_id"]if source_id and created_at < _as_int(row["thread_created_at"]):original = (source_id, created_at, row["role"])@@ -418,9 +411,8 @@ def _fold_messages(conn, zone) -> _MessageFold:usage = usage if isinstance(usage, dict) else {}timing = timing if isinstance(timing, dict) else {}- # llama.cpp reports its own counters under serverTimings when the- # provider sends no usage chunk. The response-details sheet already- # falls back to these, so the totals here have to as well.+ # llama.cpp reports its own counters under serverTimings when the provider sends no usage chunk, and+ # the response-details sheet already falls back to these.server = metadata.get("serverTimings")server = server if isinstance(server, dict) else {}@@ -431,7 +423,6 @@ def _fold_messages(conn, zone) -> _MessageFold:total_tokens = _as_int(usage.get("totalTokens"))if completion_tokens == 0:# Local engines occasionally omit the usage chunk; the adapter's- # own token count is the next best estimate.completion_tokens = _as_int(timing.get("tokenCount"))if total_tokens == 0:total_tokens = prompt_tokens + completion_tokens@@ -445,11 +436,8 @@ def _fold_messages(conn, zone) -> _MessageFold:fold.tool_calls += _as_int(timing.get("toolCallCount"))message_tokens = total_tokens- # responseDetails carries the model that actually answered, which- # differs from the requested checkpoint whenever a provider routes- # or resolves an alias. contextUsage.modelId is the request, so it- # is only the fallback. The thread's model_id is never used: it- # tracks the current selection, not the one that ran.+ # responseDetails carries the model that actually answered, which differs from the requested+ # checkpoint whenever a provider routes or resolves an alias.details = metadata.get("responseDetails")details = details if isinstance(details, dict) else {}model_id = _clean_str(details.get("responseModelId")) or _clean_str(@@ -555,18 +543,16 @@ def _training_stats(conn) -> dict[str, Any]:""").fetchone()- # A resumed run continues its source's step and token counters from the- # checkpoint, so both absolute totals already include the source's work.- # Only a run superseded by a resume is dropped: create_run's claim sets- # resume_blocked while leaving output_dir intact, whereas cancelling clears- # output_dir, so a cancelled run keeps contributing the work it did do.+ # A resumed run continues its source's counters, so only a run superseded by a resume is dropped:+ # create_run's claim sets resume_blocked while leaving output_dir intact, whereas cancelling+ # clears it, so a cancelled run keeps the work it did do.steps = conn.execute(f"SELECT COALESCE(SUM(r.final_step), 0) FROM training_runs r WHERE NOT ({_superseded()})").fetchone()[0]- # num_tokens is state.num_input_tokens_seen, a running total logged at each- # step, so summing the samples multiplies the real figure. Take each run's- # final counter, the same value get_run_metrics reports.+ # num_tokens is state.num_input_tokens_seen, a running total logged at each step, so summing the+ # samples multiplies the real figure; take each run's final counter, the value get_run_metrics+ # reports.tokens = conn.execute(f"""SELECT COALESCE(SUM(run_tokens), 0) FROM (@@ -603,7 +589,6 @@ def _training_stats(conn) -> dict[str, Any]:{"id": item["id"],# A renamed run keeps the name the user gave it; otherwise fall- # back to the short model label rather than the full repo id."name": _clean_str(item["display_name"]) or _model_label(item["model_name"] or ""),"modelLabel": _model_label(item["model_name"] or ""),"datasetLabel": _model_label(item["dataset_name"] or ""),@@ -681,14 +666,13 @@ def compute_profile_stats(_merge_api_activity(fold, api_fold)training = _training_stats(conn)- # "Today" has to match the buckets above, or the newest column and the- # current streak drift by a day whenever the caller is elsewhere.+ # "Today" has to match the buckets above, or the newest column and the current streak drift by a day+ # whenever the caller is elsewhere.today = (_local_stamp(int(time.time() * 1000), zone) or datetime.now()).date()streak = _streaks(set(fold.by_day.keys()), today)daily = _daily_series(fold, today, days)- # The grid stops at today and the streaks ignore anything later, so the- # day-based headlines have to as well. A skewed client clock would+ # The grid stops at today and the streaks ignore anything later, so a skewed client clock would# otherwise name a peak day that is nowhere in the chart.past_days = {day: bucket for day, bucket in fold.by_day.items() if day <= today}peak_day = max(past_days.items(), key = lambda item: item[1]["tokens"], default = None)diff --git a/studio/backend/storage/providers_db.py b/studio/backend/storage/providers_db.pyindex 5d49bef2fb7..55cb9a25461 100644--- a/studio/backend/storage/providers_db.py+++ b/studio/backend/storage/providers_db.py@@ -113,8 +113,8 @@ def provider_bundle_transaction() -> Iterator[sqlite3.Connection]:a new endpoint with the previous key (or the inverse) while a provider editis in progress."""- # Ensure both tables exist before opening the transaction. The credential- # module commits schema initialization on its own connection.+ # Ensure both tables exist before opening the transaction. The credential module commits schema+ # initialization on its own connection.from storage import credential_secretscredential_secrets.ensure_schema()diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.pyindex 9641eda868b..1e53fda315f 100644--- a/studio/backend/storage/rag_db.py+++ b/studio/backend/storage/rag_db.py@@ -47,15 +47,12 @@ class RagExtensionUnavailable(RuntimeError):_schema_lock = threading.Lock()_schema_ready = False-# The dylib is either there or it is not, and the UI polls the KB list on a timer, so-# one warning per process says everything the repeat lines would. Same shape as the-# per-job throttle in hub/services/snapshot_progress.py.+# The dylib is either there or it is not, and the UI polls the KB list on a timer, so one warning+# per process says everything the repeats would._unavailable_lock = threading.Lock()_unavailable_warned = False-# Set once a connection has actually loaded the extension, so the request gate can-# answer without reopening the database. Only the positive verdict is kept: a failure-# stays retried per connection exactly as it was before, so a one-off cannot latch RAG-# off for the rest of the session.+# Only the positive verdict is kept: a failure stays retried per connection, so a one-off cannot+# latch RAG off for the rest of the session._extension_loaded = False@@ -246,8 +243,8 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:cols = {r[1] for r in conn.execute("PRAGMA table_info(documents)").fetchall()}if "project_id" not in cols:conn.execute("ALTER TABLE documents ADD COLUMN project_id TEXT")- # Lazy upgrade: which embedder produced a document's vectors (NULL = legacy,- # assumed current). Dedupe re-ingests when it no longer matches.+ # Which embedder produced a document's vectors (NULL = legacy, assumed current); dedupe re-ingests+ # when it no longer matches.if "embedding_model" not in cols:conn.execute("ALTER TABLE documents ADD COLUMN embedding_model TEXT")# Folder ownership makes crash cleanup unambiguous without changing retrieval.@@ -255,24 +252,22 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:conn.execute("ALTER TABLE documents ADD COLUMN linked_folder_id TEXT")if "linked_relative_path" not in cols:conn.execute("ALTER TABLE documents ADD COLUMN linked_relative_path TEXT")- # How many messages an archived turn was rendered from. NULL for everything else and- # for older archives, which fall back to counting role labels in the rendered text.+ # NULL for everything else and for older archives, which fall back to counting role labels in the rendered text.if "archive_messages" not in cols:conn.execute("ALTER TABLE documents ADD COLUMN archive_messages INTEGER")- # An archived turn group's position in its conversation. NULL elsewhere and for older- # archives, which fall back to created_at ordering. Not backfilled: created_at cannot+ # NULL for older archives, which fall back to created_at ordering. Not backfilled: created_at cannot# recover the order within a compaction epoch.if "archive_ordinal" not in cols:conn.execute("ALTER TABLE documents ADD COLUMN archive_ordinal INTEGER")- # Partial, so it is empty until a chat is compacted and the MAX() that allocates the- # next ordinal is an index probe rather than a scan.+ # Partial, so it is empty until a chat is compacted and the MAX() that allocates the next ordinal is+ # an index probe rather than a scan.conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_archive_ordinal ""ON documents(scope, archive_ordinal) WHERE archive_ordinal IS NOT NULL")# After the ALTER that adds the column on an older database. Partial, so it holds only- # folder-owned rows and is empty with nothing linked, which keeps the lexical fast-path- # gate an index probe rather than a scan of documents.+ # folder-owned rows and is empty with nothing linked, which keeps the lexical fast-path gate an+ # index probe rather than a scan.conn.execute("CREATE INDEX IF NOT EXISTS idx_documents_linked_folder ""ON documents(linked_folder_id) WHERE linked_folder_id IS NOT NULL"@@ -312,9 +307,8 @@ def get_connection() -> sqlite3.Connection:ensure_dir(db_path.parent)conn = sqlite3.connect(str(db_path))conn.row_factory = sqlite3.Row- # Wait for a lock instead of erroring immediately: a figure/scan-heavy ingest can- # hold its connection across many seconds of vision calls, and a concurrent ingest- # or autoinject read would otherwise hit "database is locked".+ # Wait for a lock instead of erroring immediately: a figure-heavy ingest can hold its connection+ # across many seconds of vision calls, and a concurrent read would hit "database is locked".conn.execute("PRAGMA busy_timeout = 5000")try:conn.enable_load_extension(True)@@ -324,8 +318,8 @@ def get_connection() -> sqlite3.Connection:conn.close()_warn_unavailable_once(exc)raise RagExtensionUnavailable(_RAG_UNAVAILABLE_MSG) from exc- # Set before the schema step: the library loaded, so RAG runs on this machine- # whatever a broken database does next. A monotonic flip, so no lock.+ # Set before the schema step: the library loaded, so RAG runs on this machine whatever a broken+ # database does next. A monotonic flip, so no lock._extension_loaded = Trueif not _schema_ready:@@ -423,17 +417,16 @@ def reconcile_orphaned_ingestion_jobs() -> int:live backend is left alone until its lease expires. No-op without RAG. Returnsthe number of jobs reset."""- # rag_available(), not RAG_AVAILABLE: a venv with the package but no vec0 binary- # would otherwise raise out of startup and be logged as a reconcile failure, when- # there is simply nothing here to reconcile.+ # rag_available(), not RAG_AVAILABLE: a venv with the package but no vec0 binary would raise out of+ # startup and be logged as a reconcile failure when there is nothing to reconcile.if not rag_available():return 0conn = get_connection()try:conn.execute("BEGIN IMMEDIATE")now = datetime.now(timezone.utc).isoformat()- # 'cancelled' is terminal too: the job stopped because its document was deleted, so- # rewriting it to failed would report a deliberate cancellation as an indexing failure.+ # "cancelled" is terminal too: the job stopped because its document was deleted, so rewriting it to+ # failed would report a deliberate cancellation as an indexing failure.rows = conn.execute("SELECT j.id, j.document_id FROM ingestion_jobs j ""WHERE j.status NOT IN ('completed', 'failed', 'cancelled') AND NOT EXISTS ("@@ -446,10 +439,8 @@ def reconcile_orphaned_ingestion_jobs() -> int:"SELECT status FROM documents WHERE id=?", (row["document_id"],)).fetchone()if doc is not None and doc["status"] == "completed":- # Worker finished indexing before the crash but didn't retire the- # job row. Mark the job completed (not failed) and keep its chunks,- # so the UI's getJob fallback after restart doesn't flag a- # searchable document as a failed ingestion.+ # The worker finished indexing before the crash but did not retire the job row: mark it completed+ # and keep its chunks, so the UI does not flag a searchable document as a failed ingestion.conn.execute("UPDATE ingestion_jobs SET status='completed', stage='done', ""progress=1.0, error=NULL WHERE id=?",@@ -466,9 +457,8 @@ def reconcile_orphaned_ingestion_jobs() -> int:"WHERE id=? AND status NOT IN ('completed', 'failed')",(row["document_id"],),)- # A failed or still-in-flight doc must not leave citable chunks- # (retrieval filters by scope, not status); also drops any chunks of a- # doc already 'failed' before the crash.+ # A failed or still-in-flight doc must not leave citable chunks, since retrieval filters by scope+ # and not status._delete_document_chunks(conn, row["document_id"])conn.execute("DELETE FROM rag_job_leases WHERE kind='ingestion' AND job_id=?",diff --git a/studio/backend/storage/research_runs_db.py b/studio/backend/storage/research_runs_db.pyindex 44f7779d475..6c4f1740191 100644--- a/studio/backend/storage/research_runs_db.py+++ b/studio/backend/storage/research_runs_db.py@@ -172,9 +172,8 @@ def _bind_assistant_locked(existing_run_id = (existing_metadata.get("researchRunId") if isinstance(existing_metadata, dict) else None)- # Only bind to an empty placeholder or this run's own message: an untagged- # reply carries text/source parts that _update_assistant drops on completion,- # so binding one silently overwrites an existing answer.+ # Only bind to an empty placeholder or this run's own message: an untagged reply carries parts+ # _update_assistant drops on completion, so binding one silently overwrites an existing answer.existing_answer = any(isinstance(part, dict)and (@@ -363,11 +362,10 @@ def rebind_cancelled(plan_revision = revision,created = now,)- # retry_count is the attempt epoch every event is stamped with, and the new question- # is a new attempt: without the bump its report would carry the stopped question's- # reasoning (get_reasoning_text joins every event at the run's current attempt) and its- # activity panel would fold the two questions together. The retry BUDGET is counted per- # question in retry(), so spending an epoch here does not spend a retry.+ # retry_count is the attempt epoch every event is stamped with, and a new question is a new+ # attempt: without the bump its report would carry the stopped question's reasoning, since+ # get_reasoning_text joins every event at the run's current attempt. The retry BUDGET is counted+ # per question, so spending an epoch here spends no retry.conn.execute("UPDATE research_runs SET user_message_id=?, assistant_message_id=?, ""status='planning', cancel_requested=0, plan_json=NULL, plan_hash=NULL, "@@ -389,8 +387,8 @@ def rebind_cancelled(conn.execute("DELETE FROM research_plan_steps WHERE run_id=?", (run_id,))conn.execute("DELETE FROM research_sources WHERE run_id=?", (run_id,))conn.execute("DELETE FROM research_document_sources WHERE run_id=?", (run_id,))- # Events replay into the activity panel, so a kept approval would show "Plan approved"- # for a question this run no longer researches.+ # Events replay into the activity panel, so a kept approval would show "Plan approved" for a+ # question this run no longer researches.conn.execute("DELETE FROM research_events WHERE run_id=? AND event_type='run.approved'", (run_id,))@@ -787,9 +785,8 @@ def retry(run_id: str, max_retries: int = 3) -> str:raise KeyError(run_id)if row["status"] not in {"failed", "cancelled"}:raise ResearchConflictError("Only failed or cancelled runs can be retried")- # Counted per question, not per run row: a thread holds one run for its lifetime and- # re-points it at each new question, so a raw retry_count would hand a fresh question- # whatever the stopped one left over -- and nothing at all once three were spent.+ # Counted per question, not per run row: a thread re-points one run at each new question, so a raw+ # retry_count would hand a fresh question whatever the stopped one left over.spent = conn.execute("SELECT COUNT(*) FROM research_events WHERE run_id=? AND event_type='run.retried' ""AND seq > COALESCE((SELECT MAX(seq) FROM research_events "@@ -871,8 +868,8 @@ def _has_claimable(now: int) -> bool:def claim_next(worker_id: str, lease_ms: int = 120_000) -> dict | None:- # Advisory only: the row can disappear between this probe and the transaction below,- # which the "row is None" branch inside already handles.+ # Advisory only: the row can disappear between this probe and the transaction below, which the "row+ # is None" branch already handles.if not _has_claimable(now_ms()):return Noneconn = get_connection()@@ -1342,7 +1339,6 @@ def wait_for_events(return eventswith _EVENTS_CHANGED:# Recheck under the condition lock so a commit cannot be missed between- # the initial query and waiting for its notification.events = list_events(run_id, after)if events:return eventsdiff --git a/studio/backend/tests/test_mcp_upgrade_compat.py b/studio/backend/tests/test_mcp_upgrade_compat.pyindex 434bc698be0..c1adc3f98a1 100644--- a/studio/backend/tests/test_mcp_upgrade_compat.py+++ b/studio/backend/tests/test_mcp_upgrade_compat.py@@ -29,6 +29,7 @@ ENV = "UNSLOTH_STUDIO_MAX_MCP_SESSIONS"# --------------------------------------------------------------------------+# Entry points an old install may still call# --------------------------------------------------------------------------@@ -45,6 +46,7 @@ def test_the_old_close_name_takes_the_same_arguments():# --------------------------------------------------------------------------+# Session cap: the name changed, the setting must not# --------------------------------------------------------------------------@@ -90,6 +92,7 @@ def test_the_cap_never_drops_below_one(monkeypatch, raw, expected):# --------------------------------------------------------------------------+# The fastmcp surface this module relies on# --------------------------------------------------------------------------@@ -127,6 +130,7 @@ def test_tool_errors_are_still_distinguishable_from_transport_errors():# --------------------------------------------------------------------------+# Rows written by an older Studio# --------------------------------------------------------------------------diff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex 12d1afeed3b..e1956b82c16 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -16,14 +16,13 @@ import os, importlib.util, platform, sysos.environ["UNSLOTH_IS_PRESENT"] = "1"-# Transformers 4.x imports TensorFlow / Flax merely because they are installed-# (`processing_utils` -> `image_transforms`), breaking Unsloth, which uses neither.-# It reads these variables once at its own import, so this has to land first. An-# explicit opt-in still wins; 5.x dropped both backends and ignores all of this.+# Transformers 4.x imports TensorFlow / Flax merely because they are installed (processing_utils+# -> image_transforms); it reads these variables once at its own import, so they have to land+# first. An explicit opt-in still wins, and 5.x ignores all of this.if "transformers" not in sys.modules:- _TRUE = {"1", "ON", "YES", "TRUE"} # Transformers' ENV_VARS_TRUE_VALUES- # Overwrite, not `setdefault`: unset means `AUTO`, i.e. "enable if installed".- # An imported backend is in use though, so opting it out breaks a `from_tf` load.+ _TRUE = {"1", "ON", "YES", "TRUE"}+ # Overwrite, not `setdefault`: unset means AUTO ("enable if installed"), but an already-imported+ # backend is in use, so opting it out breaks a `from_tf` load.for _var, _modules, _opt_ins in (("USE_TF", ("tensorflow",), ("USE_TF", "FORCE_TF_AVAILABLE")),("USE_FLAX", ("flax", "jax"), ("USE_FLAX",)),@@ -35,12 +34,10 @@ if "transformers" not in sys.modules:os.environ[_var] = "0"del _TRUE, _var, _modules, _opt_inselse:- # Transformers derives `_tf_available` / `_flax_available` from `find_spec` alone,- # so clearing the cached flags keeps `image_transforms` off a broken backend.- # `"transformers" in sys.modules` does not mean it finished importing (Python- # publishes the module before its body runs), so write the variables too: inert- # once read, decisive in that window, unconditional since waiting deadlocks.- _TRUE = {"1", "ON", "YES", "TRUE"} # Transformers' ENV_VARS_TRUE_VALUES+ # Transformers derives _tf_available / _flax_available from find_spec alone, and being in+ # sys.modules does not mean its body ran, so clear the cached flags AND write the variables:+ # inert once read, decisive in that window, unconditional since waiting deadlocks.+ _TRUE = {"1", "ON", "YES", "TRUE"}_import_utils = sys.modules.get("transformers.utils.import_utils")for _var, _flag, _const, _modules, _opt_ins, _cached in ((@@ -64,14 +61,14 @@ else:continueif any(os.environ.get(_v, "").upper() in _TRUE for _v in _opt_ins):continue- # An opt-in can be consumed and then restored, so read the snapshot- # Transformers used. Env `USE_FLAX` lands in the constant `USE_JAX`.+ # An opt-in can be consumed and then restored, so read the snapshot Transformers used. Env USE_FLAX+ # lands in the constant USE_JAX.if any(str(getattr(_import_utils, _v, "")).upper() in _TRUE for _v in _cached):continueos.environ[_var] = "0"try:- # `import_utils` copies the env into `USE_TF` / `USE_JAX` (lines 102-104)- # and derives the flags at 264 / 355, so mid-body only the constant works.+ # import_utils copies the env into USE_TF / USE_JAX (lines 102-104) and derives the flags at+ # 264 / 355, so mid-body only the constant works.if hasattr(_import_utils, _const):setattr(_import_utils, _const, "0")# Absent on 5.x, and a module proxy can refuse the write.@@ -81,16 +78,14 @@ else:passdel _TRUE, _import_utils, _var, _flag, _const, _modules, _opt_ins, _cached-# Relax Metal's context-store timeout before MLX modules can initialize Metal.-# Keep an explicit user value authoritative.+# Relax Metal's context-store timeout before MLX modules can initialize Metal; an explicit user+# value stays authoritative.if platform.system() == "Darwin" and platform.machine() == "arm64":os.environ.setdefault("AGX_RELAX_CDM_CTXSTORE_TIMEOUT", "1")+# Legacy Windows consoles (cp1252) cannot encode Unsloth's emoji/box-drawing glyphs and crash with+# UnicodeEncodeError; errors="replace" guarantees no crash on an unencodable glyph.# ── Windows console UTF-8 safety ─────────────────────────────────────────────-# Legacy Windows consoles (cp1252) can't encode Unsloth's emoji/box-drawing-# glyphs and crash with UnicodeEncodeError. Force stdout/stderr to UTF-8 only on-# Windows and only when not already UTF-8; no-op elsewhere. errors="replace"-# guarantees we never crash on an unencodable glyph.if platform.system() == "Windows":import sys as _sysfor _name in ("stdout", "stderr"):@@ -125,9 +120,8 @@ def _bytes_to_gb(value):def _is_mlx_available():- # Transitional import barrier: keep non-Apple-Silicon imports from touching- # unsloth_zoo until unsloth_zoo.mlx is import-safe on GPU hosts. Then this- # can collapse back to the centralized zoo runtime call below.+ # Transitional import barrier: keep non-Apple-Silicon imports from touching unsloth_zoo until+ # unsloth_zoo.mlx is import-safe on GPU hosts.if (os.environ.get("UNSLOTH_FORCE_GPU_PATH", "0") == "1"or platform.system() != "Darwin"@@ -146,8 +140,8 @@ def _is_mlx_available():_IS_MLX = _is_mlx_available()if _IS_MLX:- # _gpu_init does this on the GPU path; the MLX path never reaches it, so- # torchao 0.18 + torch < 2.10 dies on `ScalingType`. No-op otherwise.+ # _gpu_init does this on the GPU path and the MLX path never reaches it, so torchao 0.18 + torch <+ # 2.10 dies on `ScalingType`.try:from .import_fixes import fix_torchao_torch_symbol_skew as _fix_torchao_fix_torchao()@@ -155,16 +149,15 @@ if _IS_MLX:except Exception:passtry:- # Same reason: MLX audio reaches xcodec2 -> torchtune -> the old- # torchao.dtypes.nf4tensor path.+ # Same reason: MLX audio reaches xcodec2 -> torchtune -> the old torchao.dtypes.nf4tensor path.from .import_fixes import fix_torchao_nf4tensor_move as _fix_nf4_fix_nf4()del _fix_nf4except Exception:passtry:- # Same reason: this branch imports transformers itself further down, so a- # --no-deps floor miss surfaces here with the same wrong remedy.+ # Same reason: this branch imports transformers itself further down, so a --no-deps floor miss would+ # surface here with the same wrong remedy.from .import_fixes import check_transformers_dependency_versions as _check_tf_deps_check_tf_deps()del _check_tf_deps@@ -177,9 +170,8 @@ if _IS_MLX:"Unsloth: MLX support requires `unsloth-zoo` with MLX modules. ""Reinstall with `pip install unsloth-zoo` or rerun install.sh.") from _e- # An older unsloth-zoo satisfies `import unsloth_zoo` but lacks the- # mlx.trainer / mlx.loader submodules. Surface a friendly install hint- # instead of a raw ImportError on the submodule path.+ # An older unsloth-zoo satisfies `import unsloth_zoo` but lacks the mlx.trainer / mlx.loader+ # submodules; surface an install hint instead of a raw ImportError.try:from unsloth_zoo.mlx.trainer import (MLXTrainer,@@ -202,9 +194,8 @@ if _IS_MLX:import types as _typesimport warnings as _warnings- # unsloth_zoo is a different distribution, pinned `>=`, so borrowing its number- # reported neither the installed core nor the latest zoo. `_version` imports nothing,- # so this stays torch-free while agreeing with the GPU path and `pip show unsloth`.+ # unsloth_zoo is a different distribution, pinned >=, so borrowing its number reported neither the+ # installed core nor the latest zoo. `_version` imports nothing, so this stays torch-free.from ._version import __version__DEVICE_TYPE = "mlx"@@ -259,8 +250,8 @@ if _IS_MLX:target = kwargs.get("device", device)if _is_mlx_cuda_device_target(target):return self- # device given by keyword: don't also pass the positional None, or the- # original raises "multiple values for 'device'" (e.g. .to(device="cpu")).+ # device given by keyword: do not also pass the positional None, or the original raises "multiple+ # values for 'device'".if "device" in kwargs:return original_to(self, *args, **kwargs)return original_to(self, device, *args, **kwargs)@@ -271,8 +262,8 @@ if _IS_MLX:_patch_mlx_batch_encoding_to_cuda()- # Load raw_text helpers without executing dataprep/__init__.py, which- # imports synthetic.py -> torch and would defeat the torch-free MLX path.+ # Load raw_text helpers without executing dataprep/__init__.py, which imports synthetic.py -> torch+ # and would defeat the torch-free MLX path.from pathlib import Path as _Path_raw_text_path = _Path(__file__).resolve().parent / "dataprep" / "raw_text.py"@@ -635,8 +626,8 @@ if _IS_MLX:try:return _normalize_mlx_optimizer_name(value)except ValueError:- # Older unsloth-zoo lacks CUDA/TRL optimizer aliases; map common- # adamw_* names so notebook defaults (optim="adamw_8bit") still work.+ # Older unsloth-zoo lacks the CUDA/TRL optimizer aliases, so map the common adamw_* names and keep+ # notebook defaults (optim="adamw_8bit") working.opt = str(getattr(value, "value", value) or "adamw").strip().lower()opt = opt.rsplit(".", 1)[-1].replace("-", "_")if opt in _MLX_ADAMW_OPTIMIZER_ALIASES:@@ -708,8 +699,7 @@ if _IS_MLX:return Falseif _is_mlx_training_args_like(args[1]):return True- # TRL callers often pass explicit defaults:- # SFTTrainer(model, None, None, train_dataset, ...)+ # TRL callers often pass explicit defaults: SFTTrainer(model, None, None, train_dataset, ...)return len(args) >= 3 and args[1] is None and (args[2] is None or callable(args[2]))def _assign_mlx_positional_kwarg(kwargs, name, value):@@ -834,9 +824,8 @@ if _IS_MLX:)max_length_value = kwargs.get("max_length", None)- # Only the canonical max_seq_length marks context length explicit; TRL- # max_length stays a compatibility alias and defers to the model's- # context length when one is available.+ # Only the canonical max_seq_length marks context length explicit; TRL max_length stays a+ # compatibility alias that defers to the model's context length.max_seq_length_explicit = (_positive_mlx_context_length(kwargs.get("max_seq_length", None)) is not None)@@ -1326,7 +1315,7 @@ if _IS_MLX:kwargs[collator_key] = collator_kwargs[collator_key]_raise_unsupported_mlx_vision_collator_kwargs(collator_kwargs)elif _is_mlx_native_text_collator(data_collator):- pass # redundant on MLX; MLXTrainer batches/masks/pads natively+ passelse:raise NotImplementedError("Unsloth MLX: custom data_collator is not supported by "@@ -1434,9 +1423,8 @@ if _IS_MLX:def train_on_responses_only(*args, **kwargs):"""Mask non-response tokens through the shared zoo dataset helper."""- # Prefer the chat_templates export, which bounds the zoo's dataset.map()- # worker count (issue #2693). It is None on a torch-free host; fall back- # so the zoo raises its own ImportError, not "NoneType is not callable".+ # Prefer the chat_templates export, which bounds the zoo's dataset.map() worker count (#2693); it is+ # None on a torch-free host, so fall back and let the zoo raise its own ImportError.from .chat_templates import train_on_responses_only as _train_on_responses_onlyif _train_on_responses_only is None:@@ -1460,9 +1448,8 @@ if _IS_MLX:safe_exports.append(name)return safe_exports- # trl trainers with no MLX implementation yet. Swap them for stubs that fail- # with a clear message instead of importing the real torch/CUDA trainer and- # crashing deep inside it, so an unmigrated GRPO/DPO/ORPO notebook is legible.+ # Stub the trl trainers with no MLX implementation yet, so an unmigrated GRPO/DPO/ORPO notebook+ # fails with a clear message instead of crashing deep inside the real torch/CUDA trainer._MLX_UNSUPPORTED_TRL_TRAINERS = ("GRPOTrainer","DPOTrainer",@@ -1516,17 +1503,13 @@ if _IS_MLX:_trl.SFTTrainer = UnslothTrainer_trl.SFTConfig = _MLXSFTConfig- # Only retarget trainers the installed trl actually exposes (don't invent- # attributes); idempotent so re-importing unsloth is a no-op.- # Decide what to stub from trl's declared exports (__all__) and already- # materialized attrs only. A getattr probe here would trigger trl's lazy- # trainer import, pulling torch and breaking `import unsloth` on torch-free- # MLX just to check existence.+ # Only retarget trainers the installed trl actually exposes; idempotent, so re-importing unsloth is a no-op.+ # Names come from trl's __all__ and already materialized attrs, never a getattr probe: resolving+ # one triggers trl's lazy trainer import and pulls torch, breaking `import unsloth` on MLX._trl_exports = set(getattr(_trl, "__all__", ()) or ())- # Stub every non-SFT trainer trl exposes, not just a fixed list, so newer- # trainers (RLOOTrainer, ...) also fail with a clear MLX message instead- # of importing the real torch trainer. Names come from __all__ so we never- # resolve them (that would trigger trl's lazy import and pull torch).+ # Stub every non-SFT trainer trl exposes, not just a fixed list, so newer trainers (RLOOTrainer, ...)+ # also fail with a clear MLX message instead of importing the real torch trainer. Names come from+ # __all__ so we never resolve them (that would trigger trl's lazy import and pull torch)._unsupported = set(_MLX_UNSUPPORTED_TRL_TRAINERS) | {_n for _n in _trl_exports if _n.endswith("Trainer") and _n != "SFTTrainer"}@@ -1595,10 +1578,9 @@ else:pass-# Both paths, GPU and MLX: a `pip install --target` / PYTHONPATH layout makes-# dill pickle whole modules by value, and every training path here builds a-# `datasets.Dataset`, which fingerprints through dill. No-op on an ordinary-# install. See `import_fixes.fix_dill_module_by_value_pickling`.+# A `pip install --target` / PYTHONPATH layout makes dill pickle whole modules by value, and every+# training path here builds a datasets.Dataset, which fingerprints through dill. See+# import_fixes.fix_dill_module_by_value_pickling.try:from .import_fixes import fix_dill_module_by_value_pickling as _fix_dill_fix_dill()diff --git a/unsloth/_compressed_quantize.py b/unsloth/_compressed_quantize.pyindex ec7564ef2f3..217cbfba253 100644--- a/unsloth/_compressed_quantize.py+++ b/unsloth/_compressed_quantize.py@@ -65,9 +65,8 @@ def compressed_ignore_patterns(config):class name) match."""ignore = ["lm_head"]- # Skip the same modules RedHatAI/NVIDIA skip for the Qwen3.5 / Qwen3-Next family (these also- # have shapes not divisible by the grouped-scheme group_size, which would otherwise error).- # No-ops elsewhere. Hybrid linear attention, VLM vision tower, and the MTP/speculative head.+ # Skip the same modules RedHatAI/NVIDIA skip for the Qwen3.5 / Qwen3-Next family: their shapes are+ # not divisible by the grouped-scheme group_size and would error. No-ops elsewhere.ignore += ["re:.*\\.linear_attn\\..*", "re:.*\\.visual\\..*", "re:.*mtp.*"]if _is_moe(config):# Keep MoE routing layers unquantized: the router gate and (Qwen) shared-expert gate.@@ -96,8 +95,8 @@ def _build_calibration_dataset(tokenizer, kind, value, num_samples, max_seq_lengexcept (ValueError, KeyError):from datasets import get_dataset_split_namestry:- # Resolve the first split name so only num_samples rows are fetched, instead of- # downloading/materializing the whole dataset just to take a small slice.+ # Resolve the first split name so only num_samples rows are fetched, instead of materializing the+ # whole dataset just to take a small slice.split = get_dataset_split_names(value)[0]ds = load_dataset(value, split = f"{split}[:{num_samples}]")except Exception:@@ -136,16 +135,16 @@ def _build_calibration_dataset(tokenizer, kind, value, num_samples, max_seq_lengcols = set(ds.column_names)if "input_ids" in cols:- # Drop non-model-input columns (e.g. a leftover 'messages' list) so llm-compressor's- # collator does not try to batch them.+ # Drop non-model-input columns (e.g. a leftover 'messages' list) so llm-compressor's collator does+ # not try to batch them.keep = {"input_ids", "attention_mask", "labels", "position_ids"}extra = [c for c in ds.column_names if c not in keep]if extra:ds = ds.remove_columns(extra)return dsif "messages" in cols:- # Base / non-chat tokenizers have no chat template; concatenate message contents instead- # of calling apply_chat_template (which would raise).+ # Base / non-chat tokenizers have no chat template, and apply_chat_template would raise, so+ # concatenate message contents instead.has_chat_template = bool(getattr(_tok, "chat_template", None))def _content_to_text(content):@@ -233,8 +232,8 @@ def main():from llmcompressor import oneshotfrom llmcompressor.modifiers.quantization import QuantizationModifier- # Import the VLM auto-class only when needed - some transformers versions lack it, and the- # text path must not fail just because that newer class is unavailable.+ # Import the VLM auto-class only when needed: some transformers versions lack it, and the text path+ # must not fail over a newer class.if args.is_vlm:from transformers import AutoProcessortry:@@ -255,8 +254,8 @@ def main():model.eval()# A tokenizer may be absent if the caller saved it separately; only calibration needs one.try:- # The tokenizer/processor has its own trust flag: consent for one component must not- # let the other's custom code run.+ # The tokenizer/processor has its own trust flag: consent for one component must not let the other's+ # custom code run.tokenizer = auto_proc.from_pretrained(args.model, trust_remote_code = args.trust_remote_code_tokenizer)@@ -268,8 +267,8 @@ def main():)tokenizer = None- # MoE models: keep the router/gate unquantized (it decides expert routing) and calibrate every- # expert even if the sample set does not route tokens to all of them.+ # MoE models: keep the router/gate unquantized and calibrate every expert even if the sample set+ # does not route tokens to all of them.config = getattr(model, "config", None)is_moe = _is_moe(config)ignore = compressed_ignore_patterns(config)@@ -286,10 +285,9 @@ def main():args.num_calibration_samples,args.max_seq_length,)- # Use the sequential pipeline: it onloads layer-by-layer, so models that do not fit in- # memory at once can still calibrate. Running here in a clean process (Unsloth's attention- # patches are absent) means tracing works; fall back to the memory-hungry "basic" pipeline- # only if tracing fails.+ # The sequential pipeline onloads layer-by-layer, so a model that does not fit at once still+ # calibrates; tracing works only because this runs in a clean process without Unsloth's attention+ # patches. Fall back to the memory-hungry "basic" pipeline only if tracing fails.try:oneshot(model = model,@@ -306,10 +304,8 @@ def main():"retrying with the 'basic' pipeline (needs the full model to fit in memory).",flush = True,)- # Free the partially-processed model before loading a fresh copy, so the fallback does- # not transiently hold two copies on GPU. llm-compressor keeps the model in a global- # session after a failed run, so reset it first; also drop the traceback frames (e) and- # the local reference that pin the model.+ # Free the partially-processed model before loading a fresh copy so the fallback never holds two+ # on GPU: llm-compressor keeps the model in a global session after a failed run.import gc as _gcimport torch as _torchdiff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex d843e179dbd..bf771f0c677 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -23,7 +23,6 @@ os.environ["UNSLOTH_IS_PRESENT"] = "1"critical_modules = ["trl", "transformers", "peft"]already_imported = [mod for mod in critical_modules if mod in sys.modules]-# Fix some issues before importing other packagesfrom .import_fixes import (fix_message_factory_issue,fix_torch_check_is_size,@@ -41,15 +40,11 @@ from .import_fixes import (fix_huggingface_hub,)-# Redirect a read-only Hugging Face cache before anything below imports-# huggingface_hub / transformers / vllm (disable_broken_vllm probes `import vllm`-# and its compiled extensions, check_fbgemm_gpu_version imports transformers,-# fix_huggingface_hub imports huggingface_hub) -- any of which would freeze Hub's-# cache constants with the un-redirected paths. unsloth_zoo runs the same redirect-# at import, but only after these probes. hf_cache.py is stdlib-only, so load it-# straight from its file without triggering the full unsloth_zoo init this early;-# the zoo's later call is an idempotent no-op. Older unsloth_zoo without it is-# skipped silently.+# Redirect a read-only Hugging Face cache before anything below imports huggingface_hub /+# transformers / vllm (disable_broken_vllm probes `import vllm`, check_fbgemm_gpu_version imports+# transformers, fix_huggingface_hub imports huggingface_hub), any of which would freeze Hub's+# cache constants with the un-redirected paths. hf_cache.py is stdlib-only, so it loads straight+# from its file without triggering the full unsloth_zoo init this early.try:import importlib.util as _importlib_utilfrom pathlib import Path as _Path@@ -79,22 +74,16 @@ disable_broken_vllm()fix_message_factory_issue()fix_torch_check_is_size()fix_torchao_torch_symbol_skew()-# The above fixes THIS process only; vLLM's model-architecture inspector-# is a subprocess that imports torchao itself and hits the same ImportError.+# The above fixes THIS process only; vLLM's model-architecture inspector is a subprocess that+# imports torchao itself and hits the same ImportError.propagate_torchao_fix_to_subprocesses()-# Warn, do not raise: this only adds the correct remedy just before transformers-# prints its misleading one, without turning a metadata-derived floor into a hard-# import failure of our own.+# Warn, do not raise: this only adds the correct remedy just before transformers prints its misleading one.check_transformers_dependency_versions()check_fbgemm_gpu_version()torchvision_compatibility_check()-# Ahead of `import unsloth_zoo` below, not with the other import fixes further-# down. unsloth_zoo's temporary_patches reach transformers.processing_utils,-# which imports transformers.audio_utils, which imports torchaudio -- so a-# torchaudio that raises at extension init takes the whole unsloth import down-# roughly 95 lines before the late block would have neutralised it. Measured:-# Kaggle-Muse_Glimmer_(30B)-GRPO died at cell 4 with the guard present but not-# yet run.+# Must precede `import unsloth_zoo`: its temporary_patches reach transformers.processing_utils ->+# audio_utils -> torchaudio, so a torchaudio raising at extension init takes the whole unsloth+# import down before the late guard would have neutralised it.disable_torchaudio_if_cuda_mismatched()fix_diffusers_warnings()fix_huggingface_hub()@@ -112,8 +101,8 @@ del torchvision_compatibility_checkdel fix_diffusers_warningsdel fix_huggingface_hub-# Unsloth patches these libraries at import time; if imported first, the-# unoptimized versions run, risking OOM or slower training.+# Unsloth patches these libraries at import time; if they are imported first the unoptimized+# versions run, risking OOM or slower training.if already_imported:# stacklevel=2 points the warning at the user's import linewarnings.warn(@@ -125,8 +114,7 @@ if already_imported:)del already_imported, critical_modules-# Pin BNB_ROCM_VERSION before bitsandbytes is first imported (`import-# unsloth_zoo` below pulls it in on ROCm hosts).+# Pin BNB_ROCM_VERSION before bitsandbytes is first imported (`import unsloth_zoo` below pulls it in on ROCm hosts).from .import_fixes import maybe_set_windows_rocm_bnb_versionmaybe_set_windows_rocm_bnb_version()@@ -137,15 +125,11 @@ del maybe_set_windows_rocm_bnb_version# Fixes https://github.com/unslothai/unsloth/issues/1266os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"-# [TODO] Check why some GPUs don't work-# "pinned_use_cuda_host_register:True,"\-# "pinned_num_register_threads:8"-from importlib.metadata import version as importlib_versionfrom importlib.metadata import PackageNotFoundError-# Check for unsloth_zoo+# Try importing PyTorch and check versiontry:unsloth_zoo_version = importlib_version("unsloth_zoo")if Version(unsloth_zoo_version) < Version("2026.8.15"):@@ -153,14 +137,6 @@ try:"Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n""Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`")- # if os.environ.get("UNSLOTH_DISABLE_AUTO_UPDATES", "0") == "0":- # try:- # os.system("pip install --upgrade --no-cache-dir --no-deps unsloth_zoo")- # except:- # try:- # os.system("pip install --upgrade --no-cache-dir --no-deps --user unsloth_zoo")- # except:- # raise ImportError("Unsloth: Please update unsloth_zoo via `pip install --upgrade --no-cache-dir --no-deps unsloth_zoo`")import unsloth_zooexcept PackageNotFoundError:raise ImportError(@@ -170,7 +146,6 @@ except:raisedel PackageNotFoundError, importlib_version-# Try importing PyTorch and check versiontry:import torchexcept ModuleNotFoundError:@@ -191,7 +166,6 @@ from unsloth_zoo.device_type import ()from .device_type import arch_lacks_bf16, hip_visible_archs-# Fix other issuesfrom .import_fixes import (fix_transformers5_bare_annotation_configs,fix_transformers_fully_masked_rows,@@ -230,19 +204,18 @@ from .import_fixes import (# Must run first: guards PretrainedConfig before vLLM defines its config classes.fix_transformers5_bare_annotation_configs()-# Probe-gated: no-ops unless this transformers really hands SDPA a query row-# that attends to nothing. Ordered here, before anything imports a model, so a-# plain `transformers.generate` in the same process is covered too -- which is-# the case unsloth #9708 was measured on.+# Probe-gated: no-ops unless this transformers really hands SDPA a query row that attends to+# nothing. Ordered here, before anything imports a model, so a plain transformers.generate in the+# same process is covered too (#9708).fix_transformers_fully_masked_rows()fix_xformers_performance_issue()-# Must run AFTER fix_xformers_performance_issue (it rewrites xformers' cutlass.py on disk, so it-# must precede any xformers import) and BEFORE models/_utils.py imports xformers.ops.+# Must run AFTER fix_xformers_performance_issue (it rewrites xformers' cutlass.py on disk) and+# BEFORE models/_utils.py imports xformers.ops.fix_flash_attn_4_namespace_shadow()fix_vllm_aimv2_issue()fix_vllm_lora_tokenizer_module()-# torchao 0.18.0 moved nf4tensor; torchtune (via xcodec2) still imports the-# old path. Lazy alias, so it costs nothing unless asked for.+# torchao 0.18.0 moved nf4tensor; torchtune (via xcodec2) still imports the old path. Lazy alias, so+# it costs nothing unless asked for.fix_torchao_nf4tensor_move()# Check vLLM + torch < 2.9.0 + SM100 compatibility BEFORE importing vLLMcheck_vllm_torch_sm100_compatibility()@@ -250,9 +223,8 @@ fix_vllm_guided_decoding_params()fix_trl_vllm_ascend()fix_vllm_pdl_blackwell()fix_triton_compiled_kernel_missing_attrs()-# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss temporary-# patches raise the dynamo recompile limits, so those settings reach the-# autograd worker threads on torch >= 2.12.+# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss patches raise the dynamo+# recompile limits, so those settings reach the autograd worker threads on torch >= 2.12.fix_dynamo_config_thread_visibility()patch_trunc_normal_precision_issue()ignore_logger_messages()@@ -270,14 +242,14 @@ patch_vllm_for_notebooks()patch_torchcodec_audio_decoder()disable_torchcodec_if_broken()disable_broken_wandb()-# Must run before patch_peft_weight_converter_compatibility: stubs the-# transformers v5 submodules peft 0.19.x imports, so the next patch can wrap-# build_peft_weight_mapping instead of being swallowed by its ImportError.+# Must run before patch_peft_weight_converter_compatibility: it stubs the transformers v5+# submodules peft 0.19.x imports, so the next patch can wrap build_peft_weight_mapping instead of+# being swallowed by its ImportError.fix_peft_transformers_tensor_parallel_import_compat()fix_peft_transformers_weight_conversion_import()patch_peft_weight_converter_compatibility()-# After peft is importable, so the already-bound `is_torchao_available` in-# peft.tuners.lora.torchao is replaced too, not just import_utils'.+# After peft is importable, so the already-bound is_torchao_available in peft.tuners.lora.torchao is+# replaced too, not just import_utils'.fix_peft_stale_torchao_import_error()patch_accelerate_recursively_apply()@@ -316,16 +288,9 @@ del patch_accelerate_recursively_apply# Torch 2.4 has including_emulationif DEVICE_TYPE == "cuda" and not torch.cuda.is_available():- # UNSLOTH_ALLOW_CPU=1 is the documented way to import on a host that has a- # CUDA-built torch and no usable device (driverless container, CI runner, a- # laptop with the runtime and no card). get_device_type() deliberately keeps- # DEVICE_TYPE at "cuda" there, so this branch is entered with nothing to- # query and torch.cuda.get_device_capability() raises out of _lazy_init().- # Ask whether a device is present before asking what it can do.- #- # No device means no capability to report, so claim the conservative answer.- # SUPPORTS_BFLOAT16 = False only costs float32; True would fail at the first- # cast. is_bf16_supported() is stubbed to match rather than left to raise.+ # UNSLOTH_ALLOW_CPU=1 keeps DEVICE_TYPE at "cuda" on a host with a CUDA-built torch and no usable+ # device, so ask whether a device is present before asking what it can do: get_device_capability()+ # would raise out of _lazy_init(). is_bf16_supported() is stubbed to match.SUPPORTS_BFLOAT16 = Falsetorch.cuda.is_bf16_supported = lambda *args, **kwargs: Falseelif DEVICE_TYPE == "cuda":@@ -360,8 +325,8 @@ elif DEVICE_TYPE == "hip":torch.cuda.is_bf16_supported = is_bf16_supporteddel old_is_bf16_supportedelif DEVICE_TYPE == "xpu":- # torch.xpu.is_bf16_supported() does not have including_emulation- # set SUPPORTS_BFLOAT16 as torch.xpu.is_bf16_supported()+ # torch.xpu.is_bf16_supported() does not have including_emulation set SUPPORTS_BFLOAT16 as+ # torch.xpu.is_bf16_supported()SUPPORTS_BFLOAT16 = torch.xpu.is_bf16_supported()# For Gradio HF Spaces?@@ -371,6 +336,7 @@ import tritonif DEVICE_TYPE == "cuda":libcuda_dirs = lambda: Noneif Version(triton.__version__) >= Version("3.0.0"):+ # Try loading bitsandbytes and tritontry:from triton.backends.nvidia.driver import libcuda_dirsexcept:@@ -378,13 +344,11 @@ if DEVICE_TYPE == "cuda":else:from triton.common.build import libcuda_dirs- # Try loading bitsandbytes and tritontry:import bitsandbytes as bnb- # Bind the submodule by name: a half-imported bitsandbytes leaves the parent- # without a `functional` attribute, which would otherwise be misreported below- # as a CUDA linking failure. See unsloth/kernels/utils.py.+ # Bind the submodule by name: a half-imported bitsandbytes leaves the parent without a `functional`+ # attribute, which would otherwise be misreported below as a CUDA linking failure.import bitsandbytes.functional as bnb_functionalexcept:print(@@ -397,12 +361,9 @@ if DEVICE_TYPE == "cuda":libcuda_dirs()except:if not torch.cuda.is_available():- # UNSLOTH_ALLOW_CPU=1 on a driverless host. DEVICE_TYPE is "cuda"- # only because the caller asked the import to survive without a- # device, so a missing libcuda is the expected state, not broken- # linkage. Repairing it would ldconfig the host's linker cache (this- # branch runs as root, the default in a container) through an- # unguarded `ls` subprocess, to link a device that is not there.+ # UNSLOTH_ALLOW_CPU=1 on a driverless host: a missing libcuda is the expected state, not broken+ # linkage, and repairing it would ldconfig the host's linker cache as root for a device that is+ # not there.passelif hasattr(os, "geteuid") and os.geteuid() == 0:warnings.warn("Unsloth: Running `ldconfig /usr/lib64-nvidia` to link CUDA.")@@ -434,12 +395,14 @@ if DEVICE_TYPE == "cuda":if bnb is not None:importlib.reload(bnb)importlib.reload(triton)+ # Same degradation as the cuda branch above: no bnb means no 4bit, not a failed `import unsloth`.try:libcuda_dirs = lambda: Noneif Version(triton.__version__) >= Version("3.0.0"):try:from triton.backends.nvidia.driver import libcuda_dirsexcept:+ # TODO: check triton for intel installed properly.passelse:from triton.common.build import libcuda_dirs@@ -466,8 +429,7 @@ elif DEVICE_TYPE == "hip":# NO-OP for rocm devicepasselif DEVICE_TYPE == "xpu":- # Same degradation as the cuda branch above: no bnb means no 4bit, not a- # failed `import unsloth`.+ # Same degradation as the cuda branch above: no bnb means no 4bit, not a failed `import unsloth`.try:import bitsandbytes as bnbexcept Exception:@@ -476,7 +438,6 @@ elif DEVICE_TYPE == "xpu":)bnb = None- # TODO: check triton for intel installed properly.passfrom .models import *@@ -497,8 +458,7 @@ from unsloth_zoo.rl_environments import (launch_openenv,)-# Patch TRL trainers for backwards compatibility. Skipped under-# UNSLOTH_ALLOW_CPU=1 (CPU-only CI): rebinding trl.SFTTrainer.__init__-# changes inspect.getsource() and corrupts downstream drift detectors.+# Skipped under UNSLOTH_ALLOW_CPU=1 (CPU-only CI): rebinding trl.SFTTrainer.__init__ changes+# inspect.getsource() and corrupts downstream drift detectors.if os.environ.get("UNSLOTH_ALLOW_CPU", "0") != "1":_patch_trl_trainer()diff --git a/unsloth/bnb_availability.py b/unsloth/bnb_availability.pyindex 9d14bbb0f30..7176aedf2ec 100644--- a/unsloth/bnb_availability.py+++ b/unsloth/bnb_availability.py@@ -40,8 +40,8 @@ _C_SYMBOLS = ("cdequantize_blockwise_fp16_nf4","cdequantize_blockwise_bf16_nf4",)-# 4bit inference is a gemv on xpu and a naive gemm elsewhere; probing the wrong pair-# would write off a perfectly good wheel.+# 4bit inference is a gemv on xpu and a naive gemm elsewhere; probing the wrong pair would write off+# a perfectly good wheel._C_SYMBOLS_XPU = ("cgemv_4bit_inference_fp16","cgemv_4bit_inference_bf16",@@ -70,8 +70,8 @@ def check_native_kernels(bnb, device_type):raise ImportError("Unsloth: `bitsandbytes` is not installed.")functional = getattr(bnb, "functional", None)if functional is None:- # A part-initialised bitsandbytes leaves the parent without the attribute while- # the submodule stays in sys.modules, which `import x.y as z` reads directly.+ # A part-initialised bitsandbytes leaves the parent without the attribute while the submodule stays+ # in sys.modules, which `import x.y as z` reads directly.import bitsandbytes.functional as functionallib = functional.libdiff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.pyindex 317b4106ac5..15d081e5c08 100644--- a/unsloth/chat_templates.py+++ b/unsloth/chat_templates.py@@ -43,8 +43,8 @@ try:standardize_data_formats,)except ImportError:- # dataset_utils pulls torch; keep chat_templates importable on torch-free- # (MLX) hosts, which expose these via the backend-specific wrappers instead.+ # dataset_utils pulls torch; keep chat_templates importable on torch-free (MLX) hosts, which expose+ # these via the backend-specific wrappers._zoo_train_on_responses_only = standardize_data_formats = Noneif _zoo_train_on_responses_only is None:@@ -57,12 +57,10 @@ else:@_functools.wraps(_zoo_train_on_responses_only)def train_on_responses_only(*args, **kwargs):- # The zoo still auto-sizes its dataset.map() workers with the uncapped- # min(max(cpu_count + 4, 2), 64) heuristic (issue #2693) and is a- # separate package, so bound the count on the way in. See- # resolve_responses_only_num_proc; the local copy is only the fallback- # for a zoo predating it. Top level, not under unsloth.utils, whose- # __init__ imports torch: this path has to stay MLX-safe.+ # The zoo auto-sizes its dataset.map() workers with the uncapped min(max(cpu_count + 4, 2), 64)+ # heuristic (#2693) and is a separate package, so bound the count on the way in; see+ # resolve_responses_only_num_proc. Top level, not under unsloth.utils, whose __init__ imports+ # torch: this path has to stay MLX-safe.try:from unsloth_zoo.dataset_num_proc import resolve_responses_only_num_procexcept ImportError:@@ -71,7 +69,7 @@ else:try:bound = _ZOO_RESPONSES_ONLY_SIGNATURE.bind_partial(*args, **kwargs)except TypeError:- # Signature drift, or a bad call. Let the zoo raise its own error.+ # Signature drift, or a bad call: let the zoo raise its own error.return _zoo_train_on_responses_only(*args, **kwargs)# The zoo returns the masking closure before reading num_proc.@@ -93,7 +91,6 @@ def _ollama_template(name: str):return OLLAMA_TEMPLATES[name]# =========================================== Unsloth-# Unsloth efficient template leverages from Zephyrunsloth_template = \"{{ bos_token }}"\"{% if messages[0]['role'] == 'system' %}"\@@ -122,8 +119,8 @@ unsloth_eos_token = "eos_token"CHAT_TEMPLATES["unsloth"] = (unsloth_template, unsloth_eos_token, False, unsloth_ollama,)DEFAULT_SYSTEM_MESSAGE["unsloth"] = "You are a helpful assistant to the user"-# =========================================== Zephyr# Zephyr has no BOS!+# =========================================== Zephyrzephyr_template = \"{% for message in messages %}"\"{% if message['role'] == 'user' %}"\@@ -142,10 +139,10 @@ zephyr_ollama = _ollama_template("zephyr")zephyr_eos_token = "eos_token"CHAT_TEMPLATES["zephyr"] = (zephyr_template, zephyr_eos_token, False, zephyr_ollama,)-DEFAULT_SYSTEM_MESSAGE["zephyr"] = None # No system message in Zephyr+DEFAULT_SYSTEM_MESSAGE["zephyr"] = None+# ChatML has no BOS and no EOS: <|im_start|> / <|im_end|> act as BOS / EOS.# =========================================== ChatML-# ChatML has no BOS and not EOS! Rather <|im_start|> and <|im_end|> acts as BOS / EOS.chatml_template = \"{% for message in messages %}"\"{% if message['role'] == 'user' %}"\@@ -164,10 +161,10 @@ chatml_ollama = _ollama_template("chatml")chatml_eos_token = "<|im_end|>"CHAT_TEMPLATES["chatml"] = (chatml_template, chatml_eos_token, True, chatml_ollama,)-DEFAULT_SYSTEM_MESSAGE["chatml"] = None # No system message in ChatML+DEFAULT_SYSTEM_MESSAGE["chatml"] = None-# =========================================== Mistral-1# Mistral Instruct doesn't allow system prompts, so we append it to the user message.+# =========================================== Mistral-1mistral_template = \"{{ bos_token }}"\"{% if messages[0]['role'] == 'system' %}"\@@ -196,10 +193,10 @@ mistral_ollama = _ollama_template("mistral")mistral_eos_token = "eos_token"CHAT_TEMPLATES["mistral"] = (mistral_template, mistral_eos_token, False, mistral_ollama,)-DEFAULT_SYSTEM_MESSAGE["mistral"] = None # No system message in Mistral+DEFAULT_SYSTEM_MESSAGE["mistral"] = None-# =========================================== Llama-2# Adds BOS to every convo! And weird <<SYS>> system messages.+# =========================================== Llama-2llama_template = \"{% if messages[0]['role'] == 'system' %}"\"{% if messages[1]['role'] == 'user' %}"\@@ -227,10 +224,10 @@ llama_ollama = _ollama_template("llama")llama_eos_token = "eos_token"CHAT_TEMPLATES["llama"] = (llama_template, llama_eos_token, False, llama_ollama,)-DEFAULT_SYSTEM_MESSAGE["llama"] = None # No system message in Llama+DEFAULT_SYSTEM_MESSAGE["llama"] = None-# =========================================== Vicuna# https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template+# =========================================== Vicunavicuna_template = \"{{ bos_token }}"\"{% if messages[0]['role'] == 'system' %}"\@@ -260,8 +257,8 @@ vicuna_eos_token = "eos_token"CHAT_TEMPLATES["vicuna"] = (vicuna_template, vicuna_eos_token, False, vicuna_ollama,)DEFAULT_SYSTEM_MESSAGE["vicuna"] = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."-# =========================================== Vicuna Old# https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template+# =========================================== Vicuna Oldvicuna_old_template = \"{{ bos_token }}"\"{% if messages[0]['role'] == 'system' %}"\@@ -293,8 +290,8 @@ DEFAULT_SYSTEM_MESSAGE["vicuna_old"] = "A chat between a curious human and an arCHAT_TEMPLATES["vicuna old"] = CHAT_TEMPLATES["vicuna_old"]DEFAULT_SYSTEM_MESSAGE["vicuna old"] = DEFAULT_SYSTEM_MESSAGE["vicuna_old"]-# =========================================== Alpaca multi turn# https://github.com/tatsu-lab/stanford_alpaca Changed for multi-turn convos+# =========================================== Alpaca multi turnalpaca_template = \"{{ bos_token }}"\"{% if messages[0]['role'] == 'system' %}"\@@ -323,10 +320,9 @@ alpaca_eos_token = "eos_token"CHAT_TEMPLATES["alpaca"] = (alpaca_template, alpaca_eos_token, False, alpaca_ollama,)DEFAULT_SYSTEM_MESSAGE["alpaca"] = "Below are some instructions that describe some tasks. Write responses that appropriately complete each request."+# |trim is required for lstrip/rstrip. <start_of_turn> maps to 106 and <end_of_turn> to 107; user+# and model are normal 1-word tokens.# =========================================== Gemma-# https://huggingface.co/google/gemma-7b-it-# Notice we must use |trim for lstrip and rstrip. <start_of_turn> maps to 106.-# <end_of_turn> maps to 107. user and model are normal 1 word tokens.gemma_template = \"{{ bos_token }}"\"{% if messages[0]['role'] == 'system' %}"\@@ -351,10 +347,10 @@ gemma_ollama = _ollama_template("gemma")gemma_eos_token = "<end_of_turn>"CHAT_TEMPLATES["gemma"] = (gemma_template, gemma_eos_token, True, gemma_ollama,)-DEFAULT_SYSTEM_MESSAGE["gemma"] = None # No system message in Gemma+DEFAULT_SYSTEM_MESSAGE["gemma"] = None+# <eos> is still more appropriate here than a ChatML end token.# =========================================== Gemma with ChatML instead-# We find using <eos> is still more appropriate!gemma_chatml_template = "{{ bos_token }}" + chatml_templategemma_chatml_ollama = _ollama_template("gemma_chatml")@@ -364,26 +360,26 @@ gemma_chatml_eos_token = ("<|im_end|>",)CHAT_TEMPLATES["gemma_chatml"] = (gemma_chatml_template, gemma_chatml_eos_token, True, gemma_chatml_ollama,)-DEFAULT_SYSTEM_MESSAGE["gemma_chatml"] = None # No system message in Gemma+DEFAULT_SYSTEM_MESSAGE["gemma_chatml"] = None-# =========================================== Gemma 2# Same as Gemma 1, but with sliding window attention!# https://ollama.com/library/gemma2/blobs/6522ca797f47+# =========================================== Gemma 2gemma2_template = gemma_templategemma2_ollama = _ollama_template("gemma2")gemma2_eos_token = "<end_of_turn>"CHAT_TEMPLATES["gemma2"] = (gemma2_template, gemma2_eos_token, True, gemma2_ollama,)-DEFAULT_SYSTEM_MESSAGE["gemma2"] = None # No system message in Gemma 2+DEFAULT_SYSTEM_MESSAGE["gemma2"] = None# =========================================== Gemma 2 with ChatML insteadgemma2_chatml_template = gemma_chatml_templategemma2_chatml_ollama = _ollama_template("gemma2_chatml")gemma2_chatml_eos_token = gemma_chatml_eos_tokenCHAT_TEMPLATES["gemma2_chatml"] = (gemma2_chatml_template, gemma2_chatml_eos_token, True, gemma2_chatml_ollama,)-DEFAULT_SYSTEM_MESSAGE["gemma2_chatml"] = None # No system message in Gemma 2+DEFAULT_SYSTEM_MESSAGE["gemma2_chatml"] = None+# The \n\n really is needed.# =========================================== Llama-3-# Weirdly \n\n is needed?llama3_template = \"{{ bos_token }}"\"{% for message in messages %}"\@@ -405,14 +401,14 @@ llama3_ollama = _ollama_template("llama-3")llama3_template_eos_token = "eos_token"CHAT_TEMPLATES["llama-3"] = (llama3_template, llama3_template_eos_token, False, llama3_ollama,)-DEFAULT_SYSTEM_MESSAGE["llama-3"] = None # No system message in Llama-3+DEFAULT_SYSTEM_MESSAGE["llama-3"] = NoneCHAT_TEMPLATES["llama3"] = (llama3_template, llama3_template_eos_token, False, llama3_ollama,)-DEFAULT_SYSTEM_MESSAGE["llama3"] = None # No system message in Llama-3+DEFAULT_SYSTEM_MESSAGE["llama3"] = None-# =========================================== Phi-3# "{{ bos_token }}"\ # Phi-3.5 removes BOS?+# =========================================== Phi-3phi3_template = \"{% for message in messages %}"\"{% if message['role'] == 'user' %}"\@@ -432,15 +428,14 @@ phi3_ollama = _ollama_template("phi-3")phi3_template_eos_token = "<|end|>"CHAT_TEMPLATES["phi-3"] = (phi3_template, phi3_template_eos_token, False, phi3_ollama,)-DEFAULT_SYSTEM_MESSAGE["phi-3"] = None # No system message in Phi-3+DEFAULT_SYSTEM_MESSAGE["phi-3"] = NoneCHAT_TEMPLATES["phi-35"] = CHAT_TEMPLATES["phi-3"]-DEFAULT_SYSTEM_MESSAGE["phi-35"] = None # No system message in Phi-3.5+DEFAULT_SYSTEM_MESSAGE["phi-35"] = NoneCHAT_TEMPLATES["phi-3.5"] = CHAT_TEMPLATES["phi-3"]-DEFAULT_SYSTEM_MESSAGE["phi-3.5"] = None # No system message in Phi-3.5+DEFAULT_SYSTEM_MESSAGE["phi-3.5"] = None-# =========================================== Llama-3.1"""No trimming in Llama 3.1 Instruct!Also an extra newline for Cutting Knowledge Date@@ -574,10 +569,10 @@ llama31_ollama = _ollama_template("llama-3.1")llama31_template_eos_token = "eos_token"CHAT_TEMPLATES["llama-3.1"] = (llama31_template, llama31_template_eos_token, False, llama31_ollama,)-DEFAULT_SYSTEM_MESSAGE["llama-3.1"] = "" # Llama3.1 default system message is empty + the dates+DEFAULT_SYSTEM_MESSAGE["llama-3.1"] = ""CHAT_TEMPLATES["llama-31"] = (llama31_template, llama31_template_eos_token, False, llama31_ollama,)-DEFAULT_SYSTEM_MESSAGE["llama-31"] = "" # Llama3.1 default system message is empty + the dates+DEFAULT_SYSTEM_MESSAGE["llama-31"] = ""for version in ("llama-3.2", "llama-3.3", "llama-32", "llama-33"):CHAT_TEMPLATES[version] = CHAT_TEMPLATES["llama-3.1"]@@ -643,19 +638,19 @@ qwen25_ollama = _ollama_template("qwen-2.5")qwen25_template_eos_token = "eos_token"qwen25_default_system_message = "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."CHAT_TEMPLATES["qwen-2.5"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,)-DEFAULT_SYSTEM_MESSAGE["qwen-2.5"] = qwen25_default_system_message # No system message in Qwen 2.5+DEFAULT_SYSTEM_MESSAGE["qwen-2.5"] = qwen25_default_system_messageCHAT_TEMPLATES["qwen-25"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,)-DEFAULT_SYSTEM_MESSAGE["qwen-25"] = qwen25_default_system_message # No system message in Qwen 2.5+DEFAULT_SYSTEM_MESSAGE["qwen-25"] = qwen25_default_system_messageCHAT_TEMPLATES["qwen25"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,)-DEFAULT_SYSTEM_MESSAGE["qwen25"] = qwen25_default_system_message # No system message in Qwen 2.5+DEFAULT_SYSTEM_MESSAGE["qwen25"] = qwen25_default_system_messageCHAT_TEMPLATES["qwen2.5"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,)-DEFAULT_SYSTEM_MESSAGE["qwen2.5"] = qwen25_default_system_message # No system message in Qwen 2.5+DEFAULT_SYSTEM_MESSAGE["qwen2.5"] = qwen25_default_system_message-# =========================================== Phi-4# "{{ bos_token }}"\ # Phi-4 removes BOS?+# =========================================== Phi-4phi4_template = \"{% for message in messages %}"\"{% if (message['role'] == 'system') %}"\@@ -680,12 +675,11 @@ phi4_ollama = _ollama_template("phi-4")phi4_template_eos_token = "<|im_end|>"CHAT_TEMPLATES["phi-4"] = (phi4_template, phi4_template_eos_token, False, phi4_ollama,)-DEFAULT_SYSTEM_MESSAGE["phi-4"] = None # No system message in Phi-4+DEFAULT_SYSTEM_MESSAGE["phi-4"] = None+# Obtained via print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))# =========================================== Gemma-3-# Obtained via-# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))gemma3_template = \"""{{ bos_token }}{%- if messages[0]['role'] == 'system' -%}@@ -734,13 +728,13 @@ gemma3_ollama = _ollama_template("gemma-3")gemma3_template_eos_token = "<end_of_turn>"CHAT_TEMPLATES["gemma-3"] = (gemma3_template, gemma3_template_eos_token, False, gemma3_ollama,)-DEFAULT_SYSTEM_MESSAGE["gemma-3"] = None # No system message in Gemma-3+DEFAULT_SYSTEM_MESSAGE["gemma-3"] = NoneCHAT_TEMPLATES["gemma3"] = (gemma3_template, gemma3_template_eos_token, False, gemma3_ollama,)-DEFAULT_SYSTEM_MESSAGE["gemma3"] = None # No system message in Gemma-3+DEFAULT_SYSTEM_MESSAGE["gemma3"] = None-# =========================================== Qwen-3# Official Qwen-3 chat template (see https://ollama.com/library/qwen3/blobs/eb4402837c78)+# =========================================== Qwen-3qwen3_template = \"""{%- if tools %}@@ -846,14 +840,13 @@ qwen3_template = \qwen3_ollama = _ollama_template("qwen-3")qwen3_template_eos_token = "<|im_end|>"CHAT_TEMPLATES["qwen-3"] = (qwen3_template, qwen3_template_eos_token, False, qwen3_ollama,)-DEFAULT_SYSTEM_MESSAGE["qwen-3"] = None # No default system message for Qwen-3+DEFAULT_SYSTEM_MESSAGE["qwen-3"] = NoneCHAT_TEMPLATES["qwen3"] = (qwen3_template, qwen3_template_eos_token, False, qwen3_ollama,)-DEFAULT_SYSTEM_MESSAGE["qwen3"] = None # No default system message for Qwen-3+DEFAULT_SYSTEM_MESSAGE["qwen3"] = None+# Obtained via print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))# =========================================== Gemma-3n-# Obtained via-# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))gemma3n_template = \"""{{ bos_token }}{%- if messages[0]['role'] == 'system' -%}@@ -903,13 +896,13 @@ gemma3n_template = \gemma3n_ollama = _ollama_template("gemma-3n")gemma3n_template_eos_token = "<end_of_turn>"CHAT_TEMPLATES["gemma-3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,)-DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = None # No system message in Gemma-3n+DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = NoneCHAT_TEMPLATES["gemma3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,)-DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None # No system message in Gemma-3n+DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None-# =========================================== Gemma-4# Gemma-4 uses <|turn>role\n...<turn|>\n format+# =========================================== Gemma-4gemma4_template = \"""{%- macro strip_thinking(text) -%}{%- set ns = namespace(result='') -%}@@ -1065,9 +1058,8 @@ DEFAULT_SYSTEM_MESSAGE["gemma-4-thinking"] = NoneCHAT_TEMPLATES["gemma4-thinking"] = (gemma4_thinking_template, gemma4_template_eos_token, False, gemma4_ollama,)DEFAULT_SYSTEM_MESSAGE["gemma4-thinking"] = None+# Obtained via print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))# =========================================== GPT-OSS-# Obtained via-# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))gptoss_template = \"""{#-In addition to the normal inputs of `messages` and `tools`, this template also accepts the@@ -1602,10 +1594,10 @@ PARAMETER top_p 1.0gptoss_template_template_eos_token = "<|return|>"CHAT_TEMPLATES["gpt-oss"] = (gptoss_template, gptoss_template_template_eos_token, False, gptoss_ollama,)-DEFAULT_SYSTEM_MESSAGE["gpt-oss"] = None # No system message in GPT-oss+DEFAULT_SYSTEM_MESSAGE["gpt-oss"] = NoneCHAT_TEMPLATES["gptoss"] = (gptoss_template, gptoss_template_template_eos_token, False, gptoss_ollama,)-DEFAULT_SYSTEM_MESSAGE["gptoss"] = None # No system message in GPT-oss+DEFAULT_SYSTEM_MESSAGE["gptoss"] = None# =========================================== Qwen3-Instructqwen3_instruct_template = \@@ -1698,7 +1690,7 @@ qwen3_instruct_template = \qwen3_template_eos_token = "<|im_end|>"CHAT_TEMPLATES["qwen3-instruct"] = (qwen3_instruct_template, qwen3_template_eos_token, False, _ollama_template("qwen3-instruct"),)-DEFAULT_SYSTEM_MESSAGE["qwen3-instruct"] = None # No system message in Qwen3+DEFAULT_SYSTEM_MESSAGE["qwen3-instruct"] = None# =========================================== Qwen3-Thinking@@ -1796,7 +1788,7 @@ CHAT_TEMPLATES["qwen3-thinking"] = (False,_ollama_template("qwen3-thinking"),)-DEFAULT_SYSTEM_MESSAGE["qwen3-thinking"] = None # No system message in Qwen3+DEFAULT_SYSTEM_MESSAGE["qwen3-thinking"] = None# =========================================== Liquid-LFM2@@ -1809,11 +1801,12 @@ liquid_lfm2_template = \liquid_lfm2_template_eos_token = "<|im_end|>"CHAT_TEMPLATES["lfm-2"] = (liquid_lfm2_template, liquid_lfm2_template_eos_token, False, None)-DEFAULT_SYSTEM_MESSAGE["lfm-2"] = None # No system message in Phi-3+DEFAULT_SYSTEM_MESSAGE["lfm-2"] = NoneCHAT_TEMPLATES["lfm-2.5"] = (liquid_lfm2_template, liquid_lfm2_template_eos_token, False, None)DEFAULT_SYSTEM_MESSAGE["lfm-2.5"] = None+# =========================================== Starling-LMstarling_template = \@@ -1833,6 +1826,7 @@ CHAT_TEMPLATES["starling"] = (starling_template, starling_template_eos_token, FaDEFAULT_SYSTEM_MESSAGE["starling"] = None+# =========================================== Yi-chatyi_chat_template = \@@ -1850,9 +1844,8 @@ yi_chat_template_eos_token = "<|endoftext|>"CHAT_TEMPLATES["yi-chat"] = (yi_chat_template, yi_chat_template_eos_token, False, yi_chat_ollama)DEFAULT_SYSTEM_MESSAGE["yi-chat"] = None-# Caller text is spliced into Jinja '...' / "..." literals: a bare quote closes the-# literal, a backslash reads as an escape, and \r becomes \n before unescaping. Backslash-# first, else it re-escapes the rest; escaping both quotes is safe in either style.+# Caller text is spliced into Jinja '...' / "..." literals: a bare quote closes the literal, a+# backslash reads as an escape, and \r becomes \n before unescaping.def _escape_jinja_literal(text):return text.replace("\\", "\\\\").replace("'", "\\'").replace('"', '\\"').replace("\r", "\\r")@@ -1861,8 +1854,8 @@ def _change_system_message(template: str, type_chat_template: str, system_messag# For predefined templates, check if default system message existsdefault_system_message = DEFAULT_SYSTEM_MESSAGE.get(f"{type_chat_template}", None)- # Custom templates have no default but may carry a {system_message} placeholder;- # fill it before the no-default return below. A missing message here is an error.+ # Custom templates have no default but may carry a {system_message} placeholder; fill it before the+ # no-default return below. A missing message here is an error.if default_system_message is None and "{system_message}" in template:if system_message is None:raise ValueError("Unsloth: You need to provide a system message for custom templates.")@@ -1880,8 +1873,8 @@ def _change_system_message(template: str, type_chat_template: str, system_messag# For predefined templates with default system messagemessage_to_use = system_message if system_message is not None else default_system_message- # Predefined templates hold {system_message} inside a Jinja literal, so escape it.- # Custom ones (above) are filled verbatim: their placeholder may sit in raw text.+ # Predefined templates hold {system_message} inside a Jinja literal, so escape it; custom ones are+ # filled verbatim, since their placeholder may sit in raw text.new_template = template.replace("{system_message}", _escape_jinja_literal(message_to_use))return new_template, message_to_use@@ -1903,8 +1896,8 @@ def get_chat_template(use_zoo_tokenizer_patch = is_mlx_backendold_tokenizer = tokenizer- # mlx-lm's TokenizerWrapper._tokenizer is the HF tokenizer, not the Rust- # backend the vocab-edit paths below need; unwrap here, re-wrap before return.+ # mlx-lm's TokenizerWrapper._tokenizer is the HF tokenizer, not the Rust backend the vocab-edit+ # paths below need; unwrap here, re-wrap before return._mlx_tokenizer_wrapper = Noneif is_mlx_backend and tokenizer.__class__.__name__ == "TokenizerWrapper":_inner_tokenizer = getattr(tokenizer, "_tokenizer", None)@@ -1917,16 +1910,6 @@ def get_chat_template(if chat_template == "chatml": chat_template = "gemma_chatml"IS_GEMMA = True- # We add a check for Llama-3- # if chat_template == "llama-3":- # tokenizer._using_llama3_template = True- # else:- # llama3_tokens = set(["<|end_header_id|>", "<|eot_id|>", "<|start_header_id|>"])- # check_llama3_tokens = llama3_tokens & set(str(x) for x in tokenizer.added_tokens_decoder.values())- # if len(check_llama3_tokens) == len(llama3_tokens):- # tokenizer._using_llama3_template = True- # pass- # pass# We first check if the tokenizer is a fast one. If not, we cannot convert this!is_fast_tokenizer = getattr(tokenizer, "is_fast", False)@@ -1936,8 +1919,7 @@ def get_chat_template(type_chat_template = Noneif type(chat_template) in (list, tuple,):- # For changing system message later- # Since it's not supported yet, we will raise an error first!+ # For changing system message later Since it's not supported yet, we will raise an error first!type_chat_template = chat_template[0].lower()chat_template, stop_word = chat_templateassert(type(chat_template) is str)@@ -1950,8 +1932,8 @@ def get_chat_template(chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template]- # Check mapping to eos_token. The template can veto the mapping, but it must not- # force it back on: `map_eos_token = False` is an explicit choice by the caller.+ # The template can veto the eos mapping, but it must not force it back on: map_eos_token = False is+ # an explicit choice by the caller.if not yes_map_eos_token and map_eos_token: map_eos_token = Falseif type(stop_word) in (list, tuple,):@@ -1962,12 +1944,10 @@ def get_chat_template(assert(type(stop_word) is str)- # gemma_chatml and gemma2_chatml build <|im_end|> by renaming <eos>, and that rename- # runs whether or not the caller opts out, while the rebuilt tokenizer only carries- # eos_token = stop_word when the mapping is on: honouring the opt-out here lets the- # tokenizer class default re-add <eos> as a fresh id past the end of the embeddings.- # Key on the mapping, not on tokenizer.eos_token, or a Gemma checkpoint whose- # eos_token is <end_of_turn> (gemma-3-270m-it, gemma-3-1b-it) slips through.+ # gemma_chatml / gemma2_chatml build <|im_end|> by renaming <eos> whether or not the caller opts+ # out, while the rebuilt tokenizer carries eos_token = stop_word only when the mapping is on, so+ # honouring the opt-out lets the class default re-add <eos> past the end of the embeddings. Key on+ # the mapping, or a Gemma checkpoint whose eos_token is <end_of_turn> slips through.if not map_eos_token and yes_map_eos_token and token_mapping is not None:logger.warning_once(f"Unsloth: {type_chat_template} builds {stop_word} by renaming existing "\@@ -1975,24 +1955,16 @@ def get_chat_template()map_eos_token = True- # Check fast tokenizerif not is_fast_tokenizer:pass- # print(- # "Unsloth: Not a fast tokenizer, so can't process it as of yet :(\n"\- # "Please log a Github issue if you want this as a new feature!\n"\- # "Your chat template will still work, but it won't add or edit tokens."- # )elif token_mapping is not None:- # token_mapping = {"<start_of_turn>" : "<|im_start|>", "<end_of_turn>" : "<|im_end|>"}- # For Gemma :)string_vocab = tokenizer._tokenizer.to_str()skipped = 0- # Only mirror applied mappings into the spm model; a skipped one would- # rename a piece the JSON never changed and desync the two.+ # Only mirror applied mappings into the spm model; a skipped one would rename a piece the JSON never+ # changed and desync the two.applied_mapping = {}for old_token, new_token in token_mapping.items():old_count = string_vocab.count(f'"{old_token}"')@@ -2046,9 +2018,9 @@ def get_chat_template(elif map_eos_token and (stop_word != "eos_token"):logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")- # HACK: replace old EOS with a new one (e.g. ChatML <|im_end|>) to- # avoid the slow lm_head/embedding retraining of new tokens.- # Idea from https://huggingface.co/cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laser+ # Replace the old EOS with a new one (e.g. ChatML <|im_end|>) to avoid slow lm_head/embedding+ # retraining of new tokens. Idea from+ # huggingface.co/cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laserold_bos_token = getattr(tokenizer, "bos_token", None)old_eos_token = getattr(tokenizer, "eos_token", None)@@ -2058,7 +2030,6 @@ def get_chat_template(string_vocab = tokenizer._tokenizer.to_str()# First check if new stop_word is in the tokenizerif stop_word in string_vocab:- # We shall swap them aroundtemporary_stop_token = "<|:__TEMP//STOP//TOKEN__:|>"string_vocab = string_vocab.replace(old_eos_token, temporary_stop_token)string_vocab = string_vocab.replace(stop_word, old_eos_token)@@ -2096,13 +2067,11 @@ def get_chat_template(f"{CHAT_TEMPLATES.keys()}")- # Careful on Gemma- # bos_token is a must or else losses become too high+ # bos_token is a must on Gemma or losses become too high.if IS_GEMMA and not chat_template.startswith(("{{ bos_token }}", "{{- bos_token }}")):chat_template = "{{ bos_token }}" + chat_template- # For ShareGPT role -> from and content -> value- # The spliced values land inside Jinja literals, so escape them.+ # The spliced ShareGPT values land inside Jinja literals, so escape them.new_chat_template = chat_template\.replace("'role'", "'" + _escape_jinja_literal(mapping["role"]) + "'")\.replace("'content'", "'" + _escape_jinja_literal(mapping["content"]) + "'")\@@ -2110,8 +2079,8 @@ def get_chat_template(.replace("'assistant'", "'" + _escape_jinja_literal(mapping["assistant"]) + "'")if use_zoo_tokenizer_patch:- # Unsloth MLX avoids the model-utils tokenizer wrapper because that- # import path pulls in Torch/GPU-specific modules before MLX training.+ # Unsloth MLX avoids the model-utils tokenizer wrapper: that import path pulls Torch/GPU-specific+ # modules in before MLX training.from unsloth_zoo.tokenizer_utils import patch_tokenizerelse:from .models._utils import patch_tokenizer@@ -2145,14 +2114,11 @@ def get_chat_template(if not same_padding_token:if old_pad_token != new_pad_token: tokenizer.pad_token = old_pad_token- # stopping_criteria = create_stopping_criteria(tokenizer, stop_word)- # Patch saving functionsif patch_saving and not is_mlx_backend:from .save import patch_saving_functionstokenizer = patch_saving_functions(tokenizer)- # Add Ollamatokenizer._ollama_modelfile = ollama_modelfiletokenizer._system_message = system_message@@ -2167,7 +2133,7 @@ def get_chat_template(getattr(tokenizer, "chat_template", None) is not None)tokenizer = _mlx_tokenizer_wrapper- return tokenizer#, stopping_criteria+ return tokenizerdef remove_special_tokens(tokenizer, prompt):@@ -2179,7 +2145,6 @@ def remove_special_tokens(tokenizer, prompt):def _parse_combined_prompt(combined_prompt, dataset):- # Find {...}possible_columns = re.findall(r"\{(.+?)\}", combined_prompt)dataset_columns = set(dataset.column_names)for column in possible_columns:@@ -2189,30 +2154,25 @@ def _parse_combined_prompt(combined_prompt, dataset):f"Only allowed columns are {list(dataset_columns)}")- # Find [[...]]optional_prompts = list(re.finditer(r"\[\[.+?\]\]", combined_prompt, flags = re.DOTALL | re.MULTILINE))optional_prompts = [(x.span(), x.group(0)) for x in optional_prompts]final_optional_prompts = []if len(optional_prompts) != 0:- # Add leftleft = optional_prompts[0]l = left[0][0]if l != 0: final_optional_prompts.append(combined_prompt[:l])- # Add in betweenfor left, right in zip(optional_prompts[:-1], optional_prompts[1:]):l, r = left[0][-1], right[0][0]final_optional_prompts.append(left)if l != r: final_optional_prompts.append(combined_prompt[l : r])final_optional_prompts.append(optional_prompts[-1])- # Add rightright = optional_prompts[-1]r = right[0][1]if r != len(combined_prompt): final_optional_prompts.append(combined_prompt[r:])else:- # Just add in the entire stringfinal_optional_prompts.append(combined_prompt)check_combined = "".join(x if type(x) is str else x[1] for x in final_optional_prompts)@@ -2255,12 +2215,7 @@ def _create_formatter(possible_columns, final_optional_prompts, user_column_nametexts = []for row_idx in range(n_rows):- # Coerce missing (None) columns to "" so they do not render as the- # literal string "None" in the emitted text. In a [[...]] block only- # the first column gates the block, so a later column can still be- # None here; required columns can be None too. Coercing at the source- # covers both; since None is now "", the gate below only needs to- # test for "" (an empty first column still drops the block).+ # Coerce missing (None) columns to "" so they do not render as the literal string "None".row_values = {column: ("" if (value := examples[column][row_idx]) is None else value)for column in columns@@ -2317,9 +2272,8 @@ def to_sharegpt(formatter = _create_formatter(possible_columns, final_optional_prompts, merged_column_name)dataset = dataset.map(formatter, batched = True, desc = "Merging columns")elif merged_column_name not in dataset.column_names:- # Without a merged_prompt there is nothing to build the input from, so the- # column has to be there already. Running the formatter anyway would fill- # merged_column_name with empty strings and silently blank every human turn.+ # Without a merged_prompt there is nothing to build the input from. Running the formatter anyway+ # would fill merged_column_name with empty strings and silently blank every human turn.raise KeyError(f"Unsloth: `to_sharegpt` needs an input column named '{merged_column_name}', "f"but the dataset has {dataset.column_names}. Pass `merged_column_name` to "@@ -2334,10 +2288,8 @@ def to_sharegpt("Unsloth: Input and output columns must have matching batch lengths. "f"Got {len(users)} {merged_column_name} rows and {len(assistants)} {output_column_name} rows.")- # A null cell is an absent value, not the word "None". _create_formatter- # already coalesces it on the merged path, and skipping the merge must- # not reintroduce it; the output column never went through that path at- # all, so it leaked here either way.+ # A null cell is an absent value, not the word "None": _create_formatter coalesces it on the merged+ # path, and the output column never went through that path at all.texts = [[{"from" : "human", "value" : "" if user is None else str(user) },@@ -2351,11 +2303,9 @@ def to_sharegpt(__convert_to_sharegpt__,batched = True,desc = "Converting to ShareGPT",- # Remove unused columns!remove_columns = dataset.column_names if remove_unused_columns else None,)- # Randomnly concat conversations to create a long stream!from datasets import concatenate_datasetsn_extensions = max(conversation_extension-1, 0)if n_extensions == 0: return dataset@@ -2367,7 +2317,6 @@ def to_sharegpt(all_shuffled.append(shuffled)dataset = concatenate_datasets(all_shuffled, axis = 1)- # Combine them into 1n_extensions += 1conversation_columns = [f"conversations{j}" for j in range(n_extensions)]def __combine_conversations__(examples):@@ -2384,7 +2333,6 @@ def to_sharegpt(__combine_conversations__,batched = True,desc = "Extending conversations",- # Remove unused columns!remove_columns = dataset.column_names if remove_unused_columns else None,)return dataset@@ -2402,13 +2350,11 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []):added_tokens_decoder = [x for x in added_tokens_decoder if x != tokenizer.bos_token]repeated_tokens = []- # Join all vocabjoined_text = "\x01\x00".join(added_tokens_decoder)for token in added_tokens_decoder:n = len(token)repeated_counts = joined_text.count(token[:n//2])- # Try finding longer than 1/2 of the token in the rest- # For eg <|reserved_special_token_0|>, <|reserved_special_token_1|>+ # Longer than half the token, for e.g. <|reserved_special_token_0|> vs <|reserved_special_token_1|>.if repeated_counts > 2:for j in range(n//2+1, n):if joined_text.count(token[:j]) < repeated_counts:@@ -2418,13 +2364,11 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []):repeated_tokens.append(token[:j])break- # Remove duplicatessplit = joined_text.split("\x01\x00")final_eos_tokens = [old for old, new in zip(added_tokens_decoder, split) if old == new]final_eos_tokens += extra_eos_tokensfinal_eos_tokens += repeated_tokens- # Remove new lines, spaces and HTML tagsfiltered_eos_tokens = []for token in final_eos_tokens:if token.count("\n") == len(token): continue@@ -2463,8 +2407,8 @@ extra_eos_tokens = None,You must use {INPUT}, {OUTPUT} twice, and {SYSTEM} is optional."""- # Strip only the left: trailing whitespace can be part of the repeated example- # (e.g. "{OUTPUT}\n"). Accidental trailing whitespace (#992) is retried on failure.+ # Strip only the left: trailing whitespace can be part of the repeated example (e.g. "{OUTPUT}\n").+ # Accidental trailing whitespace (#992) is retried on failure.chat_template = chat_template.lstrip()assert(tokenizer is not None)@@ -2496,31 +2440,28 @@ extra_eos_tokens = None,"Unsloth: Your tokenizer does not have an EOS token? Please provide one via extra_eos_tokens!")- # Check tokenizer typestokenizer_name = tokenizer.name_or_path.lower()if tokenizer_name.startswith(("unsloth/llama-3-8b-instruct", "unsloth/llama-3-70b-instruct")):- # Add <|eot_id|>extra_eos_tokens.append("<|eot_id|>")elif ("<|eot_id|>" in extra_eos_tokens or "<|eot_id|>" in chat_template) and \tokenizer_name.startswith(("unsloth/llama-3-8b", "unsloth/llama-3-70b")):- # Warnlogger.warning("Unsloth: Base llama-3 models did not train <|eot_id|>.\n"\"Please use the instruct version or use <|end_of_text|>")- # dict.fromkeys, not set: `extra_eos_tokens.insert(0, tokenizer.eos_token)` above- # sets a priority that `extra_eos_tokens[0]` relies on when appending the EOS.+ # dict.fromkeys, not set: extra_eos_tokens.insert(0, tokenizer.eos_token) above sets a priority that+ # extra_eos_tokens[0] relies on when appending the EOS.extra_eos_tokens = list(dict.fromkeys(extra_eos_tokens))count_eos = 0for eos in extra_eos_tokens:count_eos += len(re.findall(r"{OUTPUT}" + re.escape(eos), chat_template))- # This forces you to provide 2 input and outputs+ # This forces the caller to provide 2 input/output pairs.final_combined_check = Falsetry:- # O(N^2) search finding 2 repeatted pieces of text+ # O(N^2) search for 2 repeated pieces of text.j = len(chat_template)-1at_least_one = Falsewhile j > 0:@@ -2536,19 +2477,15 @@ extra_eos_tokens = None,# Must be equivalent to leftfinal_combined_check = True- # Repeatted textinstruction_response = chat_template[j:]if instruction_response.count("{INPUT}") != 1 or instruction_response.count("{OUTPUT}") != 1:raise RuntimeError(error_msg)- # 1st System, Instruction, Output pairleft = chat_template[:j]- # 2nd Instruction, Output pairright = chat_template[j:]final_combined_check = left if final_combined_check else chat_template- # Isolate input# Regex alternations prefer the first match, so match prefix tokens last.eos_tokens_for_matching = sorted(extra_eos_tokens, key = len, reverse = True)extra_eos_tokens_regex = "|".join(f"(?:{re.escape(x)})" for x in eos_tokens_for_matching)@@ -2563,14 +2500,11 @@ extra_eos_tokens = None,input_end = input_end.span(0)[1]input_part = right[:input_end]- # Isolate outputoutput_part = right[input_end:]- # Isolate systemwhere_system = left.find(input_part)system_part = left[:where_system if where_system != -1 else len(left)]- # Check if the user provided a correct promptcombined = system_part + input_part + output_partif combined != final_combined_check:combined_changed = combined .replace('\n', '\\n')@@ -2582,8 +2516,8 @@ extra_eos_tokens = None,f"{left_changed}")except:- # Accidental trailing whitespace (#992) desyncs the two-example detection,- # so retry once without it. Templates that parse as-is are never altered.+ # Accidental trailing whitespace (#992) desyncs the two-example detection, so retry once without it.+ # Templates that parse as-is are never altered.rstripped_chat_template = chat_template.rstrip()if rstripped_chat_template != chat_template:try:@@ -2648,9 +2582,8 @@ extra_eos_tokens = None,eos = extra_eos_tokens[0]output_part = output_part + eos- # Ollama modelfile parts- # Check bos_token is in system prompt+ # bos_token must be in the system prompt.ollama_system = system_parthas_bos_token = Falsealways_bos_token = False@@ -2659,7 +2592,6 @@ extra_eos_tokens = None,if ollama_system.startswith(tokenizer.bos_token):has_bos_token = Trueollama_system = ollama_system[len(tokenizer.bos_token):]- # Check systemif "{SYSTEM}" in ollama_system:system_modelfile = "{{ if .System }}" + ollama_system.replace("{SYSTEM}", "{{ .System }}") + "{{ end }}"else:@@ -2667,20 +2599,17 @@ extra_eos_tokens = None,input_modelfile = "{{ if .Prompt }}" + input_part .replace("{INPUT}", "{{ .Prompt }}") + "{{ end }}"output_modelfile = output_part.replace("{OUTPUT}", "{{ .Response }}")- # Ollama EOSollama_eos = get_ollama_eos_tokens(tokenizer, extra_eos_tokens)ollama_eos = '\n'.join(f'PARAMETER stop "{eos}"' for eos in ollama_eos)- # Add temperature and min_p to counteract gibberish+ # Temperature and min_p counteract gibberish.ollama_eos += "\nPARAMETER temperature 1.5\nPARAMETER min_p 0.1"- # Ollama modelfilepart = '"""'modelfile = 'FROM {__FILE_LOCATION__}\n\n'\'TEMPLATE ' + part + system_modelfile + input_modelfile + output_modelfile + \part + '\n\n' + ollama_eos- # HF Jinja Chat templatedef process(part, which, content = "message['content']"):# Escape the literal pieces only; the placeholder becomes Jinja syntax below.part = which.join(_escape_jinja_literal(piece) for piece in part.split(which))@@ -2709,7 +2638,6 @@ extra_eos_tokens = None,"{{ '" + _escape_jinja_literal(output_part[:output_part.find("{OUTPUT}")]) + "' }}"\"{% endif %}"- # Now add system prompt to jinjaif len(system_part) != 0:# Strip the BOS while raw: an escaped bos_token no longer matches.if has_bos_token:@@ -2735,14 +2663,13 @@ extra_eos_tokens = None,"{% set loop_messages = messages %}"\"{% endif %}"elif "{SYSTEM}" in system_part:- # Only bind loop_messages when the template can render a caller system- # message. A static prefix with no {SYSTEM} must still raise, not drop it.+ # Only bind loop_messages when the template can render a caller system message; a static prefix with+ # no {SYSTEM} must still raise, not drop it.partial_system += "{% else %}"\"{% set loop_messages = messages %}"\"{% endif %}"else:- # Emit a static prefix on both branches so the collapse below- # makes it unconditional.+ # Emit a static prefix on both branches so the collapse below makes it unconditional.partial_system += "{% else %}"\"{{ " + system_expr + " }}"\"{% set loop_messages = messages %}"\@@ -2937,7 +2864,6 @@ def test_chat_templates():{"role": "user", "content": " No it's 100% 5! "},]- # Zephyrfrom transformers import AutoTokenizertemplate = zephyr_templatecorrect_tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-beta")@@ -2946,7 +2872,6 @@ def test_chat_templates():our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)assert(correct_prompt == our_prompt)- # Chatmltemplate = chatml_templatecorrect_tokenizer = AutoTokenizer.from_pretrained("teknium/OpenHermes-2.5-Mistral-7B")correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)@@ -2954,7 +2879,6 @@ def test_chat_templates():our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)assert(correct_prompt == our_prompt)- # Mistraltemplate = mistral_templatecorrect_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)@@ -2962,7 +2886,6 @@ def test_chat_templates():our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)assert(correct_prompt == our_prompt)- # Llamatemplate = llama_templatecorrect_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-2-7b-chat")correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)@@ -2970,7 +2893,6 @@ def test_chat_templates():our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)assert(correct_prompt == our_prompt)- # Vicunatry:from fastchat.conversation import get_conv_templateexcept:@@ -3003,17 +2925,15 @@ def test_chat_templates():correct_prompt = correct_tokenizer.bos_token + correct_prompt.get_prompt()correct_tokenizer.chat_template = templateour_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)- # We add </s> ourselves+ # </s> is added here rather than by the template.assert(correct_prompt == our_prompt.replace("</s>", ""))- # Gemmacorrect_tokenizer = AutoTokenizer.from_pretrained("unsloth/gemma-7b-it")correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)correct_tokenizer.chat_template = gemma_templateour_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)assert(our_prompt == correct_prompt)- # Llama-3template = llama3_templatecorrect_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-3-8b-Instruct")correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)@@ -3021,7 +2941,6 @@ def test_chat_templates():our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)assert(correct_prompt == our_prompt)- # Phi-3template = phi3_templatecorrect_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)@@ -3055,8 +2974,8 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf")### Response:{}""".format(- "Describe the city given eloquently.", # instruction- "The lost city of Atlantis.", # input+ "Describe the city given eloquently.",+ "The lost city of Atlantis.","", # output - leave this blank for generation!)prompts = [ prompt, ]@@ -3067,7 +2986,7 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf")prompts.append(prompt)for prompt in prompts:- # Use a list of args with shell=False so prompt content is passed literally.+ # A list of args with shell=False so prompt content is passed literally.command = ["./llama.cpp/llama-cli","-m", gguf_model,diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.pyindex 73d17bba977..37f204e9e47 100644--- a/unsloth/dataprep/raw_text.py+++ b/unsloth/dataprep/raw_text.py@@ -90,8 +90,8 @@ class RawTextDataLoader:)all_chunks.extend(chunks)if not all_chunks:- # All files empty/whitespace: raise like load_from_file instead of- # create_causal_dataset([]) returning a 0-row text-column dataset.+ # All files empty/whitespace: raise like load_from_file instead of create_causal_dataset([])+ # returning a 0-row text-column dataset.raise ValueError("All files are empty or contain only whitespace")return self.create_causal_dataset(all_chunks)@@ -108,7 +108,6 @@ class RawTextDataLoader:def create_causal_dataset(self, chunks):"""Create dataset for causal language modeling"""if chunks and isinstance(chunks[0], dict):- # Already-tokenized chunks: reshape for Dataset.from_dictinput_ids = [chunk["input_ids"] for chunk in chunks]attention_mask = [chunk["attention_mask"] for chunk in chunks]# Labels == input_ids for causal LM@@ -121,7 +120,6 @@ class RawTextDataLoader:})else:- # Text strings (backward compatibility)return Dataset.from_dict({"text": chunks})def smart_chunk_text(@@ -150,9 +148,8 @@ class RawTextDataLoader:f"stride ({stride}) must be smaller than chunk_size ({chunk_size}) to progress the chunking loop")- # Skip empty/whitespace text before tokenizing: BPE/SentencePiece emit- # real tokens for spaces/newlines, so a len(tokens)==0 check misses it- # and would yield a degenerate lone-EOS sample. Mirrors load_from_file.+ # Skip empty/whitespace text before tokenizing: BPE/SentencePiece emit real tokens for+ # spaces/newlines, so a len(tokens)==0 check misses it and would yield a degenerate lone-EOS sample.if not text or not text.strip():return []@@ -194,7 +191,7 @@ class RawTextDataLoader:chunk_tokens.tolist() if hasattr(chunk_tokens, "tolist") else list(chunk_tokens))- # Append EOS on the last or a full chunk+ # Append EOS on the last or a full chunk.if end_idx == len(tokens) or len(chunk_tokens_list) == chunk_size:eos_token_id = getattr(self.tokenizer, "eos_token_id", None)if eos_token_id is not None:@@ -204,10 +201,9 @@ class RawTextDataLoader:chunks.append({"input_ids": chunk_tokens_list, "attention_mask": attention_mask})else:- # Decode back to text (backward compatibility)chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True)- # Append EOS on the last or a full chunk+ # Append EOS on the last or a full chunk.if end_idx == len(tokens) or len(chunk_tokens) == chunk_size:eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else ""chunk_text += eos_token@@ -223,16 +219,15 @@ class RawTextDataLoader:def _read_file_by_format(self, file_path, file_format):"""Read file content based on detected format."""- # utf-8-sig: Windows tooling (PowerShell's Out-File, Excel's "CSV UTF-8") prepends- # a BOM that plain utf-8 keeps as a leading character. Without a BOM it decodes- # exactly like utf-8.+ # utf-8-sig: Windows tooling (PowerShell Out-File, Excel "CSV UTF-8") prepends a BOM that plain+ # utf-8 keeps as a leading character. Without a BOM it decodes exactly like utf-8.with open(file_path, "r", encoding = "utf-8-sig") as f:if file_format == "plain_text" or file_format == "markdown":return f.read()elif file_format == "json_lines":if Path(file_path).suffix.lower() == ".json":- # A .json file is a single JSON document (commonly a list- # of records), so parsing it per line drops the whole file.+ # A .json file is a single JSON document (commonly a list of records), so parsing it per line drops+ # the whole file.try:parsed = json.load(f)records = parsed if isinstance(parsed, list) else [parsed]@@ -241,8 +236,8 @@ class RawTextDataLoader:f.seek(0)records = self._iter_json_lines(f)else:- # A .jsonl file is one JSON value per line: stay streaming so- # a large file is never held in memory all at once.+ # A .jsonl file is one JSON value per line: stay streaming so a large file is never held in memory+ # at once.records = self._iter_json_lines(f)lines = []for data in records:@@ -260,7 +255,6 @@ class RawTextDataLoader:return "\n\n".join(texts)return ""- # Cache text fields/columns for better performance_TEXT_FIELDS = ("text", "content", "message", "body", "description", "prompt")_TEXT_COLUMNS = _TEXT_FIELDS@@ -277,8 +271,8 @@ class RawTextDataLoader:def _extract_text_from_json(self, data):"""Extract text from JSON object using common field names."""- # Skip non-object lines (str/list/number): `field in data` would be a- # substring/membership test, not a key lookup, and `data[field]` raises.+ # Skip non-object lines (str/list/number): `field in data` would be a substring/membership test, not+ # a key lookup, and `data[field]` raises.if not isinstance(data, dict):return ""for field in self._TEXT_FIELDS:@@ -311,7 +305,6 @@ def _iter_column(dataset, column):class TextPreprocessor:- # Compile regex patterns once for better performance_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+")_INVALID_CHARS_PATTERN = re.compile(r"[^\x20-\x7E\n]")_MULTIPLE_SPACES_PATTERN = re.compile(r"[ ]{2,}")@@ -336,8 +329,8 @@ class TextPreprocessor:"""Extract specific sections (e.g., code blocks, quotes)"""sections = []for pattern in patterns:- # Compile pattern on first use and cache? Well, patterns are user-provided,- # so just use re.findall with compiled flags+ # Compile pattern on first use and cache? Well, patterns are user-provided, so just use re.findall+ # with compiled flagsmatches = re.findall(pattern, text, re.MULTILINE | re.DOTALL)sections.extend(matches)return sections@@ -373,8 +366,8 @@ class TextPreprocessor:"warnings": [],}- # `column_names` is HF Dataset-only; None means a mapping-like (DataFrame,- # dict, custom __getitem__), which this method has always accepted.+ # `column_names` is HF Dataset-only; None means a mapping-like (DataFrame, dict, custom+ # __getitem__), which this method has always accepted.column_names = getattr(dataset, "column_names", None)if column_names is None or "text" in column_names:texts = _iter_column(dataset, "text")@@ -402,35 +395,30 @@ class TextPreprocessor:stats["empty_samples"] += 1continue- # Check for encoding issuestry:text.encode("utf-8")except UnicodeEncodeError:stats["encoding_issues"] += 1- # Calculate lengthslength = len(text)text_lengths.append(length)stats["min_length"] = min(stats["min_length"], length)stats["max_length"] = max(stats["max_length"], length)- # Check for repeated contenttext_hash = hash(text.strip())if text_hash in seen_texts:stats["repeated_content"] += 1else:seen_texts.add(text_hash)- # Calculate average lengthif text_lengths:stats["avg_length"] = sum(text_lengths) / len(text_lengths)- # No sample had content, so min_length is still its float("inf") seed. Report 0,- # like max_length and avg_length beside it, rather than handing back an infinity.+ # No sample had content, so min_length is still its float("inf") seed; report 0 like max_length and+ # avg_length beside it.if stats["min_length"] == float("inf"):stats["min_length"] = 0- # Generate warningsif stats["empty_samples"] > 0:stats["warnings"].append(f"Found {stats['empty_samples']} empty samples")@@ -440,8 +428,8 @@ class TextPreprocessor:if stats["encoding_issues"] > 0:stats["warnings"].append(f"Found {stats['encoding_issues']} encoding issues")- # Guard on text_lengths, not on min_length: with nothing measured it is now 0,- # and zero measured samples is not "some samples are very short".+ # Guard on text_lengths, not on min_length: with nothing measured it is now 0, and zero measured+ # samples is not "some samples are very short".if text_lengths and stats["min_length"] < 10:stats["warnings"].append("Some samples are very short (< 10 characters)")diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.pyindex befb5f013a4..514953aa9b6 100644--- a/unsloth/dataprep/synthetic.py+++ b/unsloth/dataprep/synthetic.py@@ -183,7 +183,7 @@ class SyntheticDataKit:float8_kv_cache = False,conservativeness = 1.0,token = None,- timeout = 1200, # maybe this is not enough for large models if we need to download+ timeout = 1200,**kwargs,):assert type(model_name) is str@@ -232,7 +232,6 @@ class SyntheticDataKit:elif dtype_val == torch.float32:dtype_val = "float32"engine_args["dtype"] = dtype_val- # Convert torch dtype to valid CLI stringif hasattr(dtype_val, "name"):engine_args["dtype"] = dtype_val.nameelif isinstance(dtype_val, str) and dtype_val.startswith("torch."):@@ -254,9 +253,7 @@ class SyntheticDataKit:for key, value in engine_args.items():flag = key.replace("_", "-")if key == "compilation_config":- # [TODO] Unsure why subprocess doesn't process json properly- # Also -O3 breaks on T4!- # subprocess_commands += ["-O3",]+ # -O3 breaks on T4.continuewhich = str(value).replace("torch.", "")if which == "True":@@ -265,10 +262,8 @@ class SyntheticDataKit:"--" + flag,]elif which == "False":- # Ignore flagpasselif which == "None":- # Ignore flagpasselse:subprocess_commands += [@@ -300,7 +295,7 @@ class SyntheticDataKit:ready_regex = None,text = False,)- # we don't print stderr to console but self.stderr_capture.tail(200) will print the last 200 lines+ # stderr is not printed to console, but self.stderr_capture.tail(200) prints the last 200 lines.self._await_vllm_server(timeout = timeout)print("vLLM Server Ready Detected")@@ -341,11 +336,9 @@ class SyntheticDataKit:"""deadline = _deadline(timeout)while True:- # Cap each lap at the time left: a flat `poll_interval` overshoots- # any shorter timeout, and readiness inside that overshoot would- # return success from an expired deadline. No deadline means full- # laps, which still notice a dead child where the bare- # `Event.wait(None)` this replaced would have hung forever.+ # Cap each lap at the time left: a flat poll_interval overshoots any shorter timeout, and+ # readiness inside that overshoot would return success from an expired deadline. No deadline means+ # full laps, which still notice a dead child where the bare Event.wait(None) this replaced hung.remaining = _remaining(deadline)if remaining is not None and remaining <= 0:self._fail_vllm_server(f"was not ready within {timeout} seconds")@@ -357,8 +350,8 @@ class SyntheticDataKit:self._fail_vllm_server(f"exited with code {returncode} before it was ready")if self.stdout_capture.has_closed():self._fail_vllm_server("closed its stdout before it was ready")- # Expiry is checked at the top, so a dead or closed child keeps- # its own message rather than being reported as a timeout.+ # Expiry is checked at the top, so a dead or closed child keeps its own message rather than being+ # reported as a timeout.def _fail_vllm_server(self, what_happened):"""Terminate the server and raise, quoting what it managed to say.@@ -412,8 +405,7 @@ class SyntheticDataKit:response = requests.get("http://localhost:8000/metrics", timeout = 5)return response.status_code == 200except requests.exceptions.RequestException:- # ConnectionError alone let a read timeout escape as a stray- # traceback out of the readiness loop.+ # ConnectionError alone let a read timeout escape as a stray traceback out of the readiness loop.return Falsedef cleanup(self):@@ -445,7 +437,6 @@ class SyntheticDataKit:torch.cuda.empty_cache()gc.collect()- # Delete vLLM module as wellif hasattr(self, "_delete_vllm"):self._delete_vllm(llm = None)@@ -477,8 +468,8 @@ class SyntheticDataKit:if max_tokens <= 5:raise RuntimeError("Generation length is way too long!")if max_tokens <= self.overlap:- # A non-positive stride (max_tokens - overlap) makes the n_chunks- # computation below divide by zero or go negative, so reject it.+ # A non-positive stride (max_tokens - overlap) makes the n_chunks computation below divide by zero+ # or go negative.raise RuntimeError(f"The chunk size (max_seq_length - 2 * max_generation_tokens - 128 = "f"{max_tokens}) must be larger than the overlap ({self.overlap}). "@@ -489,21 +480,16 @@ class SyntheticDataKit:# Get left and right boundarieslength = len(input_ids)if length <= max_tokens:- # The whole document fits in one chunk window, so emit it as a single- # chunk. Routing it through the multi-chunk path below would drop it- # (the linspace/stack pairing emits one fewer range than boundary- # points) or, for a document shorter than the overlap, slice the wrong- # tokens via negative start indices. Empty doc -> no chunk.+ # The whole document fits one chunk window: the multi-chunk path below would drop it (the+ # linspace/stack pairing emits one fewer range than boundary points) or, for a document shorter+ # than the overlap, slice the wrong tokens via negative start indices.boundaries = [[0, length]] if length > 0 else []else:- # length > max_tokens > overlap here, so length - overlap > 0 and the- # linspace boundaries below are always non-negative.- # Minimal count: overlapping chunks cover `length` in- # ceil((length - overlap) / stride) chunks, not ceil(length / stride)- # which over-splits just past a stride multiple.+ # Minimal count: overlapping chunks cover `length` in ceil((length - overlap) / stride) chunks, not+ # ceil(length / stride), which over-splits just past a stride multiple.n_chunks = int(np.ceil((length - self.overlap) / (max_tokens - self.overlap)))- # n_chunks + 1 points: [:-1]/[1:] pairing yields n_chunks ranges; using- # n_chunks points gave one fewer, oversized chunk (over max_tokens).+ # n_chunks + 1 points: the [:-1]/[1:] pairing yields n_chunks ranges; n_chunks points gave one+ # fewer, oversized chunk (over max_tokens).boundaries = np.ceil(np.linspace(0, length - self.overlap, n_chunks + 1)).astype(int)boundaries = np.stack((boundaries[:-1], (boundaries + self.overlap)[1:])).Tboundaries = np.minimum(boundaries, length).tolist()diff --git a/unsloth/dataset_num_proc.py b/unsloth/dataset_num_proc.pyindex f891074d3b3..38b4df82a79 100644--- a/unsloth/dataset_num_proc.py+++ b/unsloth/dataset_num_proc.py@@ -43,31 +43,28 @@ __all__ = ["resolve_responses_only_num_proc",]-# Escape hatch: a positive integer forces that count verbatim (no cap, no-# start-method veto), "0"/"none" forces in-process tokenization.+# Escape hatch: a positive integer forces that count verbatim (no cap, no start-method veto),+# "0"/"none" forces in-process tokenization.NUM_PROC_ENV_VAR = "UNSLOTH_DATASET_NUM_PROC"-# Upper bound for the *auto* count only; raise it via NUM_PROC_ENV_VAR. The old-# min(max(cpu_count + 4, 2), 64) forked up to 64 workers, each handed its own-# dill-pickled tokenizer closure and an Arrow shard over a pipe. Measured on-# 8000 rows, Qwen2.5 fast tokenizer, 3 reps: None 6.3s, 1 9.5s, 8 8.1s,-# 32 14.2s, 64 21.7s -- more workers were slower than none at every size.+# Upper bound for the AUTO count only; raise it via NUM_PROC_ENV_VAR. The old+# min(max(cpu_count + 4, 2), 64) forked up to 64 workers, each handed its own dill-pickled+# tokenizer closure and an Arrow shard over a pipe: measured on 8000 rows, more workers were+# slower than none at every size (None 6.3s against 64 workers 21.7s).AUTO_NUM_PROC_CAP = 8-# ~680 MB peak RSS per worker on that run, flat across counts; 1 GB for headroom.+# ~680 MB peak RSS per worker, flat across counts; 1 GB for headroom.WORKER_MEMORY_BUDGET_GB = 1.0-# Share of *available* RAM tokenization may spend; the rest belongs to the run-# that follows. This is what bounds issue #2693, where OOM-killed workers-# surface only as "One of the subprocesses has abruptly died during map-# operation", the real cause discarded.+# Share of AVAILABLE RAM tokenization may spend; the rest belongs to the run that follows. This is+# what bounds #2693, where OOM-killed workers surface only as "One of the subprocesses has+# abruptly died during map operation".MEMORY_BUDGET_FRACTION = 0.5-# Mirrors _MIN_ROWS_FOR_MULTIPROC, a local inside-# dataset_utils.train_on_responses_only (hence not importable): below it that-# helper maps a split in-process, unless handed an explicit count.-# resolve_responses_only_num_proc needs the threshold to keep that guard; a-# canary in tests/test_dataset_num_proc.py catches drift.+# Mirrors _MIN_ROWS_FOR_MULTIPROC, a local inside dataset_utils.train_on_responses_only (hence not+# importable): below it that helper maps a split in-process unless handed an explicit count.+# resolve_responses_only_num_proc needs the threshold to keep that guard; a canary in+# tests/test_dataset_num_proc.py catches drift.ZOO_MIN_ROWS_FOR_MULTIPROC = 5_000# Warn at most once per process per distinct reason.@@ -105,9 +102,8 @@ def _unpinned_default_start_method(module) -> Optional[str]:try:name = module.context._default_context._default_context._name- # Only if the platform actually offers it: a Windows runner answered- # 'fork' while get_all_start_methods() was ['spawn'], which would read- # Windows as forkable -- the spawn re-import loop of #3211 / #3397.+ # Only if the platform actually offers it: a Windows runner answered 'fork' while+ # get_all_start_methods() was ['spawn'], which would read Windows as forkable (#3211 / #3397).if isinstance(name, str) and name and (not methods or name in methods):return nameexcept Exception:@@ -140,6 +136,7 @@ def multiprocessing_start_method() -> Optional[str]:try:return _module_start_method(module_name)except Exception:+ # Skip rather than let it mask a sized sibling.continuereturn None@@ -187,9 +184,8 @@ CGROUP_ROOT = "/sys/fs/cgroup"def _cgroup_first_line(path: str) -> Optional[str]:try:- # encoding named explicitly: these are ASCII kernel files, but a- # locale-dependent read crashes or produces mojibake on a Windows- # console codepage, and CI polices every read/write for it.+ # encoding named explicitly: a locale-dependent read of these ASCII kernel files crashes or produces+ # mojibake on a Windows console codepage, and CI polices every read/write for it.with open(path, "r", encoding = "utf-8") as f:return f.readline().strip()except OSError:@@ -250,8 +246,8 @@ def _cgroup_free_bytes_unaided() -> Optional[int]:if os.path.isdir(CGROUP_ROOT):rel = None- # The v2 line is "0::<path>"; under systemd hybrid mode v1 lines share- # the file, so scan rather than taking the first.+ # The v2 line is "0::<path>"; under systemd hybrid mode v1 lines share the file, so scan rather than+ # taking the first.for line in lines:if line.startswith("0::"):rel = line[3:].strip()@@ -309,10 +305,9 @@ def _cgroup_free_bytes() -> Optional[int]:_read_first_line,)except Exception:- # An older unsloth_zoo has no such private helpers. Do the same pairing- # here rather than reading the public limit alone: an 8GB cgroup with 6GB- # already resident would otherwise report 8GB free, and memory pressure- # is the one condition this ceiling exists for.+ # An older unsloth_zoo has no such private helpers. Do the same pairing here rather than reading+ # the public limit alone: an 8GB cgroup with 6GB already resident would otherwise report 8GB free,+ # and memory pressure is the one condition this ceiling exists for.return _cgroup_free_bytes_unaided()def _used(path):@@ -404,8 +399,7 @@ def _clamp_by_memory(num_proc: int) -> Optional[int]:"""affordable = _affordable_workers()if affordable is None:- # No memory reading, so honour the request rather than serialising a- # machine that may be perfectly capable.+ # No memory reading, so honour the request rather than serialising a machine that may be perfectly capable.return num_procif affordable < 2:@@ -531,17 +525,15 @@ def get_dataset_num_proc(A worker count >= 2, or this layer's in-process sentinel (``None`` at acall site, ``1`` at the config layer)."""- # 1. The environment override wins over everything, uncapped and unvetoed,- # so a user who knows their workload is fork-safe is never downgraded.+ # 1. The environment override wins over everything, uncapped and unvetoed, so a user who knows+ # their workload is fork-safe is never downgraded.env_set, env_value = _from_environment()if env_set:- # In-process still has to be encoded for this layer, or- # UNSLOTH_DATASET_NUM_PROC=0 -- the hatch the dead-worker message- # recommends -- would inflate a config instead of removing workers.+ # In-process still has to be encoded for this layer, or UNSLOTH_DATASET_NUM_PROC=0 (the hatch the+ # dead-worker message recommends) would inflate a config instead of removing workers.return _serial(serial_as_none) if env_value is None else env_value- # 2. Workers are unusable whatever was requested; see- # `_workers_unusable_reason` for the two refusals.+ # 2. Workers are unusable whatever was requested; see _workers_unusable_reason.unusable = _workers_unusable_reason()if unusable is not None:if isinstance(desired, int) and not isinstance(desired, bool) and desired > 1:@@ -553,13 +545,13 @@ def get_dataset_num_proc()return _serial(serial_as_none)- # 3. Normalise "no multiprocessing" requests. `1` is the trap: callers pass- # it meaning "serial" and datasets >= 4.1 hands them a Pool(1).+ # 3. Normalise "no multiprocessing" requests. `1` is the trap: callers pass it meaning "serial"+ # and datasets >= 4.1 hands them a Pool(1).if isinstance(desired, int) and not isinstance(desired, bool) and desired <= 1:return _serial(serial_as_none)- # 4. Auto-size when no usable request was made, then bound by memory. A- # non-int, or a bool (an int subclass), is not a request.+ # 4. Auto-size when no usable request was made, then bound by memory. A non-int, or a bool (an int+ # subclass), is not a request.if desired is None or not isinstance(desired, int) or isinstance(desired, bool):num_proc = _auto_num_proc()else:@@ -570,9 +562,9 @@ def get_dataset_num_proc(return num_proc-# The message datasets raises when a pool worker dies. It never reads the child's-# exit status, so an OOM kill, a segfault and a genuine exception all arrive as-# this one string -- which is why #2693 looks untraceable.+# The message datasets raises when a pool worker dies. It never reads the child's exit status, so+# an OOM kill, a segfault and a genuine exception all arrive as this one string, which is why+# #2693 looks untraceable._WORKER_DIED = "subprocesses has abruptly died"@@ -634,7 +626,6 @@ def _largest_split_rows(trainer) -> Optional[int]:try:rows = len(split)except Exception:- # Skip rather than let it mask a sized sibling.continuemeasured = Truelargest = max(largest, rows)@@ -699,8 +690,8 @@ def resolve_responses_only_num_proc(trainer, num_proc):have had: worth it only when a split is big enough to have beenparallelized at all, which is what the row check below establishes."""- # Mirror that helper's own test, so "explicit" means the same on both sides- # (it treats bools as auto, since type(True) is not int).+ # Mirror that helper's own test, so "explicit" means the same on both sides (it treats bools as+ # auto, since type(True) is not int).was_auto = num_proc is None or type(num_proc) is not introws = _largest_split_rows(trainer)small = rows is None or rows < ZOO_MIN_ROWS_FOR_MULTIPROC@@ -708,11 +699,7 @@ def resolve_responses_only_num_proc(trainer, num_proc):if not was_auto:resolved = get_dataset_num_proc(num_proc, serial_as_none = False)if resolved == 1 and small:- # Serial, and every split is under the threshold, so the helper's own- # guard runs each of them in-process, which an older unsloth_zoo- # would not do for the 1 expressible here: reaching this with- # UNSLOTH_DATASET_NUM_PROC=0 and a config sentinel of 1 was the- # escape hatch still building a Pool(1) there.+ # Serial with every split under the threshold, so the helper's own guard runs each in-process.return Nonereturn _serial_for_the_zoo(resolved)@@ -721,10 +708,8 @@ def resolve_responses_only_num_proc(trainer, num_proc):env_set, env_value = _from_environment()if env_set and env_value is not None:return env_value- # Otherwise it would have gone in-process anyway, and its guard yields- # None, which is more in-process than the 1 expressible here. Safe whatever- # the start method: every split here is under the threshold, so its own- # per-split guard serialises them all.+ # Otherwise it would have gone in-process anyway, and its guard yields None, which is more in-+ # process than the 1 expressible here. Safe whatever the start method.return num_procreturn _serial_for_the_zoo(get_dataset_num_proc(None, serial_as_none = False))diff --git a/unsloth/device_type.py b/unsloth/device_type.pyindex ebb3001e840..d6793ed421b 100644--- a/unsloth/device_type.py+++ b/unsloth/device_type.py@@ -60,13 +60,12 @@ def is_hip():@functools.cachedef get_device_type():- # MLX first: torch is never imported on the MLX runtime, so claiming "cuda" here- # would NameError in get_device_count. Matches unsloth/__init__.py and- # unsloth_zoo.device_type, which both pick MLX ahead of the CPU fallback.+ # MLX first: torch is never imported on the MLX runtime, so claiming "cuda" here would NameError in+ # get_device_count. Matches unsloth/__init__.py and unsloth_zoo.device_type.if _IS_MLX:return "mlx"- # Test-only CPU fallback: report "cuda" so every DEVICE_TYPE == "cuda"- # branch behaves identically. Read once per process (function is cached).+ # Test-only CPU fallback: report "cuda" so every DEVICE_TYPE == "cuda" branch behaves identically.+ # Read once per process (function is cached).if os.environ.get("UNSLOTH_ALLOW_CPU", "0") == "1":return "cuda"if hasattr(torch, "cuda") and torch.cuda.is_available():@@ -75,7 +74,6 @@ def get_device_type():return "cuda"elif hasattr(torch, "xpu") and torch.xpu.is_available():return "xpu"- # Check torch.acceleratorif hasattr(torch, "accelerator"):if not torch.accelerator.is_available():raise NotImplementedError("Unsloth cannot find any torch accelerator? You need a GPU.")@@ -110,28 +108,19 @@ def get_device_count():DEVICE_COUNT: int = get_device_count()-# 4-bit quantization requires a block size of 64-# | Device Type | Warp Size | Block Size |-# |-----------------|-----------|------------|-# | CUDA | 32 | 32 |-# | Radeon (Navi) | 32 | 32 |-# | Instinct (MI) | 64 | 32 |-#-# Since bitsandbytes 0.49.0, pre-quantized models with 64 blockwise now works-# on Radeon GPUs, but not Instinct MI300x for eg-# See https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1748-#-# Since bitsandbytes 0.49.2, blocksize=64 4-bit quantization is supported on-# CDNA (MI Instinct / gfx9xx) GPUs as well-# See https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1856+# 4-bit quantization requires a block size of 64: Instinct (MI) has a warp size of 64 against 32+# elsewhere. Since bitsandbytes 0.49.0 pre-quantized 64-blockwise models work on Radeon (Navi)+# but not Instinct (bitsandbytes-foundation/bitsandbytes#1748); since 0.49.2 blocksize=64 4-bit+# is supported on CDNA (MI Instinct / gfx9xx) too (#1856).+# |-----------------|-----------|------------|ALLOW_PREQUANTIZED_MODELS: bool = True# HSA_STATUS_ERROR_EXCEPTION checks - sometimes AMD fails for BnBALLOW_BITSANDBYTES: bool = True-# Unusable bitsandbytes on any backend, not just hip: clear the flags the loader reads-# before it picks a 4bit checkpoint. A guarded import, not find_spec, since importable-# is not usable - from 0.46 a dead native library still resolves every ctypes handle to-# a closure that raises only when called, so 4bit would die mid-run, not fall back here.+# Unusable bitsandbytes on any backend, not just hip: clear the flags the loader reads before it+# picks a 4bit checkpoint. A guarded import, not find_spec, since importable is not usable: from+# 0.46 a dead native library still resolves every ctypes handle to a closure that raises only when+# called, so 4bit would die mid-run rather than fall back here.try:import bitsandbytes as _bnb_probeexcept Exception:@@ -142,10 +131,8 @@ else:ALLOW_PREQUANTIZED_MODELS = FalseALLOW_BITSANDBYTES = Falsedel _bnb_probe-# gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this-# legacy GCN arch (ROCm dropped it after 6.3) - compiled graphs crash or miscompile-# while the eager path trains fine. Default compile off; setdefault so a user-# override wins.+# gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this legacy GCN arch+# (ROCm dropped it after 6.3), so compiled graphs crash or miscompile while eager trains fine.if DEVICE_TYPE == "hip":try:_gcn_arch = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0].strip().lower()@@ -174,7 +161,7 @@ if DEVICE_TYPE == "hip":passelif Version(bitsandbytes.__version__) >= Version("0.49.0"):try:- # Pre-quantized bitsandbytes models use blocksize 64, so we need to check the GPU+ # Pre-quantized bitsandbytes models use blocksize 64.from bitsandbytes.cextension import ROCM_WARP_SIZE_64ALLOW_PREQUANTIZED_MODELS = not ROCM_WARP_SIZE_64except Exception as e:diff --git a/unsloth/disk_utils.py b/unsloth/disk_utils.pyindex 43110475d33..684b58bd6a1 100644--- a/unsloth/disk_utils.py+++ b/unsloth/disk_utils.py@@ -61,11 +61,9 @@ except ImportError:_TRUE = ("1", "true", "yes", "on")def is_kaggle_environment():- # Kept byte for byte in step with unsloth_zoo.disk_utils: only a real- # kernel sets KAGGLE_KERNEL_RUN_TYPE, and only the Kaggle image has a- # /kaggle/working. KAGGLE_USERNAME / KAGGLE_KEY / KAGGLE_CONFIG_DIR- # are what people export for the Kaggle CLI on their own machines,- # and must never be mistaken for one.+ # Kept in step with unsloth_zoo.disk_utils: only a real kernel sets KAGGLE_KERNEL_RUN_TYPE and+ # only the Kaggle image has /kaggle/working, whereas KAGGLE_USERNAME / KAGGLE_KEY /+ # KAGGLE_CONFIG_DIR are what people export for the Kaggle CLI on their own machines.override = os.environ.get("UNSLOTH_IS_KAGGLE", None)if override is not None:return str(override).strip().lower() in _TRUE@@ -100,10 +98,9 @@ except ImportError:return Nonedef logical_numel(param, name = ""):- # Packed storage cannot be unpacked without the zoo's knowledge of the- # packing schemes, so this reports what `numel()` says, exactly as the- # code did before it asked. Barely reachable: `model_16bit_bytes` is 0- # here, and every caller returns before sizing anything.+ # Packed storage cannot be unpacked without the zoo's knowledge of the packing schemes, so report+ # what numel() says, exactly as the code did before it asked. Barely reachable: model_16bit_bytes+ # is 0 here and every caller returns before sizing anything.try:return int(param.numel())except Exception:@@ -116,8 +113,8 @@ except ImportError:return 0def estimate_gguf_export_bytes(*args, **kwargs):- # 0 means "unmeasurable", and every caller treats that as "do not- # block". An old unsloth_zoo therefore behaves exactly as it did.+ # 0 means "unmeasurable", and every caller treats that as "do not block", so an old unsloth_zoo+ # behaves exactly as it did.return 0def kaggle_tmp_redirect(save_directory, *args, **kwargs):diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 7af9b08e8e9..c161babd18a 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -56,7 +56,7 @@ def Version(version):raise ValueError(f"Could not parse version: {version}")new_version = new_version.group(0).rstrip(".")if new_version != version:- new_version += ".1" # Add .1 for dev / alpha / beta / rc+ new_version += ".1"return TrueVersion(new_version)except:from inspect import getframeinfo, stack@@ -145,9 +145,10 @@ def suppress_cuda_printf():if not UNSLOTH_ENABLE_LOGGING:import sys- # Apply to stderr for FBGEMM and CUTLASS errors+ # Applied to stderr for FBGEMM (pytorch/FBGEMM utils.py#L43-L52) and CUTLASS SM90-vs-SM100 MMA / arch / TMA errors.sys.stderr = HidePrintMessage(sys.stderr)- # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52+ # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/ge+ # mm/triton_gemm/utils.py#L43-L52sys.stderr.add_filter("TMA benchmarks will be running")# CUTLASS/FBGEMM MMA instruction error on SM90 vs SM100 (Blackwell) GPUs# https://github.com/NVIDIA/cutlass/blob/main/include/cutlass/gemm/kernel/sm90_gemm_tma_warpspecialized.hpp@@ -157,15 +158,14 @@ if not UNSLOTH_ENABLE_LOGGING:# CUTLASS TMA-related errors when not targeting correct architecturesys.stderr.add_filter("Trying to use tma without CUTE_ARCH_TMA")# torchao logs a cosmetic "Skipping import of cpp extensions" WARNING on torch < 2.11. The- # bnb-4bit / Unsloth paths don't use torchao's cpp kernels, so drop only that record rather- # than raising the whole torchao logger to ERROR.+ # bnb-4bit / Unsloth paths do not use its cpp kernels, so drop that one record rather than+ # raising the whole torchao logger to ERROR.logging.getLogger("torchao").addFilter(HideLoggingMessage("Skipping import of cpp extensions due to incompatible torch version"))- # torch >= 2.11 path: torchao dlopens each prebuilt _C*.so and logs "Failed to load- # .../_C*.so" when one can't (ABI tag mismatch in the wheel, e.g. a cp310 .so under a- # cp312 runtime on Colab, or an arch-specific kernel the GPU lacks). It falls back to- # non-cpp paths and Unsloth doesn't use these kernels, so drop the cosmetic record.+ # torch >= 2.11: torchao dlopens each prebuilt _C*.so and logs a failure on an ABI tag+ # mismatch (a cp310 .so under cp312) or a missing arch kernel, then falls back to non-cpp+ # paths Unsloth does not use.logging.getLogger("torchao").addFilter(HideLoggingMessage("Failed to load "))# SyntaxWarning: invalid escape sequence '\.'warnings.filterwarnings("ignore", message = "invalid escape sequence", category = SyntaxWarning)@@ -193,7 +193,6 @@ if not UNSLOTH_ENABLE_LOGGING:category = DeprecationWarning,)- # SWIG builtin type warnings (from bitsandbytes/triton SWIG bindings)warnings.filterwarnings("ignore",message = r"builtin type Swig.*has no __module__ attribute",@@ -285,8 +284,8 @@ def fix_torch_check_is_size():return-# Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'-# MUST do this at the start primarily due to tensorflow causing issues+# Fix "AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'" first, mainly+# because tensorflow causes issues.def fix_message_factory_issue():try:import google.protobuf.message_factory@@ -328,7 +327,7 @@ def fix_message_factory_issue():pass-# Fix Xformers performance issues since 0.0.25+# Fix xformers performance issues since 0.0.25; see facebookresearch/xformers#1176 (comment 2545829591).def fix_xformers_performance_issue():spec = importlib.util.find_spec("xformers")if spec is None:@@ -359,18 +358,14 @@ def fix_xformers_performance_issue():logger.info(f"Unsloth: Failed patching Xformers with error = {str(e)}")-# flash-attn 4 ships `flash_attn/cute/` with NO `flash_attn/__init__.py`, so `flash_attn`-# resolves as a namespace package with no `flash_attn.flash_attn_interface`. xformers gates on-# `find_spec("flash_attn")` and then imports that submodule unguarded, so `import xformers.ops`-# raises, `models/_utils.py` swallows it into `xformers = None`, HAS_XFORMERS goes False and-# every fast-path model silently drops to SDPA. Measured on a B200 at seq_len 8192 with-# Qwen3-0.6B + LoRA: 547 ms/step / 2.69 GB peak with xformers versus 2154 ms/step / 19.02 GB-# peak on SDPA -- 3.9x slower, 7x memory, no message. flash-attn 2 has no wheel for torch 2.9 /-# cp313, so flash-attn 4 is what a Blackwell user reaches for.-#-# The repair imports xformers ONCE with `flash_attn` hidden from `find_spec`, so xformers takes-# the next branch of its own elif chain exactly as it would with no flash-attn at all, and the-# working module is cached in `sys.modules`. Nothing is written to any third-party package.+# flash-attn 4 ships flash_attn/cute/ with no __init__.py, so flash_attn is a namespace package+# without flash_attn.flash_attn_interface. xformers gates on find_spec then imports that submodule+# unguarded, so `import xformers.ops` raises, _utils.py swallows it into xformers = None,+# HAS_XFORMERS goes False and every fast-path model drops to SDPA: measured on a B200 at seq_len+# 8192, 547 ms/step and 2.69 GB peak against 2154 ms/step and 19.02 GB on SDPA. The repair imports+# xformers ONCE with flash_attn hidden from find_spec, so it takes the next branch of its own elif+# chain and the working module is cached in sys.modules. Nothing is written to any third-party+# package._FLASH_ATTN_INTERFACE_NAME = "flash_attn_interface"_FLASH_ATTN_INTERFACE_MODULE = "flash_attn." + _FLASH_ATTN_INTERFACE_NAME_FA4_NAMESPACE_WARNED = [False]@@ -464,13 +459,12 @@ def fix_flash_attn_4_namespace_shadow():if _flash_attn_layout() != "flash_attn_4_only":returnif importlib.util.find_spec("xformers") is None:- # Nothing to protect: no xformers means SDPA regardless, and transformers'- # `is_flash_attn_2_available()` is False here (the distribution is `flash-attn-4`, so- # the metadata lookup for `flash_attn` misses).+ # Nothing to protect: no xformers means SDPA anyway, and is_flash_attn_2_available() is False+ # here, since the distribution is flash-attn-4 and the flash_attn metadata lookup misses.returnif "xformers.ops.fmha.flash" in sys.modules:- # Already imported successfully. A FAILED import leaves nothing in sys.modules, so this- # does not mask the case we are here to fix.+ # Already imported successfully. A FAILED import leaves nothing in sys.modules, so this does+ # not mask the case we are here to fix.returnreal_find_spec = importlib.util.find_spec@@ -480,9 +474,9 @@ def fix_flash_attn_4_namespace_shadow():return Nonereturn real_find_spec(name, package)- # Process-global swap, scoped to this one import and restored in `finally`. Any other thread- # calling find_spec("flash_attn*") in the ~1.0s window is told the package is absent, which- # is the honest answer for the FA2 namespace xformers is asking about anyway.+ # Process-global swap, scoped to this one import and restored in `finally`. Another thread+ # calling find_spec("flash_attn*") in the ~1.0s window is told it is absent, which is honest+ # for the FA2 namespace xformers asks about.importlib.util.find_spec = _find_spec_without_flash_attntry:import xformers.ops # noqa: F401@@ -669,7 +663,7 @@ def fix_transformers5_bare_annotation_configs():original = PretrainedConfig.__dict__.get("__init_subclass__")if original is None:return- # `__init_subclass__` is an implicit classmethod; unwrap to the function.+ # __init_subclass__ is an implicit classmethod; unwrap to the function.original_func = getattr(original, "__func__", original)def __init_subclass__(cls, *args, **kwargs):@@ -779,14 +773,12 @@ def _sdpa_mask_leaves_rows_fully_masked():sdpa_mask = getattr(masking_utils, "sdpa_mask", None)if sdpa_mask is None:return False- # Unwrap first: a second call must probe the ORIGINAL, or the patch we- # already installed reports the bug as fixed and a reload silently drops it.+ # Unwrap first: a second call must probe the ORIGINAL, or the installed patch reports the bug+ # as fixed and a reload silently drops it.sdpa_mask = getattr(sdpa_mask, "__wrapped__", sdpa_mask)- # The query axis is named differently across the versions we support: 5.x- # takes `q_length`, while 4.57.6 binds `sdpa_mask` to `sdpa_mask_recent_torch`,- # which takes `cache_position` instead. Ask the signature rather than- # guessing -- a probe that raises TypeError answers "no bug" for a reason- # that has nothing to do with the bug.+ # The query axis is named differently per version: 5.x takes q_length, while 4.57.6 binds+ # sdpa_mask to sdpa_mask_recent_torch, which takes cache_position. Ask the signature, since a+ # probe that raises TypeError answers "no bug" for an unrelated reason.kwargs = {"batch_size": 2,"kv_length": 2,@@ -809,9 +801,8 @@ def _sdpa_mask_leaves_rows_fully_masked():mask = sdpa_mask(**kwargs)if mask is None or mask.is_floating_point():return False- # Inside the guard as well: reading a mask's truth value is what fails- # on a meta or otherwise unmaterialised tensor, and a probe that raises- # would take `import unsloth` down with it.+ # Inside the guard too: reading a mask's truth value is what fails on a meta or unmaterialised+ # tensor, and a probe that raises would take `import unsloth` down with it.return bool((~mask.any(dim = -1)).any())except Exception:# Signature moved under us. Patching blind would be worse than not.@@ -869,21 +860,16 @@ def fix_transformers_fully_masked_rows():def sdpa_mask(*args, **kwargs):return _unmask_rows_attending_to_nothing(original(*args, **kwargs))- # `functools.wraps` sets __wrapped__, but set it explicitly: the probe and- # the tests both reach for it, and a future wraps-less edit must not- # silently make the patch un-probeable and un-undoable.+ # functools.wraps sets __wrapped__, but set it explicitly: the probe and the tests both read+ # it, and a wraps-less edit must not make the patch un-probeable and un-undoable.sdpa_mask.__wrapped__ = original- # The mark travels ON the wrapper, so the guard above reads the live- # binding and a reload that drops it is re-patched rather than skipped.+ # The mark travels ON the wrapper, so the guard above reads the live binding and a reload that+ # drops it is re-patched rather than skipped.setattr(sdpa_mask, _SDPA_MASK_PATCH_FLAG, True)try:- # BOTH bindings, and they are not the same object. `eager_mask` calls- # the module global by name at call time, so it picks this up; the- # interface captured the original function in `_global_mapping` when- # the class body ran, so it needs re-registering. Confirmed by search:- # `sdpa_mask` is defined once and no other transformers module- # references it.+ # BOTH bindings, which are not the same object: eager_mask calls the module global by name at+ # call time, while the interface captured the original in _global_mapping at class-body time.masking_utils.sdpa_mask = sdpa_maskinterface = getattr(masking_utils, "ALL_MASK_ATTENTION_FUNCTIONS", None)if interface is not None:@@ -902,8 +888,7 @@ def fix_vllm_aimv2_issue():spec = importlib.util.find_spec("vllm")if spec is None:return- # A findable spec with unreadable dist metadata (broken/partial vllm install)- # must not crash unsloth import; every other vllm probe here guards this too.+ # A findable spec with unreadable dist metadata (broken/partial vllm install) must not crash `import unsloth`.try:vllm_version = importlib_version("vllm")except Exception as e:@@ -946,10 +931,9 @@ def fix_vllm_aimv2_issue():logger.info(f"Unsloth: Failed patching vLLM with error = {str(e)}")-# vLLM >= 0.22 (PR #35024) deleted `vllm.transformers_utils.tokenizer`, but an-# older unsloth_zoo still imports it unguarded and crashes (issue #6385). Supply-# a stub via a meta path finder appended AFTER the real finders, so it only-# activates when vLLM no longer ships the module.+# vLLM >= 0.22 (PR #35024) deleted vllm.transformers_utils.tokenizer, which an older+# unsloth_zoo still imports unguarded (#6385). Stub it with a meta path finder appended AFTER+# the real ones, so it only fires when vLLM no longer ships the module._VLLM_LORA_TOKENIZER_MODULE = "vllm.transformers_utils.tokenizer"_VLLM_TOKENIZER_STUB_SENTINEL = "__unsloth_vllm_tokenizer_stub__"@@ -1033,9 +1017,7 @@ def fix_vllm_guided_decoding_params():if importlib.util.find_spec("vllm") is None:return- # GuidedDecodingParmas is renamed to StructuredOutputsParams in vLLM- # https://github.com/vllm-project/vllm/pull/22772/files- # trl still wants to use GuidedDecodingParams. This is a temporary patch till trl updates+ # GuidedDecodingParams was renamed to StructuredOutputsParams in vLLM (vllm#22772); trl still wants the old name.try:import vllmexcept (ImportError, OSError) as e:@@ -1058,17 +1040,10 @@ def fix_vllm_guided_decoding_params():def fix_trl_vllm_ascend():- # transformers >= 4.48's `_is_package_available(name)` returns a tuple- # (bool, version_or_None). TRL caches that tuple in module-level- # `_*_available` flags and the matching `is_*_available()` accessors- # return the tuple directly. A non-empty tuple is always truthy, so- # `if is_X_available():` fires even when X is absent, triggering an- # unconditional `import X` that fails. The surfaced case is- # `vllm_ascend` (blocks `from trl import GRPOConfig, GRPOTrainer`- # outside Huawei Ascend hosts); `llm_blender`, `deepspeed`, `joblib`- # share the same shape. Coerce every tuple-cached flag in- # trl.import_utils to bool; the existing accessors that just return- # the cached value then naturally yield a bool.+ # transformers >= 4.48's _is_package_available returns (bool, version_or_None), which TRL+ # caches and returns directly. A non-empty tuple is truthy, so `if is_X_available():` fires+ # for an absent X and triggers a failing import -- vllm_ascend blocked `from trl import+ # GRPOConfig` off Ascend hosts. Coerce every tuple-cached flag to bool.if importlib.util.find_spec("trl") is None:returntry:@@ -1084,7 +1059,6 @@ def fix_trl_vllm_ascend():def ignore_logger_messages():- # Ignore Environment variable `HF_TOKEN` is settry:from huggingface_hub._login import logger as huggingface_hub_loggerhuggingface_hub_logger.addFilter(HideLoggingMessage("`HF_TOKEN`"))@@ -1094,10 +1068,8 @@ def ignore_logger_messages():def patch_ipykernel_hf_xet():- # HF-XET == 1.1.10 and ipykernel == 7.0.0 / 7.0.1 causes issues- # See https://github.com/huggingface/xet-core/issues/526- # 2025-10-13T20:37:33.028737Z ERROR Python exception updating progress:, error: PyErr { type: <class 'LookupError'>, value: LookupError(<ContextVar name='shell_parent' at 0x7535b4cebd80>), traceback: Some(<traceback object at 0x753408489f40>) }, caller: "src/progress_update.rs:313"- # at /home/runner/work/xet-core/xet-core/error_printer/src/lib.rs:28+ # HF-XET 1.1.10 with ipykernel 7.0.0 / 7.0.1 raises LookupError on ContextVar 'shell_parent';+ # see huggingface/xet-core#526.if importlib.util.find_spec("hf_xet") is None:returnif importlib.util.find_spec("ipykernel") is None:@@ -1106,14 +1078,8 @@ def patch_ipykernel_hf_xet():returnipykernel_version = Version(importlib_version("ipykernel"))- if (- (Version(importlib_version("hf_xet")) == Version("1.1.10"))- and (- (ipykernel_version == Version("7.0.0"))- or (- ipykernel_version == Version("7.0.1")- ) # 7.0.1 seems to also break with LookupError: <ContextVar name='shell_parent' at 0x7a9775143ec0>- )+ if (Version(importlib_version("hf_xet")) == Version("1.1.10")) and (+ (ipykernel_version == Version("7.0.0")) or (ipykernel_version == Version("7.0.1"))):print("#### Unsloth: `hf_xet==1.1.10` and `ipykernel==7.0.0` or `ipykernel==7.0.1` breaks progress bars. Using ASCII progress bars.\n"@@ -1125,8 +1091,7 @@ def patch_ipykernel_hf_xet():def patch_trackio():- # Set some environment variables to customize the Trackio dashboard for experiment tracking- # See https://github.com/unslothai/notebooks/pull/110+ # Customize the Trackio dashboard for experiment tracking; see unslothai/notebooks#110.os.environ["TRACKIO_LOGO_LIGHT_URL"] = ("https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20black%20text.png")@@ -1149,28 +1114,21 @@ def patch_datasets():)-# psutil divides the pmgr "voltage-statesN-sram" IORegistry tables by 1e6 to-# reach MHz, but Apple switched them from Hz to kHz on M4, so psutil <= 7.2.2-# reads a 4.5 GHz M4 Pro back as 4 MHz (unslothai/unsloth#8519). Upstream fix is-# giampaolo/psutil#2824, merged and unreleased; its heuristics are mirrored here.-#-# Unsloth's backend keeps the same correction in-# studio/backend/utils/hardware/hardware.py, since the API server never imports-# unsloth. Keep both in sync; delete both once a fixed psutil is our floor.-#+# psutil divides the pmgr voltage-states tables by 1e6 for MHz, but Apple switched them from+# Hz to kHz on M4, so psutil <= 7.2.2 reads a 4.5 GHz M4 Pro as 4 MHz (#8519). Upstream fix+# giampaolo/psutil#2824 is merged and unreleased; mirrored here and in+# studio/backend/utils/hardware/hardware.py. Keep both in sync; delete once psutil is fixed.# Apple clocks are 0.6-4.6 GHz, so a raw Hz entry sits above 1e8 and kHz below._APPLE_CPU_FREQ_UNIT_THRESHOLD = 100_000_000_APPLE_MIN_PLAUSIBLE_CPU_MHZ = 500_APPLE_MAX_PLAUSIBLE_CPU_MHZ = 20000-# Below this a table is a GPU/NPU rail: above every Apple GPU peak so far, under-# the slowest CPU cluster shipped (M1 E-core, 2064 MHz).+# Below this a table is a GPU/NPU rail: above every Apple GPU peak so far, under the slowest+# CPU cluster shipped (M1 E-core, 2064 MHz)._APPLE_CPU_CLUSTER_MIN_PEAK_MHZ = 2000_APPLE_VOLTAGE_STATES_KEY = re.compile(r"^voltage-states\d+-sram$")-# Fixed for the life of the host, so probe once. The sentinel separates "not-# probed yet" from "probed, unavailable".+# Fixed for the life of the host, so probe once. The sentinel separates "not probed yet" from "probed, unavailable"._apple_cpu_freq_range = "unprobed"-# At import, not on first use: two threads reaching a lazy initialiser build a-# lock each and then exclude nothing.+# At import, not on first use: two threads reaching a lazy initialiser build a lock each and then exclude nothing._apple_cpu_freq_lock = threading.Lock()@@ -1217,8 +1175,8 @@ def _apple_cpu_freq_range_mhz():if _apple_cpu_freq_range != "unprobed":return _apple_cpu_freq_range- # Unlocked, every thread spawns its own ioreg and a slow failing probe landing- # last overwrites a good reading with None.+ # Unlocked, every thread spawns its own ioreg and a slow failing probe landing last overwrites+ # a good reading with None.with _apple_cpu_freq_lock:if _apple_cpu_freq_range != "unprobed":return _apple_cpu_freq_range@@ -1257,9 +1215,9 @@ def _corrected_apple_cpu_freq(sample):# macOS has no per-instant clock, so psutil reports the peak as `current`.return sample._replace(current = peak, min = low, max = peak)if not usable:- return sample # 0, negative or NaN and no tables: nothing to say- # No tables: recover the magnitude instead. psutil truncates in integer- # arithmetic, so this lands on the GHz step (4 -> 4000 MHz), not the peak.+ return sample+ # No tables: recover the magnitude instead. psutil truncates in integer arithmetic, so this+ # lands on the GHz step (4 -> 4000 MHz), not the peak.return sample._replace(current = current * 1000,min = getattr(sample, "min", 0.0) * 1000,@@ -1318,13 +1276,12 @@ def patch_psutil_cpu_freq():try:result = original_cpu_freq(*args, **kwargs)except TypeError:- # Arguments psutil does not take are the caller's mistake, not psutil- # declining to answer. This function replaces psutil's globally, so- # swallowing that would hide the error everywhere, not just here.+ # Arguments psutil does not take are the caller's mistake, not psutil declining to answer.+ # This replaces psutil's function globally, so swallowing would hide the error everywhere.raiseexcept Exception:- # psutil raises on M5, whose renumbered tables are not at the indexes- # it hardcodes. With no tables either, its error is the honest answer.+ # psutil raises on M5, whose renumbered tables are not at the indexes it hardcodes. With no+ # tables either, its error is the honest answer.stand_in = _from_tables(_percpu_requested(args, kwargs))if stand_in is None:raise@@ -1354,8 +1311,8 @@ def check_fbgemm_gpu_version():fbgemm_gpu_version = importlib_version("fbgemm_gpu_genai")except:return- # We noticed some SegFault or bad alloc errors on lower versions of fbgemm_gpu.- # Instead of raising an error, disable FBGEMM and fall back to Triton kernels.+ # Lower fbgemm_gpu versions segfault or bad-alloc, so disable FBGEMM and fall back to Triton+ # rather than raise.if Version(fbgemm_gpu_version) < Version("1.4.0"):os.environ["UNSLOTH_HAS_FBGEMM"] = "0"logger.info(@@ -1373,8 +1330,7 @@ def patch_enable_input_require_grads():import inspectfrom transformers import PreTrainedModel- # Only patch the new variant that iterates over self.modules().- # Ref: https://github.com/huggingface/transformers/pull/41993/files#diff-6b72b98c4c2dcfc6cc606843917733f5d858374fbc22a735ff483bbc0c1e63eaL1979-R1996+ # Only patch the new variant that iterates over self.modules(); see huggingface/transformers#41993.try:original_source = inspect.getsource(PreTrainedModel.enable_input_require_grads)except:@@ -1399,8 +1355,7 @@ def patch_enable_input_require_grads():try:input_embeddings = module.get_input_embeddings()except NotImplementedError:- # Vision models may not implement get_input_embeddings (e.g. GLM- # V4.6 skips only `self.visual`); skip them+ # Vision models may not implement get_input_embeddings (GLM V4.6 skips only self.visual).continueif input_embeddings is None:@@ -1459,9 +1414,8 @@ def patch_unsafe_trainer_rng_load():"(CVE-2026-1839 / CVE-2025-32434); upgrade to torch >= 2.6.")- # Install one process-wide torch.load shim that stays inert unless the calling- # thread is inside _load_rng_state, so we gate only at the real rng load with- # no global-swap race and no effect on other torch.load callers.+ # One process-wide torch.load shim, inert unless the calling thread is inside _load_rng_state,+ # so the gate applies at the real rng load with no global-swap race.if not getattr(torch.load, "_unsloth_rng_guard", False):_orig_load = torch.load_rng_active = threading.local()@@ -1503,9 +1457,8 @@ def _is_custom_torch_build(raw_version_str):local = raw_version_str.split("+", 1)[1]if not local:return False- # Use fullmatch so the entire local identifier must match, not just a prefix.- # cu/rocm require a trailing digit (e.g. cu124, rocm6.3). cpu/xpu are exact.- # Case-insensitive since some builds may use uppercase.+ # fullmatch, so the whole local identifier matches: cu/rocm need a trailing digit (cu124,+ # rocm6.3), cpu/xpu are exact, and case-insensitive since some builds use uppercase.return not re.fullmatch(r"cu\d[\d.]*|rocm\d[\d.]*|cpu|xpu", local, re.IGNORECASE)@@ -1531,11 +1484,9 @@ _TORCHVISION_ABI_MARKERS = ("torchvision.io.video","torchvision.io._video",)-# A loader failure is a torchvision break only when it names torchvision or the-# torch libraries it links. The probe below imports torchvision where nothing-# used to, so a box whose torchvision cannot load for an UNRELATED reason (a-# missing CUDA library, say) must keep importing unsloth exactly as before-# instead of being handed a hard "reinstall torchvision".+# A loader failure means torchvision only when it names torchvision or the torch libraries it+# links: the probe below imports torchvision where nothing used to, so a box broken for an+# unrelated reason must keep importing unsloth, not get "reinstall torchvision"._LOADER_FAILURE_MARKERS = ("undefined symbol", "cannot open shared object file")_TORCH_LIBRARY_MARKERS = ("torchvision", "libtorch", "libc10", "_C.so", "c10::")@@ -1556,23 +1507,14 @@ def _is_broken_torchvision_error(error) -> bool:return False-# PyPI carries exactly one torchvision build per release and it is one CUDA-# family: its `_C.so` links libcudart, libc10_cuda and libtorch_cuda. Every-# other build -- CPU, XPU, ROCm, and every CUDA family but PyPI's -- lives only-# on download.pytorch.org, under an index named after torch's own local tag.-# `--no-deps` keeps the installed torch, so an unqualified pin swaps a working-# wheel for PyPI's and raises the very `operator torchvision::nms does not-# exist` this command is handed out to clear. A tag we cannot turn into an-# index (a vendor build: this repo's Radeon extras install-# `torch 2.9.1+rocm7.2.0.lw.git7e1940d4` beside a repo.radeon.com torchvision;-# or a source build) has no wheel to name at all, and neither does a-# prerelease, whose stable-looking companion is synthesised from the release-# numbers alone.+# PyPI carries one torchvision build per release, for one CUDA family; every other build lives+# on download.pytorch.org under an index named after torch's local tag. With --no-deps an+# unqualified pin swaps a working wheel for PyPI's and raises the very+# "operator torchvision::nms does not exist" it was handed out to clear._TORCH_BACKEND_INDEX = re.compile(r"cpu|xpu|cu\d+|rocm\d+(?:\.\d+)*", re.IGNORECASE)-# conda keeps the backend in the build string (`py3.12_cuda12.4_cudnn9_0`) and-# leaves the version plain, so a conda torch is indistinguishable from a PyPI-# one by version alone. Its own ledger tells them apart:-# `conda-meta/<name>-<version>-<build>.json`, one per installed package.+# conda keeps the backend in the build string (py3.12_cuda12.4_cudnn9_0) and leaves the version+# plain, so a conda torch looks like a PyPI one by version alone; its+# conda-meta/<name>-<version>-<build>.json tells them apart._CONDA_TORCH_PACKAGES = ("pytorch", "pytorch-cpu", "pytorch-gpu", "libtorch")@@ -1585,13 +1527,12 @@ def _torch_local_tag(torch_version_raw):def _torch_is_conda_managed(torch_version_raw):"""Did conda install this torch, rather than pip?"""conda_meta = os.path.join(sys.prefix, "conda-meta")- # A conda version never carries the `+tag`, but strip it so a pip torch- # sitting in a conda prefix is still matched on its release numbers.+ # A conda version never carries the +tag, but strip it so a pip torch inside a conda prefix is+ # still matched on its release numbers.version = (torch_version_raw or "").split("+", 1)[0]if not version or not os.path.isdir(conda_meta):return False- # Pinned to this exact version, so an unrelated `pytorch-lightning-*.json`- # cannot answer for torch.+ # Pinned to this exact version, so an unrelated pytorch-lightning-*.json cannot answer for torch.prefixes = tuple(f"{name}-{version}-" for name in _CONDA_TORCH_PACKAGES)try:entries = os.listdir(conda_meta)@@ -1602,17 +1543,15 @@ def _torch_is_conda_managed(torch_version_raw):def _has_no_matching_public_wheel(torch_version_raw):try:- # `.is_prerelease`, not a substring list: `2.11.0a1` and `2.11.0b2` are- # prereleases that no `a0`/`b0` match would catch.+ # .is_prerelease, not a substring list: 2.11.0a1 and 2.11.0b2 are prereleases no a0/b0 match would catch.if TrueVersion(torch_version_raw).is_prerelease:return Trueexcept Exception:return Truelocal = _torch_local_tag(torch_version_raw)if not local:- # An absent tag means PyPI for a pip install, but conda never writes one- # either, and its torch may be CPU, ROCm or a CUDA family PyPI does not- # ship. `--no-deps` would leave that torch beside PyPI's torchvision.+ # An absent tag means PyPI for a pip install, but conda never writes one either, and its torch+ # may be CPU, ROCm or a CUDA family PyPI does not ship.return _torch_is_conda_managed(torch_version_raw)return not _TORCH_BACKEND_INDEX.fullmatch(local)@@ -1642,10 +1581,8 @@ def _torchvision_repair_command(required = None, torch_version_raw = None):if required is None:spec = "torchvision"elif len(required) >= 3:- # Exact, because the pair is exact: torchvision 0.22.0 requires torch- # 2.7.0 and 0.22.1 requires torch 2.7.1. A `0.22.*` wildcard on a- # torch 2.7.0 host resolves 0.22.1, and `--no-deps` then keeps the torch- # that does not match it, rebuilding the mismatch the command repairs.+ # Exact, because the pair is: torchvision 0.22.0 needs torch 2.7.0 and 0.22.1 needs 2.7.1, so+ # a 0.22.* wildcard resolves 0.22.1 while --no-deps keeps the mismatched torch.spec = f"torchvision=={required[0]}.{required[1]}.{required[2]}"else:spec = f"torchvision=={required[0]}.{required[1]}.*"@@ -1711,8 +1648,7 @@ def torchvision_compatibility_check():except Exception:return- # Known compatibility table (ground truth, takes precedence over formula).- # See https://pytorch.org/get-started/previous-versions/+ # Ground truth, takes precedence over the formula; see pytorch.org/get-started/previous-versions/TORCH_TORCHVISION_COMPAT = {(2, 9): (0, 24),(2, 8): (0, 23),@@ -1736,9 +1672,8 @@ def torchvision_compatibility_check():if required is None:return- # Carry torch's own patch into the companion: the two move together- # (2.7.0/0.22.0, 2.7.1/0.22.1), so the repair command can name one wheel- # instead of a minor-wide range it cannot then satisfy under `--no-deps`.+ # Carry torch's own patch into the companion: they move together (2.7.0/0.22.0, 2.7.1/0.22.1),+ # so the repair names one wheel instead of a minor-wide range --no-deps cannot satisfy.if len(torch_release) >= 3:required = (required[0], required[1], torch_release[2])@@ -1752,7 +1687,6 @@ def torchvision_compatibility_check():_probe_torchvision_binary(torch_version_raw, torchvision_version_raw, required)return- # Version mismatch detectedmessage = (f"Unsloth: torch=={torch_version_raw} requires "f"torchvision>={required_tv_str}, "@@ -1766,15 +1700,15 @@ def torchvision_compatibility_check():torchvision_version_raw)- # Detect nightly/dev/alpha/beta/rc builds from the raw version string.- # These often have version mismatches that are expected.+ # Nightly/dev/alpha/beta/rc builds mismatch expectedly, so those and source builds only warn;+ # a stable mismatch fails fast to prevent runtime operator errors._pre_tags = (".dev", "a0", "b0", "rc", "alpha", "beta", "nightly")is_prerelease = any(t in torch_version_raw for t in _pre_tags) or any(t in torchvision_version_raw for t in _pre_tags)- # Only downgrade to warning for custom/source or prerelease builds.- # Stable mismatches should fail fast to prevent runtime operator errors.+ # Only downgrade to warning for custom/source or prerelease builds. Stable mismatches should fail fast to+ # prevent runtime operator errors.if is_custom or is_prerelease:reason = "custom/source build" if is_custom else "pre-release build"logger.warning(@@ -1816,8 +1750,8 @@ def _unsatisfied_transformers_requirements():except Exception:continue # Unparseable requirement line - ignore it, never guess.- # extra = "" drops optional-extra requirements and inapplicable- # python_version / sys_platform gates - packages the user is right not to have.+ # extra = "" drops optional-extra requirements and inapplicable python_version / sys_platform+ # gates: packages the user is right not to have.if requirement.marker is not None:try:if not requirement.marker.evaluate({"extra": ""}):@@ -1828,10 +1762,8 @@ def _unsatisfied_transformers_requirements():try:installed = importlib_version(requirement.name)except PackageNotFoundError:- # Absent, which `--no-deps` causes as readily as a stale version.- # transformers checks its base requirements at its own root import and- # raises PackageNotFoundError carrying the same misleading hint, so an- # absent one belongs here - floor or no floor, it is not optional.+ # Absent, which --no-deps causes as readily as a stale version: transformers checks its base+ # requirements at root import and raises PackageNotFoundError with the same misleading hint.unsatisfied.append((requirement.name, str(requirement.specifier), None))continueexcept Exception:@@ -1841,8 +1773,8 @@ def _unsatisfied_transformers_requirements():continue # Installed, and no floor it could fall below.try:- # Parse explicitly: SpecifierSet.contains() reports a non-PEP440 version- # as "not contained" rather than raising, which would be a false positive.+ # Parse explicitly: SpecifierSet.contains() reports a non-PEP440 version as "not contained"+ # rather than raising, which would be a false positive.installed_version = TrueVersion(installed)except Exception:continue # Not a PEP 440 version - we cannot judge it, so stay quiet.@@ -1876,9 +1808,8 @@ def check_transformers_dependency_versions():):returntry:- # find_spec RAISES ValueError, rather than returning None, for a transformers- # in sys.modules with `__spec__` None or unset - a stub, or one mid-teardown.- # Nothing here is worth failing `import unsloth` over.+ # find_spec RAISES ValueError rather than returning None for a transformers in sys.modules with+ # __spec__ None or unset (a stub, or one mid-teardown). Not worth failing the import over.if importlib.util.find_spec("transformers") is None:returnexcept Exception:@@ -2064,11 +1995,11 @@ def fix_triton_compiled_kernel_missing_attrs():except (ImportError, ModuleNotFoundError):return- # Only needed when the CompiledKernel class lacks num_ctas as a direct attr- # but has metadata (triton >= 3.6.0 with torch < 2.10)+ # Only needed when CompiledKernel lacks num_ctas as a direct attr but has metadata+ # (triton >= 3.6.0 with torch < 2.10)._ck_cls = triton_compiler.CompiledKernelif hasattr(_ck_cls, "num_ctas"):- return # Old triton with direct attrs -- no patch needed+ return_orig_init = _ck_cls.__init__@@ -2121,9 +2052,8 @@ def fix_dynamo_config_thread_visibility():mirrored_modules = ("torch._dynamo.config", "torch._inductor.config")- # config.patch(...) and config.load_config(...) also assign via __setattr__, but- # their writes are thread-local by design; a per-thread depth counter marks them- # so they are not mirrored into the process-global default.+ # config.patch() and config.load_config() also assign via __setattr__, but their writes are+ # thread-local by design, so a per-thread depth counter keeps them out of the global default.import threading_scoped_depth = threading.local()@@ -2141,7 +2071,7 @@ def fix_dynamo_config_thread_visibility():def _patched_patch(self, *args, **kwargs):ctx = original_patch(self, *args, **kwargs)try:- cls = type(ctx) # patch() builds a fresh ConfigPatch class each call+ cls = type(ctx)if not getattr(cls, "__unsloth_patch_wrapped__", False):_enter0, _exit0 = cls.__enter__, cls.__exit__@@ -2206,9 +2136,9 @@ def fix_dynamo_config_thread_visibility():_patched_setattr.__unsloth_patched__ = TrueConfigModule.__setattr__ = _patched_setattr- # No replay of existing overrides: unsloth installs this before it sets any- # dynamo/inductor config, so the wrapper mirrors every later assignment. Replaying- # would also bake a still-active config.patch override into the global default.+ # No replay of existing overrides: unsloth installs this before setting any dynamo/inductor+ # config, so the wrapper mirrors every later assignment, and replaying would bake a+ # still-active config.patch override into the global default.logger.info("Unsloth: Patched torch config modules so dynamo/inductor settings ""(e.g. recompile_limit) apply across threads on torch >= 2.12."@@ -2307,7 +2237,6 @@ def check_vllm_torch_sm100_compatibility():This check runs early (before vLLM import) to provide a helpful error messageinstead of a cryptic std::bad_alloc crash."""- # vLLM installed? (without importing it)if importlib.util.find_spec("vllm") is None:return@@ -2318,7 +2247,6 @@ def check_vllm_torch_sm100_compatibility():except Exception:return # Can't determine torch version, skip check- # Any SM100 (Blackwell) GPU?try:import torch@@ -2372,7 +2300,6 @@ def fix_vllm_pdl_blackwell():if importlib.util.find_spec("vllm") is None:return- # Any SM100 (Blackwell) GPU? Fix applies globally via env var + monkey-patch.try:import torch@@ -2399,13 +2326,12 @@ def fix_vllm_pdl_blackwell():except (ImportError, OSError, ModuleNotFoundError, ValueError):return False- # PDL-related modules present?has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils")has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op")has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op")if not has_utils and not has_expand_op and not has_shrink_op:- # Old vLLM version without PDL support - nothing to patch+ # Old vLLM version without PDL support: nothing to patch.return# vLLM version already includes the fix?@@ -2421,7 +2347,6 @@ def fix_vllm_pdl_blackwell():except Exception as e:logger.debug(f"Unsloth: vLLM version check failed ({e}), applying PDL workaround.")- # Apply the PDL fixos.environ["TRITON_DISABLE_PDL"] = "1"def fake_supports_pdl(*args, **kwargs):@@ -2435,8 +2360,7 @@ def fix_vllm_pdl_blackwell():patched.append(name)patched_names.add(name)- # Patch the source module (utils.py) where supports_pdl is defined. It uses- # @lru_cache, so clear the cache to avoid stale results.+ # Patch the source module (utils.py) where supports_pdl is defined, and clear its @lru_cache to avoid stale results.try:utils_module = importlib.import_module("vllm.lora.ops.triton_ops.utils")if hasattr(utils_module, "supports_pdl"):@@ -2477,7 +2401,7 @@ def fix_vllm_pdl_blackwell():f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - patched: {', '.join(patched)}")else:- # Just set the env var - vLLM might be an older version without supports_pdl+ # Just set the env var: vLLM might be an older version without supports_pdl.logger.info(f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})")@@ -2586,8 +2510,8 @@ def disable_torchcodec_if_broken():import warningswarnings.warn(mismatch_hint, stacklevel = 2)except Exception:- # Warning filters promoted to errors must not abort the disable- # fallback below (e.g. PYTHONWARNINGS=error, pytest -W error).+ # Warning filters promoted to errors (PYTHONWARNINGS=error, pytest -W error) must not abort the+ # disable fallback below.passtry:import importlib.util@@ -2597,7 +2521,7 @@ def disable_torchcodec_if_broken():# RuntimeError on dlopen failure; OSError covers chained libavutil.so misses.from torchcodec.decoders import AudioDecoderexcept (ImportError, RuntimeError, OSError):- # transformers: flip flag (<5) and/or rebind lru_cache'd func (>=5).+ # transformers: flip the flag (<5) and/or rebind the lru_cache'd func (>=5).try:import transformers.utils.import_utils as tf_import_utils@@ -2624,8 +2548,8 @@ def disable_torchcodec_if_broken():except ImportError:pass- # Drop half-loaded entries and seat the absence sentinel. After this,- # import torchcodec raises ModuleNotFoundError and find_spec returns None.+ # Drop half-loaded entries and seat the absence sentinel: after this, `import torchcodec` raises+ # ModuleNotFoundError and find_spec returns None.for _stale in [nfor n in list(sys.modules)@@ -2675,8 +2599,7 @@ def disable_torchaudio_if_cuda_mismatched():returnexcept (RuntimeError, OSError) as exc:if "different CUDA versions" not in str(exc) and "torchaudio" not in str(exc).lower():- # Some other failure. Not this function's business, and swallowing- # it would hide a real one behind a message about CUDA versions.+ # Some other failure: swallowing it would hide a real one behind a message about CUDA versions.raisetry:import warnings@@ -2697,14 +2620,10 @@ def disable_torchaudio_if_cuda_mismatched():tf_import_utils._torchaudio_available = Falseexcept AttributeError:pass- # `speech` is transformers' composite backend and it is nothing but- # torchaudio (`is_speech_available` returns `is_torchaudio_available()`).- # On 4.x both read one module global, so setting the flag above covers- # them. On 5.x each is separately `@lru_cache`d, so a `speech` answer- # computed before this repair stays True and `requires_backends(...,- # "speech")` waves callers on into a torchaudio that is now a None- # sentinel -- a raw crash instead of the unavailable-backend path this- # whole function exists to restore. Clear both caches.+ # `speech` is transformers' composite backend and is nothing but torchaudio (is_speech_available+ # returns is_torchaudio_available()). On 4.x both read one module global; on 5.x each is+ # separately lru_cached, so an answer computed before this repair stays True and+ # requires_backends(..., "speech") waves callers into a torchaudio that is now a None sentinel.for _name in ("is_torchaudio_available", "is_speech_available"):is_avail = getattr(tf_import_utils, _name, None)if is_avail is None:@@ -2753,21 +2672,19 @@ def disable_broken_wandb():try:import wandbexcept Exception:- # wandb is installed but broken - patch all checkers to skip it+ # wandb is installed but broken: patch every checker to skip it.logger.info("Unsloth: wandb is installed but broken (likely a protobuf version mismatch). ""Disabling wandb to prevent import errors. To fix, run: pip install --upgrade wandb")_wandb_false = lambda: False- # Patch transformers' is_wandb_available (used by most trl trainers)try:import transformers.integrations.integration_utils as tf_integrationtf_integration.is_wandb_available = _wandb_falseexcept (ImportError, AttributeError):pass- # Patch accelerate's is_wandb_available. Patch both the source module and- # the re-export namespace, since `from accelerate.utils import- # is_wandb_available` reads accelerate.utils, not accelerate.utils.imports.+ # Patch accelerate.utils.imports and the accelerate.utils re-export, since+ # `from accelerate.utils import is_wandb_available` reads the latter.try:import accelerate.utils.imports as acc_importsacc_imports.is_wandb_available = _wandb_false@@ -2778,25 +2695,17 @@ def disable_broken_wandb():acc_utils.is_wandb_available = _wandb_falseexcept (ImportError, AttributeError):pass- # Set env var as additional fallbackos.environ["WANDB_DISABLED"] = "true"-# ----------------------------------------------------------------------------# peft 0.19.x + transformers 4.x drift-# ----------------------------------------------------------------------------# peft 0.19.x's ``peft/utils/transformers_weight_conversion.py`` unconditionally-# imports ``transformers.conversion_mapping`` and ``transformers.core_model_loading``-# at module top. Neither submodule exists on transformers <5, so the import-# explodes with ModuleNotFoundError -- silently swallowed by the bare except-# in ``patch_peft_weight_converter_compatibility`` below. Fix: when (and only-# when) the import is broken, stub the two missing submodules with the symbols-# peft pulls at module top. The stubs are inert at runtime because peft itself-# only calls into them behind ``if is_transformers_ge_v5:`` gates.+# peft 0.19.x's transformers_weight_conversion.py imports transformers.conversion_mapping and+# transformers.core_model_loading at module top; neither exists on transformers <5, so the+# import raises ModuleNotFoundError, swallowed by the bare except below. Stub the two+# submodules only when broken; peft calls them only behind `if is_transformers_ge_v5:`.++# Stamped on stub modules so a second call is a strict no-op and third parties can introspect __unsloth_stub__.# ----------------------------------------------------------------------------# Stamped on stub modules so a second call is a strict no-op and so third-# parties can introspect ``__unsloth_stub__`` to detect our patch._UNSLOTH_STUB_SENTINEL = "__unsloth_stub__"_PEFT_TENSOR_PARALLEL_FALLBACK_SYMBOLS = ("ALL_PARALLEL_STYLES",@@ -2962,12 +2871,11 @@ def _build_transformers_conversion_mapping_stub():backfill a REAL module missing only some of them."""mod = _make_peft_stub_module("transformers.conversion_mapping")- # peft does ``.copy()`` + keyed assignment at module top; real dict suffices.+ # peft does .copy() plus keyed assignment at module top; a real dict suffices.mod._MODEL_TO_CONVERSION_PATTERN = {}def get_checkpoint_conversion_mapping(model_type, *args, **kwargs):- # ``None`` = peft's "no conversion registered"; both callsites- # early-return on it.+ # None is peft's "no conversion registered"; both callsites early-return on it.return Nonedef get_model_conversion_mapping(model, *args, **kwargs):@@ -2993,7 +2901,7 @@ def _install_transformers_conversion_mapping_stub():try:parent.conversion_mapping = modexcept Exception:- # Frozen parent: sys.modules entry is enough for ``from ... import``.+ # Frozen parent: the sys.modules entry is enough for `from ... import`.passreturn mod@@ -3095,8 +3003,8 @@ def _install_transformers_core_model_loading_stub():return mod-# Names peft's transformers_weight_conversion imports at module top level; a-# real module missing ANY of them breaks that import as hard as an absent one.+# Names peft's transformers_weight_conversion imports at module top; a real module missing ANY+# of them breaks that import as hard as an absent one._PEFT_REQUIRED_SYMBOLS = {"transformers.conversion_mapping": ("_MODEL_TO_CONVERSION_PATTERN",@@ -3153,8 +3061,8 @@ def _backfill_missing_peft_symbols(name):return tuple(added)-# An empty pattern is peft's own starting point; the rest stand in for real-# upstream behaviour, so those are worth warning about.+# An empty pattern is peft's own starting point; the rest stand in for real upstream+# behaviour, so those are worth warning about._PEFT_INERT_BACKFILL_IS_FINE = frozenset(("_MODEL_TO_CONVERSION_PATTERN",))@@ -3190,7 +3098,7 @@ def fix_peft_transformers_weight_conversion_import():if importlib.util.find_spec("peft") is None:return None- # Already importable? Either we patched, or transformers is v5+.+ # Already importable: either we patched, or transformers is v5+.try:importlib.import_module("peft.utils.transformers_weight_conversion")return False@@ -3203,7 +3111,7 @@ def fix_peft_transformers_weight_conversion_import():):return Falseexcept ImportError as exc:- # Older Python ImportError has no `.name`; string-match instead.+ # Older Python ImportError has no .name; string-match instead.msg = str(exc)if ("transformers.conversion_mapping" not in msg@@ -3229,17 +3137,13 @@ def fix_peft_transformers_weight_conversion_import():_install_transformers_core_model_loading_stub()patched_any = True- # Present but incomplete. transformers 5.x kept both modules and dropped- # names peft still imports at module top -- `cannot import name- # '_MODEL_TO_CONVERSION_PATTERN' from 'transformers.conversion_mapping'`.- # The stubs above only fire when a module is ABSENT, so that case fell- # through here and no-oped. Backfill the missing names onto the real- # module: strictly additive, so a transformers that still defines them is- # untouched, and nothing else on either side changes.+ # Present but incomplete: transformers 5.x kept both modules and dropped names peft still+ # imports at module top. The stubs above only fire when a module is ABSENT, so backfill the+ # missing names onto the real module, which is strictly additive.patched_any = _backfill_missing_conversion_symbols() or patched_any- # An importable submodule can still lack individual symbols; backfill just- # those names rather than replacing a real module wholesale.+ # An importable submodule can still lack individual symbols; backfill just those rather than+ # replacing a real module wholesale.backfilled = {}for _submodule in _PEFT_REQUIRED_SYMBOLS:added = _backfill_missing_peft_symbols(_submodule)@@ -3253,11 +3157,10 @@ def fix_peft_transformers_weight_conversion_import():)if not patched_any:- # Real submodules present and complete; failure was for another reason.+ # Real submodules present and complete; the failure was for another reason.return False- # Force a fresh import now that stubs are in place. Drop any cached- # ``None`` entry first so importlib retries.+ # Force a fresh import now that stubs are in place, dropping any cached None entry first so importlib retries.pkg = "peft.utils.transformers_weight_conversion"if pkg in sys.modules and sys.modules[pkg] is None:del sys.modules[pkg]@@ -3276,8 +3179,8 @@ def fix_peft_transformers_weight_conversion_import():return True-# What peft.utils.transformers_weight_conversion imports at module top. Kept-# beside the stubs so the two lists cannot drift apart.+# What peft.utils.transformers_weight_conversion imports at module top, kept beside the stubs+# so the two lists cannot drift apart._PEFT_CONVERSION_SYMBOLS = {"transformers.conversion_mapping": ("_MODEL_TO_CONVERSION_PATTERN",@@ -3296,23 +3199,18 @@ _PEFT_CONVERSION_SYMBOLS = {),}-# Of those, the ones peft calls rather than merely imports. The stubs are-# deliberately inert: on transformers <5 the whole module is ours and peft's-# converter never runs. Landing an inert body on a REAL transformers is a-# different matter -- peft would call it and get a wrong answer, so these are-# replaced by a placeholder that says what is wrong instead.+# Of those, the ones peft calls rather than merely imports. An inert body on a REAL+# transformers would be called and answer wrongly, so those get a placeholder that says so._PEFT_CONVERSION_RUNTIME_SYMBOLS = frozenset(("transformers.core_model_loading.dot_natural_key","transformers.core_model_loading.rename_source_key","transformers.core_model_loading.WeightRenaming","transformers.core_model_loading.WeightConverter",- # `build_peft_weight_mapping` buckets its entries with- # `isinstance(op, Concatenate)` / `isinstance(op, MergeModulelist)` and- # builds `Transpose(dim0 = 0, dim1 = 1)` outright, so an inert stub does not- # merely fail to help: the isinstance arms go quiet and the conversion is- # skipped. `ConversionOps` stays import-only -- peft subclasses it at module- # top and never asks about instances of it.+ # build_peft_weight_mapping buckets entries with isinstance(op, Concatenate) /+ # isinstance(op, MergeModulelist) and builds Transpose outright, so an inert stub silences the+ # isinstance arms and the conversion is skipped. ConversionOps stays import-only: peft+ # subclasses it and never asks about instances."transformers.core_model_loading.Concatenate","transformers.core_model_loading.MergeModulelist","transformers.core_model_loading.Transpose",@@ -3339,12 +3237,9 @@ def _unsupported_conversion_symbol(qualified, donor_value = None):"transformers that still exports it, or a peft that does not need it.")if isinstance(donor_value, type):- # `isinstance` has to raise, not answer False. peft buckets its- # conversion entries by type, and a placeholder that quietly matches- # nothing drops the operations instead of reporting that it cannot do- # the job. Subclassing still works: peft does `class PeftConcatenate- # (Concatenate)` at module top, and creating a class does not construct- # one.+ # isinstance has to raise, not answer False: peft buckets conversion entries by type, and a+ # placeholder that quietly matches nothing drops the operations instead of reporting that it+ # cannot do the job. Subclassing still works, since creating a class constructs nothing.class _RefusingMeta(type):def __instancecheck__(cls, instance):raise RuntimeError(message)@@ -3369,23 +3264,14 @@ def _unsupported_conversion_symbol(qualified, donor_value = None):return _refuse-# The model types peft actually acts on, and what it converts them as. Only two-# base patterns reach a rewrite -- `_MOE_TARGET_MODULE_MAPPING` and-# `_MOE_FUSED_TARGETS` are keyed on `mixtral` and `qwen2_moe` alone, and-# `_convert_peft_config_moe` returns early for anything else -- so this is the-# whole set of lookups that must not be answered with a silent None.-# Snapshotted from transformers `conversion_mapping._MODEL_TO_CONVERSION_PATTERN`,-# because the case this file handles is that map being gone. Names are the-# reason it is a list and not a rule: `deepseek_v3`, `dots1`, `longcat_flash`,-# `minimax`, `mellum`, `qwen3_next`, `solar_open` and `flex_olmo` are all fused-# MoE and none of them say so.+# The model types peft acts on, snapshotted from conversion_mapping._MODEL_TO_CONVERSION_PATTERN+# because the case handled here is that map being gone. A list and not a rule: deepseek_v3,+# dots1, longcat_flash, minimax, mellum, qwen3_next, solar_open and flex_olmo are all fused MoE+# and none of them say so._PEFT_MOE_CONVERSION_PATTERNS = {- # The two base patterns map to themselves, and leaving them out was not the- # harmless omission it looked like: `mixtral` says nothing about MoE, so the- # substring hint answered the default for it -- the silent None this stand-in- # exists to prevent -- and the drift test failed outright on transformers- # 5.5.0, which pyproject permits. `qwen3_5_moe` is here for 5.3.0, where it- # is a separate key; the hint would catch that one, but only by its name.+ # The two base patterns map to themselves, and omitting them was not harmless: mixtral says+ # nothing about MoE, so the substring hint answered the default and the drift test failed on+ # transformers 5.5.0. qwen3_5_moe is here for 5.3.0, where it is a separate key."mixtral": "mixtral","qwen2_moe": "qwen2_moe","qwen3_5_moe": "qwen2_moe",@@ -3415,13 +3301,10 @@ _PEFT_MOE_CONVERSION_PATTERNS = {"solar_open": "qwen2_moe",}-# MoE-named model types whose conversion family is NOT one of the two fused ones,-# so peft's `_convert_peft_config_moe` finds no `_MOE_TARGET_MODULE_MAPPING` entry-# and returns without a rewrite. The substring hint below raised for all three-# purely on the name, turning a load that works into a hard error. Every one is a-# shipping model: `qwen3_5_moe_text` converts as `qwen3_5_text`, and both Granite-# MoE variants as `granitemoe`. Checked before the hint, never instead of the-# snapshot above, so a genuinely fused type still refuses.+# MoE-named types whose conversion family is NOT one of the two fused ones, so peft's+# _convert_peft_config_moe finds no mapping entry and returns without a rewrite. The substring+# hint below raised for all three on the name alone. Checked before the hint, never instead of+# the snapshot above._PEFT_MOE_NAMED_NOT_FUSED = frozenset(("granitemoehybrid",@@ -3430,17 +3313,10 @@ _PEFT_MOE_NAMED_NOT_FUSED = frozenset())-# The other half of the same carve-out: MoE-named model types that are not in the-# conversion map AT ALL. peft's `.get()` answers None for them and skips the target-# rewrite, so a refusal here breaks an ordinary adapter load -- `qwen3_vl_moe` and-# `lfm2_moe` are both supported models that the substring hint caught. Kept apart-# from the set above so its upstream canary keeps comparing like with like: that-# one is "in the map, mapped elsewhere", this one is "not in the map".-#-# Every `*moe*` / `*mixtral*` model type absent from `_MODEL_TO_CONVERSION_PATTERN`-# in BOTH 5.3.0 and 5.5.0. A name absent from one only (`afmoe`, `qwen3_5_moe`)-# stays fused: refusing a type that was fused costs an error message, answering-# None for one is the silent mis-conversion this exists to prevent.+# The other half of the carve-out: MoE-named types not in the conversion map AT ALL, where+# peft's .get() answers None and skips the rewrite, so a refusal breaks an ordinary adapter+# load (qwen3_vl_moe, lfm2_moe). A name absent from only one of 5.3.0 / 5.5.0 stays fused:+# refusing a fused type costs a message, answering None is a silent mis-conversion._PEFT_MOE_NAMED_NOT_CONVERTED = frozenset(("ernie4_5_vl_moe",@@ -3455,9 +3331,8 @@ _PEFT_MOE_NAMED_NOT_CONVERTED = frozenset())-# How many of those pairs a candidate map has to agree with before we believe it-# is the conversion map under a new name. Three, so a coincidence does not pass-# and a version that has renamed or dropped a handful of model types still does.+# How many pairs a candidate map must match before we believe it is the conversion map under a+# new name. Three: a coincidence does not pass, a few renamed types still do._CONVERSION_MAP_MATCHES = 3@@ -3517,19 +3392,10 @@ class _UnavailableConversionPatternMap(dict):"peft that does not need the conversion.")- # Only fused-MoE lookups are unsafe to answer with a silent None. peft reaches- # `_convert_peft_config_moe` for ANY model type that has a checkpoint conversion- # mapping, not just MoE ones, and for those a None is the correct answer: the function- # returns without a MoE target rewrite, which is what it would do with the real map- # too. Raising for all of them would break ordinary adapter loads to guard a case they- # are not in.- #- # The snapshot is the list, not a rule over the name: eleven of the twenty-four fused- # MoE model types say nothing about MoE in their names (`deepseek_v3`, `dots1`,- # `longcat_flash`, `minimax`, `mellum`, `qwen3_next`, `solar_open`, `flex_olmo` among- # them), so a substring test answered the default for exactly the checkpoints this- # exists to protect. The substring hints stay on top of it, for a fused MoE model type- # added after this snapshot that does follow the naming convention.+ # Only fused-MoE lookups are unsafe to answer with a silent None: peft reaches+ # _convert_peft_config_moe for any type with a conversion mapping, and there None is what the+ # real map gives too, so raising for all would break ordinary adapter loads. The substring+ # hints sit on top of the snapshot, for a fused type added later that follows the convention._MOE_HINTS = ("moe", "mixtral")def _is_moe(self, key):@@ -3587,8 +3453,8 @@ def _backfill_conversion_symbols_once(builders, added):missing = [s for s in symbols if not hasattr(real, s)]if not missing:continue- # Build the stub off to the side rather than installing it, so the real- # module keeps its identity and everything else it exports.+ # Build the stub off to the side rather than installing it, so the real module keeps its+ # identity and everything else it exports.saved = sys.modules.pop(name, None)try:donor = builders[name]()@@ -3600,8 +3466,8 @@ def _backfill_conversion_symbols_once(builders, added):for symbol in missing:qualified = f"{name}.{symbol}"if symbol == "_MODEL_TO_CONVERSION_PATTERN":- # peft copies this and looks families up in it, so the stub's- # empty dict silently drops every alias. Recover the real one.+ # peft copies this and looks families up in it, so the stub's empty dict silently drops every+ # alias. Recover the real one.recovered = _recover_conversion_pattern_map(real)if recovered is None:logger.warning(@@ -3610,12 +3476,10 @@ def _backfill_conversion_symbols_once(builders, added):"targets for fused MoE checkpoints. Adapters for other ""architectures are unaffected.")- # An empty dict is the one shape that fails SILENTLY. peft does- # `_MODEL_TO_CONVERSION_PATTERN.copy()` at import and then- # `.get(model_type, None)`, and a None makes `_convert_peft_config_moe`- # return early, so every affected adapter loads with its legacy targets- # unconverted and no message anywhere. Every other runtime symbol here is- # backfilled as fail-on-use for exactly that reason; this one now matches.+ # An empty dict is the one shape that fails SILENTLY: peft does+ # _MODEL_TO_CONVERSION_PATTERN.copy() at import then .get(model_type, None), and a None makes+ # _convert_peft_config_moe return early, so every affected adapter loads with legacy targets+ # unconverted and no message anywhere.setattr(real,symbol,@@ -3624,9 +3488,8 @@ def _backfill_conversion_symbols_once(builders, added):added.append(qualified)continueif qualified in _PEFT_CONVERSION_RUNTIME_SYMBOLS:- # peft calls this one. The stub bodies exist to make the import- # work on transformers <5, where peft's converter never runs;- # on a real transformers it would run and answer wrongly.+ # peft calls this one. The stub bodies exist to make the import work on transformers <5, where+ # peft's converter never runs; on a real transformers it would run and answer wrongly.setattr(real,symbol,@@ -3651,15 +3514,10 @@ def _backfill_missing_conversion_symbols():"transformers.core_model_loading": _install_transformers_core_model_loading_stub,}added = []- # One pass is not enough when the two drifts coincide. conversion_mapping- # imports names from core_model_loading at its own module top, so while- # core_model_loading is still missing them, importing conversion_mapping- # raises and the pass skips it. Backfilling core_model_loading later in the- # same pass unblocks that import, but nothing comes back for it, and- # _gpu_init calls this guard once, so an installation carrying both drifts- # stayed broken. Repeat while a pass both adds a symbol and leaves a module- # unimportable; each pass adds at least one symbol, so the bound is the- # number of modules and there is no way to spin.+ # One pass is not enough when the drifts coincide: conversion_mapping imports names from+ # core_model_loading at module top, so while those are missing its import raises and the pass+ # skips it, and _gpu_init calls this guard once. Repeat while a pass adds a symbol and leaves+ # a module unimportable; each pass adds one, so the bound is the module count.for _attempt in range(len(_PEFT_CONVERSION_SYMBOLS) + 1):before = len(added)skipped = _backfill_conversion_symbols_once(builders, added)@@ -3828,8 +3686,8 @@ def _log_rocm_detection(message):@functools.lru_cache(1)def _is_rocm_torch_build() -> bool:- # Most official ROCm wheels include a local version suffix like +rocmX.Y.- # Some custom/source builds do not, so we fall back to runtime hints.+ # Most official ROCm wheels carry a +rocmX.Y local version, but some custom or source builds do+ # not, so fall back to runtime hints.try:torch_version_raw = str(importlib_version("torch")).lower()if "rocm" in torch_version_raw:@@ -3859,7 +3717,6 @@ def _is_rocm_torch_build() -> bool:def _iter_amdgpu_asic_id_table_candidates():- # Try torch-adjacent ids table paths first without importing torch.try:torch_spec = importlib.util.find_spec("torch")except Exception:@@ -3922,17 +3779,14 @@ def configure_amdgpu_asic_id_table_path():return None-# ----------------------------------------------------------------------------# bitsandbytes Windows ROCm fix: cextension.py runs get_rocm_gpu_arch()-# (bnb >= 0.47) and get_rocm_warpsize() (0.49.x) at import, shelling out to-# rocminfo / hipinfo.exe via PATH. Neither is on PATH on Windows (AMD torch-# wheels put hipInfo.exe in venv Scripts), so every import logs ERROR +-# WARNING, ROCM_GPU_ARCH becomes "unknown", and warp size defaults to 64:-# wrong on RDNA (wave 32), breaking 4-bit blocksizes and-# ALLOW_PREQUANTIZED_MODELS. Upstream fix unmerged (bitsandbytes#1969), so a-# MetaPathFinder swaps both helpers for torch-device-props-first versions-# right after bitsandbytes.cuda_specs executes, before cextension reads-# them. Must run before `import unsloth_zoo` (imports bnb on ROCm).+# bitsandbytes Windows ROCm fix: cextension.py calls get_rocm_gpu_arch() (bnb >= 0.47) and+# get_rocm_warpsize() (0.49.x) at import, shelling out to rocminfo / hipInfo.exe via PATH. Neither+# is on PATH on Windows (AMD torch wheels put hipInfo.exe in venv Scripts), so ROCM_GPU_ARCH+# becomes "unknown" and warp size defaults to 64, wrong on RDNA (wave 32) and breaking 4-bit+# blocksizes and ALLOW_PREQUANTIZED_MODELS. Upstream fix unmerged (bitsandbytes#1969), so a+# MetaPathFinder swaps both helpers right after bitsandbytes.cuda_specs executes. Must run before+# `import unsloth_zoo`.+# ---------------------------------------------------------------------------_BNB_CUDA_SPECS_MODULE = "bitsandbytes.cuda_specs"@@ -4032,7 +3886,7 @@ def _unsloth_get_rocm_gpu_arch():props = _torch_rocm_device_props()if props is not None:try:- # gcnArchName may carry feature flags, e.g. "gfx90a:sramecc+:xnack-"+ # gcnArchName may carry feature flags, e.g. "gfx90a:sramecc+:xnack-".arch = str(props.gcnArchName).split(":")[0].strip()if arch.startswith("gfx"):return arch@@ -4136,9 +3990,8 @@ class _BnbCudaSpecsPatchLoader(importlib.abc.Loader):def exec_module(self, module):self._loader.exec_module(module)- # Patch after the module body ran, before cextension calls it. The- # finder stays on sys.meta_path (same lifecycle as the blockers- # above) so importlib.reload(bitsandbytes.cuda_specs) re-patches.+ # Patch after the module body ran, before cextension calls it. The finder stays on+ # sys.meta_path so importlib.reload(bitsandbytes.cuda_specs) re-patches.try:_patch_bnb_cuda_specs_module(module)except Exception as e:@@ -4163,8 +4016,8 @@ class _BnbCudaSpecsPatchFinder(importlib.abc.MetaPathFinder):):if fullname != _BNB_CUDA_SPECS_MODULE:return None- # Delegate to remaining finders (editable installs, frozen apps)- # and wrap the loader that would actually be used.+ # Delegate to the remaining finders (editable installs, frozen apps) and wrap the loader that+ # would actually be used.spec = Nonefor finder in sys.meta_path:if finder is self or getattr(finder, _BNB_ROCM_FIX_FINDER_SENTINEL, False):@@ -4308,8 +4161,8 @@ def _is_broken_vllm_error(error) -> bool:)) or ("vllm" in message and "undefined symbol" in message):return True- # Forced extension load raises the bare loader error (no "vllm._C"- # wrapper); match any .so failure as callers feed only vLLM imports.+ # A forced extension load raises the bare loader error with no "vllm._C" wrapper, so match any+ # .so failure; callers feed only vLLM imports.if "cannot open shared object file" in message:return Truecurrent = getattr(current, "__cause__", None) or getattr(current, "__context__", None)@@ -4319,8 +4172,8 @@ def _is_broken_vllm_error(error) -> bool:_VLLM_RELEASES_URL = "https://github.com/vllm-project/vllm/releases"_VLLM_INSTALL_DOCS_URL = "https://docs.vllm.ai/en/latest/getting_started/installation/gpu/"-# A plain release version, e.g. "0.23.0". Anything else (rc / dev / post builds)-# has no matching GitHub release asset, so we never name a wheel for it.+# A plain release version, e.g. "0.23.0". Anything else (rc / dev / post) has no matching+# GitHub release asset, so never name a wheel for it._VLLM_RELEASE_VERSION_RE = re.compile(r"^[0-9]+(?:\.[0-9]+)*$")# Normalises platform.machine() onto the arch spelling used in wheel names.@@ -4337,19 +4190,13 @@ def _both_arches(manylinux_tag):return {"x86_64": manylinux_tag, "aarch64": manylinux_tag}-# vLLM ships exactly one CUDA build per release as the unsuffixed "default" wheel-# and the other under a "+cuXXX" local version tag. Which CUDA major is the default-# flipped at 0.20.0 (release notes: "CUDA 13.0 default"), the "+cuXXX" tag has been-# cu130 and then cu129, and the manylinux tag has moved manylinux1 -> 2_31 -> 2_35 ->-# 2_34 -> 2_24 -> 2_28, differing between the two wheels of the same release. So the-# asset name cannot be derived from a formula: vLLM's own docs document a-# "+cu${CUDA_VERSION}" pattern that 404s (vllm-project/vllm#37847), and no release-# has ever published a "+cu128" wheel.-#+# vLLM ships one CUDA build per release unsuffixed and the other under a +cuXXX tag, and which+# major is default flipped at 0.20.0 while the manylinux tag moved and differs between the two+# wheels of one release. The name cannot be derived: vLLM's documented "+cu${CUDA_VERSION}"+# pattern 404s (vllm-project/vllm#37847) and no release published a +cu128 wheel.# Each entry is (min_version, max_version, {cuda_major: (local_tag, {arch: manylinux})}),-# transcribed from the actual release assets. A CUDA major or arch that is absent was-# never published for that range. Extend the table when a new vLLM release lands; until-# then newer versions fall back to the release page instead of a fabricated filename.+# transcribed from the real release assets; an absent major or arch was never published for+# that range. Extend on a new release; until then newer versions fall back to the release page._VLLM_WHEEL_ASSETS = (("0.11.0",@@ -4406,9 +4253,8 @@ _VLLM_WHEEL_ASSETS = (),)-# From this release on, the default (unsuffixed) wheel is the CUDA 13 build and the-# CUDA 12 build carries "+cu129", so we can still name the right variant for a release-# newer than the table above even though its manylinux tag is unknown.+# From this release on, the default wheel is the CUDA 13 build and the CUDA 12 one carries+# +cu129, so the right variant can be named even when the manylinux tag is unknown._VLLM_CUDA13_DEFAULT_SINCE = "0.20.0"@@ -4474,9 +4320,18 @@ def _get_vllm_cuda_mismatch_message(error):# Detect what CUDA version is actually available on the systemsystem_cuda_display = None # Human-readable, e.g. "13.0"system_cuda_major = None+ # A random name is never reused, so a failed attempt would be litter rather than something the next run+ # overwrites.+ # torchao also hangs a handler off aten._grouped_mm at import time, and that operator only exists from torch+ # 2.8. Same skew, different lookup: supplying the torch.nn.functional names above does not help it.+ # Second guard for the same hazard, in case the alias is the file rather than the directory: never chain to+ # this very hook.+ # Ask the import system rather than probing for a filename, so the package (`sitecustomize/__init__.py`) and+ # .pyc forms chain too.+ # Debug mode calls find_device(...).type on gather/broadcast inputstry:import torch- cuda_version = torch.version.cuda # e.g. "13.0" or "12.8"+ cuda_version = torch.version.cudaif cuda_version:system_cuda_display = cuda_versionsystem_cuda_major = int(str(cuda_version).split(".")[0])@@ -4664,8 +4519,8 @@ def _clear_vllm_modules():sys.modules.pop(module_name, None)-# vLLM's compiled extensions. A CUDA-major ABI break hits all of them, so-# probing the eagerly-loaded _C and its siblings reliably trips it.+# vLLM's compiled extensions: a CUDA-major ABI break hits all of them, so probing the eagerly+# loaded _C and its siblings reliably trips it._VLLM_COMPILED_EXTENSIONS = ("vllm._C","vllm._C_stable_libtorch",@@ -4692,9 +4547,8 @@ def disable_broken_vllm(error = None):try:import vllm # noqa: F401- # Lazy vLLM lets a bare `import vllm` succeed even when an extension- # is ABI-broken; force-load each to surface the .so failure here.- # A missing one raises ModuleNotFoundError (skipped below).+ # Lazy vLLM lets a bare `import vllm` succeed with an ABI-broken extension; force-load each to+ # surface the .so failure here. A missing one raises ModuleNotFoundError (skipped below).for _ext in _VLLM_COMPILED_EXTENSIONS:try:importlib.import_module(_ext)@@ -4839,8 +4693,7 @@ def maybe_set_windows_rocm_bnb_version():if sys.platform != "win32":return Noneif os.environ.get("UNSLOTH_SKIP_BNB_ROCM_VERSION") == "1":- # Real opt-out: drop our seeded default (marker present); explicit- # user values carry no marker and are kept.+ # Real opt-out: drop our seeded default (marker present); explicit user values carry no marker and are kept.if os.environ.get("UNSLOTH_BNB_ROCM_VERSION_SOURCE") == "sitecustomize":os.environ.pop("BNB_ROCM_VERSION", None)os.environ.pop("UNSLOTH_BNB_ROCM_VERSION_SOURCE", None)@@ -4888,7 +4741,7 @@ def patch_accelerate_recursively_apply():if type(data).__name__ == "EmptyLogits":cls = type(data)if cls.__eq__ is object.__eq__:- # Debug mode compares gathered metadata across ranks with ==+ # Debug mode compares gathered metadata across ranks with ==.cls.__eq__ = lambda self, other: type(other).__name__ == "EmptyLogits"return datareturn original_recursively_apply(func, data, *args, **kwargs)@@ -4935,7 +4788,7 @@ def patch_accelerate_recursively_apply():device = _search(data)if device is None and found_sentinel:- # Debug mode calls find_device(...).type on gather/broadcast inputs+ # Debug mode calls find_device(...).type on gather/broadcast inputs.try:from accelerate.state import PartialStatereturn PartialState().device@@ -4954,13 +4807,11 @@ def patch_accelerate_recursively_apply():pass-# The one ImportError worth answering False to. Matching "torchao" anywhere is-# too wide: "No module named 'torchao.quantization'" and a missing-# libtorchao_ops_cuda.so also say it, and both mean genuinely broken rather than-# merely old, so they must keep propagating. Match the version complaint itself.-# peft's current wording is the first alternative; the rest are how other-# libraries phrase the same sentence, so an upstream reword does not silently-# turn this back into a raise.+# The one ImportError worth answering False to. Matching "torchao" anywhere is too wide: a+# missing submodule ("No module named 'torchao.quantization'") or libtorchao_ops_cuda.so also+# says it and both mean genuinely broken, so match the version complaint itself. peft's wording+# is first; the rest are how other libraries phrase it, so a reword does not silently turn this+# back into a raise._TORCHAO_STALE_VERSION_ERROR = re.compile(r"incompatible version of torchao"r"|torchao.{0,120}?only versions?\s+(?:above|below|>=|<=)"@@ -4999,8 +4850,7 @@ def fix_peft_stale_torchao_import_error():try:return original(*args, **kwargs)except ImportError as exc:- # Only the version complaint; any other torchao import failure is- # a real problem and must still surface.+ # Only the version complaint; any other torchao import failure is a real problem and must still surface.message = str(exc)if _TORCHAO_STALE_VERSION_ERROR.search(message) is None:raise@@ -5023,9 +4873,9 @@ def fix_peft_stale_torchao_import_error():except Exception:return False- # `from peft.import_utils import is_torchao_available` binds the original- # into each importing module, so patching import_utils alone would leave- # peft.tuners.lora.torchao, the actual caller, still raising.+ # `from peft.import_utils import is_torchao_available` binds the original into each importing+ # module, so patching import_utils alone leaves the real caller, peft.tuners.lora.torchao,+ # still raising.for mod_name, mod in tuple(sys.modules.items()):if not mod_name.startswith("peft") or mod is None:continue@@ -5037,11 +4887,9 @@ def fix_peft_stale_torchao_import_error():return patched-# Every name torchao 0.18.0 imports from torch.nn.functional, from-# grep -rh "from torch.nn.functional import" torchao/-# scaled_grouped_mm is on the path of a plain `import torchao`.-# scaled_dot_product_attention is listed for completeness only: it exists on-# every supported torch, and the loop below skips symbols torch provides.+# Every name torchao 0.18.0 imports from torch.nn.functional. scaled_grouped_mm is on the path+# of a plain `import torchao`; scaled_dot_product_attention is listed for completeness, and the+# loop below skips symbols torch provides._TORCHAO_TORCH_SYMBOLS = ("ScalingType","SwizzleType",@@ -5081,18 +4929,16 @@ def _make_torch_symbol_placeholder(name, detail):return placeholder-# The same skew one layer down: torchao 0.18 does `@implements([aten.-# _grouped_mm.default])` at module scope in float8/float8_tensor.py, and that-# op only arrived in torch 2.8, so older torch raises AttributeError on the-# lookup. The torch.nn.functional names above do not help; this goes through-# torch.ops, so the schema itself has to exist.+# The same skew one layer down: torchao 0.18 does @implements([aten._grouped_mm.default]) at+# module scope in float8/float8_tensor.py, and that op only arrived in torch 2.8, so older torch+# raises AttributeError. This goes through torch.ops, so the schema itself has to exist._ATEN_GROUPED_MM_SCHEMA = ("_grouped_mm(Tensor self, Tensor mat2, Tensor? offs=None, ""Tensor? bias=None, ScalarType? out_dtype=None) -> Tensor")-# Module level on purpose: a torch.library.Library deregisters everything it-# defined once collected, so a local would undo itself.+# Module level on purpose: a torch.library.Library deregisters everything it defined once+# collected, so a local would undo itself._aten_grouped_mm_library = None@@ -5150,8 +4996,7 @@ def _ensure_aten_grouped_mm(detail):try:import torch- # FRAGMENT adds to a namespace someone else owns; DEF would try to- # claim "aten" outright and be rejected.+ # FRAGMENT adds to a namespace someone else owns; DEF would try to claim "aten" outright and be rejected.library = torch.library.Library("aten", "FRAGMENT")library.define(_ATEN_GROUPED_MM_SCHEMA)library.impl("_grouped_mm", _refuse, "CompositeExplicitAutograd")@@ -5219,9 +5064,8 @@ def fix_torchao_torch_symbol_skew():torch_version,)- # The aten-op half of the same skew. Independent of the loop above: a torch- # can have every functional symbol and still lack the operator, or vice- # versa, so neither result gates the other.+ # The aten-op half of the same skew, independent of the loop above: a torch can have every+ # functional symbol and still lack the operator, or the reverse.op_detail = (f"torchao {torchao_version} registers a handler for it at "f"import time, but torch {torch_version} does not provide it "@@ -5239,15 +5083,10 @@ def fix_torchao_torch_symbol_skew():return bool(patched)-# vLLM inspects model architectures in a separate process, which imports-# torchao itself, never sees a parent monkey-patch, and fails with the same-# ImportError as the generic "Model architectures ['...'] failed to be-# inspected".-#-# `sitecustomize` is the one hook that reaches a process we do not launch:-# `site` imports it at interpreter startup off PYTHONPATH, which subprocesses-# inherit. A `.pth` would also work, but only inside a real site directory,-# which a library has no business writing into.+# vLLM inspects architectures in a separate process that imports torchao itself, never sees a+# parent monkey-patch, and fails the same way. sitecustomize is the one hook reaching a process+# we do not launch: site imports it at startup off the inherited PYTHONPATH. A .pth would need+# a real site directory, which a library has no business writing into._SUBPROCESS_FIX_DIRNAME = "unsloth_subprocess_import_fix"@@ -5280,8 +5119,8 @@ def _subprocess_fix_directory():raise RuntimeError("refusing a subprocess fix directory owned by another user: " + directory)- # chmod then re-read: some network and FUSE mounts ignore mode bits- # and report success without changing anything.+ # chmod then re-read: some network and FUSE mounts ignore mode bits and report success without+ # changing anything.if stat.S_IMODE(info.st_mode) & 0o022:try:os.chmod(directory, 0o700)@@ -5531,8 +5370,8 @@ def _write_hook_atomically(target, source):stream.write(payload)os.replace(tmp, target) # atomicexcept BaseException:- # A random name is never reused, so a failed attempt would be- # litter rather than something the next run overwrites.+ # A random name is never reused, so a failed attempt is litter rather than something the next+ # run overwrites.try:os.unlink(tmp)except Exception:@@ -5608,11 +5447,9 @@ def propagate_torchao_fix_to_subprocesses():try:import torch.nn.functional as F- # Both halves of the skew must be absent for there to be nothing to do.- # Today a torch missing the operator also misses the functional symbols- # (2.8 vs 2.10), so the second check never fires alone; it keeps the- # gate correct if that ever stops holding. Our own patches do not count- # as torch being new enough: the in-process fix ran first.+ # Both halves of the skew must be absent for there to be nothing to do. Today a torch missing+ # the operator also misses the functional symbols (2.8 vs 2.10), so the second check never+ # fires alone. Our own patches do not count: the in-process fix ran first.if (all(_torch_really_has(F, n) for n in _TORCHAO_TORCH_SYMBOLS)and _aten_grouped_mm_library is None@@ -5626,11 +5463,10 @@ def propagate_torchao_fix_to_subprocesses():directory = _subprocess_fix_directory()target = os.path.join(directory, "sitecustomize.py")source = _subprocess_sitecustomize_source()- # Rewrite only when it differs, so concurrent runs do not fight and a- # reader never sees a truncated file. Matching contents only count as- # evidence when the file is ours: see the helper. A directory in the- # way makes os.replace raise, which the handler below turns into "no- # subprocess fix" rather than into a hook we do not trust.+ # Rewrite only when it differs, so concurrent runs do not fight and a reader never sees a+ # truncated file. Matching contents count as evidence only when the file is ours. A directory+ # in the way makes os.replace raise, which becomes "no subprocess fix" rather than a hook we+ # do not trust.if _existing_hook_is_trustworthy(target):try:existing = open(target, "r", encoding = "utf-8").read()@@ -5648,14 +5484,12 @@ def propagate_torchao_fix_to_subprocesses():)return None- # os.pathsep, not ":" -- Windows uses ";".+ # os.pathsep, not ":": Windows uses ";".current = os.environ.get("PYTHONPATH", "")- # An empty component is an import location, not padding: it is what- # `PYTHONPATH="$PYTHONPATH:/opt/lib"` leaves behind when PYTHONPATH was- # unset, and CPython reads it as the cwd, so dropping it would take an- # import location away from every descendant. A SET-BUT-EMPTY PYTHONPATH is- # the opposite case: CPython ignores it entirely, so it must not become a- # lone "" component, which would ADD the cwd instead.+ # An empty component is an import location, not padding: PYTHONPATH="$PYTHONPATH:/opt/lib"+ # leaves one when PYTHONPATH was unset, and CPython reads it as the cwd. A SET-BUT-EMPTY+ # PYTHONPATH is the opposite: CPython ignores it, so it must not become a lone "" that ADDS+ # the cwd.parts = current.split(os.pathsep) if current else []if directory not in parts:os.environ["PYTHONPATH"] = os.pathsep.join([directory] + parts)@@ -5668,13 +5502,10 @@ def propagate_torchao_fix_to_subprocesses():return directory-# torchao 0.18.0 moved `torchao/dtypes/nf4tensor.py` under-# `quantization/quantize_/workflows/nf4/`, but torchtune (and xcodec2 through-# it) still imports the old path and dies with ModuleNotFoundError.-#-# Same shape as the vLLM tokenizer stub above: a meta path finder APPENDED-# after the real ones, so an older torchao that still ships the module wins,-# and the alias resolves lazily so `import unsloth` pays nothing.+# torchao 0.18.0 moved torchao/dtypes/nf4tensor.py under quantization/quantize_/workflows/nf4/,+# but torchtune (and xcodec2 through it) still imports the old path. Same shape as the vLLM+# tokenizer stub: a meta path finder APPENDED after the real ones, so an older torchao wins and+# the alias resolves lazily._TORCHAO_NF4_OLD = "torchao.dtypes.nf4tensor"_TORCHAO_NF4_NEW = "torchao.quantization.quantize_.workflows.nf4.nf4_tensor"_TORCHAO_NF4_SENTINEL = "__unsloth_torchao_nf4_alias__"@@ -5688,21 +5519,17 @@ class _TorchaoNF4AliasLoader(importlib.abc.Loader):self.real_spec = Nonedef create_module(self, spec):- # Return the RELOCATED module itself, not a stub with a hand-copied- # surface: torchtune then sees whatever torchao actually ships, and- # this cannot rot as symbols are added.+ # Return the RELOCATED module itself, not a stub with a hand-copied surface: torchtune then+ # sees whatever torchao ships, and this cannot rot as symbols are added.module = importlib.import_module(_TORCHAO_NF4_NEW)- # module_from_spec is about to overwrite this shared object's __spec__- # with the old-name one (_bootstrap.py assigns __spec__ unconditionally,- # unlike every other attribute), which would leave find_spec reporting- # the old name for the new module and make reload run the no-op- # exec_module below instead of the file.+ # module_from_spec is about to overwrite this shared object's __spec__ with the old-name one+ # (_bootstrap.py assigns it unconditionally), leaving find_spec reporting the old name and+ # making reload run the no-op exec_module below instead of the file.self.real_spec = getattr(module, "__spec__", None)return moduledef exec_module(self, module):- # Already imported, so nothing to execute. Put back the __spec__- # module_from_spec just clobbered.+ # Already imported, so nothing to execute. Put back the __spec__ module_from_spec just clobbered.if self.real_spec is not None:try:module.__spec__ = self.real_spec@@ -5747,29 +5574,18 @@ def fix_torchao_nf4tensor_move():sys.meta_path.append(_TorchaoNF4AliasFinder())-# `datasets` fingerprints through dill, and `dill._dill._is_builtin_module`-# pickles a module by reference only if its `__file__` starts with a sys-# prefix, ends with an extension suffix, or contains the literal-# `site-packages`. An install satisfying none of the three -- `pip install-# --target <dir>`, a PYTHONPATH overlay, a Lambda layer, a vendored tree ---# gets every package pickled BY VALUE. `Dataset.from_dict` then walks-# `datasets/utils/_dill.py:_save_arrowTable` -> `create_arrowTable` -> its-# globals -> the pyarrow MODULE, and dies on pyarrow's Cython `MonthDayNano`,-# whose `__module__` is `builtins`:-#-# PicklingError: Can't pickle <class 'MonthDayNano'>:-# it's not found as builtins.MonthDayNano-#-# Nothing in the traceback names the install layout, and every unsloth training-# path builds a dataset. Reproduced on CPU against a byte-identical package-# tree with the DIRECTORY NAME as the only variable: the plain `--target`-# directory raised, a copy named `site-packages` returned a fingerprint.+# `datasets` fingerprints through dill, and dill._dill._is_builtin_module pickles a module by+# reference only if its __file__ starts with a sys prefix, ends with an extension suffix, or+# contains the literal `site-packages`. An install matching none (pip install --target, a+# PYTHONPATH overlay, a vendored tree) is pickled BY VALUE, and Dataset.from_dict then walks+# datasets/utils/_dill.py:_save_arrowTable -> create_arrowTable -> its globals -> the pyarrow+# MODULE, dying on pyarrow's Cython MonthDayNano, whose __module__ is `builtins`. Reproduced with+# the DIRECTORY NAME as the only variable._DILL_FIX_SENTINEL = "_unsloth_dill_by_reference_fix"_DILL_FIX_ENV = "UNSLOTH_DISABLE_DILL_FIX"-# Never widened for these: dill's whole by-value contract is about the module-# the user is working IN, and `python -m pkg` gives `__main__` a real `__spec__`-# that would otherwise satisfy the rule below.+# Never widened for these: dill's by-value contract is about the module the user works IN, and+# `python -m pkg` gives __main__ a real __spec__ that would satisfy the rule below._DILL_NEVER_BY_REFERENCE = frozenset(("__main__", "__mp_main__"))@@ -5785,12 +5601,9 @@ def _dill_path_pickles_by_value(origin):real = os.path.realpath(origin)except Exception:return False- # The LITERAL path only, as dill does: it reads `'site-packages' in- # module.__file__` and resolves only for the prefix comparisons. Searching- # `real` too would answer "not affected" for a symlink into a- # site-packages-named directory dill still pickles by value, and since this- # gate decides whether the live predicate is consulted at all, the original- # PicklingError would simply stand.+ # The LITERAL path only, as dill does: it reads 'site-packages' in module.__file__ and resolves+ # only for the prefix comparisons. Searching `real` too would answer "not affected" for a+ # symlink into a site-packages-named directory dill still pickles by value.if "site-packages" in origin:return Falsefor name in ("base_prefix", "base_exec_prefix", "exec_prefix", "prefix", "real_prefix"):@@ -5835,9 +5648,9 @@ def _dill_install_root(origin):except Exception:return Noneparent = os.path.dirname(real)- # Any initializer, not just the source one: a bytecode-only deployment- # gives `pyarrow/__init__.pyc`, and matching `__init__.py` exactly would- # leave the root at `.../pyarrow`, where no sibling metadata is ever found.+ # Any initializer, not just the source: a bytecode-only deployment gives pyarrow/__init__.pyc,+ # and matching __init__.py exactly would leave the root at .../pyarrow, where no sibling+ # metadata is found.if os.path.splitext(os.path.basename(real))[0] == "__init__":parent = os.path.dirname(parent)return parent or None@@ -5880,9 +5693,8 @@ def _dill_distribution_paths(root):full = os.path.realpath(os.path.join(base, *rel.split("/")))except Exception:return- # Inside the root, and not the metadata itself. `installed-files.txt`- # entries are relative to the egg-info dir and start with `..`, so- # containment is checked after resolving, never on the raw text.+ # Inside the root, and not the metadata itself. installed-files.txt entries are relative to the+ # egg-info dir and start with `..`, so containment is checked after resolving.if not full.startswith(prefix):returnhead = full[len(prefix) :].split(os.sep, 1)[0]@@ -5895,8 +5707,7 @@ def _dill_distribution_paths(root):continuemeta = os.path.join(root, entry)listed = False- # RECORD is the one file a wheel install always leaves behind;- # installed-files.txt is its egg-info equivalent.+ # RECORD is the one file a wheel install always leaves behind; installed-files.txt is its egg-info equivalent.for record, base in (("RECORD", root), ("installed-files.txt", meta)):path = os.path.join(meta, record)if not os.path.isfile(path):@@ -5911,11 +5722,9 @@ def _dill_distribution_paths(root):breakif listed:continue- # No file list anywhere. A `top_level.txt` name is honoured only where- # it resolves to ONE file: `dill` -> `dill.py` is unambiguous, `google`- # is a directory this metadata cannot account for. Declining the- # package case costs the original loud PicklingError, not a silently- # pinned fingerprint.+ # No file list anywhere. A top_level.txt name is honoured only when it resolves to ONE file:+ # `dill` -> dill.py is unambiguous, `google` is a directory the metadata cannot account for.+ # Declining costs the original loud PicklingError, not a silently pinned fingerprint.top_level = os.path.join(meta, "top_level.txt")try:with open(top_level, encoding = "utf-8") as f:@@ -5966,9 +5775,8 @@ def _dill_module_is_importable_by_name(module, files = ()):return Falseif real in files:return True- # A deployment that compiles to adjacent bytecode and drops the sources- # leaves RECORD naming `pkg/__init__.py` while the live spec points at- # `pkg/__init__.pyc`. Same installed file, so the source answers for it.+ # A deployment that compiles to adjacent bytecode and drops the sources leaves RECORD naming+ # pkg/__init__.py while the live spec points at pkg/__init__.pyc. Same installed file.stem, ext = os.path.splitext(real)return ext in (".pyc", ".pyo") and stem + ".py" in files@@ -5992,8 +5800,7 @@ def fix_dill_module_by_value_pickling():if original is None or getattr(original, _DILL_FIX_SENTINEL, False):return False- # Ask dill itself before touching anything: the gate above is a copy of- # dill's rule and a copy can go stale.+ # Ask dill itself before touching anything: the gate above is a copy of dill's rule and a copy can go stale.probe = Noneroots = []for name in ("datasets", "pyarrow"):@@ -6017,15 +5824,10 @@ def fix_dill_module_by_value_pickling():if probe is None or not roots:return False- # Every off-prefix path entry carrying installed metadata, not just the one- # `datasets` or `pyarrow` happens to live in: a deployment can spread its- # layers, and a transform reaching a dependency in a second layer hits the- # very failure this patch prevents. Ownership is unchanged, so a wider- # search admits more DEPENDENCIES and nothing else, and a root with no- # metadata contributes nothing rather than falling back to "everything here- # is a dependency" -- a loud crash beats a stale cache. Read once, at patch- # time: re-scanning per pickled module would put directory listings inside- # every fingerprint.+ # Every off-prefix path entry carrying installed metadata, not just the one datasets or pyarrow+ # lives in: layers can be spread, and a transform reaching a dependency in a second layer hits+ # the very failure this prevents. A root with no metadata contributes nothing. Read once, at+ # patch time: re-scanning per module would put directory listings in every fingerprint.for entry in list(sys.path):if not entry or not os.path.isdir(entry):continue@@ -6059,8 +5861,8 @@ def fix_dill_module_by_value_pickling():setattr(_is_builtin_module, _DILL_FIX_SENTINEL, True)_dill_module._is_builtin_module = _is_builtin_module- # `dill.session` binds the name at import time, so patching the defining- # module alone leaves that copy on the original.+ # dill.session binds the name at import time, so patching the defining module alone leaves that+ # copy on the original.session = sys.modules.get("dill.session")if session is not None and getattr(session, "_is_builtin_module", None) is original:session._is_builtin_module = _is_builtin_modulediff --git a/unsloth/kernels/__init__.py b/unsloth/kernels/__init__.pyindex 7ff472112f6..301efd28044 100644--- a/unsloth/kernels/__init__.py+++ b/unsloth/kernels/__init__.py@@ -44,7 +44,9 @@ from .fast_lora import (apply_lora_o,fast_lora_forward,)-from .fp8 import * # Patch FbgmemFP8Linear/FP8Linear forwards before model creation, so compiled non-fast-inference models are covered too+from .fp8 import * # Patch FbgmemFP8Linear/FP8Linear forwards before model creation, so compiled non-fast-inference++# models are covered toofrom .utils import (fast_dequantize,fast_gemv,diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.pyindex 0d3bf08c356..6616e070604 100644--- a/unsloth/kernels/cross_entropy_loss.py+++ b/unsloth/kernels/cross_entropy_loss.py@@ -77,7 +77,7 @@ def _cross_entropy_forward(label_idx = tl.load(labels_ptr).to(tl.int32)logits = tl.load(logits_ptr + col_offsets, mask = mask, other = -float("inf")).to(tl.float32)- # Go logit scaling for Cohere: t * x+ # Cohere logit scaling t * x; Gemma 2 softcapping t * tanh(1/t * x).if DO_LOGIT_SCALING:logits = LOGIT_SCALE * logits# Do logit softcapping for Gemma 2: t * tanh(1/t * x)@@ -162,10 +162,10 @@ def _chunked_cross_entropy_forward(label_idx = tl.load(labels_ptr).to(tl.int32)logits = tl.load(logits_ptr + col_offsets, mask = mask, other = -float("inf")).to(tl.float32)- # Go logit scaling for Cohere: t * x+ # Cohere logit scaling t * x; Gemma 2 softcapping t * tanh(1/t * x).+ # Do logit scaling for Cohereif DO_LOGIT_SCALING:logits = LOGIT_SCALE * logits- # Do logit softcapping for Gemma 2: t * tanh(1/t * x)if DO_SOFTCAPPING:logits = SOFTCAP * triton_tanh(logits / SOFTCAP)@@ -173,14 +173,11 @@ def _chunked_cross_entropy_forward(logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))if chunk_idx == 0:- # logsumexp(chunked_logsumexp) - x- # Do the -x separately+ # logsumexp(chunked_logsumexp) - x, with the -x done separately.if label_idx != -100:x = tl.load(logits_ptr + label_idx).to(tl.float32)- # Go logit scaling for Cohere: t * xif DO_LOGIT_SCALING:x = LOGIT_SCALE * x- # Do logit softcapping for Gemma 2: t * tanh(1/t * x)if DO_SOFTCAPPING:x = SOFTCAP * triton_tanh(x / SOFTCAP)loss = -1.0 * x@@ -244,7 +241,7 @@ def _cross_entropy_backward(x = tl.load(logits_ptr + col_offsets, mask = mask, other = -float("inf")).to(tl.float32)- # Do logit scaling for Cohere+ # d/dx [s * x] = s for Cohere scaling; d/dx [t * tanh(1/t * x)] = 1 - tanh^2(1/t * x) for Gemma 2 softcapping.if DO_LOGIT_SCALING:# d/dx [s * x] = sx = x * LOGIT_SCALE@@ -272,7 +269,7 @@ def _cross_entropy_backward(# d/dx [t * tanh(1/t * x)] = 1 - tanh^2(1/t * x)y = y * (1.0 - partial * partial)- # If y == 0: dC/dx = 0 ==> we already masked it to be = 0, so dloss = 0.+ # If y == 0 then dC/dx = 0, and it is already masked to 0, so dloss = 0.tl.store(logits_ptr + col_offsets, dloss * y, mask = mask)@@ -310,7 +307,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):BLOCK_SIZE: intnum_warps: intif n_chunks == 1:- # For small vocabs <= 65336 like Llama, Mistral+ # Small vocabs <= 65336 (Llama, Mistral) versus large vocabs like Gemma 256K below.BLOCK_SIZE, num_warps = calculate_settings(vocab_size)if is_cdna():num_warps = num_warps // 2@@ -363,9 +360,8 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):LOGIT_SCALE = logit_scaling,num_warps = 32 if not is_cdna() else 16,)- # logsumexp(chunked_logsumexp) - x- # Do the -x separately- logsumexp = torch.logsumexp(logsumexp, dim = 1) # Row sum+ # logsumexp(chunked_logsumexp) - x, with the -x done separately.+ logsumexp = torch.logsumexp(logsumexp, dim = 1)losses += logsumexplosses.masked_fill_(labels == -100, 0) # Don't forget to mask padding out!@@ -455,13 +451,12 @@ if (Version(torch.__version__) < Version("2.4.0")) and not hasattr(fast_cross_entropy_loss = torch._disable_dynamo(fast_cross_entropy_loss)-# Patch CE Losses in transformersdef patch_loss_functions(torch_compile = True):_patch_loss_functions(fast_cross_entropy_loss, torch_compile = torch_compile)- # Redirect LOSS_MAPPING aliases still pointing at stock ForCausalLMLoss- # (e.g. ForConditionalGeneration for Qwen3.5, Csm...). unsloth_zoo also- # does this; remove once the floor pin passes unslothai/unsloth-zoo#656.+ # Redirect LOSS_MAPPING aliases still pointing at stock ForCausalLMLoss (e.g.+ # ForConditionalGeneration for Qwen3.5, Csm). unsloth_zoo also does this; remove once the floor+ # pin passes unslothai/unsloth-zoo#656.try:import transformers.loss.loss_utils as _lu_unsloth_loss = _lu.LOSS_MAPPING.get("ForCausalLM")diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex f5d85e80885..05692e58925 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -164,48 +164,29 @@ class LoRA_MLP(torch.autograd.Function):d_upA = torch.empty_like(upA)d_upB = torch.empty_like(upB)- # Down projection LoRA weights- # d_downA = h.t() @ (dY @ downB.t())- # d_downB = (downA.t() @ h.t()) @ dY- # d_downA *= downS- # d_downB *= downS+ # d_downA = h.t() @ (dY @ downB.t()), d_downB = (downA.t() @ h.t()) @ dY, both scaled by downS.d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)- # Up projection LoRA weights- # d_upA = X.t() @ (df @ upB.t())- # d_upB = (upA.t() @ X.t()) @ df- # d_upA *= upS- # d_upB *= upS+ # d_upA = X.t() @ (df @ upB.t()), d_upB = (upA.t() @ X.t()) @ df, both scaled by upS.d_upA.addmm_(X.t(), df @ upB.t(), alpha = upS, beta = 0)d_upB.addmm_(upA.t() @ X.t(), df, alpha = upS, beta = 0)- # Gate projection LoRA weights- # d_gateA = X.t() @ (de @ gateB.t())- # d_gateB = (gateA.t() @ X.t()) @ de- # d_gateA *= gateS- # d_gateB *= gateS+ # d_gateA = X.t() @ (de @ gateB.t()), d_gateB = (gateA.t() @ X.t()) @ de, both scaled by gateS.d_gateA.addmm_(X.t(), de @ gateB.t(), alpha = gateS, beta = 0)d_gateB.addmm_(gateA.t() @ X.t(), de, alpha = gateS, beta = 0)- # dX = matmul_lora(df, upW.t(), upW_quant, upB, upA, upS)- # dX += matmul_lora(de, gateW.t(), gateW_quant, gateB, gateA, gateS)+ # dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.upW = fast_dequantize(upW.t(), upW_quant)dX = torch.matmul(df, upW.t(), out = X if ctx.inplace else None)del upW- # dX += df @ upB.to(dtype).t() @ (upS * upA.to(dtype).t())dX.addmm_(df @ upB.t(), upA.t(), alpha = upS)gateW = fast_dequantize(gateW.t(), gateW_quant)- # dX += de @ gateW.t()dX.addmm_(de, gateW.t())del gateW- # dX += de @ gateB.to(dtype).t() @ (gateS * gateA.to(dtype).t())dX.addmm_(de @ gateB.t(), gateA.t(), alpha = gateS)- # gateW, gateW_quant, gateA, gateB, gateS,- # upW, upW_quant, upA, upB, upS,- # downW, downW_quant, downA, downB, downS,return (dX.view(batch, seq_len, hd),None,@@ -387,9 +368,8 @@ class LoRA_QKV(torch.autograd.Function):):dtype = X.dtype- # bitsandbytes 8-bit matmul expects 2D inputs.- # TorchInductor/AOTAutograd fails on 3D tensors during backward,- # so we explicitly flatten the sequence dimension.+ # bitsandbytes 8-bit matmul expects 2D inputs, and TorchInductor/AOTAutograd fails on 3D tensors+ # during backward, so flatten the sequence dimension explicitly.orig_shape = X.shapeX_for_matmul = Xif X.dim() == 3:@@ -459,8 +439,7 @@ class LoRA_QKV(torch.autograd.Function):QA, QB, KA, KB, VA, VB = QA.t(), QB.t(), KA.t(), KB.t(), VA.t(), VB.t()- ### Weight projection LoRA weights- # See our blogpost for more details.+ # Weight projection LoRA derivatives; see the Unsloth blogpost.d_QA = torch.empty_like(QA)d_QB = torch.empty_like(QB)d_KA = torch.empty_like(KA)@@ -468,57 +447,33 @@ class LoRA_QKV(torch.autograd.Function):d_VA = torch.empty_like(VA)d_VB = torch.empty_like(VB)- # Q Projection- # d_QA = X.t() @ (dQ @ QB.t())- # d_QB = (QA.t() @ X.t()) @ dQ- # d_QA *= QS- # d_QB *= QS+ # d_QA = X.t() @ (dQ @ QB.t()), d_QB = (QA.t() @ X.t()) @ dQ, both scaled by QS; K and V below are+ # identical with their own scales.d_QA.addmm_(X.t(), dQ @ QB.t(), alpha = QS, beta = 0)d_QB.addmm_(QA.t() @ X.t(), dQ, alpha = QS, beta = 0)- # K Projection- # d_KA = X.t() @ (dK @ KB.t())- # d_KB = (KA.t() @ X.t()) @ dK- # d_KA *= KS- # d_KB *= KSd_KA.addmm_(X.t(), dK @ KB.t(), alpha = KS, beta = 0)d_KB.addmm_(KA.t() @ X.t(), dK, alpha = KS, beta = 0)- # V Projection- # d_VA = X.t() @ (dV @ VB.t())- # d_VB = (VA.t() @ X.t()) @ dV- # d_VA *= VS- # d_VB *= VSd_VA.addmm_(X.t(), dV @ VB.t(), alpha = VS, beta = 0)d_VB.addmm_(VA.t() @ X.t(), dV, alpha = VS, beta = 0)- # Combine derivatives to find dX- # dQ+ # Combine the per-projection derivatives into dX.QW = fast_dequantize(QW.t(), QW_quant)dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace else None)del QW- # dX += (dQ @ QB.to(dtype).t() @ (QS * QA.to(dtype).t()))dX.addmm_(dQ @ QB.t(), QA.t(), alpha = QS)- # dKKW = fast_dequantize(KW.t(), KW_quant)- # dX += dK @ KW.t()dX.addmm_(dK, KW.t())del KW- # dX += dK @ KB.to(dtype).t() @ (KS * KA.to(dtype).t())dX.addmm_(dK @ KB.t(), KA.t(), alpha = KS)- # dVVW = fast_dequantize(VW.t(), VW_quant)- # dX += dV @ VW.t()dX.addmm_(dV, VW.t())del VW- # dX += dV @ VB.to(dtype).t() @ (VS * VA.to(dtype).t())dX.addmm_(dV @ VB.t(), VA.t(), alpha = VS)- # QW, QW_quant, QA, QB, QS,- # KW, KW_quant, KA, KB, KS,- # VW, VW_quant, VA, VB, VS,return (dX.view(batch, seq_len, hd),None,@@ -630,12 +585,7 @@ class LoRA_W(torch.autograd.Function):d_A = torch.empty_like(A)d_B = torch.empty_like(B)- ### Weight projection LoRA weights- # Weight projection- # d_A = X.t() @ (dY @ B.t())- # d_B = (A.t() @ X.t()) @ dY- # d_A *= S- # d_B *= S+ # d_A = X.t() @ (dY @ B.t()), d_B = (A.t() @ X.t()) @ dY, both scaled by S.d_A.addmm_(X.t(), dY @ B.t(), alpha = S, beta = 0)d_B.addmm_(A.t() @ X.t(), dY, alpha = S, beta = 0)@@ -643,10 +593,8 @@ class LoRA_W(torch.autograd.Function):W = fast_dequantize(W.t(), W_quant)dX = dY @ W.t()del W- # dX += dY @ B.to(dtype).t() @ (S * A.to(dtype).t())dX.addmm_(dY @ B.t(), A.t(), alpha = S)- # W, W_quant, A, B, Sreturn dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None@@ -692,8 +640,7 @@ def fast_lora_forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:passresult = self.base_layer(x, *args, **kwargs)- # Per Tim Dettmers: for 4bit, defensively clone -- backprop can fail on a- # manipulated view (may be fixed in newer PyTorch, untested).+ # Per Tim Dettmers: for 4bit, defensively clone, since backprop can fail on a manipulated view.result = result.clone()for active_adapter in self.active_adapters:diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.pyindex 0e386ac4aa9..f4d4493aa72 100644--- a/unsloth/kernels/flex_attention.py+++ b/unsloth/kernels/flex_attention.py@@ -46,25 +46,23 @@ if not HAS_FLEX_ATTENTION:n_kv_heads = self.config.num_key_value_headsn_groups = self.num_key_value_groups+ # Grouped query attention# Grouped query attentionK = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)K = K.reshape(bsz, n_heads, q_len, head_dim)V = V.reshape(bsz, n_heads, q_len, head_dim)- # See https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e- # Gemma 9b should use 256 and not 224 (hs / nah). 27b uses the below- # We default to using the config file itself- # s = self.config.hidden_size // self.config.num_attention_heads+ # Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,+ # so default to the config. See google/gemma_pytorch commit 03e6575.s = self.config.query_pre_attn_scalart = self.config.attn_logit_softcapping- Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype) # Follow Keras exactly+ Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype)A = torch.matmul(Q, K.transpose(2, 3))- A = t * torch.tanh(A / t) # Logit softcapping+ A = t * torch.tanh(A / t)A += causal_mask[:q_len, :q_len]- # Much slower in torch compile!- # A.masked_fill_(causal_mask[:q_len, :q_len], -float("inf"))+ # Much slower under torch compile than the masked_fill_ it replaces.A = torch.nn.functional.softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)A = torch.matmul(A, V)A = A.transpose(1, 2).contiguous()@@ -74,9 +72,8 @@ if not HAS_FLEX_ATTENTION:create_flex_attention_causal_mask = Nonecreate_flex_attention_sliding_window_mask = Noneelse:- # See https://github.com/pytorch-labs/attention-gym/blob/main/examples/flex_attn.ipynb- # for more examples- # BSD 3-Clause License Copyright (c) 2023, Driss Guessous, Horace He et al+ # See pytorch-labs/attention-gym examples/flex_attn.ipynb. BSD 3-Clause License Copyright (c) 2023,+ # Driss Guessous, Horace He et al.import functools, mathdef generate_tanh_softcap(t):@@ -152,20 +149,17 @@ def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)n_kv_heads = self.config.num_key_value_headsn_groups = self.num_key_value_groups- # Grouped query attentionK = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)K = K.reshape(bsz, n_heads, q_len, head_dim)V = V.reshape(bsz, n_heads, q_len, head_dim)- # See https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e- # Gemma 9b should use 256 and not 224 (hs / nah). 27b uses the below- # We default to using the config file itself- # s = self.config.hidden_size // self.config.num_attention_heads+ # Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,+ # so default to the config. See google/gemma_pytorch commit 03e6575.s = self.config.query_pre_attn_scalart = self.config.attn_logit_softcapping- Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype) # Follow Keras exactly+ Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype)A = torch_matmul(Q, K.transpose(2, 3))# Logit softcapping@@ -173,8 +167,7 @@ def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)torch_tanh(A, out = A)A *= tA += causal_mask[:q_len, :q_len]- # Much slower in torch compile!- # A.masked_fill_(causal_mask[:q_len, :q_len], -float("inf"))+ # Much slower under torch compile than the masked_fill_ it replaces.A = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)A = torch_matmul(A, V)A = A.transpose(1, 2).contiguous()diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex a55eb2b0f83..73e007ff25b 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -83,8 +83,7 @@ def weight_dequant_kernel(x_ptr, s_ptr, y_ptr, M, N, BLOCK_SIZE: tl.constexpr):n = tl.cdiv(N, BLOCK_SIZE)offs_m = pid_m * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)offs_n = pid_n * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)- # tl.arange is int32, so offs_m * N overflows for tensors with more than- # 2**31 elements (e.g. flattened MoE expert stacks); index in int64.+ # tl.arange is int32, so offs_m * N overflows past 2**31 elements (flattened MoE expert stacks); index in int64.offs = offs_m[:, None].to(tl.int64) * N + offs_n[None, :].to(tl.int64)mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)x = tl.load(x_ptr + offs, mask = mask).to(tl.float32)@@ -128,26 +127,26 @@ def weight_dequant(if x.shape[0] == s.shape[0]:y = x.to(dtype) * s.to(dtype)elif x.shape[1] == s.shape[0]:- # sometimes, this is called with the transpose of the weight. Adjust for that.+ # Sometimes called with the transpose of the weight.y = x.t().to(dtype) * s.to(dtype)y = y.t()else:raise ValueError(f"Incompatible shapes {x.shape = }, {s.shape = }")return y- # Block quantized weight: scale shape is (ceil(m/block_m), ceil(n/block_n))else:+ # Block quantized weight: scale shape is (ceil(m/block_m), ceil(n/block_n)).return weight_dequant_block(x, s, dtype = dtype)-# Copied from https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/inference/kernel.py+# Copied from huggingface.co/deepseek-ai/DeepSeek-V3 inference/kernel.py@triton.jitdef act_quant_kernel(x_ptr, y_ptr, s_ptr, BLOCK_SIZE: tl.constexpr):pid = tl.program_id(axis = 0)offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)x = tl.load(x_ptr + offs).to(tl.float32)s = tl.max(tl.abs(x)) / 448.0- # All-zero row: keep scale at 1 so LoRA's zero dY doesn't become NaN- # (a deviation from the original implementation).+ # All-zero row: keep scale at 1 so LoRA's zero dY does not become NaN (a deviation from the original+ # implementation).s = 1.0 if s == 0 else sy = x / sy = y.to(y_ptr.dtype.element_ty)@@ -170,23 +169,20 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, torreturn y, s-# Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/quantization/fp8_kernel.py+# Adapted from sgl-project/sglang python/sglang/srt/layers/quantization/fp8_kernel.py@triton.jitdef _w8a8_block_fp8_matmul(- # Pointers to inputs and outputA,B,C,As,Bs,- # Shape for matmulM,N,K,# Block size for block-wise quantizationgroup_n,group_k,- # Stride for inputs and outputstride_am,stride_ak,stride_bk,@@ -197,7 +193,6 @@ def _w8a8_block_fp8_matmul(stride_As_k,stride_Bs_k,stride_Bs_n,- # Meta-parametersBLOCK_SIZE_M: tl.constexpr,BLOCK_SIZE_N: tl.constexpr,BLOCK_SIZE_K: tl.constexpr,@@ -340,10 +335,9 @@ def torchao_block_matmul(return out.to(output_dtype)-# fbgemm <=1.3.0 silently corrupts blockwise outputs for some shapes-# (fixed upstream in 1.4.0), so never use it for block FP8.-# Preference: fbgemm (>=1.4.0) > torchao > triton (similar outputs/losses).-# torchao is ~3x faster than the triton kernel but 15-30% slower than fbgemm (H100).+# fbgemm <= 1.3.0 silently corrupts blockwise outputs for some shapes (fixed in 1.4.0), so never+# use it for block FP8. Preference: fbgemm (>=1.4.0) > torchao > triton (similar outputs/losses);+# torchao is ~3x faster than triton but 15-30% slower than fbgemm on H100.fp8_block_matmul = (torchao_block_matmul if torchao_blockwise_gemm is not None else w8a8_block_fp8_matmul_triton)@@ -355,20 +349,19 @@ def _blockwise_weight_dequant_any_shape(weight, weight_scale, block_size, out_dtm, n = weight.shapeif weight_scale.dtype not in (torch.float32, torch.float16, torch.bfloat16):weight_scale = weight_scale.to(torch.float32) # e.g. float8_e8m0fnu scales break triton+ # Per-tensor quant: expand scalar to (ceil(m/128), ceil(n/128)) block shapeif weight_scale.numel() == 1:- # Per-tensor scale: the normal forward stashes the un-expanded scalar,- # which repeat_interleave cannot grow to (m, n). Scale directly.+ # Per-tensor scale: the normal forward stashes the un-expanded scalar, which repeat_interleave+ # cannot grow to (m, n).return (weight.to(torch.float32) * weight_scale.float()).to(out_dtype)if m % block_size[0] != 0 or n % block_size[1] != 0 or block_size[0] != block_size[1]:- # Uneven tiling, or rectangular blocks. The triton kernel uses a single- # BLOCK_SIZE for both axes and derives the column scale stride from it, so- # it mis-indexes the scale when block_size[0] != block_size[1]. Expand the- # per-block scales in torch, which handles both dimensions independently.+ # Uneven tiling, or rectangular blocks: the triton kernel uses a single BLOCK_SIZE for both axes+ # and derives the column scale stride from it, so it mis-indexes when block_size[0] != [1].s_full = weight_scale.repeat_interleave(block_size[0], 0)[:m]s_full = s_full.repeat_interleave(block_size[1], 1)[:, :n]return (weight.to(torch.float32) * s_full).to(out_dtype)- # Even tiling with square blocks: block-quant dequant with the real block size- # (weight_dequant would silently default to 128 and dequantize wrongly).+ # Even tiling with square blocks: pass the real block size, since weight_dequant would silently+ # default to 128 and dequantize wrongly.return weight_dequant_block(weight, weight_scale, block_size = block_size[0], dtype = out_dtype)@@ -378,8 +371,8 @@ class FP8BlockQuantLinear(torch.autograd.Function):m, n = weight.shapeif weight_scale.dtype not in (torch.float32, torch.float16, torch.bfloat16):- # Upcast (e.g. e8m0) returns a fresh tensor and drops any Python- # attribute, so carry block_size across the cast for the lookup below.+ # Upcast (e.g. e8m0) returns a fresh tensor and drops any Python attribute, so carry block_size+ # across the cast._scale_block_size = getattr(weight_scale, "block_size", None)weight_scale = weight_scale.to(torch.float32) # e8m0 scales break triton dtype mappingif _scale_block_size is not None:@@ -388,7 +381,6 @@ class FP8BlockQuantLinear(torch.autograd.Function):# Original scale, saved for backward before any transformationoriginal_weight_scale = weight_scale- # Per-tensor quant: expand scalar to (ceil(m/128), ceil(n/128)) block shapeif weight_scale.numel() == 1:block_size = [128, 128]num_blocks_m = triton.cdiv(m, block_size[0])@@ -404,7 +396,7 @@ class FP8BlockQuantLinear(torch.autograd.Function):if triton.cdiv(m, block_size[0]) != p or triton.cdiv(n, block_size[1]) != q:if triton.cdiv(m, block_size[0]) == q and triton.cdiv(n, block_size[1]) == p:weight_scale = weight_scale.T- original_weight_scale = weight_scale # Update for transposed case+ original_weight_scale = weight_scaleelse:raise ValueError(f"Weight shape {weight.shape} and scales shape {weight_scale.shape} is not compatible with block size {block_size}"@@ -414,9 +406,8 @@ class FP8BlockQuantLinear(torch.autograd.Function):weight = weight.contiguous()if X.shape[-1] % block_size[1] != 0:- # Hidden dim not divisible by the activation block: dequant + plain matmul.- # Use the original (un-expanded) scale so a scalar per-tensor scale keeps- # the fast scalar path in both forward and backward.+ # Hidden dim not divisible by the activation block: dequant plus plain matmul, using the un-expanded+ # scale so a scalar per-tensor scale keeps the fast path in forward and backward.W_deq = _blockwise_weight_dequant_any_shape(weight, original_weight_scale, block_size, X.dtype)@@ -435,7 +426,7 @@ class FP8BlockQuantLinear(torch.autograd.Function):output_dtype = X.dtype,)ctx.weight = weight- ctx.weight_scale = original_weight_scale # Save original for backward+ ctx.weight_scale = original_weight_scalectx.block_size = block_sizereturn output.to(X.dtype)@@ -466,15 +457,13 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):if weight.shape[0] == weight_scale.shape[0] and (weight.shape[0] % 8 == 0 and weight.shape[1] % 8 == 0):- # The kernel needs weight dims divisible by 8 (else `cutlass cannot- # implement`). Padding + f8f8bf16 is slower than dequant + bf16 matmul,- # so f8f8bf16_rowwise runs only for proper, divisible-by-8 shapes.+ # The kernel needs weight dims divisible by 8 (else "cutlass cannot implement"), and padding plus+ # f8f8bf16 is slower than dequant plus bf16 matmul.- # quantize_fp8_per_row squashes leading dims; save the shape first+ # quantize_fp8_per_row squashes leading dims; save the shape first.output_shape = (*x.shape[:-1], -1)- # x_quantized/x_scale may land on a different device than x (FBGEMM- # quantize.cu#L1237). Moving them here produces gibberish; move the- # output instead. Compute runs on weight's device regardless.+ # x_quantized/x_scale may land on a different device than x (FBGEMM quantize.cu#L1237); moving them+ # here produces gibberish, so move the output instead.x_quantized, x_scale = torch.ops.fbgemm.quantize_fp8_per_row(x.view(-1, x.shape[-1]).contiguous(),scale_ub = getattr(weight, "input_scale_ub", None),@@ -497,8 +486,8 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):elif (weight.shape[0] != weight_scale.shape[0] and weight.shape[1] == weight_scale.shape[0]) or (weight.shape[0] % 8 != 0 or weight.shape[1] % 8 != 0):- # Transposed weight/scale (backward dY@W) or non-divisible-by-8 shape- # (e.g. Qwen 2.5 VL 7B gate proj 3420x1280): dequant is preferred.+ # Transposed weight/scale (backward dY@W) or a non-divisible-by-8 shape (Qwen 2.5 VL 7B gate proj+ # 3420x1280): dequant is preferred.W_deq = weight_dequant(weight, weight_scale).Toutput = torch_matmul(x, W_deq)output = output + bias if bias is not None else output@@ -553,14 +542,13 @@ class FP8_fbgemm_block_linear(torch.autograd.Function):m, n = weight.shapep, q = weight_scale.shape if weight_scale.ndim == 2 else (0, 0)fits = triton.cdiv(m, bs_n) == p and triton.cdiv(n, bs_k) == q- # numel() == 1 like fp8_linear and the dequant helper, but a (1, 1) that really- # is this weight's grid still fits, so only a true per-tensor scale falls back.+ # numel() == 1 like fp8_linear and the dequant helper, but a (1, 1) that really is this weight's+ # grid still fits, so only a true per-tensor scale falls back.per_tensor = weight_scale.numel() == 1 and not fitsif not per_tensor:fits_transposed = triton.cdiv(n, bs_n) == p and triton.cdiv(m, bs_k) == q- # Backward passes W.t() (fast_lora's downW.t()), so its block axes swap- # too; at m == n only the stride tells the two grids apart. Transpose the- # scale, not W, which X still has to matmul.+ # Backward passes W.t() (fast_lora's downW.t()), so its block axes swap too; at m == n only the+ # stride tells the two grids apart. Transpose the scale, not W, which X still has to matmul.if fits_transposed and (not fits or (bs_n != bs_k and weight.stride(0) == 1)):weight_scale = weight_scale.Tbs_n, bs_k = bs_k, bs_n@@ -569,9 +557,8 @@ class FP8_fbgemm_block_linear(torch.autograd.Function):f"Weight shape {weight.shape} and scales shape {weight_scale.shape} is not compatible with block size {bs_n, bs_k}")- # f8f8bf16_blockwise takes only 128x128x128 blocks, float32 scale grids and- # (measured on H800 / fbgemm 1.4.0) in_features % 16 == 0, out_features % 8- # == 0. Anything else raises, so dequant + matmul as FP8BlockQuantLinear does.+ # f8f8bf16_blockwise takes only 128x128x128 blocks, float32 scale grids and (measured on H800 /+ # fbgemm 1.4.0) in_features % 16 == 0, out_features % 8 == 0; anything else raises.kernel_supported = (not per_tensorand weight_scale.dtype == torch.float32@@ -632,9 +619,8 @@ def fp8_fbgemm_block_linear(def test_has_fbgemm():- # Probe whether the faster FBGEMM works on this GPU. RTX 4090/5090 and- # SM100 (Blackwell B200/B100) fail with CUTLASS SM90 kernels.- # [TODO] Investigate with TorchAO why FBGEMM fails on consumer GPUs+ # Probe whether the faster FBGEMM works on this GPU: RTX 4090/5090 and SM100 (Blackwell B200/B100)+ # fail with CUTLASS SM90 kernels.M, N, K = 128, 128, 128xq = torch.ones(M, K, dtype = torch.float8_e4m3fn, device = "cuda")wq = xq@@ -665,6 +651,7 @@ def test_has_fbgemm():if is_cutlass_cuda_error:print("Unsloth: FBGEMM on the current GPU cannot load - will switch to Triton kernels")+ # Row/channel FP8: 2D scale shaped (n, 1)else:print(f"Unsloth: FBGEMM on the current GPU cannot load with error = {e} - will switch to Triton kernels"@@ -681,11 +668,11 @@ if "UNSLOTH_HAS_FBGEMM" not in os.environ:try:import fbgemm_gpu- # >=1.4.0 is fast and accurate (older versions corrupt some shapes whatever the- # input values); ~15% faster than torchao. Probe it: consumer GPUs fail.+ # >= 1.4.0 is fast and accurate (older versions corrupt some shapes whatever the input values) and+ # ~15% faster than torchao, but consumer GPUs fail, so probe it.if Version(fbgemm_gpu.__version__) >= Version("1.4.0"):- # Suppress CUDA printf during probe: on Blackwell (SM100), FBGEMM's- # SM90 CUTLASS kernel floods stdout with "Arch conditional MMA" before aborting.+ # Suppress CUDA printf during the probe: on Blackwell (SM100) FBGEMM's SM90 CUTLASS kernel floods+ # stdout with "Arch conditional MMA" before aborting.from unsloth.import_fixes import suppress_cuda_printfwith suppress_cuda_printf():_has_fbgemm = test_has_fbgemm()@@ -706,10 +693,10 @@ def fp8_linear(weight_scale,bias = None,):- # Per-tensor (scalar scale) or block FP8 (2D scale, multiple columns)+ # Per-tensor (scalar scale) or block FP8 (2D scale, multiple columns).if weight_scale.numel() == 1 or (weight_scale.ndim == 2 and weight_scale.shape[1] > 1):out = fp8_block_quant_linear(X, weight, weight_scale)- # Row/channel FP8: 2D scale shaped (n, 1)+ # Row/channel FP8: 2D scale shaped (n, 1).else:out = fbgemm_fp8_linear(X, weight, weight_scale, bias)return out@@ -728,11 +715,10 @@ if FbgemmFp8Linear is not None:if FP8Linear is not None:FP8Linear.forward = module_forward_patch(fp8_block_quant_linear, "weight_scale_inv")-# FP8GroupedLinear's fused grouped matmul has no autograd formula, so training-# backward fails. In training, use a custom autograd Function: dequant the frozen-# fp8 weight for a differentiable bmm, saving only the fp8 weight + scale and-# unwrapping TP shards; eval keeps the fused kernel. Gate on self.training (not-# is_grad_enabled) so the grad-checkpoint no-grad forward and its recompute match.+# FP8GroupedLinear's fused grouped matmul has no autograd formula, so training backward fails:+# dequant the frozen fp8 weight for a differentiable bmm while eval keeps the fused kernel. Gate+# on self.training, not is_grad_enabled, so the grad-checkpoint no-grad forward and its recompute+# match.if FP8GroupedLinear is not None:_fp8_grouped_forward_orig = FP8GroupedLinear.forwarddiff --git a/unsloth/kernels/geglu.py b/unsloth/kernels/geglu.pyindex 0ed325afa0d..f7174c8efd7 100644--- a/unsloth/kernels/geglu.py+++ b/unsloth/kernels/geglu.py@@ -21,7 +21,7 @@ from .utils import (torch_gpu_device,)-# signed int32 max is 2**31-1 so num_elements cannot exceed 2**31+# signed int32 max is 2**31-1, so num_elements cannot exceed 2**31.NUM_INT32_ELEMENTS = 2**31SAFE_INT32_BUFFER_MULTIPLIER = 4BLOCK_SIZE = 1024@@ -40,16 +40,14 @@ def _exact_forward_kernel(offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)mask = offsets < n_elements- # f = 1/2 * e * (1 + erf(1/sqrt(2) * e))- # h = f * up+ # f = 1/2 * e * (1 + erf(1/sqrt(2) * e)), h = f * up.e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)+ g_row = tl.load(g + offsets, mask = mask, other = 0)f_row = 0.5 * e_row * (tl.math.erf(tl.math.rsqrt(2.0) * e_row) + 1.0)f_row = f_row.to(g_row.dtype) # Exact copy from HFh_row = f_row * g_row- # Store htl.store(h + offsets, h_row, mask = mask)@@ -93,12 +91,11 @@ def _exact_backward_kernel(offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)mask = offsets < n_elements- DW_row = tl.load(DW + offsets, mask = mask, other = 0) # .to(tl.float32)+ DW_row = tl.load(DW + offsets, mask = mask, other = 0)e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)+ g_row = tl.load(g + offsets, mask = mask, other = 0)- # Break e_row away for reuse- # f = 1/2 * e * (1 + erf(1/sqrt(2) * e))+ # Break e_row away for reuse f = 1/2 * e * (1 + erf(1/sqrt(2) * e))f_partial_row = 0.5 * (tl.math.erf(tl.math.rsqrt(2.0) * e_row) + 1.0)f_row = f_partial_row * e_row@@ -110,17 +107,16 @@ def _exact_backward_kernel(# dg = DW * gdg_row = DW_row * g_row- # df/de = 1/2 * (1 + erf(1/sqrt(2) * e)) + 1/sqrt(2*pi) * e * exp(-1/2 * e^2)- t = 0.3989422804014327 # 1/sqrt(2*pi)+ # df/de = 1/2 * (1 + erf(1/sqrt(2) * e)) + 1/sqrt(2*pi) * e * exp(-1/2 * e^2).+ t = 0.3989422804014327df_de = f_partial_row + t * e_row * tl.exp(-0.5 * e_row * e_row)de_row = dg_row.to(tl.float32) * df_dede_row = de_row.to(DW_row.dtype)- # Store derivatives in buffers- tl.store(DW + offsets, h_row, mask = mask) # h = f * g- tl.store(e + offsets, df_row, mask = mask) # df = DW * f- tl.store(g + offsets, de_row, mask = mask) # de+ tl.store(DW + offsets, h_row, mask = mask)+ tl.store(e + offsets, df_row, mask = mask)+ tl.store(g + offsets, de_row, mask = mask)def geglu_exact_backward_kernel(DW, e, g):@@ -151,19 +147,16 @@ def _approx_forward_kernel(offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)mask = offsets < n_elements- # f = 1/2 * e * (1 + tanh( sqrt(2/pi) * (x + 0.044715 * x^3 ) ))- # f = 1/2 * e * (1 + tanh( sqrt(2/pi) * x * (1 + 0.044715 * x^2 ) ))- # h = f * up- s = 0.7978845608028654 # math.sqrt(2 / math.pi)+ # f = 1/2 * e * (1 + tanh(sqrt(2/pi) * x * (1 + 0.044715 * x^2))), h = f * up.+ s = 0.7978845608028654e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)+ g_row = tl.load(g + offsets, mask = mask, other = 0)f_row = 0.5 * e_row * (triton_tanh(s * e_row * (1.0 + 0.044715 * e_row * e_row)) + 1.0)f_row = f_row.to(g_row.dtype) # Exact copy from HFh_row = f_row * g_row- # Store htl.store(h + offsets, h_row, mask = mask)@@ -211,19 +204,19 @@ def _approx_backward_kernel(offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)mask = offsets < n_elements- DW_row = tl.load(DW + offsets, mask = mask, other = 0) # .to(tl.float32)+ DW_row = tl.load(DW + offsets, mask = mask, other = 0)e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)+ g_row = tl.load(g + offsets, mask = mask, other = 0)- # See https://www.desmos.com/calculator/nqprfoni6x- s = 0.7978845608028654 # math.sqrt(2 / math.pi)- a = s * e_row # a = sqrt(2 / pi) * x- b = a * 0.044715 * e_row * e_row # b = a * 0.044715 * x^2+ # See desmos.com/calculator/nqprfoni6x+ s = 0.7978845608028654+ a = s * e_row+ b = a * 0.044715 * e_row * e_rowT = 1.0 + triton_tanh(a + b)T2 = 0.5 * T- # Q = 0.5 * -T * (T - 2.0) * (a + 3.0 * b)+ # Q = 0.5 * -T * (T - 2.0) * (a + 3.0 * b), where a = sqrt(2/pi) * x and b = a * 0.044715 * x^2.Q2 = -T2 * (T - 2.0) * (a + 3.0 * b)- df_de = T2 + Q2 # 1/2 * (T + Q)+ df_de = T2 + Q2# f = 1/2 * e * (1 + tanh( sqrt(2/pi) * (x + 0.044715 * x^3 ) ))f_row = T2 * e_row@@ -238,10 +231,9 @@ def _approx_backward_kernel(de_row = dg_row.to(tl.float32) * df_dede_row = de_row.to(DW_row.dtype)- # Store derivatives in buffers- tl.store(DW + offsets, h_row, mask = mask) # h = f * g- tl.store(e + offsets, df_row, mask = mask) # df = DW * f- tl.store(g + offsets, de_row, mask = mask) # de+ tl.store(DW + offsets, h_row, mask = mask)+ tl.store(e + offsets, df_row, mask = mask)+ tl.store(g + offsets, de_row, mask = mask)def geglu_approx_backward_kernel(DW, e, g):diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex 4da6c8bf9a9..e3ad0fe5f79 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -45,17 +45,16 @@ def layernorm_forward(r += row_idxmu += row_idx- # According to https://pytorch.org/torchtune/stable/_modules/torchtune/modules/layer_norm.html#Fp32LayerNorm, all modules- # are in float32!+ # Per torchtune's Fp32LayerNorm, all modules are in float32.X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)b_row = tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)mean_X = tl.sum(X_row, axis = 0) / n_cols- # (X[0] - mean) == -mean so we need to mask it out+ # (X[0] - mean) == -mean, so mask it out.XX = tl.where(mask, X_row - mean_X, 0)row_var = tl.sum(XX * XX, axis = 0) / n_cols- # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm+ # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm.eps_f32 = tl.full((), eps, tl.float32)inv_var = tl.math.rsqrt(row_var + eps_f32)tl.store(r, inv_var)@@ -78,7 +77,7 @@ def layernorm_backward(eps: tl.constexpr,BLOCK_SIZE: tl.constexpr,):- # Approximately follows https://github.com/karpathy/llm.c/blob/master/doc/layernorm/layernorm.md+ # Approximately follows karpathy/llm.c doc/layernorm/layernorm.mdrow_idx = tl.program_id(0)col_offsets = tl.arange(0, BLOCK_SIZE)mask = col_offsets < n_cols@@ -88,8 +87,7 @@ def layernorm_backward(r += row_idxmu += row_idx- # According to https://pytorch.org/torchtune/stable/_modules/torchtune/modules/layer_norm.html#Fp32LayerNorm, all modules- # are in float32!+ # Per torchtune's Fp32LayerNorm, all modules are in float32.dY_row = tl.load(dY + col_offsets, mask = mask, other = 0).to(tl.float32)X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)@@ -197,7 +195,6 @@ def test_layernorm(YY = torch.randn((bsz, seqlen, dim), dtype = dtype, device = "cuda", requires_grad = True)Y.backward(YY)correct_grad = X.grad.clone()- # from unsloth.kernels import fast_layernormY = fast_layernorm(layernorm, XX)Y.backward(YY)assert torch.dist(correct_grad, XX.grad).item() <= 0.1diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.pyindex 775c67d2ab9..150d0c8df34 100644--- a/unsloth/kernels/moe/autotune_cache.py+++ b/unsloth/kernels/moe/autotune_cache.py@@ -73,7 +73,7 @@ def load_cached_config(cache_key: str) -> Optional[Dict[str, Any]]:with open(cache_file, "r", encoding = "utf-8") as f:cached_data = json.load(f)- # Invalidate if device capability changed+ # Invalidate if device capability changed.current_device_capability = torch.cuda.get_device_capability()if cached_data.get("device_capability") != current_device_capability:logger.info("Device capability changed, invalidating cache")@@ -167,11 +167,9 @@ def get_or_autotune_moe_kernels(logger.info(f"Using in-memory cached MoE kernel configs: {cache_key}")return _kernel_config_cache[cache_key]- # Try to load from diskif not force_autotune:cached_data = load_cached_config(cache_key)if cached_data is not None:- # Reconstruct config objects from cachetry:from .grouped_gemm.kernels.tuning import (KernelConfigForward,@@ -206,7 +204,6 @@ def get_or_autotune_moe_kernels(_kernel_config_cache[cache_key] = configs_autotune_completed[cache_key] = True- # Save to diskconfig_fwd, config_bwd_dx, config_bwd_dw = configssave_cached_config(cache_key,@@ -244,7 +241,7 @@ def _run_moe_autotuning("""Run the actual auto-tuning for MoE kernels."""device = "cuda"- # Fixed token count avoids OOMs and seq_len dependency; we ignore the passed seq_len here+ # Fixed token count avoids OOMs and a seq_len dependency, so the passed seq_len is ignored.num_tokens = 4096total_tokens = num_tokens * top_k@@ -309,7 +306,6 @@ def _run_moe_autotuning(use_tma_store = triton_config_fwd.kwargs.get("USE_TMA_STORE", False),)- # Autotune backward dX kernellogger.info("Autotuning backward dX kernel...")dummy_grad = torch.randn(total_tokens, 2 * intermediate_dim, device = device, dtype = dtype)_ = grouped_gemm_dX(@@ -324,6 +320,7 @@ def _run_moe_autotuning()triton_config_bwd_dx = _autotuned_grouped_gemm_dX_kernel.best_config+ # Safe Backward Configs: 64x64x256config_bwd_dx = KernelConfigBackward_dX(BLOCK_SIZE_M = triton_config_bwd_dx.kwargs["BLOCK_SIZE_M"],BLOCK_SIZE_N = triton_config_bwd_dx.kwargs["BLOCK_SIZE_N"],@@ -335,7 +332,6 @@ def _run_moe_autotuning(use_tma_store = triton_config_bwd_dx.kwargs.get("USE_TMA_STORE", False),)- # Autotune backward dW kernellogger.info("Autotuning backward dW kernel...")_ = grouped_gemm_dW(X = hidden_states,@@ -376,7 +372,7 @@ def _get_heuristic_configs() -> Tuple[Any, Any, Any]:KernelConfigBackward_dW,)- # Safe Forward Config: 64x128x128 (Fits A100 SMEM)+ # Safe forward config 64x128x128 fits A100 SMEM; the backward pair below is 64x64x256.config_fwd = KernelConfigForward(BLOCK_SIZE_M = 64,BLOCK_SIZE_N = 128,@@ -390,7 +386,6 @@ def _get_heuristic_configs() -> Tuple[Any, Any, Any]:use_tma_store = False,)- # Safe Backward Configs: 64x64x256config_bwd_dx = KernelConfigBackward_dX(BLOCK_SIZE_M = 64,BLOCK_SIZE_N = 64,diff --git a/unsloth/kernels/moe/benchmark/benchmark_fused_moe.py b/unsloth/kernels/moe/benchmark/benchmark_fused_moe.pyindex 5c22800bdbf..dd68e7ffc50 100644--- a/unsloth/kernels/moe/benchmark/benchmark_fused_moe.py+++ b/unsloth/kernels/moe/benchmark/benchmark_fused_moe.py@@ -59,7 +59,6 @@ def run_benchmark_forward(X = torch.randn(bs, seqlen, hidden_size, dtype = dtype, device = device, requires_grad = True)- # Forwardbench_forward_ref = lambda: ref_model(X) # noqa: E731bench_forward_fused = lambda: tt_model(X) # noqa: E731@@ -100,7 +99,7 @@ def run_benchmark_backward(output, _ = ref_model(X)- # Prevent autotuning forward pass+ # Prevent autotuning the forward pass.from grouped_gemm.kernels.forward import _autotuned_grouped_gemm_forward_kernel_autotuned_grouped_gemm_forward_kernel.configs = _autotuned_grouped_gemm_forward_kernel.configs[@@ -108,7 +107,6 @@ def run_benchmark_backward(]test_output, _ = tt_model(X_test)- # Benchgrad_output = torch.randn_like(output)bench_backward_ref = lambda: output.backward(grad_output, retain_graph = True) # noqa: E731bench_backward_fused = lambda: test_output.backward(grad_output, retain_graph = True) # noqa: E731@@ -140,7 +138,7 @@ def setup_model(if isinstance(config, Qwen3MoeConfig):ref_model = Qwen3MoeSparseMoeBlock(config).to(device, dtype)- # Triton kernel grouped gemm version of MoE Block -- this is what we're testing+ # Triton kernel grouped gemm version of the MoE block, the thing under test.tt_model = Qwen3MoeFusedGroupedGEMMBlock.from_hf(ref_model,permute_x = permute_x,@@ -302,7 +300,6 @@ if __name__ == "__main__":mode = args.modeif args.autotune:- # logging.basicConfig(level=logging.INFO)print(f"Benchmarking {model_id} {mode}: seqlen={args.seqlen}, dtype={args.dtype}, permute_x={args.permute_x}, permute_y={args.permute_y}, autotune")@@ -321,9 +318,8 @@ if __name__ == "__main__":end_time = time.time()print(f"Total time: {end_time - start_time:.4f} seconds")- # NOTE: prefer the autotuner for now. The MoE block needs 2 forward- # configs (gate_up_proj, down_proj) and 4 backward (dW + dX each); this- # benchmark only supports 1 config at a time, so it's suboptimal here.+ # Prefer the autotuner: the MoE block needs 2 forward configs (gate_up_proj, down_proj) and 4+ # backward (dW + dX each), and this benchmark supports only 1 config at a time.else:assert False, "Use autotune for now"kernel_configs = create_kernel_configs(args, args.permute_x, args.permute_y)diff --git a/unsloth/kernels/moe/benchmark/utils.py b/unsloth/kernels/moe/benchmark/utils.pyindex 7f119a7e6f4..13514a6873a 100644--- a/unsloth/kernels/moe/benchmark/utils.py+++ b/unsloth/kernels/moe/benchmark/utils.py@@ -32,7 +32,7 @@ def create_merged_results(test_config_cols = list(test_config_dict.keys())for col in test_config_cols:df[col] = test_config_dict[col]- # Reorder columns so that test config cols are first+ # Reorder columns so the test config columns come first.df = df[test_config_cols + kernel_result_cols]return dfdiff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.pyindex 3ec5b80e2b5..a4b789594c7 100644--- a/unsloth/kernels/moe/grouped_gemm/interface.py+++ b/unsloth/kernels/moe/grouped_gemm/interface.py@@ -33,17 +33,15 @@ ch.setFormatter(formatter)logger.addHandler(ch)-# Precompute TMA support to avoid graph breaks-# TMA requires both:-# 1. NVIDIA GPU with capability >= 9 (Hopper+)-# 2. Triton version with TMA API (make_tensor_descriptor or _experimental_make_tensor_descriptor)+# Precompute TMA support to avoid graph breaks: it needs an NVIDIA GPU with capability >= 9+# (Hopper+) and a triton with the TMA API (make_tensor_descriptor).def _check_tma_support():if DEVICE_TYPE in ("xpu", "hip"):return Falseimport triton.language as tlgpu_supports_tma = torch.cuda.get_device_capability()[0] >= 9- # Support both old experimental and new stable API names+ # Support both the old experimental and the new stable API names.triton_has_tma_api = hasattr(tl, "make_tensor_descriptor") or hasattr(tl, "_experimental_make_tensor_descriptor")@@ -52,7 +50,7 @@ def _check_tma_support():_SUPPORTS_TMA = _check_tma_support()-# Check if triton.set_allocator is available (Triton 3.0+)+# triton.set_allocator is Triton 3.0+._HAS_SET_ALLOCATOR = hasattr(triton, "set_allocator")@@ -124,7 +122,7 @@ def grouped_gemm_forward(fuse_mul_post: bool = False,# Autotuning -- overrides manual kernel params when Trueautotune: bool = False,- # Kernel tuning params (must be tuned, else poor performance)+ # Kernel tuning params: must be tuned, else poor performance.BLOCK_SIZE_M: int = 32,BLOCK_SIZE_N: int = 32,BLOCK_SIZE_K: int = 32,@@ -133,9 +131,8 @@ def grouped_gemm_forward(use_tma_load_w: bool = False,use_tma_load_x: bool = False,use_tma_store: bool = False,- # software pipelining; no effect until loop is re-written+ # Software pipelining; no effect until the loop is re-written.flatten: bool = True,- # debuggingdebug: bool = False,) -> torch.Tensor:"""@@ -175,7 +172,7 @@ def grouped_gemm_forward(assert not (permute_y and use_tma_store), "Cannot use both TMA store and permute_y"if use_tma_load_x:- # TMA load for activations, TMA gather only supported on Blackwell++ # TMA load for activations; TMA gather is Blackwell+ only.assert not permute_x, "Cannot use both use_tma_load_x and permute_x"use_tma = use_tma_load_w or use_tma_load_x or use_tma_store@@ -186,7 +183,7 @@ def grouped_gemm_forward(use_tma_store = Falseif use_tma or autotune:- # Respect global persistent allocator if set+ # Respect the global persistent allocator if set.if _HAS_SET_ALLOCATOR and not getattr(triton, "_unsloth_allocator_set", False):def alloc_fn(size: int, alignment: int, stream: int):@@ -197,7 +194,6 @@ def grouped_gemm_forward(if W.ndim == 3:num_experts = W.shape[0]N = W.shape[1]- # K = W.shape[2]else:num_experts = m_sizes.shape[0]N = W.shape[0] // num_experts@@ -244,8 +240,6 @@ def grouped_gemm_forward(print(f"DEBUG::GROUPED_GEMM {topk_weights.tolist()} {gather_indices.tolist()}")y = torch.empty((total_tokens, N), device = X.device, dtype = X.dtype)- # if total_tokens == 0 or N == 0:- # return yNUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count@@ -253,8 +247,6 @@ def grouped_gemm_forward(return (NUM_SMS,)if not autotune:- # BLOCK_SIZE_K = min(K, BLOCK_SIZE_K)- # BLOCK_SIZE_N = min(N, BLOCK_SIZE_N)passif debug:@@ -264,22 +256,18 @@ def grouped_gemm_forward(print(f"DEBUG::GROUPED_GEMM {m_sizes.tolist()} {(gather_indices // topk).tolist()}")kernel_args = {- # Inputs"x_ptr": X,"w_ptr": W,"m_sizes_ptr": m_sizes,"gather_indices_ptr": gather_indices,"topk_weights_ptr": topk_weights,- # Output"y_ptr": y,- # Problem shapes"NUM_TOKENS": num_tokens,"NUM_EXPERTS": num_experts,"TOPK": topk,"N": N,"K": K,"NUM_SMS": NUM_SMS,- # Gather / Scatter"PERMUTE_X": permute_x,"PERMUTE_Y": permute_y,# TopK weight merging@@ -364,10 +352,10 @@ def grouped_gemm_dX(assert m_sizes.is_contiguous()assert m_sizes.ndim == 1+ # Preconditions# Preconditionsassert not (permute_x and permute_y), "Cannot permute both X and Y"- # Note that this is flipped from the forward pass- # If we permuted y in the forward, we need to permute on load in the backward+ # Flipped from the forward pass: a y permuted in the forward must be permuted on load in the backward.assert not (permute_y and use_tma_load_dy), "Cannot use both TMA load and permute_y"assert not (permute_x and use_tma_store), "Cannot use both TMA store and permute_x"@@ -379,11 +367,10 @@ def grouped_gemm_dX(use_tma_store = Falseif use_tma or autotune:- # Respect global persistent allocator if set+ # Respect the global persistent allocator if set.if _HAS_SET_ALLOCATOR and not getattr(triton, "_unsloth_allocator_set", False):def alloc_fn(size: int, alignment: int, stream: int):- # print(f"DEBUG::GROUPED_GEMM alloc_fn {size=} {alignment=} {stream=}")return torch.empty(size, device = "cuda", dtype = torch.int8)triton.set_allocator(alloc_fn)@@ -400,13 +387,12 @@ def grouped_gemm_dX(M_total, N_grad = dY.shapeN_total, K = W.shape- # N = N_total // num_expertsassert N_grad == N, f"Grad_output N ({N_grad}) must match weight N ({N})"assert M_total % topk == 0, f"M_total ({M_total}) must be divisible by topk ({topk})"num_tokens = M_total // topk- # the kernel only reads gather_indices under permute_x / permute_y, so it stays optional otherwise+ # The kernel only reads gather_indices under permute_x / permute_y, so it stays optional otherwise.if permute_x or permute_y:assert (gather_indices is not None@@ -414,19 +400,17 @@ def grouped_gemm_dX(total_tokens = gather_indices.shape[0] if gather_indices is not None else M_totalassert total_tokens == M_total, f"Total tokens ({total_tokens}) must match M_total ({M_total})"- # Note that the output shape is [NUM_TOKENS * TOPK, K] even when `permute_x` is True since we need to accumulate gradients across all experts chosen by the token.- # This will be done in a post-processing step reduction step.+ # The output shape stays [NUM_TOKENS * TOPK, K] even under permute_x, since gradients must+ # accumulate across all experts a token chose; reduced in a post-processing step.output_shape = (total_tokens, K)dX = torch.zeros(output_shape, device = dY.device, dtype = dY.dtype)- NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count # if not debug else 1+ NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_countdef grid(META):return (NUM_SMS,)if not autotune:- # BLOCK_SIZE_N = min(N_grad, BLOCK_SIZE_N)- # BLOCK_SIZE_K = min(K, BLOCK_SIZE_K)passif debug:@@ -436,23 +420,20 @@ def grouped_gemm_dX(print(f"DEBUG::GROUPED_GEMM {m_sizes.tolist()}")kernel_args = {- # Inputs"dY_ptr": dY,"w_ptr": W,"gather_indices_ptr": gather_indices,"m_sizes_ptr": m_sizes,- # Output"dX_ptr": dX,- # Problem sizes"NUM_EXPERTS": num_experts,"NUM_TOKENS": num_tokens,"TOPK": topk,"N": N,"K": K,"NUM_SMS": NUM_SMS,- # Gather / Scatter"PERMUTE_X": permute_x,"PERMUTE_Y": permute_y,+ # Loop pipelining"FLATTEN": flatten,}if not autotune:@@ -529,7 +510,6 @@ def grouped_gemm_dW(dY = dY.contiguous()m_sizes = m_sizes.contiguous()- # Preconditionsassert not (permute_x and permute_y), "Cannot permute both X and Y"assert not (permute_y and use_tma_load_dy), "Cannot use both TMA load and permute_y"assert not (permute_x and use_tma_load_x), "Cannot use both TMA load and permute_x"@@ -542,7 +522,7 @@ def grouped_gemm_dW(use_tma_store = Falseif use_tma or autotune:- # Respect global persistent allocator if set+ # Respect the global persistent allocator if set.if _HAS_SET_ALLOCATOR and not getattr(triton, "_unsloth_allocator_set", False):def alloc_fn(size: int, alignment: int, stream: int):@@ -566,7 +546,6 @@ def grouped_gemm_dW(num_tokens = total_tokens // topknum_experts = m_sizes.shape[0]- # Get dimensions_, K = X.shapeM_grad, N = dY.shape@@ -575,8 +554,6 @@ def grouped_gemm_dW(dW = torch.zeros((num_experts, N, K), device = X.device, dtype = X.dtype)if not autotune:- # BLOCK_SIZE_N = min(N, BLOCK_SIZE_N)- # BLOCK_SIZE_K = min(K, BLOCK_SIZE_K)passdef grid(META):@@ -603,24 +580,19 @@ def grouped_gemm_dW(m_start += m_sizes[i]kernel_args = {- # Inputs"x_ptr": X,"dY_ptr": dY,"m_sizes_ptr": m_sizes,"gather_indices_ptr": gather_indices,- # Output"dW_ptr": dW,- # Problem sizes"NUM_TOKENS": num_tokens,"TOPK": topk,"NUM_EXPERTS": num_experts,"N": N,"K": K,"NUM_SMS": NUM_SMS,- # Gather / Scatter"PERMUTE_X": permute_x,"PERMUTE_Y": permute_y,- # Loop pipelining"FLATTEN": flatten,}@@ -683,7 +655,7 @@ class GroupedGemm(torch.autograd.Function):ctx.dX_only = dX_onlyctx.dW_only = dW_only- # NOTE: we don't save topk_weights for backward since we do not support training with fused_mul+ # topk_weights is not saved for backward: training with fused_mul is unsupported.ctx.save_for_backward(X, W, m_sizes, gather_indices)fwd_config = {}@@ -707,7 +679,7 @@ class GroupedGemm(torch.autograd.Function):permute_x = permute_x,permute_y = permute_y,fuse_mul_post = fuse_mul_post,- # Autotune -- this will override the manual kernel config if true+ # Autotune overrides the manual kernel config when true.autotune = autotune,# Manual kernel config**fwd_config,@@ -760,7 +732,7 @@ class GroupedGemm(torch.autograd.Function):topk = topk,permute_x = permute_x,permute_y = permute_y,- # Autotune -- this will override the manual kernel config if true+ # Autotune overrides the manual kernel config when true.autotune = autotune,# Manual kernel config**bwd_dW_config,@@ -788,7 +760,7 @@ class GroupedGemm(torch.autograd.Function):topk = topk,permute_x = permute_x,permute_y = permute_y,- # Autotune -- this will override the manual kernel config if true+ # Autotune overrides the manual kernel config when true.autotune = autotune,# Manual kernel config**bwd_dX_config,@@ -802,19 +774,19 @@ class GroupedGemm(torch.autograd.Function):return (dX,dW,- None, # m_sizes- None, # gather_indices- None, # topk- None, # permute_x- None, # permute_y- None, # topk_weights- None, # fuse_mul_post- None, # kernel_config_fwd- None, # kernel_config_bwd_dX- None, # kernel_config_bwd_dW- None, # autotune- None, # dX_only- None, # dW_only+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,)@@ -895,7 +867,6 @@ def grouped_gemm(kernel_config_bwd_dW: KernelConfigBackward_dW = None,autotune: bool = False,is_first_gemm: bool = True,- # Only for debuggingdX_only: bool = False,dW_only: bool = False,):diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py b/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.pyindex bea59a4548d..d6e04ca7688 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py@@ -53,24 +53,23 @@ def _triton_supports_tma():"""Check if current Triton version supports TMA API."""import triton.language as tl- # Check for both old experimental and new stable API names+ # Both the old experimental and the new stable API names.return hasattr(tl, "make_tensor_descriptor") or hasattr(tl, "_experimental_make_tensor_descriptor")-# TMA disabled for now: incompatible with permute_x/permute_y in the MoE-# grouped GEMM passes. Re-enable once resolved.-_TRITON_HAS_TMA = False # _triton_supports_tma()+# TMA disabled for now: incompatible with permute_x/permute_y in the MoE grouped GEMM passes.+_TRITON_HAS_TMA = Falsedef get_forward_configs(BLOCK_M = DEFAULT_M_BLOCK_SIZES,BLOCK_N = DEFAULT_N_BLOCK_SIZES,BLOCK_K = DEFAULT_K_BLOCK_SIZES,- TMA_LOAD_X = None, # Auto-detect if not specified- TMA_LOAD_W = None, # Auto-detect if not specified- TMA_STORE = False, # NOTE: TMA_STORE is disabled for now+ TMA_LOAD_X = None,+ TMA_LOAD_W = None,+ TMA_STORE = False,num_warps = DEFAULT_NUM_WARPS,num_stages = DEFAULT_NUM_STAGES,num_ctas = DEFAULT_NUM_CTAS,@@ -149,14 +148,13 @@ def get_dX_kernel_configs(BLOCK_M = DEFAULT_M_BLOCK_SIZES,BLOCK_N = DEFAULT_N_BLOCK_SIZES,BLOCK_K = DEFAULT_K_BLOCK_SIZES,- TMA_LOAD_dY = None, # Auto-detect if not specified- TMA_LOAD_W = None, # Auto-detect if not specified- TMA_STORE = False, # NOTE: TMA_STORE is disabled for now+ TMA_LOAD_dY = None,+ TMA_LOAD_W = None,+ TMA_STORE = False,num_warps = DEFAULT_NUM_WARPS,num_stages = DEFAULT_NUM_STAGES,num_ctas = DEFAULT_NUM_CTAS,):- # Auto-detect TMA supportif TMA_LOAD_dY is None:TMA_LOAD_dY = _TRITON_HAS_TMAif TMA_LOAD_W is None:@@ -232,11 +230,10 @@ def get_dW_kernel_configs(num_warps = DEFAULT_NUM_WARPS,num_stages = DEFAULT_NUM_STAGES,num_ctas = DEFAULT_NUM_CTAS,- TMA_LOAD_dY = None, # Auto-detect if not specified- TMA_LOAD_X = None, # Auto-detect if not specified+ TMA_LOAD_dY = None,+ TMA_LOAD_X = None,TMA_STORE = False,):- # Auto-detect TMA supportif TMA_LOAD_dY is None:TMA_LOAD_dY = _TRITON_HAS_TMAif TMA_LOAD_X is None:@@ -344,7 +341,6 @@ def common_prune_criteria(config: triton.Config, kwargs: dict, dtype):permute_y = kwargs["PERMUTE_Y"]tokens_per_expert = num_tokens // num_experts- # use_tma = [k for k in config.kwargs.keys() if k.startswith("USE_TMA_")]MIN_BLOCK_SIZE_M = DEFAULT_M_BLOCK_SIZES[0]if exceeds_smem_capacity(num_stages, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, dtype, smem_size@@ -354,8 +350,6 @@ def common_prune_criteria(config: triton.Config, kwargs: dict, dtype):return Trueif permute_x and permute_y:return True- # if not supports_tma() and any(use_tma):- # return Truereturn False@@ -382,7 +376,7 @@ def prune_kernel_configs_fwd(configs: list[triton.Config], args, **kwargs):if common_prune_criteria(config, kwargs, dtype):continueif config.kwargs["USE_TMA_LOAD_X"] and kwargs["PERMUTE_X"]:- # Dynamically disable TMA_LOAD_X for permuted X+ # Dynamically disable TMA_LOAD_X for permuted X, and TMA_LOAD_dY for permuted Y.config.kwargs["USE_TMA_LOAD_X"] = Falseif config.kwargs["USE_TMA_STORE"] and kwargs["PERMUTE_Y"]:continuediff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.pyindex b78ea58bd04..d6a9faebea2 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py@@ -46,9 +46,9 @@ TODO:@triton.jitdef _grouped_gemm_dX_kernel(- dY_ptr, # [M_total, N]- w_ptr, # [E, N, K]- dX_ptr, # [M_total, K]+ dY_ptr,+ w_ptr,+ dX_ptr,gather_indices_ptr,m_sizes_ptr,# problem sizes@@ -73,14 +73,12 @@ def _grouped_gemm_dX_kernel(output_dtype = dX_ptr.dtype.element_tytidx = tl.program_id(0)- # This removes the need for predication along N in the GEMM main loop+ # Removes the need for predication along N in the GEMM main loop.tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")- # Create TMA descriptors for loading sorted tokens- # When using TMA load, we don't permute_x, so shape should be [TOTAL_TOKENS, K]- # Also, we are defining a single global descriptor with single block shape- # Need to check that this does not result in errors when crossing expert boundaries+ # A single global TMA descriptor with one block shape; TMA load never permutes x, so the shape is+ # [TOTAL_TOKENS, K]. Unverified across expert boundaries.if USE_TMA_LOAD_dY:dY_desc = tl.make_tensor_descriptor(dY_ptr,@@ -110,16 +108,14 @@ def _grouped_gemm_dX_kernel(m_end = m_start + m_sizeif m_size > 0:- # Advance n offset to the weights for that respective expertn_start = expert_idx * N- # N_start_offset = g.to(tl.int64) * N- # tiles for this group's GEMM+ # N_start_offset = g.to(tl.int64) * N tiles for this group's GEMMnum_m_tiles = tl.cdiv(m_size, BLOCK_SIZE_M)num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)num_tiles_per_expert = num_m_tiles * num_k_tilesif USE_TMA_STORE:- # Need to define descript within loop to predicate store along M+ # Define the descriptor inside the loop to predicate the store along M.tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")dX_desc = tl.make_tensor_descriptor(dX_ptr,@@ -128,8 +124,8 @@ def _grouped_gemm_dX_kernel(block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],)- # Bounds are relative to tiles processed so far, so we only handle- # this expert's tiles and never exceed the total across experts.+ # Bounds are relative to tiles processed so far, so only this expert's tiles are handled and the+ # total across experts is never exceeded.while tidx >= processed_tiles and tidx < (processed_tiles + num_tiles_per_expert):group_index = tidx - processed_tiles@@ -140,7 +136,6 @@ def _grouped_gemm_dX_kernel(if PERMUTE_X or PERMUTE_Y:# These will be used for loading and storing in permuted ordergather_offsets = tile_m_idx * BLOCK_SIZE_M + m_block_range- # indices_to_gather = m_start + gather_offsetsindices_to_gather = m_start + tl.max_contiguous(tl.multiple_of(gather_offsets % m_size, BLOCK_SIZE_M),BLOCK_SIZE_M,@@ -155,35 +150,27 @@ def _grouped_gemm_dX_kernel(row_mask = gather_offsets < m_sizerow_mask = row_mask[:, None]- # We only take into account the following two cases: (PERMUTE_X and NOT PERMUTE_Y) and (NOT PERMUTE_X and PERMUTE_Y)- # Hence, we can make the following simplifying assumptions when loading and storing- # Note the different strides between the two cases: the offsets for loading and storing are flipped and the strides must also be adjusted+ # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so load/store offsets+ # are simply flipped between the two cases, with the strides adjusted to match.if PERMUTE_X:- # Case where we permuted on load in the forward pass (typically first grouped GEMM in MoE MLP)- load_a_idx = (- indices_to_gather[:, None] * N- ) # Load in contiguous (expert grouped) order- store_idx = (- expert_token_offsets * K- ) # Permute on store from expert -> token order+ # Permuted on load in the forward pass (typically the first grouped GEMM in the MoE+ # MLP), so load+ # contiguous and permute on store.+ load_a_idx = indices_to_gather[:, None] * N+ store_idx = expert_token_offsets * Kelse:- # Case where we permuted on store in the forward pass (typically second grouped GEMM in MoE MLP)- load_a_idx = (- expert_token_offsets * N- ) # Permute on load from token -> expert order- store_idx = indices_to_gather[:, None] * K # Store in contiguous order+ # Permuted on store in the forward pass (typically the second grouped GEMM), so permute+ # on load and+ # store contiguous.+ load_a_idx = expert_token_offsets * N+ store_idx = indices_to_gather[:, None] * Kelse:- # # Position in full matrix - needed for TMA- # m_offset = (M_start + (tile_m_idx * BLOCK_SIZE_M)).to(tl.int32)- # k_offset = (tile_k_idx * BLOCK_SIZE_K).to(tl.int32)- # Offsets *relative* to the *current* expert -- m_start will then advance to this expert's start token+ # Offsets relative to the CURRENT expert; m_start then advances to this expert's start token.offs_am = tile_m_idx * BLOCK_SIZE_M + m_block_range- # [M, N] @ [N, K] -> [M, K] => Stride for A is N, stride for B is K- # We need two additional offsets:- # 1. For A, m_start to advance to this expert's start token- # 2. For B, n_start to advance to this expert's weights since we are passing in an [E, N, K] weight matrix+ # [M, N] @ [N, K] -> [M, K], so A strides by N and B by K, plus m_start for A's expert start token+ # and n_start for B's slice of the [E, N, K] weight matrix.row_offsets_a = m_start + offs_am[:, None]load_a_idx = row_offsets_a * Nstore_idx = row_offsets_a * K@@ -195,13 +182,9 @@ def _grouped_gemm_dX_kernel(offs_bk = tile_k_idx * BLOCK_SIZE_K + k_block_rangeif not USE_TMA_LOAD_W:row_offsets_b = n_start + n_block_range- # offs_bn = n_start + n_block_range- # row_offsets_b = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N)w_ptrs = w_ptr + row_offsets_b[:, None] * K + offs_bk[None, :]- # TODO: check whether predication along K is needed since we checked that K is divisible by BLOCK_SIZE_K in the forward kernel- # col_mask = offs_bk[None, :] < K- store_mask = row_mask # & col_mask+ store_mask = row_maskaccumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_K), dtype = tl.float32)@@ -214,30 +197,29 @@ def _grouped_gemm_dX_kernel(dY = dY_desc.load([m_start + tile_m_idx * BLOCK_SIZE_M, n_offset])if not USE_TMA_LOAD_W:- w = tl.load(w_ptrs) # , mask=col_mask)+ w = tl.load(w_ptrs)else:w = w_desc.load([expert_idx, n_offset, tile_k_idx * BLOCK_SIZE_K])w = tl.reshape(w, (BLOCK_SIZE_N, BLOCK_SIZE_K))- # TODO: check if predication along K is needed since we checked that K is divisible by BLOCK_SIZE_K in the forward kernel# [M, N] @ [N, K] -> [M, K]dY = dY.to(w.dtype)- accumulator += tl.dot(dY, w) # NOTE: no transpose of b+ accumulator += tl.dot(dY, w)# Advance A along contiguous dimensionif not USE_TMA_LOAD_dY:dY_ptrs += BLOCK_SIZE_N- # Note we are no longer advancing B along contiguous dimension since weights are arranged as [N, K]- # Instead, we need to stride by K to advance to the [N_BLOCK_SIZE, K_BLOCK_SIZE] tile+ # B is no longer advanced along the contiguous dimension: weights are [N, K], so stride by K to+ # reach the next [N_BLOCK_SIZE, K_BLOCK_SIZE] tile.if not USE_TMA_LOAD_W:w_ptrs += BLOCK_SIZE_N * KdX = accumulator.to(output_dtype)- # Writing out a BLOCK_M x BLOCK_K tile, so we need to stride by K+ # A BLOCK_M x BLOCK_K tile is written out, so stride by K.if USE_TMA_STORE:- offset_m = tile_m_idx * BLOCK_SIZE_M # .to(tl.int32)- offset_k = tile_k_idx * BLOCK_SIZE_K # .to(tl.int32)+ offset_m = tile_m_idx * BLOCK_SIZE_M+ offset_k = tile_k_idx * BLOCK_SIZE_KdX_desc.store([m_start + offset_m, offset_k], dX)else:tl.store(@@ -256,7 +238,7 @@ def _grouped_gemm_dX_kernel(_autotuned_grouped_gemm_dX_kernel = triton.autotune(configs = get_dX_kernel_configs(),prune_configs_by = {"early_config_prune": prune_dX_configs},- # NOTE: NUM_TOKENS removed from key to avoid recompilation for every sequence length+ # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length.key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],)(_grouped_gemm_dX_kernel)@@ -327,7 +309,7 @@ def _grouped_gemm_dW_kernel(strides = [K, 1],block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],)- # Output tiles per expert, since each expert weight matrix is [N, K]+ # Output tiles per expert, since each expert weight matrix is [N, K].num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)output_tiles_per_expert = num_n_tiles * num_k_tiles@@ -336,7 +318,7 @@ def _grouped_gemm_dW_kernel(block_range_n = tl.arange(0, BLOCK_SIZE_N)block_range_k = tl.arange(0, BLOCK_SIZE_K)- # NOTE: Important that N % BLOCK_SIZE_N == 0 and K % BLOCK_SIZE_K == 0 when using TMA store+ # N % BLOCK_SIZE_N == 0 and K % BLOCK_SIZE_K == 0 are required when using TMA store.if USE_TMA_STORE:tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")@@ -347,7 +329,7 @@ def _grouped_gemm_dW_kernel(block_shape = [1, BLOCK_SIZE_N, BLOCK_SIZE_K],)- for tile_idx in range(tidx, output_tiles_per_expert, NUM_SMS): # , flatten=FLATTEN):+ for tile_idx in range(tidx, output_tiles_per_expert, NUM_SMS):# Output tile indextile_n_idx = tile_idx % num_n_tilestile_k_idx = tile_idx // num_n_tiles@@ -356,24 +338,23 @@ def _grouped_gemm_dW_kernel(n_offset = tile_n_idx * BLOCK_SIZE_Nk_offset = tile_k_idx * BLOCK_SIZE_K- # For storing- # TODO: Check whether the k mask is needed since we statically check that K is divisible by BLOCK_SIZE_K in the forward kernel- # ditto for n_maskn_mask = block_range_n + n_offset < Nk_mask = block_range_k + k_offset < Knk_mask = n_mask[:, None] & k_mask[None, :]m_end = 0for expert_idx in range(NUM_EXPERTS):- # We need to instantiate a fresh accumulator for each expert+ # A fresh accumulator per expert.accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_K), dtype = acc_dtype)m_start = m_end- # Need to figure out why this cast is needed, otherwise compiler complains about mismatching types+ # The cast is needed or the compiler complains about mismatching types; reason unknown.m_size = tl.load(m_sizes_ptr + expert_idx).to(tl.int32)m_end = m_start + m_size- # NOTE: when storing the result, we need to offset by n_start since we are storing the result for this expert to the global [E, N, K] weight matrix+ # Offset by n_start: the result goes into this expert's slice of the global [E, N, K] weight matrix.+ # NOTE: when storing the result, we need to offset by n_start since we are storing the result for+ # this expert to the global [E, N, K] weight matrixn_start = expert_idx * Nstore_row_offs = n_start + n_offset + block_range_n@@ -403,15 +384,12 @@ def _grouped_gemm_dW_kernel(if PERMUTE_X or PERMUTE_Y:# These will be used for loading and storing in permuted order- gather_offsets = (- tile_m_idx + block_range_m- ) # NOTE: tile_m_idx is already strided by BLOCK_SIZE_M+ gather_offsets = tile_m_idx + block_range_mindices_to_gather = m_start + tl.max_contiguous(tl.multiple_of(gather_offsets % m_size, BLOCK_SIZE_M),BLOCK_SIZE_M,)- # indices_to_gather = m_start + gather_offsetsexpert_token_idx = tl.load(gather_indices_ptr + indices_to_gather,mask = indices_to_gather < TOTAL_TOKENS,@@ -421,18 +399,16 @@ def _grouped_gemm_dW_kernel(# Masks for permuted load and storerow_load_mask = gather_offsets < m_size- # We only take into account the following two cases: (PERMUTE_X and NOT PERMUTE_Y) and (NOT PERMUTE_X and PERMUTE_Y)- # Hence, we can make the following simplifying assumptions when loading and storing- # Note the different strides between the two cases: the offsets for loading and storing are flipped and the strides must also be adjusted+ # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so+ # load/store offsets+ # are flipped between the two cases, with the strides adjusted.if PERMUTE_X:x_row_load_idx = ((expert_token_offsets // TOPK) * K- ) # Permute on load from token -> expert order, divide by TOPK to index from original number of tokens+ ) # Permute on load: token to expert order, /TOPK for the original count.dY_row_load_idx = m_offsets[:, None] * Nelse:- x_row_load_idx = (- indices_to_gather[:, None] * K- ) # Load in contiguous order (no permutation on load)+ x_row_load_idx = indices_to_gather[:, None] * KdY_row_load_idx = expert_token_offsets * Nelse:@@ -460,21 +436,18 @@ def _grouped_gemm_dW_kernel()accumulator += tl.dot(- dY.T.to(x.dtype), # [BLOCK_N, BLOCK_M]- x, # [BLOCK_M, BLOCK_K]+ dY.T.to(x.dtype),+ x,)y = accumulator.to(output_dtype)if USE_TMA_STORE:- # Need to expand dims to match [E, N, K] shape+ # Expand dims to match the [E, N, K] shape.y = tl.expand_dims(y, 0)dW_desc.store([expert_idx, n_offset, k_offset], y)else:tl.store(- dW_ptr- # + (n_offset + offs_n)[:, None] * K- + store_row_offs[:, None] * K- + (k_offset + block_range_k)[None, :],+ dW_ptr + store_row_offs[:, None] * K + (k_offset + block_range_k)[None, :],y,mask = nk_mask,)@@ -483,6 +456,6 @@ def _grouped_gemm_dW_kernel(_autotuned_grouped_gemm_dW_kernel = triton.autotune(configs = get_dW_kernel_configs(),prune_configs_by = {"early_config_prune": prune_kernel_configs_backward_dW},- # NOTE: NUM_TOKENS removed from key to avoid recompilation for every sequence length+ # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length.key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],)(_grouped_gemm_dW_kernel)diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py b/unsloth/kernels/moe/grouped_gemm/kernels/forward.pyindex cb1f22d0b85..0a4010e2881 100644--- a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py+++ b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py@@ -11,15 +11,8 @@ from .autotuning import ()-#-# PERMUTE_X -> permute tokens so that they are ordered by expert-# PERMUTE_Y -> permute output so that they are ordered by token-# These are effectively the same thing: the former loads in permuted order, the latter stores in permuted order => we only need to define the permutation indices once-# In the former, we use these row indices when loading X-# For the latter, we use these row indices when storing Y-# FUSE_MUL -> multiply routed outputs by their respective weights-# topk_weights are in token order-# Only account for the case when X is in expert order and we are permuting Y when fusing mul -- this precondition is checked in the interface+# PERMUTE_X loads tokens in expert order, PERMUTE_Y stores output in token order: the same+# permutation indices either way.@triton.jitdef _grouped_gemm_forward_kernel(x_ptr,@@ -57,14 +50,11 @@ def _grouped_gemm_forward_kernel(SHOULD_PERMUTE: tl.constexpr = PERMUTE_X or PERMUTE_YSHOULD_FUSE_MUL: tl.constexpr = FUSE_MUL_PRE or FUSE_MUL_POSTSHOULD_PERMUTE_OR_FUSE: tl.constexpr = SHOULD_PERMUTE or SHOULD_FUSE_MUL- # tl.static_print("SHOULD_PERMUTE", PERMUTE_X, PERMUTE_Y, FUSE_MUL_PRE, FUSE_MUL_POST, SHOULD_PERMUTE, SHOULD_FUSE, SHOULD_PERMUTE_OR_FUSE)tidx = tl.program_id(0)output_dtype: tl.dtype = y_ptr.dtype.element_ty- # Create TMA descriptors for loading sorted tokens- # When using TMA load, we don't permute_x, so shape should be [TOTAL_TOKENS, K]- # Also, we are defining a single global descriptor with single block shape- # Need to check that this does not result in errors when crossing expert boundaries+ # A single global TMA descriptor with one block shape; TMA load never permutes x, so the shape is+ # [TOTAL_TOKENS, K]. Unverified across expert boundaries.if USE_TMA_LOAD_X:x_desc = tl.make_tensor_descriptor(x_ptr,@@ -98,10 +88,10 @@ def _grouped_gemm_forward_kernel(num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)num_tiles_per_expert = num_m_tiles * num_n_tiles- # Need to create tma_store within loop since we need to predicate stores based on m_size+ # The tma_store is created inside the loop so stores can be predicated on m_size.if USE_TMA_STORE:y_desc = tl.make_tensor_descriptor(- y_ptr, # + m_start * N,+ y_ptr,shape = [m_end, N],strides = [N, 1],block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_N],@@ -129,21 +119,16 @@ def _grouped_gemm_forward_kernel(expert_token_offsets = expert_token_idx[:, None]# Masks for permuted load and store-row_mask = gather_offsets < m_sizerow_mask = row_mask[:, None]- # row_mask = indices_to_gather < m_end- # row_mask = row_mask[:, None]-- # We only take into account the following two cases: (PERMUTE_X and NOT PERMUTE_Y) and (NOT PERMUTE_X and PERMUTE_Y)- # Hence, we can make the following simplifying assumptions when loading and storing- # Note the different strides between the two cases: the offsets for loading and storing are flipped and the strides must also be adjusted+ # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so load/store offsets+ # are flipped between the two cases, with the strides adjusted.if PERMUTE_X:load_idx = ((expert_token_offsets // TOPK) * K- ) # Permute on load from token -> expert order, divide by TOPK to index from original number of tokens- store_idx = indices_to_gather[:, None] * N # Store in contiguous order+ ) # Permute on load from token to expert order, dividing by TOPK to index the original token count.+ store_idx = indices_to_gather[:, None] * Nelse:off_am = tile_m_idx * BLOCK_SIZE_Mif not PERMUTE_Y:@@ -157,19 +142,11 @@ def _grouped_gemm_forward_kernel(if PERMUTE_Y:if not USE_TMA_LOAD_X:- load_idx = (- indices_to_gather[:, None] * K- ) # Load in contiguous order (no permutation on load)- # offs_am = off_am + m_block_range- # row_mask = offs_am[:, None] < m_size- store_idx = (- expert_token_offsets * N- ) # Permute on store from expert -> token order-- # We always load topk weights in expert order- # In the pre-multiplication case, we multiply permuted hidden states by weights before the first gemm- # In the post-multiplication case, we multiply permuted hidden states by weights after the second gemm- # In either case, the hidden states are grouped by expert, so we always permute on load of topk weights+ load_idx = indices_to_gather[:, None] * K+ store_idx = expert_token_offsets * N++ # topk weights are always loaded in expert order: pre-multiplication scales hidden states before+ # the first gemm and post-multiplication after the second, both grouped by expert.if SHOULD_FUSE_MUL:topk_load_idx = expert_token_offsets@@ -215,10 +192,9 @@ def _grouped_gemm_forward_kernel(y = accumulator.to(output_dtype)- # NOTE: order of fusing multiplication is important- # Fusing before accumulator dtype conversion results in numerical diffs+ # Order of the fused multiplication matters: fusing before the accumulator dtype conversion changes+ # the numerics.if FUSE_MUL_POST:- # Check for correct broadcastingtopk_weights = tl.load(topk_weights_ptr + topk_load_idx, mask = row_mask)y *= topk_weights.to(output_dtype)@@ -226,8 +202,8 @@ def _grouped_gemm_forward_kernel(store_mask = row_mask & (offs_bn[None, :] < N)if USE_TMA_STORE:- offset_m = tile_m_idx * BLOCK_SIZE_M # .to(tl.int32)- offset_n = tile_n_idx * BLOCK_SIZE_N # .to(tl.int32)+ offset_m = tile_m_idx * BLOCK_SIZE_M+ offset_n = tile_n_idx * BLOCK_SIZE_Ny_desc.store([m_start + offset_m, offset_n], y)else:tl.store(@@ -243,8 +219,8 @@ def _grouped_gemm_forward_kernel(_autotuned_grouped_gemm_forward_kernel = triton.autotune(configs = get_forward_configs(),prune_configs_by = {"early_config_prune": prune_kernel_configs_fwd},- # NOTE: NUM_TOKENS removed from key to avoid recompilation for every sequence length- # The kernel handles variable token counts via m_sizes and tile-based processing+ # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length; the kernel+ # handles variable token counts via m_sizes and tile-based processing.key = ["NUM_EXPERTS","N",diff --git a/unsloth/kernels/moe/grouped_gemm/reference/layers/llama4_moe.py b/unsloth/kernels/moe/grouped_gemm/reference/layers/llama4_moe.pyindex a1d94d2c7d1..9170515f641 100644--- a/unsloth/kernels/moe/grouped_gemm/reference/layers/llama4_moe.py+++ b/unsloth/kernels/moe/grouped_gemm/reference/layers/llama4_moe.py@@ -155,7 +155,7 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):hidden_states = hidden_states.view(-1, hidden_dim)if self.overlap_router_shared:- # Marker for all prior ops on default stream+ # Marker for all prior ops on the default stream.self.default_event.record()router_logits, routing_weights, selected_experts = self.run_router(hidden_states)@@ -166,7 +166,8 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):if self.overlap_router_shared:with torch.cuda.stream(self.shared_expert_stream):- # Ensure prior kernels on default stream complete+ # Ensure prior kernels on the default stream complete, that hidden states stay valid on this stream,+ # and that the shared expert stays valid on the default stream.self.default_event.wait()shared_expert_out = self.shared_expert(hidden_states)@@ -189,8 +190,8 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):hidden_states = hidden_states.sum(dim = 1)hidden_states_after_weight_merge = hidden_states.view(-1, hidden_dim)- # Token counts per expert + gather indices (token->expert order).- # Auxiliary structs; not recorded in the autograd graph.+ # Token counts per expert plus gather indices (token to expert order): auxiliary structs, not+ # recorded in the autograd graph.token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(selected_experts)@@ -199,7 +200,6 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):hidden_states = permute(hidden_states_after_weight_merge, gather_indices, self.top_k)assert hidden_states.shape == (total_tokens, hidden_dim)- # Start expert computationfirst_gemm = torch_grouped_gemm(X = hidden_states, W = self.experts.gate_up_proj, m_sizes = token_counts_by_expert)@@ -214,10 +214,8 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):)assert second_gemm.shape == (total_tokens, hidden_dim)- # Post-processinghidden_states_unpermute = unpermute(second_gemm, gather_indices)assert hidden_states_unpermute.shape == (total_tokens, hidden_dim)- # grouped_gemm_out = hidden_states.view(batch_size, sequence_length, hidden_dim)final_out = hidden_states_unpermute + shared_expert_out@@ -331,7 +329,7 @@ class Llama4TritonTextMoe(Llama4GroupedGemmTextMoe):hidden_states = hidden_states.view(-1, hidden_dim)if self.overlap_router_shared:- # Marker for all prior ops on default stream+ # Marker for all prior ops on the default stream.self.default_event.record()router_logits, routing_weights, selected_experts = self.run_router(hidden_states)@@ -342,7 +340,8 @@ class Llama4TritonTextMoe(Llama4GroupedGemmTextMoe):if self.overlap_router_shared:with torch.cuda.stream(self.shared_expert_stream):- # Ensure prior kernels on default stream complete+ # Ensure prior kernels on the default stream complete, that hidden states stay valid on this stream,+ # and that the shared expert stays valid on the default stream.self.default_event.wait()shared_expert_out = self.shared_expert(hidden_states)@@ -365,8 +364,8 @@ class Llama4TritonTextMoe(Llama4GroupedGemmTextMoe):hidden_states = hidden_states.sum(dim = 1)hidden_states = hidden_states.view(-1, hidden_dim)- # Token counts per expert + gather indices (token->expert order).- # Auxiliary structs; not recorded in the autograd graph.+ # Token counts per expert plus gather indices (token to expert order): auxiliary structs, not+ # recorded in the autograd graph.token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(selected_experts)@@ -375,7 +374,6 @@ class Llama4TritonTextMoe(Llama4GroupedGemmTextMoe):hidden_states = permute(hidden_states, gather_indices, self.top_k)assert hidden_states.shape == (total_tokens, hidden_dim)- # Start expert computationhidden_states = grouped_gemm(X = hidden_states,W = self.experts.gate_up_proj,diff --git a/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py b/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.pyindex da3faa5f93c..47bd5548e9f 100644--- a/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py+++ b/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py@@ -44,7 +44,7 @@ class GroupedGEMMResult:intermediate: torch.Tensorsecond_gemm: torch.Tensorhidden_states_unpermute: torch.Tensor- hidden_states: torch.Tensor # final output+ hidden_states: torch.Tensorclass Qwen3MoeGroupedGEMMBlock(torch.nn.Module):@@ -70,10 +70,8 @@ class Qwen3MoeGroupedGEMMBlock(torch.nn.Module):config.moe_intermediate_size,)- # gatingself.gate = torch.nn.Parameter(gate)- # expertsself.gate_up_proj = torch.nn.Parameter(gate_up_proj, requires_grad = True)self.down_proj = torch.nn.Parameter(down_proj, requires_grad = True)self.act_fn = ACT2FN[config.hidden_act]@@ -157,8 +155,8 @@ class Qwen3MoeGroupedGEMMBlock(torch.nn.Module):router_logits, routing_weights, selected_experts = self.run_router(hidden_states)- # Token counts per expert + gather indices (token->expert order).- # Auxiliary structs; not recorded in the autograd graph.+ # Token counts per expert plus gather indices (token to expert order): auxiliary structs, not+ # recorded in the autograd graph.token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(selected_experts)@@ -167,7 +165,6 @@ class Qwen3MoeGroupedGEMMBlock(torch.nn.Module):hidden_states = permute(hidden_states, gather_indices, self.top_k)assert hidden_states.shape == (total_tokens, hidden_dim)- # Start expert computationfirst_gemm = torch_grouped_gemm(X = hidden_states, W = self.gate_up_proj, m_sizes = token_counts_by_expert)@@ -274,13 +271,13 @@ class Qwen3MoeFusedGroupedGEMMBlock(Qwen3MoeGroupedGEMMBlock):hidden_states = hidden_states.view(-1, hidden_dim)router_logits, routing_weights, selected_experts = self.run_router(hidden_states)- # Token counts per expert + gather indices (token->expert order).- # Auxiliary structs; not recorded in the autograd graph.+ # Token counts per expert plus gather indices (token to expert order): auxiliary structs, not+ # recorded in the autograd graph.token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(selected_experts)- # When permute_x is set, the permute fuses into the first gemm prologue+ # With permute_x set, the permute fuses into the first gemm's prologue.if not self.permute_x:hidden_states = permute(hidden_states, gather_indices, self.top_k)# Start expert computationdiff --git a/unsloth/kernels/moe/grouped_gemm/reference/moe_block.py b/unsloth/kernels/moe/grouped_gemm/reference/moe_block.pyindex 2412425c1a8..49c970a3f18 100644--- a/unsloth/kernels/moe/grouped_gemm/reference/moe_block.py+++ b/unsloth/kernels/moe/grouped_gemm/reference/moe_block.py@@ -96,17 +96,15 @@ class Qwen3MoeFusedGroupedGEMMBlock(Qwen3MoeGroupedGEMMBlock):hidden_states = hidden_states.view(-1, hidden_dim)router_logits, routing_weights, selected_experts = self.run_router(hidden_states)- # Pre-processing- # 1. Compute tokens per expert and indices for gathering tokes from token order to expert order- # NOTE: these are auxiliary data structs which don't need to be recorded in autograd graph+ # Tokens per expert and the token-order to expert-order gather indices: auxiliary structs, not+ # recorded in the autograd graph.token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(selected_experts)- # 2. permute_x -> permutation will be fused in prologue of first grouped gemm+ # permute_x fuses the permutation into the first grouped gemm's prologue.if not self.permute_x:hidden_states = permute(hidden_states, gather_indices, self.top_k)- # Start expert computationhidden_states = grouped_gemm(X = hidden_states,W = self.gate_up_proj,@@ -141,8 +139,7 @@ class Qwen3MoeFusedGroupedGEMMBlock(Qwen3MoeGroupedGEMMBlock):dX_only = self.dX_only,)- # Post-processing- # 1. Unpermute from expert order to token order+ # Post-processing 1. Unpermute from expert order to token orderif not self.permute_y:hidden_states = unpermute(hidden_states, gather_indices)diff --git a/unsloth/kernels/moe/grouped_gemm/reference/moe_ops.py b/unsloth/kernels/moe/grouped_gemm/reference/moe_ops.pyindex 41866de347c..510d89c96e4 100644--- a/unsloth/kernels/moe/grouped_gemm/reference/moe_ops.py+++ b/unsloth/kernels/moe/grouped_gemm/reference/moe_ops.py@@ -20,7 +20,7 @@ def permute(X: torch.Tensor, gather_indices: torch.Tensor, topk: int):"""assert gather_indices.ndim == 1X = X.view(-1, X.shape[-1])- # Shortcut for topk == 1+ # Shortcut for topk == 1.if topk == 1:return X[gather_indices]@@ -94,7 +94,7 @@ def get_routing_indices(min = 0,max = num_experts,)- # token_indices_experts_sorted shape (bs*slen*top_k,)+ # token_indices_experts_sorted has shape (bs * slen * top_k,).gather_indices = torch.argsort(selected_experts.view(-1), stable = True)if return_scatter_indices:scatter_indices = gather_indices.argsort()@@ -136,13 +136,11 @@ def torch_grouped_gemm(if m_size > 0:m_end = m_start + m_size- # Extract group input- # m_size x K+ # Extract group input m_size x KX_g = X[m_start:m_end]- # N x KW_g = W[g]- # Y_g = X_g @ W_g.T -> [m_size, N]+ # Y_g = X_g @ W_g.T -> [m_size, N].W_g = W_g.T if transpose else W_gY_g = X_g @ W_gdiff --git a/unsloth/kernels/moe/tests/common.py b/unsloth/kernels/moe/tests/common.pyindex f35a2fc9c34..b8574a82fb8 100644--- a/unsloth/kernels/moe/tests/common.py+++ b/unsloth/kernels/moe/tests/common.py@@ -89,7 +89,7 @@ TOLERANCE = {}-# from https://github.com/triton-lang/triton/blob/main/bench/triton_bench/testing.py+# From triton-lang/triton bench/triton_bench/testing.pydef assert_equal(ref, tri):if isinstance(ref, torch.Tensor):assert torch.all(ref == tri), f"tensors not equal {ref} != {tri}"@@ -265,15 +265,13 @@ def remove_feature_flags(return pruned_configs-# Test Configs-TOPK = [1, 4]NUM_EXPERTS = [4, 16]TEST_MODEL_SIZES = [- (32, 32), # Debug- (128, 128), # Small- (512, 512), # Medium+ (32, 32),+ (128, 128),+ (512, 512),]SMALL_MODEL_CONFIGS = [diff --git a/unsloth/kernels/moe/tests/moe_utils.py b/unsloth/kernels/moe/tests/moe_utils.pyindex 3b87fe45a40..801557c96d6 100644--- a/unsloth/kernels/moe/tests/moe_utils.py+++ b/unsloth/kernels/moe/tests/moe_utils.py@@ -34,13 +34,13 @@ def rebind_experts_to_shared_buffer(moe_block: Qwen3MoeSparseMoeBlock, config: Qbuffer_gate = torch.empty(num_experts, interm_size, hidden_size, device = device, dtype = dtype)buffer_down = torch.empty(num_experts, hidden_size, interm_size, device = device, dtype = dtype)- # Copy existing expert weights into buffers+ # Rebind expert weights to views in the shared bufferfor i, expert in enumerate(moe_block.experts):buffer_up[i].copy_(expert.up_proj.weight.data)buffer_gate[i].copy_(expert.gate_proj.weight.data)buffer_down[i].copy_(expert.down_proj.weight.data)- # Rebind expert weights to views in the shared buffer+ # Rebind expert weights to views in the shared buffer.for i, expert in enumerate(moe_block.experts):expert.up_proj.weight = torch.nn.Parameter(buffer_up[i])expert.gate_proj.weight = torch.nn.Parameter(buffer_gate[i])@@ -117,13 +117,11 @@ def check_gate_up_proj_grad(assert ref_gate_proj_grad is not Noneassert ref_up_proj_grad is not None- # Extract gradientstest_gate_proj_grad = grouped_gemm_block.gate_up_proj.grad[i, :moe_intermediate_size]test_up_proj_grad = grouped_gemm_block.gate_up_proj.grad[i, moe_intermediate_size:]assert test_gate_proj_grad is not Noneassert test_up_proj_grad is not None- # Sanity check shapesassert (ref_gate_proj_grad.shape == test_gate_proj_grad.shape), f"{ref_gate_proj_grad.shape} != {test_gate_proj_grad.shape}"@@ -131,7 +129,6 @@ def check_gate_up_proj_grad(ref_up_proj_grad.shape == test_up_proj_grad.shape), f"{ref_up_proj_grad.shape} != {test_up_proj_grad.shape}"- # Check gradientsdiff = (ref_gate_proj_grad - test_gate_proj_grad).abs().max()if not torch.allclose(ref_gate_proj_grad, test_gate_proj_grad, atol = atol, rtol = rtol):print(f"expert {i} gate_proj_grad_diff: {diff.detach().cpu().item():.6f}")@@ -199,7 +196,6 @@ def check_expert_grads(ref_grads.shape == test_grads.shape), f"{field}: {ref_grads.shape} != {test_grads.shape}"- # Test each expertfor i in range(ref_grads.shape[0]):ref_grad = ref_grads[i]test_grad = test_grads[i]@@ -208,7 +204,6 @@ def check_expert_grads(ref_grad, test_grad, atol = atol, rtol = rtol), f"{field}[{i}] diff: {diff.detach().cpu().item():.6f}"- # Test all expertsdiff = (ref_grads - test_grads).abs().max()if verbose:print(f"{field} diff: {diff.detach().cpu().item():.6f}")@@ -238,7 +233,6 @@ def check_fwd(rtol: float,verbose: bool = False,):- # First check hidden states (output)ref_output = ref_result.outputtest_output = test_result.outputdiff = (ref_output - test_output).abs().max()@@ -248,7 +242,6 @@ def check_fwd(ref_output, test_output, atol = atol, rtol = rtol), f"output diff: {diff.detach().cpu().item():.6f}"- # Check router logitsref_router_logits = ref_result.router_logitstest_router_logits = test_result.router_logitsdiff = (ref_router_logits - test_router_logits).abs().max()@@ -272,8 +265,8 @@ def check_grouped_gemm_results(test_value = getattr(fused_result, field.name)diff = (ref_value - test_value).abs().max()- # second_gemm in torch grouped gemm is not yet unpermuted so comparing the fused unpermuted second_gemm will result in error- # instead the hidden_states_unpermute should match since hidden_states_unpermute for the fused result is the same as second_gemm+ # The torch grouped gemm's second_gemm is not yet unpermuted, so compare hidden_states_unpermute+ # instead, which matches the fused result's second_gemm.if field.name == "second_gemm" and permute_y:continue@@ -404,18 +397,17 @@ class Qwen3MoeFusedGroupedGEMMBlock(Qwen3MoeGroupedGEMMBlock):hidden_states = hidden_states.view(-1, hidden_dim)router_logits, routing_weights, selected_experts = self.run_router(hidden_states)- # Pre-processing: token counts per expert + token-order -> expert-order- # gather indices (auxiliary, not recorded in the autograd graph).+ # Token counts per expert plus token-order to expert-order gather indices: auxiliary, not recorded+ # in the autograd graph.token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(selected_experts)- # permute_x fuses the permutation into the first grouped gemm's prologue+ # permute_x fuses the permutation into the first grouped gemm's prologue.if not self.permute_x:hidden_states = permute(hidden_states, gather_indices, self.top_k)assert hidden_states.shape == (total_tokens, hidden_dim)- # Start expert computationfirst_gemm = grouped_gemm(X = hidden_states,W = self.gate_up_proj,diff --git a/unsloth/kernels/moe/tests/test_grouped_gemm.py b/unsloth/kernels/moe/tests/test_grouped_gemm.pyindex 7ff47d07678..865df9f2007 100644--- a/unsloth/kernels/moe/tests/test_grouped_gemm.py+++ b/unsloth/kernels/moe/tests/test_grouped_gemm.py@@ -43,8 +43,7 @@ from .common import (SEED = 0-# Only certain (permute_x, permute_y, use_W1) combinations are valid; see the-# module string below for the full rationale.+# Only certain (permute_x, permute_y, use_W1) combinations are valid; see the module string below.def check_valid_config(permute_x,permute_y,@@ -165,8 +164,8 @@ def _test_grouped_gemm_forward(topk_weights, topk_ids = calculate_topk(gating_output, topk, use_sigmoid = use_sigmoid, renormalize = renormalize)- topk_weights = topk_weights.view(-1) # num_tokens * topk- topk_ids = topk_ids.view(-1) # num_tokens * topk+ topk_weights = topk_weights.view(-1)+ topk_ids = topk_ids.view(-1)expert_token_counts, gather_indices = get_routing_indices(topk_ids, num_experts = E)assert len(gather_indices) == total_tokens@@ -189,7 +188,7 @@ def _test_grouped_gemm_forward(else:X_test = Xperm- # No need to run all configs for tests, otherwise takes too long+ # Running all configs would take too long.if autotune:from grouped_gemm.kernels.forward import _autotuned_grouped_gemm_forward_kernelif num_autotune_configs is not None:@@ -197,7 +196,6 @@ def _test_grouped_gemm_forward(_autotuned_grouped_gemm_forward_kernel.configs[:num_autotune_configs])- # Use autograd.Function interfaceif use_autograd:from grouped_gemm.interface import grouped_gemmkernel_config_fwd = KernelConfigForward(@@ -228,7 +226,6 @@ def _test_grouped_gemm_forward(autotune = autotune,is_first_gemm = use_W1,)- # Use manual interfaceelse:test_output = grouped_gemm_forward(X = X_test,@@ -257,8 +254,8 @@ def _test_grouped_gemm_forward(if permute_y:ref_output = unpermute(ref_output, gather_indices)if fuse_mul_post:- # if we don't permute_y, then test output is permuted with topk weights applied- # the ref output needs to be unpermuted before multiplying by topk weights since topk weights are in token order+ # Without permute_y the test output is permuted with topk weights applied, so the reference output+ # must be unpermuted before multiplying by topk weights, which are in token order.if not permute_y:ref_output = unpermute(ref_output, gather_indices)test_output = unpermute(test_output, gather_indices)@@ -269,7 +266,7 @@ def _test_grouped_gemm_forward(), f"Grouped gemm forward failed: {(ref_output - test_output).abs().max().item():.6f}"-# NOTE: Fuse multiplication of topk weights is only supported for inference and not training, although this may change in the future; not currently tested.+# Fusing the topk-weight multiplication is inference-only, not training, and is not tested.@pytest.mark.parametrize("kernel_config",KERNEL_CONFIGS_FWD,@@ -457,7 +454,7 @@ def _test_grouped_gemm_backward_dX(if autotune and model_config.intermediate_size <= 128 and model_config.hidden_size <= 128:pytest.skip("Skipping autotuning for small model configs")- # Prevent OOM for large intermediate sizes+ # Prevent OOM for large intermediate sizes.if model_config.intermediate_size > 2048:model_config.intermediate_size = 1024if model_config.hidden_size > 2048:@@ -502,8 +499,8 @@ def _test_grouped_gemm_backward_dX(topk_weights, topk_ids = calculate_topk(gating_output, topk, use_sigmoid = use_sigmoid, renormalize = renormalize)- topk_weights = topk_weights.view(-1) # num_tokens * topk- topk_ids = topk_ids.view(-1) # num_tokens * topk+ topk_weights = topk_weights.view(-1)+ topk_ids = topk_ids.view(-1)expert_token_counts, gather_indices = get_routing_indices(topk_ids, num_experts = E)assert len(gather_indices) == total_tokens@@ -512,7 +509,7 @@ def _test_grouped_gemm_backward_dX(atol, rtol = TOLERANCE[X.dtype]Xperm = permute(X, gather_indices, topk)- # Need to retain grad otherwise grad is not propagated+ # Grad is not propagated without retain_grad.X.retain_grad()W.retain_grad()Xperm.retain_grad()@@ -607,10 +604,9 @@ def _test_grouped_gemm_backward_dX(test_output.backward(grad_output)assert X_.grad is not None- # NOTE:need to handle grad differenlty in this case due to errors arising to do how torch autograd handles unpermute and sum reduction- # the grad of Xperm unpermuted and reduced across topk should match X_.grad- # However, both will have a numerical difference with that of ref_grad- # This is due to the fact that torch autograd handles unpermute and sum reduction differently see: https://discuss.pytorch.org/t/permute-unpermute-gradient/219557 else:+ # Grad needs different handling here: Xperm's grad unpermuted and reduced across topk matches+ # X_.grad, but both differ numerically from ref_grad because autograd handles unpermute and sum+ # reduction differently. See discuss.pytorch.org/t/permute-unpermute-gradient/219557if permute_x and use_W1:X_grad_unperm = unpermute(Xperm.grad, gather_indices)manual_grad_check = X_grad_unperm.view(num_tokens, topk, K).sum(dim = 1)@@ -647,11 +643,10 @@ def _test_grouped_gemm_backward_dX(num_warps = num_warps,num_stages = num_stages,flatten = flatten,- # debug=True,)- # if permute_x and use_W1 (first grouped GEMM) then the kernel should have unpermuted the dX- # therefore we need to unpermute the ref_grad to compare to the output of the kernel+ # With permute_x and use_W1 (first grouped GEMM) the kernel has already unpermuted dX, so unpermute+ # ref_grad to compare.if permute_x and use_W1:ref_grad = unpermute(ref_grad, gather_indices)@@ -665,12 +660,11 @@ def _test_grouped_gemm_backward_dX(), f"Grouped gemm manual backward_dX outputs mismatch: {diff:.6f}"if permute_x and use_W1:- # Show that reduction results in diffs- # First calculate X.grad manually by backpropping through unpermuted ref_grad+ # Show that reduction results in diffs First calculate X.grad manually by backpropping through+ # unpermuted ref_graddX_ref_check = ref_grad.view(num_tokens, topk, K).sum(dim = 1)# Do the same for the actual output of the kerneldX_test_check = dX_test.view(num_tokens, topk, K).sum(dim = 1)- # Show diffs for each combinationdiff_ref_check = (X.grad - dX_ref_check).abs().max().item()diff_test_check = (X.grad - dX_test_check).abs().max().item()diff_check_test = (dX_ref_check - dX_test_check).abs().max().item()@@ -679,8 +673,8 @@ def _test_grouped_gemm_backward_dX()-# NOTE: We reduce the size of the Llama4 model configs to prevent OOM-# Important to note that for the full model size (5120, 8192), the tests do result in diffs on the order of 1e-2.+# The Llama4 model configs are shrunk to prevent OOM; at the full size (5120, 8192) the tests do+# show diffs on the order of 1e-2.@pytest.mark.parametrize("kernel_config",KERNEL_CONFIGS_BWD_dX,@@ -759,7 +753,7 @@ def test_grouped_gemm_backward_dX_autotune(use_W1: bool,num_autotune_configs: int,):- # TMA loads / stores will be autotuned+ # TMA loads / stores are autotuned._test_grouped_gemm_backward_dX(data_config = data_config,model_config = model_config,@@ -792,7 +786,7 @@ def test_grouped_gemm_backward_dX_autotune_autograd(use_W1: bool,num_autotune_configs: int,):- # TMA loads / stores will be autotuned+ # TMA loads / stores are autotuned._test_grouped_gemm_backward_dX(data_config = data_config,model_config = model_config,@@ -881,8 +875,8 @@ def _test_grouped_gemm_backward_dW(topk_weights, topk_ids = calculate_topk(gating_output, topk, use_sigmoid = use_sigmoid, renormalize = renormalize)- topk_weights = topk_weights.view(-1) # num_tokens * topk- topk_ids = topk_ids.view(-1) # num_tokens * topk+ topk_weights = topk_weights.view(-1)+ topk_ids = topk_ids.view(-1)expert_token_counts, gather_indices = get_routing_indices(topk_ids, num_experts = E)assert len(gather_indices) == total_tokens@@ -892,7 +886,7 @@ def _test_grouped_gemm_backward_dW(Xperm = permute(X, gather_indices, topk)Xperm_test = Xperm.detach().clone().requires_grad_(True)- # Need to retain grad otherwise grad is not propagated+ # Grad is not propagated without retain_grad.X.retain_grad()W.retain_grad()Xperm.retain_grad()@@ -903,8 +897,8 @@ def _test_grouped_gemm_backward_dW(ref_output = torch_grouped_gemm(X = Xperm, W = W, m_sizes = expert_token_counts)assert ref_output.shape == output_shape- # if permute_y then the assumption is that the output of grouped_gemm was unpermuted on store- # Therefore we have to unpermute before backpropping to ensure proper alignment+ # With permute_y the grouped_gemm output was unpermuted on store, so unpermute before backpropping+ # to keep the alignment.if permute_y:ref_output = unpermute(ref_output, gather_indices)@@ -913,7 +907,6 @@ def _test_grouped_gemm_backward_dW(assert X.grad is not Noneassert W.grad is not None- # Test backward kernel directlyX_ = X_test if permute_x else Xperm_testif debug:diff --git a/unsloth/kernels/moe/tests/test_llama4_moe.py b/unsloth/kernels/moe/tests/test_llama4_moe.pyindex ccc04b692a7..56cc51d6a6c 100644--- a/unsloth/kernels/moe/tests/test_llama4_moe.py+++ b/unsloth/kernels/moe/tests/test_llama4_moe.py@@ -32,7 +32,7 @@ LLAMA4_SCOUT_ID = "meta-llama/Llama-4-Scout-17B-16E"SEED = 42SEQ_LENS = [1024]DTYPES = [torch.bfloat16]-# Reduce the number of autotuning configs to prevent excessive runtime+# Reduce the number of autotuning configs to prevent excessive runtime.NUM_AUTOTUNE_CONFIGS = 50@@ -162,7 +162,7 @@ def test_llama4_ref(permute_x: bool,permute_y: bool,overlap_router_shared: bool,- model_config: Llama4TextConfig, # test fixture+ model_config: Llama4TextConfig,bs: int = 1,device = "cuda",precision = ".6f",@@ -177,10 +177,8 @@ def test_llama4_ref(check_diff = partial(_check_diff, atol = atol, rtol = rtol, precision = precision, verbose = verbose)check_grads = partial(_check_grads, atol = atol, rtol = rtol, precision = precision, verbose = verbose)- # Reference op -- HFllama4_ref = Llama4TextMoe(model_config).to(dtype = dtype, device = device)- # Torch grouped gemm implllama4_gg_ref = Llama4GroupedGemmTextMoe(model_config, overlap_router_shared = overlap_router_shared).to(dtype = dtype, device = device)diff --git a/unsloth/kernels/moe/tests/test_qwen3_moe.py b/unsloth/kernels/moe/tests/test_qwen3_moe.pyindex c601ebcfc71..7c0675f0d9d 100644--- a/unsloth/kernels/moe/tests/test_qwen3_moe.py+++ b/unsloth/kernels/moe/tests/test_qwen3_moe.py@@ -96,7 +96,7 @@ SEED = 42SEQ_LENS = [1024]DTYPES = [torch.bfloat16]-# Reduce the number of autotuning configs to prevent excessive runtime+# Reduce the number of autotuning configs to prevent excessive runtime.NUM_AUTOTUNE_CONFIGS = 50@@ -120,10 +120,9 @@ def test_qwen3_moe(hidden_size = config.hidden_sizebs = 1atol, rtol = TOLERANCES[dtype]- # Reference op -- HFmoe_block = Qwen3MoeSparseMoeBlock(config).to(device, dtype)- # Torch-native grouped gemm version of MoE Block -- for sanity checking+ # Torch-native grouped gemm version of the MoE block, for sanity checking.grouped_gemm_block = Qwen3MoeGroupedGEMMBlock.from_hf(moe_block).to(device, dtype)grouped_gemm_block.check_weights(moe_block)@@ -153,7 +152,7 @@ def test_qwen3_moe(kernel_config_bwd_dW = Nonekernel_config_bwd_dX = None- # Triton kernel grouped gemm version of MoE Block -- this is what we're testing+ # Triton kernel grouped gemm version of the MoE block, the thing under test.fused_gemm_block = Qwen3MoeFusedGroupedGEMMBlock.from_hf(moe_block,permute_x = permute_x,@@ -167,7 +166,6 @@ def test_qwen3_moe(X = torch.randn(bs, seqlen, hidden_size, dtype = dtype, device = device, requires_grad = True)- # Forwardref_result = run_forward(moe_block, X, is_grouped_gemm = False)grouped_result = run_forward(grouped_gemm_block, X, is_grouped_gemm = True)fused_result = run_forward(fused_gemm_block, X, is_grouped_gemm = True)@@ -178,15 +176,13 @@ def test_qwen3_moe(char = "=",num_chars = 100,):- # Sanity checks-with annotated_context("Checking HF vs torch grouped gemm MoE forward outputs..."):check_fwd(ref_result, grouped_result, atol, rtol, verbose = False)with annotated_context("Checking torch grouped gemm MoE vs fused grouped gemm MoE forward outputs..."):- # We implement a custom check for grouped gemm results to test each of the intermediate results for easier debugging+ # Custom check over the grouped gemm intermediates, for easier debugging.check_grouped_gemm_results(grouped_result.grouped_gemm_result,fused_result.grouped_gemm_result,@@ -195,11 +191,9 @@ def test_qwen3_moe(rtol = rtol,verbose = False,)- # Actual testwith annotated_context("Checking HF vs fused grouped gemm MoE forward outputs..."):check_fwd(ref_result, fused_result, atol, rtol, verbose = True)- # Backwardgrad_output = torch.randn_like(ref_result.output)ref_backward_result = run_backward(moe_block, grad_output, output = ref_result.output, X = ref_result.X@@ -220,7 +214,6 @@ def test_qwen3_moe(char = "=",num_chars = 100,):- # Sanity checkswith annotated_context("Checking HF vs torch grouped gemm MoE grads..."):check_grads(ref_backward_result, grouped_backward_result, atol, rtol, verbose = False)with annotated_context(@@ -234,7 +227,6 @@ def test_qwen3_moe(verbose = False,)- # Actual testwith annotated_context("Checking HF vs fused grouped gemm MoE grads..."):check_grads(ref_backward_result, fused_backward_result, atol, rtol, verbose = True)diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 5fc867e0cb3..13c8863332d 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -46,10 +46,10 @@ def _rms_layernorm_forward(r += row_idx * r_row_strideX_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)- W_row = tl.load(W + col_offsets, mask = mask, other = 0) # .to(tl.float32)+ W_row = tl.load(W + col_offsets, mask = mask, other = 0)row_var = tl.sum(X_row * X_row, axis = 0) / n_cols- # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm+ # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm.eps_f32 = tl.full((), eps, tl.float32)inv_var = tl.math.rsqrt(row_var + eps_f32)tl.store(r, inv_var)@@ -70,7 +70,6 @@ def _rms_layernorm_backward(W_row_stride: tl.constexpr,r,r_row_stride: tl.constexpr,- # dW, dW_row_stride,n_cols: tl.constexpr,eps: tl.constexpr,GEMMA: tl.constexpr,@@ -134,9 +133,8 @@ def _gemma_rms_layernorm_forward(eps: tl.constexpr,BLOCK_SIZE: tl.constexpr,):- # Copies https://github.com/google-deepmind/gemma/blob/main/gemma/layers.py#L31- # and https://github.com/keras-team/keras-nlp/blob/v0.8.2/keras_nlp/models/gemma/rms_normalization.py#L33- # exactly. Essentially all in float32!+ # Copies google-deepmind/gemma layers.py#L31 and keras-nlp gemma/rms_normalization.py#L33 exactly:+ # essentially all in float32.row_idx = tl.program_id(0)col_offsets = tl.arange(0, BLOCK_SIZE)mask = col_offsets < n_cols@@ -149,7 +147,7 @@ def _gemma_rms_layernorm_forward(W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)row_var = tl.sum(X_row * X_row, axis = 0) / n_cols- # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm+ # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm.eps_f32 = tl.full((), eps, tl.float32)inv_var = tl.math.rsqrt(row_var + eps_f32)tl.store(r, inv_var)@@ -214,7 +212,6 @@ class Fast_RMS_Layernorm(torch.autograd.Function):n_rows: intn_cols: intn_rows, n_cols = dY.shape- # dW = XdX = torch.empty_like(dY) if ctx.GEMMA else dYwith torch_gpu_device(dY.device):@@ -229,7 +226,6 @@ class Fast_RMS_Layernorm(torch.autograd.Function):W.stride(0),r,r.stride(0),- # dW, dW.stride(0),n_cols,ctx.eps,GEMMA = ctx.GEMMA,@@ -240,7 +236,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):return dX, None, None, None-# [TODO] Unsure why RMS Layernorm is not torch.compiling properly+# RMS Layernorm does not torch.compile properly; reason unknown.@torch.compiler.disabledef fast_rms_layernorm(layernorm,@@ -320,7 +316,6 @@ def test_rms_layernorm(YY = torch.randn((bsz, seqlen, dim), dtype = dtype, device = "cuda", requires_grad = True)Y.backward(YY)correct_grad = X.grad.clone()- # from unsloth.kernels import fast_rms_layernormY = fast_rms_layernorm(layernorm, XX)Y.backward(YY)assert torch.amax(correct_grad - XX.grad).item() <= 0.05diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.pyindex 7acab02e8e3..282b2623bf8 100644--- a/unsloth/kernels/rope_embedding.py+++ b/unsloth/kernels/rope_embedding.py@@ -138,19 +138,19 @@ def _rope_embedding()if BACKWARD_PASS:- # See our blog post for more info.+ # See the Unsloth blog post for more info.sin1 = -sin1# [TODO] Autotune ROPE_GROUP_SIZE to be 1, 2, 4, 8head_start = group_head_position * ROPE_GROUP_SIZEhead_end = min((head_start + ROPE_GROUP_SIZE), n_heads)- # 10% Faster kernel from [HuyNguyen-hust](https://github.com/unslothai/unsloth/pull/238)+ # 10% faster kernel from HuyNguyen-hust, unslothai/unsloth#238.for k in range(head_start, head_end):offs_q1 = row_position * Q_row_stride + k * head_dim + col_offsetsoffs_q2 = row_position * Q_row_stride + k * head_dim + col_offsets + half_head_dim- # For Gemma - sometimes RoPE must be done in float32 and not bfloat16+ # Gemma sometimes needs RoPE done in float32 rather than bfloat16.Q1 = tl.load(Q + offs_q1, mask = mask, other = 0).to(sin1.dtype)Q2 = tl.load(Q + offs_q2, mask = mask, other = 0).to(sin1.dtype)@@ -181,9 +181,9 @@ class Fast_RoPE_Embedding(torch.autograd.Function):n_rows, n_cols = Q.shapeassert seq_len <= cos.shape[0]- # [TODO] Changing blocksize to head_dim//2 seems to have- # some concurrency / un-deterministic issues.- BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2) # (head_dim//2)+ # Changing blocksize to head_dim//2 showed concurrency / non-deterministic issues; group_size too+ # large also hurts performance.+ BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2)# group_size = 4 # 4 or 8, too large group_size can hurt performance.div: int@@ -261,7 +261,7 @@ class Fast_RoPE_Embedding(torch.autograd.Function):)-# [TODO] Unsure why RoPE Embedding is not torch.compiling properly+# RoPE embedding does not torch.compile properly; reason unknown.@torch.compiler.disabledef fast_rope_embedding(Q,@@ -289,12 +289,12 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):batch, n_heads_Q, seq_len, head_dim = Q.shape_, n_heads_K, _, _ = K.shape- # Inplace rotary embedding is generally fine+ # Inplace rotary embedding is generally fine.Q_out = Q.clone() if not Q.is_contiguous() else QK_out = K.clone() if not K.is_contiguous() else Kif has_indices:- # TRL's rotary indices are always in int32, so casting is just for safety+ # TRL's rotary indices are always int32, so the cast is only for safety.rope_ptr = rope_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)else:rope_ptr = cos.new_empty(1, dtype = torch.int32)@@ -357,7 +357,7 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):rope_ptr = ctx.rope_indices if ctx.has_indices else ctx.cos.new_empty(1, dtype = torch.int32)- # Inplace rotary embedding is generally fine+ # Inplace rotary embedding is generally fine.dQ_out = dQ.clone() if not dQ.is_contiguous() else dQdK_out = dK.clone() if not dK.is_contiguous() else dK@@ -403,19 +403,17 @@ class Slow_RoPE_Embedding(torch.autograd.Function):@staticmethoddef forward(ctx, Q, cos, sin, position_ids):if position_ids is not None:- # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.- cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]- sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]- cos = cos[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim]- sin = sin[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim]+ # The first two dimensions of cos and sin are always 1, so squeeze them.+ cos = cos.squeeze(1).squeeze(0)+ sin = sin.squeeze(1).squeeze(0)+ cos = cos[position_ids].unsqueeze(2)+ sin = sin[position_ids].unsqueeze(2)- # Q * cos + rotate_half(Q) * sin+ # Q * cos + rotate_half(Q) * sin, with the transposed rotate_half in the backward.half = Q.shape[-1] // 2RH_Q = torch.cat((-Q[..., half:], Q[..., :half]), dim = -1)Q *= cosQ.addcmul_(RH_Q, sin)- # RH_Q *= sin- # Q += RH_Qctx.save_for_backward(cos, sin)return Q@@ -427,8 +425,6 @@ class Slow_RoPE_Embedding(torch.autograd.Function):RH_dY = torch.cat((dY[..., half:], -dY[..., :half]), dim = -1)dY *= cosdY.addcmul_(RH_dY, sin)- # RH_dY *= sin- # dY += RH_dYreturn dY, None, None, Nonediff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.pyindex 6d74a868fea..213839aeee0 100644--- a/unsloth/kernels/swiglu.py+++ b/unsloth/kernels/swiglu.py@@ -17,7 +17,7 @@ import triton.language as tlimport torchfrom .utils import calculate_settings, torch_gpu_device-# signed int32 max is 2**31-1 so num_elements cannot exceed 2**31+# signed int32 max is 2**31-1, so num_elements cannot exceed 2**31.NUM_INT32_ELEMENTS = 2**31SAFE_INT32_BUFFER_MULTIPLIER = 4BLOCK_SIZE = 1024@@ -35,15 +35,14 @@ def _fg_kernel(e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.mask = offsets < n_elementse_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)+ g_row = tl.load(g + offsets, mask = mask, other = 0)- # f = e * sigmoid(e)- f_row = e_row * tl.sigmoid(e_row) # e_row / (1 + tl.exp(-e_row))+ # f = e * sigmoid(e), h = f * g.+ f_row = e_row * tl.sigmoid(e_row)f_row = f_row.to(g_row.dtype) # Exact copy from HF# h = f * gh_row = f_row * g_row- # Store htl.store(h + offsets, h_row, mask = mask)@@ -83,13 +82,12 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEoffsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)mask = offsets < n_elements- DW_row = tl.load(DW + offsets, mask = mask, other = 0) # .to(tl.float32)+ DW_row = tl.load(DW + offsets, mask = mask, other = 0)e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)+ g_row = tl.load(g + offsets, mask = mask, other = 0)- # e = e.float()- # se = 1.0 / (1.0 + torch.exp(-e))- se_row = tl.sigmoid(e_row) # 1.0 / (1.0 + tl.exp(-e_row))+ # se = sigmoid(e), f = se * e, df = DW * f, dg = DW * g, de = dg * se * (1 + e * (1 - se)).+ se_row = tl.sigmoid(e_row)# f = (se * e).to(dtype)f_row = se_row * e_rowf_row = f_row.to(DW_row.dtype)@@ -103,10 +101,9 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEde_row = dg_row.to(tl.float32) * se_row * (1.0 + e_row * (1.0 - se_row))de_row = de_row.to(DW_row.dtype)- # Store derivatives in buffers- tl.store(DW + offsets, h_row, mask = mask) # h = f * g- tl.store(e + offsets, df_row, mask = mask) # df = DW * f- tl.store(g + offsets, de_row, mask = mask) # de+ tl.store(DW + offsets, h_row, mask = mask)+ tl.store(e + offsets, df_row, mask = mask)+ tl.store(g + offsets, de_row, mask = mask)def swiglu_DWf_DW_dfg_kernel(DW, e, g):diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 839eb9db84e..312baa66b53 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -33,7 +33,7 @@ from ..bnb_availability import native_kernels_readyfrom .fp8 import weight_dequant, fp8_linearimport functools-# torch.cuda.amp.custom_fwd is deprecated >= 2.4+# torch.cuda.amp.custom_fwd is deprecated from 2.4.import torchtorch_Tensor = torch.Tensor@@ -49,12 +49,15 @@ else:torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "cuda")torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "cuda")+# DEVICE_COUNT == 0 = no visible accelerator (e.g. CPU-only CI runner). The consumer functions below only index+# these arrays during real GPU work, so empty containers are safe -- they just need to be defined so the module+# imports cleanly.if DEVICE_TYPE == "xpu":torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "xpu")torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "xpu")-# tl.math.tanh now is libdevice.tanh+# tl.math.tanh is now libdevice.tanh.import tritonimport triton.language as tl@@ -68,7 +71,7 @@ if Version(triton.__version__) >= Version("3.0.0"):else:triton_tanh = tl.math.tanh- # No casting in old Triton versions+ # No casting in old Triton versions.@triton.jitdef triton_cast(x, dtype):return x.to(dtype)@@ -137,15 +140,13 @@ HAS_CUDA_STREAM = Falsetry:import bitsandbytes as bnb- # If an earlier `import bitsandbytes` died inside __init__, CPython evicts only- # the parent from sys.modules and keeps its submodules, so this retry re-executes- # __init__ without rebinding `bnb.functional`. `import x.y as z` reads sys.modules- # directly and survives that, plain attribute access does not.+ # If an earlier `import bitsandbytes` died inside __init__, CPython evicts only the parent from+ # sys.modules and keeps its submodules, so this retry re-executes __init__ without rebinding+ # bnb.functional. `import x.y as z` reads sys.modules directly and survives that.import bitsandbytes.functional as bnb_functionalexcept Exception:- # device_type.py already degrades to 16bit/full finetuning when bnb is missing- # (e.g. gfx906, whose generic wheel has no kernels). Keep the import working and- # fail only if a 4bit path is actually entered.+ # device_type.py already degrades to 16bit/full finetuning when bnb is missing (gfx906, whose+ # generic wheel has no kernels), so keep the import working and fail only if a 4bit path is entered.bnb = Nonebnb_functional = None@@ -178,19 +179,17 @@ else:return nullcontext()-# INTEL GPU Specific Logicif DEVICE_TYPE == "xpu":_gpu_getCurrentRawStream = torch._C._xpu_getCurrentRawStreamelif DEVICE_TYPE == "mlx":def _gpu_getCurrentRawStream(_index = 0):return 0-# NVIDIA GPU Default Logicelif hasattr(torch._C, "_cuda_getCurrentRawStream"):_gpu_getCurrentRawStream = torch._C._cuda_getCurrentRawStreamelse:- # CPU-only torch wheel (no compiled CUDA backend). _get_tensor_stream- # is only invoked during real GPU work, so a no-op binding is safe.+ # CPU-only torch wheel (no compiled CUDA backend). _get_tensor_stream is only invoked during real+ # GPU work, so a no-op binding is safe.def _gpu_getCurrentRawStream(_index = 0):return 0@@ -202,16 +201,12 @@ def _get_tensor_stream(tensor: torch_Tensor) -> c_void_p:return c_void_p(_gpu_getCurrentRawStream(tensor.device.index))-# Get array of CUDA streams and other buffersglobal CUDA_STREAMSglobal XPU_STREAMSglobal WEIGHT_BUFFERSglobal ABSMAX_BUFFERS-# DEVICE_COUNT == 0 = no visible accelerator (e.g. CPU-only CI runner).-# The consumer functions below only index these arrays during real GPU-# work, so empty containers are safe -- they just need to be defined so-# the module imports cleanly.+# DEVICE_COUNT == 0 means no visible accelerator (CPU-only CI runner).if DEVICE_TYPE == "xpu":if DEVICE_COUNT > 0:_XPU_STREAMS = {@@ -237,7 +232,6 @@ elif DEVICE_TYPE == "mlx":WEIGHT_BUFFERS = []ABSMAX_BUFFERS = []else:- # NVIDIA GPU Default Logicif DEVICE_COUNT > 0:_CUDA_STREAMS = {(index := torch.cuda.device(i).idx): ctypes.c_void_p(@@ -257,12 +251,11 @@ else:WEIGHT_BUFFERS = []ABSMAX_BUFFERS = []-# Bitsandbytes operationsctypes_c_int = ctypes.c_intctypes_c_int32 = ctypes.c_int32-# Same verdict device_type.py used to clear ALLOW_BITSANDBYTES, applied to the binds-# themselves. 0.45.5 leaves `functional.lib = None` when the native library fails to-# load, so these lookups would kill `import unsloth` instead of degrading to 16bit.+# Same verdict device_type.py used to clear ALLOW_BITSANDBYTES, applied to the binds themselves:+# 0.45.5 leaves functional.lib = None when the native library fails to load, so these lookups+# would kill `import unsloth` instead of degrading to 16bit.if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):cdequantize_blockwise_fp32 = _bnb_requiredcdequantize_blockwise_fp16_nf4 = _bnb_required@@ -275,8 +268,7 @@ else:cdequantize_blockwise_bf16_nf4 = bnb_functional.lib.cdequantize_blockwise_bf16_nf4if DEVICE_TYPE == "xpu":- # https://github.com/bitsandbytes-foundation/bitsandbytes/blob/c3b8de268fdb55a88f92feada23fc811a1e6877a/bitsandbytes/backends/xpu/ops.py#L115- # for xpu, inference gemv using above link+ # xpu inference gemv, per bitsandbytes backends/xpu/ops.py#L115.cgemm_4bit_inference_naive_fp16 = bnb_functional.lib.cgemv_4bit_inference_fp16cgemm_4bit_inference_naive_bf16 = bnb_functional.lib.cgemv_4bit_inference_bf16else:@@ -298,7 +290,6 @@ torch_float16 = torch.float16torch_bfloat16 = torch.bfloat16-# Check whether torchao can be imported to get Float8Tensorif importlib.util.find_spec("torchao") is not None:try:from torchao.quantization import Float8Tensor@@ -317,11 +308,10 @@ def QUANT_STATE(W):return getattr(W, "quant_state", None)-# fp8 weight dtypes. A `weight_scale` / `weight_scale_inv` should only be treated as a-# quant state when the weight itself is still fp8. compressed-tensors layers expose an-# already-dequantized bf16 weight at forward time while keeping a `weight_scale` around;-# reading that as a quant state routes a bf16 weight into the bitsandbytes fast_gemv /-# fast_dequantize path, which then reads a missing `absmax` and crashes.+# fp8 weight dtypes: a weight_scale / weight_scale_inv is only a quant state while the weight+# itself is still fp8. compressed-tensors layers expose an already-dequantized bf16 weight at+# forward time while keeping a weight_scale, and reading that as a quant state routes it into+# fast_gemv / fast_dequantize, which then reads a missing absmax and crashes._FP8_WEIGHT_DTYPES = tuple(dtypefor dtype in (@@ -336,21 +326,18 @@ def get_lora_parameters(proj):"""Return (weight, weight quant_state, lora A, lora B, lora scale).With QAT enabled, also fake-quantizes the base layer and lora weights."""- # For DPO or disabled adapters- base_layer = getattr(- proj, "base_layer", proj- ) # (proj.base_layer if hasattr(proj, "base_layer") else proj)+ # For DPO or disabled adapters.+ base_layer = getattr(proj, "base_layer", proj)W = base_layer.weight- # Optionally apply fake quantization to base layer weights for QAT+ # Optionally apply fake quantization to base layer weights for QAT.if hasattr(base_layer, "weight_fake_quantizer"):weight_fake_quantizer = getattr(base_layer, "weight_fake_quantizer", None)if weight_fake_quantizer is not None:W = weight_fake_quantizer(W)- # Get quant state for 4bit or FP8. Only fall back to a weight_scale(_inv) when the- # weight is still fp8; a bf16 weight (e.g. a decompressed compressed-tensors layer)- # must not carry a scale as its quant state or fast_gemv will crash on it.+ # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed+ # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.W_quant = getattr(W, "quant_state", None)if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:W_quant = getattr(base_layer, "weight_scale_inv", None)@@ -358,14 +345,12 @@ def get_lora_parameters(proj):W_quant = getattr(base_layer, "weight_scale", None)if getattr(base_layer, "quant_method", None) == "fp8":- # we need to somehow store and pass this information :)W.block_size = getattr(base_layer, "block_size", [128, 128])- # A decompressed compressed-tensors layer keeps quant_method == "fp8" while its- # weight is back to bf16, so it has no quant state to carry the block size.+ # A decompressed compressed-tensors layer keeps quant_method == "fp8" while its weight is back to+ # bf16, so it has no quant state to carry the block size.if W_quant is not None:W_quant.block_size = W.block_size- # if not hasattr(proj, "disable_adapters") or proj.disable_adapters or proj.merged:if getattr(proj, "disable_adapters", True) or proj.merged:return W, W_quant, None, None, None@@ -374,7 +359,7 @@ def get_lora_parameters(proj):adapter = getattr(proj, "active_adapter", ("default"))adapter = adapter[0]- # Optionally apply fake quantization to lora weights for QAT+ # Optionally apply fake quantization to lora weights for QAT.lora_A_linear = proj.lora_A[adapter]lora_B_linear = proj.lora_B[adapter]A = lora_A_linear.weight@@ -398,15 +383,12 @@ def get_lora_parameters(proj):def get_lora_parameters_bias(proj):- # For DPO or disabled adapters- base_layer = getattr(- proj, "base_layer", proj- ) # (proj.base_layer if hasattr(proj, "base_layer") else proj)+ # For DPO or disabled adapters.+ base_layer = getattr(proj, "base_layer", proj)W = base_layer.weight- # Get quant state for 4bit or FP8. Only fall back to a weight_scale(_inv) when the- # weight is still fp8; a bf16 weight (e.g. a decompressed compressed-tensors layer)- # must not carry a scale as its quant state or fast_gemv will crash on it.+ # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed+ # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.W_quant = getattr(W, "quant_state", None)if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:W_quant = getattr(base_layer, "weight_scale_inv", None)@@ -414,14 +396,12 @@ def get_lora_parameters_bias(proj):W_quant = getattr(base_layer, "weight_scale", None)if getattr(base_layer, "quant_method", None) == "fp8":- # we need to somehow store and pass this information :)W.block_size = getattr(base_layer, "block_size", [128, 128])- # A decompressed compressed-tensors layer keeps quant_method == "fp8" while its- # weight is back to bf16, so it has no quant state to carry the block size.+ # A decompressed compressed-tensors layer keeps quant_method == "fp8" while its weight is back to+ # bf16, so it has no quant state to carry the block size.if W_quant is not None:W_quant.block_size = W.block_size- # if not hasattr(proj, "disable_adapters") or proj.disable_adapters or proj.merged:if getattr(proj, "disable_adapters", True) or proj.merged:return W, W_quant, None, None, None, base_layer.bias@@ -451,7 +431,6 @@ def _maybe_fake_quantize_activations(X: torch.Tensor, proj: torch.nn.Module) ->return X-# INTEL GPU Specific Logicif DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:@torch.inference_mode@@ -469,8 +448,7 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:if W.dtype == torch.float8_e4m3fn:return weight_dequant(W, quant_state)if type(quant_state) is not list:- # New quant_state as a class- # https://github.com/TimDettmers/bitsandbytes/pull/763/files+ # New quant_state as a class, per TimDettmers/bitsandbytes#763.absmax = quant_state.absmaxshape = quant_state.shapedtype = quant_state.dtype@@ -491,7 +469,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:XPU_STREAM = XPU_STREAMS[device_index]n_elements_absmax = absmax.numel()- # Create weight matrixif use_global_buffer:# Use same buffers for faster inferencesize = shape[0] * shape[1]@@ -530,7 +507,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:requires_grad = False,)- # NF4 dequantization of statisticsptr_out_absmax = get_ptr(out_absmax)with torch_gpu_device(device):cdequantize_blockwise_fp32(@@ -544,7 +520,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:)out_absmax += offset- # Dequantize Wfx = (cdequantize_blockwise_fp16_nf4if dtype == torch_float16@@ -559,11 +534,10 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:ctypes_c_int(out.numel()),XPU_STREAM,)- # Careful returning transposed data+ # Careful returning transposed data.is_transposed = True if W.shape[0] == 1 else Falsereturn out.t() if is_transposed else out-# NVIDIA GPU Default Logicelif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:@torch.inference_mode@@ -580,8 +554,7 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:if W.dtype == torch.float8_e4m3fn:return weight_dequant(W, quant_state)if type(quant_state) is not list:- # New quant_state as a class- # https://github.com/TimDettmers/bitsandbytes/pull/763/files+ # New quant_state as a class, per TimDettmers/bitsandbytes#763.absmax = quant_state.absmaxshape = quant_state.shapedtype = quant_state.dtype@@ -604,9 +577,7 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:n_elements_absmax = absmax.numel()- # Create weight matrixif use_global_buffer:- # Use same buffers for faster inferencesize = shape[0] * shape[1]global WEIGHT_BUFFERSglobal ABSMAX_BUFFERS@@ -644,7 +615,6 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:)pass- # NF4 dequantization of statisticsptr_out_absmax = get_ptr(out_absmax)with torch_gpu_device(device):cdequantize_blockwise_fp32(@@ -658,7 +628,6 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:)out_absmax += offset- # Dequantize Wfx = (cdequantize_blockwise_fp16_nf4if dtype == torch_float16@@ -674,7 +643,7 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:CUDA_STREAM,)pass- # Careful returning transposed data+ # Careful returning transposed data.is_transposed = True if W.shape[0] == 1 else Falsereturn out.t() if is_transposed else out@@ -695,7 +664,7 @@ else:if W.dtype == torch.float8_e4m3fn:return weight_dequant(W, quant_state)if type(quant_state) is not list:- # New quant_state as a class+ # New quant_state as a class, per TimDettmers/bitsandbytes#763.# https://github.com/TimDettmers/bitsandbytes/pull/763/filesabsmax = quant_state.absmaxshape = quant_state.shape@@ -707,7 +676,6 @@ else:code2 = state2.codeblocksize2 = state2.blocksizeelse:- # Old quant_state as a list of listsabsmax, shape, dtype, blocksize, compressed_stats, _, _ = quant_stateoffset, state2 = compressed_statsabsmax2, code2, blocksize2, _, _, _, _ = state2@@ -716,7 +684,6 @@ else:n_elements_absmax = absmax.numel()device = W.device- # Create weight matrixif out is None:out = torch_empty(shape, dtype = dtype, device = device, requires_grad = False)else:@@ -726,7 +693,6 @@ else:n_elements_absmax, dtype = torch_float32, device = device, requires_grad = False)- # Do dequantizationptr_out_absmax = get_ptr(out_absmax)cdequantize_blockwise_fp32(get_ptr(code2),@@ -752,14 +718,13 @@ else:ctypes_c_int(out.numel()),)- # Careful returning transposed data+ # Careful returning transposed data.is_transposed = True if W.shape[0] == 1 else Falsereturn out.t() if is_transposed else outpass-# INTEL GPU Specific Logicif DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:def fast_gemv(@@ -770,13 +735,10 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:):if quant_state is None:return torch_matmul(X, W, out = out)- # For fast X @ W where seq_len == 1- # From https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L1469+ # Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763._, q_len, hd = X.shape- # assert(q_len == 1)if type(quant_state) is not list:- # https://github.com/TimDettmers/bitsandbytes/pull/763/filesabsmax = quant_state.absmaxshape = quant_state.shapedtype = quant_state.dtype@@ -796,7 +758,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:device_index = device.indexXPU_STREAM = XPU_STREAMS[device_index]- # assert(dtype == X.dtype)bout = shape[0]if out is None:@@ -809,9 +770,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:dtype = dtype,device = device,)- # else:- # assert(out.shape == (1, 1, bout,))- # passif DEVICE_TYPE == "xpu":m = 1@@ -879,13 +837,10 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:):if quant_state is None:return torch_matmul(X, W, out = out)- # For fast X @ W where seq_len == 1- # From https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L1469+ # Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763._, q_len, hd = X.shape- # assert(q_len == 1)if type(quant_state) is not list:- # https://github.com/TimDettmers/bitsandbytes/pull/763/filesabsmax = quant_state.absmaxshape = quant_state.shapedtype = quant_state.dtype@@ -906,7 +861,6 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:device_index = device.indexCUDA_STREAM = CUDA_STREAMS[device_index]- # assert(dtype == X.dtype)bout = shape[0]if out is None:@@ -919,9 +873,6 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:dtype = dtype,device = device,)- # else:- # assert(out.shape == (1, 1, bout,))- # passn = 1m = shape[0]@@ -987,13 +938,10 @@ else:):if quant_state is None:return torch_matmul(X, W, out = out)- # For fast X @ W where seq_len == 1- # From https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L1469+ # Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763._, q_len, hd = X.shape- # assert(q_len == 1)if type(quant_state) is not list:- # https://github.com/TimDettmers/bitsandbytes/pull/763/filesabsmax = quant_state.absmaxshape = quant_state.shapedtype = quant_state.dtype@@ -1009,7 +957,6 @@ else:offset, state2 = compressed_statsabsmax2, code2, blocksize2, _, _, _, _ = state2pass- # assert(dtype == X.dtype)bout = shape[0]device = W.device@@ -1023,9 +970,6 @@ else:dtype = dtype,device = device,)- # else:- # assert(out.shape == (1, 1, bout,))- # passn = 1m = shape[0]@@ -1100,7 +1044,6 @@ def fast_linear_forward(W = fast_dequantize(W.t(), W_quant, use_global_buffer = True)out = torch_matmul(X, W, out = out)- # Add in LoRA weightsif lora_A is not None:out_dim = out.shape[2]dtype = X.dtype@@ -1146,8 +1089,7 @@ def matmul_lora(if isinstance(W, Float8Tensor):assert W.ndim == 2if W.block_size[0] == W.shape[0] and W.block_size[1] == 1:- # Rowwise scaling becomes colwise after transpose, so this detects- # the backward pass. TODO: avoid calling matmul_lora in backward.+ # Rowwise scaling becomes colwise after transpose, so this detects the backward pass.W = W.dequantize()else:W = W.contiguous()@@ -1161,10 +1103,8 @@ def matmul_lora(del Wif A is not None:- # LoRA is enabledA, B = A.t(), B.t()XA = torch_matmul(X, A.to(dtype))out.addmm_(XA, B.to(dtype), alpha = s)- # out += (X @ A.to(dtype)) @ (s * B.to(dtype))return out.view(batch, seq_len, -1) if reshape else outdiff --git a/unsloth/models/_attn_mask_compat.py b/unsloth/models/_attn_mask_compat.pyindex df6d2c7e70b..b38bab35d7e 100644--- a/unsloth/models/_attn_mask_compat.py+++ b/unsloth/models/_attn_mask_compat.py@@ -36,13 +36,11 @@ import torchfrom transformers.utils.import_utils import is_torchdynamo_compilingtry:- # `is_tracing` exists only from Transformers 5.0.0, so the fallback below is- # the live path for all of 4.x, not just the 4.51.3 floor. The fallback- # mirrors what 4.x `modeling_attn_mask_utils` computes inline- # (`torch.jit.is_tracing() or isinstance(x, torch.fx.Proxy) or- # is_torchdynamo_compiling()`), so the data-dependent `torch.all(...)`- # branches stay skipped under JIT / FX / Dynamo and SDPA path selection is- # unchanged.+ # `is_tracing` exists only from Transformers 5.0.0, so the fallback below is the live path for all+ # of 4.x, not just the 4.51.3 floor. It mirrors what 4.x modeling_attn_mask_utils computes inline+ # (torch.jit.is_tracing() or isinstance(x, torch.fx.Proxy) or is_torchdynamo_compiling()), so the+ # data-dependent torch.all(...) branches stay skipped under JIT / FX / Dynamo and SDPA path+ # selection is unchanged.from transformers.utils.import_utils import is_tracing # type: ignore[attr-defined]except ImportError:@@ -188,9 +186,9 @@ class AttentionMaskConverter:expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)- # 0-dim tensor, not a Python float: a float literal lowers to an fp32- # scalar and breaks ExecuTorch edge-dialect export on fp16/bf16 masks.- # Matches upstream from 4.53.0 (huggingface/transformers#38637).+ # 0-dim tensor, not a Python float: a float literal lowers to an fp32 scalar and breaks ExecuTorch+ # edge-dialect export on fp16/bf16 masks. Matches upstream from 4.53.0+ # (huggingface/transformers#38637).inverted_mask = torch.tensor(1.0, dtype = dtype) - expanded_maskreturn inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)@@ -277,13 +275,10 @@ def _prepare_4d_causal_attention_mask_for_sdpa(key_value_length = key_value_length,)- # Attend to all tokens in masked rows, e.g. the first rows under left- # padding. Required by F.scaled_dot_product_attention's memory-efficient- # path: https://github.com/pytorch/pytorch/issues/110213- # Scoped to this branch to match upstream. At function scope it also runs- # on the `attention_mask is None` path, where it is a no-op numerically- # but materialises `to_causal_4d`'s stride-0 view into a dense- # [bsz, 1, q, kv] tensor.+ # Attend to all tokens in masked rows (the first rows under left padding), required by+ # F.scaled_dot_product_attention's memory-efficient path (pytorch/pytorch#110213). Scoped to this+ # branch to match upstream: at function scope it also runs on the attention_mask is None path,+ # where it materialises to_causal_4d's stride-0 view into a dense [bsz, 1, q, kv] tensor.if not is_tracing_ and expanded_4d_mask.device.type in ["cuda", "xpu"]:expanded_4d_mask = AttentionMaskConverter._unmask_unattended(expanded_4d_mask, min_dtype = torch.finfo(inputs_embeds.dtype).mindiff --git a/unsloth/models/_custom_dtype.py b/unsloth/models/_custom_dtype.pyindex 5df5b719efe..0deb02ce82f 100644--- a/unsloth/models/_custom_dtype.py+++ b/unsloth/models/_custom_dtype.py@@ -34,8 +34,7 @@ import osimport torch-# A table instead of `eval`, so the field NAMES a dtype rather than being an-# arbitrary expression.+# A table instead of eval, so the field NAMES a dtype rather than being an arbitrary expression.DTYPE_ALIASES = {"None": None,"none": None,@@ -53,9 +52,8 @@ DTYPE_ALIASES = {"fp32": torch.float32,}-# `unsloth_zoo==2026.8.15`, which this package's floor resolves to, still `eval`s the-# dtype field, and `eval("fp16")` is a NameError. So a field this package accepts is-# canonicalised to the one spelling both readers evaluate.+# unsloth_zoo==2026.8.15, which this package's floor resolves to, still evals the dtype field, and+# eval("fp16") is a NameError, so canonicalise to the one spelling both readers evaluate._CANONICAL_DTYPE_NAMES = {None: "None",torch.float16: "torch.float16",diff --git a/unsloth/models/_uma_safetensors.py b/unsloth/models/_uma_safetensors.pyindex 38d8b7d33a5..f53c934383d 100644--- a/unsloth/models/_uma_safetensors.py+++ b/unsloth/models/_uma_safetensors.py@@ -92,10 +92,7 @@ def patch_unified_memory_safetensors_load():return Truedef _clone_move(tensor, device):- # Clone into a regular CPU allocation to restore fast pinned-DMA, then- # move. The clone transiently doubles the tensor's CPU footprint and can- # OOM a low-memory UMA box; fall back to the direct, allocation-free move- # (a genuine non-memory error re-raises identically from it).+ # Clone into a regular CPU allocation to restore fast pinned-DMA, then move.try:return tensor.clone().to(device, non_blocking = False)except (MemoryError, RuntimeError):diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 34883a18c31..f8b6b1ffa6f 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -45,8 +45,6 @@ __all__ = ["create_boolean_mask","torch_amp_custom_fwd","torch_amp_custom_bwd",- # "accelerate_old_send_to_device",- # "accelerate_new_send_to_device","patch_gradient_accumulation_fix","apply_accepts_loss_kwargs_fix","patch_compiling_bitsandbytes",@@ -274,12 +272,11 @@ def _unsloth_install_pretrain_detector(model):return modelmarker = getattr(model, "_unsloth_pretrain_marker", None)if isinstance(marker, dict):- # A live hook is already recording: keep it (no duplicates) and DON'T clear seen -- a- # grad-enabled probe may have already flagged the poisoned cache, and a re-entrant- # get_peft_model/patch_peft_model call must not erase that before train() resets.+ # A live hook is already recording: keep it and do NOT clear seen, since a grad-enabled probe+ # may already have flagged the poisoned cache and a re-entrant call must not erase that.if "hook" in marker:return model- # Marker exists but its hook was torn down -> reinstall fresh, so reset seen.+ # Marker exists but its hook was torn down, so reinstall fresh and reset seen.marker["seen"] = Falseelse:marker = {"seen": False}@@ -289,8 +286,8 @@ def _unsloth_install_pretrain_detector(model):return modeldef _mark(_module, _inp):- # Only a grad-enabled forward poisons the AOTAutograd backward-graph cache; a no-grad- # probe builds no backward graph, so treat it as clean (avoids a needless dynamo reset).+ # Only a grad-enabled forward poisons the AOTAutograd backward-graph cache; a no-grad probe+ # builds no backward graph, so treat it as clean.if torch.is_grad_enabled():marker["seen"] = True@@ -302,20 +299,17 @@ def _unsloth_install_pretrain_detector(model):def _unsloth_reset_stray_compile_cache(self):- # A manual forward/backward under torch.compile BEFORE trainer.train() (e.g. a grad-norm- # probe) caches a forward + AOTAutograd backward graph in a one-off context; reusing it- # poisons training with NaN/zero gradients. If such a forward was seen and compile is on,- # drop the compiled-graph cache so training recompiles cleanly. No-op on the normal path.- # Module-level (not just inside the RL trainer template) so the SFT auto-packing wrapper and- # the plain-Trainer loop can import and run it too.+ # A manual forward/backward under torch.compile BEFORE trainer.train() caches a forward plus+ # AOTAutograd backward graph in a one-off context, and reusing it poisons training with+ # NaN/zero gradients, so drop the cache. Module-level, so the SFT auto-packing wrapper and the+ # plain-Trainer loop can run it too.import osmodel = getattr(self, "model", None)if model is None:return# The detector hook can sit on any wrapper in the chain, and the probe may have run on a- # different one than self.model, so walk the chain: detect a "seen" marker anywhere and- # collect every marker to tear down below.+ # different one than self.model, so walk the chain.markers = []seen = False_curr = model@@ -384,7 +378,7 @@ def apply_unsloth_gradient_checkpointing(use_gradient_checkpointing, max_seq_lenThe effective use_gradient_checkpointing value (may change from "unsloth" to True)"""if use_gradient_checkpointing == "unsloth":- # Offloading not worth it below ~512; standard gc is faster (crossover ~384-512).+ # Offloading is not worth it below ~512; standard gc is faster (crossover ~384-512).if max_seq_length < 512:unpatch_unsloth_smart_gradient_checkpointing()return True@@ -392,39 +386,29 @@ def apply_unsloth_gradient_checkpointing(use_gradient_checkpointing, max_seq_lenpatch_unsloth_smart_gradient_checkpointing(dtype = dtype)return "unsloth"elif use_gradient_checkpointing in (True, False):- # User explicitly set True or False - unpatch any previous "unsloth" patching+ # User explicitly set True or False, so unpatch any previous "unsloth" patching.unpatch_unsloth_smart_gradient_checkpointing()return use_gradient_checkpointingreturn use_gradient_checkpointing-# Models that don't work with flex_attention as the global Transformers-# attention implementation:-# GPT-OSS: training uses the custom flex sink patch, but inference intentionally-# falls back to eager because flex decoding gives incorrect outputs.-# Mllama: BlockMask Q_LEN!=KV_LEN ValueError on decode.-# NemotronH: hybrid Mamba-2 + Transformer, raises NotImplementedError.-# Gemma3N: timm vision wrappers don't support flex_attention.-# ModernBERT: create_block_mask with _compile=True hits CUDA illegal memory-# access on some GPU architectures (B200). Falls back to eager safely.+# Models that break with flex_attention as the global attention implementation: GPT-OSS+# inference (wrong decode outputs), Mllama (BlockMask Q_LEN != KV_LEN on decode), NemotronH+# (hybrid Mamba-2), Gemma3N (timm vision wrappers), ModernBERT (CUDA illegal access on B200)._FLEX_EXCLUDED_MODELS = ("gpt_oss", "mllama", "nemotron_h", "modernbert")_FLEX_PREFERRED_MODELS = ("gemma3", "gemma3_text", "shieldgemma2")_SDPA_EXCLUDED_MODELS = ("gpt_oss", "deepseek_v4")-# The loader (loader.py) forces supports_sdpa=False for these because their bundled-# SDPA modules are wrong. Kept here, not in loader.py, so _is_sdpa_excluded can honor-# them without a loader -> _utils import cycle (loader.py already imports from _utils-# and re-exports this name for callers like sentence_transformer.py). Entries are matched-# as substrings against a comma-joined model_types string ending in a comma, so "gemma3,"-# matches a distinct "gemma3" entry but not "gemma3n", and "gemma3_text" matches the-# EmbeddingGemma text model.+# The loader forces supports_sdpa=False for these, whose bundled SDPA modules are wrong; kept+# here so _is_sdpa_excluded can honor them without a loader -> _utils cycle. Matched as+# substrings of a comma-joined string, so "gemma3," matches gemma3 but not gemma3n.DISABLE_SDPA_MODEL_NAMES = [- "gemma3,", # Add comma bc gemma3 will match gemma3n+ "gemma3,","gemma3_text", # Gemma3TextModel (EmbeddingGemma) - substring match, keep underscore"gpt_oss",]_FLASH_EXCLUDED_MODELS = ("gpt_oss", "deepseek_v4")-# deepseek_v4's custom attention is sdpa/flash-incompatible; force eager, and-# excluded above so an explicit sdpa/flash request cannot re-enable the crash.+# deepseek_v4's custom attention is sdpa/flash-incompatible: force eager, and it is excluded+# above so an explicit request cannot re-enable the crash._EAGER_ONLY_PREFIXES = ("gemma3n", "deepseek_v4")_FLASH_ATTENTION_MAX_HEAD_DIM = 256_FLASH_ATTENTION_DISABLED_WARNED = set()@@ -435,20 +419,16 @@ def _is_flex_excluded(model_type):def _is_sdpa_disabled_by_name(model_type):- # Mirror the loader's DISABLE_SDPA_MODEL_NAMES check: loader.py builds- # model_types_all = ",".join(model_types) + "," and tests `name in model_types_all`.- # Rebuild the same trailing-comma form for a single model_type so the match is- # identical (e.g. "gemma3," matches "gemma3" but not "gemma3n", and "gemma3_text"- # still matches "gemma3_text").+ # Mirror the loader's DISABLE_SDPA_MODEL_NAMES check, which joins model_types with a trailing+ # comma, so a single model_type matches identically.model_types_all = model_type.lower() + ","return any(name.lower() in model_types_all for name in DISABLE_SDPA_MODEL_NAMES)def _is_sdpa_excluded(model_type):- # SDPA is known-broken for these models, so an explicit sdpa request must not- # re-enable it. Two sources: _SDPA_EXCLUDED_MODELS (resolver-level, e.g. gpt_oss)- # and DISABLE_SDPA_MODEL_NAMES (loader-level, e.g. gemma3 / gemma3_text, which the- # loader also forces to supports_sdpa=False).+ # SDPA is known-broken for these, so an explicit sdpa request must not re-enable it. Two sources:+ # _SDPA_EXCLUDED_MODELS (resolver, e.g. gpt_oss) and DISABLE_SDPA_MODEL_NAMES (loader, e.g.+ # gemma3 / gemma3_text).lowered = model_type.lower()return lowered in _SDPA_EXCLUDED_MODELS or _is_sdpa_disabled_by_name(lowered)@@ -468,10 +448,9 @@ def _is_eager_only(model_type):return any(model_type.startswith(p) for p in _EAGER_ONLY_PREFIXES)-# Ampere. Below it the flex HOP runs its eager `sdpa_dense` fallback, whose-# backward does `softmax_scores.to(query.dtype) @ grad_out` -- casting the scores-# but not `grad_out`. Such a card also forces fp16 here, so query is Half against-# a Float grad and the matmul is refused.+# Ampere. Below it the flex HOP runs its eager sdpa_dense fallback, whose backward casts+# softmax_scores but not grad_out; such a card also forces fp16, so a Half query meets a Float+# grad and the matmul is refused._FLEX_ATTENTION_MIN_CAPABILITY = (8, 0)@@ -532,12 +511,10 @@ def _config_get(return config.get(field_name, default)return getattr(config, field_name, default)except Exception:- # transformers 5.x heterogeneous configs (Gemma 3n / Gemma 4, anything- # with `per_layer_config`) raise AmbiguousGlobalPerLayerAttributeError,- # not AttributeError, on a global read of a per-layer field, so a getattr- # default does not cover it. This killed model load on transformers- # 5.15.0. Not caught by name: the class does not exist on 4.x, and a- # config is third-party code that may raise anything.+ # transformers 5.x heterogeneous configs (Gemma 3n / Gemma 4, anything with per_layer_config)+ # raise AmbiguousGlobalPerLayerAttributeError, not AttributeError, on a global read of a+ # per-layer field, so a getattr default does not cover it; this killed model load on 5.15.0.+ # Not caught by name: the class does not exist on 4.x.return default@@ -546,19 +523,16 @@ def _get_per_layer_values(config, field_name):per_layer = _config_get(config, "per_layer_config", None)if per_layer is None or isinstance(per_layer, (str, bytes)):return []- # Three shapes: a live `_PerLayerConfigView` (a Sequence, not a list/tuple);- # `to_dict`/config.json, a mapping of zero-padded layer index to overrides- # like `{"04": {"head_dim": 512}}`; and Unsloth's SimpleNamespace wrap of it.- # Iterating a mapping walks the indices, not the overrides, so the probe- # would answer 256 where the object form answers 512.+ # Three shapes: a live _PerLayerConfigView (a Sequence), the to_dict/config.json mapping of+ # zero-padded layer index to overrides, and Unsloth's SimpleNamespace wrap. Iterating a mapping+ # walks the indices, so the probe would answer 256 where the object form answers 512.if isinstance(per_layer, dict):per_layer = list(per_layer.values())else:try:iter(per_layer)except TypeError:- # Namespace form. `iter` not `__iter__`: the view may be an- # old-style sequence with only `__getitem__`.+ # `iter`, not `__iter__`: the view may be an old-style sequence with only __getitem__.per_layer = list(vars(per_layer).values()) if hasattr(per_layer, "__dict__") else []values = []try:@@ -627,9 +601,8 @@ def _collect_attention_head_dims(config):"local_head_dim","kv_head_dim",):- # Per-layer first: on a heterogeneous config the global read refuses,- # and no head dim reads as "no reason to disable Flash Attention" on- # exactly the models whose layers may exceed its ceiling.+ # Per-layer first: on a heterogeneous config the global read refuses, and no head dim reads as+ # "no reason to disable Flash Attention" on exactly the models that may exceed its ceiling.candidates = _get_per_layer_values(config, field_name)candidates.append(_config_get(config, field_name, None))for value in candidates:@@ -699,14 +672,12 @@ def _disable_flash_attention_if_needed(if disable_reason is None:return attn_implementation- # Only an implementation passed by the caller counts as an explicit request.- # Values read from the config are synthesized by the loaders (the language path- # seeds the config with attn_implementation="sdpa") or come from Transformers- # defaults, so they must not be treated as a deliberate user choice.+ # Only an implementation passed by the caller is an explicit request: config values are+ # synthesized by the loaders or come from Transformers defaults.explicit_request = attn_implementation- # Off for a float32 load: with no flash-specific reason the config never steered the- # choice, so a config-seeded "eager" must not drag an fp32 load from sdpa down to eager.+ # Off for a float32 load: with no flash-specific reason the config never steered the choice, so+ # a config-seeded "eager" must not drag an fp32 load from sdpa down to eager.requested_attn_implementation = attn_implementationif honor_config_attn_implementation:if requested_attn_implementation is None:@@ -719,13 +690,9 @@ def _disable_flash_attention_if_needed(model_type = _config_get(config, "model_type", "")- # The disable reason is flash-specific: honor an explicit non-flash request from- # the caller instead of downgrading it. SDPA is honored unless the model's SDPA is- # known-broken - _SDPA_EXCLUDED_MODELS (e.g. gpt_oss) or DISABLE_SDPA_MODEL_NAMES- # (e.g. gemma3 / gemma3_text); flex_attention- # is honored only when it is actually usable, since supports_flex_attention already- # rejects the excluded/broken/unavailable configs. This keeps an explicit request- # from selecting a backend the repo marks as wrong.+ # The disable reason is flash-specific, so honor an explicit non-flash request. SDPA is honored+ # unless known-broken, flex only when supports_flex_attention accepts it, so an explicit+ # request cannot select a backend the repo marks as wrong.if explicit_request == "sdpa" and not _is_sdpa_excluded(model_type.lower()):return _set_attn_impl(config, "sdpa")if explicit_request == "flex_attention" and supports_flex_attention:@@ -796,13 +763,13 @@ def resolve_model_class(auto_model, config):def _is_family_text_decoder(parent_model_type, text_model_type):- # True only for the family's own text variant (gemma3 -> gemma3_text); a generic- # reused decoder (llava -> llama) would load random weights, so keep the full model.+ # True only for the family's own text variant (gemma3 -> gemma3_text); a generic reused decoder+ # (llava -> llama) would load random weights, so keep the full model.return bool(parent_model_type) and str(text_model_type).startswith(parent_model_type)def _get_text_only_config(model_config, model_name):- # Text sub-config of a vision-language config so FastLanguageModel skips the vision tower (PR #5816).+ # Text sub-config of a vision-language config so FastLanguageModel skips the vision tower (#5816).text_config = Noneif hasattr(model_config, "get_text_config"):text_config = model_config.get_text_config()@@ -810,7 +777,7 @@ def _get_text_only_config(model_config, model_name):text_config = getattr(model_config, "text_config", None)if text_config is None:raise ValueError(f"Cannot load {model_name} as text-only; use FastVisionModel")- # Carry over quantization_config; copy first since get_text_config() shares the parent's object.+ # Carry over quantization_config; copy first, since get_text_config() shares the parent's object.qc = getattr(model_config, "quantization_config", None)if qc is not None and getattr(text_config, "quantization_config", None) is None:text_config = copy.copy(text_config)@@ -819,8 +786,8 @@ def _get_text_only_config(model_config, model_name):def _remap_text_only_skip_modules(qc):- # Remap llm_int8_skip_modules off the VLM wrapper prefix (language_model.model.* ->- # model.*) after text-only stripping, and drop vision/audio entries. See PR #5816.+ # Remap llm_int8_skip_modules off the VLM wrapper prefix (language_model.model.* -> model.*)+ # after text-only stripping, and drop vision/audio entries (#5816).is_dict = isinstance(qc, dict)skip = (qc.get("llm_int8_skip_modules") if is_dict else getattr(qc, "llm_int8_skip_modules", None)@@ -861,9 +828,9 @@ def _remap_text_only_skip_modules(qc):def _get_text_only_key_mapping(parent_config, text_config):- # transformers >=5 stopped auto-stripping the VLM wrapper prefix (base_model_prefix- # changed language_model -> model), so remap the text weights onto the decoder keys.- # None on tf <5 (still strips; a mapping would break the load) or non-family. See PR #5816.+ # transformers >= 5 stopped stripping the VLM wrapper prefix (base_model_prefix went+ # language_model -> model), so remap text weights onto the decoder keys. None on tf <5, which+ # still strips and would break the load with a mapping (#5816).if Version(transformers_version) < Version("5.0.0"):return Noneif not _is_family_text_decoder(@@ -923,9 +890,8 @@ def resolve_attention_implementation(if _is_eager_only(model_type):attn_impl = _set_attn_impl(config, "eager")elif prefers_flex_attention and supports_flex_attention:- # Models in _FLEX_PREFERRED_MODELS (gemma3 family) prefer flex_attention- # over flash. Caller can still override by passing- # requested_attn_implementation="sdpa" (handled below).+ # _FLEX_PREFERRED_MODELS (the gemma3 family) prefer flex_attention over flash; a caller can+ # still override with requested_attn_implementation="sdpa".attn_impl = _set_attn_impl(config, "flex_attention")elif not flash_attention_disabled and HAS_FLASH_ATTENTION and supports_flash_attention:attn_impl = _set_attn_impl(config, "flash_attention_2")@@ -948,16 +914,14 @@ def resolve_attention_implementation(elif supports_sdpa:attn_impl = _set_attn_impl(config, "sdpa")elif supports_flex_attention:- # Flex is only a fallback for models that don't support SDPA- # (e.g. some custom configurations). Without this fallback such- # models would land on eager.+ # Flex is only a fallback for models that do not support SDPA; without it such models would land on eager.attn_impl = _set_attn_impl(config, "flex_attention")else:attn_impl = _set_attn_impl(config, "eager")- # float32 only rules out flash, unlike config-driven reasons which say something about the- # model itself. So an explicit non-flash request must skip the flash fallback ladder, which- # answers differently: gemma3 + "sdpa" resolves to eager, but the ladder says flex_attention.+ # float32 only rules out flash, unlike config-driven reasons, which say something about the+ # model itself, so an explicit non-flash request skips the flash fallback ladder: gemma3 ++ # "sdpa" resolves to eager, but the ladder says flex_attention.reroute_request_through_flash_fallback = flash_attention_disabled and not (float32_is_only_disable_reasonand not _is_flash_attention_requested(requested_attn_implementation)@@ -977,15 +941,9 @@ def resolve_attention_implementation(final_attn_impl = requested_attn_implementation_set_attn_impl(config, final_attn_impl)- # A caller who explicitly passes requested_attn_implementation="sdpa" keeps it even- # on a conservatively unsupported model, mirroring _disable_flash_attention_if_needed- # which honors an explicit sdpa request. The exception is a model whose SDPA is- # known-broken - _SDPA_EXCLUDED_MODELS (e.g. gpt_oss) or DISABLE_SDPA_MODEL_NAMES- # (e.g. gemma3 / gemma3_text, which the loader also forces to supports_sdpa=False):- # an explicit request must not re-enable it, so it still downgrades to eager, just- # like flex falls back for _FLEX_EXCLUDED_MODELS. A synthesized/default sdpa- # (requested is None, so the value came from the model resolution above or the- # config) also downgrades.+ # An explicit "sdpa" is kept even on a conservatively unsupported model, except where SDPA is+ # known-broken, which still downgrades to eager just as flex falls back for+ # _FLEX_EXCLUDED_MODELS. A synthesized default sdpa (requested is None) also downgrades.honor_explicit_sdpa = requested_attn_implementation == "sdpa" and not _is_sdpa_excluded(model_type)@@ -1029,7 +987,6 @@ def _run_temporary_patches(phase):_run_temporary_patches("init")# =============================================-# Disable some warnings which can get annoyingwarnings.filterwarnings(action = "ignore", category = UserWarning, module = "torch")warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "torch")warnings.filterwarnings(action = "ignore", category = UserWarning, module = "huggingface_hub")@@ -1051,8 +1008,8 @@ logging.getLogger("transformers.tokenization_utils_base").setLevel(logging.CRITITORCHAO_MSG = "Error: torchao not found, please install with `pip install torchao`"-# Artifacts a Transformers/PEFT load never reads (ONNX/TF/Flax/CoreML/GGUF/training state), skipped-# when prewarming so a mixed-format repo is not pulled in full.+# Artifacts a Transformers/PEFT load never reads (ONNX/TF/Flax/CoreML/GGUF/training state),+# skipped when prewarming so a mixed-format repo is not pulled in full._PREFETCH_IGNORE_PATTERNS = ("*.onnx","onnx/*",@@ -1076,8 +1033,8 @@ _PREFETCH_IGNORE_PATTERNS = ()-# Repo-root tokenizer / config / processor files from_pretrained reads from root even when weights-# load from a subfolder. Exact names (no wildcard) so they match only root-level files.+# Repo-root tokenizer / config / processor files from_pretrained reads from root even when+# weights load from a subfolder. Exact names, so they match only root-level files._ROOT_AUX_PREFETCH_PATTERNS = ("config.json","generation_config.json",@@ -1090,7 +1047,8 @@ _ROOT_AUX_PREFETCH_PATTERNS = ("vocab.txt","merges.txt","spiece.model",- # More VOCAB_FILES_NAMES the slow tokenizer may fetch (DeBERTa-v2, Whisper, Mistral, XLM-R/mBART, Marian, FSMT/XLM, GPT-2).+ # More VOCAB_FILES_NAMES the slow tokenizer may fetch (DeBERTa-v2, Whisper, Mistral,+ # XLM-R/mBART, Marian, FSMT/XLM, GPT-2)."spm.model","normalizer.json","tokenizer.model.v3",@@ -1110,24 +1068,23 @@ _ROOT_AUX_PREFETCH_PATTERNS = ("preprocessor_config.json","processor_config.json","video_preprocessor_config.json", # Qwen2.5-VL-style video processors- # trust_remote_code auto_map can name any module, so warm every *.py (tiny; none in a non-remote repo).+ # trust_remote_code auto_map can name any module, so warm every *.py."*.py","*.tiktoken", # tiktoken vocab (e.g. Qwen's qwen.tiktoken))-# Files a PEFT adapter load reads: config + weights (glob covers sharded adapters). Any merged-# full-model weights the repo also ships match none of these.+# Files a PEFT adapter load reads: config plus weights, the glob covering sharded adapters. Any+# merged full-model weights the repo ships match none of these._ADAPTER_PREFETCH_PATTERNS = ("adapter_config.json","adapter_model*",)-# Weight files in a SUBDIRECTORY. A bare root load reads only root weights, so ignoring these drops-# alternate-precision/experimental dirs (fp16/, experimental/). "*/*" spans "/" (HF fnmatch), so nested-# weights match while root "model.safetensors" is kept. Only applied when weights_at_root (diffusion-# keeps weights in subfolders).+# Weight files in a SUBDIRECTORY: a bare root load reads only root weights, so ignoring these+# drops alternate-precision dirs. "*/*" spans "/" in HF fnmatch, so nested weights match while+# root model.safetensors is kept. Only under weights_at_root: diffusion uses subfolders._SUBDIR_WEIGHT_IGNORE_PATTERNS = ("*/*.safetensors","*/*.bin",@@ -1147,8 +1104,8 @@ def _in_requested_load_scope(filename, subfolder):return "/" not in filename # root load: no directory component-# .safetensors training-state files that are NOT model weights (e.g. optimizer.safetensors next to a-# real pytorch_model.bin); counting them as "model safetensors present" would drop the needed .bin.+# .safetensors training-state files that are NOT model weights (optimizer.safetensors beside a+# real pytorch_model.bin); counting them as model safetensors would drop the needed .bin._NON_MODEL_WEIGHT_STEMS = frozenset({"optimizer",@@ -1169,7 +1126,7 @@ def _is_model_weight_safetensors(filename):return Falseif name.startswith("adapter_"):return False- # Stem before first dot: "optimizer.safetensors" -> "optimizer" (real shards kept); rng_state via prefix.+ # Stem before the first dot: "optimizer.safetensors" -> "optimizer" (real shards kept); rng_state via prefix.stem = name.split(".", 1)[0].lower()if stem in _NON_MODEL_WEIGHT_STEMS or stem.startswith("rng_state"):return False@@ -1352,20 +1309,19 @@ def _prefetch_ignore_patterns(or (from_flax and pattern == "*.msgpack"))]- # Drop the format the load will not read (the other doubles the download); skipped for a whole- # multi-component snapshot (see docstring).+ # Drop the format the load will not read, since the other doubles the download; skipped for a+ # whole multi-component snapshot.whole_multi_component = not weights_at_root and not (isinstance(subfolder, str) and subfolder.strip("/"))if whole_multi_component:passelif st_module_paths and (from_tf or from_flax or use_safetensors is not None):- # An explicit format request cannot be scoped: no repo listing is fetched on these branches, and- # the globs span "/", so they would prune a declared ST module's only weight. Keep both formats,- # the same trade whole_multi_component already makes.+ # An explicit format request cannot be scoped: no repo listing is fetched here and the globs+ # span "/", so they would prune a declared ST module's only weight. Keep both formats.passelif from_tf or from_flax:- # TF / Flax loads never read the PyTorch formats; drop safetensors and .bin.+ # TF / Flax loads never read the PyTorch formats.ignore_patterns.extend(("*.safetensors",@@ -1375,10 +1331,10 @@ def _prefetch_ignore_patterns())elif use_safetensors is True:- # Explicit safetensors: load never reads .bin (no model_info call needed).+ # Explicit safetensors: the load never reads .bin, so no model_info call is needed.ignore_patterns.extend(("*.bin", "*.bin.index.json"))elif use_safetensors is False:- # Explicit .bin: load never reads safetensors.+ # Explicit .bin: the load never reads safetensors.ignore_patterns.extend(("*.safetensors", "*.safetensors.index.json"))else:# Auto: skip .bin only once in-scope safetensors are confirmed (best-effort; any failure keeps both).@@ -1395,8 +1351,8 @@ def _prefetch_ignore_patterns(.siblingsor [])- # Count only in-scope model-weights safetensors (not adapters/sidecars): variant-matching if- # a variant is requested, else canonical, proving the .bin redundant.+ # Count only in-scope model-weight safetensors, not adapters or sidecars, variant-matching when+ # a variant is requested, else canonical, to prove the .bin redundant.has_safetensors = any(_is_model_weight_safetensors(sibling.rfilename)and _in_requested_load_scope(sibling.rfilename, subfolder)@@ -1459,12 +1415,11 @@ def maybe_prefetch_hf_snapshot(if not isinstance(model_name, str) or not model_name:return False- # Local path: nothing to download. Expand ~ first (os.path.exists does not).+ # Local path: nothing to download. Expand ~ first, which os.path.exists does not.model_path = os.path.expanduser(model_name)if os.path.isdir(model_path) or os.path.exists(model_path):return False- # Looks local but not yet on disk (e.g. an uncreated output dir): not a Hub repo id, so leave it- # for from_pretrained rather than download it.+ # Looks local but is not yet on disk (an uncreated output dir): not a Hub repo id, so leave it for from_pretrained.if (os.path.isabs(model_path)or model_name.startswith(("~", "./", "../", ".\\", "..\\"))@@ -1481,9 +1436,9 @@ def maybe_prefetch_hf_snapshot(if fast_inference: # vLLM has its own download pathreturn False- # A sentence-transformers repo can hold weights at the root AND in declared module subfolders- # (2_Dense/...). Probed once here and shared, since both the subdir prune below and the redundant- # format prune span "/" and would drop those weights.+ # A sentence-transformers repo can hold weights at the root AND in declared module subfolders.+ # Probed once and shared, since the subdir prune and the format prune both span "/" and would+ # drop those weights.st_module_paths = ()if (weights_at_root@@ -1497,7 +1452,7 @@ def maybe_prefetch_hf_snapshot(cache_dir = cache_dir,)# tokenizer-only / adapter-only warms allow-list exact files below, so the weight-format ignore- # list (and its auto-branch model_info call) is skipped.+ # list and its model_info call are skipped.ignore_patterns = (Noneif tokenizer_only or adapter_only or gguf_file@@ -1514,12 +1469,12 @@ def maybe_prefetch_hf_snapshot(st_module_paths = st_module_paths,))- # Narrow the warm to what the load reads (skip extra checkpoints/precisions); every branch still warms- # root tokenizer/config/custom-code so those never fall in-process.+ # Narrow the warm to what the load reads; every branch still warms root tokenizer, config and+ # custom code so those never fall in-process.allow_patterns = Noneif gguf_file:- # gguf_file=NAME reads exactly that GGUF, but the static ignore list drops *.gguf; so warm just- # that file (plus root aux), under <subfolder>/ if set.+ # gguf_file=NAME reads exactly that GGUF, but the static ignore list drops *.gguf, so warm that+ # file plus root aux._gguf_path = (f"{subfolder.strip('/')}/{gguf_file}"if isinstance(subfolder, str) and subfolder.strip("/")@@ -1530,11 +1485,11 @@ def maybe_prefetch_hf_snapshot(# A distinct tokenizer repo: warm only tokenizer / config / vocab files, never its weights.allow_patterns = list(_ROOT_AUX_PREFETCH_PATTERNS)elif adapter_only:- # A PEFT adapter load reads only adapter_config.json + adapter_model.* (plus root aux), not any- # merged weights the repo may also publish.+ # A PEFT adapter load reads only adapter_config.json, adapter_model.* and root aux, not any+ # merged weights the repo may publish.allow_patterns = [*_ADAPTER_PREFETCH_PATTERNS, *_ROOT_AUX_PREFETCH_PATTERNS]- # PeftModel reads one format (safetensors when present): explicit use_safetensors wins, else- # prefer safetensors when shipped (best-effort; any failure keeps both).+ # PeftModel reads one format: an explicit use_safetensors wins, else prefer safetensors when+ # the repo ships them.if use_safetensors is False:ignore_patterns = ["adapter_model*.safetensors",@@ -1545,15 +1500,12 @@ def maybe_prefetch_hf_snapshot():ignore_patterns = ["adapter_model*.bin", "adapter_model*.bin.index.json"]elif isinstance(subfolder, str) and subfolder.strip("/"):- # subfolder=X: load resolves every weight under X/, so warm that subfolder (plus root aux).+ # subfolder=X: the load resolves every weight under X/, so warm that subfolder plus root aux.allow_patterns = [f"{subfolder.strip('/')}/*", *_ROOT_AUX_PREFETCH_PATTERNS]elif weights_at_root:- # A bare load reads only root weights: drop subdir weights (fp16/, checkpoint dirs) while keeping- # subdir configs. Diffusion leaves weights_at_root False.- #- # Except when the repo is both: a sentence-transformers model with a root weight AND weighted- # module subfolders (2_Dense/...), which the ST load reads too. Dropping those turned a- # satisfiable request into one that could never succeed. See _st_weighted_subfolder_paths.+ # A bare load reads only root weights, so drop subdir weights and keep subdir configs. Except+ # a sentence-transformers model with a root weight AND weighted module subfolders, which the ST+ # load also reads: dropping those made the request unsatisfiable.if not st_module_paths:ignore_patterns = [*(ignore_patterns or []), *_SUBDIR_WEIGHT_IGNORE_PATTERNS]try:@@ -1590,7 +1542,7 @@ class HideLoggingMessage(logging.Filter):return not (self.text in x.getMessage())-# Replace warning messages (analogous to HideLoggingMessage but for warnings.warn)+# Replace warning messages, analogous to HideLoggingMessage but for warnings.warn.class ReplaceWarningMessage:"""Intercepts warnings.warn calls and replaces matching messages with Unsloth branded ones.@@ -1698,7 +1650,7 @@ if not UNSLOTH_ENABLE_LOGGING:except:pass-# The speedups for torchdynamo mostly come with GPU Ampere or higher and which is not detected here.+# The torchdynamo speedups mostly need Ampere or higher, which is not detected here.from transformers.training_args import logger as transformers_training_args_loggertransformers_training_args_logger.addFilter(HideLoggingMessage("The speedups"))@@ -1717,7 +1669,7 @@ transformers_trainer_logger.addFilter(HideLoggingMessage("No label_names"))transformers_trainer_logger.addFilter(HideLoggingMessage("The tokenizer has new"))del transformers_trainer_logger-# Strip the "[transformers] " prefix transformers>=5 adds; keep the messages.+# Strip the "[transformers] " prefix transformers >= 5 adds; keep the messages.try:try:import transformers.utils.logging as _tf_log@@ -1732,7 +1684,6 @@ try:except Exception:pass-# Using the default loss: `ForCausalLMLoss`.try:from transformers.modeling_utils import logger as transformers_modeling_utils_loggertransformers_modeling_utils_logger.addFilter(HideLoggingMessage("ForCausalLMLoss"))@@ -1740,7 +1691,6 @@ try:except:pass-# The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function.try:from accelerate.utils.modeling import logger as accelerate_utils_modeling_loggeraccelerate_utils_modeling_logger.addFilter(HideLoggingMessage("The model weights are not tied"))@@ -1748,7 +1698,6 @@ try:except:pass-# Setting `pad_token_id` to `eos_token_id`try:from transformers.generation.utils import (logger as transformers_generation_utils_logger,@@ -1763,7 +1712,18 @@ try:except:pass-# The following generation flags are not valid and may be ignored:+# Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl"+# Hide HF Hub unauthenticated request warnings+# We detected that you are using `from_pretrained` with a meta device context manager or+# `torch.set_default_device('meta')+# `use_cache=True` is incompatible with gradient checkpointing+# Using a slow image processor as `use_fast`+# UserWarning: Logical operators 'and' and 'or' are deprecated for non-scalar tensors; please use '&' or '|'+# instead Will be fixed in torch 2.8.1 https://github.com/pytorch/pytorch/issues/158463+# You passed `quantization_config` or equivalent parameters+# MXFP4 quantization requires triton >= 3.4.0+# Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed.+# Gemma4 It is strongly recommended to train Gemma4 models with the `eager`try:from transformers.generation.configuration_utils import (logger as configuration_logger,@@ -1773,7 +1733,6 @@ try:except:pass-# Gemma3 It is strongly recommended to train Gemma3 models with the `eager`try:from transformers.models.gemma3.modeling_gemma3 import logger as gemma3_loggergemma3_logger.addFilter(HideLoggingMessage("strongly recommended"))@@ -1781,7 +1740,6 @@ try:except:pass-# Gemma4 It is strongly recommended to train Gemma4 models with the `eager`try:from transformers.models.gemma4.modeling_gemma4 import logger as gemma4_loggergemma4_logger.addFilter(HideLoggingMessage("strongly recommended"))@@ -1789,7 +1747,6 @@ try:except:pass-# Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed.try:from huggingface_hub.file_download import logger as hub_loggerhub_logger.addFilter(HideLoggingMessage("hf_xet"))@@ -1797,7 +1754,6 @@ try:except:pass-# MXFP4 quantization requires triton >= 3.4.0try:from transformers.quantizers.quantizer_mxfp4 import logger as mxfp4_loggermxfp4_logger.addFilter(HideLoggingMessage("requires triton"))@@ -1805,7 +1761,6 @@ try:except:pass-# You passed `quantization_config` or equivalent parameterstry:warnings.filterwarnings(action = "ignore",@@ -1816,8 +1771,7 @@ try:except:pass-# UserWarning: Logical operators 'and' and 'or' are deprecated for non-scalar tensors; please use '&' or '|' instead-# Will be fixed in torch 2.8.1 https://github.com/pytorch/pytorch/issues/158463+# Logical operators 'and'/'or' deprecated for non-scalar tensors; fixed in torch 2.8.1, pytorch/pytorch#158463.try:warnings.filterwarnings(action = "ignore",@@ -1828,7 +1782,6 @@ try:except:pass-# Using a slow image processor as `use_fast`try:from transformers.processing_utils import logger as processing_utils_loggerprocessing_utils_logger.addFilter(HideLoggingMessage("`use_fast`"))@@ -1836,7 +1789,6 @@ try:except:pass-# Using a slow image processor as `use_fast`try:from transformers.models.auto.image_processing_auto import (logger as processing_utils_logger,@@ -1846,7 +1798,6 @@ try:except:pass-# `use_cache=True` is incompatible with gradient checkpointingtry:from transformers.trainer import logger as trainer_loggertrainer_logger.addFilter(HideLoggingMessage("`use_cache=True`"))@@ -1854,7 +1805,6 @@ try:except:pass-# `use_cache=True` is incompatible with gradient checkpointingtry:from transformers.utils.generic import logger as trainer_loggertrainer_logger.addFilter(HideLoggingMessage("`use_cache=True`"))@@ -1862,7 +1812,6 @@ try:except:pass-# We detected that you are using `from_pretrained` with a meta device context manager or `torch.set_default_device('meta')try:from transformers.modeling_utils import logger as modeling_utils_loggermodeling_utils_logger.addFilter(HideLoggingMessage("anti-pattern"))@@ -1870,13 +1819,12 @@ try:except:pass-# Errors out on-# Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint+# Errors out on "Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint".from transformers.modeling_utils import logger as transformers_logger-# Faster safetensors loads on UMA (integrated) GPUs; lazy gate keeps this import-# fork-safe (no CUDA init). No-op off-UMA. Opt out: UNSLOTH_DISABLE_UMA_CLONE_LOAD=1.+# Faster safetensors loads on UMA GPUs; the lazy gate keeps this import fork-safe (no CUDA+# init). No-op off-UMA, opt out with UNSLOTH_DISABLE_UMA_CLONE_LOAD=1.from ._uma_safetensors import patch_unified_memory_safetensors_loadpatch_unified_memory_safetensors_load()@@ -1947,7 +1895,6 @@ try:except:pass-# Hide HF Hub unauthenticated request warningstry:from huggingface_hub.utils._http import logger as hf_http_loggerhf_http_logger.addFilter(HideLoggingMessage("You are sending unauthenticated requests"))@@ -1955,7 +1902,7 @@ try:except:pass-# Replace PEFT target_parameters warning with Unsloth branded message for MoE models+# Replace the PEFT target_parameters warning with an Unsloth-branded message for MoE models.ReplaceWarningMessage.add_rule(match_text = "target_parameters",replacement = (@@ -1965,7 +1912,6 @@ ReplaceWarningMessage.add_rule(category = RuntimeWarning,)-# Patch get_model_param_count to record correct 4bit / 8bitfrom transformers.trainer_pt_utils import is_deepspeed_zero3_enabled@@ -2013,13 +1959,12 @@ transformers.trainer_pt_utils.get_model_param_count = get_model_param_countimport transformers.trainertransformers.trainer.get_model_param_count = get_model_param_count-# =============================================-# =============================================-# Edits all Config files to enable RoPE Scaling for all models+# Edits all Config files to enable RoPE Scaling for all models.# Transformers had to update for Mistral Nemo 12b since Attention is (5120, 4096) now.+# =============================================def patch_mistral_nemo_config(config):if "head_dim (" not in config:add_head_dim = (@@ -2038,14 +1983,13 @@ def patch_mistral_nemo_config(config):try:- # Some Config files use layer_type_validation- # for eg Gemma-2, so we must import it to stop errors.+ # Some Config files use layer_type_validation (Gemma-2), so it must be imported to stop errors.from transformers.configuration_utils import layer_type_validationexcept:passtry:- # Transformers 5.0+ uses RotaryEmbeddingConfigMixin as a base class for configs+ # Transformers 5.0+ uses RotaryEmbeddingConfigMixin as a base class for configs.from transformers.modeling_rope_utils import RotaryEmbeddingConfigMixinexcept:pass@@ -2068,11 +2012,10 @@ model_architectures = ["falcon_h1",]-# Transformers 5.x uses class-level annotations with @strict, @auto_docstring,-# and interval() in config classes. exec(inspect.getsource(...)) fails because-# those symbols are not in scope. Skip the exec-based config patching for 5.x-# since those configs already use rope_parameters (the v5 replacement for-# rope_scaling).+# Transformers 5.x config classes use class-level annotations with @strict, @auto_docstring and+# interval(), so exec(inspect.getsource(...)) fails with those symbols out of scope. Skip the+# exec-based patching there; those configs already use rope_parameters, the v5 replacement for+# rope_scaling._skip_config_exec_patch = Version(transformers_version) >= Version("5.0.0")for model_name in model_architectures:@@ -2080,7 +2023,7 @@ for model_name in model_architectures:breakconfig_filepath = f"transformers.models.{model_name}.configuration_{model_name}"model_filepath = f"transformers.models.{model_name}.modeling_{model_name}"- config_filename = f"{model_name.title().replace('_', '')}Config" # qwen3 arch folder is qwen3_moe but config is Qwen3Config. Need to remove underscore(_) for now+ config_filename = f"{model_name.title().replace('_', '')}Config" # qwen3_moe -> Qwen3Config: drop the underscore.try:exec(f"from {config_filepath} import {config_filename}", globals())except:@@ -2106,7 +2049,7 @@ for model_name in model_architectures:config,)- # Just for Mistral Nemo+ # Just for Mistral Nemo.if model_name == "mistral":if Version(transformers_version) <= Version("4.42.4"):config = patch_mistral_nemo_config(config)@@ -2117,10 +2060,9 @@ for model_name in model_architectures:exec(f"{config_filepath}.{config_filename} = {config_filename}", globals())except Exception:continue-# =============================================+# torch.cuda.amp.custom_fwd is deprecated from 2.4.# =============================================-# torch.cuda.amp.custom_fwd is deprecated >= 2.4torch_version = torch.__version__if DEVICE_TYPE in ("cuda", "hip"):if Version(torch_version) < Version("2.4.0"):@@ -2135,32 +2077,12 @@ elif DEVICE_TYPE == "xpu":else:torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "xpu")torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "xpu")-# =============================================-# =============================================-# Fix KeyError: 'Cache only has 0 layers, attempted to access layer with index 0'-# import transformers.cache_utils-# if hasattr(transformers.cache_utils, "DynamicCache") and \-# transformers.cache_utils.DynamicCache.__getitem__.__name__ != "__cache_utils_getitem__":--# source = inspect.getsource(transformers.cache_utils.DynamicCache.__getitem__)-# start = source.find("def")-# spaces = start*" "-# source = source.split("\n")-# source = "\n".join(x[start:] for x in source)-# where = source.find("raise KeyError")-# source = source[:where] + \-# f"if len(self) == 0:\n{spaces}{spaces}"\-# " raise RuntimeError('Unsloth: You must call `FastLanguageModel.for_inference(model)` before doing inference for Unsloth models.')\n" + \-# f"{spaces}{spaces}else:\n{spaces}{spaces}{spaces}" + source[where:]-# source = source.replace("__getitem__", "__cache_utils_getitem__", 1)-# exec(source)-# transformers.cache_utils.DynamicCache.__getitem__ = __cache_utils_getitem__-# pass-# =============================================+# Fix KeyError: 'Cache only has 0 layers, attempted to access layer with index 0'.++# Weird Databricks errors.# =============================================-# Weird Databricks errorsfrom transformers.utils import is_openai_availableif is_openai_available():@@ -2176,16 +2098,13 @@ if is_openai_available():transformers.utils.is_openai_available = _is_openai_available# =============================================-# Get Flash Attention v2 if Ampere (RTX 30xx, A100)from transformers import AutoTokenizerfrom transformers.utils.import_utils import _is_package_availabledef _package_available(pkg_name: str) -> bool:- # transformers >= 5.x makes `_is_package_available` always return a- # `(exists, version)` tuple, which is truthy even when the package is- # absent; older versions returned a plain bool. Normalise to a bool so- # callers don't take "package present" branches for missing packages.+ # transformers >= 5.x makes _is_package_available always return an (exists, version) tuple,+ # truthy even when absent; older versions returned a plain bool.result = _is_package_available(pkg_name)if isinstance(result, tuple):return bool(result[0])@@ -2197,11 +2116,9 @@ HAS_FLASH_ATTENTION = FalseHAS_FLASH_ATTENTION_SOFTCAPPING = Falseif DEVICE_TYPE == "cuda" and not torch.cuda.is_available():- # UNSLOTH_ALLOW_CPU=1 keeps DEVICE_TYPE "cuda" on driverless hosts, so ask- # whether a device is present before asking what it can do. The three flags- # keep the defaults set just above: no device means no bfloat16 support to- # claim (False only costs float32, True fails at the first cast) and Flash- # Attention is a CUDA kernel that could not run here anyway.+ # UNSLOTH_ALLOW_CPU=1 keeps DEVICE_TYPE "cuda" on driverless hosts, so ask whether a device is+ # present before asking what it can do. No device means no bfloat16 to claim (False costs only+ # float32, True fails at the first cast) and no Flash Attention either.SUPPORTS_BFLOAT16 = FalseHAS_FLASH_ATTENTION = FalseHAS_FLASH_ATTENTION_SOFTCAPPING = False@@ -2212,7 +2129,7 @@ elif DEVICE_TYPE == "cuda":if major_version >= 8:SUPPORTS_BFLOAT16 = Trueif _package_available("flash_attn"):- # Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl"+ # Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl" and for softcapping; see #1437.try:try:# See https://github.com/unslothai/unsloth/issues/1437@@ -2238,7 +2155,6 @@ elif DEVICE_TYPE == "cuda":"Using Xformers instead. No performance changes will be seen.")- # Stop Flash Attention from importing!import transformers.utils.import_utilstransformers.utils.import_utils.is_flash_attn_2_available = (@@ -2257,6 +2173,7 @@ elif DEVICE_TYPE == "cuda":elif DEVICE_TYPE == "hip":SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported()if _package_available("flash_attn"):+ # Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl" and for softcapping; see #1437.# Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl"try:try:@@ -2283,7 +2200,6 @@ elif DEVICE_TYPE == "hip":"Using Xformers instead. No performance changes will be seen.")- # Stop Flash Attention from importing!import transformers.utils.import_utilstransformers.utils.import_utils.is_flash_attn_2_available = lambda *args, **kwargs: (@@ -2297,9 +2213,7 @@ elif DEVICE_TYPE == "hip":elif DEVICE_TYPE == "xpu":SUPPORTS_BFLOAT16 = True-# =============================================-# Get Xformers-# Silence xformers CUDA mismatch warnings before import+# Silence xformers CUDA mismatch warnings before import.try:_xformers_logger = logging.getLogger("xformers")_xformers_logger.setLevel(logging.ERROR)@@ -2309,11 +2223,10 @@ except:try:from xformers import __version__ as xformers_version- # Xformers <= 0.0.32.post2 has a broken FA3 dispatch on Blackwell/RTX 50x GPUs.- # The FA3 check used `capability >= (9, 0)` which matches SM 10.0/11.0/12.0,- # causing sm_90a kernels to be attempted on non-Hopper GPUs (CUDA error in- # flash_fwd_launch_template.h:188). Fixed in 0.0.33 with `<= (9, 0)`.- # See https://github.com/facebookresearch/xformers/issues/1329+ # Xformers <= 0.0.32.post2 dispatches FA3 wrongly on Blackwell/RTX 50x: `capability >= (9, 0)`+ # matches SM 10.0/11.0/12.0 and runs sm_90a kernels on non-Hopper GPUs (CUDA error in+ # flash_fwd_launch_template.h:188). Fixed in 0.0.33 with `<= (9, 0)`; see+ # facebookresearch/xformers#1329.if DEVICE_TYPE == "cuda":major_version, minor_version = torch.cuda.get_device_capability()if (f"{major_version}.{minor_version}" in ("10.0", "11.0", "12.0")) and (@@ -2328,8 +2241,8 @@ try:"```\n")- # Temporarily disable 0.0.27 and higher - inference issues- if False: # Version(xformers_version) >= Version("0.0.27"):+ # Temporarily disable 0.0.27 and higher: inference issues.+ if False:raise ImportError("Unsloth: If you are in Colab, we updated the top cell install instructions - please change it to below ""then press Disconnect Runtime and then Restart it.\n"@@ -2366,7 +2279,7 @@ try:from xformers._cpp_lib import _register_extensionstry:- _register_extensions() # Check if C++ modules are loaded correctly+ _register_extensions()except Exception as error:raise ImportError("Unsloth: Xformers was not installed correctly.\n"@@ -2390,11 +2303,10 @@ except Exception as e:xformers_attention = Nonexformers_version = None-# Check TRL versionfrom trl import __version__ as trl_version-# Unsloth now supports all TRL versions!-if False: # Version(trl_version) >= Version("0.9.0"):+# Unsloth now supports all TRL versions.+if False:raise ImportError("Unsloth: If you are in Colab, we updated the top cell install instructions - please change it to below ""then press Disconnect Runtime and then Restart it.\n"@@ -2408,43 +2320,23 @@ if False: # Version(trl_version) >= Version("0.9.0"):"Please downgrade TRL via `pip install --force-reinstall trl")+# Fix new Xformers versions "TypeError: Multiple dispatch failed for 'torch._ops.aten.to.dtype_layout'".++# Transformers 4.46 breaks dynamic caching; this is a hack.# =============================================-# Fix new Xformers versions TypeError: Multiple dispatch failed for 'torch._ops.aten.to.dtype_layout'-# accelerate_old_send_to_device = None-# accelerate_new_send_to_device = None-# if xformers_version is not None and Version(xformers_version) >= Version("0.0.27"):-# import accelerate.utils.operations-# if hasattr(accelerate.utils.operations, "send_to_device") and \-# accelerate.utils.operations.send_to_device.__name__ != "_fixed_send_to_device":-# accelerate_old_send_to_device = accelerate.utils.operations.send_to_device-# from accelerate.utils.operations import *-# send_to_device = inspect.getsource(accelerate.utils.operations.send_to_device)-# send_to_device = re.sub(-# r"([ ]{4,})return tensor\.to\(device\)",-# r"\1try: return tensor.to(device)\n\1except: return tensor",-# send_to_device,-# ).replace("def send_to_device", "def _fixed_send_to_device")-# exec(send_to_device)-# # accelerate.utils.operations.send_to_device = _fixed_send_to_device-# accelerate_new_send_to_device = _fixed_send_to_device-# pass-# pass--# Transformers 4.46 breaks dynamic caching. This is a hackimport transformers.generation.configuration_utilsif hasattr(transformers.generation.configuration_utils, "ALL_CACHE_IMPLEMENTATIONS"):if type(transformers.generation.configuration_utils.ALL_CACHE_IMPLEMENTATIONS) is list:if "dynamic" not in transformers.generation.configuration_utils.ALL_CACHE_IMPLEMENTATIONS:transformers.generation.configuration_utils.ALL_CACHE_IMPLEMENTATIONS.append("dynamic")-# =============================================+# Torch compile settings.# =============================================-# Torch compile settingsUNSLOTH_COMPILE_DEBUG = os.environ.get("UNSLOTH_COMPILE_DEBUG", "0") == "1"UNSLOTH_COMPILE_MAXIMUM = os.environ.get("UNSLOTH_COMPILE_MAXIMUM", "0") == "1"UNSLOTH_COMPILE_IGNORE_ERRORS = os.environ.get("UNSLOTH_COMPILE_IGNORE_ERRORS", "1") == "1"-# Just remove max_autotune_gemm warning+# Just remove the max_autotune_gemm warning.from torch._inductor.runtime.hints import DeviceProperties@@ -2499,7 +2391,7 @@ del acceleratedef patch_regional_compilation():- # Regional torch 2.5 Recompilation - weirdly very slow??+ # Regional torch 2.5 recompilation is weirdly very slow, and only works on torch 2.5.if torch.nn.ModuleList.__name__ == "UnslothModuleList":return# Only works for torch 2.5@@ -2533,8 +2425,6 @@ def patch_regional_compilation():# =============================================--def prepare_model_for_kbit_training(model: Any,use_gradient_checkpointing: Optional = True,@@ -2552,14 +2442,16 @@ def prepare_model_for_kbit_training()+# LoraLayer.update_layer downcasts PEFT layers to float16; mixed precision needs float32.# =============================================-# Weirdly LoraLayer.update_layer downcasts PEFT layers to float16??-# For mixed precision, we need it to be in float32 not float16.from peft import __version__ as peft_versionfrom peft.utils.integrations import dequantize_module_weightif Version(peft_version) < Version("0.12.0"):+ # Fix up incorrect downcasting of LoRA weightsfrom peft.tuners.lora.layer import LoraLayer++ # Prefer filesystem markers (harder to misidentify) before env-key matchingtry:source = inspect.getsource(LoraLayer.update_layer)text = "if weight is not None:\n"@@ -2574,7 +2466,6 @@ if Version(peft_version) < Version("0.12.0"):source = source.replace("def update_layer", "def LoraLayer_update_layer")exec(source, globals())- # Fix up incorrect downcasting of LoRA weightsfrom peft.tuners.lora.layer import LoraLayerLoraLayer.update_layer = LoraLayer_update_layer@@ -2630,18 +2521,16 @@ import psutildef _get_statistics(statistics = None, force_download = True):- # We log some basic stats about which environment is being used.- # We simply download a README.md file from HF - all data is made public.- # This is simply so we can check if some envs are broken or not.- # You can disable this by commenting the below out+ # Basic stats on which environment is in use: a README.md is downloaded from HF, all data+ # public, so broken envs can be detected. Disable with UNSLOTH_DISABLE_STATISTICS.n_cpus = psutil.cpu_count(logical = False)keynames = "\n" + "\n".join(os.environ.keys())- # Check modelscope for down detectionglobal USE_MODELSCOPEUSE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1"+ # Fallback to env-key detectionif statistics is None:- # Prefer filesystem markers (harder to misidentify) before env-key matching+ # Prefer filesystem markers (harder to misidentify) before env-key matching.try:from pathlib import Pathif Path("/kaggle/working").exists():@@ -2653,7 +2542,6 @@ def _get_statistics(statistics = None, force_download = True):except Exception:pass- # Fallback to env-key detectionif statistics is None:if "\nKAGGLE_" in keynames:statistics = "kaggle"@@ -2667,10 +2555,8 @@ def _get_statistics(statistics = None, force_download = True):statistics = "aws"elif "\nAZURE_" in keynames:statistics = "azure"- # elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp"elif "\nINVOCATION_ID" in keynames:statistics = "lambda"- # else: statistics = "other"else:def try_vllm_check():@@ -2732,15 +2618,12 @@ def _get_statistics(statistics = None, force_download = True):)except Exception:logger.debug("Unsloth: stats_check failed with an exception.")- # Don't retry without a time limit — would freeze offline+ # Do not retry without a time limit: it would freeze offline.def get_statistics(local_files_only = False):- # We log some basic stats about which environment is being used.- # This is also to check if HuggingFace is down or not!- # We simply download a README.md file from HF - all data is made public.- # This is simply so we can check if some envs are broken or not.- # You can disable this by setting UNSLOTH_DISABLE_STATISTICS+ # Basic stats on which environment is in use, and whether HuggingFace is down: a README.md is+ # downloaded from HF, all data public. Disable with UNSLOTH_DISABLE_STATISTICS.import osif (@@ -2798,33 +2681,16 @@ def get_statistics(local_files_only = False):# =============================================-# Fixes Bitsandbytes to remove missing warningsfrom transformers.utils.quantization_config import (BitsAndBytesConfig,QuantizationMethod,)-# Importing this module twice in one process must not raise.-#-# This block reads BitsAndBytesConfig.__init__'s source, rewrites it, and (at-# the bottom of the section) installs the result over the original. The-# replacement is built with exec(), so it has no file on disk. On a SECOND-# import, inspect.getsource() is therefore handed our own exec'd function,-# linecache finds nothing to read for it, and the import dies with-#-# OSError: could not get source code-#-# taking every later `from unsloth...` in that process with it. Anything that-# drops unsloth from sys.modules and imports it again reaches this: on a clean-# tree, `pytest tests/vllm_compat/test_extended_module_imports.py-# tests/python/test_vision_lora_targeting.py` is 30 failures, and it is only-# luck of the xdist scheduling that a full run does not usually pair them.-#-# The patch is idempotent by nature -- when it has already run, the wanted end-# state is the one already in place -- so recognising our own handiwork and-# standing down is the fix. The marker is set on the function rather than-# tracked in a module global because a re-import gets fresh globals but the same-# transformers class object.+# Importing this module twice must not raise. This block rewrites BitsAndBytesConfig.__init__+# via exec(), so it has no file on disk: on a SECOND import inspect.getsource() is handed our+# own function and dies with "OSError: could not get source code", taking every later+# `from unsloth...` with it. The patch is idempotent, so recognising our own work and standing+# down is the fix; the marker lives on the function, since a re-import gets fresh globals._bnb_init = BitsAndBytesConfig.__init__if getattr(_bnb_init, "__unsloth_patched__", False):@@ -2834,8 +2700,7 @@ else:try:BitsAndBytesConfig__init__ = inspect.getsource(_bnb_init)except (OSError, TypeError):- # Someone else replaced __init__ with something sourceless. Leaving- # theirs alone is strictly better than failing the import.+ # Someone else replaced __init__ with something sourceless; leaving theirs alone beats failing the import.BitsAndBytesConfig__init__ = Noneif BitsAndBytesConfig__init__ is None:@@ -2873,13 +2738,12 @@ if DEVICE_COUNT == 1 and int(os.environ.get("WORLD_SIZE", "1")) <= 1:accelerate.accelerator.Accelerator.distributed_type = property(lambda self: DistributedType.NO)-# to move multiple tensors to the same devicedef move_to_device(target_device, *tensors):"""Move tensors to target_device (returns same objects if already there)."""if isinstance(target_device, int):target_device = torch.device(target_device)elif isinstance(target_device, str):- # if string we expect it to be a device name like "cuda:0"+ # If a string, expect a device name like "cuda:0".target_device = torch.device(target_device)elif isinstance(target_device, torch.device):pass@@ -2897,9 +2761,9 @@ def move_to_device(target_device, *tensors):import transformers.utils.quantization_configtransformers.utils.quantization_config.BitsAndBytesConfig.__init__ = _BitsAndBytesConfig__init__-# =============================================-# Offloading to disk for modules (lm_head, embed_tokens)+# Offloading to disk for modules (lm_head, embed_tokens).+# =============================================import pickle@@ -2921,7 +2785,7 @@ def offload_to_disk(pickle_module = pickle,pickle_protocol = pickle.HIGHEST_PROTOCOL,)- # We must use weights_only = False due to pickling+ # weights_only = False is required due to pickling.offloaded_W = torch.load(filename, map_location = "cpu", mmap = True, weights_only = False)offloaded_W._offloaded_file_location = filenamereturn offloaded_W@@ -2953,7 +2817,7 @@ def offload_output_embeddings(model, temporary_location: str = "_unsloth_temporareturn-# Fixes a weird Torch 2.3 bug which says T4s have bfloat16+# Fixes a weird Torch 2.3 bug which says T4s have bfloat16.def is_bfloat16_supported():return SUPPORTS_BFLOAT16@@ -3009,7 +2873,7 @@ def is_vLLM_available():return _package_available("vllm")-# Patches models to add RoPE Scaling+# Patches models to add RoPE Scaling.def patch_linear_scaling(model_name = "gemma2",rope_module = None,@@ -3032,7 +2896,7 @@ def patch_linear_scaling(try:function = inspect.getsource(attention_module.__init__)except:- # Most likely already patched!+ # Most likely already patched.return None, Nonewhere = function.find("def")function = function.split("\n")@@ -3082,7 +2946,7 @@ def patch_linear_scaling(return init_name, function-# Patches for Llama-3 LlamaExtendedRotaryEmbedding+# Patches for Llama-3 LlamaExtendedRotaryEmbedding.def patch_llama_rope_scaling(model_name = "llama",rope_module = None,@@ -3111,7 +2975,7 @@ def patch_llama_rope_scaling(try:function = inspect.getsource(attention_module.__init__)except:- # Most likely already patched!+ # Most likely already patched.return None, Nonewhere = function.find("def")function = function.split("\n")@@ -3185,7 +3049,6 @@ def patch_llama_rope_scaling(def create_boolean_mask(n = 4096, sliding_window = 2048):- # Creates a boolean mask for attentionmask = torch.ones(n, n, dtype = torch.bool)if sliding_window == 0:return torch.triu(mask, diagonal = 1, out = mask)@@ -3242,12 +3105,10 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):if "num_items_in_batch" in kwargs:num_items_in_batch = kwargs["num_items_in_batch"]if num_items_in_batch is None:- # Remove it since the model does not support it!kwargs.pop("num_items_in_batch")elif "num_items_in_batch" not in inputs:inputs["num_items_in_batch"] = num_items_in_batch- # Get gradient accumulation steps if possibleif (num_items_in_batch is Noneand getattr(getattr(self, "args", self), "gradient_accumulation_steps", 1) != 1@@ -3264,8 +3125,8 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):"Using gradient accumulation will be very slightly less accurate.\n""Read more on gradient accumulation issues here: https://unsloth.ai/blog/gradient")- # Gemma3 multimodal models in transformers 5.x require token_type_ids during training.- # For text-only SFT, token_type_ids should be all zeros (no image tokens).+ # Gemma3 multimodal models on transformers 5.x require token_type_ids during training; for+ # text-only SFT they are all zeros.if "token_type_ids" not in inputs and "input_ids" in inputs:_inner = modelfor _attr in ("base_model", "model", "model"):@@ -3277,7 +3138,7 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):_has_ccm = _mod is not None and hasattr(_mod, "create_causal_mask_mapping")if _has_ccm and _inner.training:inputs["token_type_ids"] = torch.zeros_like(inputs["input_ids"])- # Gemma4 uses mm_token_type_ids (not token_type_ids) for VLM masking+ # Gemma4 uses mm_token_type_ids, not token_type_ids, for VLM masking.if "mm_token_type_ids" not in inputs and "input_ids" in inputs:_inner = modelfor _attr in ("base_model", "model", "model"):@@ -3295,8 +3156,7 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):def patch_gradient_accumulation_fix(Trainer):- # Fixes gradient accumulation- # Fixes Output 0 of UnslothFusedLossBackward is a view and is being modified inplace.+ # Fixes "Output 0 of UnslothFusedLossBackward is a view and is being modified inplace" and gradient accumulation.import inspectif hasattr(Trainer, "get_batch_samples"):@@ -3312,16 +3172,15 @@ def patch_gradient_accumulation_fix(Trainer):if Trainer.get_batch_samples.__name__ != "_unsloth_get_batch_samples":Trainer.get_batch_samples = _unsloth_get_batch_samples- # Also fix passing in num_items_in_batchif not hasattr(Trainer, "_old_compute_loss"):- # Fix transformers 4.57.0 causing `Output 0 of UnslothFusedLossBackward is a view and is being modified inplace.`+ # Fix transformers 4.57.0 raising "Output 0 of UnslothFusedLossBackward is a view and is being+ # modified inplace".function = inspect.getsource(Trainer.compute_loss)if "loss *=" in function or "loss*=" in function:where = function.find("def")function = function.split("\n")function = "\n".join(x[where:] for x in function)- # Import all variables that need importingimport transformers.traineritems_in_trainer = dir(transformers.trainer)@@ -3336,7 +3195,9 @@ def patch_gradient_accumulation_fix(Trainer):globals(),)- # Replace loss*= with loss = loss *+ # Replace loss *= with loss = loss *.+ # Fix when num_items_in_batch is nothing+ # https://github.com/huggingface/transformers/pull/35207function = re.sub(r"loss[\s]{0,}\*\=","loss = loss *",@@ -3354,7 +3215,6 @@ def patch_gradient_accumulation_fix(Trainer):"`pip install --upgrade --no-cache-dir --no-deps unsloth transformers git+https://github.com/huggingface/trl.git`")- # Also fix up loss scaling ie negate loss *= self.args.gradient_accumulation_stepsif not (Trainer.training_step.__name__ == "_unsloth_training_step"or "num_items_in_batch" not in inspect.signature(Trainer.training_step).parameters@@ -3364,7 +3224,6 @@ def patch_gradient_accumulation_fix(Trainer):function = function.split("\n")function = "\n".join(x[where:] for x in function)- # Import all variables that need importingimport transformers.traineritems_in_trainer = dir(transformers.trainer)@@ -3377,23 +3236,21 @@ def patch_gradient_accumulation_fix(Trainer):globals(),)- # Accelerate does / self.args.gradient_accumulation_steps internally, so if we already- # summed it up and did the division before hand, we have to negate it.+ # Accelerate divides by args.gradient_accumulation_steps internally, so a sum already divided+ # here has to be negated.function = function.replace("loss *= self.args.gradient_accumulation_steps","if num_items_in_batch is not None: loss *= self.args.gradient_accumulation_steps",)function = function.replace("def training_step", "def _unsloth_training_step", 1)- # Fix 4.47.0 issue where num_items_in_batch was removed- # See https://github.com/huggingface/transformers/pull/35121+ # Fix 4.47.0 removing num_items_in_batch (huggingface/transformers#35121) and the case where it+ # is nothing (huggingface/transformers#35207).function = function.replace("if self.model_accepts_loss_kwargs:","if False:",)- # Fix when num_items_in_batch is nothing- # https://github.com/huggingface/transformers/pull/35207function = re.sub(r"else:\n"r"([\s]{4,})self\.accelerator\.backward\(loss, \*\*kwargs\)\n"@@ -3409,12 +3266,10 @@ def patch_gradient_accumulation_fix(Trainer):exec(function, globals())Trainer.training_step = _unsloth_training_step- # Settle any deferred compile-mode switch at the start of every step.- # On recompile-limit exhaustion unsloth_zoo defers the switch to eager- # instead of flipping mid-call: non-reentrant checkpointing packs the- # forward compiled and would recompute it eagerly, aborting the backward- # with "Something went unexpectedly wrong in activation checkpoint".- # Between steps nothing is half-packed, so the switch is free.+ # Settle any deferred compile-mode switch at the start of every step: on recompile-limit+ # exhaustion unsloth_zoo defers the switch to eager rather than flipping mid-call, since+ # non-reentrant checkpointing packs the forward compiled and would recompute it eagerly,+ # aborting the backward. Between steps nothing is half-packed.if not getattr(Trainer, "_unsloth_settles_eager_fallbacks", False):try:from unsloth_zoo.temporary_patches.utils import (@@ -3437,10 +3292,9 @@ def patch_gradient_accumulation_fix(Trainer):Trainer.training_step = _unsloth_training_step_settling_fallbacksTrainer._unsloth_settles_eager_fallbacks = True- # Wrap Trainer.__init__: (1) pre-init, shadow accepts_loss_kwargs on whatever- # model was passed in (covers PEFT wrapping done after FastModel.from_pretrained);- # (2) post-init, clamp accelerator GA to 1 for the transformers 5.0-5.5- # GradientAccumulationPlugin regression. No-op on 4.x and 5.6+. See #4982.+ # Wrap Trainer.__init__: pre-init, shadow accepts_loss_kwargs on the model passed in (covers+ # PEFT wrapping after from_pretrained); post-init, clamp accelerator GA to 1 for the+ # transformers 5.0-5.5 GradientAccumulationPlugin regression. No-op on 4.x and 5.6+ (#4982).if not getattr(Trainer, "_unsloth_init_wrapped_for_accelerate_gas", False):_original_trainer_init = Trainer.__init__@@ -3476,7 +3330,7 @@ def patch_gradient_accumulation_fix(Trainer):def _unsloth_compile_cache_leaves():- # Accepts `UNSLOTH_COMPILE_LOCATION` overrides (the env var unsloth_zoo honors).+ # Accepts UNSLOTH_COMPILE_LOCATION overrides, the env var unsloth_zoo honors.leaves = {"unsloth_compiled_cache", "unsloth_cache", "unsloth_compiled"}loc = os.environ.get("UNSLOTH_COMPILE_LOCATION", "") or ""loc = loc.rstrip("/\\")@@ -3486,8 +3340,8 @@ def _unsloth_compile_cache_leaves():def _forward_is_unsloth_compiled(model):- # True iff forward was installed from the Unsloth compile cache directory.- # __module__ stays as the transformers module, so check co_filename.+ # True iff forward was installed from the Unsloth compile cache directory. __module__ stays the+ # transformers module, so check co_filename.leaves = _unsloth_compile_cache_leaves()def check(m):@@ -3522,8 +3376,8 @@ def _forward_is_unsloth_compiled(model):def _find_concrete_accepts_loss_kwargs(model):- # Walk wrapper chain for first class that declares accepts_loss_kwargs in its- # own __mro__ dict. Avoids PEFT __getattr__ forwarding and our own shadow.+ # Walk the wrapper chain for the first class declaring accepts_loss_kwargs in its own __mro__+ # dict, avoiding PEFT __getattr__ forwarding and our own shadow.seen = set()m = modelfor _ in range(6):@@ -3545,8 +3399,8 @@ def _find_concrete_accepts_loss_kwargs(model):def _shadow_accepts_loss_kwargs(model, value):- # Set the attribute at every wrapper level so HF's hasattr check resolves- # regardless of where accelerator / peft unwrap lands.+ # Set the attribute at every wrapper level so HF's hasattr check resolves wherever accelerator+ # or peft unwrap lands.seen = set()m = modelfor _ in range(8):@@ -3566,10 +3420,9 @@ def _shadow_accepts_loss_kwargs(model, value):def apply_accepts_loss_kwargs_fix(model):- # Shadow the correct accepts_loss_kwargs on the model so HF Trainer picks it- # up via hasattr(unwrapped_model, ...). Replaces the old Trainer.__init__- # source rewrite. Priority: compiled forward -> True; else first class attr- # in wrapper chain; else leave HF default. Issue #4982.+ # Shadow accepts_loss_kwargs on the model so HF Trainer sees it via hasattr(unwrapped_model).+ # Priority: compiled forward -> True, else the first class attr in the chain, else the HF+ # default (#4982).if _forward_is_unsloth_compiled(model):_shadow_accepts_loss_kwargs(model, True)return "True (Unsloth compiled forward)"@@ -3627,10 +3480,9 @@ def unsloth_compile_transformers(return_logits = False,unsloth_force_compile = False,):- # Again here, not only at import: a dotenv or config loader can populate- # `UNSLOTH_FORCE_CUSTOM_DTYPE` after this package was imported, and the compiler- # below reaches the older `unsloth_zoo` reader that still `eval`s the dtype field.- # Idempotent, so a registered or already sanitized value is unchanged.+ # Again here, not only at import: a dotenv or config loader can set UNSLOTH_FORCE_CUSTOM_DTYPE+ # after this package was imported, and the compiler reaches the older zoo reader that evals the+ # dtype field. Idempotent.from ._custom_dtype import neutralize_inherited_custom_dtypeneutralize_inherited_custom_dtype()@@ -3655,8 +3507,8 @@ def unsloth_compile_transformers(supports_sdpa = [True]- # Run patches BEFORE compiler so class replacements (e.g. GptOssTopKRouter,- # GptOssExperts) are in place before the compiler caches references to them.+ # Run patches BEFORE the compiler so class replacements (GptOssTopKRouter, GptOssExperts) are+ # in place before it caches references to them._run_temporary_patches("pre_compile")for model_type in model_types:@@ -3688,13 +3540,12 @@ def unsloth_compile_transformers(return_logits = return_logits,supports_sdpa = supports_sdpa,)- # Redo patches which override compiler+ # Redo patches which override the compiler._run_temporary_patches("post_compile")return model_types, supports_sdpa[0]-# We need an empty logits flag to warn people logits will not be returned anymore unless asked ie-# os.environ['UNSLOTH_RETURN_LOGITS'] = '1'+# An empty logits flag warns that logits are no longer returned unless os.environ['UNSLOTH_RETURN_LOGITS'] = '1'.LOGITS_ERROR_STRING = ("Unsloth: Logits are empty from 2024.11 onwards. To get raw logits again, please "'set the environment variable `UNSLOTH_RETURN_LOGITS` to `"1" BEFORE starting to train ie before `trainer.train()`. For example:\n'@@ -3730,7 +3581,8 @@ class EmptyLogits:return LOGITS_ERROR_STRINGdef __reduce__(self):- # Stateless pickling so gather_object works on the sentinel+ # Stateless pickling so gather_object works on the sentinel, and gathered copies compare equal+ # in accelerate debug mode.return (type(self), ())def __eq__(self, other):@@ -3749,8 +3601,8 @@ for j, function in enumerate(functions):exec(f"EMPTY_LOGITS.{function} = raise_{j}", globals(), locals())except:continue-# The loop above stomps pickle hooks with stubs returning None, which breaks-# gather_object on EMPTY_LOGITS in distributed runs. Restore default pickling.+# The loop above stomps pickle hooks with stubs returning None, breaking gather_object on+# EMPTY_LOGITS in distributed runs; restore default pickling.for function in ("__reduce__", "__reduce_ex__", "__getstate__", "__setstate__"):try:delattr(EMPTY_LOGITS, function)@@ -3828,7 +3680,7 @@ def fast_inference_setup(model_name, model_config):patch_vllm()if model_name.endswith("unsloth-bnb-4bit"):if not vllm_dynamic_quant_supported(model_name, model_config):- # Instead use -bnb-4bit variant+ # Instead use the -bnb-4bit variant.logger.warning_once(f"Unsloth: Switching from Unsloth dynamic quant to normal quant since\n"f"we do not yet support fast inference for {model_name}"@@ -3866,14 +3718,16 @@ def patch_peft_fast_inference(model):model.fast_generate = model.model.fast_generatemodel.fast_generate_batches = model.model.fast_generate_batches- # load_lora copies into vLLM's own adapter tensors, so it needs an engine.+ # load_lora copies into vLLM's own adapter tensors, so it needs an engine; with one, the Zoo+ # helper stays, since vLLM reads the adapter back through its own loader.from unsloth_zoo.vllm_utils import load_loramodel.load_lora = functools.partial(load_lora, model)- # An engine keeps the Zoo helper it has always had: vLLM reads the saved- # adapter back through its own LoRA loader, so what that file may carry- # is its call, not one to change here.+ # Without an engine there was no `save_lora` at all, and `save_lora` needs none: it is `save_pretrained`+ # over the adapter. Gating it on the engine gave `fast_inference = False` GRPO runs `AttributeError:+ # 'Lfm2ForCausalLM' object has no attribute 'save_lora'`. Set only when absent, so a model carrying its+ # own keeps it.if not hasattr(model, "save_lora"):try:from unsloth_zoo.vllm_utils import save_lora@@ -3882,11 +3736,9 @@ def patch_peft_fast_inference(model):if save_lora is not None:model.save_lora = functools.partial(save_lora, model)- # Without an engine there was no `save_lora` at all, and `save_lora` needs- # none: it is `save_pretrained` over the adapter. Gating it on the engine- # gave `fast_inference = False` GRPO runs `AttributeError:- # 'Lfm2ForCausalLM' object has no attribute 'save_lora'`.- # Set only when absent, so a model carrying its own keeps it.+ # Without an engine there was no save_lora at all, and it needs none (it is save_pretrained+ # over the adapter): gating it on the engine gave fast_inference = False GRPO runs an+ # AttributeError. Set only when absent.if not hasattr(model, "save_lora"):model.save_lora = functools.partial(save_lora_adapter, model)@@ -3913,7 +3765,7 @@ except:class TorchAOConfig:qat_scheme: Optional[str] = "int4"- # Each (config, filter_fn) pair defines a quantization rule+ # Each (config, filter_fn) pair defines a quantization rule.base_config_and_filter_fns: List[Tuple["AOBaseConfig", Optional[Callable[[torch.nn.Module, str], bool]]]] = field(@@ -3925,7 +3777,6 @@ class TorchAOConfig:])- # Optional transformation to apply before quantization setupprequantization_transform: Optional[Callable[[torch.nn.Module], None]] = None@@ -3936,34 +3787,30 @@ def _untie_input_output_embeddings(model: torch.nn.Module) -> None:Model is modified in-place."""- # 1) Persist setting in configif hasattr(model.config, "tie_word_embeddings"):model.config.tie_word_embeddings = False- # 2) Find input and output embeddingsin_emb = model.get_input_embeddings()out_proj = model.get_output_embeddings() or getattr(model, "lm_head", None)if out_proj is None:raise AttributeError("Couldn't locate output projection (lm_head).")- # (Optional) sanity: shapes should match [vocab, hidden]assert (out_proj.weight.shape == in_emb.weight.shape), f"Shape mismatch: out_proj {out_proj.weight.shape} vs in_emb {in_emb.weight.shape}"- # 3) Only clone if they are actually tied (shared storage)+ # Clone only if actually tied (shared storage), then prevent automatic re-tying and verify no+ # shared storage remains.if out_proj.weight.data_ptr() == in_emb.weight.data_ptr():with torch.no_grad():W = in_emb.weight.detach().clone()out_proj.weight = torch.nn.Parameter(W) # new storage, keeps dtype/device- # 4) Prevent future automatic re-tyingdef _no_tie(self):returnmodel.tie_weights = _no_tie.__get__(model, model.__class__)- # 5) Verify no shared storageassert out_proj.weight.data_ptr() != in_emb.weight.data_ptr(), "Embeddings still tied!"@@ -3989,7 +3836,7 @@ def _convert_torchao_model(model):for base_config, filter_fn in model._torchao_config.base_config_and_filter_fns:quantize_(model, QATConfig(base_config, step = "convert"), filter_fn = filter_fn)- # Default filter function used for quantize_+ # Default filter function used for quantize_.if filter_fn is None:if "_default" in module_to_fqn_dict:raise ValueError("Cannot use multiple default quantization configs")@@ -4034,8 +3881,8 @@ def _prepare_model_for_qat(except ImportError:raise ImportError(TORCHAO_MSG)- # Gemma3 models have issues with int8 embedding quantization due to their- # large vocabulary size (262144). Auto-switch to int4 weight-only instead.+ # Gemma3 int8 embedding quantization struggles with its 262144-token vocabulary, so auto-switch+ # to int4 weight-only.if qat_scheme == "int8-int4":model_types = get_transformers_model_type(model.config)is_gemma3 = any("gemma3" in mt or "gemma_3" in mt for mt in model_types)@@ -4129,10 +3976,8 @@ def _prepare_model_for_qat(except ImportError:raise ImportError(TORCHAO_MSG)- # IntxWeightOnlyConfig already defaults to- # `mapping_type = MappingType.SYMMETRIC`, so we intentionally do not- # import `MappingType` here. Matches the upstream Cactus runtime- # int8 / per-group-32 / symmetric weight-only configuration.+ # IntxWeightOnlyConfig already defaults to MappingType.SYMMETRIC, so MappingType is deliberately+ # not imported. Matches the upstream Cactus int8 / per-group-32 / symmetric configuration.group_size = 32base_config = IntxWeightOnlyConfig(weight_dtype = torch.int8,@@ -4143,13 +3988,8 @@ def _prepare_model_for_qat(and m.in_features >= group_sizeand m.in_features % group_size == 0)- # Warn if any Linear layer is skipped by the cactus filter because- # its in_features is not divisible by `group_size`. torchao's- # PerGroup(32) quantizer rejects non-divisible widths at- # `quantize_()` time, so the filter excludes those layers to keep- # the QAT prepare step from crashing. Surface that silently-skipped- # coverage gap to the user so they know some Linears will stay in- # full precision during training.+ # torchao's PerGroup(32) quantizer rejects in_features not divisible by group_size, so the+ # filter excludes those Linears; warn, or the gap is silent and they stay full precision.skipped_cactus_layers = [namefor name, module in model.named_modules()@@ -4178,7 +4018,6 @@ def _prepare_model_for_qat(else:torchao_config = qat_scheme- # Save Torchao metadata everywhereinner_model = modelwhile hasattr(inner_model, "model"):inner_model._torchao_config = torchao_config@@ -4194,7 +4033,7 @@ def _prepare_model_for_qat(def patch_hf_quantizer():- # To tell hf trainer that the quantized model is trainable+ # Tell the HF trainer that the quantized model is trainable.def make_trainable(self):return True@@ -4228,20 +4067,20 @@ patch_hf_quantizer()def verify_fp8_support_if_applicable(model_config):quant_method = get_quant_type(model_config)if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE != "cuda":+ # In case of block quantized, we allow L4 because we fall back to torchao kernels.raise ValueError(f"Unsloth: FP8 quantization is only supported on CUDA GPUs. You are using {DEVICE_TYPE}.")- # [TODO] Need to add FP8 support for Intel XPUsif DEVICE_TYPE == "cuda":major_version, minor_version = torch.cuda.get_device_capability()if quant_method == "fbgemm_fp8" and major_version < 9:- # While L4 does support FP8 as data type, it doesn't have fbgemm (package) support yet. So we restrict it.+ # L4 has FP8 as a data type but no fbgemm support yet, except block quantized, where torchao+ # kernels are the fallback.raise ValueError(f"Unsloth: FBGEMM FP8 quantization is only supported on H100 and higher GPUs. L4 is not supported. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details.")if quant_method == "fp8" and major_version * 10 + minor_version < 89:- # In case of block quantized, we allow L4 because we fall back to torchao kernels.raise ValueError(f"Unsloth: FP8 quantization is only supported on L4 and higher GPUs with compute capability 8.9 or higher. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details.")@@ -4285,9 +4124,6 @@ def hf_login(token: Optional[str] = None) -> Optional[str]:# =============================================-# MoE (Mixture of Experts) Detection and LoRA Utilities--def is_moe_model(model) -> bool:"""Detect if a model is a Mixture of Experts (MoE) model.@@ -4300,17 +4136,14 @@ def is_moe_model(model) -> bool:"""config = getattr(model, "config", model)- # Different MoE models use different config attribute names:- # - Qwen3-MoE: num_experts- # - GLM4-MoE: n_routed_experts, num_local_experts- # - Mixtral: num_local_experts+ # MoE configs name the count differently: num_experts (Qwen3-MoE), n_routed_experts /+ # num_local_experts (GLM4-MoE), num_local_experts (Mixtral).num_experts = Nonefor attr in ("num_experts", "n_routed_experts", "num_local_experts"):num_experts = getattr(config, attr, None)if num_experts is not None:break- # Check text_config for VL modelsif num_experts is None and hasattr(config, "text_config"):for attr in ("num_experts", "n_routed_experts", "num_local_experts"):num_experts = getattr(config.text_config, attr, None)@@ -4361,8 +4194,8 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]:return {target_modules}is_regex = re.search(r"[*+?()[\]{}|\\^$]", target_modules) is not None- # Key detection on the mlp/ffn/experts path segment (absent from an- # attention-only regex), never on q/k/v/o leaves alone.+ # Key detection on the mlp/ffn/experts path segment, absent from an attention-only regex, never+ # on q/k/v/o leaves alone.targets_mlp_path = any(tag in target_modules for tag in ("mlp", "ffn", "feed_forward", "experts"))@@ -4372,13 +4205,12 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]:named = {name for name in _MOE_BROAD_MLP_TARGETS if name in target_modules}if named:return named- # A generic projection under an mlp path (e.g. ".*mlp.*proj"): any proj- # occurrence that is not an attention leaf name.+ # A generic projection under an mlp path (".*mlp.*proj"): any proj occurrence that is not an attention leaf name.if re.search(r"(?<![qkvo]_)(?<!out_)(?<!in_)proj", target_modules):return set(_MOE_BROAD_MLP_TARGETS)- # The auto regex on fused-expert models lists only attention Linears as- # leaves; its mlp tag block is the remaining MLP-intent signal. A regex- # like "(mlp|self_attn).(q_proj|o_proj)" has neither and stays attention-only.+ # The auto regex on fused-expert models lists only attention Linears, so its mlp tag block is+ # the remaining MLP-intent signal; "(mlp|self_attn).(q_proj|o_proj)" has neither and stays+ # attention-only.if "mlp|feed_forward|ffn|dense" in target_modules:return set(_MOE_BROAD_MLP_TARGETS)@@ -4410,7 +4242,6 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[strreturn Noneconfig = getattr(model, "config", model)- # Get num_experts from various possible config attributesnum_experts = Nonefor attr in ("num_experts", "n_routed_experts", "num_local_experts"):num_experts = getattr(config, attr, None)@@ -4448,10 +4279,9 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[stralternate_name = "experts.down_proj",)- # gate_up_proj combines gate_proj and up_proj; also match the fused name directly.- # Only target a fused expert Parameter that exists: per-expert Linear layouts- # (e.g. gpt-oss bnb-4bit) have no fused Parameter and are handled by- # get_moe_target_modules, so skip them rather than pass PEFT a dead path.+ # gate_up_proj combines gate_proj and up_proj. Target only a fused expert Parameter that+ # exists: per-expert Linear layouts (gpt-oss bnb-4bit) have none and go through+ # get_moe_target_modules, so skip them rather than hand PEFT a dead path.if "gate_proj" in target_set or "up_proj" in target_set or "gate_up_proj" in target_set:if _moe_parameter_exists(model, gate_up_name):moe_params.append(gate_up_name)@@ -4512,11 +4342,9 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:if not hasattr(model, "named_modules"):return []- # Scope the returned suffixes to the requested projection leaves, matching- # get_moe_target_parameters: gate_proj/up_proj/gate_up_proj map to the fused- # gate_up ModuleList (e.g. gate_up_projs); down_proj maps to the down ModuleList- # (e.g. down_projs). A down-only (or gate/up-only) request must not pull in the- # other projection.+ # Scope the suffixes to the requested leaves, matching get_moe_target_parameters:+ # gate/up/gate_up map to the fused gate_up ModuleList and down_proj to the down one, so a+ # down-only request must not pull in the other projection.want_gate_up = bool(target_set & {"gate_proj", "up_proj", "gate_up_proj"})want_down = "down_proj" in target_set@@ -4525,10 +4353,9 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:if not isinstance(module, torch.nn.ModuleList) or len(module) == 0:continueparent, _, leaf = name.rpartition(".")- # ModuleList directly under an ``experts`` container, holding only Linear- # leaves (bnb Linear4bit / Linear8bitLt subclass nn.Linear). After PEFT has- # wrapped the experts the child is a LoRA layer whose ``base_layer`` is the- # Linear, so accept that too (keeps this idempotent across a re-wrapped model).+ # ModuleList directly under an `experts` container holding only Linear leaves (bnb Linear4bit+ # subclasses nn.Linear). After PEFT wraps the experts the child is a LoRA layer whose+ # base_layer is the Linear, so accept that too and stay idempotent.if not parent.endswith("experts"):continueif not all(@@ -4545,7 +4372,7 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:continueif is_gate_up and not want_gate_up:continue- # One entry per expert index; ``leaf.<i>`` matches expert i in every layer.+ # One entry per expert index; leaf.<i> matches expert i in every layer.for expert_index in range(len(module)):targets.add(f"{leaf}.{expert_index}")@@ -4735,10 +4562,9 @@ def _drop_tied_output_module(model, modules_to_save, ensure_weight_tying):or not EMBEDDING_MODULES <= _embedding_leaves(modules_to_save)):return modules_to_save- # Bare leaf, not the caller's spelling: peft 0.18 ties `tied_weight_keys` minus whole- # `modules_to_save` entries, so a qualified `model.embed_tokens` matches nothing there- # and it reconstructs no output module, leaving the head frozen. PEFT resolves the- # bare name by suffix on every version, so normalizing is safe.+ # Bare leaf, not the caller's spelling: peft 0.18 ties tied_weight_keys minus whole+ # modules_to_save entries, so a qualified model.embed_tokens matches nothing and no output+ # module is reconstructed, leaving the head frozen. PEFT resolves bare names by suffix.kept = [x for x in modules_to_save if _embedding_leaf(x) != "lm_head"]normalized = []for entry in kept:@@ -4748,8 +4574,8 @@ def _drop_tied_output_module(model, modules_to_save, ensure_weight_tying):elif _embedding_name_is_unambiguous(model, entry):normalized.append(leaf)else:- # Rewriting would widen the scope to another subtree's embedding. Keep- # both entries: two copies beats a silently frozen head.+ # Rewriting would widen the scope to another subtree's embedding, so keep both entries: two+ # copies beats a silently frozen head.return modules_to_savereturn normalized@@ -4798,15 +4624,15 @@ def make_fast_generate_wrapper(original_generate):@functools.wraps(original_generate)def _fast_generate_wrapper(*args, **kwargs):def _has_sampling_params(a):- # SamplingParams passed directly or inside a positional list/tuple+ # SamplingParams passed directly or inside a positional list/tuple.return type(a).__name__ == "SamplingParams" or (isinstance(a, (list, tuple))and any(type(i).__name__ == "SamplingParams" for i in a))def _is_vllm_prompt(a):- # str prompt, a vLLM prompt dict (prompt / prompt_token_ids / prompt_embeds /- # multi_modal_data), or a list/tuple of those+ # A str prompt, a vLLM prompt dict (prompt / prompt_token_ids / prompt_embeds /+ # multi_modal_data), or a list or tuple of those.head = a[0] if isinstance(a, (list, tuple)) and len(a) > 0 else areturn isinstance(head, str) or (isinstance(head, dict)@@ -4816,7 +4642,7 @@ def make_fast_generate_wrapper(original_generate):))- # vLLM-only; also catch SamplingParams passed positionally (fast_generate(prompt, params))+ # vLLM-only; also catches SamplingParams passed positionally.if "sampling_params" in kwargs or any(_has_sampling_params(a) for a in args):raise ValueError("Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). "@@ -4830,12 +4656,9 @@ def make_fast_generate_wrapper(original_generate):"Since `fast_inference=False`, LoRA weights are already merged into the model.")- # A vLLM-style prompt (string, {"prompt":..., "multi_modal_data":...} dict, or a list/tuple- # of either) only works under vLLM; tokenize first when fast_inference=False. A positional- # arg may be HF token ids, so check it conservatively with _is_vllm_prompt. The `prompts` /- # `prompt_token_ids` / `prompt_embeds` keywords are vLLM-only names that HuggingFace generate- # does not accept, so any of them being present is a vLLM-style call (even a bare token list,- # or an explicit None from a defaulted kwargs dict), hence membership rather than a value check.+ # A vLLM-style prompt only works under vLLM, so tokenize first when fast_inference=False. A+ # positional arg may be HF token ids, so check conservatively. The prompts / prompt_token_ids /+ # prompt_embeds keywords are vLLM-only names HF generate rejects, hence a membership test.vllm_prompt_kwarg = any(k in kwargs for k in ("prompts", "prompt_token_ids", "prompt_embeds"))@@ -4851,21 +4674,15 @@ def make_fast_generate_wrapper(original_generate):" output = model.fast_generate(**inputs.to('cuda'), max_new_tokens=64, temperature=1.0)")- # Call original generatereturn original_generate(*args, **kwargs)return _fast_generate_wrapper-# Fix llm_int8_skip_modules not being respected for VLMs with dynamic quantization.-# Dynamic quant checkpoints (eg gemma-3-4b-it-unsloth-bnb-4bit) encode skip paths as-# "language_model.model.layers.*", but the live module tree surfaces them as-# "model.language_model.layers.*". This prefix mismatch causes should_convert_module-# to miss the skip list, so modules meant to stay in 16-bit get wrapped in Linear4bit-# without a quant_state, producing "Skipping ... no quant_state found" warnings.-# We patch should_convert_module to expand both the module name and the skip patterns-# into all equivalent alias forms before delegating to the original matcher.-# Ref: https://github.com/unslothai/unsloth/issues/4208+# llm_int8_skip_modules was ignored for VLMs with dynamic quantization: those checkpoints spell+# skip paths "language_model.model.layers.*" while the live tree has+# "model.language_model.layers.*", so should_convert_module missed them and 16-bit modules were+# wrapped in Linear4bit with no quant_state. Expand both into all aliases first (#4208).import transformers.quantizers.quantizers_utils as _quantizers_utilsif (diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.pyindex 0ac2bed66b5..d21a4e9327d 100644--- a/unsloth/models/cohere.py+++ b/unsloth/models/cohere.py@@ -75,7 +75,7 @@ def fast_layernorm_inference(return XX.to(X.dtype)-# QK norm in Cohere+# QK norm in Cohere.def CohereAttention_fast_forward(self,hidden_states: torch.Tensor,@@ -90,7 +90,6 @@ def CohereAttention_fast_forward(*args,**kwargs,) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:- # Clear inferenceif hasattr(self, "paged_attention"):del self.paged_attention_Kdel self.paged_attention_V@@ -123,14 +122,13 @@ def CohereAttention_fast_forward(if past_key_value is not None:kv_seq_len += past_key_value[0].shape[-2]- # Extend RoPE dynamically to fit in VRAM+ # Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.if position_embeddings:cos, sin = position_embeddingselse:cos, sin = self.rotary_emb.get_cached(kv_seq_len, Q.device.index)rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")- # Useful for LongRoPEQ, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)if past_key_value is not None:@@ -138,7 +136,6 @@ def CohereAttention_fast_forward(V = torch.cat([past_key_value[1], V], dim = 2)past_key_value = (K, V) if use_cache else None- # Attention moduleuse_varlen = seq_info is not None and past_key_value is Nonebackend = select_attention_backend(use_varlen)attention_config = AttentionConfig(@@ -152,8 +149,8 @@ def CohereAttention_fast_forward("softmax_scale": getattr(self, "softmax_scale", None),},)- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse- # (KV cache / padding mask) raises. None => byte-identical default.+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /+ # padding mask) raises. None means the byte-identical default._pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)context = AttentionContext(bsz = bsz,@@ -176,7 +173,7 @@ def CohereAttention_fast_forward(return attn_output, attn_weights, past_key_value-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L590+# Ported from transformers models/llama/modeling_llama.py#L590def CohereDecoderLayer_fast_forward(self,hidden_states: torch.Tensor,@@ -191,14 +188,13 @@ def CohereDecoderLayer_fast_forward(*args,**kwargs,):- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:+ if use_cache and hasattr(self, "_flag_for_generation"):out_weight = torch.empty(self.input_layernorm.weight.shape,dtype = torch.float32,device = f"{DEVICE_TYPE_TORCH}:0",)- # Self Attentionresidual = hidden_stateshidden_states = fast_layernorm_inference(self.input_layernorm, hidden_states, out_weight)hidden_states_attention, self_attn_weights, present_key_value = self.self_attn(@@ -213,7 +209,6 @@ def CohereDecoderLayer_fast_forward(**kwargs,)- # Fully Connectedhidden_states_mlp = fast_swiglu_inference(self.mlp, hidden_states)residual += hidden_states_attentionresidual += hidden_states_mlp@@ -233,7 +228,6 @@ def CohereDecoderLayer_fast_forward(**kwargs,)- # Fully Connectedhidden_states_mlp = self.mlp(hidden_states)hidden_states = residual + hidden_states_attention + hidden_states_mlp@@ -247,7 +241,7 @@ def CohereDecoderLayer_fast_forward(from math import sqrt as math_sqrt-KV_CACHE_INCREMENT = 256 # KV Cache update size+KV_CACHE_INCREMENT = 256torch_nn_functional_softmax = torch.nn.functional.softmaxtorch_matmul = torch.matmul@@ -270,15 +264,12 @@ def CohereAttention_fast_forward_inference(n_groups = self.num_key_value_groupsn_kv_heads = self.config.num_key_value_headshead_dim = self.head_dim- # assert(n_kv_heads * n_groups == n_heads)hidden_size = self.config.hidden_sizeattention_size = n_heads * head_dimseq_len = K1.shape[-2]kv_seq_len = seq_len + 1- # Prefill phase- # if not hasattr(self, "paged_attention"):if do_prefill:self.paged_attention = torch.empty((KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),@@ -301,7 +292,7 @@ def CohereAttention_fast_forward_inference((bsz, n_heads, 1, head_dim), dtype = dtype, device = f"{DEVICE_TYPE_TORCH}:0")- # Mistral Nemo 12b has weird dimensions+ # Mistral Nemo 12b has weird dimensions.if attention_size != hidden_size:self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = f"{DEVICE_TYPE_TORCH}:0"@@ -316,7 +307,7 @@ def CohereAttention_fast_forward_inference()self.scalar = 1.0 / math_sqrt(self.head_dim)self.half_head_dim = head_dim // 2- # Cohere has QK layernorms+ # Cohere has QK layernorms.if self.use_qk_norm:self.q_norm_out_weight = torch.empty(self.q_norm.weight.shape,@@ -355,10 +346,8 @@ def CohereAttention_fast_forward_inference(Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight)Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight)- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]cos = cos[position_ids].unsqueeze(1)@@ -372,18 +361,13 @@ def CohereAttention_fast_forward_inference(Qn *= cosQn.addcmul_(RH_Q, sin)- RH_K = RH_Q[- :, :n_kv_heads, :, :- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")+ RH_K = RH_Q[:, :n_kv_heads, :, :]RH_K[:, :, :, :h] = Kn[:, :, :, h:]RH_K[:, :, :, h:] = Kn[:, :, :, :h]RH_K[:, :, :, :h].neg_()Kn *= cosKn.addcmul_(RH_K, sin)- # New KV cache- # Kn = torch.cat([K1, Kn], dim = 2)- # Vn = torch.cat([V1, Vn], dim = 2)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)@@ -393,14 +377,14 @@ def CohereAttention_fast_forward_inference(sliding_window = getattr(self.config, "sliding_window", None)if sliding_window is not None and kv_seq_len > sliding_window:start = kv_seq_len - sliding_window- Knn = Kn[:, :, start:, :] # .contiguous()- Vnn = Vn[:, :, start:, :] # .contiguous()+ Knn = Kn[:, :, start:, :]+ Vnn = Vn[:, :, start:, :]if attention_mask is not None:attention_mask = attention_mask[..., start:]else:Knn, Vnn = Kn, Vn- # Grouped query attention+ # Grouped query attention._, _, cached_len, _ = Knn.shapeif n_groups != 1:Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)@@ -408,14 +392,12 @@ def CohereAttention_fast_forward_inference(Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)- # Attentionif bsz == 1:- Qn *= (- self.scalar- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows+ Qn *= self.scalar+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805+ # (comment 2153349963).A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)A = torch_matmul(A, Vnn, out = Qn)else:A = scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = False)@@ -425,8 +407,7 @@ def CohereAttention_fast_forward_inference(return A, (Kn, Vn)-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825-# @torch.inference_mode+# Ported from transformers models/llama/modeling_llama.py#L825def CohereModel_fast_forward_inference(self,input_ids,@@ -455,7 +436,7 @@ def CohereModel_fast_forward_inference(seq_len,sliding_window = getattr(self.config, "sliding_window", None),)- # Pre-convert to bool once for all layers (avoids per-layer .eq(0))+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).if attention_mask is not None and attention_mask.dtype != torch.bool:attention_mask = attention_mask.eq(0)else:diff --git a/unsloth/models/diffusion.py b/unsloth/models/diffusion.pyindex 679046de4ac..8df1a65d1a5 100644--- a/unsloth/models/diffusion.py+++ b/unsloth/models/diffusion.py@@ -38,22 +38,22 @@ from .loader_utils import (__all__ = ["FastDiffusionModel", "DIFFUSION_MODEL_TYPES", "is_diffusion_model_type"]-# transformers model_type strings routed to this slow path+# transformers model_type strings routed to this slow path.DIFFUSION_MODEL_TYPES = ("diffusion_gemma", "diffusion_gemma4")-# Default LoRA targets: standard nn.Linear modules in the shared Gemma-4 backbone. The 128 MoE experts-# are fused 3D Parameters (gate_up_proj/down_proj), not nn.Linear, so PEFT LoRA cannot target them.+# Default LoRA targets are the standard nn.Linear modules in the shared Gemma-4 backbone: the 128+# MoE experts are fused 3D Parameters, not nn.Linear, so PEFT LoRA cannot target them.DIFFUSION_LORA_TARGETS = ["q_proj","k_proj","v_proj",- "o_proj", # attention+ "o_proj","gate_proj","up_proj",- "down_proj", # dense (non-expert) MLP+ "down_proj",]-# Vision tower uses a custom Linear with the same suffix names; exclude it so only the text path is wrapped.+# The vision tower uses a custom Linear with the same suffix names; exclude it so only the text path is wrapped.DIFFUSION_LORA_EXCLUDE = r".*(vision_tower|embed_vision).*"@@ -133,9 +133,8 @@ def _load_diffusion_config(from transformers import DiffusionGemma4Configaliased = DiffusionGemma4Config.from_dict(cd)- # Only this object knows the rewrite happened; the checkpoint on disk still says- # `diffusion_gemma`, so anything rebuilding the config from `model_name` -- the- # planner -- hits the AutoConfig failure just caught here.+ # Only this object knows the rewrite happened: the checkpoint on disk still says diffusion_gemma,+ # so anything rebuilding the config from model_name hits the AutoConfig failure just caught.aliased._unsloth_legacy_alias = Truereturn aliased@@ -197,8 +196,8 @@ class FastDiffusionModel:model_cls = _resolve_diffusion_model_class(config)- # Prefetch the whole repo root so the weight load is a cache hit. No subfolder: the pipeline- # loads every component subfolder, so narrowing would leave unet/vae/text_encoder to Xet.+ # Prefetch the whole repo root so the weight load is a cache hit. No subfolder: the pipeline loads+ # every component subfolder, so narrowing would leave unet/vae/text_encoder to Xet.maybe_prefetch_hf_snapshot(model_name,token = token,@@ -208,14 +207,14 @@ class FastDiffusionModel:fast_inference = False,force_download = kwargs.get("force_download", False),use_safetensors = kwargs.get("use_safetensors"),- # Forward variant (e.g. "fp16") so the warm keeps variant weights.+ # Forward the variant (e.g. "fp16") so the warm keeps variant weights.variant = kwargs.get("variant"),)- # Optional bitsandbytes quant. The MoE experts (3D Parameters) are not nn.Linear so bnb skips- # them; only attention + dense MLP Linears quantize, lm_head/embeddings stay full precision.- # Before the plan: the skip list becomes `modules_to_not_convert`, and sizing- # those at 4 bits while they load in compute dtype OOMs a tight map.+ # Optional bitsandbytes quant: the MoE experts (3D Parameters) are not nn.Linear so bnb skips+ # them, and lm_head/embeddings stay full precision. Before the plan, since the skip list becomes+ # modules_to_not_convert and sizing those at 4 bits while they load in compute dtype OOMs a tight+ # map.qcfg = Noneif load_in_4bit or load_in_8bit:from transformers import BitsAndBytesConfig@@ -236,12 +235,8 @@ class FastDiffusionModel:else:qcfg = BitsAndBytesConfig(load_in_8bit = True)- # Same leaf-level resolution as llama.py and vision.py: an unresolved "unsloth"- # becomes torch.device("unsloth") in transformers and raises instead of loading.- #- # A legacy `diffusion_gemma` checkpoint loads only because the config above rewrites- # the type in memory. The planner takes a name and rebuilds from the checkpoint, so- # it hits the same error and would report it as a generic planning failure.+ # Same leaf-level resolution as llama.py and vision.py: an unresolved "unsloth" becomes+ # torch.device("unsloth") in transformers and raises instead of loading.device_map = resolve_unsloth_device_map(requested_device_map(device_map),model_name,@@ -256,8 +251,8 @@ class FastDiffusionModel:device_map_planner_kwargs,kwargs,),- # This leaf popped `local_files_only` off kwargs above, so the helper gets the- # resolved value rather than the caller's raw mapping.+ # This leaf popped local_files_only off kwargs above, so the helper gets the resolved value rather+ # than the caller's raw mapping.**planner_hub_kwargs({"cache_dir": cache_dir,@@ -291,7 +286,7 @@ class FastDiffusionModel:# Match the load's weight format to the warm (None/auto already matches).if kwargs.get("use_safetensors") is not None:load_kwargs["use_safetensors"] = kwargs["use_safetensors"]- # Forward variant to the real load so it reads the warmed variant weights.+ # Forward the variant to the real load so it reads the warmed variant weights.if kwargs.get("variant") is not None:load_kwargs["variant"] = kwargs["variant"]@@ -312,7 +307,7 @@ class FastDiffusionModel:if not return_tokenizer:return model, None- # Prefer the processor (chat template + tokenizer); fall back to a bare tokenizer. Returned as+ # Prefer the processor (chat template plus tokenizer), falling back to a bare tokenizer, returned as# "tokenizer" to match the Unsloth (model, tokenizer) contract.try:tokenizer = AutoProcessor.from_pretrained(@@ -354,10 +349,8 @@ class FastDiffusionModel:if target_modules is None:target_modules = DIFFUSION_LORA_TARGETS- # NOTE: use_dora (and any other LoraConfig kwarg outside this allowlist,- # e.g. use_rslora) is silently dropped here. Unsloth does not reach this- # path today, so it's untested/unsupported on diffusion models; add it- # to the allowlist below if that changes.+ # use_dora, and any other LoraConfig kwarg outside this allowlist, is silently dropped: Unsloth does+ # not reach this path today, so it is untested on diffusion models.lora_kwargs = dict(r = r,lora_alpha = lora_alpha,diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.pyindex d0c9e09b23f..f50eaabb329 100644--- a/unsloth/models/falcon_h1.py+++ b/unsloth/models/falcon_h1.py@@ -84,7 +84,6 @@ def FalconH1Attention_fast_forward(*args,**kwargs,) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:- # Clear inferenceif hasattr(self, "paged_attention"):del self.paged_attention_Kdel self.paged_attention_V@@ -108,7 +107,7 @@ def FalconH1Attention_fast_forward(V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)seq_info = get_packed_info_from_kwargs(kwargs, hidden_states.device)- # Falcon H1 multiplies key states by a multiplier+ # Falcon H1 multiplies key states by a multiplier.K = K * self.config.key_multiplierQ = Q.transpose(1, 2)@@ -118,7 +117,7 @@ def FalconH1Attention_fast_forward(if past_key_value is not None:kv_seq_len += past_key_value[0].shape[-2]- # Extend RoPE dynamically to fit in VRAM+ # Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.if position_embeddings and kv_seq_len <= position_embeddings[0].shape[0]:cos, sin = position_embeddingselse:@@ -135,7 +134,6 @@ def FalconH1Attention_fast_forward(V = torch.cat([past_key_value[1], V], dim = 2)past_key_value = (K, V) if use_cache else None- # Attention modulewindow = (-1, -1)use_varlen = (attention_mask is None@@ -229,15 +227,12 @@ def FalconH1Attention_fast_forward_inference(n_groups = self.num_key_value_groupsn_kv_heads = self.config.num_key_value_headshead_dim = self.head_dim- # assert(n_kv_heads * n_groups == n_heads)hidden_size = self.config.hidden_sizeattention_size = n_heads * head_dimseq_len = K1.shape[-2]kv_seq_len = seq_len + 1- # Prefill phase- # if not hasattr(self, "paged_attention"):device = hidden_states.deviceif do_prefill:self.paged_attention = torch.empty(@@ -253,7 +248,7 @@ def FalconH1Attention_fast_forward_inference(self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)- # Mistral Nemo 12b has weird dimensions+ # Mistral Nemo 12b has weird dimensions.if attention_size != hidden_size:self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)else:@@ -282,25 +277,17 @@ def FalconH1Attention_fast_forward_inference(Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0])Kn.mul_(self.config.key_multiplier)Vn = fast_linear_forward(self.v_proj, Xn, out = self.temp_KV[1])- Qn = Qn.view(- bsz, 1, n_heads, head_dim- ) # .transpose(1, 2) # we will transpose after normalisation- Kn = Kn.view(- bsz, 1, n_kv_heads, head_dim- ) # .transpose(1, 2) # we will transpose after normalisation+ Qn = Qn.view(bsz, 1, n_heads, head_dim)+ Kn = Kn.view(bsz, 1, n_kv_heads, head_dim)Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)Qn = Qn.transpose(1, 2)Kn = Kn.transpose(1, 2)- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)-- # Need to do it prior 2 steps before hitting full on short KV cache- # or else error+ # Must happen 2 steps before hitting full on a short KV cache, or it errors.self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]cos = cos[position_ids].unsqueeze(1)@@ -310,22 +297,17 @@ def FalconH1Attention_fast_forward_inference(RH_Q = self.RH_QRH_Q[:, :, :, :h] = Qn[:, :, :, h:]RH_Q[:, :, :, h:] = Qn[:, :, :, :h]- RH_Q[:, :, :, :h].neg_() # torch.neg(RH_Q[:,:,:,:h], out = RH_Q[:,:,:,:h])+ RH_Q[:, :, :, :h].neg_()Qn *= cosQn.addcmul_(RH_Q, sin)- RH_K = RH_Q[- :, :n_kv_heads, :, :- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")+ RH_K = RH_Q[:, :n_kv_heads, :, :]RH_K[:, :, :, :h] = Kn[:, :, :, h:]RH_K[:, :, :, h:] = Kn[:, :, :, :h]- RH_K[:, :, :, :h].neg_() # torch.neg(RH_K[:,:,:,:h], out = RH_K[:,:,:,:h])+ RH_K[:, :, :, :h].neg_()Kn *= cosKn.addcmul_(RH_K, sin)- # New KV cache- # Kn = torch.cat([K1, Kn], dim = 2)- # Vn = torch.cat([V1, Vn], dim = 2)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)@@ -335,14 +317,14 @@ def FalconH1Attention_fast_forward_inference(sliding_window = getattr(self.config, "sliding_window", None)if sliding_window is not None and kv_seq_len > sliding_window:start = kv_seq_len - sliding_window- Knn = Kn[:, :, start:, :] # .contiguous()- Vnn = Vn[:, :, start:, :] # .contiguous()+ Knn = Kn[:, :, start:, :]+ Vnn = Vn[:, :, start:, :]if attention_mask is not None:attention_mask = attention_mask[..., start:]else:Knn, Vnn = Kn, Vn- # Grouped query attention+ # Grouped query attention._, _, cached_len, _ = Knn.shapeif bsz == 1 or not SDPA_HAS_GQA and n_groups != 1:Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)@@ -350,14 +332,12 @@ def FalconH1Attention_fast_forward_inference(Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)- # Attentionif bsz == 1:- Qn *= (- self.scalar- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows+ Qn *= self.scalar+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805+ # (comment 2153349963).A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)A = torch_matmul(A, Vnn, out = Qn)else:if SDPA_HAS_GQA:@@ -374,7 +354,7 @@ def FalconH1Attention_fast_forward_inference(return A, (Kn, Vn)-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/falcon_h1/modeling_falcon_h1.py+# Ported from transformers models/falcon_h1/modeling_falcon_h1.pydef FalconH1DecoderLayer_fast_forward(self,hidden_states: torch.Tensor,@@ -433,7 +413,6 @@ def FalconH1DecoderLayer_fast_forward(hidden_states += residual- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm_inference(self.pre_ff_layernorm, hidden_states)hidden_states = fast_swiglu_inference(self.feed_forward, hidden_states)@@ -466,10 +445,9 @@ def FalconH1DecoderLayer_fast_forward(hidden_states = mamba_hidden_states + attention_hidden_states- # residual connection after attention + Mamba+ # Residual connection after attention + Mamba.hidden_states = residual + hidden_states- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm(self.pre_ff_layernorm, hidden_states)hidden_states = self.feed_forward(hidden_states)@@ -487,7 +465,7 @@ def _FalconH1_fast_forward_inference(attention_fast_forward_inference = FalconH1Attention_fast_forward_inference,mlp_fast_forward_inference = fast_swiglu_inference,):- # Makes attention and MLP customisable (e.g. qwen3/cohere custom attention).+ # Makes attention and MLP customisable (qwen3/cohere custom attention).def FalconH1Model_fast_forward_inference_custom(self,input_ids,@@ -537,7 +515,7 @@ def _FalconH1_fast_forward_inference(next_decoder_cache = []for idx, decoder_layer in enumerate(self.model.layers):- residual.copy_(X) # residual = X+ residual.copy_(X)X = fast_rms_layernorm_inference(decoder_layer.input_layernorm,X,@@ -565,7 +543,7 @@ def _FalconH1_fast_forward_inference(X += residual- residual.copy_(X) # residual = X+ residual.copy_(X)X = fast_rms_layernorm_inference(decoder_layer.pre_ff_layernorm,X,@@ -602,7 +580,8 @@ def _FalconH1_fast_forward_inference(return FalconH1Model_fast_forward_inference_custom-# Separate prepare_inputs_for_generation for Hybrid FalconH1+# Separate prepare_inputs_for_generation for Hybrid FalconH1, which has its own cache type+# FalconHybridMambaAttentionDynamicCache.def _fast_prepare_inputs_for_generation(self,input_ids,@@ -617,11 +596,9 @@ def _fast_prepare_inputs_for_generation(# Overwritten -- has a unique cache type, `FalconHybridMambaAttentionDynamicCache`empty_past_kv = past_key_values is None- # If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens- # Exception 1: when passing input_embeds, input_ids may be missing entries- # Exception 2: some generation methods do special slicing of input_ids, so we don't need to do it here- # Exception 3: with synced GPUs cache_position may go out of bounds, but we only want dummy token in that case.- # (we can't check exception 3 while compiling)+ # Slice input_ids through cache_position to keep only unprocessed tokens, except: with+ # input_embeds input_ids may be missing entries; some generation methods slice input_ids+ # themselves; and with synced GPUs cache_position may go out of bounds.if not empty_past_kv:if (inputs_embeds is not None # Exception 1@@ -634,16 +611,6 @@ def _fast_prepare_inputs_for_generation(input_ids.shape[1] != cache_position.shape[0]): # Default case (the "else", a no op, is Exception 2)input_ids = input_ids[:, cache_position]- # TODO: Wire up Cache to work for inference.- # else:- # past_key_values = FalconHybridMambaAttentionDynamicCache(- # self.config,- # input_ids.shape[0],- # self.dtype,- # devices=[- # self.model.layers[i].mamba.conv1d.weight.device for i in range(self.config.num_hidden_layers)- # ],- # )if attention_mask is not None and position_ids is None:# create position_ids on the fly for batch generation@@ -652,7 +619,7 @@ def _fast_prepare_inputs_for_generation(if not empty_past_kv:position_ids = position_ids[:, -input_ids.shape[1] :]- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step+ # inputs_embeds are only used in the 1st generation step.if inputs_embeds is not None and empty_past_kv:model_inputs = {"inputs_embeds": inputs_embeds}else:@@ -712,17 +679,14 @@ class FastFalconH1Model(FastLlamaModel):PeftModelForCausalLM.forward = PeftModel_fast_forwardfix_prepare_inputs_for_generation(FalconH1ForCausalLM)- # Solves https://github.com/unslothai/unsloth/issues/168- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.- # https://github.com/huggingface/transformers/pull/27931- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,+ # huggingface/transformers#27931), so the old rotary embeddings are retained.import transformers.models.falcon_h1.modeling_falcon_h1transformers.models.falcon_h1.modeling_falcon_h1.FalconH1RotaryEmbedding = (LlamaRotaryEmbedding)- # Avoids the float64 RMSNorm compile kernel that fails on Intel Arc DG2 (issue #6555).+ # Avoids the float64 RMSNorm compile kernel that fails on Intel Arc DG2 (#6555).patch_falcon_h1_rms_layernorm()returndiff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.pyindex df05176e0ba..05725726c38 100644--- a/unsloth/models/gemma.py+++ b/unsloth/models/gemma.py@@ -63,23 +63,18 @@ torch_nn_functional_gelu = torch.nn.functional.geludef fast_geglu_inference(self, X):- # gate = self.gate_proj(X)- # up = self.up_proj(X)bsz, _, hd = X.shape- # mlp_size = self.config.intermediate_size- # temp = torch.empty((2, bsz, 1, mlp_size), dtype = X.dtype, device = "cuda:0")- gate = fast_linear_forward(self.gate_proj, X) # , out = temp[0])- up = fast_linear_forward(self.up_proj, X) # , out = temp[1])+ gate = fast_linear_forward(self.gate_proj, X)+ up = fast_linear_forward(self.up_proj, X)gate = torch_nn_functional_gelu(gate, approximate = "tanh")gate *= up- # X = self.down_proj(gate)down = fast_linear_forward(self.down_proj, gate, out = up[:, :, :hd])return down-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L590+# Ported from transformers models/llama/modeling_llama.py#L590def GemmaDecoderLayer_fast_forward(self,hidden_states: torch.Tensor,@@ -93,14 +88,13 @@ def GemmaDecoderLayer_fast_forward(*args,**kwargs,):- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:+ if use_cache and hasattr(self, "_flag_for_generation"):out_weight = torch.empty(self.input_layernorm.weight.shape,dtype = torch.float32,device = f"{DEVICE_TYPE_TORCH}:0",)- # Self Attentionresidual = hidden_stateshidden_states = fast_rms_layernorm_inference_gemma(self.input_layernorm, hidden_states, out_weight@@ -118,7 +112,6 @@ def GemmaDecoderLayer_fast_forward()hidden_states += residual- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm_inference_gemma(self.post_attention_layernorm, hidden_states, out_weight@@ -141,7 +134,6 @@ def GemmaDecoderLayer_fast_forward()hidden_states = residual + hidden_states- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states, gemma = True)hidden_states = self.mlp(hidden_states)@@ -158,8 +150,7 @@ def GemmaDecoderLayer_fast_forward(from math import sqrt as math_sqrt-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825-# @torch.inference_mode+# Ported from transformers models/llama/modeling_llama.py#L825def GemmaModel_fast_forward_inference(self,input_ids,@@ -179,8 +170,7 @@ def GemmaModel_fast_forward_inference(input_ids = input_ids[:, : self.max_seq_length]hidden_states = self.model.embed_tokens(input_ids)hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config)))- # 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32- # 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32+ # 3072**0.5 is 55.5000 in bfloat16 against 55.4256 in float32, and 2048**0.5 is 45.2500 against 45.2548.hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype)bsz, q_len, hd = hidden_states.shape@@ -193,11 +183,11 @@ def GemmaModel_fast_forward_inference(hidden_states,seq_len,)- # Pre-convert to bool once for all layers (avoids per-layer .eq(0))+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).if attention_mask is not None and attention_mask.dtype != torch.bool:attention_mask = attention_mask.eq(0)- # Compute rotary_seq_len once to avoid per-layer GPU-CPU sync from .item()+ # Compute rotary_seq_len once to avoid a per-layer GPU-CPU sync from .item().rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)next_decoder_cache = []@@ -242,27 +232,24 @@ def GemmaModel_fast_forward_inference()-# Follows line by line https://github.com/google-deepmind/gemma/blob/main/gemma/positional_embeddings.py#L45-# Formulates cos and sin differently from Llama!+# Follows google-deepmind/gemma positional_embeddings.py#L45 line by line, which formulates cos and+# sin differently from Llama.class GemmaFixedRotaryEmbedding(torch.nn.Module):- # Fixes https://github.com/huggingface/transformers/pull/28837- # https://github.com/microsoft/DeepSpeed/issues/4932- # The precision of RoPE buffers is not correct, so we cast to int64.+ # RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.def __init__(self,dim = None,max_position_embeddings = 2048,base = 10000,device = None,- config = None, # [TODO] Hack to pass in config - need to remove later+ config = None,):super().__init__()- # In transformers 5.0+, RotaryEmbedding(config) passes config as first positional arg (dim)+ # In transformers 5.0+, RotaryEmbedding(config) passes config as the first positional arg (dim).if config is None and dim is not None and hasattr(dim, "max_position_embeddings"):config = dimdim = Noneif config is not None:- # [TODO] Hack to pass in config - need to remove laterbase = _get_rope_theta(config, default = base)partial_rotary_factor = (config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0@@ -275,7 +262,7 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):self.dim = dimself.max_position_embeddings = max_position_embeddingsself.base = base- # Dynamic RoPE we first set it to a max of 4 * 8192 tokens then we iteratively grow this+ # Dynamic RoPE starts at a max of 4 * 8192 tokens and grows iteratively in increments of 8192.self.current_rope_size = min(4 * 8192, self.max_position_embeddings)self.multi_gpu_cos_cached = [None] * DEVICE_COUNTself.multi_gpu_sin_cached = [None] * DEVICE_COUNT@@ -288,7 +275,6 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):dtype = torch.get_default_dtype(),)- # dummy so that patch_utils doesn't fail for nowself.cos_cached = torch.empty(1, device = get_current_device(), dtype = torch.get_default_dtype())@@ -297,8 +283,8 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):)def _set_cos_sin_cache(self, seq_len, device, dtype):- # Note: on the original Llama codebase, these tensors are created on the target device (and not on CPU) and- # in FP32. They are applied (multiplied) in FP32 as well.+ # The original Llama codebase creates these on the target device in FP32 and multiplies in FP32; the+ # difference here is explicit division t/x rather than t * (1/x).self.current_rope_size = seq_len# The difference is we do division explicitly instead of t * (1/x) ie we do t/x.@@ -311,9 +297,9 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):radians_new = radians_new.squeeze(0)emb = torch.cat((radians_new, radians_new), dim = -1)- # We must do RoPE in float32!- cos = emb.cos().to(device = device, non_blocking = True) # , dtype = dtype)- sin = emb.sin().to(device = device, non_blocking = True) # , dtype = dtype)+ # RoPE must be done in float32.+ cos = emb.cos().to(device = device, non_blocking = True)+ sin = emb.sin().to(device = device, non_blocking = True)self.multi_gpu_cos_cached[device.index] = cosself.multi_gpu_sin_cached[device.index] = sinreturn cos, sin@@ -324,7 +310,6 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):position_ids = None,seq_len = None,):- # x: [bs, num_attention_heads, seq_len, head_size]if seq_len is not None and seq_len > self.current_rope_size:self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)@@ -358,9 +343,7 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):"""LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""- # Fixes https://github.com/huggingface/transformers/pull/28837- # https://github.com/microsoft/DeepSpeed/issues/4932- # The precision of RoPE buffers is not correct, so we cast to int64.+ # RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.def __init__(self,dim = None,@@ -368,7 +351,7 @@ class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):base = 10000,device = None,scaling_factor = 1.0,- config = None, # [TODO] Hack to pass in config - need to remove later+ config = None,):self.scaling_factor = scaling_factorsuper().__init__(@@ -380,8 +363,8 @@ class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):)def _set_cos_sin_cache(self, seq_len, device, dtype):- # Note: on the original Llama codebase, these tensors are created on the target device (and not on CPU) and- # in FP32. They are applied (multiplied) in FP32 as well.+ # The original Llama codebase creates these on the target device in FP32 and multiplies in FP32; the+ # difference here is explicit division t/x rather than t * (1/x).self.current_rope_size = seq_len# The difference is we do division explicitly instead of t * (1/x) ie we do t/x.@@ -395,9 +378,9 @@ class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):radians_new = radians_new.squeeze(0)emb = torch.cat((radians_new, radians_new), dim = -1)- # We must do RoPE in float32!- cos = emb.cos().to(device = device, non_blocking = True) # , dtype = dtype)- sin = emb.sin().to(device = device, non_blocking = True) # , dtype = dtype)+ # RoPE must be done in float32.+ cos = emb.cos().to(device = device, non_blocking = True)+ sin = emb.sin().to(device = device, non_blocking = True)self.multi_gpu_cos_cached[device.index] = cosself.multi_gpu_sin_cached[device.index] = sinreturn cos, sin@@ -424,11 +407,8 @@ class FastGemmaModel(FastLlamaModel):PeftModelForCausalLM.forward = PeftModel_fast_forwardfix_prepare_inputs_for_generation(GemmaForCausalLM)- # Solves https://github.com/unslothai/unsloth/issues/168- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.- # https://github.com/huggingface/transformers/pull/27931- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,+ # huggingface/transformers#27931), so the old rotary embeddings are retained.import transformers.models.gemma.modeling_gemmatransformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding = GemmaFixedRotaryEmbedding@@ -440,18 +420,15 @@ class FastGemmaModel(FastLlamaModel):tokenizer,correct_dtype = None,):- # Gemma does not downcast RoPE+ # Gemma does not downcast RoPE.model, tokenizer = patch_model_and_tokenizer(model, tokenizer, downcast_rope = False, correct_dtype = correct_dtype)- # Add 1 to weight- # return output * (1 + self.weight)- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/gemma/modeling_gemma.py#L89+ # Gemma returns output * (1 + self.weight); see transformers models/gemma/modeling_gemma.py#L89.from transformers.models.gemma.modeling_gemma import GemmaRMSNorm- # Freeze all parameters except LoRA- # We do this first since += 1 seems to not be liked by requires_grad = True+ # Freeze all parameters except LoRA first, since += 1 does not agree with requires_grad = True.for name, param in model.named_parameters():if ".lora_A." in name or ".lora_B." in name:param.requires_grad_(True)@@ -461,15 +438,11 @@ class FastGemmaModel(FastLlamaModel):# Patch RMS Layernormfor name, module in model.named_modules():if isinstance(module, GemmaRMSNorm):- # Must be in float32- # https://github.com/keras-team/keras-nlp/blob/v0.8.2/keras_nlp/models/gemma/rms_normalization.py#L36- # module = module.to(torch.float32)- # Leave + 1 to Triton kernel itself- # module.weight += 1.0 # return output * (1 + self.weight)+ # Must be in float32 (keras-nlp gemma/rms_normalization.py#L36); the +1 is left to the Triton kernel+ # itself.if not hasattr(module, "variance_epsilon"):module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon- # Clear deleted GPU itemsimport gcfor _ in range(3):diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex 00849b62afc..2d1ab52bc91 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -69,7 +69,7 @@ if HAS_FLASH_ATTENTION_SOFTCAPPING:from flash_attn import flash_attn_func-# Logit softcapping+# Logit softcapping.def Gemma2Attention_fast_forward(self,hidden_states: torch.Tensor,@@ -83,7 +83,6 @@ def Gemma2Attention_fast_forward(*args,**kwargs,) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:- # Clear inferenceif hasattr(self, "paged_attention"):del self.paged_attention_Kdel self.paged_attention_V@@ -117,7 +116,7 @@ def Gemma2Attention_fast_forward(rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")if rope_position_ids is not None:- # Useful for LongRoPE+ # Useful for LongRoPE.cos_var, sin_var = self.rotary_emb.get_cached(kv_seq_len, device_index)Q, K = fast_rope_embedding(Q, K, cos_var, sin_var, rope_position_ids)else:@@ -128,7 +127,7 @@ def Gemma2Attention_fast_forward(V = torch.cat([past_key_value[1], V], dim = 2)past_key_value = (K, V) if use_cache else None- # Only enable if the attention_mask is True+ # Only enable if the attention_mask is True.use_sliding_window = kwargs.get("use_sliding_window")has_sliding_window = (use_sliding_window@@ -169,10 +168,8 @@ def Gemma2Attention_fast_forward(},)- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse- # (KV cache / padding mask) raises. None => byte-identical default. gemma2 is- # sliding-window and softcapped: the engage gate caps spans at the window and- # excludes softcap models entirely, so PG never engages here.+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /+ # padding mask) raises. None means the byte-identical default._pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)context = AttentionContext(bsz = bsz,@@ -201,7 +198,7 @@ def Gemma2Attention_fast_forward(return A, None, past_key_value-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L590+# Ported from transformers models/llama/modeling_llama.py#L590def Gemma2DecoderLayer_fast_forward(self,hidden_states: torch.Tensor,@@ -215,14 +212,13 @@ def Gemma2DecoderLayer_fast_forward(*args,**kwargs,):- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:+ if use_cache and hasattr(self, "_flag_for_generation"):out_weight = torch.empty(self.input_layernorm.weight.shape,dtype = torch.float32,device = f"{DEVICE_TYPE_TORCH}:0",)- # Self Attentionresidual = hidden_stateshidden_states = fast_rms_layernorm_inference_gemma(self.input_layernorm, hidden_states, out_weight@@ -244,7 +240,6 @@ def Gemma2DecoderLayer_fast_forward()hidden_states += residual- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm_inference_gemma(self.pre_feedforward_layernorm, hidden_states, out_weight@@ -271,7 +266,6 @@ def Gemma2DecoderLayer_fast_forward(hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states, gemma = True)hidden_states = residual + hidden_states- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm(self.pre_feedforward_layernorm, hidden_states, gemma = True@@ -292,7 +286,7 @@ def Gemma2DecoderLayer_fast_forward(from math import sqrt as math_sqrt-KV_CACHE_INCREMENT = 256 # KV Cache update size+KV_CACHE_INCREMENT = 256torch_nn_functional_softmax = torch.nn.functional.softmaxtorch_matmul = torch.matmultorch_tanh = torch.tanh@@ -317,7 +311,6 @@ def Gemma2Attention_fast_forward_inference(n_groups = self.num_key_value_groupsn_kv_heads = self.config.num_key_value_headshead_dim = self.head_dim- # assert(n_kv_heads * n_groups == n_heads)hidden_size = self.config.hidden_sizeattention_size = n_heads * head_dim@@ -325,8 +318,6 @@ def Gemma2Attention_fast_forward_inference(kv_seq_len = seq_len + 1device = hidden_states.device- # Prefill phase- # if not hasattr(self, "paged_attention"):if do_prefill:self.paged_attention = torch.empty((KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),@@ -340,16 +331,14 @@ def Gemma2Attention_fast_forward_inference(self.temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)- # Only for Gemma2+ # Only for Gemma2.self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)self.attention = torch.empty((bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device)- # See https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e- # Gemma 9b should use 256 and not 224 (hs / nah). 27b uses the below- # We default to using the config file itself- # s = self.config.hidden_size // self.config.num_attention_heads+ # Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,+ # so default to the config. See google/gemma_pytorch commit 03e6575.self.scalar = 1.0 / math_sqrt(self.config.query_pre_attn_scalar)# self.scalar = 1.0 / math_sqrt(self.config.hidden_size // self.config.num_attention_heads)self.half_head_dim = head_dim // 2@@ -376,10 +365,8 @@ def Gemma2Attention_fast_forward_inference(Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]cos = cos[position_ids].unsqueeze(1)@@ -393,18 +380,13 @@ def Gemma2Attention_fast_forward_inference(Qn *= cosQn.addcmul_(RH_Q, sin)- RH_K = RH_Q[- :, :n_kv_heads, :, :- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")+ RH_K = RH_Q[:, :n_kv_heads, :, :]RH_K[:, :, :, :h] = Kn[:, :, :, h:]RH_K[:, :, :, h:] = Kn[:, :, :, :h]RH_K[:, :, :, :h].neg_()Kn *= cosKn.addcmul_(RH_K, sin)- # New KV cache- # Kn = torch.cat([K1, Kn], dim = 2)- # Vn = torch.cat([V1, Vn], dim = 2)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)@@ -414,12 +396,12 @@ def Gemma2Attention_fast_forward_inference(sliding_window = self.config.sliding_windowif use_sliding_window and kv_seq_len > sliding_window:start = kv_seq_len - sliding_window- Knn = Kn[:, :, start:, :] # .contiguous()- Vnn = Vn[:, :, start:, :] # .contiguous()+ Knn = Kn[:, :, start:, :]+ Vnn = Vn[:, :, start:, :]else:Knn, Vnn = Kn, Vn- # Grouped query attention+ # Grouped query attention._, _, cached_len, _ = Knn.shapeif n_groups != 1:Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)@@ -427,29 +409,25 @@ def Gemma2Attention_fast_forward_inference(Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)- # Attention- # [TODO] Gemma2 uses manual matmul for all batch sizes since SDPA lacks- # softcapping (tanh logit scaling). If PyTorch adds a softcap param to- # SDPA, consider SDPA for bsz > 1 to match the llama/qwen3 pattern.- Qn *= (- self.scalar- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows+ # Gemma2 uses manual matmul for all batch sizes since SDPA lacks softcapping (tanh logit scaling);+ # if PyTorch adds a softcap param, consider SDPA for bsz > 1 to match llama/qwen3.+ Qn *= self.scalar+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805 (comment 2153349963).A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])- # Softcapping must happen BEFORE the mask is applied.- # Reference: google-deepmind/gemma _modules.py and transformers gemma2 eager_attention_forward+ # Softcapping must happen BEFORE the mask is applied; see google-deepmind/gemma _modules.py and+ # transformers gemma2 eager_attention_forward.A *= self.reciprocal_tA.tanh_()- A *= self.t # Logit softcapping+ A *= self.tif attention_mask is not None and isinstance(attention_mask, torch.Tensor):- # Slice mask to match K/V when sliding window is active+ # Slice mask to match K/V when sliding window is active.if attention_mask.shape[-1] != A.shape[-1]:attention_mask = attention_mask[:, :, :, -A.shape[-1] :]A += attention_mask- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)A = torch_matmul(A, Vnn, out = Qn)A = A.transpose(1, 2)A = A.reshape(bsz, 1, attention_size)@@ -457,8 +435,7 @@ def Gemma2Attention_fast_forward_inference(return A, (Kn, Vn)-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825-# @torch.inference_mode+# Ported from transformers models/llama/modeling_llama.py#L825def Gemma2Model_fast_forward_inference(self,input_ids,@@ -478,8 +455,7 @@ def Gemma2Model_fast_forward_inference(input_ids = input_ids[:, : self.max_seq_length]hidden_states = self.model.embed_tokens(input_ids)hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config)))- # 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32- # 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32+ # 3072**0.5 is 55.5000 in bfloat16 against 55.4256 in float32, and 2048**0.5 is 45.2500 against 45.2548.hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype)bsz, q_len, hd = hidden_states.shape@@ -507,8 +483,7 @@ def Gemma2Model_fast_forward_inference(GA = attention_masknext_decoder_cache = []for idx, decoder_layer in enumerate(self.model.layers):- # For pipeline parallelism, we need to move all tensors to the same device- # note that this movement is once per GPU in PP+ # For pipeline parallelism every tensor must be on the same device; this movement happens once per GPU in PP.device_index = getattr(decoder_layer, "_per_layer_device_index", 0)hidden_states, position_ids = move_to_device(device_index, hidden_states, position_ids)@@ -582,11 +557,8 @@ class FastGemma2Model(FastLlamaModel):PeftModelForCausalLM.forward = PeftModel_fast_forwardfix_prepare_inputs_for_generation(Gemma2ForCausalLM)- # Solves https://github.com/unslothai/unsloth/issues/168- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.- # https://github.com/huggingface/transformers/pull/27931- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,+ # huggingface/transformers#27931), so the old rotary embeddings are retained.import transformers.models.gemma2.modeling_gemma2transformers.models.gemma2.modeling_gemma2.Gemma2RotaryEmbedding = GemmaFixedRotaryEmbedding@@ -598,36 +570,28 @@ class FastGemma2Model(FastLlamaModel):tokenizer,correct_dtype = None,):- # Gemma does not downcast RoPE+ # Gemma does not downcast RoPE.model, tokenizer = patch_model_and_tokenizer(model, tokenizer, downcast_rope = False, correct_dtype = correct_dtype)- # Add 1 to weight- # return output * (1 + self.weight)- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/gemma/modeling_gemma.py#L89+ # Gemma returns output * (1 + self.weight); see transformers models/gemma/modeling_gemma.py#L89.from transformers.models.gemma2.modeling_gemma2 import Gemma2RMSNorm- # Freeze all parameters except LoRA- # We do this first since += 1 seems to not be liked by requires_grad = True+ # Freeze all parameters except LoRA first, since += 1 does not agree with requires_grad = True.for name, param in model.named_parameters():if ".lora_A." in name or ".lora_B." in name:param.requires_grad_(True)else:param.requires_grad_(False)- # Patch RMS Layernormfor name, module in model.named_modules():if isinstance(module, Gemma2RMSNorm):- # Must be in float32- # https://github.com/keras-team/keras-nlp/blob/v0.8.2/keras_nlp/models/gemma/rms_normalization.py#L36- # module = module.to(torch.float32)- # Leave + 1 to Triton kernel itself- # module.weight += 1.0 # return output * (1 + self.weight)+ # Must be in float32 (keras-nlp gemma/rms_normalization.py#L36); the +1 is left to the Triton kernel+ # itself.if not hasattr(module, "variance_epsilon"):module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon- # Clear deleted GPU itemsimport gcfor _ in range(3):diff --git a/unsloth/models/glm4_moe.py b/unsloth/models/glm4_moe.pyindex 503afbce4b6..146d3b03f9c 100644--- a/unsloth/models/glm4_moe.py+++ b/unsloth/models/glm4_moe.py@@ -43,7 +43,7 @@ import torch.nn.functional as Ffrom typing import Optional, Tuplefrom ..kernels import fast_rms_layernorm-# grouped_gemm expects its parent dir on sys.path+# grouped_gemm expects its parent dir on sys.path.HAS_GROUPED_GEMM = Falsetry:import sys@@ -55,8 +55,8 @@ try:if _moe_path not in sys.path:sys.path.insert(0, _moe_path)- # Import first to apply the TMA compatibility shim (patches triton.language- # for both old and new TMA API names)+ # Import first to apply the TMA compatibility shim, which patches triton.language for both old and+ # new TMA API names.import grouped_gemm # noqa: F401 - triggers TMA compatibility shimfrom grouped_gemm.interface import grouped_gemm@@ -72,7 +72,6 @@ except ImportError as e:warnings.warn(f"Grouped GEMM not available: {e}. MoE will use fallback implementation.")-# Import transformers GLM4 MoE Lite classestry:from transformers.models.glm4_moe_lite.modeling_glm4_moe_lite import (Glm4MoeLiteAttention,@@ -89,7 +88,7 @@ try:except ImportError:HAS_GLM4_MOE = False- # Create dummy classes for type checking+ # Dummy classes for type checking.class Glm4MoeLiteAttention:pass@@ -137,7 +136,7 @@ def Glm4MoeLiteMoE_fast_forward(self, hidden_states):router_logits = self.gate(hidden_states) # [num_tokens, n_routed_experts]topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)- # Sigmoid router returns fp32; cast weights to hidden_states dtype (e.g. bf16)+ # The sigmoid router returns fp32; cast weights to the hidden_states dtype.topk_weights = topk_weights.to(hidden_states.dtype)with torch.no_grad():@@ -146,7 +145,7 @@ def Glm4MoeLiteMoE_fast_forward(self, hidden_states):)if HAS_GROUPED_GEMM:- # Under autocast hidden_states may be fp32 while weights are bf16+ # Under autocast hidden_states may be fp32 while weights are bf16.hidden_states = hidden_states.to(self.experts.gate_up_proj.dtype)# gate_up_proj: [num_tokens, hidden_dim] -> [total_tokens, 2*intermediate_dim]@@ -162,7 +161,7 @@ def Glm4MoeLiteMoE_fast_forward(self, hidden_states):is_first_gemm = True,)- # Activation: SiLU(gate) * up+ # SiLU(gate) * up.gate, up = intermediate.chunk(2, dim = -1)intermediate = torch_nn_functional_silu(gate) * up@@ -179,7 +178,7 @@ def Glm4MoeLiteMoE_fast_forward(self, hidden_states):is_first_gemm = False,)- # Merge topk weights: [num_tokens, top_k, hidden_dim] -> [num_tokens, hidden_dim]+ # Merge topk weights: [num_tokens, top_k, hidden_dim] -> [num_tokens, hidden_dim].hidden_states = (expert_output.view(num_tokens, self.top_k, hidden_dim) * topk_weights.unsqueeze(-1)).sum(dim = 1)@@ -242,10 +241,9 @@ def Glm4MoeLiteNaiveMoe_fast_forward(with torch.no_grad():token_counts_by_expert, gather_indices = get_routing_indices(top_k_index, self.num_experts)- # Under autocast hidden_states may be fp32 while weights are bf16+ # Under autocast hidden_states may be fp32 while weights are bf16.hidden_states = hidden_states.to(self.gate_up_proj.dtype)- # First grouped GEMM: gate_up_projintermediate = grouped_gemm(X = hidden_states,W = self.gate_up_proj,@@ -262,7 +260,6 @@ def Glm4MoeLiteNaiveMoe_fast_forward(gate, up = intermediate.chunk(2, dim = -1)intermediate = self.act_fn(gate) * up- # Second grouped GEMM: down_projexpert_output = grouped_gemm(X = intermediate,W = self.down_proj,@@ -315,13 +312,11 @@ def Glm4MoeLiteDecoderLayer_fast_forward()hidden_states = residual + hidden_states- # MLP/MoEresidual = hidden_stateshidden_states = fast_rms_layernorm_inference(self.post_attention_layernorm, hidden_states)hidden_states = self.mlp(hidden_states)hidden_states = residual + hidden_stateselse:- # Training pathresidual = hidden_stateshidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)hidden_states, _ = self.self_attn(@@ -336,7 +331,6 @@ def Glm4MoeLiteDecoderLayer_fast_forward()hidden_states = residual + hidden_states- # MLP/MoEresidual = hidden_stateshidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)hidden_states = self.mlp(hidden_states)@@ -370,15 +364,13 @@ class FastGLM47Model(FastLlamaModel):"Please upgrade with: pip install --upgrade transformers")- # Patch MoE forward with grouped GEMM (TMA compat handled by- # grouped_gemm/__init__.py)+ # Patch MoE forward with grouped GEMM; TMA compat is handled by grouped_gemm/__init__.py.if HAS_GROUPED_GEMM:Glm4MoeLiteNaiveMoe.forward = Glm4MoeLiteNaiveMoe_fast_forwardGlm4MoeLiteMoE.forward = Glm4MoeLiteMoE_fast_forward- # Attention/rope/decoder/model forwards are NOT patched: GLM4 uses MLA- # with different projection names and lacks extend_rope_embedding, so the- # Llama-compatible infrastructure doesn't apply.+ # Attention/rope/decoder/model forwards are NOT patched: GLM4 uses MLA with different projection+ # names and lacks extend_rope_embedding, so the Llama-compatible infrastructure does not apply.return@@ -397,7 +389,7 @@ class FastGLM47Model(FastLlamaModel):trust_remote_code = False,**kwargs,):- # Used by loader, not passed to model+ # Used by the loader, not passed to the model.kwargs.pop("unsloth_force_compile", None)return FastLlamaModel.from_pretrained(diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 1f4afa37697..6cb8f1a3195 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -32,8 +32,8 @@ from .llama import ()from .mistral import *-# Without bnb, peft stops exporting its 4bit LoRA layer too. Both names only feed-# isinstance checks, so placeholders nothing can match are exact stand-ins.+# Without bnb, peft stops exporting its 4bit LoRA layer too. Both names only feed isinstance checks,+# so placeholders nothing can match are exact stand-ins.try:from bitsandbytes.nn import Linear4bit as Bnb_Linear4bitfrom peft.tuners.lora import Linear4bit as Peft_Linear4bit@@ -92,7 +92,6 @@ def GraniteAttention_fast_forward(*args,**kwargs,) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:- # Clear inferenceif hasattr(self, "paged_attention"):del self.paged_attention_Kdel self.paged_attention_V@@ -125,7 +124,7 @@ def GraniteAttention_fast_forward(cos, sin = position_embeddingsrope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")if rope_position_ids is not None:- # Useful for LongRoPE+ # Useful for LongRoPE.Q, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)else:Q, K = fast_rope_embedding(Q, K, cos, sin)@@ -135,7 +134,6 @@ def GraniteAttention_fast_forward(V = torch.cat([past_key_value[1], V], dim = 2)past_key_value = (K, V) if use_cache else None- # Attention moduleuse_varlen = attention_mask is None and seq_info is not None and past_key_value is Nonebackend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)@@ -172,8 +170,8 @@ def GraniteAttention_fast_forward(},)- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse- # (KV cache / padding mask) raises. None => byte-identical default.+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /+ # padding mask) raises. None means the byte-identical default._pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)context = AttentionContext(bsz = bsz,@@ -216,7 +214,7 @@ def GraniteDecoderLayer_fast_forward(else self.config.residual_multiplier)- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:+ if use_cache and hasattr(self, "_flag_for_generation"):residual = hidden_stateshidden_states = fast_rms_layernorm_inference(self.input_layernorm, hidden_states)hidden_states, self_attn_weights, present_key_value = self.self_attn(@@ -234,7 +232,6 @@ def GraniteDecoderLayer_fast_forward()hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm_inference(self.post_attention_layernorm, hidden_states)hidden_states = fast_swiglu_inference(self.mlp, hidden_states)@@ -256,7 +253,6 @@ def GraniteDecoderLayer_fast_forward()hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)hidden_states = self.mlp(hidden_states)@@ -272,7 +268,7 @@ def GraniteDecoderLayer_fast_forward(from math import sqrt as math_sqrt-KV_CACHE_INCREMENT = 256 # KV Cache update size+KV_CACHE_INCREMENT = 256torch_nn_functional_softmax = torch.nn.functional.softmaxtorch_matmul = torch.matmultorch_tanh = torch.tanh@@ -301,7 +297,6 @@ def GraniteAttention_fast_forward_inference(n_groups = self.num_key_value_groupsn_kv_heads = self.config.num_key_value_headshead_dim = self.head_dim- # assert(n_kv_heads * n_groups == n_heads)hidden_size = self.config.hidden_sizeattention_size = n_heads * head_dim@@ -309,8 +304,6 @@ def GraniteAttention_fast_forward_inference(kv_seq_len = seq_len + 1device = hidden_states.device- # Prefill phase- # if not hasattr(self, "paged_attention"):if do_prefill:self.paged_attention = torch.empty((KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),@@ -351,10 +344,8 @@ def GraniteAttention_fast_forward_inference(Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)cos, sin = position_embeddings- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]cos, sin = cos[position_ids], sin[position_ids]@@ -367,24 +358,19 @@ def GraniteAttention_fast_forward_inference(Qn *= cosQn.addcmul_(RH_Q, sin)- RH_K = RH_Q[- :, :n_kv_heads, :, :- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")+ RH_K = RH_Q[:, :n_kv_heads, :, :]RH_K[:, :, :, :h] = Kn[:, :, :, h:]RH_K[:, :, :, h:] = Kn[:, :, :, :h]RH_K[:, :, :, :h].neg_()Kn *= cosKn.addcmul_(RH_K, sin)- # New KV cache- # Kn = torch.cat([K1, Kn], dim = 2)- # Vn = torch.cat([V1, Vn], dim = 2)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)- # Grouped query attention+ # Grouped query attention._, _, cached_len, _ = Kn.shapeif bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):Kn = Kn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)@@ -392,7 +378,6 @@ def GraniteAttention_fast_forward_inference(Kn = Kn.reshape(bsz, n_heads, cached_len, head_dim)Vn = Vn.reshape(bsz, n_heads, cached_len, head_dim)- # Attentionif bsz == 1:Qn *= self.scalingA = torch_matmul(Qn, Kn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])@@ -428,8 +413,7 @@ def GraniteAttention_fast_forward_inference(return A, (Kn, Vn)-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825-# @torch.inference_mode+# Ported from transformers models/llama/modeling_llama.py#L825def GraniteModel_fast_forward_inference(self,input_ids,@@ -456,7 +440,7 @@ def GraniteModel_fast_forward_inference(hidden_states,seq_len,)- # Pre-convert to bool once for all layers (avoids per-layer .eq(0))+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).if attention_mask is not None and attention_mask.dtype != torch.bool:attention_mask = attention_mask.eq(0)else:@@ -510,9 +494,8 @@ class GraniteRotaryEmbedding(LlamaRotaryEmbedding):def patched_init(original_init):def new_init(self, *args, **kwargs):- # GraniteModel_fast_forward_inference can't reach residual_multiplier/config,- # so stash the whole config here to pass it around. See:- # https://github.com/huggingface/transformers/blob/e5fd865ebae062b7cf03a81b8c6affeb39f30bec/src/transformers/models/granite/modeling_granite.py#L243+ # GraniteModel_fast_forward_inference cannot reach residual_multiplier/config, so stash the whole+ # config here to pass it around. See transformers models/granite/modeling_granite.py#L243config = kwargs.get("config", args[0] if args else None)if config is not None:self.config = config@@ -555,8 +538,8 @@ class FastGraniteModel(FastLlamaModel):tokenizer,correct_dtype = None,):- # Torch.compile fails on embedding matrix??- # Workaround randomnly fixes it for torch versions < 2.2+ # Torch.compile fails on the embedding matrix; this workaround randomly fixes it for torch < 2.2,+ # and the same is done for lm_head.model.model.embed_tokens = torch.nn.Embedding.from_pretrained(model.model.embed_tokens.weight)@@ -570,7 +553,7 @@ class FastGraniteModel(FastLlamaModel):lm_head.out_features = lm_head.weight.shape[0]model.lm_head = lm_head- # Granite has tied weights! This means lm_head == embed_tokens+ # Granite has tied weights, so lm_head == embed_tokens.if model.model.embed_tokens.weight.data_ptr() != model.lm_head.weight.data_ptr():lm_head = torch.nn.Linear(1, 1, bias = None)del lm_head.weight@@ -579,8 +562,8 @@ class FastGraniteModel(FastLlamaModel):lm_head.out_features = lm_head.weight.shape[0]model.lm_head = lm_head- # Also patch all dtypes - BnB seems to not allocate the correct type?- # BnB default dtype seems to be float16!+ # BnB does not allocate the correct type: its default dtype is float16, so patch all dtypes. See+ # TimDettmers/bitsandbytes#763.correct_dtype = lm_head.weight.dtypefor name, module in model.named_modules():@@ -594,7 +577,7 @@ class FastGraniteModel(FastLlamaModel):else:# https://github.com/TimDettmers/bitsandbytes/pull/763/filesquant_state.dtype = correct_dtype- # Downcast RoPE embedding to correct data type+ # Downcast RoPE embedding to the correct data type.if name.endswith("rotary_emb") or hasattr(module, "cos_cached"):if hasattr(module, "cos_cached") and (module.cos_cached.dtype != correct_dtype):module.cos_cached = module.cos_cached.to(correct_dtype)@@ -606,7 +589,6 @@ class FastGraniteModel(FastLlamaModel):module.short_cos_cached = module.short_cos_cached.to(correct_dtype)module.short_sin_cached = module.short_sin_cached.to(correct_dtype)- # Clear deleted GPU itemsimport gcfor _ in range(3):diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex a4d47d27736..dc3993f2927 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -85,7 +85,7 @@ from ..device_type import ()transformers_version = Version(transformers_version)-# Transformers moved rotary embeddings out of all attention layers+# Transformers moved rotary embeddings out of all attention layers.IS_ATTENTION_REFACTOR = transformers_version > Version("4.47.1")try:from transformers.modeling_layers import GradientCheckpointingLayer@@ -105,7 +105,6 @@ from ..kernels import *from ..tokenizer_utils import *from .vision import FastBaseModel-# Final patching codefrom transformers.models.llama.modeling_llama import (LlamaAttention,LlamaDecoderLayer,@@ -141,15 +140,14 @@ import typestry:from huggingface_hub.utils import get_tokenexcept:- # Old HF Hub versions <= 0.0.25+ # Old HF Hub versions <= 0.0.25.from huggingface_hub.utils._token import get_tokenfrom triton import __version__ as triton_version-# Not `xformers is not None`: attention_dispatch probes the install and turns HAS_XFORMERS-# off when the library imports but has no kernel that runs here. Recomputing it from the-# import alone left the model code on the xFormers path the dispatcher had already left ---# and Mistral answers "xFormers" by skipping the 4D sliding-window mask, so every sequence-# longer than config.sliding_window attended to the whole causal history on SDPA instead.+# Not `xformers is not None`: attention_dispatch turns HAS_XFORMERS off when the library imports+# but has no kernel that runs here. Recomputing from the import alone left the model on the+# xFormers path, and Mistral then skipped the 4D sliding-window mask, so every sequence longer+# than config.sliding_window attended to the whole causal history on SDPA.BlockDiagonalCausalMask = xformers.attn_bias.BlockDiagonalCausalMask if HAS_XFORMERS else None@@ -167,9 +165,9 @@ def original_apply_o(self, X):from math import sqrt as math_sqrt-KV_CACHE_INCREMENT = 512 # KV Cache update size+KV_CACHE_INCREMENT = 512torch_nn_functional_softmax = torch.nn.functional.softmax-# SDPA has GQA internally+# SDPA has GQA internally.SDPA_HAS_GQA = "enable_gqa" in scaled_dot_product_attention.__doc__from peft.utils.other import ModulesToSaveWrapper@@ -195,16 +193,13 @@ def _offload_frozen_module_for_training(new_dtype = module.modules_to_save.default.weight.dtypeif new_dtype == torch.float16:- # See https://github.com/unslothai/unsloth/pull/1200- # Tesla T4 must use float32 and not float16+ # Tesla T4 must use float32 and not float16; see #1200.new_dtype = torch.float32- # `device_type` is a bare type like "cuda", and `.to("cuda")` resolves to the- # CURRENT device, which is cuda:0, so on a split model it drags the trainable- # copy off its own card and the forward then mixes devices. Prefer the index- # the copy already has; under the default `use_gradient_checkpointing =- # "unsloth"` it has none, because the disk offload above rebuilt it on CPU,- # so fall back to the placement recorded before that. Only then the bare type.+ # `device_type` is bare ("cuda") and .to("cuda") resolves to the CURRENT device, so on a split+ # model it drags the trainable copy off its card. Prefer the index the copy already has; under+ # use_gradient_checkpointing="unsloth" it has none (rebuilt on CPU), so fall back to the+ # placement recorded before that, then the bare type.wanted_type = torch.device(device_type).typetarget_device = device_typefor candidate in (module.modules_to_save.default.weight.device, original_device):@@ -215,13 +210,11 @@ def _offload_frozen_module_for_training(module.modules_to_save.default.to(device = target_device, dtype = new_dtype, non_blocking = True)module.modules_to_save.default.requires_grad_(True)- # [TODO] Move old module to CPU - should be disk!if offload_device is not None:module.original_module.to(device = offload_device, non_blocking = True)module.original_module.requires_grad_(False)-# Fix new HF's inference codedef _fast_prepare_inputs_for_generation(self,input_ids,@@ -232,7 +225,7 @@ def _fast_prepare_inputs_for_generation(past_key_values = kwargs.get("past_key_values", None)original_attention_mask = attention_mask- # Only use inputs_embeds on the first step (no cache). Fixes issue #3798.+ # Only use inputs_embeds on the first step (no cache); fixes #3798.use_inputs_embeds = inputs_embeds is not None and past_key_values is Noneif input_ids is not None and input_ids.numel() > 0:@@ -246,12 +239,10 @@ def _fast_prepare_inputs_for_generation(device = "cuda" if torch.cuda.is_available() else "cpu"if past_key_values is not None:- # Check for uninitialized DynamicCacheif len(past_key_values) == 0:past_key_values = Nonekwargs["past_key_values"] = Noneuse_inputs_embeds = inputs_embeds is not None- # New since 4.56elif hasattr(past_key_values, "get_seq_length") and past_key_values.get_seq_length() == 0:past_key_values = Nonekwargs["past_key_values"] = None@@ -272,7 +263,7 @@ def _fast_prepare_inputs_for_generation(if hasattr(past_key_values, "get_seq_length"):past_len = int(past_key_values.get_seq_length())else:- # legacy tuple cache: (layer, (K,V))+ # Legacy tuple cache: (layer, (K, V)).past_len = int(past_key_values[0][0].shape[-2])max_cache_len = None@@ -283,7 +274,6 @@ def _fast_prepare_inputs_for_generation(m = past_key_values.get_max_length()max_cache_len = int(m) if m is not None else None- # ensure cache_positioncache_position = kwargs.get("cache_position", None)if cache_position is None:kwargs["cache_position"] = torch.arange(@@ -296,7 +286,6 @@ def _fast_prepare_inputs_for_generation(if hasattr(cache_position, "device") and cache_position.device != device:kwargs["cache_position"] = cache_position.to(device)- # Get to the base modelbase_model = selfif hasattr(base_model, "base_model_prefix"):base_model = getattr(base_model, base_model.base_model_prefix)@@ -309,7 +298,7 @@ def _fast_prepare_inputs_for_generation(sig = inspect.signature(inspect.unwrap(fn))return "device" in sig.parametersexcept:- # transformers <= 4.51.3 includes device arg but > 4.51.3 does not+ # transformers <= 4.51.3 includes the device arg; later versions do not.return transformers_version < Version("4.52.0")base_model._unsloth_mask_needs_device = _check_needs_device(@@ -373,7 +362,6 @@ def _fast_prepare_inputs_for_generation(def fix_prepare_inputs_for_generation(module):- # Fix prepare_inputs_for_generationif hasattr(module, "prepare_inputs_for_generation"):module.prepare_inputs_for_generation = _fast_prepare_inputs_for_generation@@ -427,15 +415,12 @@ def LlamaAttention_fast_forward_inference(n_groups = self.num_key_value_groupsn_kv_heads = self.config.num_key_value_headshead_dim = self.head_dim- # assert(n_kv_heads * n_groups == n_heads)hidden_size = self.config.hidden_sizeattention_size = n_heads * head_dimseq_len = K1.shape[-2]kv_seq_len = seq_len + 1- # Prefill phase- # if not hasattr(self, "paged_attention"):device = hidden_states.deviceif do_prefill:self.paged_attention = torch.empty(@@ -451,7 +436,7 @@ def LlamaAttention_fast_forward_inference(self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)- # Mistral Nemo 12b has weird dimensions+ # Mistral Nemo 12b has weird dimensions.if attention_size != hidden_size:self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)else:@@ -483,15 +468,11 @@ def LlamaAttention_fast_forward_inference(Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)-- # Need to do it prior 2 steps before hitting full on short KV cache- # or else error+ # Must be done 2 steps before hitting full on a short KV cache, or it errors.if position_ids.dim() == 1:position_ids = position_ids[:, None]- # Transformers 5.x accumulates position_ids as [batch, full_seq_len] across- # decode steps; single-token inference only needs the last position.+ # Transformers 5.x accumulates position_ids as [batch, full_seq_len] across decode steps;+ # single-token inference only needs the last position.if position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]position_ids = position_ids.to(Qn.device)@@ -509,40 +490,34 @@ def LlamaAttention_fast_forward_inference(RH_Q = self.RH_QRH_Q[:, :, :, :h] = Qn[:, :, :, h:]RH_Q[:, :, :, h:] = Qn[:, :, :, :h]- RH_Q[:, :, :, :h].neg_() # torch.neg(RH_Q[:,:,:,:h], out = RH_Q[:,:,:,:h])+ RH_Q[:, :, :, :h].neg_()Qn *= cosQn.addcmul_(RH_Q, sin)- RH_K = RH_Q[- :, :n_kv_heads, :, :- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")+ RH_K = RH_Q[:, :n_kv_heads, :, :]RH_K[:, :, :, :h] = Kn[:, :, :, h:]RH_K[:, :, :, h:] = Kn[:, :, :, :h]- RH_K[:, :, :, :h].neg_() # torch.neg(RH_K[:,:,:,:h], out = RH_K[:,:,:,:h])+ RH_K[:, :, :, :h].neg_()Kn *= cosKn.addcmul_(RH_K, sin)- # New KV cache- # Kn = torch.cat([K1, Kn], dim = 2)- # Vn = torch.cat([V1, Vn], dim = 2)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)- # Handle sliding windows+ # Handle sliding windows, from transformers models/mistral/modeling_mistral.py#L193sliding_window = getattr(self.config, "sliding_window", None)if sliding_window is not None and kv_seq_len > sliding_window:- # From https://github.com/huggingface/transformers/blob/main/src/transformers/models/mistral/modeling_mistral.py#L193start = kv_seq_len - sliding_window- Knn = Kn[:, :, start:, :] # .contiguous()- Vnn = Vn[:, :, start:, :] # .contiguous()+ Knn = Kn[:, :, start:, :]+ Vnn = Vn[:, :, start:, :]if attention_mask is not None:attention_mask = attention_mask[..., start:]else:Knn, Vnn = Kn, Vn- # Grouped query attention+ # Grouped query attention._, _, cached_len, _ = Knn.shapeif bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)@@ -550,27 +525,23 @@ def LlamaAttention_fast_forward_inference(Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)- # when qlen==vlen and attn_mask is None, we should use causal attention+ # When qlen == vlen and attn_mask is None, causal attention is the right choice.Q_len = Qn.shape[-2]K_len = Knn.shape[-2]if attention_mask is None and Q_len == K_len:is_causal = Trueelse:is_causal = False- # Attentionif bsz == 1:- Qn *= (- self.scalar- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows+ Qn *= self.scalar+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805.A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)A = torch_matmul(A, Vnn, out = Qn)- # --- attention_mask fixup for SDPA if user passes 2D padding mask+ # attention_mask fixup for SDPA when the user passes a 2D padding mask.else:if attention_mask is not None and attention_mask.dim() == 2:attention_mask = attention_mask[:, None, None, :].to(torch.bool)- # is it more appropriate to use _prepare_4d_causal_attention_mask_for_sdpa?elif (attention_mask is not Noneand attention_mask.dim() == 4@@ -609,11 +580,7 @@ def fast_swiglu_inference(gate_multiplier = None,down_multiplier = None,):- # gate = self.gate_proj(X)- # up = self.up_proj(X)bsz, _, hd = X.shape- # mlp_size = self.config.intermediate_size- # temp = torch.empty((2, bsz, 1, mlp_size), dtype = X.dtype, device = "cuda:0")gate = fast_linear_forward(self.gate_proj, X, out = temp_gate)@@ -625,7 +592,6 @@ def fast_swiglu_inference(gate = torch_nn_functional_silu(gate, inplace = True)gate *= up- # X = self.down_proj(gate)down = fast_linear_forward(self.down_proj, gate, out = up[:, :, :hd])if down_multiplier is not None:@@ -684,7 +650,7 @@ def fast_rms_layernorm_inference_gemma(return XX.to(X.dtype)-# Normal layernorm with mean removal+# Normal layernorm with mean removal.@torch.compile(fullgraph = False, dynamic = True, options = torch_compile_options)def fast_layernorm_compiled(layernorm, X):old_dtype = X.dtype@@ -699,7 +665,7 @@ def fast_layernorm_compiled(layernorm, X):return X.to(old_dtype)-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L320+# Ported from transformers models/llama/modeling_llama.py#L320def LlamaAttention_fast_forward(self,hidden_states: torch.Tensor,@@ -714,7 +680,6 @@ def LlamaAttention_fast_forward(*args,**kwargs,) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:- # Clear inferenceif hasattr(self, "paged_attention"):del self.paged_attention_Kdel self.paged_attention_V@@ -754,11 +719,6 @@ def LlamaAttention_fast_forward(if rope_position_ids is None and seq_info is not None:rope_position_ids = kwargs.get("position_ids")- # Q, K = (- # fast_rope_embedding(Q, K, cos, sin)- # if rope_position_ids is None- # else inplace_rope_embedding(Q, K, cos, sin, rope_position_ids)- # )Q, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)if past_key_value is not None:@@ -766,11 +726,10 @@ def LlamaAttention_fast_forward(V = torch.cat([past_key_value[1], V], dim = 2)past_key_value = (K, V) if use_cache else None- # Attention moduleuse_varlen = seq_info is not None and past_key_value is Nonebackend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)- # should dropout be hardcoded to 0.0?+ # Should dropout be hardcoded to 0.0?config = AttentionConfig(backend = backend,n_kv_heads = n_kv_heads,@@ -778,9 +737,9 @@ def LlamaAttention_fast_forward(flash_dense_kwargs = {"causal": True},flash_varlen_kwargs = {"dropout_p": 0.0, "causal": True},)- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward (same route- # as packed_seq_lengths); misuse (KV cache / padding mask) raises. None => byte-identical- # default. Reuse of this forward also carries the branch to qwen2 & gemma.+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward (same route as+ # packed_seq_lengths); misuse (KV cache / padding mask) raises. None means the byte-identical+ # default. Reuse of this forward carries the branch to qwen2 and gemma._pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)context = AttentionContext(bsz = bsz,@@ -802,7 +761,7 @@ def LlamaAttention_fast_forward(return attn_output, attn_weights, past_key_value-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L590+# Ported from transformers models/llama/modeling_llama.py#L590def LlamaDecoderLayer_fast_forward(self,hidden_states: torch.Tensor,@@ -847,7 +806,6 @@ def LlamaDecoderLayer_fast_forward()hidden_states += residual- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm_inference(self.post_attention_layernorm, hidden_states)hidden_states = fast_swiglu_inference(self.mlp, hidden_states)@@ -869,7 +827,6 @@ def LlamaDecoderLayer_fast_forward()hidden_states = residual + hidden_states- # Fully Connectedresidual = hidden_stateshidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)hidden_states = self.mlp(hidden_states)@@ -883,7 +840,7 @@ def LlamaDecoderLayer_fast_forward(return outputs-# https://github.com/unslothai/unsloth/issues/404#issuecomment-2323473452+# See unslothai/unsloth#404 (comment 2323473452).__DTYPE_MAP = {"float32": torch.float32,torch.float32: torch.float32,@@ -894,7 +851,7 @@ __DTYPE_MAP = {}-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825+# Ported from transformers models/llama/modeling_llama.py#L825def LlamaModel_fast_forward(self,input_ids: Optional[torch.LongTensor] = None,@@ -923,7 +880,6 @@ def LlamaModel_fast_forward(return_dict = return_dict if return_dict is not None else self.config.use_return_dict- # retrieve input_ids and inputs_embedsif input_ids is not None and inputs_embeds is not None:raise ValueError("Unsloth: You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time"@@ -965,8 +921,8 @@ def LlamaModel_fast_forward(past_key_values_length = past_key_values[0][0].shape[2]seq_length_with_past = seq_length_with_past + past_key_values_length- # We already handle KV cache position_ids ourselves.- if False: # (past_key_values_length != 0):+ # KV cache position_ids are handled here already.+ if False:position_ids = torch.arange(past_key_values_length,seq_length + past_key_values_length,@@ -975,7 +931,7 @@ def LlamaModel_fast_forward()position_ids = position_ids.unsqueeze(0).view(-1, seq_length)elif position_ids is not None:- position_ids = position_ids.view(-1, seq_length).to(torch.int32) # .long()+ position_ids = position_ids.view(-1, seq_length).to(torch.int32)else:position_ids = None@@ -983,13 +939,13 @@ def LlamaModel_fast_forward(if position_ids.shape[0] != batch_size:position_ids = position_ids.repeat((batch_size, 1))- # Embed positionsif inputs_embeds is None:inputs_embeds = self.embed_tokens(input_ids)inputs_embeds = inputs_embeds.to(_get_dtype(dtype_from_config(self.config)))- # Normalized from Gemma+ # Normalized from Gemma: cast to bfloat16/float16 to match it exactly, since 3072**0.5 is+ # 55.5000 in bfloat16 against 55.4256 in float32.IS_GEMMA = self.config.model_type.startswith("gemma")IS_GEMMA2 = self.config.model_type.startswith("gemma2")IS_COHERE = self.config.model_type.startswith("cohere")@@ -999,14 +955,10 @@ def LlamaModel_fast_forward(train_embed_tokens = self.embed_tokens.weight.requires_gradif IS_GEMMA:- # Match Gemma exactly by casting to bfloat16 / float16- # inputs_embeds *= math_sqrt(self.config.hidden_size)- # Ie 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32- # & 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32normalizer = torch.tensor(math_sqrt(self.config.hidden_size), dtype = inputs_embeds.dtype)if train_embed_tokens:- # Careful we must not do an inplace op!+ # Careful: this must not be an inplace op.inputs_embeds = inputs_embeds * normalizerelse:inputs_requires_grad = inputs_embeds.requires_grad@@ -1016,12 +968,10 @@ def LlamaModel_fast_forward(elif inputs_requires_grad:inputs_embeds.requires_grad_(False)inputs_embeds *= normalizer- # inputs_embeds *= math_sqrt(self.config.hidden_size)if inputs_requires_grad:inputs_embeds.requires_grad_(True)- # Fix up attention mask by setting elements to 0- # Specifically for DPO+ # Fix up the attention mask by setting elements to 0, specifically for DPO.if (getattr(self, "_has_no_labels", False) is Trueand (attention_mask is not None)@@ -1030,8 +980,7 @@ def LlamaModel_fast_forward(and (not train_embed_tokens)and self.training):- # Careful for inference the attention_mask is size (1, kv_seq_len)- # Whilst the input_embeds is size (1, 1, 4096)+ # Careful: for inference the attention_mask is size (1, kv_seq_len) while input_embeds is (1, 1, 4096).inputs_requires_grad = inputs_embeds.requires_gradif not inputs_embeds.is_leaf:inputs_embeds = inputs_embeds.detach()@@ -1043,16 +992,12 @@ def LlamaModel_fast_forward(if inputs_requires_grad:inputs_embeds.requires_grad_(True)- # Ignore attention_maskif attention_mask is None:padding_mask = Noneelif self.training:attention_mask = Nonepadding_mask = Noneelse:- # if 0 in attention_mask:- # padding_mask = attention_mask- # else:padding_mask = Noneattention_mask = _prepare_4d_causal_attention_mask_for_sdpa(@@ -1062,9 +1007,7 @@ def LlamaModel_fast_forward(past_key_values_length,sliding_window = getattr(self.config, "sliding_window", None),)- # Must NOT convert to bool - weirdly this causes stuff to error out!- # if attention_mask is not None:- # attention_mask = attention_mask.to(torch.bool)+ # Must NOT convert to bool; that weirdly causes errors.hidden_states = inputs_embedsif IS_GRANITE or IS_FALCON_H1: # granite has embedding multiplier@@ -1072,30 +1015,23 @@ def LlamaModel_fast_forward(if past_key_values is None and self.training:use_cache = False- # if use_cache:- # logger.warning_once(- # "Unsloth: `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`"- # )- # use_cache = False- # decoder layersall_hidden_states = () if output_hidden_states else Noneall_self_attns = () if output_attentions else Nonenext_decoder_cache = () if use_cache else None- # Gradient checkpointing methods (ie sqrt)+ # Gradient checkpointing methods (ie sqrt).if hasattr(self, "_gradient_checkpointing_boundaries"):boundaries = self._gradient_checkpointing_boundarieselse:boundaries = None- # Check checkpointing methodgradient_checkpointing = Falseif self.gradient_checkpointing and self.training and not use_cache:gradient_checkpointing = True- # Gemma2 has alternating SWA and global attn+ # Gemma2 has alternating SWA and global attn.use_static_mask = Truedynamic_SWA_mask = Nonedynamic_GA_mask = None@@ -1104,12 +1040,9 @@ def LlamaModel_fast_forward(self.SWA_mask = Trueself.GA_mask = Falseelif attention_mask is not None:- # Fixes https://github.com/unslothai/unsloth/issues/853- # Unsloth needs a 2D mask, not a [2, 1, n, n] mask!+ # Unsloth needs a 2D mask, not [2, 1, n, n] (#853), converted to float not bool+ # (pytorch/pytorch#103749).- # https://github.com/pytorch/pytorch/issues/103749- # Need to convert to float and not using bool- # attention_mask = (1.0 - attention_mask.float()) * torch.finfo(inputs_embeds.dtype).mindynamic_SWA_mask = _prepare_4d_causal_attention_mask_for_sdpa(attention_mask,(batch_size, seq_length),@@ -1128,16 +1061,13 @@ def LlamaModel_fast_forward(elif not hasattr(self, "SWA_mask"):if HAS_FLEX_ATTENTION:- # Use Flex Attention instead!self.SWA_mask = create_flex_attention_sliding_window_mask(self.max_seq_length, self.config.sliding_window)self.GA_mask = create_flex_attention_causal_mask(self.max_seq_length)else:- n = self.max_seq_length # self.config.max_position_embeddings- # masked_fill is making stuff slower!- # self. GA_mask = create_boolean_mask(n = n, sliding_window = 0)- # self.SWA_mask = create_boolean_mask(n = n, sliding_window = self.config.sliding_window)+ n = self.max_seq_length+ # masked_fill makes things slower.self.SWA_mask = (AttentionMaskConverter(is_causal = True,@@ -1174,8 +1104,8 @@ def LlamaModel_fast_forward(IS_ATTENTION_REFACTORand (hasattr(self, "rotary_emb") or not hasattr(self.layers[0].self_attn, "rotary_emb"))) or IS_GRANITE:- # position_embeddings is mandatory on main: https://github.com/huggingface/transformers/pull/34858- # granite always had the attention refactor, so let it always use this path.+ # position_embeddings is mandatory on main (huggingface/transformers#34858), and granite always+ # had the attention refactor, so it always takes this path.self.rotary_emb.extend_rope_embedding(hidden_states, self.config.max_position_embeddings)position_embeddings = self.rotary_emb.get_cached(self.config.max_position_embeddings, hidden_states.device.index@@ -1183,7 +1113,6 @@ def LlamaModel_fast_forward(else:position_embeddings = None- # Go through every layer!for idx, decoder_layer in enumerate(self.layers):if output_hidden_states:all_hidden_states += (hidden_states,)@@ -1244,7 +1173,6 @@ def LlamaModel_fast_forward(if output_attentions:all_self_attns += (layer_outputs[1],)- # Final layernormif use_cache:if IS_FALCON_H1:hidden_states = fast_rms_layernorm_inference(self.final_layernorm, hidden_states)@@ -1277,7 +1205,7 @@ def LlamaModel_fast_forward()-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825+# Ported from transformers models/llama/modeling_llama.py#L825def _LlamaModel_fast_forward_inference(attention_fast_forward_inference = LlamaAttention_fast_forward_inference,mlp_fast_forward_inference = fast_swiglu_inference,@@ -1300,7 +1228,6 @@ def _LlamaModel_fast_forward_inference(X = X.to(_get_dtype(dtype_from_config(self.config)))bsz, q_len, hd = X.shapeassert q_len == 1- # Get saved buffers to reduce memory movementresidual = torch.empty((bsz, q_len, hd), dtype = torch.float32, device = f"{DEVICE_TYPE_TORCH}:0")@@ -1327,13 +1254,13 @@ def _LlamaModel_fast_forward_inference(seq_len,sliding_window = getattr(self.config, "sliding_window", None),)- # Pre-convert to bool once for all layers (avoids per-layer .eq(0))+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).if attention_mask is not None and attention_mask.dtype != torch.bool:attention_mask = attention_mask.eq(0)else:attention_mask = None- # Compute rotary_seq_len once to avoid per-layer GPU-CPU sync from .item()+ # Compute rotary_seq_len once to avoid a per-layer GPU-CPU sync from .item().rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)next_decoder_cache = []@@ -1341,7 +1268,7 @@ def _LlamaModel_fast_forward_inference(for idx, decoder_layer in enumerate(self.model.layers):device_index = getattr(decoder_layer, "_per_layer_device_index", 0)X, residual, position_ids = move_to_device(device_index, X, residual, position_ids)- residual.copy_(X) # residual = X+ residual.copy_(X)X = fast_rms_layernorm_inference(decoder_layer.input_layernorm,X,@@ -1360,7 +1287,7 @@ def _LlamaModel_fast_forward_inference()X += residual- residual.copy_(X) # residual = X+ residual.copy_(X)X = fast_rms_layernorm_inference(decoder_layer.post_attention_layernorm,X,@@ -1395,7 +1322,7 @@ def _LlamaModel_fast_forward_inference(return LlamaModel_fast_forward_inference_custom-# For ensuring backwards compatibility, we create LlamaModel_fast_forward_inference that is consumed by other models+# LlamaModel_fast_forward_inference exists for backwards compatibility and is consumed by other models.LlamaModel_fast_forward_inference = _LlamaModel_fast_forward_inference()@@ -1441,7 +1368,6 @@ def CausalLM_fast_forward(fast_forward_inference):else self.config.output_hidden_states)return_dict = return_dict if return_dict is not None else self.config.use_return_dict- # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)self.model._has_no_labels = labels is Noneoutputs = self.model(input_ids = input_ids,@@ -1471,12 +1397,11 @@ def CausalLM_fast_forward(fast_forward_inference):else:num_logits_to_keep = max(num_logits_to_keep, logits_to_keep)- # Move items to same device as lm_headhidden_states = hidden_states.to(lm_head_device)if labels is not None:labels = labels.to(lm_head_device)- # Output last hidden states without logits if asked+ # Output last hidden states without logits if asked.if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":if num_logits_to_keep != 0:hidden_states = hidden_states[:, -num_logits_to_keep:, :]@@ -1495,9 +1420,9 @@ def CausalLM_fast_forward(fast_forward_inference):logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(dtype))else:RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"- # < 1024 Normal Unsloth uses less VRAM!+ # Below 1024 normal Unsloth uses less VRAM.if bsz * q_len <= 1024 and not RETURN_LOGITS:- # Use unsloth_fused_ce_loss which actually calculates the best chunk size to reduce VRAM usage+ # unsloth_fused_ce_loss calculates the best chunk size to reduce VRAM usage.RETURN_LOGITS = Falseif not RETURN_LOGITS and labels is not None:@@ -1508,24 +1433,17 @@ def CausalLM_fast_forward(fast_forward_inference):if self.config.model_type == "falcon_h1":hidden_states = hidden_states * self.config.lm_head_multiplier- # Packed-boundary guard on raw labels (the fused kernel shifts internally).- # This branch RETURNS, so mask_packed_sequence_boundaries() below is dead on- # packed training paths: it needs UNSLOTH_RETURN_LOGITS=1, which itself forces- # packing off (unsloth/trainer.py). Out-of-place, no-op without packed_seq_lengths.+ # Packed-boundary guard on raw labels (the fused kernel shifts internally). This branch RETURNS,+ # so mask_packed_sequence_boundaries() below is dead on packed paths: it needs+ # UNSLOTH_RETURN_LOGITS=1, which forces packing off. Out-of-place, no-op without+ # packed_seq_lengths.labels = mask_packed_boundary_labels(labels,kwargs.get("packed_seq_lengths"),)- ### DISABLED since T4 breaks- # OutOfResources: out of resource: shared memory, Required: 98304, Hardware limit: 65536. Reducing block sizes or `num_stages` may help.- # loss = fused_linear_cross_entropy(- # hidden_states = hidden_states,- # lm_weight = lm_head,- # labels = labels,- # num_items_in_batch = n_items,- # logit_softcapping = logit_softcapping,- # )+ # fused_linear_cross_entropy is disabled: T4 raises OutOfResources (needs 98304 shared memory+ # against a 65536 limit).loss = unsloth_fused_ce_loss(trainer = None,hidden_states = hidden_states,@@ -1540,8 +1458,7 @@ def CausalLM_fast_forward(fast_forward_inference):logit_softcapping = logit_softcapping,)if not return_dict:- # Fused CE never materializes `logits`; use EMPTY_LOGITS- # like the return_dict branch below (fixes #2068).+ # Fused CE never materializes logits; use EMPTY_LOGITS like the return_dict branch below (#2068).output = (EMPTY_LOGITS,) + outputs[1:]return (loss,) + output if loss is not None else output@@ -1558,12 +1475,9 @@ def CausalLM_fast_forward(fast_forward_inference):logits = logits.to(_get_dtype(dtype_from_config(self.config)))loss = None- # Which field carries the scale is per family (cohere multiplies by logit_scale,- # granite divides by logits_scaling, falcon_h1 multiplies by lm_head_multiplier).- # The planner sizes the head's card from the same answer, so a new family is taught- # once, not twice.- # granite: https://github.com/huggingface/transformers/blob/4d1d0f29a493098e6bc6b904b82e29cb331827f5/src/transformers/models/granite/modeling_granite.py#L1103- # cohere: https://github.com/huggingface/transformers/blob/4d1d0f29a493098e6bc6b904b82e29cb331827f5/src/transformers/models/cohere/modeling_cohere.py#L1176+ # Which field carries the scale is per family (cohere logit_scale, granite logits_scaling,+ # falcon_h1 lm_head_multiplier). The planner sizes the head's card from the same answer, so a+ # new family is taught once, not twice.if detect_logit_transforms is not None:_transforms = detect_logit_transforms(self.config)logit_softcapping = _transforms["logit_softcapping"]@@ -1580,10 +1494,6 @@ def CausalLM_fast_forward(fast_forward_inference):if labels is not None:shift_logits = logits- # if not hasattr(self, "extra_ignored_labels"):- # # Fixes https://github.com/unslothai/unsloth/issues/10- # self.extra_ignored_labels = torch.full((self.max_seq_length, 1), -100, device = "cuda:0")- # passshift_labels = torch.empty_like(labels)shift_labels[..., :-1] = labels[..., 1:]shift_labels[..., -1] = -100@@ -1591,7 +1501,6 @@ def CausalLM_fast_forward(fast_forward_inference):shift_labels,kwargs.get("packed_seq_lengths"),)- # shift_labels = torch.hstack((labels[..., 1:], self.extra_ignored_labels[:labels.shape[0]]))n_items = kwargs.get("num_items_in_batch", None)if n_items is None:n_items = kwargs.get("n_items", None)@@ -1774,9 +1683,9 @@ def _compute_config_rope_inv_freq(config, rope_scaling):original_rope_scaling = rope_scalingrope_scaling = _rope_scaling_as_dict(rope_scaling)rope_type = rope_scaling.get("rope_type", None) or rope_scaling.get("type", None)- # "default"/unset means unscaled RoPE. transformers >=5 reports- # rope_type="default" for every plain config and dropped "default" from- # ROPE_INIT_FUNCTIONS, so compute it directly instead of warning per load.+ # "default"/unset means unscaled RoPE: transformers >= 5 reports rope_type="default" for every+ # plain config and dropped "default" from ROPE_INIT_FUNCTIONS, so compute it directly instead+ # of warning per load.if rope_type in (None, "default"):return _vanilla_inv_freq_from_config(config).to(dtype = torch.float32, device = "cpu"), 1.0try:@@ -1809,31 +1718,25 @@ def _compute_config_rope_inv_freq(config, rope_scaling):return None, 1.0-# Solves https://github.com/unslothai/unsloth/issues/168-# Static KV Cache was introduced in 4.38.0, causing training to be much slower.-# Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.-# https://github.com/huggingface/transformers/pull/27931-# https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py+# Static KV Cache landed in 4.38.0 and made training much slower (#168,+# huggingface/transformers#27931), so the old rotary embeddings are retained.class LlamaRotaryEmbedding(torch.nn.Module):- # Fixes https://github.com/huggingface/transformers/pull/28837- # https://github.com/microsoft/DeepSpeed/issues/4932- # The precision of RoPE buffers is not correct, so we cast to int64.+ # RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.def __init__(self,dim = None,max_position_embeddings = 2048,base = 10000,device = None,- config = None, # [TODO] Hack to pass in config - need to remove later+ config = None,):super().__init__()# cos/sin multiplier (1.0 except yarn / longrope); set before any cache build.self.attention_scaling = 1.0- # Base-class-from-config path (modern transformers): derive inv_freq like- # transformers so config.rope_scaling is not dropped (#2405). Scaled- # subclasses are excluded to avoid double-scaling.+ # Base-class-from-config path (modern transformers): derive inv_freq like transformers so+ # config.rope_scaling is not dropped (#2405). Scaled subclasses excluded to avoid+ # double-scaling.if config is not None:- # [TODO] Hack to pass in config - need to remove laterbase = _get_rope_theta(config, default = base)partial_rotary_factor = (config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0@@ -1849,7 +1752,7 @@ class LlamaRotaryEmbedding(torch.nn.Module):self.base = base# Kept so the v5 rope repair can rebuild the scaled inv_freq (#2405).self._unsloth_rope_config = config- # Dynamic RoPE we first set it to a max of 4 * 8192 tokens then we iteratively grow this+ # Dynamic RoPE starts at a max of 4 * 8192 tokens and grows iteratively in increments of 8192.self.current_rope_size = min(4 * 8192, self.max_position_embeddings)self.multi_gpu_cos_cached = [None] * DEVICE_COUNTself.multi_gpu_sin_cached = [None] * DEVICE_COUNT@@ -1857,7 +1760,6 @@ class LlamaRotaryEmbedding(torch.nn.Module):inv_freq = self._unsloth_recompute_inv_freq()self.register_buffer("inv_freq", inv_freq, persistent = False)- # Build here to make `torch.jit.trace` work.for device_idx in range(DEVICE_COUNT):self._set_cos_sin_cache(seq_len = self.current_rope_size,@@ -1865,7 +1767,6 @@ class LlamaRotaryEmbedding(torch.nn.Module):dtype = torch.get_default_dtype(),)- # dummy so that patch_utils doesn't fail for nowself.cos_cached = torch.empty(1, device = get_current_device(), dtype = torch.get_default_dtype())@@ -1878,8 +1779,8 @@ class LlamaRotaryEmbedding(torch.nn.Module):return inv_freqdef _unsloth_recompute_inv_freq(self):- # Config scaling (llama3/yarn) first, else vanilla + subclass scaling.- # Shared by __init__ and the v5 rope repair so they cannot diverge.+ # Config scaling (llama3/yarn) first, else vanilla plus subclass scaling. Shared by __init__ and+ # the v5 rope repair so they cannot diverge.config = getattr(self, "_unsloth_rope_config", None)config_inv_freq = Nonerope_scaling = getattr(config, "rope_scaling", None) if config is not None else None@@ -1901,8 +1802,7 @@ class LlamaRotaryEmbedding(torch.nn.Module):return tdef _set_cos_sin_cache(self, seq_len, device, dtype):- # Note: on the original Llama codebase, these tensors are created on the target device (and not on CPU) and- # in FP32. They are applied (multiplied) in FP32 as well.+ # The original Llama codebase creates these on the target device in FP32 and multiplies in FP32.self.current_rope_size = seq_lent = torch.arange(self.current_rope_size, device = self.inv_freq.device, dtype = torch.int64@@ -1910,10 +1810,10 @@ class LlamaRotaryEmbedding(torch.nn.Module):t = self._apply_time_scaling(t)freqs = torch.outer(t, self.inv_freq)- # Different from paper, but it uses a different permutation in order to obtain the same calculation+ # Different from the paper, but it uses a different permutation to obtain the same calculation.emb = torch.cat((freqs, freqs), dim = -1)- # Applied here so attention_scaling survives extend_rope_embedding rebuilds;- # default 1.0 keeps unscaled paths bit-identical.+ # Applied here so attention_scaling survives extend_rope_embedding rebuilds; the 1.0 default+ # keeps unscaled paths bit-identical.cos = (emb.cos() * self.attention_scaling).to(dtype = dtype, device = device, non_blocking = True)sin = (emb.sin() * self.attention_scaling).to(dtype = dtype, device = device, non_blocking = True)self.multi_gpu_cos_cached[device.index] = cos@@ -1926,7 +1826,6 @@ class LlamaRotaryEmbedding(torch.nn.Module):position_ids = None,seq_len = None,):- # x: [bs, num_attention_heads, seq_len, head_size]if seq_len is not None and seq_len > self.current_rope_size:self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)@@ -1948,7 +1847,7 @@ class LlamaRotaryEmbedding(torch.nn.Module):def extend_rope_embedding(self, x, seq_len):if seq_len <= self.current_rope_size:return- # Iteratively grow by increments of 8192+ # Iteratively grow by increments of 8192.self.current_rope_size = ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192for device_idx in range(DEVICE_COUNT):self._set_cos_sin_cache(@@ -1959,9 +1858,7 @@ class LlamaRotaryEmbedding(torch.nn.Module):class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):"""LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""- # Fixes https://github.com/huggingface/transformers/pull/28837- # https://github.com/microsoft/DeepSpeed/issues/4932- # The precision of RoPE buffers is not correct, so we cast to int64.+ # RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.def __init__(self,dim = None,@@ -1969,7 +1866,7 @@ class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):base = 10000,device = None,scaling_factor = 1.0,- config = None, # [TODO] Hack to pass in config - need to remove later+ config = None,):self.scaling_factor = scaling_factorsuper().__init__(@@ -1985,8 +1882,7 @@ class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):return t / self.scaling_factor-# See https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/rotary_embedding.py#L736-# For Llama 3.1+# For Llama 3.1; see vllm model_executor/layers/rotary_embedding.py#L736class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):def __init__(self,@@ -1994,7 +1890,7 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):max_position_embeddings = 2048,base = 10000,device = None,- config = None, # [TODO] Hack to pass in config - need to remove later+ config = None,):super().__init__(dim = dim,@@ -2004,12 +1900,10 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):config = config,)- # From https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/api/model.py#L41+ # From meta-llama/llama-models llama3_1/api/model.py#L41def _apply_inv_freq_scaling(self, freqs: torch.Tensor):- # llama3 factors from config; Llama-3.1 defaults when built without one- # (legacy codegen path). Hardcoding 8 is wrong for e.g. Llama-3.2 (32).- # v5 renames rope_scaling -> rope_parameters; read either so the factor- # survives even if the rope_scaling back-compat shim is dropped.+ # llama3 factors come from the config, with Llama-3.1 defaults when built without one: hardcoding+ # 8 is wrong for Llama-3.2 (32). v5 renames rope_scaling to rope_parameters, so read either.config = getattr(self, "_unsloth_rope_config", None)rope_scaling = _rope_scaling_as_dict(getattr(config, "rope_scaling", None) or getattr(config, "rope_parameters", None) or {}@@ -2038,7 +1932,7 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):class LongRopeRotaryEmbedding(torch.nn.Module):- # For Phi 3.5 128K https://huggingface.co/microsoft/Phi-3.5-mini-instruct/blob/main/modeling_phi3.py+ # For Phi 3.5 128K; see microsoft/Phi-3.5-mini-instruct modeling_phi3.pydef __init__(self,dim = None,@@ -2048,7 +1942,7 @@ class LongRopeRotaryEmbedding(torch.nn.Module):short_factor = None,long_factor = None,device = None,- config = None, # [TODO] Hack to pass in config - need to remove later+ config = None,):super().__init__()assert short_factor is not None@@ -2056,7 +1950,6 @@ class LongRopeRotaryEmbedding(torch.nn.Module):assert type(original_max_position_embeddings) is intif config is not None:- # [TODO] Hack to pass in config - need to remove laterbase = _get_rope_theta(config, default = base)partial_rotary_factor = (config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0@@ -2069,15 +1962,14 @@ class LongRopeRotaryEmbedding(torch.nn.Module):self.max_position_embeddings = max_position_embeddingsself.original_max_position_embeddings = original_max_position_embeddingsself.base = base- # Dynamic RoPE we first set it to a max of 4 * 8192 tokens then we iteratively grow this+ # Dynamic RoPE starts at a max of 4 * 8192 tokens and grows iteratively.self.current_rope_size = min(original_max_position_embeddings, self.max_position_embeddings)self.multi_gpu_short_cos_cached = [None] * DEVICE_COUNTself.multi_gpu_short_sin_cached = [None] * DEVICE_COUNTself.multi_gpu_long_cos_cached = [None] * DEVICE_COUNTself.multi_gpu_long_sin_cached = [None] * DEVICE_COUNT- # Long RoPE similar to RoPE except short sequences have 1 cos / sin- # and long sequences have another cos / sin+ # Long RoPE is like RoPE except short sequences have one cos/sin and long sequences another.inv_freq_shape = (torch.arange(0, self.dim, 2, dtype = torch.int64, device = "cpu").float() / self.dim)@@ -2086,7 +1978,7 @@ class LongRopeRotaryEmbedding(torch.nn.Module):short_inv_freq = 1.0 / (short_factor * self.base**inv_freq_shape)long_inv_freq = 1.0 / (long_factor * self.base**inv_freq_shape)- # Phi-3 Scale factor+ # Phi-3 scale factor.scale = self.max_position_embeddings / self.original_max_position_embeddingsif scale <= 1.0:scaling_factor = 1.0@@ -2096,12 +1988,10 @@ class LongRopeRotaryEmbedding(torch.nn.Module):)self.scaling_factor = scaling_factor- # Short and long inv_freqself.register_buffer("short_inv_freq", short_inv_freq, persistent = False)self.register_buffer("long_inv_freq", long_inv_freq, persistent = False)- # Build here to make `torch.jit.trace` work.- # Initialize short sequences cache for all devices+ # Build here to make torch.jit.trace work, initializing the short-sequence cache for all devices.dtype = torch.bfloat16 if is_bfloat16_supported() else torch.float16t = torch.arange(original_max_position_embeddings,@@ -2122,7 +2012,6 @@ class LongRopeRotaryEmbedding(torch.nn.Module):self.multi_gpu_short_cos_cached[device_idx] = cos_cachedself.multi_gpu_short_sin_cached[device_idx] = sin_cached- # dummy so that patch_utils doesn't fail for nowself.short_cos_cached = torch.empty(1, device = get_current_device(), dtype = torch.get_default_dtype())@@ -2137,14 +2026,12 @@ class LongRopeRotaryEmbedding(torch.nn.Module):)def _set_cos_sin_cache(self, seq_len, device, dtype):- # Note: on the original Llama codebase, these tensors are created on the target device (and not on CPU) and- # in FP32. They are applied (multiplied) in FP32 as well.+ # The original Llama codebase creates these on the target device in FP32 and multiplies in FP32.self.current_rope_size = seq_lent = torch.arange(self.current_rope_size, device = self.long_inv_freq.device, dtype = torch.int64).float()- # Long sequencesfreqs = torch.outer(t, self.long_inv_freq)emb = torch.cat((freqs, freqs), dim = -1)cos_cached = (emb.cos() * self.scaling_factor).to(@@ -2163,7 +2050,6 @@ class LongRopeRotaryEmbedding(torch.nn.Module):position_ids = None,seq_len = None,):- # x: [bs, num_attention_heads, seq_len, head_size]if seq_len is not None and seq_len > self.current_rope_size:self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)@@ -2198,7 +2084,7 @@ class LongRopeRotaryEmbedding(torch.nn.Module):def extend_rope_embedding(self, x, seq_len):if seq_len <= self.current_rope_size:return- # Iteratively grow by increments of 8192+ # Iteratively grow by increments of 8192.self.current_rope_size = ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192for device_idx in range(DEVICE_COUNT):self._set_cos_sin_cache(@@ -2207,10 +2093,9 @@ class LongRopeRotaryEmbedding(torch.nn.Module):def unsloth_fast_generate(self, *args, **kwargs):- # Restore training mode after generation if we started in itrestore_training_mode = self.training- # Snapshot the real GC mode (e.g. "unsloth") before for_inference clears it,- # so the restore preserves it rather than collapsing to a plain bool.+ # Snapshot the real GC mode (e.g. "unsloth") before for_inference clears it, so the restore+ # preserves it rather than collapsing to a plain bool.use_gradient_checkpointing = next((v for v in (getattr(m, "gradient_checkpointing", False) for m in self.modules()) if v),False,@@ -2218,9 +2103,8 @@ def unsloth_fast_generate(self, *args, **kwargs):FastLlamaModel.for_inference(self)- # Unpack BatchEncoding passed as input_ids (old notebooks do- # generate(input_ids=tokenizer(...))). v5 generate() calls .shape on it and- # crashes; unpack into separate kwargs so v4 and v5 both work.+ # Unpack a BatchEncoding passed as input_ids (old notebooks do generate(input_ids=tokenizer(...))):+ # v5 generate() calls .shape on it and crashes._maybe_encoding = kwargs.get("input_ids", None)if (_maybe_encoding is not None@@ -2244,16 +2128,11 @@ def unsloth_fast_generate(self, *args, **kwargs):"You will need to do long context extension by increasing the `max_seq_length` in `FastLanguageModel.from_pretrained`.")- # Must patch accelerate for Xformers- # if accelerate_new_send_to_device is not None:- # import accelerate.utils.operations- # accelerate.utils.operations.send_to_device = accelerate_new_send_to_device- # pass+ # Must patch accelerate for Xformers.- # For newer HFkwargs["cache_implementation"] = "dynamic"- # transformers 4.50 renamed num_logits_to_keep -> logits_to_keep; pop both,- # re-emit under the spelling forward() accepts.+ # transformers 4.50 renamed num_logits_to_keep to logits_to_keep; pop both and re-emit under the+ # spelling forward() accepts._provided_num = kwargs.pop("num_logits_to_keep", None)_provided_logits = kwargs.pop("logits_to_keep", None)_provided = _provided_logits if _provided_logits is not None else _provided_num@@ -2262,7 +2141,7 @@ def unsloth_fast_generate(self, *args, **kwargs):_has_new = "logits_to_keep" in _fwd_params_has_old = "num_logits_to_keep" in _fwd_paramsexcept (TypeError, ValueError):- # Opaque forward: keep the caller's spelling, default to new.+ # Opaque forward: keep the caller's spelling, defaulting to the new one._has_old = _provided_num is not None and _provided_logits is None_has_new = not _has_oldif _has_new:@@ -2270,28 +2149,21 @@ def unsloth_fast_generate(self, *args, **kwargs):elif _has_old:kwargs["num_logits_to_keep"] = _provided if _provided is not None else 1- # Remove token_type_idskwargs.pop("token_type_ids", None)- # Check pad_tokenmodel_eos_token_id = getattr(self.config, "eos_token_id", None)if model_eos_token_id is not None and hasattr(model_eos_token_id, "__iter__"):model_eos_token_id = model_eos_token_id[0]kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)- # Mixed precision autocast+ # Mixed precision autocast.with (_get_inference_mode_context_manager(self),torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),):output = self._old_generate(*args, **kwargs)- # Return accelerate back- # if accelerate_new_send_to_device is not None:- # accelerate.utils.operations.send_to_device = accelerate_old_send_to_device- # pass-if restore_training_mode:FastLlamaModel.for_training(self,@@ -2348,11 +2220,8 @@ class FastLlamaModel:PeftModelForCausalLM.forward = PeftModel_fast_forwardfix_prepare_inputs_for_generation(LlamaForCausalLM)- # Solves https://github.com/unslothai/unsloth/issues/168- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.- # https://github.com/huggingface/transformers/pull/27931- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,+ # huggingface/transformers#27931), so the old rotary embeddings are retained.import transformers.models.llama.modeling_llamatransformers.models.llama.modeling_llama.LlamaRotaryEmbedding = LlamaRotaryEmbedding@@ -2417,11 +2286,10 @@ class FastLlamaModel:raise RuntimeError("Unsloth: `unsloth_vllm_standby` is True, but environment variable `UNSLOTH_VLLM_STANDBY` is not set to 1!")- # Only vLLM cannot take a revision; fast_inference may have just been cleared- # above, and a num_labels load stays in-process, so both can honour the pin.+ # Only vLLM cannot take a revision, and fast_inference may have just been cleared above while a+ # num_labels load stays in-process, so both can honour the pin. vLLM fetches the default branch,+ # so pinning only the config and tokenizer would mix two refs in one model.if _vllm_will_load_weights(fast_inference, num_labels) and revision is not None:- # vLLM fetches the default branch, so pinning only the config and tokenizer- # would mix two refs in one model.logger.warning_once(f"Unsloth: Ignoring revision = `{revision}` since vLLM loads weights from ""the default branch. Use `fast_inference = False` to load a pinned revision."@@ -2455,7 +2323,6 @@ class FastLlamaModel:print(statistics)- # Warn about fast transfersif "HF_HUB_ENABLE_HF_TRANSFER" in os.environ:old_hf_transfer = os.environ["HF_HUB_ENABLE_HF_TRANSFER"]if old_hf_transfer in ("False", "false"):@@ -2472,7 +2339,7 @@ class FastLlamaModel:os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"model_patcher.pre_patch()- # For debugging - we use a download counter to see if environments are not breaking or if HF is down+ # A download counter, to see whether environments are breaking or HF is down.get_statistics(kwargs.get("local_files_only", False))if dtype is None:@@ -2480,22 +2347,16 @@ class FastLlamaModel:elif dtype == torch.bfloat16 and not SUPPORTS_BFLOAT16:logger.warning_once("Device does not support bfloat16. Will change to float16.")dtype = torch.float16- # elif dtype == torch.float16 and SUPPORTS_BFLOAT16:- # logger.warning_once("Device supports bfloat16 but you selected float16. Will change to bfloat16.")- # dtype = torch.bfloat16assert dtype == torch.float16 or dtype == torch.bfloat16 or dtype == torch.float32- # RoPE Scaling- # Respect a user-provided config so it is the single config object used- # everywhere below; otherwise HF would receive it again through **kwargs- # alongside our own config= and fail with a duplicate-kwarg TypeError.+ # Respect a user-provided config so it is the single config object used everywhere below; else+ # HF gets it again through **kwargs alongside our config= and fails with a duplicate kwarg.user_config = kwargs.pop("config", None)if user_config is not None:model_config = user_config- # model_name may have been remapped to a prequantized repo whose- # checkpoint needs its quantization_config; graft it onto the user- # config or the 4bit weights load without their quant state.+ # model_name may have been remapped to a prequantized repo whose checkpoint needs its+ # quantization_config; graft it on, or the 4bit weights load without their quant state.if getattr(model_config, "quantization_config", None) is None:_checkpoint_config = AutoConfig.from_pretrained(model_name,@@ -2527,8 +2388,8 @@ class FastLlamaModel:)# Prefetch the repo (killable child) so the weight load is a cache hit. Runs after the- # AutoConfig/model-class check so an unsupported repo fails on its small config fetch.- # Warm the revision the load uses, or the repo downloads twice.+ # AutoConfig/model-class check so an unsupported repo fails on its small config fetch, and warms+ # the revision the load uses or the repo downloads twice._prefetched = maybe_prefetch_hf_snapshot(model_name,token = token,@@ -2543,21 +2404,20 @@ class FastLlamaModel:use_safetensors = kwargs.get("use_safetensors"),from_tf = kwargs.get("from_tf", False),from_flax = kwargs.get("from_flax", False),- # Bare load reads only ROOT weights; skip subdir weights. Ignored when a subfolder is set.+ # A bare load reads only ROOT weights, so skip subdir weights; ignored when a subfolder is set.weights_at_root = True,variant = kwargs.get("variant"), # forward so the warm keeps the variant .bingguf_file = kwargs.get("gguf_file"), # forward so the warm fetches the GGUF (else ignored))- # Child did the forced download; clear the flag so the load reuses the warm cache.+ # The child did the forced download; clear the flag so the load reuses the warm cache.if _prefetched and kwargs.get("force_download", False):kwargs["force_download"] = False- # Tokenizer always loads in-process. Resolve the cache_dir the tokenizer load will actually- # use, mirroring load_correct_tokenizer: without an explicit cache_dir, Colab/Kaggle route to- # a special tokenizer cache (huggingface_tokenizers_cache / Kaggle tmp), NOT the HF-default- # cache the base snapshot warmed. So the base warm does not cover the tokenizer there.+ # The tokenizer always loads in-process, and without an explicit cache_dir Colab/Kaggle route it+ # to a special tokenizer cache rather than the one the base snapshot warmed, so resolve the+ # cache_dir the tokenizer load will actually use.from ..tokenizer_utils import (IS_COLAB_ENVIRONMENT,IS_KAGGLE_ENVIRONMENT,@@ -2573,9 +2433,8 @@ class FastLlamaModel:_tokenizer_cache_dir = "huggingface_tokenizers_cache"elif IS_KAGGLE_ENVIRONMENT:_tokenizer_cache_dir = os.path.join(KAGGLE_TMP, "huggingface_tokenizers_cache")- # Warm the tokenizer repo into the cache the load will use whenever the base warm did not- # cover it: a distinct tokenizer repo, fast_inference (base warm skipped), or a tokenizer- # cache_dir that differs from the base-warm cache_dir (Colab/Kaggle special cache).+ # Warm the tokenizer repo into the cache the load will use whenever the base warm did not cover+ # it: a distinct tokenizer repo, fast_inference (base warm skipped), or a differing cache_dir._warm_tokenizer_repo = (isinstance(_tokenizer_repo, str)and bool(_tokenizer_repo)@@ -2603,7 +2462,7 @@ class FastLlamaModel:passhas_rope_scaling = True- # If max_seq_length is not specified, use maximum from config+ # If max_seq_length is not specified, use the maximum from the config.if max_seq_length is None:max_seq_length = model_max_seq_length@@ -2642,24 +2501,20 @@ class FastLlamaModel:)from unsloth_zoo.utils import get_quant_type- # Extract load_in_8bit from kwargs if providedload_in_8bit = kwargs.get("load_in_8bit", False)- # Check and disable bitsandbytes loading if model has non-bitsandbytes quantization+ # Disable bitsandbytes loading if the model has non-bitsandbytes quantization.load_in_4bit, load_in_8bit, _ckpt_quant_method = check_and_disable_bitsandbytes_loading(model_config, load_in_4bit = load_in_4bit, load_in_8bit = load_in_8bit)- # Correct UNSLOTH_MODEL_NAME's bnb tokens now that the effective bnb state is known- # (the per-load env was built before remap/disable). gpt-oss only; no-op otherwise.+ # Correct UNSLOTH_MODEL_NAME's bnb tokens now the effective bnb state is known (the per-load env+ # was built before remap/disable). gpt-oss only.sync_unsloth_model_name_bnb_flags(load_in_4bit, load_in_8bit)- # `num_labels` sends the load to AutoModelForSequenceClassification, whose `score`- # replaces the `lm_head` the planner named off the repo's config, so `dispatch_model`- # refuses the map: "does not give any device for ... score.weight".+ # num_labels sends the load to AutoModelForSequenceClassification, whose `score` replaces the+ # lm_head the planner named off the repo's config, so dispatch_model refuses the map. The weights+ # load against user_config while the planner rebuilds the repo's from model_name._planner_skip_reason = None- # The weights load against `user_config`; the planner rebuilds the repo's from- # `model_name`. A changed `num_hidden_layers` or `vocab_size` then gets a map for a- # different model, which dispatches short or budgets short.if user_config is not None:_planner_skip_reason = ("a caller-supplied config may not describe the repo the planner rebuilds"@@ -2673,15 +2528,12 @@ class FastLlamaModel:or "num_labels loads a task head the repo config does not describe")- # A prequantized checkpoint is always sized by the list in its own config.json:- # `merge_quantization_configs` overlays loading attributes for GPTQ / AWQ /- # AutoRound / FbgemmFp8 / CompressedTensors / Mxfp4 and never for bitsandbytes, so- # neither the merge below (#5027) nor any argument here reaches the plan. Modules- # the load then keeps dense are charged at 4bit, budgeting the head device short.- #- # Worth refusing over only for mamba, whose `out_proj` stacks are GiBs of error. The- # other reachable gap is MoE routers, ~70 MiB even on Qwen3-235B-A22B (4096 x 128- # experts x 94 layers), inside the planner's own 256 MiB safety margin.+ # A prequantized checkpoint is always sized by its own config.json: merge_quantization_configs+ # overlays loading attributes for GPTQ / AWQ / AutoRound / FbgemmFp8 / CompressedTensors / Mxfp4+ # and never for bitsandbytes, so neither the merge below (#5027) nor any argument here reaches the+ # plan. Worth refusing over only for mamba, whose out_proj stacks are GiBs of error; the other+ # reachable gap is MoE routers, ~70 MiB even on Qwen3-235B-A22B, inside the planner's 256 MiB+ # margin.if _planner_skip_reason is None and IS_FALCON_H1 and _ckpt_quant_method == "bitsandbytes":_bundled = getattr(model_config, "quantization_config", None)_bundled_skip = (@@ -2695,9 +2547,8 @@ class FastLlamaModel:"load adds, so the plan would size dense weights at 4bit")- # Here, not in loader.py: the mapper up there can still substitute the repo (a- # -bnb-4bit name resolving to its 16-bit twin), so a plan sized for the name the- # caller gave is the wrong plan for the one actually loaded.+ # Here, not in loader.py: the mapper there can still substitute the repo (a -bnb-4bit name+ # resolving to its 16-bit twin), so a plan sized for the caller's name is the wrong plan.device_map = resolve_unsloth_device_map(requested_device_map(device_map),model_name,@@ -2709,11 +2560,11 @@ class FastLlamaModel:trust_remote_code = trust_remote_code,**planner_hub_kwargs(kwargs),revision = revision,- # The dtype the load gets, not the checkpoint's: `from_pretrained` overrides- # config.json, and planning the wrong one mis-sizes weights 2x either way.+ # The dtype the load gets, not the checkpoint's: from_pretrained overrides config.json, and+ # planning the wrong one mis-sizes weights 2x either way.**add_dtype_kwargs(dtype),- # The caller's own config, still untouched in kwargs here, overrides the- # flags: loader.py clears them whenever it forwards one.+ # The caller's own config, still untouched in kwargs here, overrides the flags: loader.py clears+ # them whenever it forwards one.**planner_quantization_kwargs(load_in_4bit = load_in_4bit,load_in_8bit = load_in_8bit,@@ -2729,7 +2580,7 @@ class FastLlamaModel:if load_in_4bit:llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES.copy()if IS_FALCON_H1:- # we cannot quantize out_proj layer due to mamba kernels: https://github.com/tiiuae/Falcon-H1/issues/13#issuecomment-2918671274+ # out_proj cannot be quantized due to mamba kernels; see tiiuae/Falcon-H1#13 (comment 2918671274).llm_int8_skip_modules.append("out_proj")bnb_config = BitsAndBytesConfig(load_in_4bit = True,@@ -2738,10 +2589,9 @@ class FastLlamaModel:bnb_4bit_compute_dtype = dtype,llm_int8_skip_modules = llm_int8_skip_modules,)- # Pre-quantized checkpoints (e.g. unsloth/Qwen3-4B-bnb-4bit) use the- # quantization_config baked into config.json, ignoring our runtime- # BitsAndBytesConfig. Merge our skip list into the bundled config so- # task heads like `score` stay in compute dtype. See unslothai/unsloth#5027.+ # Pre-quantized checkpoints use the quantization_config baked into config.json, ignoring our+ # runtime BitsAndBytesConfig, so merge our skip list into it to keep task heads like `score` in+ # compute dtype (#5027).if _ckpt_quant_method == "bitsandbytes" and _ckpt_qcfg is not None:if isinstance(_ckpt_qcfg, dict):_ckpt_skip = list(_ckpt_qcfg.get("llm_int8_skip_modules") or [])@@ -2759,12 +2609,11 @@ class FastLlamaModel:except Exception:pass- # https://huggingface.co/togethercomputer/LLaMA-2-7B-32K/discussions/12- # RoPE Scaling's max_position_embeddings must be updated+ # RoPE Scaling's max_position_embeddings must be updated; see togethercomputer/LLaMA-2-7B-32K discussion 12.max_position_embeddings = max(max_seq_length, model_max_seq_length)kwargs.pop("attn_implementation", None) # No need since we auto call it- # Cannot be None, since HF now checks for the config+ # Cannot be None, since HF now checks for the config.if load_in_4bit:kwargs["quantization_config"] = bnb_config@@ -2773,13 +2622,12 @@ class FastLlamaModel:raise_handler = RaiseUninitialized()try:if num_labels is not None:- # Transformers 5.x @strict config classes reject unexpected kwargs- # like num_labels and max_position_embeddings. Set on the config- # object directly and pass config= instead.+ # Transformers 5.x @strict config classes reject unexpected kwargs like num_labels and+ # max_position_embeddings, so set them on the config object and pass config= instead.set_task_config_attr(model_config, "num_labels", num_labels)if max_position_embeddings is not None:model_config.max_position_embeddings = max_position_embeddings- # Pop config-level attrs that would be rejected by @strict model init+ # Pop config-level attrs that would be rejected by @strict model init.for _cfg_key in ("id2label", "label2id", "rope_scaling"):_cfg_val = kwargs.pop(_cfg_key, None)if _cfg_val is not None:@@ -2791,16 +2639,14 @@ class FastLlamaModel:model_name,config = model_config,device_map = device_map,- # torch_dtype = dtype, # transformers changed torch_dtype to dtype- # quantization_config = bnb_config,token = token,trust_remote_code = trust_remote_code,attn_implementation = preferred_attn_impl,revision = revision,**kwargs,)- # Defensive: ensure the task head is in a floating dtype, guarding- # against any path leaving it as integer storage. See unslothai/unsloth#5027.+ # Defensive: ensure the task head is in a floating dtype, guarding against any path leaving it+ # as integer storage (#5027).for _head_name in ("score", "classifier", "qa_outputs"):_head = getattr(model, _head_name, None)if (@@ -2819,13 +2665,13 @@ class FastLlamaModel:offload_embedding = False,fast_inference = fast_inference,)- # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200).+ # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200), reading+ # scales from the same revision as the weights._restore_dropped_fp8_scales(model,model_name,local_files_only = kwargs.get("local_files_only", False),token = token,- # Read scales from the same revision as the weights.revision = revision,subfolder = kwargs.get("subfolder"),cache_dir = kwargs.get("cache_dir"),@@ -2833,9 +2679,8 @@ class FastLlamaModel:)elif not fast_inference:if user_config is not None:- # Transformers 5.x @strict model init rejects extra kwargs next- # to config=; set the override on the config and pass the single- # config object through so user overrides reach the actual load.+ # Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override+ # on the config and pass the single config object through.if max_position_embeddings is not None:model_config.max_position_embeddings = max_position_embeddingsmodel = AutoModelForCausalLM.from_pretrained(@@ -2852,8 +2697,6 @@ class FastLlamaModel:model = AutoModelForCausalLM.from_pretrained(model_name,device_map = device_map,- # torch_dtype = dtype, # transformers changed torch_dtype to dtype- # quantization_config = bnb_config,token = token,max_position_embeddings = max_position_embeddings,trust_remote_code = trust_remote_code,@@ -2861,7 +2704,6 @@ class FastLlamaModel:revision = revision,**kwargs,)- # Attach dispatch hooks for bnb multi-device loads.from unsloth.models.vision import _attach_bnb_multidevice_hooks_attach_bnb_multidevice_hooks(@@ -2871,13 +2713,13 @@ class FastLlamaModel:offload_embedding = False,fast_inference = False,)- # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200).+ # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200), reading+ # scales from the same revision as the weights._restore_dropped_fp8_scales(model,model_name,local_files_only = kwargs.get("local_files_only", False),token = token,- # Read scales from the same revision as the weights.revision = revision,subfolder = kwargs.get("subfolder"),cache_dir = kwargs.get("cache_dir"),@@ -2920,10 +2762,8 @@ class FastLlamaModel:load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]pass- # Load vLLM firstllm = load_vllm(**load_vllm_kwargs)- # Convert to HF format_, quant_state_dict = get_vllm_state_dict(llm,config = model_config,@@ -2938,10 +2778,9 @@ class FastLlamaModel:model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)finally:raise_handler.remove()- # Return old flagos.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer- # Counteract saved tokenizers+ # Counteract saved tokenizers.tokenizer_name = model_name if tokenizer_name is None else tokenizer_name# Route the tokenizer load to the custom cache_dir the prefetch warmed._tokenizer_cache_kwargs = {}@@ -2961,12 +2800,10 @@ class FastLlamaModel:model, tokenizer = patch_tokenizer(model, tokenizer)model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)- # Patch up QKV / O and MLPfor idx, layer in enumerate(model.model.layers):layer.self_attn.apply_qkv = original_apply_qkvlayer.self_attn.apply_o = original_apply_o- # Patch Trainerfrom transformers.trainer import Trainertry:@@ -2998,8 +2835,7 @@ class FastLlamaModel:spaces = re.search(r"\n([\s\t]{1,})", original_debug).group(0)[1:]front_spaces = re.match(r"([\s\t]{1,})", inner_training_loop).group(0)- # Cannot use \\ since it will cause a SyntaxWarning in Python 3.12- # Instead use chr(92) == \\+ # Cannot use a backslash escape (SyntaxWarning in Python 3.12), so use chr(92).debug_info = """debug_info = \\f"==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = {len(set(p.device for p in model.parameters()))}\\n"\\f" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\n"\\@@ -3042,11 +2878,10 @@ class FastLlamaModel:"is_torch_tpu_available()","False",)- # Wire the stray-forward compile-cache reset into the plain Trainer path: get_peft_model- # arms the pre-train detector for every LoRA model, but only the TRL SFT/RL wrappers run- # the reset. A grad-enabled probe before a bare transformers.Trainer.train() would- # otherwise keep the poisoned Dynamo cache and leave the detector hook installed. Anchored- # on the first body statement; a no-op (and harmless) if upstream drops that line.+ # Wire the stray-forward compile-cache reset into the plain Trainer path: get_peft_model arms+ # the detector for every LoRA model, but only the TRL SFT/RL wrappers run the reset, so a+ # grad-enabled probe before a bare Trainer.train() would keep the poisoned Dynamo cache.+ # Anchored on the first body statement; a harmless no-op if upstream drops that line.inner_training_loop = inner_training_loop.replace("self.accelerator.free_memory()","self.accelerator.free_memory()\n"@@ -3059,18 +2894,15 @@ class FastLlamaModel:exec(inner_training_loop, globals())Trainer._inner_training_loop = _fast_inner_training_loop- # Save max_seq_lengthmodel.max_seq_length = max_seq_lengthm = modelwhile hasattr(m, "model"):m.max_seq_length = max_seq_lengthm = m.modelm.max_seq_length = max_seq_length- # Save to modules as wellfor module in model.modules():module.max_seq_length = max_seq_length- # We check the tokenizer first for errorsif fix_tokenizer:tokenizer = check_tokenizer(model = model,@@ -3083,27 +2915,24 @@ class FastLlamaModel:)patch_saving_functions(tokenizer)- # Fix up config for transformers uploading PEFT- # Not necessary anymore since we require transformers>=4.37!+ # Not necessary since we requires transformers >= 4.37if False:name = model.config._name_or_pathif name.startswith("unsloth/") and name.endswith("-bnb-4bit"):name = name[: len(name) - len("-bnb-4bit")]model.config.update({"_name_or_path": name})- # Log Unsloth version for future fastpaths for inference+ # Log the Unsloth version for future inference fastpaths.model.config.update({"unsloth_version": __version__})- # Add save modulespatch_saving_functions(model)Trainer._inner_training_loop = _fast_inner_training_loop- # Fix gradient accumulation. See issue #4982.+ # Fix gradient accumulation; see #4982.apply_accepts_loss_kwargs_fix(model)patch_gradient_accumulation_fix(Trainer)- # Save tokenizer for inference purposes- tokenizer.padding_side = "left" # Force inference+ tokenizer.padding_side = "left"internal_model = modelwhile hasattr(internal_model, "model"):internal_model._saved_temp_tokenizer = tokenizer@@ -3113,13 +2942,12 @@ class FastLlamaModel:# Prevent Transformers Trainer from auto-wrapping Unsloth LoRA models in DP._mark_unsloth_disable_data_parallel(model)- # For transformers > 4.47.1, we need to add rotary_emb to all attention layers+ # For transformers > 4.47.1, rotary_emb must be added to all attention layers.if IS_ATTENTION_REFACTOR or hasattr(model.model, "rotary_emb"):rotary_emb = model.model.rotary_embfor layer in model.model.layers:layer.self_attn.rotary_emb = rotary_emb- # Add for_inference and for_trainingmodel.for_training = functools.partial(FastLlamaModel.for_training, model)model.for_inference = functools.partial(FastLlamaModel.for_inference, model)m = model@@ -3128,16 +2956,14 @@ class FastLlamaModel:m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model- # Patch generateis_classification = "Classification" in str(type(model))if not is_classification and model.generate.__name__ != "unsloth_fast_generate":model._old_generate = model.generateunsloth_fast_generate.__doc__ = model._old_generate.__doc__model.generate = types.MethodType(unsloth_fast_generate, model)- # Zero weight[padding_idx] only for embeddings NOT tied to lm_head: when- # tied, zeroing the row forces pad logit = 0, which beats the (negative)- # logits of real tokens (e.g. Gemma) and makes the decoder emit <pad>.- # Skip if eos_token == pad_token to avoid zeroing the EOS embedding.+ # Zero weight[padding_idx] only for embeddings NOT tied to lm_head: when tied, zeroing the row+ # forces pad logit = 0, which beats the negative logits of real tokens (Gemma) and makes the+ # decoder emit <pad>. Skipped when eos_token == pad_token.eos_token_id = getattr(tokenizer, "eos_token_id", None) if tokenizer is not None else Nonepad_token_id = getattr(tokenizer, "pad_token_id", None) if tokenizer is not None else Noneif tokenizer is not None and eos_token_id != pad_token_id:@@ -3151,7 +2977,6 @@ class FastLlamaModel:and getattr(module, "padding_idx", None) is not None):if module.padding_idx < module.weight.shape[0]:- # Skip if tied to lm_headif (lm_head_weight is not Noneand module.weight.data_ptr() == lm_head_weight.data_ptr()@@ -3159,9 +2984,9 @@ class FastLlamaModel:continuemodule.weight[module.padding_idx] = 0- # LAST: `post_patch` replaces the embedding modules and the QKV/MLP- # patching below it replaces the forwards a hook wraps, so an earlier- # attach is lost. Skipped under vLLM, which owns the weights.+ # LAST: post_patch replaces the embedding modules and the QKV/MLP patching below replaces the+ # forwards a hook wraps, so an earlier attach is lost. Skipped under vLLM, which owns the+ # weights.if not fast_inference:try:from unsloth.models.vision import _repair_dispatch_hooks@@ -3222,7 +3047,6 @@ class FastLlamaModel:**kwargs,):if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1":- # Check for other PEFT args in kwargsfor peft_arg, flag in (("finetune_vision_layers", False),("finetune_language_layers", True),@@ -3256,13 +3080,12 @@ class FastLlamaModel:)if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")- # Full finetuning still compiles, so a stray pre-train forward can poison the- # cache; install the detector here too (it is idempotent).+ # Full finetuning still compiles, so a stray pre-train forward can poison the cache; install the+ # detector here too (idempotent)._unsloth_install_pretrain_detector(model)return modeltransformers_set_seed(random_state)- # Apply gradient checkpointing with smart heuristicsmax_seq = getattr(model, "max_seq_length", 512)dtype = model.get_input_embeddings().weight.dtypeuse_gradient_checkpointing = apply_unsloth_gradient_checkpointing(@@ -3277,7 +3100,6 @@ class FastLlamaModel:if isinstance(model, PeftModelForCausalLM) or isinstance(model, PeftModelForSequenceClassification):- # Check if exactly the same and then pass through!assert hasattr(model, "peft_config")peft_config = model.peft_config["default"].to_dict()@@ -3295,18 +3117,15 @@ class FastLlamaModel:for param in check_parameters:check_all = check_all and (peft_config[param] == eval(param))- # use_dora arrives via **kwargs, not a named parameter of this- # function, so it can't go in check_parameters (which uses- # eval(param) and needs each entry to be a bound local name).- # Compare it explicitly so a DoRA request against an existing- # plain-LoRA adapter doesn't silently pass through unchanged.+ # use_dora arrives via **kwargs, not as a named parameter, so it cannot go in check_parameters+ # (which evals each entry as a bound local); compare it explicitly, or a DoRA request against an+ # existing plain-LoRA adapter silently passes through unchanged.check_all = check_all and (bool(peft_config.get("use_dora", False)) == bool(kwargs.get("use_dora", False)))- # Everything the stored adapter trains, wherever PEFT filed it: tying moves- # the counterpart to modules_to_tie, which to_dict() drops, so read the- # config object.+ # Everything the stored adapter trains, wherever PEFT filed it: tying moves the counterpart to+ # modules_to_tie, which to_dict() drops, so read the config object.requested_modules_to_save = modules_to_savemodules_to_save = list(peft_config["modules_to_save"] or [])old_target_modules = list(peft_config["target_modules"]) + modules_to_save@@ -3314,8 +3133,8 @@ class FastLlamaModel:getattr(model.peft_config["default"], "modules_to_tie", None) or [])- # The new side is what THIS call asks for, through the same redirect. Built- # from the stored lists instead, a narrowed request would compare equal.+ # The new side is what THIS call asks for, through the same redirect; built from the stored+ # lists instead, a narrowed request would compare equal.requested_targets, requested_saved, _ = _redirect_embedding_targets(target_modules,requested_modules_to_save,@@ -3323,13 +3142,13 @@ class FastLlamaModel:)new_target_modules = list(requested_targets) + list(requested_saved or [])if isinstance(model, PeftModelForSequenceClassification):- # PeftModelForSequenceClassification.__init__ extends modules_to_save with- # these unconditionally, so the stored config names modules the caller- # never asked for. Mirror it, or an unchanged request looks different.+ # PeftModelForSequenceClassification.__init__ extends modules_to_save with these+ # unconditionally, so the stored config names modules the caller never asked for. Mirror it, or+ # an unchanged request looks different.new_target_modules += ["classifier", "score"]- # Tying is not in check_parameters and leaves both lists looking the same, so- # compare it separately. On what it actually does, not what was asked: on an- # untied model the request changes nothing either way.+ # Tying is not in check_parameters and leaves both lists looking the same, so compare it+ # separately, on what it does rather than what was asked: on an untied model the request changes+ # nothing either way._requested_ties = _effective_weight_tying(model,requested_saved,@@ -3338,17 +3157,15 @@ class FastLlamaModel:_stored_ties = bool(getattr(model.peft_config["default"], "modules_to_tie", None))check_all = check_all and (_requested_ties == _stored_ties)if _requested_ties:- # PEFT files the counterpart under modules_to_tie, which the old side- # counts, so name it here even when the caller listed only one side.+ # PEFT files the counterpart under modules_to_tie, which the old side counts, so name it here+ # even when the caller listed only one side.new_target_modules += list(EMBEDDING_MODULES)- # Per-expert Linear MoE experts (e.g. gpt-oss bnb-4bit) were auto-added to the- # saved target_modules when the adapter was first created. Recompute them so a- # repeat get_peft_model call with the same args stays idempotent instead of- # tripping the mismatch below. No-op for non per-expert-Linear models.+ # Per-expert Linear MoE experts (gpt-oss bnb-4bit) were auto-added to the saved target_modules+ # at creation, so recompute them and keep a repeat get_peft_model call idempotent.new_target_modules += get_moe_target_modules(model, target_modules)- # Now check! Fold away PEFT's model.embed_tokens alias, and only that one:- # layers.0.q_proj is a real target and is not layers.1.q_proj.+ # Fold away PEFT's model.embed_tokens alias, and only that one: layers.0.q_proj is a real target+ # and is not layers.1.q_proj.def _leaf_name(module):return _embedding_leaf(module) or module@@ -3362,11 +3179,8 @@ class FastLlamaModel:)if check_all:- # Simply pass through!logger.warning("Unsloth: Already have LoRA adapters! We shall skip this step.")- # Offload!- # [TODO] First offload lm_head and embed_tokens to CPU (should be disk!!)if "embed_tokens" in new_target_modules:print("Unsloth: Training embed_tokens in mixed precision to save VRAM")@@ -3381,8 +3195,8 @@ class FastLlamaModel:model.get_output_embeddings(), DEVICE_TYPE_TORCH)- # Pre-wrapped PEFT model passes through here; still arm the detector so an RL- # trainer can reset a compile cache poisoned by a pre-train forward.+ # A pre-wrapped PEFT model passes through here; still arm the detector so an RL trainer can+ # reset a compile cache poisoned by a pre-train forward._unsloth_install_pretrain_detector(model)# This branch returns before patch_peft_model, so record here too;# apply_unsloth_gradient_checkpointing above already re-patched global state to match (#4735).@@ -3449,7 +3263,6 @@ class FastLlamaModel:assert type(use_rslora) is boolif use_rslora:if not SUPPORTS_RSLORA:- # We manually check for PEFTimport peftraise RuntimeError(f"Unsloth: Your PEFT version of {peft.__version__} does not support `use_rslora`.\n"@@ -3477,8 +3290,8 @@ class FastLlamaModel:train_lm_head = Falsetrain_embed_tokens = Falsefinal_modules = []- # LoRA on these never trains (see _redirect_embedding_targets); modules_to_save- # does, and is what embedding_learning_rate matches.+ # LoRA on these never trains (see _redirect_embedding_targets); modules_to_save does, and is+ # what embedding_learning_rate matches.target_modules, modules_to_save, _moved_embedding_modules = _redirect_embedding_targets(target_modules,modules_to_save,@@ -3521,10 +3334,9 @@ class FastLlamaModel:"Beware - your finetuning might be noticeably slower!")- # Check if we added new tokens!if hasattr(model, "_need_to_train_embeddings"):- # Check if embed_tokens/lm_head are already being trained- # (either as LoRA targets in final_modules or via modules_to_save)+ # embed_tokens/lm_head may already be trained, either as LoRA targets in final_modules or via+ # modules_to_save._embed_already_trained = train_embed_tokens or "embed_tokens" in final_modules_lm_head_already_trained = train_lm_head or "lm_head" in final_modulesif not _lm_head_already_trained or not _embed_already_trained:@@ -3533,7 +3345,7 @@ class FastLlamaModel:"train the lm_head and embed_tokens.\nWe must turn it on for you.")- # Only add to modules_to_save if not already a LoRA target+ # Only add to modules_to_save if not already a LoRA target.if not _embed_already_trained:train_embed_tokens = Trueif modules_to_save is None:@@ -3548,18 +3360,8 @@ class FastLlamaModel:elif "lm_head" not in modules_to_save:modules_to_save.append("lm_head")- # Check for Llama-3- # if hasattr(model._saved_temp_tokenizer, "_using_llama3_template"):- # if not train_embed_tokens and not train_lm_head:- # raise RuntimeError("")-- # First fix untrained tokens- # Wrong - can cause reserved tokens to pop out!!- # if train_embed_tokens or train_lm_head:- # fix_untrained_tokens(model, eps = 1e-16)- # pass+ # Fixing untrained tokens here is wrong: it can cause reserved tokens to pop out.- # Check modules_to_saveif modules_to_save is not None:for module in modules_to_save:# By leaf: PEFT resolves model.embed_tokens to the same module.@@ -3577,7 +3379,6 @@ class FastLlamaModel:vllm_engine = Noneif hasattr(model, "vllm_engine"):- # Fast inference!vllm_engine = model.vllm_enginevllm_fast_generate = model.fast_generatevllm_fast_generate_batches = model.fast_generate_batches@@ -3592,15 +3393,15 @@ class FastLlamaModel:"Unsloth: Currently fast inference does not work with using biases for LoRA.")- # Does not get lora yet, so get name from model, not base model+ # Does not get lora yet, so take the name from model, not base model.is_classification = "Classification" in str(type(model))- # Auto-detect MoE models and populate target_parameters for expert layers+ # Auto-detect MoE models and populate target_parameters for expert layers.if target_parameters is None:target_parameters = get_moe_target_parameters(model, target_modules)- # Per-expert Linear expert layouts (e.g. gpt-oss bnb-4bit) are Linear modules,- # not fused Parameters, so target them via target_modules. No-op otherwise.+ # Per-expert Linear expert layouts (gpt-oss bnb-4bit) are Linear modules, not fused Parameters,+ # so target them via target_modules._moe_module_targets = get_moe_target_modules(model, target_modules)if _moe_module_targets:_added = [t for t in _moe_module_targets if t not in final_modules]@@ -3635,7 +3436,7 @@ class FastLlamaModel:ensure_weight_tying = ensure_weight_tying,**kwargs,)- # Older PEFT has no `ensure_weight_tying`; passing it would TypeError.+ # Older PEFT has no ensure_weight_tying; passing it would TypeError.if "ensure_weight_tying" not in inspect.signature(LoraConfig).parameters:del arguments["ensure_weight_tying"]if not SUPPORTS_LOFTQ:@@ -3646,7 +3447,6 @@ class FastLlamaModel:_saved_temp_tokenizer = model._saved_temp_tokenizerlora_config = LoraConfig(**arguments)- # First offload lm_head and embed_tokens to diskinput_embeddings_device = model.get_input_embeddings().weight.deviceif is_classification:output_embeddings_device = model.score.weight.device@@ -3658,7 +3458,6 @@ class FastLlamaModel:print("Unsloth: Offloading input_embeddings to disk to save VRAM")offload_input_embeddings(model, temporary_location)- # Remove old items to save VRAMfor _ in range(3):gc.collect()clean_gpu_cache()@@ -3667,7 +3466,6 @@ class FastLlamaModel:print("Unsloth: Offloading output_embeddings to disk to save VRAM")offload_output_embeddings(model, temporary_location)- # Remove old items to save VRAMfor _ in range(3):gc.collect()clean_gpu_cache()@@ -3684,10 +3482,8 @@ class FastLlamaModel:)except Exception as _exc:logger.warning_once(f"Unsloth: could not lift adapter hooks: {_exc}")- # Fix LoraConfig.auto_mapping is Nonefix_lora_auto_mapping(model)- # Apply QAT + LoRA if specifiedif qat_scheme is not None:print("Unsloth: Applying QAT to mitigate quantization degradation")model = FastLlamaModel._prepare_for_qat(model, qat_scheme)@@ -3707,7 +3503,6 @@ class FastLlamaModel:if not hasattr(source_module, "weight"):returnweight = source_module.weight- # Remove existing registration to avoid "attribute already exists"if "weight" in getattr(target_module, "_parameters", {}):target_module._parameters.pop("weight")if hasattr(target_module, "weight"):@@ -3720,7 +3515,8 @@ class FastLlamaModel:)target_module.register_parameter("weight", weight)- # Tie trainable copies created by ModulesToSaveWrapper first (these are used in forward)+ # Tie the trainable copies created by ModulesToSaveWrapper first, since those are used in+ # forward, then the original_module references if present.if hasattr(input_embeddings, "modules_to_save") and hasattr(output_embeddings, "modules_to_save"):@@ -3732,7 +3528,6 @@ class FastLlamaModel:input_embeddings.modules_to_save.default,)- # Tie original_module references as well if presentif hasattr(input_embeddings, "original_module") and hasattr(output_embeddings, "original_module"):@@ -3767,7 +3562,6 @@ class FastLlamaModel:original_device = output_embeddings_device,)- # Patch tokenizer to pad to the rightinternal_model = modelwhile hasattr(internal_model, "model"):if hasattr(internal_model, "_saved_temp_tokenizer"):@@ -3778,14 +3572,12 @@ class FastLlamaModel:# Prevent Transformers Trainer from auto-wrapping Unsloth LoRA models in DP._mark_unsloth_disable_data_parallel(model)- # Clear deleted GPU itemsfor _ in range(3):gc.collect()clean_gpu_cache()patch_peft_fast_inference(model)- # Add for_inference and for_trainingmodel.for_training = functools.partial(FastLlamaModel.for_training, model)model.for_inference = functools.partial(FastLlamaModel.for_inference, model)m = model@@ -3793,18 +3585,17 @@ class FastLlamaModel:m.for_training = functools.partial(FastBaseModel.for_training, m)m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model- # Detect a stray pre-train forward so train() can drop the torch.compile- # graph cache it would otherwise poison (see prepare_for_training_mode).+ # Detect a stray pre-train forward so train() can drop the torch.compile graph cache it would+ # otherwise poison (see prepare_for_training_mode)._unsloth_install_pretrain_detector(model)model = _exclude_rope_inv_freq_from_ddp(model)return model@staticmethoddef patch_peft_model(model, use_gradient_checkpointing = "unsloth"):- # Persist the effective GC mode so the trainer restores it verbatim: for_inference()- # clears the module flags every GRPO step, and a plain TrainingArguments defaults it to- # False, which would otherwise silently disable it at train time (#4735). Recorded here,- # not in get_peft_model, so adapters loaded via loader.py's from_pretrained path are covered.+ # Persist the effective GC mode so the trainer restores it verbatim: for_inference() clears the+ # module flags every GRPO step, and TrainingArguments defaults it to False, which would silently+ # disable it at train time (#4735). Recorded here so loader.py's from_pretrained path is covered.model._unsloth_gradient_checkpointing = use_gradient_checkpointingif os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1":return FastBaseModel.patch_peft_model(@@ -3816,7 +3607,6 @@ class FastLlamaModel:):raise TypeError("Unsloth: Your model needs to call `.get_peft_model` first!")- # Get activation functionmodel_type = model.config.model_typeif model_type == "llama":@@ -3848,26 +3638,21 @@ class FastLlamaModel:use_reentrant = True,)- # Fix up config for transformers uploading PEFTfor active_adapter in model.peft_config.keys():- # Not necessary since we requires transformers >= 4.37if False:name = model.peft_config[active_adapter].base_model_name_or_pathif name.startswith("unsloth/") and name.endswith("-bnb-4bit"):name = name[: len(name) - len("-bnb-4bit")]model.peft_config[active_adapter].base_model_name_or_path = namepass- # Add revision to enable future fast inference paths- # [TODO] Bugs out!see https://github.com/unslothai/unsloth/issues/492- # model.peft_config[active_adapter].revision = f"unsloth"+ # Adding a revision to enable future fast inference paths bugs out; see #492.from transformers.trainer import Trainerif Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":raise RuntimeError("Unsloth: Unsuccessfully patched Trainer! Please file a bug report!")- # Fix loftq issues- # loftq_config must not = None, but rather {}+ # loftq_config must not be None, but rather {}.all_configs = model.peft_configfor key, current_config in all_configs.items():if hasattr(current_config, "loftq_config") and current_config.loftq_config is None:@@ -3876,7 +3661,6 @@ class FastLlamaModel:current_config = current_config.__class__(**new_args)all_configs[key] = current_config- # Do patchingn_mlp = 0n_qkv = 0n_o = 0@@ -3885,11 +3669,10 @@ class FastLlamaModel:model.active_adapters[0] if hasattr(model, "active_adapters") else model.active_adapter)- # Get dropout and biaslora_dropout = model.peft_config[active_adapter].lora_dropoutbias = model.peft_config[active_adapter].bias- # We also do not inplace edit QKV for Cohere!+ # QKV is not inplace-edited for Cohere._apply_lora_mlp = (functools.partial(apply_lora_mlp, inplace = False)if model_type == "cohere"@@ -3899,10 +3682,8 @@ class FastLlamaModel:if lora_dropout == 0 and bias == "none":for idx, layer in enumerate(model.model.model.layers):if model_type != "falcon_h1":- # LoRAMLP.apply doesn't have functionality for gate and down multipliers yet.- # Don't patch falcon h1 for the time being.+ # LoRAMLP.apply has no gate/down multiplier support yet, so falcon h1 is not patched for now.- # MLP patchingmlp_module = layer.mlpgate_proj = mlp_module.gate_projup_proj = mlp_module.up_proj@@ -3919,9 +3700,9 @@ class FastLlamaModel:and (len(getattr(up_proj, "lora_magnitude_vector", []) or []) == 0)and (len(getattr(down_proj, "lora_magnitude_vector", []) or []) == 0)):- # https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module+ # See stackoverflow.com/questions/50599045 on replacing a function within a class of a module.if hasattr(mlp_module, "_unsloth_forward"):- # then we've patched the mlp to use TiledMLP+ # Then the mlp has been patched to use TiledMLP.mlp_module._unsloth_forward = types.MethodType(_apply_lora_mlp, mlp_module)@@ -3934,7 +3715,6 @@ class FastLlamaModel:"are not enabled or a bias term (like in Qwen) is used.")- # QKV attention patchingq_proj = layer.self_attn.q_projk_proj = layer.self_attn.k_projv_proj = layer.self_attn.v_proj@@ -3960,7 +3740,6 @@ class FastLlamaModel:"are not enabled or a bias term (like in Qwen) is used.")- # O attention patchingo_proj = layer.self_attn.o_projif (hasattr(o_proj, "lora_A")@@ -3981,21 +3760,18 @@ class FastLlamaModel:)patch_saving_functions(model)- # Patch cross entropy loss labels- # Fixes https://github.com/unslothai/unsloth/issues/10+ # Patch cross entropy loss labels; fixes #10.max_seq_length = model.max_seq_length- # extra_ignored_labels = torch.full((max_seq_length, 1), -100, device = "cuda:0")- # model.model.extra_ignored_labels = extra_ignored_labelsinternal_model = modelwhile hasattr(internal_model, "model"):internal_model.max_seq_length = max_seq_lengthinternal_model = internal_model.modelinternal_model.max_seq_length = max_seq_length- # Save to modules as well+ # Since transformers 4.53, must turn on explicitly+ # Since transformers 4.53, must turn off explicitlyfor module in model.modules():module.max_seq_length = max_seq_length- # Patch tokenizer to pad to the rightinternal_model = modelwhile hasattr(internal_model, "model"):if hasattr(internal_model, "_saved_temp_tokenizer"):@@ -4004,14 +3780,12 @@ class FastLlamaModel:if hasattr(internal_model, "_saved_temp_tokenizer"):internal_model._saved_temp_tokenizer.padding_side = "right"- # Clear deleted GPU itemsfor _ in range(3):gc.collect()clean_gpu_cache()patch_peft_fast_inference(model)- # Add for_inference and for_trainingmodel.for_training = functools.partial(FastLlamaModel.for_training, model)model.for_inference = functools.partial(FastLlamaModel.for_inference, model)m = model@@ -4019,8 +3793,6 @@ class FastLlamaModel:m.for_training = functools.partial(FastBaseModel.for_training, m)m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model- # Detect a stray pre-train forward so train() can drop the torch.compile- # graph cache it would otherwise poison (see prepare_for_training_mode)._unsloth_install_pretrain_detector(model)return model@@ -4036,10 +3808,8 @@ class FastLlamaModel:m.gradient_checkpointing = Falseif hasattr(m, "training"):m.training = False- # Pad tokenizer to the leftif hasattr(m, "_saved_temp_tokenizer"):m._saved_temp_tokenizer.padding_side = "left"- # Set a flag for generation!m._flag_for_generation = Truem = model@@ -4049,12 +3819,11 @@ class FastLlamaModel:_for_inference(m)model.eval() # to turn off training on modules deeper in- # Since transformers 4.53, must turn off explicitly+ # Since transformers 4.53, this must be turned off explicitly.for module in model.modules():if hasattr(module, "gradient_checkpointing"):module.gradient_checkpointing = False- # Also disable training for embeddings for NEFTuneif hasattr(model, "get_input_embeddings"):embeddings = model.get_input_embeddings()if hasattr(embeddings, "training"):@@ -4064,8 +3833,8 @@ class FastLlamaModel:if hasattr(embeddings, "training"):embeddings.training = False- # Restore use_cache values that prepare_model_for_training disabled- # for gradient checkpointing (older unsloth_zoo has no restore helper)+ # Restore use_cache values that prepare_model_for_training disabled for gradient checkpointing+ # (older unsloth_zoo has no restore helper).try:from unsloth_zoo.training_utils import restore_use_cacherestore_use_cache(model)@@ -4080,7 +3849,6 @@ class FastLlamaModel:"Unsloth: I think you're passing a tokenizer, not the model to for_training!")- # Delete all fast inference lorasfor param in model.parameters():if hasattr(param, "_fast_lora"):del param._fast_lora@@ -4090,13 +3858,11 @@ class FastLlamaModel:m.gradient_checkpointing = use_gradient_checkpointingif hasattr(m, "training"):m.training = True- # Pad tokenizer to the leftif hasattr(m, "_saved_temp_tokenizer"):m._saved_temp_tokenizer.padding_side = "right"- # Set a flag for generation!if hasattr(m, "_flag_for_generation"):try:- # A PEFT wrapper delegates the read but owns nothing to delete+ # A PEFT wrapper delegates the read but owns nothing to delete.del m._flag_for_generationexcept AttributeError:pass@@ -4108,12 +3874,11 @@ class FastLlamaModel:_for_training(m)model.train() # to turn on training on modules deeper in- # Since transformers 4.53, must turn on explicitly+ # Since transformers 4.53, this must be turned on explicitly.for module in model.modules():if hasattr(module, "gradient_checkpointing"):module.gradient_checkpointing = use_gradient_checkpointing- # Also re-enable training for embeddings for NEFTuneif hasattr(model, "get_input_embeddings"):embeddings = model.get_input_embeddings()if hasattr(embeddings, "training"):@@ -4123,8 +3888,8 @@ class FastLlamaModel:if hasattr(embeddings, "training"):embeddings.training = True- # Re-disable use_cache if prepare_model_for_training had disabled it- # and for_inference restored it (record only exists after a disable)+ # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored+ # it; the record only exists after a disable.if (use_gradient_checkpointingand getattr(model, "_unsloth_use_cache_originals", None) is not None@@ -4140,7 +3905,7 @@ class FastLlamaModel:from .rl import PatchFastRL# Auto-enable grouped-GEMM MoE (tf<5 ModuleList experts) on built / PEFT'd models. Wrap the-# loader leaves before PatchFastRL so downstream patchers see the wrapped versions. Guarded.+# loader leaves before PatchFastRL so downstream patchers see the wrapped versions.try:from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moeFastLlamaModel.from_pretrained = staticmethod(diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 3ceecbdce70..0313cc75c2b 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -21,9 +21,8 @@ from ._utils import (USE_MODELSCOPE,get_transformers_model_type,hf_login,- # Single source of truth is _utils.py; re-exported here so callers doing- # `from unsloth.models.loader import DISABLE_SDPA_MODEL_NAMES` keep working and so- # _is_sdpa_excluded (in _utils) can honor it without a loader -> _utils cycle.+ # Single source of truth is _utils.py; re-exported so callers importing it from here keep+ # working and so _is_sdpa_excluded can honor it without a loader -> _utils cycle.DISABLE_SDPA_MODEL_NAMES,)from ._custom_dtype import register_custom_dtype@@ -59,7 +58,7 @@ except:try:from huggingface_hub.utils import get_tokenexcept:- # For older versions of huggingface_hub+ # For older versions of huggingface_hub.from huggingface_hub.utils._token import get_tokenimport importlib.utilfrom ..device_type import (@@ -72,7 +71,7 @@ from ..device_type import (ALLOW_BITSANDBYTES,)-# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!+# huggingface/transformers#26037 allows 4 bit loading.from unsloth_zoo.utils import Version, _get_dtypefrom unsloth_zoo.hf_utils import dtype_from_configfrom unsloth_zoo.tiled_mlp import patch_tiled_mlp@@ -90,8 +89,7 @@ SUPPORTS_FALCON_H1 = transformers_version >= Version("4.53.0")SUPPORTS_GEMMA3N = transformers_version >= Version("4.53.0")SUPPORTS_GPTOSS = transformers_version >= Version("4.55.0")SUPPORTS_GEMMA4 = transformers_version >= Version("5.5.0")-# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq).-# See _fix_rope_inv_freq() below for details.+# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq); see _fix_rope_inv_freq() below._NEEDS_ROPE_FIX = transformers_version >= Version("5.0.0")if SUPPORTS_GEMMA:from .gemma import FastGemmaModel@@ -121,12 +119,11 @@ from ._utils import (maybe_prefetch_hf_snapshot,)-# Source of truth is unsloth_zoo.model_lists. Re-exported so callers doing-# `from unsloth.models.loader import FORCE_FLOAT32` keep working. The fallback-# list is also unioned in so a newer unsloth still forces float32 for these-# archs when paired with an older unsloth_zoo that predates them (upgrade skew).+# Source of truth is unsloth_zoo.model_lists, re-exported for callers importing FORCE_FLOAT32+# from here. The fallback list is unioned in so a newer unsloth still forces float32 for these+# archs against an older zoo._FORCE_FLOAT32_FALLBACK = [- "gemma3,", # Add comma bc gemma3 will match gemma3n+ "gemma3,","gemma3text", # Gemma3TextModel (EmbeddingGemma, standalone text-only Gemma3)"gemma3n","gemma4", # Gemma4 (gemma4 / gemma4_text): float16 NaNs grad norms in the backward@@ -145,7 +142,7 @@ for _mt in _FORCE_FLOAT32_FALLBACK:FORCE_FLOAT32.append(_mt)global DISABLE_COMPILE_MODEL_NAMES-# Must be alphabetically sorted for each entry+# Must be alphabetically sorted for each entry.def _strip_unsloth_bnb_4bit_suffix(model_name: str) -> str:@@ -283,8 +280,8 @@ DISABLE_COMPILE_MODEL_NAMES = []# Architectures with gated-deltanet (linear attention) layers. Unsloth bundles the-# flash-linear-attention Triton kernels (unsloth_zoo/_vendored/fla), so no install is-# needed; transformers uses the much slower pure PyTorch path only when they can't be enabled.+# flash-linear-attention Triton kernels, so no install is needed; transformers falls back to+# the much slower pure PyTorch path only when they cannot be enabled.FLA_MODEL_TYPE_PREFIXES = ("qwen3_next", "qwen3_5", "kimi_linear", "olmo_hybrid")_fla_advised = False@@ -316,11 +313,9 @@ def _maybe_advise_fla_install(model_types):except Exception:return_fla_advised = True- # Prefer unsloth_zoo's specific reason when it disabled the kernels on purpose.- # The generic text below blames CUDA / torch / triton minimums, all of which are- # satisfied on e.g. an H100 that hit the fla #640 Triton miscompile, so it would- # send that user looking in the wrong place. try/except keeps an unsloth pinned- # against an older unsloth_zoo working.+ # Prefer unsloth_zoo's reason when it disabled the kernels on purpose: the generic text blames+ # CUDA / torch / triton minimums, all satisfied on an H100 that hit the fla #640 Triton+ # miscompile, so it would point that user in the wrong direction.try:from unsloth_zoo.temporary_patches.fla_vendor import fla_unavailable_reasonreason = fla_unavailable_reason()@@ -350,9 +345,8 @@ def _fix_rope_inv_freq(model):return modelfor name, module in model.named_modules():- # Unsloth's LlamaRotaryEmbedding and subclasses (Extended, LinearScaling,- # Granite). Native v5 rotary classes (Gemma3, etc.) have original_inv_freq- # which v5's _init_weights() uses to restore inv_freq, so they are fine.+ # Unsloth's LlamaRotaryEmbedding and subclasses (Extended, LinearScaling, Granite). Native v5+ # rotary classes carry original_inv_freq, which v5's _init_weights() uses to restore inv_freq.if (hasattr(module, "inv_freq")and hasattr(module, "base")@@ -381,7 +375,7 @@ def _fix_rope_inv_freq(model):dtype = torch.get_default_dtype(),)- # LongRopeRotaryEmbedding (Phi-3.5 style with short_inv_freq + long_inv_freq)+ # LongRopeRotaryEmbedding (Phi-3.5 style with short_inv_freq + long_inv_freq).elif (hasattr(module, "short_inv_freq")and hasattr(module, "long_inv_freq")@@ -456,11 +450,11 @@ class FastLanguageModel(FastLlamaModel):qat_scheme = None,load_in_fp8 = False, # fp8 LoRA (True, False, 'block')unsloth_tiled_mlp = False,- text_only = False, # Skip vision/audio towers and load only the text decoder+ text_only = False,*args,**kwargs,):- # Respect user-provided quantization_config (e.g. BitsAndBytesConfig)+ # Respect a user-provided quantization_config (e.g. BitsAndBytesConfig).quantization_config = kwargs.get("quantization_config", None)if quantization_config is not None:if isinstance(quantization_config, dict):@@ -476,9 +470,8 @@ class FastLanguageModel(FastLlamaModel):load_in_8bit = Trueload_in_4bit = False- # Login to allow private models- # Before normalization: dtype is also derived below from a 4bit config's- # compute dtype, which is not a request for the whole model.+ # Login to allow private models. Before normalization: dtype is also derived below from a 4bit+ # config's compute dtype, which is not a request for the whole model.user_float32 = _requested_float32(dtype)token = hf_login(token)# Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.@@ -495,16 +488,15 @@ class FastLanguageModel(FastLlamaModel):if isinstance(bnb_compute_dtype, torch.dtype):dtype = bnb_compute_dtype- # Distributed-safe device placement for quantized models.- # In multi-GPU (torchrun), each rank must load the model on its own device- # to avoid Accelerate device relocation errors with quantized weights.+ # Distributed-safe placement for quantized models: under torchrun each rank must load on its+ # own device, else Accelerate raises device relocation errors on quantized weights.is_quantized = load_in_4bit or load_in_8bit or load_in_fp8device_map = requested_device_map(device_map)if is_quantized and isinstance(device_map, str):distributed_device_map, is_dist = prepare_device_map()if is_dist:- # One whole model per rank; sharding one across the ranks' GPUs as well- # would have every rank fighting for the same cards.+ # One whole model per rank; sharding one across the ranks' GPUs too would have every rank+ # fighting for the same cards.device_map = distributed_device_map# @_offline_aware_load already forced offline when needed; delegations inherit it.@@ -526,12 +518,12 @@ class FastLanguageModel(FastLlamaModel):use_gradient_checkpointing = use_gradient_checkpointing,resize_model_vocab = resize_model_vocab, # [TODO] No effectrevision = revision,- return_logits = False, # Return logits+ return_logits = False,fullgraph = True, # No graph breaksuse_exact_model_name = use_exact_model_name,offload_embedding = offload_embedding,float32_mixed_precision = float32_mixed_precision,- # Pass vLLM/inference parameters+ # Pass vLLM/inference parameters.fast_inference = fast_inference,gpu_memory_utilization = gpu_memory_utilization,float8_kv_cache = float8_kv_cache,@@ -564,7 +556,7 @@ class FastLanguageModel(FastLlamaModel):)if DEVICE_TYPE_TORCH == "cuda":for i in range(DEVICE_COUNT):- # [TODO] DGX Spark vLLM breaks+ # DGX Spark vLLM breaks.if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():print("Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"@@ -573,14 +565,12 @@ class FastLanguageModel(FastLlamaModel):fast_inference = Falsebreak- # bitsandbytes unusable (absent, or unstable as on some AMD stacks). This is- # a capability check, so it is not gated on use_exact_model_name: that only- # suppresses repo-name remapping and cannot make bitsandbytes available.+ # bitsandbytes unusable (absent, or unstable on some AMD stacks). A capability check, so not+ # gated on use_exact_model_name, which only suppresses repo-name remapping.if not ALLOW_BITSANDBYTES:- # A user-supplied config sets load_in_4bit/8bit above and is forwarded- # in kwargs, so clearing the flags alone still rebuilds the bnb- # quantizer downstream. Only drop it when it asks for bnb: a GPTQ /- # AWQ / fp8 / torchao config must pass through untouched.+ # A user config sets load_in_4bit/8bit above and is forwarded in kwargs, so clearing the flags+ # alone still rebuilds the bnb quantizer downstream. Drop it only when it asks for bnb: a+ # GPTQ / AWQ / fp8 / torchao config must pass through untouched._quant_cfg = kwargs.get("quantization_config", None)if isinstance(_quant_cfg, dict):_wants_bnb = bool(@@ -603,8 +593,8 @@ class FastLanguageModel(FastLlamaModel):"Unsloth: `bitsandbytes` is unavailable here - disabling 4bit/8bit. ""16bit LoRA and full finetuning still work.")- # 8bit is bitsandbytes too: leaving either set sends the request on to- # Transformers, which builds the bnb quantizer and fails there.+ # 8bit is bitsandbytes too: leaving either set sends the request to Transformers, which builds+ # the bnb quantizer and fails there.load_in_4bit = Falseload_in_8bit = Falseif _wants_bnb:@@ -637,28 +627,26 @@ class FastLanguageModel(FastLlamaModel):else:assert new_model_name is not Nonemodel_name = new_model_name- # If mapper resolved to a pre-quantized FP8 model, disable- # on-the-fly quantization to avoid double quantization+ # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid+ # quantizing twice.if load_in_fp8 != False and new_model_name != old_model_name:load_in_fp8 = False# Only this block honours use_exact_model_name; the transforms below do not.mapper_moved_name = model_name != old_model_name- # Check if pre-quantized models are allowed- # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)+ # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2, and our pre-quants use blocksize = 64.if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):model_name = _strip_unsloth_bnb_4bit_suffix(model_name)- # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set- # Say so: dropping the flags in silence surfaces much later as an- # out-of-memory failure whose message never mentions quantization.+ # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set. Say+ # so: dropping the flags silently resurfaces as an OOM whose message never mentions+ # quantization.if model_name.lower().endswith("-bf16") and (load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))):- # A user quantization_config stays in **kwargs and still quantizes.- # `load_in_4bit` defaults to True, so a set flag is not proof of a- # request; an explicit `load_in_16bit` IS one, for exactly this load.+ # A user quantization_config stays in **kwargs and still quantizes. load_in_4bit defaults to+ # True, so a set flag is no proof of a request; an explicit load_in_16bit is.if (not load_in_16bitand (load_in_4bit or load_in_8bit or load_in_fp8 != False)@@ -684,13 +672,12 @@ class FastLanguageModel(FastLlamaModel):base_revision = _revision_for_resolved_repo(revision, model_name, old_model_name, mapper_moved_name)- # The PeftConfig probe reads the adapter repo, which peft loads in-process, so it- # keeps the ref even when vLLM takes the base model's away just after.+ # The PeftConfig probe reads the adapter repo, which peft loads in-process, so it keeps the ref+ # even when vLLM takes the base model's away just after.adapter_revision = base_revision- # vLLM fetches the default branch, so the pin is dead for the weights. Drop it before- # the probe: model_types picks the architecture class off that config, so a ref the- # weights are not at dispatches the wrong one. llama.py's predicate spares the- # in-process pre-Volta and num_labels fallbacks, which can still honour the pin.+ # vLLM fetches the default branch, so the pin is dead for the weights. Drop it before the+ # probe: model_types picks the architecture class off that config, so a ref the weights are not+ # at dispatches the wrong one. llama.py's predicate spares the in-process fallbacks.if base_revision is not None and _vllm_will_load_weights(fast_inference, kwargs.get("num_labels")):@@ -700,7 +687,6 @@ class FastLanguageModel(FastLlamaModel):)base_revision = None- # First check if it's a normal model via AutoConfigfrom huggingface_hub.utils import (disable_progress_bars,enable_progress_bars,@@ -769,7 +755,7 @@ class FastLanguageModel(FastLlamaModel):# Old transformers versions checkboth_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32- # Error out if both LoRA and normal model config exists.+ # Error out if both LoRA and normal model config exist.if both_exist:raise RuntimeError("Unsloth: Your repo has a LoRA adapter and a base model.\n"@@ -793,8 +779,7 @@ class FastLanguageModel(FastLlamaModel):f"AutoConfig error: {autoconfig_error}\n\n"f"PeftConfig error: {peft_error}\n\n")- # Chain an offline-related cause if either probe had one, so @_offline_aware_load- # still retries from cache (e.g. adapter repo: permanent AutoConfig 404 + transient PeftConfig).+ # Chain an offline-related cause if either probe had one, so @_offline_aware_load still retries from cache._cause = next((e@@ -812,12 +797,11 @@ class FastLanguageModel(FastLlamaModel):if len(model_types) == 1:model_type = model_types[0]else:- # Leave as tuple if more than one arch+ # Leave as tuple if more than one arch.model_type = model_types# New transformers need to check manually.if SUPPORTS_LLAMA32 and is_model and is_peft:- # Check if folder exists locallyif os.path.isdir(model_name):exist_adapter_config = os.path.exists(os.path.join(model_name, "adapter_config.json")@@ -825,14 +809,11 @@ class FastLanguageModel(FastLlamaModel):exist_config = os.path.exists(os.path.join(model_name, "config.json"))both_exist = exist_adapter_config and exist_configelse:- # Both AutoConfig and PeftConfig loaded successfully from this- # remote repo, so both config.json and adapter_config.json- # definitely exist -- no need for an extra HfFileSystem network call.+ # Both AutoConfig and PeftConfig loaded from this remote repo, so config.json and+ # adapter_config.json both exist and no extra HfFileSystem call is needed.both_exist = True- # Get base model for PEFT:if is_peft:- # Check base model again for PEFTmodel_name = peft_config.base_model_name_or_pathif not use_exact_model_name:model_name = get_model_name(@@ -842,21 +823,19 @@ class FastLanguageModel(FastLlamaModel):token = token,trust_remote_code = trust_remote_code,)- # Check if pre-quantized models are allowed- # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)+ # Are pre-quantized models allowed? AMD Instinct GPUs need blocksize = 128 on bitsandbytes+ # < 0.49.2, and our pre-quants use 64.if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):model_name = _strip_unsloth_bnb_4bit_suffix(model_name)- # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set- # Say so: dropping the flags in silence surfaces much later as an- # out-of-memory failure whose message never mentions quantization.+ # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set. Say+ # so: dropping the flags silently resurfaces as an OOM that never mentions quantization.if model_name.lower().endswith("-bf16") and (load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))):- # A user quantization_config stays in **kwargs and still quantizes.- # `load_in_4bit` defaults to True, so a set flag is not proof of a- # request; an explicit `load_in_16bit` IS one, for exactly this load.+ # A user quantization_config stays in **kwargs and still quantizes. load_in_4bit defaults to+ # True, so a set flag is no proof of a request; an explicit load_in_16bit is.if (not load_in_16bitand (load_in_4bit or load_in_8bit or load_in_fp8 != False)@@ -947,21 +926,7 @@ class FastLanguageModel(FastLlamaModel):f"to obtain the latest transformers build, then restart this session.")dispatch_model = FastQwen3Model if model_type == "qwen3" else FastQwen3MoeModel- # elif model_type == "falcon_h1":- # dispatch_model = FastFalconH1Model- # if not SUPPORTS_FALCON_H1:- # raise ImportError(- # f"Unsloth: Your transformers version of {transformers_version} does not support FalconH1.\n"\- # f"The minimum required version is 4.50.3.\n"\- # f'Try `pip install --upgrade "transformers>=4.50.3"`\n'\- # f"to obtain the latest transformers build, then restart this session."\- # )- # Temporary disable optimized Cohere until errors match- # elif model_type == "cohere":- # dispatch_model = FastCohereModel- # Temporary disable optimized Granite until errors match- # elif model_type == "granite":- # dispatch_model = FastGraniteModel+ # Optimized Cohere and Granite paths are disabled until their errors match.else:delegated, tokenizer = FastModel.from_pretrained(model_name = old_model_name,@@ -980,12 +945,12 @@ class FastLanguageModel(FastLlamaModel):use_gradient_checkpointing = use_gradient_checkpointing,resize_model_vocab = resize_model_vocab, # [TODO] No effectrevision = revision,- return_logits = False, # Return logits+ return_logits = False,fullgraph = True, # No graph breaksuse_exact_model_name = use_exact_model_name,offload_embedding = offload_embedding,float32_mixed_precision = float32_mixed_precision,- # Pass vLLM/inference parameters+ # Pass vLLM/inference parameters.fast_inference = fast_inference,gpu_memory_utilization = gpu_memory_utilization,float8_kv_cache = float8_kv_cache,@@ -1001,19 +966,18 @@ class FastLanguageModel(FastLlamaModel):)return _mark_requested_float32(delegated, user_float32), tokenizer- # Apply gradient checkpointing with smart heuristicsuse_gradient_checkpointing = apply_unsloth_gradient_checkpointing(use_gradient_checkpointing, max_seq_length, dtype)- # Keep the local checkpoint dir as tokenizer when self-sufficient (see _resolve_checkpoint_tokenizer_name).+ # Keep the local checkpoint dir as tokenizer when self-sufficient; see _resolve_checkpoint_tokenizer_name.tokenizer_name = _resolve_checkpoint_tokenizer_name(old_model_name, kwargs)if fast_inference:fast_inference, model_name = fast_inference_setup(model_name, model_config)- # model_name can move once more here. Skip for PEFT: model_name is then the base- # model, and `revision` names the adapter that PeftModel.from_pretrained loads below.+ # model_name can move once more here. Skip for PEFT: model_name is then the base model, and+ # `revision` names the adapter PeftModel.from_pretrained loads below.if not is_peft:base_revision = _revision_for_resolved_repo(base_revision, model_name, old_model_name, mapper_moved_name@@ -1027,17 +991,16 @@ class FastLanguageModel(FastLlamaModel):load_in_4bit_kwargs = Falseload_in_8bit_kwargs = False- # Mirror FastModel: bitsandbytes < 0.46.0 needs dynamo disabled.- # Best effort: never crash the load (old unsloth_zoo without the- # zoo #710 fix raises NameError here on Python 3.13).+ # Mirror FastModel: bitsandbytes < 0.46.0 needs dynamo disabled. Best effort, never crash the+ # load (an old zoo without the #710 fix raises NameError here on Python 3.13).try:patch_compiling_bitsandbytes()except Exception as e:print(f"Unsloth: Could not patch bitsandbytes for torch.compile - {e}")- # The optimized path has never carried `offload_embedding`, so a request for one is- # dropped rather than honoured; say so instead of leaving the caller to infer it- # from memory use. `"auto"` stays quiet: it promises a decision, and off is one.+ # The optimized path never carried offload_embedding, so a request for one is dropped rather+ # than honoured; say so instead of leaving the caller to infer it from memory use. "auto"+ # stays quiet: it promises a decision, and off is one.if offload_embedding != OFFLOAD_EMBEDDING_AUTO and offload_embedding:print("Unsloth: Not offloading embeddings; the optimized path for this "@@ -1075,8 +1038,8 @@ class FastLanguageModel(FastLlamaModel):if resize_model_vocab is not None:model.resize_token_embeddings(resize_model_vocab)- # `resize_token_embeddings` rebuilds the embedding, dropping `_hf_hook`;- # this is the last module swap of all, after every repair above.+ # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module+ # swap of all, after every repair above.try:from unsloth.models.vision import _repair_dispatch_hooks_repaired = _repair_dispatch_hooks(model)@@ -1106,11 +1069,11 @@ class FastLanguageModel(FastLlamaModel):)if load_in_4bit:- # Fix up bitsandbytes config, but respect user-provided quantization_config+ # Fix up the bitsandbytes config, but respect a user-provided quantization_config.if quantization_config is None:- # `load_in_4bit` is the requested flag, not the effective one: a non-bnb- # checkpoint (MXFP4/gptq/awq) had bnb disabled by check_and_disable, so stamping a- # synthetic bnb config would corrupt its real one. Only stamp bnb/unquantized.+ # load_in_4bit is the requested flag, not the effective one: a non-bnb checkpoint+ # (MXFP4/gptq/awq) had bnb disabled by check_and_disable, so a synthetic bnb config would+ # corrupt its real one.try:from unsloth_zoo.utils import get_quant_type_stamp_bnb = get_quant_type(model.config) in (None, "bitsandbytes")@@ -1119,14 +1082,14 @@ class FastLanguageModel(FastLlamaModel):if _stamp_bnb:compute_dtype = dtype_from_config(model.config)quantization_config = {- # Sometimes compute_dtype is not a string!!+ # compute_dtype is sometimes not a string."bnb_4bit_compute_dtype": compute_dtype,"bnb_4bit_quant_type": "nf4","bnb_4bit_use_double_quant": True,"llm_int8_enable_fp32_cpu_offload": False,"llm_int8_has_fp16_weight": False,- # Whatever the load really used. None here would describe a layout- # that never existed, and saving it makes the adapter unreloadable.+ # Whatever the load really used. None here describes a layout that never existed, and saving it+ # makes the adapter unreloadable."llm_int8_skip_modules": _loaded_skip_modules(model.config),"llm_int8_threshold": 6.0,"load_in_4bit": True,@@ -1144,9 +1107,9 @@ class FastLanguageModel(FastLlamaModel):_tag_model_with_fp8_torchao_config(model, fp8_mode)if is_peft:- # From https://github.com/huggingface/peft/issues/184- # Now add PEFT adapters- # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet.+ # From huggingface/peft#184+ # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet, and it always+ # loads in-process, so warm it even under fast_inference._prefetched = maybe_prefetch_hf_snapshot(old_model_name,token = token,@@ -1156,15 +1119,15 @@ class FastLanguageModel(FastLlamaModel):# Adapter always loads in-process via PeftModel, so warm it even under fast_inference.fast_inference = False,force_download = kwargs.get("force_download", False),- # Leave use_safetensors auto (inheriting base format could skip a safetensors-only- # adapter). adapter_only restricts the warm to the adapter files + root aux.+ # Leave use_safetensors auto, since inheriting the base format could skip a safetensors-only+ # adapter; adapter_only restricts the warm to adapter files plus root aux.adapter_only = True,)- # Child did the forced download; clear the flag so the load reuses the warm cache.+ # The child did the forced download; clear the flag so the load reuses the warm cache.if _prefetched and kwargs.get("force_download", False):kwargs["force_download"] = False- # Forward cache_dir so the load reads the warmed adapter. No subfolder (that targets the- # base checkpoint; adapters live at the root).+ # Forward cache_dir so the load reads the warmed adapter. No subfolder: that targets the base+ # checkpoint, and adapters live at the root.peft_load_kwargs = {}if kwargs.get("cache_dir") is not None:peft_load_kwargs["cache_dir"] = kwargs["cache_dir"]@@ -1178,15 +1141,14 @@ class FastLanguageModel(FastLlamaModel):trust_remote_code = trust_remote_code,**peft_load_kwargs,)- # Patch it as well!model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing)try:from .vision import _lift_endpoint_hooks_onto_adapters_lift_endpoint_hooks_onto_adapters(model)except Exception:pass # never block loading on a placement nicety- # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back- # to the original loop, an attention-only adapter keeps the grouped path. Guarded.+ # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back to the+ # original loop, an attention-only adapter keeps the grouped path.try:from unsloth_zoo.temporary_patches.moe_grouped_modulelist import (auto_enable_grouped_moe,@@ -1195,8 +1157,7 @@ class FastLanguageModel(FastLlamaModel):except Exception:pass # optional speedup; never block model loading- # Patch Tiled MLP- # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}""+ # Patch Tiled MLP; enable by setting UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}".patch_tiled_mlp_choice = os.environ.get("UNSLOTH_TILED_MLP", "arctic" if unsloth_tiled_mlp else "0")@@ -1205,8 +1166,8 @@ class FastLanguageModel(FastLlamaModel):model = _fix_rope_inv_freq(model)model = _exclude_rope_inv_freq_from_ddp(model)- # This path never sets UNSLOTH_FORCE_FLOAT32, so answer False rather than- # leave the trainer reading whatever a later load writes to the environment.+ # This path never sets UNSLOTH_FORCE_FLOAT32, so answer False rather than let the trainer read+ # whatever a later load writes to the environment.model = _mark_forced_float32(model, False)# Full finetuning delegates to FastModel above, so this path is always LoRA.model = _mark_full_finetuning(model, False)@@ -1275,7 +1236,7 @@ class FastModel(FastBaseModel):use_gradient_checkpointing = "unsloth",resize_model_vocab = None, # [TODO] No effectrevision = None,- return_logits = False, # Return logits+ return_logits = False,fullgraph = True, # No graph breaksuse_exact_model_name = False,auto_model = None,@@ -1295,12 +1256,12 @@ class FastModel(FastBaseModel):load_in_fp8 = False, # fp8 LoRA (True, False, 'block')unsloth_tiled_mlp = False,target_parameters = None, # For MoE expert parameters- text_only = False, # Skip vision/audio towers and load only the text decoder+ text_only = False,*args,**kwargs,):user_config = kwargs.pop("config", None)- # Respect user-provided quantization_config (e.g. BitsAndBytesConfig)+ # Respect a user-provided quantization_config (e.g. BitsAndBytesConfig).quantization_config = kwargs.get("quantization_config", None)if quantization_config is not None:if isinstance(quantization_config, dict):@@ -1316,9 +1277,8 @@ class FastModel(FastBaseModel):load_in_8bit = Trueload_in_4bit = False- # Login to allow private models- # Before normalization: dtype is also derived below from a 4bit config's- # compute dtype, which is not a request for the whole model.+ # Login to allow private models. Before normalization: dtype is also derived below from a 4bit+ # config's compute dtype, which is not a request for the whole model.user_float32 = _requested_float32(dtype)token = hf_login(token)if whisper_language is not None:@@ -1348,9 +1308,9 @@ class FastModel(FastBaseModel):assert load_in_fp8 in (True, False, "block")patch_compiled_autograd()- # Same best-effort wrapper as the FastLanguageModel path: unsloth_zoo's- # patch imports bitsandbytes unconditionally, so on a host without it this- # raised before the capability fallback below could take the 16bit path.+ # Same best-effort wrapper as the FastLanguageModel path: unsloth_zoo's patch imports+ # bitsandbytes unconditionally, so on a host without it this raised before the capability+ # fallback could take the 16bit path.try:patch_compiling_bitsandbytes()except Exception as e:@@ -1365,14 +1325,11 @@ class FastModel(FastBaseModel):load_in_fp8 = Falseload_in_16bit = False- # bitsandbytes unusable (absent, or unstable as on some AMD stacks). This is- # a capability check, so it is not gated on use_exact_model_name: that only- # suppresses repo-name remapping and cannot make bitsandbytes available.+ # bitsandbytes unusable (absent, or unstable on some AMD stacks). A capability check, so not+ # gated on use_exact_model_name, which only suppresses repo-name remapping.if not ALLOW_BITSANDBYTES:- # A user-supplied config sets load_in_4bit/8bit above and is forwarded- # in kwargs, so clearing the flags alone still rebuilds the bnb- # quantizer downstream. Only drop it when it asks for bnb: a GPTQ /- # AWQ / fp8 / torchao config must pass through untouched.+ # A user config sets load_in_4bit/8bit above and is forwarded in kwargs, so clearing the flags+ # alone still rebuilds the bnb quantizer downstream. Drop it only when it asks for bnb._quant_cfg = kwargs.get("quantization_config", None)if isinstance(_quant_cfg, dict):_wants_bnb = bool(@@ -1395,8 +1352,8 @@ class FastModel(FastBaseModel):"Unsloth: `bitsandbytes` is unavailable here - disabling 4bit/8bit. ""16bit LoRA and full finetuning still work.")- # 8bit is bitsandbytes too: leaving either set sends the request on to- # Transformers, which builds the bnb quantizer and fails there.+ # 8bit is bitsandbytes too: leaving either set sends the request to Transformers, which builds+ # the bnb quantizer and fails there.load_in_4bit = Falseload_in_8bit = Falseif _wants_bnb:@@ -1422,16 +1379,15 @@ class FastModel(FastBaseModel):if qat_scheme == "phone-deployment":qat_scheme = "int8-int4"- # Distributed-safe device placement for quantized models.- # In multi-GPU (torchrun), each rank must load the model on its own device- # to avoid Accelerate device relocation errors with quantized weights.+ # Distributed-safe placement for quantized models: under torchrun each rank must load on its+ # own device, else Accelerate raises device relocation errors.is_quantized = load_in_4bit or load_in_8bit or load_in_fp8device_map = requested_device_map(device_map)if is_quantized and isinstance(device_map, str):distributed_device_map, is_dist = prepare_device_map()if is_dist:- # One whole model per rank; sharding one across the ranks' GPUs as well- # would have every rank fighting for the same cards.+ # One whole model per rank; sharding one across the ranks' GPUs too would have every rank+ # fighting for the same cards.device_map = distributed_device_mapif fast_inference:@@ -1442,7 +1398,7 @@ class FastModel(FastBaseModel):)if DEVICE_TYPE_TORCH == "cuda":for i in range(DEVICE_COUNT):- # [TODO] DGX Spark vLLM breaks+ # DGX Spark vLLM breaks.if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():print("Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"@@ -1474,28 +1430,26 @@ class FastModel(FastBaseModel):else:assert new_model_name is not Nonemodel_name = new_model_name- # If mapper resolved to a pre-quantized FP8 model, disable- # on-the-fly quantization to avoid double quantization+ # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid+ # quantizing twice.if load_in_fp8 != False and new_model_name != old_model_name:load_in_fp8 = False# Only this block honours use_exact_model_name; the transforms below do not.mapper_moved_name = model_name != old_model_name- # Check if pre-quantized models are allowed- # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)+ # Are pre-quantized models allowed? AMD Instinct GPUs need blocksize = 128 on bitsandbytes+ # < 0.49.2, and our pre-quants use 64.if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):model_name = _strip_unsloth_bnb_4bit_suffix(model_name)- # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set- # Say so: dropping the flags in silence surfaces much later as an- # out-of-memory failure whose message never mentions quantization.+ # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set. Say+ # so: dropping the flags silently resurfaces as an OOM that never mentions quantization.if model_name.lower().endswith("-bf16") and (load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))):- # A user quantization_config stays in **kwargs and still quantizes.- # `load_in_4bit` defaults to True, so a set flag is not proof of a- # request; an explicit `load_in_16bit` IS one, for exactly this load.+ # A user quantization_config stays in **kwargs and still quantizes. load_in_4bit defaults to+ # True, so a set flag is no proof of a request; an explicit load_in_16bit is.if (not load_in_16bitand (load_in_4bit or load_in_8bit or load_in_fp8 != False)@@ -1513,7 +1467,6 @@ class FastModel(FastBaseModel):load_in_fp8 = Falseload_in_16bit = True- # Check modelscopeif USE_MODELSCOPE and not os.path.exists(model_name):from modelscope import snapshot_downloadmodel_name = snapshot_download(model_name)@@ -1522,13 +1475,12 @@ class FastModel(FastBaseModel):base_revision = _revision_for_resolved_repo(revision, model_name, old_model_name, mapper_moved_name)- # The PeftConfig probe reads the adapter repo, which peft loads in-process, so it- # keeps the ref even when vLLM takes the base model's away just after.+ # The PeftConfig probe reads the adapter repo, which peft loads in-process, so it keeps the ref+ # even when vLLM takes the base model's away just after.adapter_revision = base_revision- # vLLM fetches the default branch, so the pin is dead for the weights. Drop it here,- # not at dispatch: model_types, auto_model and the text-only decision all come off the- # config probed below, so a ref the weights are not at dispatches the wrong model.- # Same predicate as FastBaseModel, whose own guard is then a no-op here.+ # vLLM fetches the default branch, so the pin is dead for the weights. Drop it here, not at+ # dispatch: model_types, auto_model and the text-only decision all come off the config probed+ # below, so a ref the weights are not at dispatches the wrong model.if base_revision is not None and fast_inference and is_vLLM_available():logger.warning_once(f"Unsloth: Ignoring revision = `{base_revision}` since vLLM loads weights "@@ -1536,7 +1488,6 @@ class FastModel(FastBaseModel):)base_revision = None- # First check if it's a normal model via AutoConfigfrom huggingface_hub.utils import (disable_progress_bars,enable_progress_bars,@@ -1555,8 +1506,8 @@ class FastModel(FastBaseModel):# @_offline_aware_load already forced offline when needed; nested calls inherit it.local_files_only = kwargs.get("local_files_only", False)- # Text-diffusion slow-path dispatch, factored so both the normal route (below) and the- # legacy-config fallback (in the AutoConfig except handler) share one call site.+ # Text-diffusion slow-path dispatch, factored so the normal route and the legacy-config+ # fallback share one call site.def _dispatch_diffusion():model, tokenizer = FastDiffusionModel.from_pretrained(model_name = model_name,@@ -1573,9 +1524,8 @@ class FastModel(FastBaseModel):revision = base_revision,**kwargs,)- # Returns before the FORCE_FLOAT32 scan and no diffusion type is on that- # list, so False. Stamped, not left unset, or the trainer reads whatever- # UNSLOTH_FORCE_FLOAT32 an earlier load wrote.+ # Returns before the FORCE_FLOAT32 scan and no diffusion type is on that list, so False.+ # Stamped, not left unset, or the trainer reads whatever an earlier load wrote.model = _mark_forced_float32(model, False)model = _mark_full_finetuning(model, full_finetuning)return _mark_requested_float32(model, user_float32), tokenizer@@ -1596,10 +1546,9 @@ class FastModel(FastBaseModel):except Exception as error:autoconfig_error = str(error)autoconfig_exc = error- # Legacy text-diffusion configs use model_type "diffusion_gemma", which current- # transformers does not register by name (it ships "diffusion_gemma4"). AutoConfig- # raises before we can dispatch; route straight to the diffusion slow path, whose- # loader aliases the legacy type to the gemma4 classes.+ # Legacy text-diffusion configs use model_type "diffusion_gemma", which current transformers+ # does not register (it ships "diffusion_gemma4"), so AutoConfig raises before dispatch. Route+ # straight to the diffusion slow path, whose loader aliases the legacy type.if "diffusion_gemma" in autoconfig_error and is_diffusion_model_type("diffusion_gemma"):return _dispatch_diffusion()if "architecture" in autoconfig_error:@@ -1637,7 +1586,7 @@ class FastModel(FastBaseModel):is_peft = False# Old transformers versions checkboth_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32- # Error out if both LoRA and normal model config exists.+ # Error out if both LoRA and normal model config exist.if both_exist:raise RuntimeError("Unsloth: Your repo has a LoRA adapter and a base model.\n"@@ -1661,8 +1610,7 @@ class FastModel(FastBaseModel):f"AutoConfig error: {autoconfig_error}\n\n"f"PeftConfig error: {peft_error}\n\n")- # Chain an offline-related cause if either probe had one, so @_offline_aware_load- # still retries from cache (e.g. adapter repo: permanent AutoConfig 404 + transient PeftConfig).+ # Chain an offline-related cause if either probe had one, so @_offline_aware_load still retries from cache._cause = next((e@@ -1680,27 +1628,20 @@ class FastModel(FastBaseModel):model_types_all = ",".join(model_types) + ","_maybe_advise_fla_install(model_types)+ # Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom+ # block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile+ # patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.# ---- Text-diffusion models (e.g. DiffusionGemma) take a transformers-only slow path. ----- # These use a custom block-diffusion `generate` and a novel backbone, so we skip Unsloth's- # autoregressive kernel/compile patching and load the unmodified HF model (bit-identical to- # naive transformers), keeping only 4bit/8bit + PEFT LoRA conveniences.if is_diffusion_model_type(model_types):return _dispatch_diffusion()- # Save model types and loading methodlowered_model_name = model_name.lower()- # Build UNSLOTH_MODEL_NAME fresh from THIS load's model types + flags; do not prepend the- # inherited os.environ value (a stale "_load_in_4bit_" from an earlier load, e.g. across a- # save->reload subprocess, would push gpt-oss onto the BnB router patch when later loading- # a 16bit checkpoint -> "weights not initialized"). Only the type tokens and the load flags- # below are consumed downstream; the raw model name/path is excluded so a path containing a- # flag sentinel cannot be misread.- #- # Encode the EFFECTIVE bnb state: a non-bnb checkpoint (MXFP4/gptq/awq) has load_in_4bit- # disabled later by check_and_disable, so recording the requested flag here would route a- # native MXFP4 gpt-oss onto the BnB router patch. This is only an EARLY best-effort (an- # adapter-only PEFT repo has model_config=None here, and the base may be remapped); the- # authoritative correction is sync_unsloth_model_name_bnb_flags(...) after check_and_disable.+ # Build UNSLOTH_MODEL_NAME fresh from THIS load's types and flags: prepending the inherited+ # value would let a stale "_load_in_4bit_" push gpt-oss onto the BnB router patch on a later+ # 16bit load. The raw model name is excluded so a path containing a sentinel is not misread.+ # Encode the EFFECTIVE bnb state: a non-bnb checkpoint has load_in_4bit disabled later by+ # check_and_disable, so the requested flag would route a native MXFP4 gpt-oss onto the BnB+ # router patch. Early best-effort; sync_unsloth_model_name_bnb_flags is authoritative.try:from unsloth_zoo.utils import get_quant_type_bnb_compatible_quant = get_quant_type(model_config) in (None, "bitsandbytes")@@ -1717,18 +1658,15 @@ class FastModel(FastBaseModel):string += "load_in_fp8"os.environ["UNSLOTH_MODEL_NAME"] = string- # Check versionsLATEST = "\nPlease use transformers via `pip install --no-deps git+https://github.com/huggingface/transformers.git`"NIGHTLY = ('\nPlease use nightly transformers via pip install --upgrade "transformers>=4.49.0"`')- # Pixtralif "pixtral" in model_types_all and transformers_version < Version("4.49.0"):raise RuntimeError("Unsloth: Pixtral only works on transformers >= 4.49.0." + LATEST)- # Qwen 2.5elif "qwen2_5" in model_types_all and transformers_version < Version("4.49.0"):raise RuntimeError("Unsloth: Qwen 2.5 only works on transformers >= 4.49.0." + LATEST)- # Gemma 4 must be before Gemma 3N and Gemma 3+ # Gemma 4 must come before Gemma 3N, and Gemma 3N before Gemma 3.elif "gemma4" in model_types_all:if not SUPPORTS_GEMMA4:raise RuntimeError("Unsloth: Gemma 4 requires transformers >= 5.5.0" + LATEST)@@ -1748,54 +1686,46 @@ class FastModel(FastBaseModel):";""from unsloth_zoo.temporary_patches.gemma3n import patch_Gemma3nConv_Embed_forwards; patch_Gemma3nConv_Embed_forwards()")- # Set norms to float32 since anyways they get upcasted to float32- # common in both gemma-3 and gemma-3n+ # Set norms to float32 since they get upcasted anyway; common to gemma-3 and gemma-3n.+ # Granite-4 rms norms are stored as 16 bit, but we upcastos.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"- # Gemma 3elif "gemma3" in model_types_all:if transformers_version < Version("4.50.0.dev0"):raise RuntimeError("Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY)- # Set norms to float32 since anyways they get upcasted to float32- # common in both gemma-3 and gemma-3nos.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"- # ROCm/HIP: Gemma3 compiled forward produces NaN on RDNA GPUs- # (gfx1100, gfx1101, gfx1102, gfx1150, gfx1151, etc.).- # Disable torch.compile for model forward; loss compilation is fine.- # See https://github.com/unslothai/unsloth/issues/3385+ # ROCm/HIP: the Gemma3 compiled forward gives NaN on RDNA GPUs (gfx1100-gfx1151), so disable+ # torch.compile for the model forward; loss compilation is fine (#3385).from unsloth.kernels.utils import is_rdnaif is_rdna():os.environ["UNSLOTH_COMPILE_DISABLE"] = "partial"- # Cohereelif "cohere2" in model_types_all and transformers_version < Version("4.50.0.dev0"):raise RuntimeError("Unsloth: Cohere's Command model only works on transformers >= 4.50.0." + NIGHTLY)- # Sesameelif "csm" in model_types_all:os.environ["UNSLOTH_COMPILE_DISABLE"] = "partial" # Inference is too slowos.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Sesame fails+ # Set down projection compute dtype to be float32 for float16 machines Set norms to float32 since+ # anyways they get upcasted to float32register_custom_dtype("all;torch.float32;torch.float16;""if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)"";")- # Granite 4elif "granitemoehybrid" in model_types_all:- # Granite-4 rms norms are stored as 16 bit, but we upcast+ # Granite-4 rms norms are stored as 16 bit, but we upcast.+ # Set norms to float32 since anyways they get upcasted to float32os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"- # OLMo 2elif "olmo2" in model_types_all and transformers_version < Version("4.50.0.dev0"):raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY)- # OLMo 3elif "olmo3" in model_types_all and transformers_version < Version("4.57.0.dev0"):raise RuntimeError("Unsloth: OLMo-3 only works on transformers >= 4.57.0." + LATEST)elif "falcon_h1" in model_types_all:- # Falcon must use float32 Triton ie TRITON_F32_DEFAULT = 'ieee'- # since Mamba kernels error out on using lower precision+ # Falcon must use float32 Triton (TRITON_F32_DEFAULT = 'ieee') since Mamba kernels error on lower precision.register_custom_dtype("float16;torch.float32;torch.float16;""if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16)"@@ -1803,8 +1733,8 @@ class FastModel(FastBaseModel):"os.environ['TRITON_F32_DEFAULT'] = 'ieee'")elif "nemotron_h" in model_types_all:- # NemotronH (hybrid Mamba-2 + Transformer) uses same Mamba kernels as Falcon-H1- # Mamba kernels need float32 Triton precision+ # NemotronH (hybrid Mamba-2 + Transformer) uses the same Mamba kernels as Falcon-H1, which need+ # float32 Triton precision.register_custom_dtype("float16;torch.float32;torch.float16;""if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16)"@@ -1813,13 +1743,10 @@ class FastModel(FastBaseModel):)elif "gpt_oss" in model_types_all:os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"- # Use the EFFECTIVE bnb state, not the raw flag: a native MXFP4 checkpoint loaded- # with the default load_in_4bit=True (e.g. openai/gpt-oss-20b by exact name) has- # bnb disabled later by check_and_disable, so the raw flag would wrongly pick the- # BnB dtype path. Mirrors the _load_in_4bit_ token gate above.+ # Use the EFFECTIVE bnb state, not the raw flag: a native MXFP4 checkpoint loaded with the+ # default load_in_4bit=True has bnb disabled later, so the raw flag picks the wrong dtype path.if not (load_in_4bit and _bnb_compatible_quant):- # Only upcast MoE biases for MXFP4, not BnB- # Set norms to float32 since anyways they get upcasted to float32+ # Only upcast MoE biases for MXFP4, not BnB.register_custom_dtype("all;None;None;""x = 'gate_up_proj_bias'\n"@@ -1833,8 +1760,6 @@ class FastModel(FastBaseModel):";")else:- # Set down projection compute dtype to be float32 for float16 machines- # Set norms to float32 since anyways they get upcasted to float32register_custom_dtype("torch.float16;torch.bfloat16;torch.float16;""if ('down_projs' in name) and hasattr(module, 'weight') and "@@ -1845,7 +1770,6 @@ class FastModel(FastBaseModel):"module._pre_set_compute_dtype = torch.float32\n"";")- # Set norms to float32 since anyways they get upcasted to float32os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"else:for check_model_name in DISABLE_COMPILE_MODEL_NAMES:@@ -1860,12 +1784,11 @@ class FastModel(FastBaseModel):breakif auto_model is not None:- # All other models need to disable static cache+ # All other models need to disable static cache.os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"# New transformers need to check manually.if SUPPORTS_LLAMA32 and is_model and is_peft:- # Check if folder exists locallyif os.path.isdir(model_name):exist_adapter_config = os.path.exists(os.path.join(model_name, "adapter_config.json")@@ -1873,32 +1796,27 @@ class FastModel(FastBaseModel):exist_config = os.path.exists(os.path.join(model_name, "config.json"))both_exist = exist_adapter_config and exist_configelse:- # Both AutoConfig and PeftConfig loaded successfully from this- # remote repo, so both config.json and adapter_config.json- # definitely exist -- no need for an extra HfFileSystem network call.+ # Both AutoConfig and PeftConfig loaded from this remote repo, so config.json and+ # adapter_config.json both exist and no extra HfFileSystem call is needed.both_exist = True- # Get base model for PEFT:if is_peft:- # Check base model again for PEFTmodel_name = peft_config.base_model_name_or_pathif not use_exact_model_name:model_name = get_model_name(model_name, load_in_4bit)- # Check if pre-quantized models are allowed- # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)+ # Are pre-quantized models allowed? AMD Instinct GPUs need blocksize = 128 on bitsandbytes+ # < 0.49.2, and our pre-quants use 64.if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):model_name = _strip_unsloth_bnb_4bit_suffix(model_name)- # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set- # Say so: dropping the flags in silence surfaces much later as an- # out-of-memory failure whose message never mentions quantization.+ # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set. Say+ # so: dropping the flags silently resurfaces as an OOM that never mentions quantization.if model_name.lower().endswith("-bf16") and (load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))):- # A user quantization_config stays in **kwargs and still quantizes.- # `load_in_4bit` defaults to True, so a set flag is not proof of a- # request; an explicit `load_in_16bit` IS one, for exactly this load.+ # A user quantization_config stays in **kwargs and still quantizes. load_in_4bit defaults to+ # True, so a set flag is no proof of a request; an explicit load_in_16bit is.if (not load_in_16bitand (load_in_4bit or load_in_8bit or load_in_fp8 != False)@@ -1943,7 +1861,7 @@ class FastModel(FastBaseModel):if model_type_arch != "siglip":breakfor disable_name in FORCE_FLOAT32:- # add comma to model_types_all matching in case of exact match for end+ # Add a comma to model_types_all so an exact match at the end still matches.if (disable_name.lower() == model_type_arch.lower().replace("-", "").replace("_", "")or disable_name.lower() in model_types_all@@ -1952,7 +1870,6 @@ class FastModel(FastBaseModel):do_forced_float32 = Truedtype = torch.bfloat16 # Change to bfloat16 loadingbreak- # Apply gradient checkpointing with smart heuristicsuse_gradient_checkpointing = apply_unsloth_gradient_checkpointing(use_gradient_checkpointing, max_seq_length, dtype)@@ -1995,9 +1912,8 @@ class FastModel(FastBaseModel):if model_type in model_types_all:supports_sdpa = False- # Keep the local checkpoint dir as tokenizer when self-sufficient (see- # _resolve_checkpoint_tokenizer_name). A VLM also needs local processor files, else- # we fall back to the base repo so its cached processor loads.+ # Keep the local checkpoint dir as tokenizer when self-sufficient; a VLM also needs local+ # processor files, else fall back to the base repo so its cached processor loads._ckpt_arch = getattr(model_config, "architectures", None) or []_ckpt_is_vlm = any(x.endswith("ForConditionalGeneration") for x in _ckpt_arch) or hasattr(model_config, "vision_config"@@ -2006,8 +1922,7 @@ class FastModel(FastBaseModel):old_model_name, kwargs, require_processor = _ckpt_is_vlm)- # Capture task intent before text_only can replace a parent VLM config- # with its nested text config.+ # Capture task intent before text_only can replace a parent VLM config with its nested text config.task_config_attrs = _get_user_task_config_attrs(user_config)for _cfg_key in ("num_labels", "id2label", "label2id", "problem_type"):_cfg_val = kwargs.get(_cfg_key, None)@@ -2017,7 +1932,6 @@ class FastModel(FastBaseModel):for _cfg_key, _cfg_val in task_config_attrs.items():set_task_config_attr(model_config, _cfg_key, _cfg_val)- # Check if VLMarchitectures = getattr(model_config, "architectures", None)if architectures is None:architectures = []@@ -2028,8 +1942,8 @@ class FastModel(FastBaseModel):if load_text_only:if hasattr(model_config, "vision_config"):text_config = _get_text_only_config(model_config, old_model_name)- # Skip the vision tower only for families with their own text decoder (Gemma 3);- # others would load random weights, so keep the full model (use FastVisionModel).+ # Skip the vision tower only for families with their own text decoder (Gemma 3); others would+ # load random weights, so keep the full model.text_class = resolve_model_class(AutoModelForCausalLM, text_config)if text_class is None or not _is_family_text_decoder(getattr(model_config, "model_type", ""),@@ -2041,16 +1955,16 @@ class FastModel(FastBaseModel):f"Loading {old_model_name} as text-only; vision/audio towers skipped. ""Use FastVisionModel for multimodal inputs.")- # Remap VLM text weights (tf >=5) while model_config is still the parent. #5816+ # Remap VLM text weights (tf >= 5) while model_config is still the parent (#5816)._apply_text_only_key_mapping(kwargs, model_config, text_config)model_config = text_configis_vlm = False- # model_config is no longer the repo's config, so anything rebuilding- # it from model_name (the device-map planner) sees a different model.+ # model_config is no longer the repo's config, so anything rebuilding it from model_name (the+ # device-map planner) sees a different model.text_only_decoder = Trueelse:is_vlm = False- # If num_labels is set, use AutoModelForSequenceClassification+ # If num_labels is set, use AutoModelForSequenceClassification.for _cfg_key, _cfg_val in task_config_attrs.items():set_task_config_attr(model_config, _cfg_key, _cfg_val)if auto_model is None:@@ -2058,16 +1972,10 @@ class FastModel(FastBaseModel):from transformers import AutoModelForSequenceClassificationauto_model = AutoModelForSequenceClassificationelif is_vlm:- # Check if the model's auto_map supports the VLM auto class.- # Some repo-code VL models register only a generic auto class and not- # AutoModelForImageTextToText/AutoModelForVision2Seq: Nemotron-VL uses- # AutoModelForCausalLM, DeepSeek-OCR uses AutoModel. Calling the VLM auto- # class on those raises "Unrecognized configuration class ... for- # AutoModelForImageTextToText", so fall back to whatever generic class the- # repo actually registered. Match the CONCRETE class name we would pass- # (AutoModelForVision2Seq aliases to AutoModelForImageTextToText on tf>=5),- # since transformers resolves remote code by that exact name -- a config- # that only registers the legacy key must still take the generic fallback.+ # Some repo-code VL models register only a generic auto class (Nemotron-VL uses+ # AutoModelForCausalLM, DeepSeek-OCR AutoModel), so the VLM auto class raises "Unrecognized+ # configuration class". Fall back to what the repo registered, matching the CONCRETE class+ # name, since transformers resolves remote code by that exact name._auto_map = getattr(model_config, "auto_map", {}) or {}_vlm_class_name = AutoModelForVision2Seq.__name___has_vlm_class = _vlm_class_name in _auto_map@@ -2087,8 +1995,8 @@ class FastModel(FastBaseModel):load_in_4bit_kwargs = Falseload_in_8bit_kwargs = False- # FastBaseModel remaps again via fast_inference_setup. Skip for PEFT: model_name is- # then the base model, and `revision` names the adapter PeftModel loads below.+ # FastBaseModel remaps again via fast_inference_setup. Skip for PEFT: model_name is then the+ # base model, and `revision` names the adapter PeftModel loads below.if not is_peft:base_revision = _revision_for_resolved_repo(base_revision, model_name, old_model_name, mapper_moved_name@@ -2121,17 +2029,16 @@ class FastModel(FastBaseModel):whisper_task = whisper_task,auto_config = model_config,auto_config_from_caller = user_config is not None,- # `resize_token_embeddings` below replaces the embedding module and hooks do- # not travel to the replacement, so an offload installed during the load would- # leave a CPU embedding feeding a GPU decoder. An explicit request is left- # alone, since that combination was already this caller's to get wrong.+ # resize_token_embeddings below replaces the embedding module and hooks do not follow, so an+ # offload installed during the load would leave a CPU embedding feeding a GPU decoder. An+ # explicit request is left alone.offload_embedding = (Falseif resize_model_vocab is not None and offload_embedding == OFFLOAD_EMBEDDING_AUTOelse offload_embedding),float32_mixed_precision = float32_mixed_precision,- # Pass vLLM/inference parameters+ # Pass vLLM/inference parameters.fast_inference = fast_inference,gpu_memory_utilization = gpu_memory_utilization,float8_kv_cache = float8_kv_cache,@@ -2147,8 +2054,8 @@ class FastModel(FastBaseModel):if resize_model_vocab is not None:model.resize_token_embeddings(resize_model_vocab)- # `resize_token_embeddings` rebuilds the embedding, dropping `_hf_hook`;- # this is the last module swap of all, after every repair above.+ # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module+ # swap of all, after every repair above.try:from unsloth.models.vision import _repair_dispatch_hooks_repaired = _repair_dispatch_hooks(model)@@ -2178,11 +2085,10 @@ class FastModel(FastBaseModel):)if load_in_4bit:- # Fix up bitsandbytes config, but respect user-provided quantization_config+ # Fix up the bitsandbytes config, but respect a user-provided quantization_config.if quantization_config is None:- # `load_in_4bit` is the requested flag, not the effective one: a non-bnb- # checkpoint (MXFP4/gptq/awq) had bnb disabled by check_and_disable, so stamping a- # synthetic bnb config would corrupt its real one. Only stamp bnb/unquantized.+ # load_in_4bit is the requested flag, not the effective one: a non-bnb checkpoint had bnb+ # disabled by check_and_disable, so a synthetic bnb config would corrupt its real one.try:from unsloth_zoo.utils import get_quant_type_stamp_bnb = get_quant_type(model.config) in (None, "bitsandbytes")@@ -2191,14 +2097,14 @@ class FastModel(FastBaseModel):if _stamp_bnb:compute_dtype = dtype_from_config(model.config)quantization_config = {- # Sometimes compute_dtype is not a string!!+ # compute_dtype is sometimes not a string."bnb_4bit_compute_dtype": compute_dtype,"bnb_4bit_quant_type": "nf4","bnb_4bit_use_double_quant": True,"llm_int8_enable_fp32_cpu_offload": False,"llm_int8_has_fp16_weight": False,- # Whatever the load really used. None here would describe a layout- # that never existed, and saving it makes the adapter unreloadable.+ # Whatever the load really used. None here describes a layout that never existed, and saving it+ # makes the adapter unreloadable."llm_int8_skip_modules": _loaded_skip_modules(model.config),"llm_int8_threshold": 6.0,"load_in_4bit": True,@@ -2216,13 +2122,10 @@ class FastModel(FastBaseModel):_tag_model_with_fp8_torchao_config(model, fp8_mode)if is_peft:- # From https://github.com/huggingface/peft/issues/184- # Now add PEFT adapters+ # From huggingface/peft#184- # Gemma4 ClippableLinear wraps nn.Linear -- PEFT can't inject LoRA- # on it directly. Monkey-patch PEFT to target the inner .linear- # child instead (same patch as vision.py training path).- # See https://github.com/huggingface/peft/issues/3129+ # Gemma4 ClippableLinear wraps nn.Linear and PEFT cannot inject LoRA on it directly, so patch+ # PEFT to target the inner .linear child (same patch as vision.py). See huggingface/peft#3129._clippable_linear_cls = Nonetry:from transformers.models.gemma4.modeling_gemma4 import (@@ -2270,7 +2173,8 @@ class FastModel(FastBaseModel):_LoraModel._create_and_replace = _patched_car- # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet.+ # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet, and it always+ # loads in-process, so warm it even under fast_inference._prefetched = maybe_prefetch_hf_snapshot(old_model_name,token = token,@@ -2280,15 +2184,15 @@ class FastModel(FastBaseModel):# Adapter always loads in-process via PeftModel, so warm it even under fast_inference.fast_inference = False,force_download = kwargs.get("force_download", False),- # Leave use_safetensors auto (inheriting base format could skip a safetensors-only- # adapter). adapter_only restricts the warm to the adapter files + root aux.+ # Leave use_safetensors auto, since inheriting the base format could skip a safetensors-only+ # adapter; adapter_only restricts the warm to adapter files plus root aux.adapter_only = True,)- # Child did the forced download; clear the flag so the load reuses the warm cache.+ # The child did the forced download; clear the flag so the load reuses the warm cache.if _prefetched and kwargs.get("force_download", False):kwargs["force_download"] = False- # Forward cache_dir so the load reads the warmed adapter. No subfolder (that targets the- # base checkpoint; adapters live at the root).+ # Forward cache_dir so the load reads the warmed adapter. No subfolder: that targets the base+ # checkpoint, and adapters live at the root.peft_load_kwargs = {}if kwargs.get("cache_dir") is not None:peft_load_kwargs["cache_dir"] = kwargs["cache_dir"]@@ -2304,11 +2208,10 @@ class FastModel(FastBaseModel):**peft_load_kwargs,)finally:- # Always restore original PEFT method, even if loading fails+ # Always restore the original PEFT method, even if loading fails.if _clippable_linear_cls is not None:_LoraModel._create_and_replace = _original_car- # Patch it as well!model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing, trust_remote_code = trust_remote_code)@@ -2317,8 +2220,8 @@ class FastModel(FastBaseModel):_lift_endpoint_hooks_onto_adapters(model)except Exception:pass # never block loading on a placement nicety- # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back- # to the original loop, an attention-only adapter keeps the grouped path. Guarded.+ # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back to the+ # original loop, an attention-only adapter keeps the grouped path.try:from unsloth_zoo.temporary_patches.moe_grouped_modulelist import (auto_enable_grouped_moe,@@ -2327,13 +2230,11 @@ class FastModel(FastBaseModel):except Exception:pass # optional speedup; never block model loading- # Apply QAT if specifiedif qat_scheme is not None:print("Unsloth: Applying QAT to mitigate quantization degradation")model = FastModel._prepare_for_qat(model, qat_scheme)- # Patch Tiled MLP- # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}""+ # Patch Tiled MLP; enable by setting UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}".patch_tiled_mlp_choice = os.environ.get("UNSLOTH_TILED_MLP", "arctic" if unsloth_tiled_mlp else "0")diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 31c60ce6d2e..ac169165e71 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -34,16 +34,15 @@ from .mapper import (_add_lower_only,)-# The alias helpers a fetched mapper.py may call, resolved to the INSTALLED-# implementations. `_get_new_mapper` reads such calls as data: it takes the table and-# the two literal strings out of the AST and applies them with these, so the fetched-# text never supplies behaviour.+# The alias helpers a fetched mapper.py may call, resolved to the INSTALLED implementations:+# _get_new_mapper reads such calls as data, taking the table and the two literal strings out of+# the AST, so the fetched text never supplies behaviour._MAPPER_HELPERS = {"_add_with_lower": _add_with_lower,"_add_lower_only": _add_lower_only,}-# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!+# huggingface/transformers#26037 allows 4 bit loading.from transformers import __version__ as transformers_versionfrom unsloth.models._utils import TorchAOConfigfrom unsloth_zoo.utils import Version, get_quant_type@@ -66,7 +65,7 @@ BAD_MAPPINGS = {def _get_torchao_fp8_config(fp8_mode):- # Lazy import so a broken optional vLLM install doesn't break `import unsloth`.+ # Lazy import so a broken optional vLLM install does not break `import unsloth`.from unsloth_zoo.vllm_utils import _get_torchao_fp8_config as _implreturn _impl(fp8_mode)@@ -120,10 +119,8 @@ def prepare_device_map():UNSLOTH_DEVICE_MAP = "unsloth"-# Same planner, different answer when it declines. Every veto path ends in "sequential",-# which fills the first device to its whole free budget -- right for a loader default,-# wrong for a caller that picked several cards on their combined capacity. Naming the-# fallback spares such a caller enumerating veto reasons that are unsloth's and can grow.+# Same planner, different answer when it declines: every veto path ends in "sequential", which+# fills the first device to its whole free budget, wrong for a caller that picked several cards.UNSLOTH_BALANCED_DEVICE_MAP = "unsloth_balanced"_PLANNED_DEVICE_MAPS = {UNSLOTH_DEVICE_MAP: "sequential", UNSLOTH_BALANCED_DEVICE_MAP: "balanced"}@@ -159,9 +156,8 @@ def planner_hub_kwargs(loader_kwargs):hub["cache_dir"] = loader_kwargs["cache_dir"]if _get_effective_local_files_only(loader_kwargs):hub["local_files_only"] = True- # Resolving a remote class is a third lookup, and the planner honours `code_revision`- # for it (`_HUB_KWARGS` in unsloth_zoo's `device_map_planner`). Left out, the plan is- # built from the default revision's code and can name a tree the model does not have.+ # Resolving a remote class is a third lookup, and the planner honours code_revision for it; left+ # out, the plan is built from the default revision's code and can name a tree the model lacks.if loader_kwargs.get("code_revision") is not None:hub["code_revision"] = loader_kwargs["code_revision"]return hub@@ -243,9 +239,8 @@ def planner_quantization_kwargs(try:from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULESexcept Exception:- # Built on every quantized load, planning or not, so an older unsloth_zoo must- # not turn a 4bit load into an ImportError. One without the shared list predates- # the planner that consumes it, so this plan was going to decline anyway.+ # Built on every quantized load, planning or not, so an older unsloth_zoo must not turn a 4bit load+ # into an ImportError. One without the shared list predates the planner that consumes it.return kwargskwargs["llm_int8_skip_modules"] = SKIP_QUANTIZATION_MODULES + list(extra_skip_modules or [])return kwargs@@ -281,8 +276,8 @@ def planner_class_mismatch_reason(loaded_class, planned_class):return f"the load builds {loaded_class.__name__}, not the planned {planned_class.__name__}"-# accelerate's `max_memory` spellings, in the order it tries them: GiB/MiB/KiB are binary,-# GB/MB/KB are decimal, and a lowercase trailing `b` on a decimal unit means bits, not bytes.+# accelerate's max_memory spellings, in the order it tries them: GiB/MiB/KiB are binary, GB/MB/KB+# decimal, and a lowercase trailing `b` on a decimal unit means bits._SIZE_UNITS = (("GIB", 2**30, False),("MIB", 2**20, False),@@ -320,7 +315,7 @@ def _as_bytes(size):amount = int(float(size[: -len(unit)]) * scale)except ValueError:return None- # Bits, if they spelled the unit "Gb" rather than "GB". Binary units have no such form.+ # Bits, if they spelled the unit "Gb" rather than "GB"; binary units have no such form.if has_bit_form and size.endswith("b"):amount //= 8return amount if amount >= 0 else None@@ -350,7 +345,7 @@ def resolve_unsloth_device_map(`skip_reason` is the caller's veto, for when only the caller can tell the plannerwould describe a different model than the load builds."""- # `isinstance` first: a caller's explicit dict is unhashable, so `in` alone raises.+ # isinstance first: a caller's explicit dict is unhashable, so `in` alone raises.if not isinstance(device_map, str) or device_map not in _PLANNED_DEVICE_MAPS:return device_map_declined = _PLANNED_DEVICE_MAPS[device_map]@@ -366,8 +361,8 @@ def resolve_unsloth_device_map(if full_finetuning:return _fallback("full finetuning does not use the quantized planner")if is_distributed():- # Every rank already owns the whole model on its own card; splitting on top of- # that puts every rank on every card, a different execution model, not a bigger one.+ # Every rank already owns the whole model on its own card; splitting on top of that puts every rank+ # on every card, a different execution model.return _fallback("each rank of a distributed launch owns its own device")if DEVICE_TYPE_TORCH != "cuda":return _fallback(f"the planner has no memory budgets for {DEVICE_TYPE_TORCH}")@@ -376,23 +371,19 @@ def resolve_unsloth_device_map(except Exception as error:return _fallback(f"the devices could not be counted ({error})")- # Popped, not forwarded: `max_memory` is a named parameter of the planner, so a copy- # left in `planner_kwargs` raises `TypeError: got multiple values for keyword argument`,- # which the handler below turns into a silent "sequential", losing the cap and the plan.- #- # A caller's mapping replaces the measured one rather than editing it: its keys are the- # devices they will let the load use. That is accelerate's reading too --- # `_init_infer_auto_device_map` takes `devices = list(max_memory.keys())` and- # `get_max_memory` never widens a supplied mapping -- so `{0: ..., 1: ...}` on a- # four-GPU host means GPUs 2 and 3 are somebody else's.+ # Popped, not forwarded: max_memory is a named parameter of the planner, so a copy left in+ # planner_kwargs raises TypeError, which the handler below turns into a silent "sequential".+ # A caller's mapping replaces the measured one rather than editing it: its keys are the devices+ # they will let the load use, as accelerate reads it too, so {0: ..., 1: ...} on a four-GPU host+ # means GPUs 2 and 3 are somebody else's.planner_kwargs = dict(planner_kwargs or {})requested_memory = planner_kwargs.pop("max_memory", None)requested_memory = dict(requested_memory) if requested_memory else None- # Read before probing: `mem_get_info` initialises a CUDA context on each device it- # touches, and a withheld card is likely busy with the workload it was withheld for.- # One that refuses (ECC error, MIG parent, Exclusive_Process) would also drop the plan- # to "sequential" over a device this load was never going to use.+ # Read before probing: mem_get_info initialises a CUDA context on each device it touches, and a+ # withheld card is likely busy with the workload it was withheld for. One that refuses (ECC error,+ # MIG parent, Exclusive_Process) would also drop the plan to "sequential" over a device this load+ # was never going to use.if requested_memory is None:probe = list(range(device_count))else:@@ -411,8 +402,8 @@ def resolve_unsloth_device_map(except Exception as error:return _fallback(f"the planner is unavailable ({error})")- # Free, not total: this process's context and anything else resident on the card make- # total an overcommit. Guarded because a card can still refuse mid-probe.+ # Free, not total: this process's context and anything else resident make total an overcommit.+ # Guarded, because a card can still refuse mid-probe.try:max_memory = {index: torch.cuda.mem_get_info(index)[0] for index in probe}except Exception as error:@@ -424,12 +415,12 @@ def resolve_unsloth_device_map(measured = max_memory.get(device)budget = _as_bytes(written)if budget is None:- # Nothing to compare against: what we measured, or for a device we never- # measured (cpu, disk) their value untouched for the planner to read.+ # Nothing to compare against: what we measured, or for a device we never measured (cpu, disk) their+ # value untouched for the planner to read.budgets[device] = measured if measured is not None else writtenelse:- # Under what is actually free: they may know of reservations we cannot- # measure, but planning above free is how a plan OOMs on dispatch.+ # Under what is actually free: they may know of reservations we cannot measure, but planning above+ # free is how a plan OOMs on dispatch.budgets[device] = budget if measured is None else min(measured, budget)max_memory = budgets@@ -465,7 +456,7 @@ def __get_model_name(if load_in_fp8 != False:if load_in_fp8 == True and (os.environ.get("UNSLOTH_HAS_FBGEMM", "0") == "1"):if lower_model_name in FLOAT_TO_FP8_ROW_MAPPER:- # Faster row scaling only works if FBGEMM works!+ # Faster row scaling only works if FBGEMM works; otherwise use the slower blockwise type.return FLOAT_TO_FP8_ROW_MAPPER[lower_model_name]elif lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER:# Otherwise we use the slower blockwise type@@ -473,8 +464,8 @@ def __get_model_name(else:if lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER:return FLOAT_TO_FP8_BLOCK_MAPPER[lower_model_name]- # No pre-quantized model found. vllm >= 0.12.0 quantizes to FP8 on the- # fly (return original name); older vllm falls through to offline quant.+ # No pre-quantized model found. vllm >= 0.12.0 quantizes to FP8 on the fly (returning the original+ # name); older vllm falls through to offline quant.if importlib.util.find_spec("vllm") is not None:import vllmif Version(vllm.__version__) >= Version("0.12.0"):@@ -494,10 +485,6 @@ def __get_model_name(elif not load_in_4bit and lower_model_name in INT_TO_FLOAT_MAPPER:new_model_name = INT_TO_FLOAT_MAPPER[lower_model_name]- # logger.warning_once(- # f"Unsloth: You passed in `{model_name}` which is a 4bit model, yet you set\n"\- # f"`load_in_4bit = False`. We shall load `{new_model_name}` instead."- # )return new_model_nameelif not load_in_4bit and lower_model_name in MAP_TO_UNSLOTH_16bit:@@ -510,10 +497,6 @@ def __get_model_name(return model_namenew_model_name = FLOAT_TO_INT_MAPPER[lower_model_name]- # logger.warning_once(- # f"Unsloth: You passed in `{model_name}` and `load_in_4bit = True`.\n"\- # f"We shall load `{new_model_name}` for 4x faster loading."- # )return new_model_namereturn None@@ -527,12 +510,12 @@ def _get_new_mapper():new_mapper = ("https://raw.githubusercontent.com/unslothai/unsloth/main/unsloth/models/mapper.py")- # Capped WHILE reading, since `requests.get` buffers the whole body first, and- # the deadline is total because `timeout` is per-read.+ # Capped WHILE reading, since requests.get buffers the whole body first, and the deadline is total+ # because timeout is per-read.byte_cap = 1_000_000deadline = time.monotonic() + 10chunks, total = [], 0- # Redirects by hand: `requests` drains each intermediate body inside `get`.+ # Redirects by hand: requests drains each intermediate body inside get.url = new_mapperfor _ in range(5):response = requests.get(url, timeout = 3, stream = True, allow_redirects = False)@@ -546,10 +529,9 @@ def _get_new_mapper():if time.monotonic() > deadline:return {}, {}, {}, {}, {}else:- # `read1`: the timeout is per SOCKET READ, so a trickling peer- # resets it forever and the deadline is never reached.+ # read1: the timeout is per SOCKET READ, so a trickling peer resets it forever and the deadline is+ # never reached. requests only decompresses inside iter_content.raw = response.raw- # `requests` only decompresses inside `iter_content`.try:raw.decode_content = Trueexcept AttributeError:@@ -561,7 +543,7 @@ def _get_new_mapper():if read_once is not None:chunk = read_once(65_536)else:- # Older urllib3 has no `read1`; one byte returns as promptly.+ # Older urllib3 has no read1; one byte returns as promptly.chunk = raw.read(1)if not chunk:break@@ -576,18 +558,17 @@ def _get_new_mapper():else:return {}, {}, {}, {}, {}new_mapper = b"".join(chunks).decode(encoding, errors = "replace")- # Never exec the response: that is arbitrary code execution inside every- # `from_pretrained` that hits an unmapped name. Only `__INT_TO_FLOAT_MAPPER` is- # data, and the tables are returned rather than written into this module.+ # Never exec the response: that is arbitrary code execution inside every from_pretrained that hits+ # an unmapped name.import ast- # `ast.parse` builds the whole tree before any literal-only check runs, so this- # is no defence against exhaustion (python/cpython#95588); the byte cap above is.+ # ast.parse builds the whole tree before any literal-only check runs, so this is no defence against+ # exhaustion (python/cpython#95588); the byte cap above is.tree = ast.parse(new_mapper)# Every module-level literal dict, in source order; chosen below.literal_bindings = []for index, node in enumerate(tree.body):- # `AnnAssign` too, or an annotation upstream turns the probe off silently.+ # AnnAssign too, or an annotation upstream turns the probe off silently.if isinstance(node, ast.AnnAssign):targets = [node.target.id] if isinstance(node.target, ast.Name) else []value = node.value@@ -619,9 +600,8 @@ def _get_new_mapper():found = literalreturn found- # Statements that really RUN at import: `ast.walk` reaches into function bodies- # and dead branches, which would fabricate a mapping. Local, because the tests- # run this body in a bare namespace.+ # Statements that really RUN at import: ast.walk reaches into function bodies and dead branches,+ # which would fabricate a mapping. Local, because the tests run this body in a bare namespace.def _constant_truth(test):"""True/False for a statically decidable condition, else None.@@ -658,7 +638,8 @@ def _get_new_mapper():shadowed = frozenset(),class_body = False,):- # What ENDED the suite: `"return"` leaves the function, `True` the suite.+ # What ENDED the suite: "return" leaves the function, True the suite, and only the statements ABOVE+ # it still see the module global.for statement in body:if class_body:# Only the statements ABOVE it still see the module global.@@ -699,7 +680,7 @@ def _get_new_mapper():yield from _executed_nodes(children, shadowed)for handler in getattr(statement, "handlers", []) or []:yield from _executed_nodes(handler.body, shadowed)- # Only the non-suite parts; a `Lambda` is excluded at the seed too.+ # Only the non-suite parts; a Lambda is excluded at the seed too.pending = [childfor child in ast.iter_child_nodes(statement)@@ -734,8 +715,7 @@ def _get_new_mapper():continuepending.append(child)- # The aliases a newer mapper.py adds live inside `build_mappers`, which the- # installed builder cannot know.+ # The aliases a newer mapper.py adds live inside build_mappers, which the installed builder cannot know.def _calls_the_builder(node):"""Whether this executed node IS the `build_mappers(...)` call.@@ -750,8 +730,8 @@ def _get_new_mapper():)builder_body = []- # From the EXECUTED nodes, with the statement each runs in: a source table- # rebound after the call is not what the module exports.+ # From the EXECUTED nodes, with the statement each runs in: a source table rebound after the call is+ # not what the module exports.exported_names = ("INT_TO_FLOAT_MAPPER","FLOAT_TO_INT_MAPPER",@@ -900,8 +880,8 @@ def _get_new_mapper():if source_table is None:source_table = _binding_at(source_name, builder_index)- # Not the end of the probe: a row-only FP8 repo cannot be expressed through the- # source table at all. A body that adds nothing is reported so, at the end.+ # Not the end of the probe: a row-only FP8 repo cannot be expressed through the source table at all,+ # so a body that adds nothing is reported so at the end.empty_base = not source_tabletables = build_mappers(source_table or {})# Restored at the call below, which REPLACES all five tables.@@ -916,8 +896,8 @@ def _get_new_mapper():"FLOAT_TO_FP8_ROW_MAPPER": tables[4],}- # Only the helper CALLS: the builder binds its own `INT_TO_FLOAT_MAPPER = {}`,- # which the whole-name rule below would read as a clear.+ # Only the helper CALLS: the builder binds its own INT_TO_FLOAT_MAPPER = {}, which the whole-name+ # rule below would read as a clear.builder_additions = [(node, shadowed)for node, shadowed in _executed_nodes(builder_body)@@ -925,8 +905,8 @@ def _get_new_mapper():and isinstance(node.func, ast.Name)and node.func.id in _MAPPER_HELPERS]- # In EXECUTION order, so a rebind after the call still empties the table.- # `rebuilt` stands where the build assigns the exports; matched by identity.+ # In EXECUTION order, so a rebind after the call still empties the table; `rebuilt` stands where the+ # build assigns the exports, matched by identity.rebuilt = object()ordered = []for node, shadowed in _executed_nodes(tree.body):@@ -944,8 +924,7 @@ def _get_new_mapper():table.clear()table.update(original)elif isinstance(node, (ast.Assign, ast.AnnAssign)):- # An annotated subscript assigns as the plain form does; a bare one- # binds nothing.+ # An annotated subscript assigns as the plain form does; a bare one binds nothing.if isinstance(node, ast.AnnAssign):if node.value is None:continue@@ -964,11 +943,11 @@ def _get_new_mapper():continueif isinstance(replacement, dict):if not replacement and not builder_called:- # An INITIALISER, not a clear: a mapper.py with no- # `build_mappers` writes `X = {}` and fills all five- # from a module-scope loop the installed builder- # reproduces. Reading it as a clear emptied every- # table and the upgrade notice stopped firing.+ # An INITIALISER, not a clear: a mapper.py with no build_mappers writes X = {}+ # and fills all five+ # from a module-scope loop the installed builder reproduces, so reading it as a+ # clear emptied+ # every table and stopped the upgrade notice firing.continuetable.clear()table.update(replacement)@@ -1012,8 +991,7 @@ def _get_new_mapper():and isinstance(node.func, ast.Attribute)and node.func.attr == "update"):- # Only a named receiver and a literal mapping; the keyword form cannot- # express keys with a slash.+ # Only a named receiver and a literal mapping; the keyword form cannot express keys with a slash.if not isinstance(node.func.value, ast.Name):continueif node.func.value.id in shadowed:@@ -1028,8 +1006,8 @@ def _get_new_mapper():if isinstance(additions, dict):table.update(additions)elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name):- # An alias not derivable from the source table, applied with the- # INSTALLED helper from literal arguments only.+ # An alias not derivable from the source table, applied with the INSTALLED helper from literal+ # arguments only.helper = _MAPPER_HELPERS.get(node.func.id)if helper is None:continue@@ -1106,7 +1084,7 @@ def get_model_name(float_to_int = FLOAT_TO_INT_MAPPER,map_to_unsloth_16bit = MAP_TO_UNSLOTH_16bit,)- # Remap "bad" names (e.g. oversized dynamic quants or MoEs)+ # Remap "bad" names (oversized dynamic quants or MoEs).if (new_model_name is not Noneand type(new_model_name) is str@@ -1114,9 +1092,8 @@ def get_model_name():new_model_name = BAD_MAPPINGS[new_model_name.lower()]elif new_model_name is None and model_name.lower() in BAD_MAPPINGS:- # Some bad names (e.g. the `-unsloth-bnb-4bit` dynamic quants) are keys- # of the mappers, not values, so the resolver returns None for them and- # the remap above is skipped; remap the input name directly instead.+ # Some bad names (the -unsloth-bnb-4bit dynamic quants) are keys of the mappers, not values, so the+ # resolver returns None and the remap above is skipped; remap the input name directly.new_model_name = BAD_MAPPINGS[model_name.lower()]if (@@ -1125,7 +1102,6 @@ def get_model_name(and model_name[0].isalnum()and not _env_says_offline() # offline: skip the remote (raw GitHub) mapper refresh):- # Try checking if a new Unsloth version allows it!(NEW_INT_TO_FLOAT_MAPPER,NEW_FLOAT_TO_INT_MAPPER,@@ -1140,8 +1116,8 @@ def get_model_name(int_to_float = NEW_INT_TO_FLOAT_MAPPER,float_to_int = NEW_FLOAT_TO_INT_MAPPER,map_to_unsloth_16bit = NEW_MAP_TO_UNSLOTH_16bit,- # the fp8 probe has to look at the FETCHED tables too, or a new fp8 repo would- # miss both here and in the installed tables and skip the upgrade message+ # The fp8 probe has to look at the FETCHED tables too, or a new fp8 repo would miss both here and in+ # the installed tables and skip the upgrade message.fp8_block = NEW_FP8_BLOCK_MAPPER,fp8_row = NEW_FP8_ROW_MAPPER,)@@ -1190,7 +1166,7 @@ def _offline_quantize_to_fp8(for x in (getattr(config, "architectures", None) or []))is_vlm = is_vlm or hasattr(config, "vision_config")- # Decide text-only before the cache name so the fp8 artifact and its path stay in sync. #5816+ # Decide text-only before the cache name so the fp8 artifact and its path stay in sync (#5816).text_config = Noneif text_only and hasattr(config, "vision_config"):from ._utils import (@@ -1209,11 +1185,11 @@ def _offline_quantize_to_fp8(is_vlm = Falsetemp_dir = tempfile.gettempdir()- # Cache text-only and full-VLM artifacts separately so neither reuses the other. #5816+ # Cache text-only and full-VLM artifacts separately so neither reuses the other (#5816).cache_name = model_name.split("/")[-1] + "-fp8-" + fp8_modeif revision is not None:- # Sanitizing is lossy (`release/v1` and `release.v1` collapse), so a digest of the- # raw ref rides along and two refs never share an artifact.+ # Sanitizing is lossy (release/v1 and release.v1 collapse), so a digest of the raw ref rides along+ # and two refs never share an artifact.digest = hashlib.sha256(revision.encode("utf-8")).hexdigest()[:12]readable = re.sub(r"[^0-9A-Za-z_-]", "_", revision)[:40]cache_name += "-rev-" + readable + "-" + digest@@ -1336,6 +1312,7 @@ def _load_fp8_weight_map(if is_local and os.path.exists(_local_path(index_file)):index_path = _local_path(index_file)elif not is_local:+ # Errors after this point are per-tensor: warn and continue, never abort or hide them.try:index_path = _remote_path(index_file)except Exception:@@ -1407,7 +1384,7 @@ def _match_fp8_module(module_by_name, base):if "language_model.model." in base:candidates.append(base.replace("language_model.model.", "model.language_model.", 1))if base.startswith("language_model."):- candidates.append("model." + base) # add model. prefix+ candidates.append("model." + base)for candidate in candidates:if candidate in module_by_name:return module_by_name[candidate]@@ -1441,8 +1418,8 @@ def _restore_dropped_fp8_scales(# A variant load reads variant-named files; skip to avoid applying default scales to them.if variant:return (0, 0)- # No fp8 params means the checkpoint was dequantized on purpose (e.g. load_in_16bit);- # re-applying a scale would corrupt those already-correct 16bit weights, so do nothing.+ # No fp8 params means the checkpoint was dequantized on purpose (load_in_16bit), and re-applying a+ # scale would corrupt those already-correct 16bit weights.if not any(p.dtype in _FP8_DTYPES for p in model.parameters()):return (0, 0)weight_map = _load_fp8_weight_map(@@ -1471,8 +1448,8 @@ def _restore_dropped_fp8_scales(if not isinstance(weight, torch.Tensor) or weight.ndim != 2:continueif weight.device.type == "meta":- # Disk-offloaded layer: weight lives on meta until forward, so it cannot be- # scaled in place here. Count and warn rather than silently leave it unscaled.+ # Disk-offloaded layer: the weight lives on meta until forward, so it cannot be scaled in place+ # here. Count and warn rather than silently leave it unscaled.offloaded += 1continueif weight.dtype in _FP8_DTYPES:@@ -1510,9 +1487,8 @@ def _restore_dropped_fp8_scales(scale = scale.to(weight.device)with torch.no_grad():if out_features % bs0 == 0 and in_features % bs1 == 0:- # Memory-frugal path: multiply block views in place against the broadcast- # fp32 scale, avoiding a full expanded scale and fp32 copy that could OOM.- # The in-place multiply promotes to fp32, matching the fallback exactly.+ # Memory-frugal path: multiply block views in place against the broadcast fp32 scale, avoiding a+ # full expanded scale and fp32 copy that could OOM.module.weight.data.view(out_blocks, bs0, in_blocks, bs1).mul_(scale[:, None, :, None])@@ -1569,8 +1545,8 @@ def check_and_disable_bitsandbytes_loading(if quant_method is None or quant_method == "bitsandbytes":return load_in_4bit, load_in_8bit, quant_method- # Model has a non-bitsandbytes quantization config (e.g., compressed-tensors, gptq, awq)- # We should disable BOTH bitsandbytes loading to avoid config conflicts+ # A non-bitsandbytes quantization config (compressed-tensors, gptq, awq) means BOTH bitsandbytes+ # loading flags must be disabled to avoid config conflicts.if load_in_4bit or load_in_8bit:if verbose:print(@@ -1620,7 +1596,6 @@ def _get_fp8_mode_and_check_settings(else:fp8_mode = load_in_fp8- # Check user settingsif fp8_mode not in ["row", "block"]:raise ValueError(f"Unsloth: `load_in_fp8` can only be 'row' or 'block', got '{fp8_mode}'")if full_finetuning:@@ -1646,8 +1621,7 @@ def _get_fp8_mode_and_check_settings("Unsloth: On the fly `load_in_fp8` requires torch 2.9.0+. Try `unsloth/Qwen3-8B` instead.")- # Check if torchao has this PR: https://github.com/pytorch/ao/pull/3158,- # which will be released in 0.15.0.+ # Check if torchao has pytorch/ao#3158, released in 0.15.0.if importlib.util.find_spec("torchao") is None:raise ValueError("Unsloth: Please install torchao for on the fly float8 to work! Try `unsloth/Qwen3-8B` instead."@@ -1662,7 +1636,7 @@ def _get_fp8_mode_and_check_settings(if Version(torchao.__version__) < Version("0.15.0"):raise ValueError(error_message)- # If fbgemm_gpu_genai is installed and old, disable FBGEMM and use Triton instead+ # If fbgemm_gpu_genai is installed and old, disable FBGEMM and use Triton kernels instead.if (importlib.util.find_spec("fbgemm_gpu") is not Noneand importlib.util.find_spec("fbgemm_gpu.experimental") is not None@@ -1679,18 +1653,12 @@ def _get_fp8_mode_and_check_settings(return fp8_mode-# Rotary inv_freq buffers are deliberately kept on CPU - Unsloth pre-builds a-# cos/sin cache per GPU instead (see LlamaRotaryEmbedding.multi_gpu_cos_cached)-# so the GPU-resident lookup never needs to move the tiny inv_freq tensor itself.-# torch.nn.parallel.DistributedDataParallel ignores device entirely when it-# broadcasts buffers across ranks, so a CPU buffer crashes NCCL's-# _broadcast_coalesced with "No backend type associated with device type cpu".-# Telling DDP to skip these specific buffers avoids that crash without moving-# inv_freq to GPU (which would break the per-GPU cache design) and without-# disabling buffer broadcast for every other module (the user's workaround).-# Re-run this after wrapping with PEFT too - the buffers' fully qualified-# names change once they sit under a PeftModel (eg "base_model.model...").-# https://github.com/unslothai/unsloth/issues/6656+# Rotary inv_freq buffers are deliberately kept on CPU: Unsloth pre-builds a cos/sin cache per GPU+# instead (LlamaRotaryEmbedding.multi_gpu_cos_cached). torch.nn.parallel.DistributedDataParallel+# ignores device when it broadcasts buffers across ranks, so a CPU buffer crashes NCCL's+# _broadcast_coalesced with "No backend type associated with device type cpu"; telling DDP to skip+# these buffers avoids that without moving inv_freq to GPU. Re-run after wrapping with PEFT, since+# the fully qualified names change under a PeftModel ("base_model.model...") (#6656)._ROTARY_INV_FREQ_BUFFER_NAMES = ("inv_freq", "short_inv_freq", "long_inv_freq")@@ -1707,16 +1675,15 @@ def _exclude_rope_inv_freq_from_ddp(model):from torch.nn.parallel import DistributedDataParallelDistributedDataParallel._set_params_and_buffers_to_ignore_for_model(model, ignored)except Exception:- # Private PyTorch API - fall back to setting the attribute DDP reads- # directly if it ever moves or changes signature.+ # Private PyTorch API: fall back to setting the attribute DDP reads directly if it ever moves or+ # changes signature.model._ddp_params_and_buffers_to_ignore = ignoredreturn model-# =============================================================================-# Offline loading - single source of truth (shared by vision.py, loader.py and-# the Unsloth exporter). Decide offline ONCE at the load boundary and force it-# ONCE around the whole load, so every nested HF call inherits it.+# Offline loading, shared by vision.py, loader.py and the Unsloth exporter: decide offline ONCE at+# the load boundary and force it ONCE around the whole load, so nested HF calls inherit it.+# =============================================================================_OFFLINE_ENV_VALUES = {"1", "true", "yes", "on"}@@ -1737,17 +1704,16 @@ def _get_effective_local_files_only(kwargs):return _env_says_offline()-# Attribute stamped on a tokenizer/processor that was loaded local-only, so a later-# save still knows. transformers takes local_files_only as an explicit from_pretrained-# parameter and never copies it into tokenizer.init_kwargs, and _offline_aware_load-# restores the offline env vars when the load window closes, so without this stamp an-# explicit local_files_only = True load is invisible by the time we save (issue #7481).+# Stamped on a tokenizer/processor loaded local-only so a later save still knows: transformers+# takes local_files_only as an explicit from_pretrained parameter and never copies it into+# tokenizer.init_kwargs (#7481). The load's cache_dir and ref travel with it, since saving+# otherwise derives a cache from HF_HUB_CACHE and drops the branch from name_or_path._LOCAL_FILES_ONLY_ATTR = "_unsloth_local_files_only"-# The load's cache_dir travels with it too: saving derives one from HF_HUB_CACHE /-# HF_HOME, which does not see a caller-supplied cache.+# The load's cache_dir travels with it too: saving derives one from HF_HUB_CACHE / HF_HOME, which does not see a+# caller-supplied cache._LOADED_CACHE_DIR_ATTR = "_unsloth_loaded_cache_dir"-# So does the ref it was read at: saving restores sentencepiece assets from-# tokenizer.name_or_path, which names the repo but not the branch.+# So does the ref it was read at: saving restores sentencepiece assets from tokenizer.name_or_path, which names+# the repo but not the branch._LOADED_REVISION_ATTR = "_unsloth_loaded_revision"@@ -1756,6 +1722,8 @@ def _mark_loaded_revision(result, revision):if revision is None:return resultfor obj in result if isinstance(result, (tuple, list)) else (result,):+ # Objects that reject new attributes (__slots__) are skipped.+ # Skip objects that reject new attributes (__slots__).try:targets = (obj, getattr(obj, "tokenizer", None))except Exception:@@ -1763,7 +1731,6 @@ def _mark_loaded_revision(result, revision):for target in targets:if target is None:continue- # Skip objects that reject new attributes (__slots__).try:setattr(target, _LOADED_REVISION_ATTR, str(revision))except Exception:@@ -1781,15 +1748,14 @@ def _mark_loaded_local_files_only(result, cache_dir = None):"""Stamp a load's local-only mode and cache_dir onto the returned objects."""for obj in result if isinstance(result, (tuple, list)) else (result,):try:- # A processor keeps the tokenizer that _has_tokenizer_model unwraps to,- # so stamp both (a wrapped model can raise from its own __getattr__).+ # A processor keeps the tokenizer that _has_tokenizer_model unwraps to, so stamp both; a wrapped+ # model can raise from its own __getattr__.targets = (obj, getattr(obj, "tokenizer", None))except Exception:targets = (obj,)for target in targets:if target is None:continue- # Objects that reject new attributes (__slots__) are skipped.try:setattr(target, _LOCAL_FILES_ONLY_ATTR, True)if cache_dir:@@ -1825,7 +1791,8 @@ def _is_offline_related_error(exc):# Match network failures by type (locale independent), not just message wording._net_types = [ConnectionError, TimeoutError, socket.gaierror, urllib.error.URLError]_offline_fnf_types = () # FileNotFoundError subclasses that count as offline- # urllib HTTPError is a URLError subclass: judge by status (5xx offline, 4xx propagates).+ # urllib HTTPError is a URLError subclass, so judge by status (5xx offline, 4xx propagates).+ # TLS/cert failures are security-sensitive (MITM, expired CA) and are never offline-retried._http_types = (urllib.error.HTTPError,)# TLS/cert failures are security-sensitive (MITM, expired CA): never offline-retry them._ssl_types = [ssl.SSLError]@@ -1883,7 +1850,7 @@ def _is_offline_related_error(exc):"connection refused","we couldn't connect to","proxyerror",- # Raw socket.gaierror DNS wording (Linux / macOS)+ # Raw socket.gaierror DNS wording (Linux / macOS)."name or service not known","temporary failure in name resolution","nodename nor servname provided",@@ -1892,14 +1859,14 @@ def _is_offline_related_error(exc):cur = excwhile cur is not None and id(cur) not in seen:seen.add(id(cur))- # TLS/cert failure (corporate MITM, expired CA): security-sensitive, never retry from- # cache. Skip this node; a deeper cause in the chain may still be a genuine outage.+ # TLS/cert failure (corporate MITM, expired CA): security-sensitive, never retry from cache. Skip+ # this node; a deeper cause in the chain may still be a genuine outage.if isinstance(cur, _ssl_types) or isinstance(getattr(cur, "reason", None), _ssl_types):cur = cur.__cause__ or cur.__context__continueis_fnf = isinstance(cur, FileNotFoundError) and not isinstance(cur, _offline_fnf_types)- # urllib HTTPError is a URLError (net type) but must be judged by status code below,- # unlike LocalEntryNotFoundError (an HfHubHTTPError that is always offline).+ # urllib HTTPError is a URLError (net type) but must be judged by status code below, unlike+ # LocalEntryNotFoundError, an HfHubHTTPError that is always offline.if (isinstance(cur, _net_types)and not is_fnf@@ -1910,10 +1877,10 @@ def _is_offline_related_error(exc):code = _http_status(cur)if code is not None and 500 <= code < 600:return True- # No status -> wording fallback (coded 4xx already decided above).+ # No status, so fall back to wording; a coded 4xx was already decided above.if code is None and not is_fnf and any(w in str(cur).lower() for w in _wording):return True- # OSError wording fallback (HTTP status already decided above).+ # OSError wording fallback; the HTTP status was already decided above.elif isinstance(cur, OSError) and not is_fnf:if any(w in str(cur).lower() for w in _wording):return True@@ -1921,17 +1888,20 @@ def _is_offline_related_error(exc):return False-# Process-wide HF offline state; the depth counter lets nested windows share one-# flip (first entrant saves originals, last exit restores). Lock guards flip/restore.+# Process-wide HF offline state; the depth counter lets nested windows share one flip (first entrant saves+# originals, last exit restores). Lock guards flip/restore._force_offline_lock = _threading.RLock()_force_offline_depth = 0-_force_offline_saved = [] # in-process module attributes-_force_offline_saved_env = {} # HF offline env-var originals+_force_offline_saved = []+_force_offline_saved_env = {}def _reset_hf_sessions():"""Clear hub's per-thread cached Sessions so the next rebuilds against the currentoffline flag. On hub 0.x the offline adapter is baked in at Session creation. Best-effort."""+ # Snapshot in-process constants BEFORE forcing the env: a module first imported here would otherwise+ # initialize its constant from the just-set "1" and we would save (then restore) True, pinning the process+ # offline after the window.try:from huggingface_hub.utils._http import reset_sessionsexcept Exception:@@ -1956,9 +1926,6 @@ def _force_hf_offline():if _force_offline_depth == 0:saved = []saved_env = {}- # Snapshot in-process constants BEFORE forcing the env: a module first imported- # here would otherwise initialize its constant from the just-set "1" and we would- # save (then restore) True, pinning the process offline after the window.try:import huggingface_hub.constants as _hfcif hasattr(_hfc, "HF_HUB_OFFLINE"):@@ -1983,6 +1950,7 @@ def _force_hf_offline():pass_force_offline_saved = saved_force_offline_saved_env = saved_env+ # Drop offline-mounted sessions so later online calls rebuild for the network.# Rebuild cached sessions so they pick up the offline adapter._reset_hf_sessions()_force_offline_depth += 1@@ -2004,7 +1972,6 @@ def _force_hf_offline():else:os.environ[_k] = _v_force_offline_saved_env = {}- # Drop offline-mounted sessions so later online calls rebuild for the network._reset_hf_sessions()@@ -2029,11 +1996,11 @@ def _restore_progress_bars(were_disabled):pass-# Every way a cache miss reaches the caller once offline mode has skipped Transformers'-# own "does not appear to have a file named" raise: the resolved path stays None and the-# next line dereferences it, so the message names the None and never the cache. Same set-# the Unsloth training worker matches (studio/backend/core/training/worker.py, #7845):-# weights come out as `endswith`, tokenizers/processors as any of the other four.+# Every way a cache miss reaches the caller once offline mode has skipped Transformers' own "does not appear to+# have a file named" raise: the resolved path stays None and the next line dereferences it, so the message names+# the None and never the cache. Same set the Unsloth training worker matches+# (studio/backend/core/training/worker.py, #7845): weights come out as `endswith`, tokenizers/processors as any+# of the other four._EMPTY_CACHE_ARTIFACTS = ("'nonetype' object has no attribute 'endswith'","'nonetype' object has no attribute 'readlines'",@@ -2059,8 +2026,8 @@ def _empty_cache_artifact(exc):while exc is not None and id(exc) not in seen:seen.add(id(exc))text = str(exc).lower()- # Gate on the None: the same TypeError wording about a real path (`not 'int'`)- # is a caller bug, not an empty cache.+ # Gate on the None: the same TypeError wording about a real path (`not 'int'`) is a caller bug, not an+ # empty cache.if ("nonetype" in text or "path 'none'" in text) and any(artifact in text for artifact in _EMPTY_CACHE_ARTIFACTS):@@ -2118,25 +2085,25 @@ def _offline_aware_load(fn):if _get_effective_local_files_only(kwargs):kwargs["local_files_only"] = Truewith _force_hf_offline():- # Stamp inside the window: the env vars are restored on exit, so the- # request has to travel on the objects themselves to reach saving.+ # Stamp inside the window: the env vars are restored on exit, so the request has to travel on+ # the objects themselves to reach saving.return _mark_loaded_local_files_only(fn(*args, **kwargs), kwargs.get("cache_dir"))- _pb_were_disabled = _progress_bars_were_disabled() # restore before any retry+ _pb_were_disabled = _progress_bars_were_disabled()try:return fn(*args, **kwargs)except Exception as e:- # Skip if not network-related, or already retried by a nested decorator- # (else outer layers reload the whole model again).+ # Skip if not network-related, or already retried by a nested decorator (else outer layers reload+ # the whole model again).if not _is_offline_related_error(e) or getattr(e, "_unsloth_offline_retried", False):raise- # Holding `e` holds its frames, and those frames hold the half-built model,- # so the collect below could not free it and a large VLM OOMed on the reload- # the retry exists to make. A wrapper's cause/context carries tracebacks over- # the SAME frames, so the whole chain has to be released, not just the top.+ # Holding `e` holds its frames, and those frames hold the half-built model, so the collect below+ # could not free it and a large VLM OOMed on the reload the retry exists to make. A wrapper's+ # cause/context carries tracebacks over the SAME frames, so the whole chain has to be released, not+ # just the top.online_error = e_release_traceback_locals(online_error)- # Retry OUTSIDE the except so the failed attempt's traceback (a partial model)- # is freed before reallocating, else a large VLM can OOM on the second load.+ # Retry OUTSIDE the except so the failed attempt's traceback (a partial model) is freed before+ # reallocating, else a large VLM can OOM on the second load.try:gc.collect()if torch.cuda.is_available():@@ -2152,8 +2119,8 @@ def _offline_aware_load(fn):with _force_hf_offline():return fn(*args, **kwargs)except Exception as e:- # A real retry failure (corrupt checkpoint, OOM) is news and goes out as- # itself; only the empty-cache artifact is worth replacing.+ # A real retry failure (corrupt checkpoint, OOM) is news and goes out as itself; only the empty-+ # cache artifact is worth replacing.if not _empty_cache_artifact(e):# Tag so an enclosing _offline_aware_load skips its own redundant retry.try:@@ -2161,22 +2128,21 @@ def _offline_aware_load(fn):except Exception:passraise- # The retry can load a whole cached model and only then trip over a missing- # tokenizer file, and this error outlives the call on the online one, so its- # frames would keep that model resident for as long as the caller holds it.+ # The retry can load a whole cached model and only then trip over a missing tokenizer file, and this+ # error outlives the call on the online one, so its frames would keep that model resident for as+ # long as the caller holds it._release_traceback_locals(e)retry_error = e- # Report the ONLINE error: this retry only runs because of it, and its own- # failure names an empty cache badly (offline mode skips Transformers'- # "does not appear to have a file named" raise, so the user saw+ # Report the ONLINE error: this retry only runs because of it, and its own failure names an empty cache+ # badly (offline mode skips Transformers' "does not appear to have a file named" raise, so the user saw# `AttributeError: 'NoneType' ... 'endswith'`).try:online_error._unsloth_offline_retried = Trueexcept Exception:pass- # Raise OUTSIDE the handler above: inside it, Python would overwrite- # `__context__` with the cache miss, and that chain is often the only thing- # that still makes the online error classifiable as network-related.+ # Raise OUTSIDE the handler above: inside it, Python would overwrite `__context__` with the cache miss,+ # and that chain is often the only thing that still makes the online error classifiable as network-+ # related.if online_error.__cause__ is None and online_error.__context__ is None:# No chain to lose, so chain the retry on where it also prints.raise online_error from retry_error@@ -2215,8 +2181,8 @@ def _resolve_hub_repo_local_dir(token = None,cache_dir = None,revision = None,- # Default closed: a "resolve local dir" helper must not download. False here- # means five filenames each retried with backoff before it gives up.+ # Default closed: a "resolve local dir" helper must not download. False here means five filenames each+ # retried with backoff before it gives up.local_files_only = True,filenames = ("tokenizer_config.json",diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.pyindex 71095b796db..6fa681f41d5 100644--- a/unsloth/models/mapper.py+++ b/unsloth/models/mapper.py@@ -1116,8 +1116,8 @@ __INT_TO_FLOAT_MAPPER = \"unsloth/Magistral-Small-2509-bnb-4bit",),},- # No Unsloth 16bit repo exists at this size, so this is a 1-tuple naming the- # real upstream, same as the other 70B and 405B rows.+ # No Unsloth 16bit repo exists at this size, so this is a 1-tuple naming the real upstream, like the+ # other 70B and 405B rows."unsloth/Apertus-70B-Instruct-2509-unsloth-bnb-4bit" : ("swiss-ai/Apertus-70B-Instruct-2509",),@@ -1311,7 +1311,6 @@ __INT_TO_FLOAT_MAPPER = \"google/functiongemma-270m-it","unsloth/functiongemma-270m-it-unsloth-bnb-4bit",),- # Ministral 3 models"unsloth/Ministral-3-3B-Instruct-2512-unsloth-bnb-4bit" : {"8" : ("mistralai/Ministral-3-3B-Instruct-2512",@@ -1426,7 +1425,7 @@ def build_mappers(__INT_TO_FLOAT_MAPPER):if type(values) is dict:assert "16" in valuesfloat16_values = values["16"]- # Float8 and other quantized types+ # Float8 and other quantized types.if "8" in values:float8_values = values["8"]assert len(float8_values) == 3@@ -1455,14 +1454,14 @@ def build_mappers(__INT_TO_FLOAT_MAPPER):for value in values:FLOAT_TO_INT_MAPPER[value] = key- # Map to Unsloth version for 16bit versions+ # Map to the Unsloth version for 16bit.if len(values) == 2:if values[0].startswith("unsloth"):_add_with_lower(MAP_TO_UNSLOTH_16bit, values[1], values[0])_add_with_lower(MAP_TO_UNSLOTH_16bit, block, values[0])_add_with_lower(MAP_TO_UNSLOTH_16bit, row, values[0])elif len(values) == 3:- # Dynamic Unsloth quantization+ # Dynamic Unsloth quantization.if values[0].startswith("unsloth"):_add_with_lower(MAP_TO_UNSLOTH_16bit, values[1], values[0])_add_with_lower(MAP_TO_UNSLOTH_16bit, values[2], values[0])@@ -1470,7 +1469,6 @@ def build_mappers(__INT_TO_FLOAT_MAPPER):_add_with_lower(MAP_TO_UNSLOTH_16bit, row, values[0])pass- # Get lowercasedlowered_key = key.lower()INT_TO_FLOAT_MAPPER[lowered_key] = values[0].lower()diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex 41b5a7c3b6b..4da95393af5 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -68,7 +68,6 @@ def MistralAttention_fast_forward(*args,**kwargs,) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:- # Clear inferenceif hasattr(self, "paged_attention"):del self.paged_attention_Kdel self.paged_attention_V@@ -96,7 +95,7 @@ def MistralAttention_fast_forward(if past_key_value is not None:kv_seq_len += past_key_value[0].shape[-2]- # Extend RoPE dynamically to fit in VRAM+ # Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.self.rotary_emb.extend_rope_embedding(V, seq_len = kv_seq_len)cos, sin = self.rotary_emb.get_cached(kv_seq_len, Q.device.index)@@ -109,11 +108,10 @@ def MistralAttention_fast_forward(V = torch.cat([past_key_value[1], V], dim = 2)past_key_value = (K, V) if use_cache else None- # Attention modulesw_cfg = getattr(self.config, "sliding_window", None)- # A non-positive window means "no local attention", the same as absent. Passing 0 through- # made window_size (0, 0) for flash and an all-false SDPA mask, since the lower bound- # q_pos - (0 - 1) sits above the causal upper bound.+ # A non-positive window means "no local attention", the same as absent: passing 0 through made+ # window_size (0, 0) for flash and an all-false SDPA mask, since q_pos - (0 - 1) sits above the+ # causal upper bound.if sw_cfg is None or sw_cfg == "null" or (isinstance(sw_cfg, int) and sw_cfg <= 0):sw = kv_seq_lenelse:@@ -133,8 +131,8 @@ def MistralAttention_fast_forward("softmax_scale": getattr(self, "softmax_scale", None),},)- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse- # (KV cache / padding mask) raises. None => byte-identical default.+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /+ # padding mask) raises. None means the byte-identical default._pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)context = AttentionContext(bsz = bsz,@@ -146,11 +144,10 @@ def MistralAttention_fast_forward(seq_info = seq_info,attention_mask = attention_mask,causal_mask = causal_mask,- # The window the flash path already gets through window_size. SDPA needs it too, and- # not only in the branch above: training takes `elif self.training: pass`, so no 4D- # mask is synthesized, and with xformers off and flash absent the local window was the- # one thing nothing carried -- every sequence past config.sliding_window silently- # attended its whole causal history.+ # The window the flash path already gets through window_size. SDPA needs it too, and not only in+ # the branch above: training takes `elif self.training: pass`, so no 4D mask is synthesized, and+ # with xformers off and flash absent every sequence past config.sliding_window silently attended+ # its whole causal history.sliding_window = None if window_size == (-1, -1) else sw,prefix_seg_info = _pg_seg,)@@ -182,6 +179,7 @@ def MistralForCausalLM_fast_forward() -> Union[Tuple, CausalLMOutputWithPast]:# PrefixGrouper brings its own mask: a synthesized causal attention_mask would trip# resolve_prefix_seg_info on the no-xFormers path and force a fallback.+ # Not using xformers - need to create attention masksif (causal_mask is Noneand past_key_values is None@@ -191,7 +189,7 @@ def MistralForCausalLM_fast_forward(sliding_window = getattr(self.config, "sliding_window", None)if HAS_XFORMERS:- # Always create causal mask for xformers+ # Always create a causal mask for xformers.if sliding_window is None or sliding_window == "null" or sliding_window <= 0:causal_mask = xformers.attn_bias.LowerTriangularMask()elif q_len <= sliding_window:@@ -201,29 +199,27 @@ def MistralForCausalLM_fast_forward([q_len] * bsz).make_local_attention(window_size = sliding_window)- # If attention_mask exists, it will be handled in the attention forward+ # If attention_mask exists, it is handled in the attention forward.elif self.training:- # LlamaModel_fast_forward's DPO embed-masking block needs the 2D- # attention_mask; it nulls the mask before attention anyway, so- # leaving it 2D is safe and avoids a 4D conversion that crashes DPO.+ # LlamaModel_fast_forward's DPO embed-masking block needs the 2D attention_mask; it nulls the mask+ # before attention anyway, so leaving it 2D avoids a 4D conversion that crashes DPO.passelse:- # Not using xformers - need to create attention masksif (sliding_window is Noneor sliding_window == "null"or sliding_window <= 0or q_len <= sliding_window):- # Fully causal mask+ # Fully causal mask.causal_mask_values = torch.triu(torch.full((q_len, q_len), -torch.inf, device = input_ids.device),diagonal = 1,)else:- # Sliding window attention+ # Sliding window attention.q_indices = torch.arange(q_len, device = input_ids.device).view(-1, 1)k_indices = torch.arange(q_len, device = input_ids.device).view(1, -1)@@ -239,7 +235,7 @@ def MistralForCausalLM_fast_forward(attention_mask = causal_mask_values[None, None, :, :].expand(bsz, 1, q_len, q_len)else:if attention_mask.dim() == 2:- # Convert 0/1 padding mask to additive format: 1->0 (keep), 0->-inf (mask)+ # Convert the 0/1 padding mask to additive form: 1 -> 0 (keep), 0 -> -inf (mask).padding_mask = torch.where(attention_mask[:, None, None, :].bool(),0.0,@@ -261,7 +257,6 @@ def MistralForCausalLM_fast_forward()return_dict = return_dict if return_dict is not None else self.config.use_return_dict- # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)self.model._has_no_labels = labels is Noneif past_key_values is not None:@@ -298,15 +293,14 @@ def MistralForCausalLM_fast_forward(if labels is not None:labels = labels.to(lm_head_device)- # Merge legacy / new spellings before branching so the decode-time- # last-token slice fires on the normal path too. Skip int max() if- # either is a tensor (HF selective-decode form).+ # Merge legacy / new spellings before branching so the decode-time last-token slice fires on the+ # normal path too. Skip int max() if either is a tensor (HF selective-decode form).if isinstance(num_logits_to_keep, torch.Tensor) or isinstance(logits_to_keep, torch.Tensor):num_logits_to_keep = 0else:num_logits_to_keep = max(num_logits_to_keep, logits_to_keep)- # If we are in GRPO mode, return raw hidden states+ # In GRPO mode, return raw hidden states.if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":if num_logits_to_keep != 0:hidden_states = hidden_states[:, -num_logits_to_keep:, :]@@ -325,9 +319,9 @@ def MistralForCausalLM_fast_forward(logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(lm_head.dtype))else:RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"- # < 1024 Normal Unsloth uses less VRAM!+ # Below 1024 normal Unsloth uses less VRAM.if bsz * q_len <= 1024 and not RETURN_LOGITS:- # Use unsloth_fused_ce_loss which actually calculates the best chunk size to reduce VRAM usage+ # unsloth_fused_ce_loss calculates the best chunk size to reduce VRAM usage.RETURN_LOGITS = Falseif not RETURN_LOGITS and labels is not None:@@ -336,20 +330,13 @@ def MistralForCausalLM_fast_forward(n_items = kwargs.get("n_items", None)logit_softcapping = getattr(self.config, "final_logit_softcapping", 0)- # Packed-boundary guard, see llama.py. This branch returns, so- # mask_packed_sequence_boundaries() below is never reached.+ # Packed-boundary guard, see llama.py. This branch returns, so mask_packed_sequence_boundaries()+ # below is never reached.labels = mask_packed_boundary_labels(labels,kwargs.get("packed_seq_lengths"),)- # loss = fused_linear_cross_entropy(- # hidden_states = hidden_states,- # lm_weight = lm_head,- # labels = labels,- # num_items_in_batch = n_items,- # logit_softcapping = logit_softcapping,- # )loss = unsloth_fused_ce_loss(trainer = None,hidden_states = hidden_states,@@ -364,8 +351,7 @@ def MistralForCausalLM_fast_forward(logit_softcapping = logit_softcapping,)if not return_dict:- # Fused CE never materializes `logits`; use EMPTY_LOGITS- # like the return_dict branch below (fixes #2068).+ # Fused CE never materializes logits; use EMPTY_LOGITS like the return_dict branch below (#2068).output = (EMPTY_LOGITS,) + outputs[1:]return (loss,) + output if loss is not None else output@@ -384,11 +370,6 @@ def MistralForCausalLM_fast_forward(loss = Noneif labels is not None:shift_logits = logits- # if not hasattr(self, "extra_ignored_labels"):- # # Fixes https://github.com/unslothai/unsloth/issues/10- # self.extra_ignored_labels = torch.full((self.max_seq_length, 1), -100, device = "cuda:0")- # pass- # shift_labels = torch.hstack((labels[..., 1:], self.extra_ignored_labels[:labels.shape[0]]))shift_labels = torch.empty_like(labels)shift_labels[..., :-1] = labels[..., 1:]shift_labels[..., -1] = -100@@ -444,10 +425,9 @@ class FastMistralModel(FastLlamaModel):scaled_rope_module = LlamaLinearScalingRotaryEmbedding,attention_module = MistralAttention,)- # Just for Mistral Nemo models!+ # Just for Mistral Nemo models.if function is not None and init_name is not None:function = patch_mistral_nemo_attention(function)- # if True:#init_name is not None:exec(function, globals())MistralAttention.__init__ = eval(init_name)MistralAttention.forward = MistralAttention_fast_forward@@ -459,11 +439,8 @@ class FastMistralModel(FastLlamaModel):PeftModelForCausalLM.forward = PeftModel_fast_forwardfix_prepare_inputs_for_generation(MistralForCausalLM)- # Solves https://github.com/unslothai/unsloth/issues/168- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.- # https://github.com/huggingface/transformers/pull/27931- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,+ # huggingface/transformers#27931), so the old rotary embeddings are retained.import transformers.models.mistral.modeling_mistraltransformers.models.mistral.modeling_mistral.MistralRotaryEmbedding = LlamaRotaryEmbeddingdiff --git a/unsloth/models/qwen2.py b/unsloth/models/qwen2.pyindex 61af235b523..dbbd5b2a062 100644--- a/unsloth/models/qwen2.py+++ b/unsloth/models/qwen2.py@@ -57,11 +57,8 @@ class FastQwen2Model(FastLlamaModel):PeftModelForCausalLM.forward = PeftModel_fast_forwardfix_prepare_inputs_for_generation(Qwen2ForCausalLM)- # Solves https://github.com/unslothai/unsloth/issues/168- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.- # https://github.com/huggingface/transformers/pull/27931- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,+ # huggingface/transformers#27931), so the old rotary embeddings are retained.import transformers.models.qwen2.modeling_qwen2transformers.models.qwen2.modeling_qwen2.Qwen2RotaryEmbedding = LlamaRotaryEmbeddingdiff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex 986c1fa115b..d41410c7a2a 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -77,7 +77,6 @@ def Qwen3Attention_fast_forward(*args,**kwargs,) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:- # Clear inferenceif hasattr(self, "paged_attention"):del self.paged_attention_Kdel self.paged_attention_V@@ -96,17 +95,13 @@ def Qwen3Attention_fast_forward(assert n_kv_heads * n_groups == n_headsQ, K, V = self.apply_qkv(self, hidden_states)- Q = Q.view(- bsz, q_len, n_heads, head_dim- ) # .transpose(1, 2) # we will transpose after normalisation- K = K.view(- bsz, q_len, n_kv_heads, head_dim- ) # .transpose(1, 2) # we will transpose after normalisation+ Q = Q.view(bsz, q_len, n_heads, head_dim)+ K = K.view(bsz, q_len, n_kv_heads, head_dim)V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)seq_info = get_packed_info_from_kwargs(kwargs, hidden_states.device)- # Qwen3 adds QKNorm (the only difference from Qwen2). A compiled norm- # mismatches Transformers' numbers, so use fast_rms_layernorm. TODO: investigate.+ # Qwen3 adds QKNorm, the only difference from Qwen2. A compiled norm mismatches Transformers'+ # numbers, so use fast_rms_layernorm.Q = fast_rms_layernorm(self.q_norm, Q)K = fast_rms_layernorm(self.k_norm, K)@@ -117,7 +112,7 @@ def Qwen3Attention_fast_forward(if past_key_value is not None:kv_seq_len += past_key_value[0].shape[-2]- # Extend RoPE dynamically to fit in VRAM+ # Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.if position_embeddings and kv_seq_len <= position_embeddings[0].shape[0]:cos, sin = position_embeddingselse:@@ -126,7 +121,6 @@ def Qwen3Attention_fast_forward(cos, sin = rotary_emb.get_cached(kv_seq_len, Q.device.index)rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")- # Useful for LongRoPEQ, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)if past_key_value is not None:@@ -134,7 +128,6 @@ def Qwen3Attention_fast_forward(V = torch.cat([past_key_value[1], V], dim = 2)past_key_value = (K, V) if use_cache else None- # Attention moduleuse_varlen = seq_info is not None and past_key_value is Nonebackend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)attention_config = AttentionConfig(@@ -148,8 +141,8 @@ def Qwen3Attention_fast_forward("softmax_scale": getattr(self, "softmax_scale", None),},)- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse- # (KV cache / padding mask) raises. None => byte-identical default.+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /+ # padding mask) raises. None means the byte-identical default._pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)context = AttentionContext(bsz = bsz,@@ -200,15 +193,12 @@ def Qwen3Attention_fast_forward_inference(n_groups = self.num_key_value_groupsn_kv_heads = self.config.num_key_value_headshead_dim = self.head_dim- # assert(n_kv_heads * n_groups == n_heads)hidden_size = self.config.hidden_sizeattention_size = n_heads * head_dimseq_len = K1.shape[-2]kv_seq_len = seq_len + 1- # Prefill phase- # if not hasattr(self, "paged_attention"):device = hidden_states.deviceif do_prefill:self.paged_attention = torch.empty(@@ -224,7 +214,7 @@ def Qwen3Attention_fast_forward_inference(self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)- # Mistral Nemo 12b has weird dimensions+ # Mistral Nemo 12b has weird dimensions.if attention_size != hidden_size:self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)else:@@ -252,12 +242,8 @@ def Qwen3Attention_fast_forward_inference(Qn = fast_linear_forward(self.q_proj, Xn, out = self.temp_QA[0])Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0])Vn = fast_linear_forward(self.v_proj, Xn, out = self.temp_KV[1])- Qn = Qn.view(- bsz, 1, n_heads, head_dim- ) # .transpose(1, 2) # we will transpose after normalisation- Kn = Kn.view(- bsz, 1, n_kv_heads, head_dim- ) # .transpose(1, 2) # we will transpose after normalisation+ Qn = Qn.view(bsz, 1, n_heads, head_dim)+ Kn = Kn.view(bsz, 1, n_kv_heads, head_dim)Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)Qn = fast_rms_layernorm_inference(self.q_norm, Qn)@@ -266,14 +252,10 @@ def Qwen3Attention_fast_forward_inference(Qn = Qn.transpose(1, 2)Kn = Kn.transpose(1, 2)- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)-- # Need to do it prior 2 steps before hitting full on short KV cache- # or else error+ # Must be done 2 steps before hitting full on a short KV cache, or it errors.self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:position_ids = position_ids[:, -1:]cos = cos[position_ids].unsqueeze(1)@@ -283,22 +265,17 @@ def Qwen3Attention_fast_forward_inference(RH_Q = self.RH_QRH_Q[:, :, :, :h] = Qn[:, :, :, h:]RH_Q[:, :, :, h:] = Qn[:, :, :, :h]- RH_Q[:, :, :, :h].neg_() # torch.neg(RH_Q[:,:,:,:h], out = RH_Q[:,:,:,:h])+ RH_Q[:, :, :, :h].neg_()Qn *= cosQn.addcmul_(RH_Q, sin)- RH_K = RH_Q[- :, :n_kv_heads, :, :- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")+ RH_K = RH_Q[:, :n_kv_heads, :, :]RH_K[:, :, :, :h] = Kn[:, :, :, h:]RH_K[:, :, :, h:] = Kn[:, :, :, :h]- RH_K[:, :, :, :h].neg_() # torch.neg(RH_K[:,:,:,:h], out = RH_K[:,:,:,:h])+ RH_K[:, :, :, :h].neg_()Kn *= cosKn.addcmul_(RH_K, sin)- # New KV cache- # Kn = torch.cat([K1, Kn], dim = 2)- # Vn = torch.cat([V1, Vn], dim = 2)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)@@ -308,14 +285,14 @@ def Qwen3Attention_fast_forward_inference(sliding_window = getattr(self.config, "sliding_window", None)if sliding_window is not None and kv_seq_len > sliding_window:start = kv_seq_len - sliding_window- Knn = Kn[:, :, start:, :] # .contiguous()- Vnn = Vn[:, :, start:, :] # .contiguous()+ Knn = Kn[:, :, start:, :]+ Vnn = Vn[:, :, start:, :]if attention_mask is not None:attention_mask = attention_mask[..., start:]else:Knn, Vnn = Kn, Vn- # when qlen==vlen and attn_mask is None, we should use causal attention+ # When qlen == vlen and attn_mask is None, causal attention is the right choice.Q_len = Qn.shape[-2]K_len = Knn.shape[-2]if attention_mask is not None and attention_mask.dim() == 2:@@ -340,7 +317,7 @@ def Qwen3Attention_fast_forward_inference(# Avoid SDPA GQA drift for batched masked decode.use_sdpa_gqa = False- # Grouped query attention+ # Grouped query attention._, _, cached_len, _ = Knn.shapeif bsz == 1 or ((not use_sdpa_gqa) and n_groups != 1):Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)@@ -348,14 +325,12 @@ def Qwen3Attention_fast_forward_inference(Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)- # Attentionif bsz == 1:- Qn *= (- self.scalar- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows+ Qn *= self.scalar+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805+ # (comment 2153349963).A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)A = torch_matmul(A, Vnn, out = Qn)else:if use_sdpa_gqa:@@ -400,8 +375,8 @@ class FastQwen3Model(FastLlamaModel):PeftModelForCausalLM.forward = PeftModel_fast_forwardfix_prepare_inputs_for_generation(Qwen3ForCausalLM)- # Retain old rotary embeddings; static KV cache (transformers 4.38.0)- # slowed training. See unslothai/unsloth#168 and transformers#27931.+ # Retain the old rotary embeddings: the static KV cache (transformers 4.38.0) slowed training.+ # See unslothai/unsloth#168 and huggingface/transformers#27931.import transformers.models.qwen3.modeling_qwen3transformers.models.qwen3.modeling_qwen3.Qwen3RotaryEmbedding = LlamaRotaryEmbeddingdiff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.pyindex 872cb60a2e7..279b74b99dc 100644--- a/unsloth/models/qwen3_moe.py+++ b/unsloth/models/qwen3_moe.py@@ -34,16 +34,7 @@ from transformers.models.qwen3_moe.modeling_qwen3_moe import ()# For Pytorch 2.1.1-# TODO: Transformers moved to `attention_interface`. So we might not need these anymore-# try:-# from transformers.models.qwen3_moe.modeling_qwen3_moe import (-# Qwen3SdpaAttention,-# Qwen3FlashAttention2,-# )-# except:-# Qwen3SdpaAttention = Qwen3Attention-# Qwen3FlashAttention2 = Qwen3Attention-# pass+# Transformers moved to attention_interface, so the Sdpa/FlashAttention2 aliases may no longer be needed.from unsloth_zoo.utils import Version, _get_dtype@@ -56,7 +47,7 @@ def Qwen3MoeSparseMoeBlock_fast_forward(temp_gate = None,temp_up = None,):- # adapted from https://github.com/huggingface/transformers/pull/36878/files#diff-0855b77fc27ad9449158a1c74953f909b011c00de7125f7c8e68d0ff209c092aR356-R370+ # Adapted from huggingface/transformers#36878 (modeling change R356-R370).bsz, seq_len, hd = X.shapeX = X.view(-1, hd)@@ -68,11 +59,11 @@ def Qwen3MoeSparseMoeBlock_fast_forward(routing_weights = torch_nn_functional_softmax(router_logits, dim = -1, dtype = torch.float32)routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim = -1)routing_weights /= routing_weights.sum(dim = -1, keepdim = True)- # cast back to the input dtype+ # Cast back to the input dtype.routing_weights = routing_weights.to(X.dtype)final_X = torch.zeros((bsz * seq_len, hd), dtype = torch.float32, device = X.device)- # One-hot the selected experts into a mask to index which expert is used+ # One-hot the selected experts into a mask indexing which expert is used.expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes = self.num_experts).permute(2, 1, 0)@@ -81,13 +72,13 @@ def Qwen3MoeSparseMoeBlock_fast_forward(expert_layer = self.experts[expert_idx]idx, top_x = torch.where(expert_mask[expert_idx])- # Index hidden states for this expert and scale by routing_weights+ # Index hidden states for this expert and scale by routing_weights.current_state = X[None, top_x].reshape(-1, hd)current_X = (expert_layer(current_state) * routing_weights[top_x, idx, None]) # Qwen3MoeMLP.forward = fast_swiglu_inference speeds this up- # index_add_ needs a tensor index, so use top_x+ # index_add_ needs a tensor index, so use top_x.final_X.index_add_(0, top_x, current_X.to(X.dtype))final_X = final_X.reshape(bsz, seq_len, hd)return final_X, router_logits@@ -110,7 +101,7 @@ def Qwen3MoeDecoderLayer_fast_forward():residual = hidden_states- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:+ if use_cache and hasattr(self, "_flag_for_generation"):residual = hidden_stateshidden_states = fast_rms_layernorm_inference(self.input_layernorm, hidden_states)hidden_states, self_attn_weights, present_key_value = self.self_attn(@@ -127,7 +118,7 @@ def Qwen3MoeDecoderLayer_fast_forward()hidden_states += residual- # MoE Router MLP+ # MoE Router MLP.residual = hidden_stateshidden_states = fast_rms_layernorm_inference(self.post_attention_layernorm, hidden_states)hidden_states, router_logits = Qwen3MoeSparseMoeBlock_fast_forward(self.mlp, hidden_states)@@ -148,7 +139,7 @@ def Qwen3MoeDecoderLayer_fast_forward()hidden_states = residual + hidden_states- # MoE Router MLP+ # MoE Router MLP.residual = hidden_stateshidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)hidden_states, router_logits = self.mlp(hidden_states)@@ -177,8 +168,6 @@ class FastQwen3MoeModel(FastQwen3Model):exec(function, globals())Qwen3MoeAttention.__init__ = eval(init_name)Qwen3MoeAttention.forward = Qwen3Attention_fast_forward- # Qwen3SdpaAttention .forward = Qwen3Attention_fast_forward- # Qwen3FlashAttention2 .forward = Qwen3Attention_fast_forwardQwen3MoeSparseMoeBlock.forward = Qwen3MoeSparseMoeBlock_fast_forwardQwen3MoeMLP.forward = fast_swiglu_inference # This is analogous to Dense models' MLPQwen3MoeDecoderLayer.forward = Qwen3MoeDecoderLayer_fast_forward@@ -187,11 +176,8 @@ class FastQwen3MoeModel(FastQwen3Model):PeftModelForCausalLM.forward = PeftModel_fast_forwardfix_prepare_inputs_for_generation(Qwen3MoeForCausalLM)- # Solves https://github.com/unslothai/unsloth/issues/168- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.- # https://github.com/huggingface/transformers/pull/27931- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py\+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,+ # huggingface/transformers#27931), so the old rotary embeddings are retained.import transformers.models.qwen3_moe.modeling_qwen3_moetransformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = (diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 65cb2621cd4..64fba41b818 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -43,13 +43,13 @@ from .rl_replacements import (torch_compile_options = {"epilogue_fusion": True,- "max_autotune": False, # Disable Triton mm kernels+ "max_autotune": False,"shape_padding": True,"trace.enabled": False,"triton.cudagraphs": False,}-# vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it)+# vLLM compatibility shim: TRL expects GuidedDecodingParams even when vLLM does not provide it.try:import vllm.sampling_params as _unsloth_vllm_spif not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"):@@ -74,13 +74,11 @@ except Exception:except Exception:trl_version = Version("0.0.0")-# Get PyTorch version for feature detectiontry:torch_version = Version(torch.__version__.split("+")[0].split("a")[0].split("b")[0])except Exception:torch_version = Version("0.0.0")-# Get transformers version for feature detectiontry:from transformers import __version__ as _transformers_version_rawtransformers_version = Version(_transformers_version_raw)@@ -155,7 +153,7 @@ def PatchRL(FastLanguageModel):try:from trl.models import unwrap_model_for_generationexcept ImportError:- # Local fallback -- TRL removed or moved this symbol+ # Local fallback: TRL removed or moved this symbol.from contextlib import contextmanager as _cm@_cm@@ -188,10 +186,8 @@ def PatchRL(FastLanguageModel):@contextmanagerdef unsloth_unwrap_model_for_generation(model, *args, **kwargs):- # why: snapshot before TRL's unwrap context manager, which calls- # gradient_checkpointing_disable() before yielding; preserve the actual- # mode value (e.g. "unsloth") rather than collapsing it to a bool, so- # the finally restore matches the caller's configured GC mode.+ # Snapshot before TRL's unwrap CM, which calls gradient_checkpointing_disable() before+ # yielding. Keep the mode value (e.g. "unsloth"), not a bool, so the finally restore matches.use_gradient_checkpointing = next((v@@ -201,11 +197,9 @@ def PatchRL(FastLanguageModel):False,)with unwrap_model_for_generation(model, *args, **kwargs) as unwrapped_model:- # Put the model in inference mode.FastLanguageModel.for_inference(model)- # We must use .clone for Unsloth since we force inference_mode- # Rather we should have used no_grad+ # .clone is required because inference_mode is forced here; no_grad would have been the better choice.original_generate = unwrapped_model.generatedef generate_with_clone(*args, **kwargs):@@ -219,7 +213,6 @@ def PatchRL(FastLanguageModel):try:yield unwrapped_modelfinally:- # Restore generate and returnunwrapped_model.generate = original_generateFastLanguageModel.for_training(model,@@ -255,9 +248,8 @@ def PatchRL(FastLanguageModel):if len(self.label_names) == 0else all(inputs.get(k) is not None for k in self.label_names))- # For CLIP-like models capable of returning loss values.- # If `return_loss` is not specified or being `None` in `inputs`, we check if the default value of `return_loss`- # is `True` in `model.forward`.+ # For CLIP-like models capable of returning loss values: if return_loss is unset in inputs, check+ # whether model.forward defaults it to True.return_loss = inputs.get("return_loss", None)if return_loss is None:return_loss = self.can_return_loss@@ -270,7 +262,7 @@ def PatchRL(FastLanguageModel):else:ignore_keys = []- # labels may be popped when computing the loss (label smoothing for instance) so we grab them first.+ # labels may be popped when computing the loss (label smoothing), so grab them first.if has_labels or loss_without_labels:labels = nested_detach(tuple(inputs.get(name) for name in self.label_names))if len(labels) == 1:@@ -278,8 +270,8 @@ def PatchRL(FastLanguageModel):else:labels = None- # Force logits during eval, but restore the user's prior setting after- # so an explicit UNSLOTH_RETURN_LOGITS="1" is not silently turned off.+ # Force logits during eval, but restore the user's prior setting after so an explicit+ # UNSLOTH_RETURN_LOGITS="1" is not silently turned off._old_return_logits = os.environ.get("UNSLOTH_RETURN_LOGITS", "0")os.environ["UNSLOTH_RETURN_LOGITS"] = "1"with torch.no_grad():@@ -317,7 +309,6 @@ def PatchRL(FastLanguageModel):logits = tuple(v for k, v in outputs.items() if k not in ignore_keys)else:logits = outputs- # TODO: this needs to be fixed and made cleaner later.if self.args.past_index >= 0:self._past = outputs[self.args.past_index - 1]os.environ["UNSLOTH_RETURN_LOGITS"] = _old_return_logits@@ -570,12 +561,10 @@ pass'''-# Marks a config class generated by Unsloth. The generated class is renamed to-# the TRL name it stands in for (see _patch_config_pickle_identity), so the-# "already patched" checks cannot go by __name__ alone any more.+# Marks an Unsloth-generated config class. It is renamed to the TRL name it stands in for,+# so the "already patched" checks cannot go by __name__ alone._UNSLOTH_PATCHED_CONFIG_FLAG = "_unsloth_patched_rl_config"-# Set on the PRISTINE config class, pointing at the Unsloth subclass that has-# taken over its module attribute.+# Set on the PRISTINE config class, pointing at the Unsloth subclass that has taken over its module attribute._UNSLOTH_CONFIG_PICKLE_TARGET = "_unsloth_config_pickle_target"@@ -638,14 +627,12 @@ def _patch_config_pickle_identity(pristine_config, patched_config):home_module = importlib.import_module(home_module_name)except Exception:return- # The rebinding above walks the trl.trainer.<x>_trainer -> <x>_config naming- # convention. The pristine class's own __module__ is what pickle consults,- # which is the same module for every stock TRL layout and the correct one for- # the trl.experimental wrappers the convention misses.+ # The rebinding above follows the trl.trainer.<x>_trainer -> <x>_config convention, but+ # pickle consults the pristine class's own __module__, which is also right for+ # trl.experimental wrappers the convention misses.current = getattr(home_module, qualname, None)if current is not pristine_config and current is not patched_config:- # Something else owns the name. Renaming the patched class now would- # only move the failure, so leave both classes as they are.+ # Something else owns the name; renaming the patched class would only move the failure.returntry:setattr(home_module, qualname, patched_config)@@ -763,13 +750,9 @@ def _column_names(dataset):source = datasettry:iterator = iter(dataset)- # `iterator is dataset` catches a bare generator and misses an- # `IterableDataset` whose `__iter__` hands back one stored generator:- # that is not the dataset, but it is still the only pass there is. Two- # `iter()` calls returning the same object covers both, and a- # `datasets.IterableDataset` restarts, so it answers False and is- # rewound rather than chained. Both calls happen before anything is- # read.+ # `iterator is dataset` misses an IterableDataset whose __iter__ returns one stored+ # generator. Two iter() calls giving the same object catches both; a datasets.IterableDataset+ # restarts, so it answers False and is rewound.single_pass = iterator is dataset or iterator is iter(dataset)row = next(iterator, None)if single_pass and row is not None:@@ -779,11 +762,8 @@ def _column_names(dataset):row = Noneif isinstance(row, dict):names.update(row.keys())- # The row too: it was read either way, and on a one-shot stream it is the- # ONLY row anything may look at. Without it `_sliceable_per_token` had no- # widths to compare and fell back to `input_ids` alone, leaving `labels`- # and `attention_mask` at their overlength size -- rows whose supervision- # no longer lines up with the tokens they describe.+ # Keep the probed row: on a one-shot stream it is the only row anything may see. Without it+ # _sliceable_per_token had no widths and cut input_ids alone, leaving labels overlength.return tuple(names), source, (row if isinstance(row, dict) else None)@@ -804,13 +784,9 @@ class _CappedBase:self._per_token = tuple(per_token)def _slice(self, row):- # Per value, per row, exactly as the `map` path does it.- # `_sliceable_per_token` judges alignment from ONE probed row, and an- # optional column that is a list there can be None -- or a different- # width -- further in. Slicing that raised inside the dataloader, which- # is a failure the caller would not have had without the cap. Only- # `input_ids` is cut unconditionally: it is the column the cap exists- # for, and the width every other one is measured against.+ # Per value, per row, like the map path: _sliceable_per_token judges from ONE row, and an+ # optional column that is a list there can be None later, which raised in the dataloader.+ # Only input_ids is cut unconditionally.if not isinstance(row, dict):return rowtry:@@ -850,11 +826,9 @@ class _CappedBase:yield cutdef __getattr__(self, attribute):- # Anything else the trainer asks for (column_names, features, ...) is- # the wrapped split's own answer. Never a dunder, and never our own- # state: a DataLoader worker pickles the split, and `__setstate__`- # looked up before `__init__` has run would recurse on `_inner`- # forever.+ # Everything else (column_names, features, ...) is the wrapped split's answer. Never a dunder+ # nor our own state: a DataLoader worker pickles the split, and __setstate__ before __init__+ # would recurse on _inner forever.inner = self.__dict__.get("_inner")if inner is None or attribute.startswith("__"):raise AttributeError(attribute)@@ -870,11 +844,8 @@ class _CappedRows(_CappedBase):def __init__(self, inner, cut, supervision, per_token):super().__init__(inner, cut, supervision, per_token)- # No supervision columns means `_keep` is True for every row, so the- # index would be `range(len(inner))` -- and building it read and- # transformed every item, which for a `with_transform` split is a whole- # extra tokenization pass before the dataloader can even start, plus an- # O(n) list. Stay lazy and let `__getitem__` map straight through.+ # With no supervision columns _keep is True for every row, so building the index would+ # transform every item -- a whole extra tokenization pass for a with_transform split.self._index = (Noneif not self._supervision@@ -927,10 +898,8 @@ def _is_stream(dataset):_SCAN_ROWS = 1024-# A producer that truncates every row itself can say so and be believed without a-# scan: scanning a lazily-tokenizing split (`with_transform`) would tokenize every-# row in `__init__`, the exact eager pass it exists to avoid. Only a cap at or-# below the enforced one counts.+# Believe a producer's own truncation claim: scanning a with_transform split tokenizes every+# row in __init__, the eager pass it avoids. Only a cap at or below the enforced one counts._TRUNCATION_ATTESTATION_ATTR = "_unsloth_truncated_to"@@ -1071,8 +1040,7 @@ def _first_row_without_consuming(dataset):return dataset[0]except Exception:return None- # A stream whose `iter` hands back the same exhausting generator cannot- # spare a row, and nothing here chains it back.+ # A stream whose iter hands back the same exhausting generator cannot spare a row, and nothing here chains it back.probe = iter(dataset)if probe is dataset or probe is iter(dataset):return None@@ -1118,35 +1086,25 @@ def _sliceable_per_token(A row that cannot be read without costing it leaves `input_ids` alone: thecolumn the cap exists for, and the one every other is measured against."""- # `input_ids` first, and the rest in a fixed order: `names` arrives as a set,- # and the `map` path reads the width off `input_ids` as it walks this list,- # so a run that happened to order `labels` ahead of it sliced the labels- # without ever comparing them to anything.+ # input_ids first, then a fixed order: `names` is a set, and the map path reads the width off+ # input_ids as it walks this list, so labels first sliced them against nothing.known = [c for c in _PER_TOKEN if c in names]- # A user-defined per-token field -- `loss_mask`, `token_weights`, a custom- # model input -- is not in the allow-list, so it kept its full length while- # `input_ids` was cut. A custom collator (the case where padding-free is off- # and this wrapper still runs) then gets mismatched lengths and either fails- # or supervises the wrong tokens. Judge those the way the row check below- # judges everything else, by alignment, but only accept a flat vector of- # scalars: that is what a per-token field is, and it keeps `messages` or a- # column of strings that happens to be as long as the row out of the slice.+ # A custom per-token field (loss_mask, token_weights) is not in the allow-list, so it stayed+ # full length while input_ids was cut and a custom collator got mismatched lengths. Judge by+ # alignment, but only a flat vector of scalars, keeping `messages` out of the slice.custom = sorted(c for c in names if c not in _PER_TOKEN)per_token = known + customif len(known) < 2 and not custom:return known- # `probed` is the row `_column_names` already read. Preferring it is what- # lets a one-shot stream align every per-token column: reading another row- # would cost the caller that example, so without it this fell back to- # `input_ids` alone and left the masks and labels overlength.+ # `probed` is the row _column_names already read. Preferring it is what lets a one-shot stream align+ # every per-token column: reading another row would cost the caller that example.row = probed if isinstance(probed, dict) else _first_row_without_consuming(dataset)if not isinstance(row, dict):return ["input_ids"] if "input_ids" in names else []try:width = len(row.get("input_ids"))except Exception:- # Nothing to measure against, so a custom column has no evidence at all- # behind it. Fall back to the named ones only.+ # Nothing to measure against, so a custom column has no evidence behind it; fall back to the named ones only.return knownkept = []for name in per_token:@@ -1154,11 +1112,8 @@ def _sliceable_per_token(if name in custom and not _is_token_vector(value, width):continuetry:- # As long as `input_ids`, which is what makes the FIRST axis the- # token axis: a `[seq_len, channels]` field slices correctly there,- # and a channel-major one (`position_ids` under mrope is- # `[3, seq_len]`) fails this check and is left alone, which is the- # shape the nested test used to be aimed at.+ # As long as input_ids, which makes the FIRST axis the token axis: [seq_len, channels] slices+ # right, and a channel-major position_ids ([3, seq_len] under mrope) fails and is left alone.if len(value) != width:continueexcept Exception:@@ -1249,9 +1204,8 @@ def _pin_pristine_sft_loss_type(config_cls):else:return Falsefield.default = "nll"- # The class attribute is the other copy of the default: dataclasses seeds it- # at class creation and a later subclass reads the field, so leave the two- # agreeing rather than half-patched.+ # The class attribute is the other copy of the default: dataclasses seeds it at class creation and a+ # later subclass reads the field, so leave the two agreeing rather than half-patched.setattr(config_cls, "loss_type", "nll")return True@@ -1294,14 +1248,9 @@ def _wrap_sft_evaluate_cap(trainer_cls):does the same and the two must agree."""columns = ["labels"] if "labels" in names else []- # The TRAINER's resolved value first. It is the one the collator uses,- # and it is present whenever TRL resolved it -- including the cases the- # generated block never runs in: a TRL whose guard did not match, or- # padding-free off from the start. Falling straight through to this- # split's own schema then read False off a late pre-tokenized split- # carrying only `input_ids` and `completion_mask`, kept the rows whose- # completion was cut away entirely, and the collator turned them into- # all -100, i.e. a NaN eval loss.+ # The TRAINER's resolved value first: the collator uses it, and it is set whenever TRL+ # resolved it. The split's own schema read False off a pre-tokenized eval split with only+ # input_ids + completion_mask, so cut-away rows survived as all -100, i.e. a NaN eval loss.only = getattr(args, "_unsloth_resolved_completion_only", None)if only is None:only = getattr(args, "_unsloth_completion_only_loss", None)@@ -1322,68 +1271,34 @@ def _wrap_sft_evaluate_cap(trainer_cls):drop_unsupervised = True,packs_late = False,):- # `evaluate()` caps the split, stores it on the trainer, and Transformers- # then calls `get_eval_dataloader`, which this module also wraps -- so the- # paired wrappers reach here twice for one call. Re-capping is a no-op at- # best, and over a one-shot stream it is destructive: `_column_names` and- # `_sliceable_per_token` each read a row to probe it, and `_CappedStream`- # hands out a fresh generator over the SAME exhausted source rather than- # rewinding, so the second pass eats the first rows instead of replaying- # them. Our own wrapper, capped the same way, is already the answer.+ # evaluate() caps the split and Transformers then calls get_eval_dataloader, which is also+ # wrapped, so both reach here in one call. Re-capping is destructive over a one-shot stream:+ # each probe reads a row and _CappedStream re-opens the SAME exhausted source.if _cap_still_holds(dataset, cap, drop_unsupervised):return datasetnames, dataset, probed = _column_names(dataset)if "input_ids" not in names:- # No tokens here yet, so there is nothing to cut. TRL does not- # tokenize a late split either -- `_prepare_dataset` runs only from- # `__init__` -- but that is its own gap and not one a truncation can- # close.+ # No tokens here yet, so there is nothing to cut. TRL does not tokenize a late split either+ # (_prepare_dataset runs only from __init__), but that is its own gap.return _mark_capped(dataset, cap, drop_unsupervised)- # `eval_packing` is consulted only where the packer can actually reach the- # split, which is why `packs_late` is passed in per entry point rather than- # read here. Up to TRL 1.6 nothing packs a late split: `evaluate()` and- # `predict()` are the base Trainer's, they call `get_eval_dataloader` /- # `get_test_dataloader` straight through, and `_prepare_dataset` never runs- # again, so skipping the cap there handed the collator the raw overlength- # rows with `max_length` already cleared. From TRL 1.7.0 the opposite is- # true for `evaluate` alone: it prepares the split itself under- # `eval_packing`, and every strategy owns the overflow -- `wrapped`- # concatenates the token stream before chunking, `bfd_split` splits an- # overlength example into more chunks -- so cutting rows at the cap first- # throws that away. `predict` and the two dataloader builders stay the base- # Trainer's on every TRL, so they always cap.+ # eval_packing is consulted only where the packer reaches the split, hence packs_late per+ # entry point. Up to TRL 1.6 nothing packs a late split; from 1.7.0 `evaluate` prepares it+ # itself and the strategy owns the overflow, so capping rows first throws that away.if packs_late and _eval_packing_on(args):- # Left FOR the packer, so it is not capped -- and must not be marked- # as if it were. `packs_late` is read off the class, but TRL only- # actually prepares a split that was PASSED to `evaluate`:- # if not self._skip_prepare_dataset and eval_dataset is not None- # and not isinstance(eval_dataset, str):- # (trl 1.9.2 sft_trainer.py:1675, same line since 1.7.0). A split- # stored on the trainer, one named by a string key, and any run with- # `skip_prepare_dataset` therefore reach `get_eval_dataloader` as- # this very object with no packer having touched it -- and that- # builder, which does cap, then read the mark through- # `_cap_still_holds` and handed the collator the overlength rows.- # Where TRL really does pack, it hands back a NEW object, so nothing- # is lost by leaving this one unmarked.+ # Left FOR the packer, so it is not capped and must not be marked as if it were. TRL only+ # prepares a split PASSED to evaluate, so a stored split or string key arrives here untouched;+ # where it really packs, it hands back a NEW object.return dataset- # A packed split carries document lengths, not tokens. Slicing `input_ids`- # under a `seq_lengths` that still describes the longer row makes- # padding-free build position ids for tokens the row no longer has, which- # is worse than not cutting: the construction-time cap refuses this shape- # too.+ # A packed split carries document lengths, not tokens: slicing input_ids under a seq_lengths that+ # still describes the longer row builds position ids for tokens the row no longer has.if "seq_lengths" in names:return _mark_capped(dataset, cap, drop_unsupervised)try:- # TRL slices [-max_length:] for `keep_end`, and so does the- # construction-time cap. Always keeping the prefix evaluates the wrong- # half of every long row for callers whose completion sits at the tail.+ # TRL slices [-max_length:] for keep_end, and so does the construction-time cap; always keeping the+ # prefix evaluates the wrong half of every long row.mode = getattr(args, "truncation_mode", "keep_start")- # keep_start and keep_end are the only two slices there are, and a- # third value silently became keep_start here, cutting every late- # row from the side the caller asked us not to. The construction- # path already refuses those; this one has to as well, and refusing- # means handing the split back untouched so the caller still has it.+ # keep_start and keep_end are the only two slices there are, and a third value silently became+ # keep_start here. Refusing means handing the split back untouched so the caller still has it.if mode not in ("keep_start", "keep_end"):print(f"Unsloth: `truncation_mode = {mode}` is not one of "@@ -1392,39 +1307,25 @@ def _wrap_sft_evaluate_cap(trainer_cls):return datasetcut = slice(-cap, None) if mode == "keep_end" else slice(None, cap)per_token = _sliceable_per_token(dataset, names, cap, probed)- # Never on the predict path. Dropping rows is right for a loss, which- # is meaningless over a row with no supervised token, and wrong for- # `predict`, whose contract is one prediction per row IN ORDER: a- # caller zipping the output back onto its own dataframe would silently- # get a shorter, shifted column.+ # Never on the predict path: dropping rows is right for a loss and wrong for predict, whose contract+ # is one prediction per row IN ORDER.supervision = _supervision_columns(args, names) if drop_unsupervised else []- # A stream has no length and cannot be rewound, so there is no prefix- # scan to skip the work with, and indexing it does not even fail- # loudly: on datasets 4.x `dataset[0]` reads 0 as a COLUMN name and- # returns an IterableColumn, whose len() then raised TypeError into- # the catch below and handed the eval call its uncapped stream back.- # map() is lazy and applies to every row it will ever yield.+ # A stream has no length, cannot be rewound, and on datasets 4.x dataset[0] reads 0 as a+ # COLUMN name rather than failing. Use map(), which is lazy over every row it will yield.overlength = Trueif not _is_stream(dataset):try:overlength = max(len(r) for r in dataset["input_ids"]) > capexcept Exception:- # The scan only exists to skip a pointless map. A split with no- # column access (a custom map-style dataset) cannot answer it, and- # that is not a reason to hand it back uncapped.+ # The scan only exists to skip a pointless map; a split with no column access cannot answer it, and+ # that is no reason to hand it back uncapped.pass- # A split already under the cap still goes through the supervision- # filter below. Being short is not the same as being supervised: a row- # whose labels are all -100, or whose active mask is all zeros, is a- # NaN loss whether or not anything had to be cut off it, and the- # construction-time cap filters those rows unconditionally too.- # Returning early on the length alone was the one way the two paths- # disagreed.+ # A split already under the cap still goes through the supervision filter below: being short is not+ # the same as being supervised, and a row whose labels are all -100 is a NaN loss either way.if not overlength and not supervision:return _mark_capped(dataset, cap, drop_unsupervised)- # A split we cannot rewrite is capped on read instead. `map` belongs to- # `datasets`, and a `with_transform` split has it but recreates its rows- # on every read, so mapping writes a table nobody reads.+ # A split we cannot rewrite is capped on read instead: map belongs to datasets, and a with_transform+ # split has it but recreates its rows on every read, so mapping writes a table nobody reads.transform = str((getattr(dataset, "format", None) or {}).get("type", "")).lower()if (not hasattr(dataset, "map")@@ -1444,12 +1345,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):_cut = cut,_cols = tuple(per_token),):- # Per value, not per column. `_sliceable_per_token` judges by ONE- # row, so an optional column that is a list there and None (or a- # different width) three rows later raised inside `map` -- and- # the broad catch below then returned the UNCAPPED split, losing- # the `input_ids` truncation too. `input_ids` is always cut; an- # auxiliary value that cannot take the same slice is left alone.+ # Per value, not per column: _sliceable_per_token judges by ONE row, so an optional column+ # that is None three rows later raised inside map and the catch returned the UNCAPPED split.out = {}width = Nonefor name in _cols:@@ -1466,17 +1363,9 @@ def _wrap_sft_evaluate_cap(trainer_cls):return outnew = dataset if not overlength else dataset.map(_slice_row)- # Truncating can leave a row with every label at -100, or a mask that is- # now all zeros, which the collator turns into all -100. A batch of- # those has no supervised token and reports a NaN loss. TRL filters them- # right after its own truncation; here `args.max_length` is already None- # so TRL does not, and the construction-time cap filters them for the- # splits it saw. Masks are applied one after another onto the same- # labels, so what survives is their intersection.- # One intersection over labels AND every active mask, not a filter each:- # the collator applies the masks ONTO the labels, so a row whose only- # supervised label sits where the mask is zero passes both filters- # separately and still goes out all -100.+ # A truncated row can end all -100, or with an all-zero mask the collator makes all -100, and+ # such a batch reports a NaN loss. Intersect labels AND every active mask, not one filter+ # each: masks are applied ONTO the labels, so separate filters still pass an all -100 row.if supervision:kept = new.filter(lambda e, c = tuple(supervision): any(@@ -1484,9 +1373,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):for v in zip(*[e[n] for n in c])))- # Hand back the caller's own split when the filter dropped- # nothing: a copy of an unchanged dataset is a new object for the- # trainer to cache and reload for no reason at all.+ # Hand back the caller's own split when the filter dropped nothing: a copy of an unchanged dataset+ # is a new object for the trainer to cache and reload for no reason.try:new = new if len(kept) == len(new) else keptexcept Exception:@@ -1529,8 +1417,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):column each time. Keep the answer, keyed on the split object and holding areference to it, so a later split cannot inherit its `id()`."""- # Carried onto args because `_cap` only ever sees those. `is not None`- # rather than truthiness: False is TRL's answer just as much as True.+ # Carried onto args because _cap only ever sees those. `is not None` rather than truthiness: False+ # is TRL's answer just as much as True.resolved = getattr(trainer, "completion_only_loss", None)if resolved is not None:try:@@ -1547,17 +1435,14 @@ def _wrap_sft_evaluate_cap(trainer_cls):trainer._unsloth_eval_cap_memo = memoexcept Exception:return _cap(dataset, cap, trainer.args, drop_unsupervised, packs_late)- # `truncation_mode` shapes the SLICE, so it belongs in the key: without- # it, evaluating once with keep_start and again with keep_end handed- # back the cached prefixes for both.+ # truncation_mode shapes the SLICE, so it belongs in the key: without it, evaluating once with+ # keep_start and again with keep_end handed back the cached prefixes for both.key = (id(dataset),drop_unsupervised,getattr(trainer.args, "truncation_mode", "keep_start"),- # `evaluate` and `get_eval_dataloader` share `drop_unsupervised` and- # see the same object in one call, but only the first may skip the cut- # under `eval_packing`, so without this the second reused the first's- # answer.+ # evaluate and get_eval_dataloader share drop_unsupervised and see the same object in one call, but+ # only the first may skip the cut under eval_packing.packs_late,)seen = memo.get(key)@@ -1566,13 +1451,9 @@ def _wrap_sft_evaluate_cap(trainer_cls):return seen[2]capped = _cap(dataset, cap, trainer.args, drop_unsupervised, packs_late)memo[key] = (dataset, cap, capped, token)- # Bounded, because every entry pins BOTH the original split and the- # capped copy for the trainer's whole lifetime -- deliberately, so a- # later split cannot inherit a freed `id()`. A caller that builds a- # fresh validation subset each epoch therefore accumulated Arrow tables- # until the host ran out. Evicting the least recently used costs nothing- # real: a run evaluates over a handful of splits, which is what the- # bound leaves room for.+ # Bounded: every entry pins both the original split and the capped copy for the trainer's+ # life, so a later split cannot inherit a freed id(). A fresh subset per epoch otherwise+ # accumulated Arrow tables until the host ran out.while len(memo) > _EVAL_CAP_MEMO_MAX:memo.pop(next(iter(memo)))return capped@@ -1584,20 +1465,14 @@ def _wrap_sft_evaluate_cap(trainer_cls):drop_unsupervised = True,packs_late = False,):- # `evaluate(eval_dataset = "validation")` is the supported way to pick one- # split out of a stored dict: `get_eval_dataloader` resolves it as- # `self.eval_dataset[eval_dataset]`. Capping the KEY is a no-op, so the- # split it names reached the collator uncapped. Cap it where it is stored- # and hand the key straight back.+ # evaluate(eval_dataset = "validation") picks one split out of a stored dict, and capping the+ # KEY is a no-op, so the split it names reached the collator uncapped.if isinstance(given, str):stored = getattr(trainer, "eval_dataset", None)if isinstance(stored, dict) and given in stored:capped = _cap_cached(trainer, stored[given], cap, drop_unsupervised, packs_late)- # Staged for the caller to swap in and OUT, never written- # through. Overwriting `stored[given]` destroyed the uncapped- # original, so a later `truncation_mode = "keep_end"` -- which- # the memo key now distinguishes on purpose -- could only re-cap- # the saved prefix and never produce the suffix asked for.+ # Staged for the caller to swap in and OUT: overwriting stored[given] destroyed the uncapped+ # original, so a later truncation_mode = "keep_end" could only re-cap the saved prefix.if capped is not stored[given]:trainer._unsloth_pending_split_swap = (stored, given, capped)return given@@ -1615,14 +1490,9 @@ def _wrap_sft_evaluate_cap(trainer_cls):def wrapped(self, *args, **kwargs):cap = getattr(getattr(self, "args", None), "max_seq_length", None)retained = getattr(getattr(self, "args", None), "max_length", None)- # A retained `max_length` is NOT proof the cap is enforced- # downstream. It is exactly what the construction block leaves- # behind when it turns padding-free OFF instead of clearing the- # cap, and TRL's collator does not truncate rows that already carry- # `input_ids`. `_prepare_dataset` runs only from `__init__`, so a- # split handed over later is never prepared either: skipping on a- # retained `max_length` let an overlength late split reach the model- # with nothing enforcing anything. Cap to whichever value is set.+ # A retained max_length does not prove the cap is enforced: it is what the construction block+ # leaves when it turns padding-free OFF, TRL's collator never truncates rows carrying+ # input_ids, and _prepare_dataset runs only from __init__.if retained is not None:cap = retainedif not cap:@@ -1638,8 +1508,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):swap = getattr(self, "_unsloth_pending_split_swap", None)if swap is None:return original(self, *args, **kwargs)- # A named split: swap the capped copy in for this call only, so- # the caller keeps the uncapped original for the next mode.+ # A named split: swap the capped copy in for this call only, so the caller keeps the uncapped+ # original for the next mode.container, key, replacement = swapself._unsloth_pending_split_swap = Noneprevious = container[key]@@ -1648,22 +1518,16 @@ def _wrap_sft_evaluate_cap(trainer_cls):return original(self, *args, **kwargs)finally:container[key] = previous- # `evaluate()` with no split passed falls back to the one stored on- # the trainer, and a caller can install or replace that after- # construction -- which is exactly where the constructor's cap can no- # longer see it. Every eval during training comes through here too.+ # evaluate() with no split falls back to the one stored on the trainer, which a caller can install+ # after construction, where the constructor's cap can no longer see it.stored = getattr(self, keyword, None) if keyword == "eval_dataset" else Noneif stored is None:return original(self, *args, **kwargs)capped = _cap_splits(self, stored, cap, drop_unsupervised, packs_late)if capped is stored:return original(self, *args, **kwargs)- # Swapped onto the trainer rather than passed down as an argument,- # because Trainer.evaluate recurses over a dict of splits by NAME when- # nothing was passed:- # eval_dataset = _eval_dataset if override else eval_dataset_name- # Passing the dict makes that an override, which changes what- # `get_eval_dataloader` is handed and what it caches.+ # Swapped onto the trainer rather than passed down: Trainer.evaluate recurses over a dict of+ # splits by NAME when nothing was passed, and passing the dict makes that an override.setattr(self, keyword, capped)try:return original(self, *args, **kwargs)@@ -1673,17 +1537,11 @@ def _wrap_sft_evaluate_cap(trainer_cls):wrapped._unsloth_eval_cap_wrapped = Truereturn wrapped- # `predict` keeps every row: its contract is one prediction per row in order.- #- # The two dataloader builders are public API and neither goes through- # `evaluate`/`predict`, so a caller doing `get_eval_dataloader(late)` reached- # the padding-free collator with `args.max_length` already cleared and- # nothing capping the split. Same wrapper, same argument position; the- # `drop_unsupervised` split follows the method it serves, since- # `get_test_dataloader` feeds `predict`.- # Only `evaluate` can hand its split to TRL's own prep, and only from 1.7.0.- # Read once, before anything is wrapped, so the probe sees TRL's method rather- # than ours.+ # predict keeps every row: one prediction per row, in order. The two dataloader builders are+ # public API and bypass evaluate/predict, so get_eval_dataloader(late) met the padding-free+ # collator with max_length already cleared and nothing capping the split.+ # Only evaluate can hand its split to TRL's own prep, and only from 1.7.0. Read once, before+ # anything is wrapped, so the probe sees TRL's method rather than ours.prepares_late = _trl_prepares_late_evals(trainer_cls)for name, keyword, drop_unsupervised, packs_late in (("evaluate", "eval_dataset", True, prepares_late),@@ -1700,17 +1558,14 @@ def _wrap_sft_evaluate_cap(trainer_cls):_UNSLOTH_RETURN_HIDDEN_STATES_SUPPORT_MARKER = "__UNSLOTH_SUPPORTS_RETURN_HIDDEN_STATES__"_UNSLOTH_GRPO_HIDDEN_STATES_WRAPPED_ATTR = "_unsloth_grpo_hidden_states_forward_wrapped"_UNSLOTH_GRPO_HIDDEN_STATES_WARNING_ATTR = "_unsloth_grpo_hidden_states_warning_issued"-# Whether the *most recent* forward handed back real logits instead of hidden-# states. The warning attribute above is warn-once bookkeeping and is never-# cleared, so it answers "ever degraded", not "degraded on the call the caller-# is about to dispatch on".+# Whether the MOST RECENT forward handed back real logits instead of hidden states: the warning+# attribute above is warn-once bookkeeping and is never cleared, so it answers "ever degraded"._UNSLOTH_GRPO_HIDDEN_STATES_DEGRADED_ATTR = "_unsloth_grpo_hidden_states_degraded"def _module_returns_logits(module):- # get_output_embeddings() is None on the decoder bodies (Qwen2Model, ...) and the- # head module on the *ForCausalLM wrappers, so it finds the head owner by behaviour- # rather than by a model-name list.+ # get_output_embeddings() is None on the decoder bodies and the head module on the *ForCausalLM+ # wrappers, so it finds the head owner by behaviour rather than by a model-name list.if module is None:return Falseget_output_embeddings = getattr(module, "get_output_embeddings", None)@@ -1734,10 +1589,9 @@ def _grpo_hidden_states_wrap_target(model):child = getattr(model, attr, None)if child is None or child is model or not hasattr(child, "forward"):continue- # Descend only into an adapter that still owns the head. A bare *ForCausalLM- # (what TRL builds GRPO's ref_model as) also has a `.model`, but that is its- # decoder body, and wrapping it leaves the head above running untouched: the- # fallback becomes a silent no-op returning [B, T, vocab], not [B, T, hidden].+ # Descend only into an adapter that still owns the head: a bare *ForCausalLM (TRL's GRPO+ # ref_model) also has .model, but that is its decoder body, so wrapping it returns+ # [B, T, vocab] and the fallback is a silent no-op.if not _module_returns_logits(child):continuereturn child@@ -1813,12 +1667,9 @@ def _get_num_logits_to_keep(forward_signature, args, kwargs):def _warn_grpo_hidden_states_fallback_once(model, message):- # The degradation flag is per call: whether the tensor this forward is about- # to return is real logits. Degradation is not a property of the model -- a- # forward that splats **kwargs into a sub-module only reached by some inputs- # (a vision tower, say) raises for the batches that reach it and succeeds for- # the rest -- so a sticky flag would keep routing later, genuine hidden- # states through the raw-logits helper, skipping the lm_head matmul.+ # The degradation flag is per call: a forward that splats **kwargs into a sub-module raises+ # only for the batches that reach it, so a sticky flag would send real hidden states through+ # the raw-logits helper.setattr(model, _UNSLOTH_GRPO_HIDDEN_STATES_DEGRADED_ATTR, True)if getattr(model, _UNSLOTH_GRPO_HIDDEN_STATES_WARNING_ATTR, False):return@@ -1861,9 +1712,8 @@ def _minimise_logits_kwarg(forward_signature, args, forward_kwargs):bound = forward_signature.bind_partial(*args, **forward_kwargs)except TypeError:return None- # A forward given `labels` positionally lands it in `bound.arguments` and- # never in `forward_kwargs`, so the lookup above cannot see it, and the loss- # the model then computes for itself would be one position wide.+ # A forward given labels positionally lands it in bound.arguments and never in forward_kwargs, so+ # the lookup above cannot see it and the loss the model computes would be one position wide.if bound.arguments.get("labels") is not None:return Noneaccepts_var_keyword = any(@@ -1874,13 +1724,8 @@ def _minimise_logits_kwarg(forward_signature, args, forward_kwargs):declared = name in forward_signature.parametersif not declared and not accepts_var_keyword:continue- # Passing it positionally already and then also by keyword is a- # TypeError. Give up entirely rather than moving to the next name: a- # caller who bound a width positionally has said what it wants, and- # reaching for the OTHER spelling would either fight that width or, on a- # forward that only understands the one it was handed and swallows the- # rest in `**kwargs`, be accepted and ignored -- no saving, and a- # non-None return arms the re-run below for nothing.+ # Positionally and by keyword is a TypeError. Give up rather than try the OTHER spelling,+ # which would fight the caller's width or be swallowed by **kwargs and silently ignored.if name in bound.arguments and name not in forward_kwargs:return Noneforward_kwargs[name] = 1@@ -1915,8 +1760,8 @@ def _drop_spare_hidden_states(outputs):elif hasattr(outputs, "hidden_states"):outputs.hidden_states = Noneexcept Exception:- # A frozen or exotic output object is not worth failing the step over;- # the caller has already taken the layer it needs.+ # A frozen or exotic output object is not worth failing the step over; the caller has already taken+ # the layer it needs.logger.debug("Unsloth: could not drop spare GRPO hidden states.",exc_info = True,@@ -1951,10 +1796,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):model_name = type(target_model).__name__def wrapped_forward(*args, **kwargs):- # accelerate's extract_model_from_parallel(keep_fp32_wrapper = False), which the- # GRPO loop calls every step, rebinds an instance-level forward as- # MethodType(forward, model), so the module arrives as a leading positional- # argument. `original_forward` is already bound, so drop it.+ # accelerate's extract_model_from_parallel(keep_fp32_wrapper = False), called every GRPO step,+ # rebinds the forward as MethodType, so the module arrives as a leading positional argument;+ # original_forward is already bound, so drop it.while len(args) != 0 and args[0] is target_model:args = args[1:]if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") != "1":@@ -1962,10 +1806,8 @@ def _install_grpo_hidden_states_forward_wrapper(model):setattr(target_model, _UNSLOTH_GRPO_HIDDEN_STATES_DEGRADED_ATTR, True)return original_forward(*args, **kwargs)- # copy: _drop_forward_kwargs_consumed_positionally hands `kwargs` straight- # back when there is nothing to drop, so mutating it below would poison the- # caller's dict and make the fallback calls further down re-send the very- # kwargs the model just rejected.+ # Copy: _drop_forward_kwargs_consumed_positionally returns kwargs unchanged when there is+ # nothing to drop, so mutating it would poison the caller's dict for the fallback calls.forward_kwargs = dict(_drop_forward_kwargs_consumed_positionally(forward_signature, args, kwargs))@@ -1988,6 +1830,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):)return original_forward(*args, **kwargs)+ # TRL 0.26+: Config may be in a separate *_config.py module+ # Thin wrapper fallback: walk the Trainer's MRO to find Config in the real implementation module (e.g.,+ # trl.experimental.bco)try:outputs = original_forward(*args, **forward_kwargs)except TypeError as error:@@ -1996,12 +1841,8 @@ def _install_grpo_hidden_states_forward_wrapper(model):if not rejected_hidden_states(message):raisereturn forward_without_hidden_states()- # The signature advertised the parameter but the forward refuses- # the value. Retry without our minimisation rather than losing- # hidden states entirely over an optimisation -- and send the retry- # through the same fallback, since a forward that splats **kwargs- # into a sub-module can refuse the logits limiter and the hidden- # states one after the other.+ # The signature advertised the parameter but the forward refuses the value: retry without the+ # minimisation, through the same fallback, rather than lose hidden states over it.forward_kwargs.pop(logits_kwarg, None)logits_kwarg = Nonetry:@@ -2019,12 +1860,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):)if logits_kwarg is None:return outputs- # `outputs.logits` is the return value now, and we asked for a single- # position because we meant to throw the logits away. GRPO drops the- # last one and slices the completion window out of what is left, so- # one position leaves it nothing. Put the caller's own limit back --- # absent means every position, which is what a model that ignores- # `logits_to_keep` would have returned anyway -- and re-run.+ # outputs.logits is the return value now, and one position was asked for only to throw the+ # logits away; GRPO drops the last and slices the completion window, so restore the caller's+ # own limit and re-run.if logits_kwarg in kwargs:forward_kwargs[logits_kwarg] = kwargs[logits_kwarg]else:@@ -2034,11 +1872,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):hidden_states = hidden_states[-1]if num_logits_to_keep != 0:hidden_states = hidden_states[:, -num_logits_to_keep:, :]- # Only the last layer is ever read. Every earlier layer is still hanging- # off `outputs`, and on a multi-GPU dispatch accelerate's- # AlignDevicesHook.post_forward walks the whole returned object and- # copies every tensor in it to the input device, so keeping them costs a- # cross-device copy per layer as well as the memory.+ # Only the last layer is read, and accelerate's AlignDevicesHook.post_forward copies every+ # tensor in the returned object to the input device, so keeping the rest costs a cross-device+ # copy per layer as well as the memory._drop_spare_hidden_states(outputs)_note_grpo_hidden_states_success(target_model)return _replace_outputs_logits(outputs, hidden_states)@@ -2108,17 +1944,14 @@ def _backport_vision_dataset_gate(RLTrainer_source):def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):- # Defensive wrapper: matches patch_trl_rl_trainers()'s try/except so- # direct callers don't see exceptions from the impl on TRL versions- # that rename or move classes (e.g. TRL 1.x trl.experimental).+ # Defensive wrapper matching patch_trl_rl_trainers()'s try/except, so direct callers do not see+ # exceptions from the impl on TRL versions that rename or move classes.try:return _patch_trl_rl_trainers_impl(trainer_file)except Exception as e:- # Warning, not info. The impl RETURNS for the benign case this swallow- # exists for (a trainer this TRL does not ship), so anything reaching- # here means the module imported and generation itself failed, and the- # run silently falls back to trl's trainer, losing Unsloth's- # compute_loss, bf16/fp16 fixup and dataset handling at once.+ # Warning, not info: the impl RETURNS for the benign case, so reaching here means generation+ # failed and the run silently falls back to trl's trainer, losing Unsloth's compute_loss,+ # bf16/fp16 fixup and dataset handling.logger.warning_once(f"Unsloth: Could not build the patched trl.trainer.{trainer_file}, "f"so training will use trl's own trainer instead: "@@ -2128,7 +1961,6 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):- # Patch for vLLM and Unsloth PEFTimport trlimport trl.trainer@@ -2138,7 +1970,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):logger.info(f"Unsloth: Could not import trl.trainer.{trainer_file}: {error}")return- # Get SFTTrainer and SFTConfig namesname = [xfor x in dir(trainer)@@ -2161,7 +1992,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)returnif len(config) != 1:- # TRL 0.26+: Config may be in a separate *_config.py module+ # TRL 0.26+: the Config may be in a separate *_config.py module, or reachable by walking the+ # Trainer's MRO to the real implementation module (trl.experimental.bco).config_module_name = trainer_file.replace("_trainer", "_config")try:config_mod = eval(f"trl.trainer.{config_module_name}")@@ -2176,8 +2008,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):except Exception:passif len(config) != 1 and len(name) == 1:- # Thin wrapper fallback: walk the Trainer's MRO to find Config- # in the real implementation module (e.g., trl.experimental.bco)try:_temp_cls = eval(f"trl.trainer.{trainer_file}.{name[0]}")for _parent in _temp_cls.__mro__[1:]:@@ -2204,7 +2034,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)return- # Get SFTTrainer, SFTConfigRLTrainer_name = name[0]RLConfig_name = config[0]try:@@ -2218,7 +2047,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):try:RLConfig = eval(f"trl.trainer.{trainer_file}.{RLConfig_name}")except Exception:- # TRL 0.26+: Config may be in a separate *_config.py module+ # TRL 0.26+: the Config may be in a separate *_config.py module, or loadable from the parent trainer's module.try:config_module_name = trainer_file.replace("_trainer", "_config")RLConfig = eval(f"trl.trainer.{config_module_name}.{RLConfig_name}")@@ -2244,7 +2073,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):logger.info(f"Unsloth: Could not load {RLConfig_name}")return- # Check nameif RLTrainer.__name__.startswith("Unsloth"):print(f"Unsloth: {RLTrainer.__name__} is already patched.")return@@ -2252,10 +2080,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):print(f"Unsloth: {RLConfig.__name__} is already patched.")return- # TRL 0.26+: Resolve thin wrappers to their experimental parent class.- # Thin wrappers are deprecation shims in trl.trainer that just forward- # *args/**kwargs to the real implementation in trl.experimental.- # Only resolve if a parent class actually lives in a trl.experimental module.+ # TRL 0.26+: resolve thin wrappers (trl.trainer shims forwarding to trl.experimental) to their+ # parent class, and only when that parent really lives in a trl.experimental module._trainer_resolved_module = Nonetry:_trainer_src = inspect.getsource(RLTrainer)@@ -2287,14 +2113,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):_parent_mod = inspect.getmodule(_parent)if _parent_mod is None:continue- # Only resolve to a parent that lives in trl.experimentalif "trl.experimental" in _parent_mod.__name__:RLConfig = _parentbreakexcept Exception:pass- # Get old sourceold_RLTrainer_source = inspect.getsource(RLTrainer)old_RLConfig_source = inspect.getsource(RLConfig)@@ -2307,7 +2131,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):# Fix _deprecate_arguments not getting imported so stop __ but not _imports = [x for x in all_imports if not x.startswith("__")]- # Get default argumentsEMPTY = inspect.Parameter.emptyprocessed = []for RLobject in [RLTrainer, RLConfig]:@@ -2345,11 +2168,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):))- # Process RLTrainer firstarguments, call_args = processed[0]RLTrainer_post = ""- # Add tokenizer if not seenif "tokenizer" not in parameters and "processing_class" in parameters:arguments += f",\n{' ' * 8}tokenizer = None"call_args = call_args.replace(@@ -2357,7 +2178,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"processing_class = tokenizer if tokenizer is not None else processing_class",)- # Edit bf16, fp16 by checking model's dtype/torch_dtype directly+ # Edit bf16, fp16 by checking the model's dtype/torch_dtype directly.extra_args = ""if "args" in call_args and "model" in call_args:mixed_precision = (@@ -2366,22 +2187,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"use_fp16 = getattr(args, 'fp16', False)\n""if type(use_fp16) is not bool: use_fp16 = False\n""force_float32 = False\n"- # device-aware bf16 check (CUDA/XPU/HIP), so V100/T4 never pick bf16- # but AMD/Intel are unaffected; fall back on older unsloth_zoo.+ # Device-aware bf16 check (CUDA/XPU/HIP), so V100/T4 never pick bf16 while AMD/Intel are unaffected;+ # fall back on older unsloth_zoo."try:\n"" from unsloth_zoo.device_type import device_is_bf16_supported as _bf16_supported\n""except Exception:\n"" _bf16_supported = torch.cuda.is_bf16_supported\n"- # FORCE_FLOAT32 models (Gemma3, gpt_oss, ...) cannot use float16. On a GPU without- # bf16 (V100/T4) keep them in float32 so they never autocast to fp16. On a bf16 GPU,- # full finetuning can still use bf16 autocast (master weights stay float32), which is- # faster and uses less memory; LoRA/QLoRA keep float32 when forced.- # Stamped by from_pretrained: the env is process wide, so a LoRA model- # loaded after this one would cost this trainer its bfloat16.+ # FORCE_FLOAT32 models (Gemma3, gpt_oss) cannot use float16: without bf16 keep float32, with+ # bf16 full finetuning may still autocast. Stamped by from_pretrained, since the env is+ # process wide and a later load would otherwise answer for this trainer."full_finetuning = getattr(model, '_unsloth_full_finetuning', None)\n""if full_finetuning is None: full_finetuning = os.environ.get('UNSLOTH_ENABLE_FULL_FINETUNING', '0') == '1'\n"- # Stamped by from_pretrained: the env is process wide, so a forced family- # loaded earlier would answer here for a model that is not forced at all.+ # Stamped by from_pretrained: the env is process wide, so a forced family loaded earlier would+ # answer here for a model that is not forced at all."model_forced_float32 = getattr(model, '_unsloth_forced_float32', None)\n""if model_forced_float32 is None: model_forced_float32 = os.environ.get('UNSLOTH_FORCE_FLOAT32', '0') == '1'\n""if model_forced_float32 and not (full_finetuning and _bf16_supported()):\n"@@ -2395,8 +2213,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"float16 = dtype == torch.float16\n""bfloat16 = dtype == torch.bfloat16\n""float32 = dtype == torch.float32\n"- # Set only when the caller passed dtype = torch.float32 themselves: a- # request, not a side effect of upcasting, and immune to a second load.+ # Set only when the caller passed dtype = torch.float32 themselves: a request, not a side effect of+ # upcasting, and immune to a second load."user_float32 = bool(getattr(model, '_unsloth_user_float32', False))\n""if full_finetuning:\n"" if bfloat16 and use_fp16: use_fp16 = False\n"@@ -2411,10 +2229,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" if hasattr(args, 'mixed_precision'): args.mixed_precision = 'no'\n"" # args.mixed_precision is a new argument which needs to be set now\n""elif (not use_bf16 and not use_fp16) and mixed_precision_dtype == 'float32' and float32 and user_float32 and not _bf16_supported():\n"- # Without bf16 the only autocast is float16, whose narrower exponent range- # overflows float32 values to inf then NaN. Gated on the explicit request:- # full finetuning upcasts to float32 itself, and float16 autocast over- # float32 master weights is the normal V100/T4 recipe (see #4082).+ # Without bf16 the only autocast is float16, whose exponent range overflows float32 to inf+ # then NaN. Gated on the explicit request: fp16 autocast over fp32 master weights is the+ # normal V100/T4 recipe (#4082)." print('Unsloth: Model is in float32 and this GPU has no bfloat16 support, so training stays in float32. Pass fp16 = True to force float16 mixed precision instead.')\n"" args.fp16 = False\n"" args.bf16 = False\n"@@ -2445,10 +2262,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += mixed_precision- # Check if per_device_eval_batch_size (default 8) bigger than bsz- # Also use FP16 / BF16 evaluation+ # Check if per_device_eval_batch_size (default 8) is bigger than bsz, and use FP16 / BF16 evaluation.if "args" in call_args:- # Check eval_dataset firstif "eval_dataset" in call_args:check_eval_dataset = ("if getattr(args, 'eval_dataset', None) is not None and "@@ -2458,7 +2273,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += check_eval_dataset- # Check if gradient accumulation bug fix is appliedcheck_ga = ("ga_steps = getattr(args, 'gradient_accumulation_steps', None)\n""if ga_steps is not None and ga_steps > 1:\n"@@ -2492,7 +2306,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += eval_changes- # Force logits to be produced if preprocess_logits_for_metrics or compute_metrics is used+ # Force logits to be produced if preprocess_logits_for_metrics or compute_metrics is used.if "model" in call_args:logits_check = ("_output_logits = False\n"@@ -2515,7 +2329,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += warnings_issued_check- # Check max_seq_lengthif "model" in call_args:length_check = ("if 'max_seq_length' not in locals() and not hasattr(args, 'max_seq_length'):\n"@@ -2534,7 +2347,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += length_check- # At this point max_seq_length might be set, but trl is moving to max_length+ # max_seq_length may be set here, but trl is moving to max_length.if trainer_file == "sft_trainer":max_length_check = ("if 'max_length' not in locals() and not hasattr(args, 'max_length'):\n"@@ -2558,31 +2371,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" print('Unsloth: We did not find `max_seq_length` or `max_length` in the model or args. We will set it to 1024.')\n"" args.max_length = 1024\n")- # TRL >= 1.0.0 refuses padding-free without packing while `max_length` is- # set, since it cannot truncate a flattened batch. Unsloth auto-enables- # padding-free and the block above always writes `args.max_length`, so- # essentially every SFT user tripped that guard.- #- # Keep the length that block resolved (`max_seq_length`, capped by the- # model, wins) and move it to wherever it will actually be enforced:- # * Unsloth's dataset prep will tokenize -> park it in `max_seq_length`,- # which prep reads, and hand TRL the None it asks for.- # * it will not (`skip_prepare_dataset`, or rows that already carry- # `input_ids` / `labels`) -> nothing would truncate, so turn- # padding-free off and keep `max_length` for TRL's own collator.- #- # The copy is unconditional on purpose: no TRL from 0.22.2 to 1.9.2- # declares `max_seq_length` on SFTConfig (only UnslothSFTConfig re-adds- # it), so a hasattr() gate would skip it for every pristine- # `trl.SFTConfig` and the clear below would drop the cap. `args` is a- # dataclass instance, so the assignment just adds the attribute and- # `to_dict()` never sees it.- #- # It must be None, not 0: TRL's guard reads `args.max_length is not None`,- # and rl_replacements.py normalises the None back to 0 inside the Zoo.- # The schema comes off the first yielded row like the Zoo does, falling- # back to `column_names` (a `with_transform` dataset reports its backing- # columns there while yielding `input_ids`).+ # TRL >= 1.0.0 refuses padding-free without packing while max_length is set, and Unsloth+ # auto-enables padding-free, so nearly every SFT user tripped that guard. Move the resolved+ # length to where it is enforced: max_seq_length when prep tokenizes, else padding-free off.+ # Unconditional copy: no TRL from 0.22.2 to 1.9.2 declares max_seq_length on SFTConfig, so a+ # hasattr() gate would skip every pristine config and the clear below would drop the cap.+ # Must be None, not 0: TRL's guard reads `args.max_length is not None`.if "`max_length` is not enforced" in old_RLTrainer_source:max_length_check += ("if getattr(args, 'padding_free', False) is True and not getattr(args, 'packing', False) "@@ -2596,20 +2390,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" _unsloth_skip_prepare = True\n"" except Exception:\n"" pass\n"- # Metadata first and a row only as a fallback, the other way round- # from before: reading a row off a one-shot training stream consumes- # it, and nothing here chains it back, so the run began at row 2.- # `iter(x) is iter(x)` marks the streams that cannot spare one --- # the same signal the cap scan and the schema probe use.- # A `with_transform` split reports its BACKING columns, so a- # transform yielding `input_ids` over a stored `text` answered- # "raw" and the cap was cleared for rows nothing then truncates.- # Its rows are rebuilt on every read, so probing one is free.- # An unprobeable stream cannot be ruled tokenized either, and- # `True` there clears the cap on the same guess: refuse instead- # and keep `max_length` for TRL's collator.- # One rule, read twice: the late `_unsloth_pretokenized` probe asks the- # same question of an eval split and must not trust its columns either.+ # Metadata first, a row only as fallback: reading one off a one-shot stream consumes it, and+ # `iter(x) is iter(x)` marks those. A with_transform split reports BACKING columns, so one+ # yielding input_ids over stored `text` read "raw"; probing its rows is free." def _unsloth_is_transformed(_ds):\n"" _f = getattr(_ds, 'format', None)\n"" _f = _f.get('type') if isinstance(_f, dict) else None\n"@@ -2632,23 +2415,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" _unsloth_prep_truncates = False\n"" except Exception:\n"" _unsloth_prep_truncates = False\n"- # Already-tokenized rows are not a dead end. TRL's own _prepare_dataset- # truncates them (sft_trainer.py: `elif args.max_length is not None:- # truncate_dataset(...)`), and the LM collator it builds passes no- # max_length, so truncation there is the ONLY thing enforcing the cap. The- # Zoo's replacement returns pre-tokenized rows untouched, so doing it here- # restores TRL's contract rather than inventing one, and padding-free can- # stay on. skip_prepare_dataset is the exception: the user asked for the- # dataset to be left alone, and TRL skips truncation there too.- # Only a MATERIALISED tokenized dataset. A with_transform dataset yields- # input_ids on access while column_names still says ["text"], so mapping it- # would truncate the wrong thing; that case keeps the fallback below.- # One predicate, applied per DATASET, because the train split and each eval- # split can be in different shapes. Truncating is only safe for a materialised- # tokenized table: raw conversational rows (messages: list[dict]) are per-row- # sequences too and would be sliced into corrupted turns, and a with_transform- # dataset recreates its rows on read, so map() writes the backing table while- # the reader keeps handing back overlength input_ids.+ # Already-tokenized rows are not a dead end: TRL's _prepare_dataset truncates them and its LM+ # collator passes no max_length, so that truncation is the only thing enforcing the cap. The+ # Zoo returns pre-tokenized rows untouched. skip_prepare_dataset is the exception.+ # Only a MATERIALISED tokenized dataset, judged per DATASET since splits differ: with_transform+ # yields input_ids while column_names still says ["text"], raw conversational rows would be+ # sliced into corrupted turns, and a transform's rows are rebuilt on every read." def _unsloth_truncatable(_ds):\n"" if _ds is None or not hasattr(_ds, 'map'): return False\n"" try:\n"@@ -2660,67 +2432,36 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" _cols = getattr(_ds, 'column_names', None)\n"" if isinstance(_cols, dict):\n"" _cols = [_c for _v in _cols.values() for _c in (_v or [])]\n"- # A packed split is out: TRL skips truncation when packing- # (`if args.max_length is not None and not packing`), and cutting- # `input_ids` under a `seq_lengths` that still describes the old- # row is worse than not cutting at all.+ # A packed split is out: TRL skips truncation when packing, and cutting input_ids under a+ # seq_lengths that still describes the old row is worse than not cutting." if 'seq_lengths' in (_cols or ()): return False\n"" return bool(_cols) and 'input_ids' in _cols\n"" except Exception:\n"" return False\n"- # Read a row BACK. Every predicate above is a guess about what the dataset- # will do; this is the one check that observes it. A split with no- # `input_ids` is raw, so prep tokenizes it with the cap and it is fine.+ # Read a row BACK: every predicate above is a guess about what the dataset will do, and this is the+ # one check that observes it." _unsloth_cap = args.max_length\n"- # TRL slices [-max_length:] for `keep_end`, which callers use when- # the completion sits at the tail of a long prompt. Consuming- # `max_length` while always keeping the prefix trained on the wrong- # half of every row.+ # TRL slices [-max_length:] for keep_end, which callers use when the completion sits at the tail of+ # a long prompt; always keeping the prefix trained on the wrong half of every row." _unsloth_truncation_mode = getattr(args, 'truncation_mode', 'keep_start') or 'keep_start'\n"- # keep_start and keep_end are the only two slices there are. TRL's SFT path- # never reads this attribute at all -- it belongs to the preference trainers --- # so nothing downstream would catch a third value, and mapping it to the- # default would silently cut from the side the caller asked us not to. Refuse- # the enforcement claim instead, the same answer this block gives for any- # split it cannot honour.+ # keep_start and keep_end are the only two slices, and TRL's SFT path never reads this+ # attribute, so a third value would go uncaught. Refuse the enforcement claim instead." _unsloth_keep_end = _unsloth_truncation_mode == 'keep_end'\n"" _unsloth_known_mode = _unsloth_truncation_mode in ('keep_start', 'keep_end')\n"" _unsloth_slice = slice(-_unsloth_cap, None) if _unsloth_keep_end else slice(None, _unsloth_cap)\n"- # Resolved here, one level out from the truncation block, because the fallback- # below reads it even when `skip_prepare_dataset` skips that block entirely.- # `eval_packing` is separate from `packing`:- # packing = args.packing if args.eval_packing is None else args.eval_packing- # (sft_trainer.py), so packing = False with eval_packing = True reaches this- # branch, which is gated on `not args.packing`. TRL then PACKS the eval split- # rather than truncating it, and every strategy owns the overflow: `wrapped`- # concatenates the stream before chunking, `bfd_split` splits an overlength- # example into more chunks. Cutting rows at the cap first throws that away.+ # Resolved outside the truncation block, since the fallback reads it even under+ # skip_prepare_dataset. eval_packing is separate from packing, so packing=False with+ # eval_packing=True lands here and TRL packs the eval split instead of truncating it." _unsloth_eval_packing = getattr(args, 'packing', False) if getattr(args, 'eval_packing', None) is None else getattr(args, 'eval_packing')\n"" _unsloth_completion_only = getattr(args, 'completion_only_loss', None)\n"- # Column names first, and a row only if reading one is free. On a- # one-shot stream this probe consumed the first TRAINING example --- # nothing chains it back here, so the run started at row 2. The- # same `iter(x) is iter(x)` signal the cap scan uses marks those,- # and an unreadable schema resolves to False, which is what TRL- # itself answers for a split with no `prompt`/`completion`.- # A `with_transform` split answers `column_names` with its BACKING- # table, so a transform storing `text` but yielding `prompt` /- # `completion` resolved to False here while TRL, which reads a- # yielded row, resolved True and applied `completion_mask`. The- # cap filters then kept rows whose completion was cut away- # entirely, and the collator turned those into all -100.- # Same rule as the two schema probes above: transformed columns- # are not evidence, so ignore them and read a row instead.+ # Column names first, a row only if free: on a one-shot stream this probe ate the first+ # TRAINING example. A with_transform split answers with its BACKING table, so one yielding+ # prompt/completion read False here while TRL read True and applied completion_mask." if _unsloth_completion_only is None:\n"" try:\n"- # A `set_format(columns = [...], output_all_columns = False)`- # split yields only the named columns while `column_names`- # still answers with the whole backing table. Reading the- # table resolved completion-only True off a `completion` the- # rows never hand over, TRL resolved False from a yielded- # row, and the cap then filtered on a `completion_mask` the- # collator ignores -- dropping valid rows, up to emptying the- # split. The format's own column list is what is yielded.+ # A set_format(output_all_columns = False) split yields only the named columns while+ # column_names still lists the whole backing table, so completion-only resolved True off a+ # completion the rows never hand over. The format's own column list is what is yielded." _unsloth_fmt = getattr(train_dataset, 'format', None)\n"" _unsloth_fmt = _unsloth_fmt if isinstance(_unsloth_fmt, dict) else {}\n"" _unsloth_shown = _unsloth_fmt.get('columns')\n"@@ -2737,22 +2478,17 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" except Exception:\n"" _unsloth_train_sample = {}\n"" _unsloth_completion_only = ('prompt' in _unsloth_train_sample and 'completion' in _unsloth_train_sample)\n"- # Parked on args so the late evaluate()/predict() cap reads the SAME value.- # It resolves from the train schema, which a split handed over later cannot- # see, and disagreeing with the collator is what leaves an all -100 row in.+ # Parked on args so the late evaluate()/predict() cap reads the SAME value: it resolves from+ # the train schema, and disagreeing with the collator leaves an all -100 row in." args._unsloth_completion_only_loss = _unsloth_completion_only\n"- # EVERY row, not the first. A short row 0 in front of a long- # row 5000 read as "within the cap", and in the fallback branch- # nothing downstream truncates it. A map-style split is read in- # full; a stream cannot be rewound, so a bounded prefix is all- # there is and the check says so rather than pretending.+ # EVERY row, not the first: a short row 0 before a long row 5000 read as within the cap. A+ # map-style split is read in full; a stream cannot be rewound, so a bounded prefix is all+ # there is and the check says so." _UNSLOTH_SCAN_ROWS = 1024\n"" def _unsloth_within_cap(_ds):\n"" if _ds is None: return True\n"- # Believe a producer's own truncation claim: scanning a- # `with_transform` split would tokenize every row in `__init__`,- # the eager pass it exists to avoid. Read from `__dict__` so a- # wrapper does not inherit the inner split's claim.+ # Believe a producer's own truncation claim: scanning a with_transform split tokenizes every+ # row in __init__. Read from __dict__ so a wrapper does not inherit the inner split's claim." _unsloth_own = getattr(_ds, '__dict__', None)\n"" if isinstance(_unsloth_own, dict):\n"" _unsloth_claim = _unsloth_own.get('_unsloth_truncated_to')\n"@@ -2761,49 +2497,33 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" try:\n"" try: _n = len(_ds)\n"" except Exception: _n = None\n"- # A single-pass stream cannot be scanned at all: reading it here IS- # consuming it, and the trainer would then get an exhausted split- # (or one short by up to 1024 rows). Two `iter()` calls handing back- # the same object is what says so -- true for a bare generator and- # for a `torch.utils.data.IterableDataset` that returns shared- # iterator state, false for a `datasets.IterableDataset`, which- # restarts. Unverifiable, so answer as the prefix case below does:- # not proven within the cap, which drops the enforcement claim and- # leaves every row where it is.+ # A single-pass stream cannot be scanned: reading it here IS consuming it, and two iter()+ # calls returning the same object say so (a datasets.IterableDataset restarts and does not).+ # Unverifiable, so answer as the prefix case: not proven within the cap." _unsloth_rows = iter(_ds)\n"" if _unsloth_rows is iter(_ds): return False\n"" _seen = 0\n"" for _row in _unsloth_rows:\n"" if 'input_ids' not in _row: return True\n"" if len(_row['input_ids']) > _unsloth_cap: return False\n"- # An unexhausted stream is UNVERIFIED, not verified. Calling the- # first 1024 fitting rows proof let a later overlength row through,- # and in the fallback branch nothing truncates a pre-tokenized row.- # A stream that can be rewritten is capped by the lazy map above and- # never reaches this; one that cannot is refused.+ # An unexhausted stream is UNVERIFIED, not verified: treating the first 1024 fitting rows as proof+ # let a later overlength row through, and nothing truncates a pre-tokenized row here." _seen += 1\n"" if _n is None and _seen >= _UNSLOTH_SCAN_ROWS: return False\n"" except Exception:\n"" return False\n"" return True\n"- # Each eval split counts. A tokenized eval split in a shape the truncation- # cannot rewrite (with_transform, or an iterable with no column_names) is- # left alone above, and prep will not re-tokenize rows that already carry- # `input_ids`, so consuming `max_length` on the strength of the train split- # alone let evaluation run over the cap.+ # Each eval split counts: one the truncation cannot rewrite is left alone above, and prep+ # never re-tokenizes rows carrying input_ids, so trusting the train split left eval uncapped." def _unsloth_splits_within_cap(_ev):\n"" _splits = list(_ev.values()) if isinstance(_ev, dict) else [_ev]\n"" return all(_unsloth_within_cap(_s) for _s in _splits)\n"- # Not train-only. `_unsloth_prep_truncates` is decided from the train- # split, so a raw train beside a pre-tokenized eval set skipped this- # whole block, consumed `max_length`, and left evaluation uncapped:- # prep does not re-tokenize rows that already carry `input_ids`.+ # Not train-only: _unsloth_prep_truncates is decided from the train split, so a raw train beside a+ # pre-tokenized eval set skipped this whole block and left evaluation uncapped." if not _unsloth_skip_prepare:\n"- # TRL forwards its preparation map_kwargs; honour the same setting so a large- # pre-tokenized dataset is not rewritten single-process. Resolved through the- # same helper as every other map site: the config layer writes "run- # in-process" as `dataset_num_proc = 1`, and datasets >= 4.1 builds a Pool(1)- # for it, forking a tokenizer worker on the host that asked for none.+ # Honour TRL's preparation map_kwargs so a large pre-tokenized dataset is not rewritten+ # single-process, through the same helper as every map site: the config layer writes serial as+ # dataset_num_proc = 1, and datasets >= 4.1 builds a Pool(1) for it." _unsloth_map_kw = {}\n"" try:\n"" try:\n"@@ -2814,12 +2534,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" except Exception:\n"" _unsloth_nproc = None\n"" if _unsloth_nproc: _unsloth_map_kw['num_proc'] = _unsloth_nproc\n"- # Same rule as TRL's truncate_dataset: slice every per-row list column, so- # input_ids, labels, attention_mask and the masks stay aligned. Written out- # rather than imported, because trl.data_utils drags in the processor stack- # and an ImportError there would silently drop the cap. A torch/numpy format- # hands batched map() tensors, where `if _col` raises on the ambiguous truth- # value, so ask for a per-row sequence and exclude str/bytes.+ # Same rule as TRL's truncate_dataset: slice every per-row list column so input_ids, labels,+ # attention_mask and the masks stay aligned. Written out rather than imported, since+ # trl.data_utils drags in the processor stack and an ImportError would drop the cap." def _unsloth_is_sequence_column(_col):\n"" try:\n"" if len(_col) == 0: return False\n"@@ -2827,24 +2544,15 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" except Exception:\n"" return False\n"" if isinstance(_first, (str, bytes)): return False\n"- # len(), not hasattr('__len__'): under set_format('torch') a scalar- # column batches to a 1-D tensor whose element is 0-dim, which HAS- # __len__ and raises on it. The later len(_v) then threw TypeError,- # the outer catch restored the overlength dataset, and a truncatable- # run died on "cannot be enforced".+ # len(), not hasattr('__len__'): under set_format('torch') a scalar column batches to a 1-D+ # tensor whose element is 0-dim, so len(_v) threw TypeError and the catch restored the+ # overlength dataset." try: len(_first)\n"" except Exception: return False\n"" return True\n"- # Per-token columns only, matched by row length against `input_ids`.- # A packed split carries `seq_lengths` -- document lengths, not tokens- # -- and slicing that by the cap left it stale, so padding-free built- # position ids for more tokens than the row now holds.- # Per VALUE, because `_unsloth_is_sequence_column` judges the- # column from its first row. An optional field that is a list- # there and None (or a scalar) further in raised TypeError out- # of `len`, the enclosing handler restored the overlength split,- # and a truncatable run died on "cannot be enforced". The late- # cap validates each row for the same reason.+ # Per-token columns only, matched by row length against input_ids: a packed split's+ # seq_lengths are document lengths, and slicing them left padding-free building position ids+ # for tokens the row no longer has. Per VALUE, since a list row 0 can be None later." def _unsloth_cut_value(_v, _r):\n"" try:\n"" if len(_v) != len(_r): return _v\n"@@ -2861,38 +2569,22 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" else:\n"" _out[_k] = [_unsloth_cut_value(_v, _r) for _v, _r in zip(_col, _ids)]\n"" return _out\n"- # A stream has no length, and `IterableDataset.map` takes no `num_proc`:- # passing one raised TypeError, the catch below restored the original,+ # A stream has no length, and IterableDataset.map takes no num_proc: passing one raised TypeError# and the run died on "cannot be enforced" instead of being truncated." def _unsloth_is_stream(_ds):\n"" try: return not hasattr(_ds, '__len__')\n"" except Exception: return True\n"- # One split, capped and then checked. A stream's map is lazy and applies- # to EVERY row it will ever yield, which is a stronger guarantee than the- # bounded prefix scan: reading 1024 rows and calling the rest verified let- # a later overlength row through, since nothing else truncates a- # pre-tokenized stream.- # Enforcement, not observation. A `with_transform` split that happens- # to sit under the cap is not enforced -- it rebuilds its rows on every- # read -- so it keeps the old answer: hold `max_length` and turn- # padding-free off. Only a split we rewrote, or a raw one the tokenizer- # pass will cut, counts.- # Schema first, and a row only when one is free. `next(iter(_ds))`- # on a single-pass stream is a row the run then trains without:- # if it reads raw the split is declared safe and training starts- # at row 2, if it reads tokenized construction rejects a stream- # the caller still owns and has already been mutated. The- # `iter(x) is iter(x)` test is the one the cap scan and the- # schema probe already use; an unprobeable stream answers True,- # which holds `max_length` rather than clearing it on a guess.+ # One split, capped and then checked. A stream's map is lazy and applies to EVERY row it will ever+ # yield, a stronger guarantee than the bounded prefix scan.+ # Enforcement, not observation: a with_transform split under the cap is not enforced, since it+ # rebuilds its rows on every read, so it keeps max_length and turns padding-free off.+ # Schema first, a row only when free: next(iter(_ds)) on a single-pass stream is a row the run+ # then trains without. An unprobeable stream answers True, holding max_length." def _unsloth_pretokenized(_ds):\n"" try:\n"- # Columns first, EXCEPT for a transform: a `with_transform` split reports- # its backing table, so one storing `text` and yielding overlength- # `input_ids` answered "raw" here. `_unsloth_truncatable` already refuses- # to rewrite it, and this answer then marked it safe anyway and cleared- # `max_length`, leaving padding-free with rows nothing truncates. Its rows- # are rebuilt on every read, so probing one below costs nothing.+ # Columns first, EXCEPT for a transform: a with_transform split reports its backing table, so+ # one storing `text` and yielding overlength input_ids read "raw" and cleared max_length. Its+ # rows are rebuilt on every read, so probing one costs nothing." _cols = None if _unsloth_is_transformed(_ds) else getattr(_ds, 'column_names', None)\n"" if isinstance(_cols, dict):\n"" _cols = [_c for _v in _cols.values() for _c in (_v or [])]\n"@@ -2902,13 +2594,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" _row = next(_probe, None)\n"" except Exception: return True\n"" return isinstance(_row, dict) and 'input_ids' in _row\n"- # Every rank reaches this before TRL's `_prepare_dataset`, and- # TRL runs its own preparation maps under `main_process_first`.- # Without the same window, eight ranks each start `num_proc`- # workers against one Arrow cache -- 64 processes doing the same- # work, and writing the same cache files at the same time. One- # rank materialises, the rest read what it wrote. A single- # process gets a no-op context manager, so nothing changes there.+ # Every rank reaches this before TRL's _prepare_dataset, which runs its own maps under+ # main_process_first. Without the same window, eight ranks start num_proc workers each against+ # one Arrow cache. A single process gets a no-op context manager." def _unsloth_rank_first():\n"" try:\n"" from accelerate import PartialState\n"@@ -2924,91 +2612,47 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" if not _unsloth_truncatable(_ds): return _ds, not _unsloth_pretokenized(_ds)\n"" _kw = {} if _unsloth_is_stream(_ds) else _unsloth_map_kw\n"" _new = _ds.map(_unsloth_truncate_rows, batched = True, **_kw)\n"- # TRL filters these immediately after truncating: a row whose- # prompt alone fills the cap has every label at -100 and- # contributes no loss, so leaving them in feeds batches with no- # supervised tokens.- # `labels` is only one of the three ways a row says which tokens- # are supervised. A completion-only or assistant-only row carries- # `completion_mask` / `assistant_masks` instead, and truncating a- # long prompt can leave that mask all zeros; TRL's collator turns- # those into all -100 and a batch made of them has no supervised- # token at all, which is a NaN loss rather than a small one.- # A mask is supervision when TRL will actually apply it, and the- # two masks are not symmetric there. DataCollatorForLanguageModeling- # guards `completion_mask` behind `self.completion_only_loss` but- # applies `assistant_masks` on presence alone:- # if self.completion_only_loss and "completion_mask" in examples[0]:- # if "assistant_masks" in examples[0]:- # (trl/trainer/sft_trainer.py, checked on 0.25.1 and v1.9.2). TRL only- # ever BUILDS that column under `assistant_only_loss`, which is why the- # flag looks like the gate, but a pre-tokenized split -- the only kind- # this branch handles -- carries whichever columns the caller put there.- # So gating on the flag left an all-zero assistant mask in place, and- # the collator turned the row into all -100: no supervised token, and a- # batch of them is a NaN loss.- # A None `completion_only_loss` is NOT "on". TRL resolves it from- # the dataset shape:- # if args.completion_only_loss is None:- # self.completion_only_loss = "prompt" in dataset_sample and "completion" in dataset_sample- # (sft_trainer.py:736). A pre-tokenized split has neither column, so- # the effective mode is False and the collator ignores the mask- # entirely; treating None as enabled deleted rows that still had- # valid full-sequence supervision, and could empty the split.+ # TRL filters these right after truncating: a row whose prompt fills the cap is all -100 and+ # contributes no loss. labels is only one of three supervision signals; completion-only and+ # assistant-only rows carry completion_mask / assistant_masks instead.+ # A mask is supervision when TRL will apply it, and the two are not symmetric:+ # DataCollatorForLanguageModeling gates completion_mask on completion_only_loss but applies+ # assistant_masks on presence, so gating on the flag left an all-zero mask and an all -100 row.+ # A None completion_only_loss is NOT "on": TRL resolves it from the dataset shape (prompt plus+ # completion), and a pre-tokenized split has neither, so treating None as enabled deleted rows+ # with valid full-sequence supervision." _unsloth_cols = getattr(_new, 'column_names', None) or ()\n"- # One mode for every split, resolved from the TRAIN sample, because that is- # what the collator uses:- # dataset_sample = next(iter(train_dataset))- # if args.completion_only_loss is None:- # self.completion_only_loss = "prompt" in dataset_sample and "completion" in dataset_sample- # (sft_trainer.py). Resolving it per split disagreed with the collator whenever- # the schemas differ: prompt/completion training data makes the collator apply- # `completion_mask`, while a pre-tokenized eval split carrying only- # `input_ids` and `completion_mask` read here as full-sequence loss, so rows- # whose mask truncated to all zeros survived and went all -100 at eval.+ # One mode for every split, from the TRAIN sample, because that is what the collator uses.+ # Per-split resolution disagreed with it whenever the schemas differ, so rows whose mask+ # truncated to all zeros survived and went all -100 at eval." _unsloth_masks = []\n"" if _unsloth_completion_only and 'completion_mask' in _unsloth_cols:\n"" _unsloth_masks.append('completion_mask')\n"" if 'assistant_masks' in _unsloth_cols:\n"" _unsloth_masks.append('assistant_masks')\n"" try:\n"- # `labels` is unconditional: it IS the supervision.- # One intersection over labels AND every active mask, not a filter each.- # The collator applies the masks ONTO the labels, so a row whose only- # supervised label sits where the mask is zero passes both filters- # separately and still goes out all -100.+ # labels is unconditional: it IS the supervision. Intersect labels AND every active mask+ # rather than filtering each, since masks are applied one after another onto the labels; zip+ # stops at the shorter, which is what an intersection means for a ragged pair." _unsloth_supervision = (['labels'] if 'labels' in _unsloth_cols else []) + _unsloth_masks\n"- # The masks are applied one after another onto the same labels, so- # what survives is their INTERSECTION. Filtering each on its own kept- # rows whose two masks light up in different positions, which TRL then- # labels all -100 -- the very rows this filter exists to drop. zip- # stops at the shorter, which is what an intersection means for a- # ragged pair.+ # The masks are applied one after another onto the same labels, so what survives is their+ # INTERSECTION. Filtering each on its own kept rows whose two masks light up in different+ # positions, which TRL then labels all -100 -- the very rows this filter exists to drop. zip+ # stops at the shorter, which is what an intersection means for a ragged pair." if _unsloth_supervision:\n"" _new = _new.filter(lambda _e, _c = tuple(_unsloth_supervision): any(all((_x != -100) if _n == 'labels' else _x for _n, _x in zip(_c, _v)) for _v in zip(*[_e[_n] for _n in _c])), **_kw)\n"" except Exception:\n"" pass\n"- # Recorded, not raised: the caller wraps these calls in a broad- # `except Exception` that would turn a raise into "could not- # truncate". The raise happens past that handler.+ # Recorded, not raised: the caller wraps these calls in a broad `except Exception` that would turn a+ # raise into "could not truncate", so the raise happens past that handler." try:\n"" if _unsloth_supervision and len(_new) == 0: _unsloth_emptied.append(1)\n"" except TypeError:\n"" pass\n"" return _new, (True if _unsloth_is_stream(_new) else _unsloth_within_cap(_new))\n"- # Resolved BEFORE the try, because the fallback below needs it even when- # the try never ran. `eval_packing` is separate from `packing`:- # packing = args.packing if args.eval_packing is None else args.eval_packing- # (sft_trainer.py), so packing = False with eval_packing = True reaches this- # branch, which is gated on `not args.packing`. TRL then PACKS the eval split- # rather than truncating it, and every strategy owns the overflow: `wrapped`- # concatenates the stream before chunking, `bfd_split` splits an overlength- # example into more chunks. Cutting rows at the cap first throws that away.- # So the packer keeps the split, and the enforcement claim is dropped rather- # than the tokens: `max_length` stays and padding-free turns off. That is- # required anyway, since packing raises on a None `max_length`, and it has to- # hold even with no eval split at construction, because one handed to- # `evaluate()` later would find `max_length` already cleared.+ # Resolved BEFORE the try, since the fallback needs it even when the try never ran.+ # eval_packing is separate from packing, so TRL may PACK the eval split instead of truncating+ # it: keep max_length and turn padding-free off, which packing requires anyway." _unsloth_emptied = []\n"" _unsloth_orig_train = train_dataset\n"" _unsloth_orig_eval = eval_dataset if 'eval_dataset' in locals() else None\n"@@ -3016,38 +2660,20 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" _unsloth_capped = _unsloth_known_mode\n"" if not _unsloth_known_mode:\n"" print('Unsloth: `truncation_mode = ' + str(_unsloth_truncation_mode) + '` is not one of keep_start / keep_end, so `max_length` is not being enforced here.')\n"- # A raw train split is tokenized with the cap by prep, so leave it alone.- # `and`, like the eval splits below. A plain assignment threw away the- # unknown-mode refusal seeded above, so a `truncation_mode` this cannot- # honour was warned about and then silently served as keep_start, with- # `max_length` cleared and padding-free left on.- # `_unsloth_known_mode` too: seeding `_unsloth_capped` false only drops the- # ENFORCEMENT claim. The slice still ran, with `_unsloth_keep_end` false for- # any unknown value, so the fallback scanned an already-trimmed split, found- # it within the cap and merely turned padding-free off -- every row silently- # cut from the start right after warning that the mode could not be honoured.- # Leaving the split alone lets that scan see the real lengths and say so.+ # A raw train split is tokenized with the cap by prep, so leave it alone. `and`, not a plain+ # assignment, which threw away the unknown-mode refusal seeded above and served an unhonoured+ # truncation_mode as keep_start.+ # _unsloth_known_mode too: clearing _unsloth_capped only drops the ENFORCEMENT claim while the+ # slice still ran, so the fallback scanned an already-trimmed split. Leaving the split alone+ # lets that scan see the real lengths." if _unsloth_known_mode and not _unsloth_prep_truncates:\n"" train_dataset, _unsloth_split_ok = _unsloth_cap_split(train_dataset)\n"" _unsloth_capped = _unsloth_capped and _unsloth_split_ok\n"- # An eval split TRL will PACK must not be truncated first. The branch- # is gated on `not args.packing`, but `eval_packing` is resolved- # separately:- # packing = args.packing if args.eval_packing is None else args.eval_packing- # (sft_trainer.py), so packing = False with eval_packing = True reaches- # here. TRL then packs the eval split instead of truncating it- # (`if packing: ... elif args.max_length is not None: truncate_dataset`),- # and the wrapped strategy concatenates the whole token stream before- # chunking it, so cutting each row at the cap first throws away every- # token past it and evaluates on a truncated corpus.- # Leaving it uncut also means the cap is not enforced for that split, and- # packing needs `max_length` anyway ("When packing is enabled,- # `max_length` can't be `None`"), so this drops the enforcement claim- # rather than the split: `max_length` stays and padding-free turns off,- # which is the same answer this block already gives for a split it- # cannot rewrite.- # Each eval split on its own: a raw one stays raw for the tokenizer pass- # that follows, and only a materialised tokenized one is cut.+ # An eval split TRL will PACK must not be truncated first: the branch is gated on+ # `not args.packing` while eval_packing resolves separately, and the wrapped strategy+ # concatenates the stream before chunking. Drop the enforcement claim, not the split.+ # Each eval split on its own: a raw one stays raw for the tokenizer pass that follows, and only a+ # materialised tokenized one is cut." if _unsloth_eval_packing or not _unsloth_known_mode:\n"" _unsloth_capped = False\n"" elif 'eval_dataset' in locals() and eval_dataset is not None:\n"@@ -3061,36 +2687,27 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" eval_dataset, _unsloth_split_ok = _unsloth_cap_split(eval_dataset)\n"" _unsloth_capped = _unsloth_capped and _unsloth_split_ok\n"" _unsloth_prep_truncates = _unsloth_capped\n"- # The splits that WERE rewritten keep their truncation: it is the cap- # the caller asked for, applied exactly as TRL's own truncate_dataset- # would. Rolling them back because a different split cannot be- # rewritten put an overlength train set back and turned a healthy run- # into "cannot be enforced". Only the claim of enforcement is dropped.+ # Splits that WERE rewritten keep their truncation: it is the cap the caller asked for, as+ # TRL's truncate_dataset would apply it. Rolling them back for a sibling that cannot be+ # rewritten put an overlength train set back; only the claim of enforcement is dropped." if not _unsloth_capped:\n"" print('Unsloth: `max_length` cannot be enforced for every split here, so padding-free batching is being turned off instead.')\n"" except Exception as _unsloth_truncate_error:\n"" train_dataset = _unsloth_orig_train\n"" if 'eval_dataset' in locals(): eval_dataset = _unsloth_orig_eval\n"- # The flag is decided from the train split, so a failure while capping- # an eval split would otherwise leave it reading "cap enforced".+ # The flag is decided from the train split, so a failure while capping an eval split would otherwise+ # leave it reading "cap enforced"." _unsloth_prep_truncates = False\n"# Never silent: a swallowed failure here reads as the cap being enforced." print('Unsloth: could not truncate the pre-tokenized dataset to `max_length` (' + str(_unsloth_truncate_error) + ').')\n"- # Outside the handler on purpose. Every row losing its supervised- # tokens means the cap sits below where the supervision starts, and- # an empty split is not something to hand onwards: every TRL 1.x- # reads `next(iter(train_dataset))` in `__init__` to resolve- # `completion_only_loss` and `_is_vision_dataset`, so it comes out- # as a bare `StopIteration` naming nothing at all.+ # Outside the handler on purpose: if every row loses its supervised tokens the cap sits below+ # where supervision starts, and every TRL 1.x reads next(iter(train_dataset)) in __init__, so+ # an empty split surfaces as a bare StopIteration naming nothing." if _unsloth_emptied:\n"" raise ValueError('Unsloth: truncating to `max_length = ' + str(args.max_length) + '` left every row with no supervised token, so there is nothing to train on. The supervised part of your rows starts past that length: raise `max_length`, or set `truncation_mode = \"keep_end\"` if the completion sits at the end of each row.')\n"- # A producer that truncates every row enforces the cap just as- # `truncate_dataset` would, so keep padding-free rather than pay the- # fallback for a guarantee already held. Unsloth's online tokenization- # is this shape: a `with_transform` view capped on read, invisible- # without reading -- and reading it all is the eager pass it avoids.- # Not under `eval_packing`: that split is meant to be overlength and- # the packer needs the `max_length` this branch clears.+ # A producer that truncates every row enforces the cap as truncate_dataset would, so keep+ # padding-free rather than pay the fallback; Unsloth's online tokenization is this shape. Not+ # under eval_packing: that split is overlength on purpose." if not _unsloth_prep_truncates and not _unsloth_eval_packing:\n"" def _unsloth_attests(_ds):\n"" if _ds is None: return True\n"@@ -3108,19 +2725,13 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" args.max_length = None\n"" max_length = None\n"" else:\n"- # Turning padding-free off keeps `max_length` for TRL's collator, and that- # collator does not truncate: for rows that already carry `input_ids`- # nothing downstream enforces the cap. Before this block existed TRL's own- # guard made the same configuration a hard error, so an observed overlength- # row must stay one rather than become a silently uncapped run.- # `skip_prepare_dataset` used to exempt this, which made it the one- # way to get a silently uncapped run: TRL then neither truncates nor- # builds its collator with a truncation length, so the oversized rows- # reach the model. The check is what decides, not the flag.- # An eval split left for the packer is overlength ON PURPOSE, so scanning it- # here turned a working eval-packing run into a hard error and denied- # `wrapped` / `bfd_split` the overflow they exist to handle. The train split- # is still scanned: nothing packs that one.+ # Turning padding-free off keeps max_length for TRL's collator, which does not truncate, so+ # rows already carrying input_ids are unenforced. TRL's own guard used to make this a hard+ # error, so an observed overlength row must stay one rather than run silently uncapped.+ # skip_prepare_dataset used to exempt this, the one way to get a silently uncapped run: TRL+ # then neither truncates nor gives its collator a truncation length.+ # An eval split left for the packer is overlength ON PURPOSE, so scanning it here turned a working+ # eval-packing run into a hard error. The train split is still scanned: nothing packs that one." _unsloth_scan_eval = None if _unsloth_eval_packing else (eval_dataset if 'eval_dataset' in locals() else None)\n"" if not (_unsloth_within_cap(train_dataset) and _unsloth_splits_within_cap(_unsloth_scan_eval)):\n"" raise ValueError('Unsloth: `max_length = ' + str(args.max_length) + '` cannot be enforced. Your dataset already carries `input_ids` and holds rows longer than that, and nothing downstream truncates pre-tokenized rows. Truncate it yourself before passing it in, or drop `max_length`.')\n"@@ -3129,7 +2740,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += max_length_check- # Enable for training and move padding side of tokenizer to right+ # Sync chat_template from processing_class to vLLM's tokenizer This fixes base models that have custom chat+ # templates applied after loadingif "model" in call_args:training_check = ("if model is not None and hasattr(model, 'for_training'):\n"@@ -3143,7 +2755,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += training_check- # Check data collator if it's correct!if "data_collator" in call_args and "train_dataset" in call_args:data_collator_check = ("__tokenizer = processing_class if 'processing_class' in locals() else tokenizer\n"@@ -3168,10 +2779,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += data_collator_check- # Also check if .pad exists -> if not, and is VLM, then change it!- # Only swap LM/Seq2Seq collators; leave preference collators- # (DPODataCollatorWithPadding etc.) alone so ORPO/DPO/CPO/KTO keep- # their own prompt/chosen/rejected handling.+ # Also swap when .pad is missing on a VLM. LM/Seq2Seq collators only: preference collators+ # (DPODataCollatorWithPadding etc.) keep their own prompt/chosen/rejected handling.pad_check = ("if not isinstance(data_collator, UnslothVisionDataCollator):\n"" if not hasattr(__tokenizer, 'pad') and hasattr(__tokenizer, 'tokenizer'):\n"@@ -3190,7 +2799,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += pad_check- # Check NEFTuneif "model" in call_args:neftune_check = ("if hasattr(self, 'neftune_hook_handle'):\n"@@ -3202,7 +2810,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)RLTrainer_post += neftune_check- # Add accelerator scaler to modelif "model" in call_args:accelerator_check = ("if hasattr(self, 'accelerator'):\n"@@ -3216,7 +2823,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)RLTrainer_post += accelerator_check- # Add enabling and disabling training modesif "model" in call_args:training_check = ("if hasattr(self, 'train'):\n"@@ -3225,8 +2831,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)RLTrainer_post += training_check- # Sync chat_template from processing_class to vLLM's tokenizer- # This fixes base models that have custom chat templates applied after loading+ # Sync chat_template from processing_class to vLLM's tokenizer, which fixes base models that have+ # custom chat templates applied after loading.if "model" in call_args:vllm_chat_template_sync = ("if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n"@@ -3238,14 +2844,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)RLTrainer_post += vllm_chat_template_sync- # Edit optional metricsother_metrics_processor = ""if trainer_file in RL_METRICS_CHANGES:process_extra_args = RL_METRICS_CHANGES[trainer_file]for process_extra_arg in process_extra_args:other_metrics_processor += process_extra_arg(old_RLTrainer_source, old_RLConfig_source)- # Add statistics as well!extra_args += ("other_metrics = []\n"f"{other_metrics_processor}\n"@@ -3253,13 +2857,11 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):f"PatchRLStatistics('{trainer_file}', other_metrics)\n")- # Patch optional argsif trainer_file in RL_EXTRA_ARGS:process_extra_args = RL_EXTRA_ARGS[trainer_file]for process_extra_arg in process_extra_args:extra_args += process_extra_arg(call_args, extra_args)- # Create RLTrainer argsextra_args = extra_args.split("\n")extra_args = "\n".join(" " * 8 + x for x in extra_args)RLTrainer_post = RLTrainer_post.split("\n")@@ -3268,18 +2870,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):RLTrainer_extra_args = extra_argsRLTrainer_call_args = call_args- # Fix RLConfig nextarguments, call_args = processed[1]extra_args = ""- # Edit GA / bsz and weight_decayreplacements = {"output_dir": None,"logging_nan_inf_filter": False,"per_device_train_batch_size": 4,"gradient_accumulation_steps": 2,- # LoRA decays A and B toward 0 so effective W = W_init + (alpha/r) * B @ A is pulled toward W_init, not 0 as in full FT.- # 0.001 keeps a small Frobenius prior |A|_F^2 + |B|_F^2 without measurably dragging the merged adapter back to base.+ # LoRA decays A and B toward 0, so W = W_init + (alpha/r) * B @ A is pulled to W_init, not 0+ # as in full FT; 0.001 keeps a small Frobenius prior without dragging the adapter to base."weight_decay": 0.001,"seed": 3407,"optim": "adamw_8bit",@@ -3290,8 +2890,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"logging_steps": 1,"max_seq_length": None,"num_generations": 8,- # "steps_per_generation" : 1, # Otherwise defaults to ga_steps which is wrong- # "generation_batch_size" : None, # Useless. If steps_per_generation set, generation_batch_size clashes+ # steps_per_generation would otherwise default to ga_steps, which is wrong, and+ # generation_batch_size clashes with it."top_k": None,"vllm_mode": "colocate","generation_kwargs": {},@@ -3303,12 +2903,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"auto_find_batch_size": False, # Auto /2 batch size - too many people complained so removing"dataloader_pin_memory": True,"padding_free": None, # None = user didn't set it, allows auto-enable detection- # Might fail so disable for now- # "dataloader_persistent_workers" : True, # Keeps dataloader in RAM- # "dataloader_prefetch_factor" : 2,- # "dataloader_num_workers" : 2, # Default is 0 means 1+ # Might fail, so persistent dataloader workers / prefetch are disabled for now.}- # warmup_ratio deprecated in transformers >= 5.0; warmup_steps accepts float+ # warmup_ratio is deprecated in transformers >= 5.0; warmup_steps accepts a float.if transformers_version >= Version("5.0.0"):replacements["warmup_steps"] = 0.1else:@@ -3320,19 +2917,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):y = f"{k} = {y},\n"arguments = re.sub(x, y, arguments)- # Fix GRPO beta default as 0.001 TRL used to be 0.04, now 0.00!- # https://github.com/huggingface/trl/pull/3516- # https://verl.readthedocs.io/en/latest/examples/config.html+ # GRPO beta default is 0.001: TRL used 0.04 and now 0.00. See huggingface/trl#3516 and the verl docs.if trainer_file == "grpo_trainer":replacements = {"loss_type": "bnpo", # Default GRPO paper"beta": 0.001, # Recommended as seen in verl"auto_find_batch_size": False, # Cannot work on GRPO- # [TODO] See https://fengyao.notion.site/off-policy-rl- # https://github.com/huggingface/trl/pull/3867 (August 7th)+ # See fengyao.notion.site/off-policy-rl and huggingface/trl#3867."vllm_importance_sampling_correction": False,- # TRL >= 1.7.0 enables the MoE router aux loss by default (0.001); the optimized- # GRPO forward does not compute it, so default off. Opt in via router_aux_loss_coef > 0.+ # TRL >= 1.7.0 enables the MoE router aux loss by default (0.001), but the optimized GRPO forward+ # does not compute it, so default it off; opt in via router_aux_loss_coef > 0."router_aux_loss_coef": 0.0,}for k, v in replacements.items():@@ -3341,15 +2935,13 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):y = f"{k} = {y},\n"arguments = re.sub(x, y, arguments)- # TRL >= 1.7.0 defaults SFT to loss_type="chunked_nll" (trl#5846), which patches- # the lm_head and calls the backbone directly. Two problems:- # 1. It bypasses the model forward, so unsloth_fused_ce_loss never runs. Ours- # chunks too and peaks 1.7-3.7GB lower on gemma-3-4b at 141-8192 tokens.- # 2. It divides by num_items_in_batch ignoring model_accepts_loss_kwargs, so- # on models setting that flag False (gemma3, qwen-vl, paligemma, glm4v)- # training_step divides by grad-accum again: loss and grads scaled 1/GA.- # Explicit loss_type= still wins. Keep scoped to sft_trainer: loss_type is an- # unrelated field in DPO/KTO/GRPO and the global dict above is applied to all.+ # TRL >= 1.7.0 defaults SFT to loss_type="chunked_nll" (trl#5846), which patches the lm_head and+ # calls the backbone directly, so unsloth_fused_ce_loss never runs (ours chunks too and peaks+ # 1.7-3.7GB lower on gemma-3-4b at 141-8192 tokens), and it divides by num_items_in_batch ignoring+ # model_accepts_loss_kwargs, so on models setting that flag False (gemma3, qwen-vl, paligemma,+ # glm4v) training_step divides by grad-accum again and loss and grads are scaled 1/GA. Explicit+ # loss_type= still wins. Kept scoped to sft_trainer, since loss_type is an unrelated field in+ # DPO/KTO/GRPO.if trainer_file == "sft_trainer":replacements = {"loss_type": "nll"}for k, v in replacements.items():@@ -3358,7 +2950,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):y = f"{k} = {y},\n"arguments = re.sub(x, y, arguments)- # Warn on too large or too small learning rateif "learning_rate" in call_args:learning_rate_check = ("if learning_rate < 1e-7: print(f'Unsloth: Your learning rate of `{learning_rate}` is too small and less than 1e-7! "@@ -3368,8 +2959,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += learning_rate_check- # Fix num_train_epochs = None causing TypeError in Trainer.__init__- # Trainer does `args.num_train_epochs > 0` which fails when None+ # Fix num_train_epochs = None causing a TypeError in Trainer.__init__, which does `args.num_train_epochs > 0`.if "num_train_epochs" in call_args:num_train_epochs_check = ("if num_train_epochs is None:\n"@@ -3377,7 +2967,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += num_train_epochs_check- # Check if max_seq_length is NOT defined (max_length is now default)+ # Check whether max_seq_length is NOT defined; max_length is now the default.if "max_seq_length" not in call_args and "max_length" in call_args:max_seq_length_pre = """max_seq_length : Optional[int] = field(default = None,@@ -3390,9 +2980,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):max_seq_length_call = ""max_seq_length_post = ""- # Add output_dir savingif "output_dir" in call_args:- # Default checkssaving_check = ("if output_dir is None and save_strategy == 'steps' and save_steps == 500:\n"" output_dir = 'unsloth_training_checkpoints'\n"@@ -3400,22 +2988,13 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += saving_check- # Edit dataset_num_proc- # The policy lives in unsloth_zoo.dataset_num_proc: it had drifted into four- # inline copies, two wrong (stdlib `multiprocessing` asked about a start- # method `datasets` takes from `multiprocess`, and `1` used as the serial- # sentinel when datasets >= 4.1 builds a Pool(1) for it). The zoo rather than- # unsloth, so generated source never imports back into its generator;- # unsloth.dataset_num_proc is the fallback for an older zoo, and the- # ladder is guarded so a stale generated file degrades to the caller's value.- # serial_as_none depends on who reads the value back. Only SFT has a- # downstream auto-sizer: unsloth_zoo.sft_prepare_dataset reads a config- # `None` as "auto-size me", so serial has to be written as `1` there and the- # map() call site (rl_replacements.py) turns it back into None. DPO, KTO,- # CPO, ORPO, Reward and PRM hand args.dataset_num_proc straight to- # Dataset.map, where nothing can inflate a None but a `1` is a Pool(1) on- # datasets >= 4.1 -- one worker with its own tokenizer copy, on a host the- # memory clamp had just declared too small for workers.+ # The worker-count policy lives in unsloth_zoo.dataset_num_proc: it had drifted into four+ # inline copies, two wrong (stdlib multiprocessing start method, and `1` as the serial+ # sentinel when datasets >= 4.1 builds a Pool(1)). In the zoo so generated source never+ # imports back into its generator.+ # serial_as_none depends on the reader: unsloth_zoo.sft_prepare_dataset reads a config None as+ # "auto-size me", so SFT writes serial as 1 and the map() call site turns it back. DPO, KTO,+ # CPO, ORPO, Reward and PRM pass it straight to Dataset.map, where 1 is a Pool(1).if "dataset_num_proc" in call_args:_serial_as_none = "False" if trainer_file == "sft_trainer" else "True"num_proc_check = (@@ -3432,7 +3011,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += num_proc_check- # Add padding if flex attention is addedif "pad_to_multiple_of" in call_args:pad_to_multiple_of = ("if os.environ.get('UNSLOTH_ENABLE_FLEX_ATTENTION', '0') == '1':\n"@@ -3444,10 +3022,11 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += pad_to_multiple_of- # Check for loss_type = dr_grpo and scale_rewards for GRPO+ # Check for loss_type = dr_grpo and scale_rewards for GRPO; DAPO uses per-token loss, so BNPO loss+ # is used. See huggingface/trl#3130 (comment 2746947835).if "loss_type" in call_args and "scale_rewards" in call_args:- # See https://github.com/huggingface/trl/issues/3130#issuecomment-2746947835- # DAPO uses per token loss so BNPO loss used+ # See https://github.com/huggingface/trl/issues/3130#issuecomment-2746947835 DAPO uses per token loss so+ # BNPO loss usedcheck_dr_grpo = ("if loss_type.lower() == 'dr_grpo':\n"" loss_type = 'dr_grpo'\n"@@ -3472,7 +3051,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += check_dr_grpo- # Check GRPO num_generations mismatch+ # Check the GRPO num_generations mismatch; if world size is not set by accelerate or torchrun at this point it is 1.if ("per_device_train_batch_size" in call_argsand "num_generations" in call_args@@ -3506,7 +3085,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += check_num_generations- # Check temperature must not be <= 0. Also stop if >= 10+ # Temperature must not be <= 0, and stop if >= 10.if "temperature" in call_args:check_temperature = ("if temperature <= 0:\n"@@ -3517,22 +3096,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)extra_args += check_temperature- # Edit config with anything extraif trainer_file in RL_CONFIG_CHANGES:process_extra_args = RL_CONFIG_CHANGES[trainer_file]for process_extra_arg in process_extra_args:extra_args += process_extra_arg(old_RLTrainer_source, old_RLConfig_source)- # Create RLConfig argsextra_args = extra_args.split("\n")extra_args = "\n".join(" " * 8 + x for x in extra_args)RLConfig_arguments = argumentsRLConfig_extra_args = extra_argsRLConfig_call_args = call_args- # TRL 0.27.0+ forces use_reentrant=False in gradient_checkpointing_kwargs.- # Unsloth gradient checkpointing requires use_reentrant=True, so we remove- # the setting after super().__init__() when it gets auto-applied.+ # TRL 0.27.0+ forces use_reentrant=False in gradient_checkpointing_kwargs, but Unsloth gradient+ # checkpointing requires True, so remove the setting after super().__init__() applies it.RLConfig_post = ""if trl_version >= Version("0.27.0"):RLConfig_post = (@@ -3542,14 +3118,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" del self.gradient_checkpointing_kwargs['use_reentrant']\n")- # Patch vLLM and other functionsRLTrainer_extras = patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports)if RLTrainer_extras is None:RLTrainer_extras = f"_Unsloth{RLTrainer_name} = {RLTrainer_name}"- # Create full moduleexec(f"from trl.trainer import ({RLTrainer_name}, {RLConfig_name},)")__RLTrainer_doc__ = eval(f"trl.trainer.{RLTrainer_name}").__doc__if __RLTrainer_doc__ is None:@@ -3558,17 +3132,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):if __RLConfig_doc__ is None:__RLConfig_doc__ = ""- # Get all pre-modulesif trainer_file in RL_PRE_ITEMS:RL_pre = "\n".join(RL_PRE_ITEMS[trainer_file])else:RL_pre = ""- # Check if SamplingParams is in thereif "SamplingParams" in old_RLTrainer_source:RL_pre = RL_pre + "\n" + inspect.getsource(vLLMSamplingParams)- # Selective log softmax and other functionsselective_log_softmax_code = inspect.getsource(selective_log_softmax)grpo_selective_log_softmax_code = inspect.getsource(grpo_selective_log_softmax)calculate_pad_tokens_in_prompt_code = inspect.getsource(calculate_pad_tokens_in_prompt)@@ -3578,7 +3149,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):align_completion_tool_mask_code = inspect.getsource(align_completion_tool_mask)autotune_batch_and_chunks_code = inspect.getsource(autotune_batch_and_chunks)sanitize_logprob_code = inspect.getsource(sanitize_logprob)- # Get final source codeRLTrainer_source = RLTrainer_replacement.format(RLTrainer_name = RLTrainer_name,__RLTrainer_doc__ = __RLTrainer_doc__,@@ -3611,19 +3181,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)if RLTrainer_name == "GRPOTrainer":- # Base torch_compile_options shared by all device types+ # Base torch_compile_options shared by all device types; CUDA adds its own, and XPU / HIP / others+ # use the base only.base_options = """torch_compile_options = {"epilogue_fusion" : True,"max_autotune" : False,"shape_padding" : True,"trace.enabled" : False,"""- # Generate torch_compile_options based on device typeif DEVICE_TYPE == "cuda":# CUDA-specific options (added to base options)cuda_options = """"triton.enable_persistent_tma_matmul": torch.cuda.get_device_capability()[0] >= 9,"""- # cutlass options were added in PyTorch 2.8.0+ # cutlass options were added in PyTorch 2.8.0.if torch_version >= Version("2.8.0"):cuda_options += """"cuda.cutlass_epilogue_fusion_enabled": torch.cuda.get_device_capability()[0] >= 9,@@ -3646,10 +3216,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):RLTrainer_source = re.sub(pattern, new_options, RLTrainer_source, flags = re.DOTALL)if trl_version >= Version("1.4.0"):- # The `elif is_peft_model(model) and args.beta != 0.0:` ref-adapter block- # was introduced in TRL 1.4.0 and is used through 1.7.x. Remove only that- # block, anchored on the final ref_param copy so we do NOT also swallow the- # following gradient-checkpointing enable_input_require_grads() block.+ # The `elif is_peft_model(model) and args.beta != 0.0:` ref-adapter block exists from TRL+ # 1.4.0 through 1.7.x. Anchored on the final ref_param copy so the following+ # enable_input_require_grads() block is not swallowed.peft_pattern = (r"\s*elif is_peft_model\(model\) and args\.beta != 0\.0:"r".*?"@@ -3665,9 +3234,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)if trl_version >= Version("1.7.0"):- # router_aux_loss_coef / aux_loss_enabled were added in TRL 1.7.0. Unsloth's- # optimized GRPO forward cannot compute the MoE router aux loss, so reject- # explicit opt-in (router_aux_loss_coef > 0) at init rather than silently ignoring it.+ # router_aux_loss_coef / aux_loss_enabled arrived in TRL 1.7.0, and the optimized GRPO forward+ # cannot compute the MoE router aux loss, so reject an explicit opt-in at init.RLTrainer_source = RLTrainer_source.replace("self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0","self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0\n"@@ -3703,11 +3271,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):flags = re.DOTALL,)- # Remove TRL 0.26.0's unconditional bfloat16 cast of trainable params. It- # hardcodes bfloat16 for QLoRA, ignoring the user's dtype and breaking- # GradScaler with fp16=True. Unsloth already handles adapter dtype via- # patch_model_and_tokenizer, so the block is unnecessary (and already a- # no-op for GRPO, whose peft init block is removed above).+ # Remove TRL 0.26.0's unconditional bfloat16 cast of trainable params: it ignores the user's+ # dtype and breaks GradScaler with fp16=True. patch_model_and_tokenizer already handles it.RLTrainer_source = RLTrainer_source.replace('if getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False):',"if False:",@@ -3727,11 +3292,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):)RLTrainer_source = RLTrainer_source.replace(original_text, new_text)- # Do NOT override _is_vlm -- let TRL detect VLM models naturally- # (forcing _is_vlm=False errors on vision datasets in TRL 0.27.1+).- # But some notebooks pass a bare tokenizer as processing_class, so TRL- # sets _is_vlm=False even for VLMs; add an architecture-based override- # before the validation check.+ # Do NOT override _is_vlm: forcing it False errors on vision datasets in TRL 0.27.1+. A bare+ # tokenizer as processing_class makes TRL set it False even for VLMs, so add an+ # architecture-based override before the validation check._vlm_check_original = (' self._is_vision_dataset = "image" in dataset_sample or "images" in dataset_sample\n'" if self._is_vision_dataset and not self._is_vlm:"@@ -3750,9 +3313,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):if _vlm_check_original in RLTrainer_source:RLTrainer_source = RLTrainer_source.replace(_vlm_check_original, _vlm_check_patched)- # TRL 0.22.x keys off _is_vlm, not _is_vision_dataset (0.24.0+), so the- # vision-only signature columns never overlap the tokenized ones. Merge- # both sets; _remove_unused_columns ignores extras.+ # TRL 0.22.x keys off _is_vlm, not _is_vision_dataset (0.24.0+), so the vision-only signature+ # columns never overlap the tokenized ones. Merge both sets; _remove_unused_columns ignores extras._sig_vlm_old = 'self._signature_columns = ["messages", "prompt", "completion", "images"]'_sig_vlm_new = ('self._signature_columns = ["messages", "prompt", "completion", "images",'@@ -3762,16 +3324,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):RLTrainer_source = _backport_vision_dataset_gate(RLTrainer_source)- # Inject model reference before _prepare_dataset for dynamic- # token_type_ids detection in sft_prepare_dataset+ # Inject the model reference before _prepare_dataset for dynamic token_type_ids detection in+ # sft_prepare_dataset._prep_pattern = r"([ \t]*)train_dataset = self\._prepare_dataset\("_prep_replacement = (r"\1self._unsloth_model_ref = model\n\1train_dataset = self._prepare_dataset(")RLTrainer_source = re.sub(_prep_pattern, _prep_replacement, RLTrainer_source, count = 1)- # Silence TRL's noisy batch_size=1 + padding-free warning (handles both- # the original "anihilate" typo and the corrected "annihilate" spelling)+ # Silence TRL's noisy batch_size=1 + padding-free warning, handling both the original "anihilate"+ # typo and the corrected spelling.for _typo in ("anihilate", "annihilate"):_idx = RLTrainer_source.find(_typo)if _idx == -1:@@ -3792,28 +3354,20 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):RLTrainer_source = RLTrainer_source[:_line_start] + RLTrainer_source[_block_end:]break- # TRL turns a plain `TrainingArguments` into its own config with- # `args = <X>Config(**dict_args)`. That name resolves through the generated- # module's globals, which are imported from TRL before the config is patched- # below, so the conversion hands back a PRISTINE config: no unsloth fields on- # it (#3931 arrives this way too), and a class that no longer answers to its- # own module attribute, so the first checkpoint save cannot pickle it. Build- # the Unsloth subclass instead. Only the construction is rewritten - the- # `isinstance(args, <X>Config)` guard next to it still wants the pristine- # class, which the subclass satisfies.+ # TRL converts a plain TrainingArguments with `args = <X>Config(**dict_args)`, resolved+ # through the generated module's globals, so it hands back a PRISTINE config: no unsloth+ # fields (#3931), and a class the first checkpoint save cannot pickle. Only the construction+ # is rewritten; the isinstance guard still wants the pristine class.RLTrainer_source = RLTrainer_source.replace(f"args = {RLConfig_name}(**dict_args)",f"args = Unsloth{RLConfig_name}(**dict_args)",)- # Remove multiple doc stringsif __RLConfig_doc__ != "" and RLTrainer_source.count(__RLTrainer_doc__) == 2:RLTrainer_source = RLTrainer_source.replace(__RLTrainer_doc__, "", 1)- # Remove multiple newlinesRLTrainer_source = re.sub(r"[\n]{3,}", "\n", RLTrainer_source)- # Create new function_resolved_module = _trainer_resolved_module or _config_resolved_module_model_location = (_resolved_module.__name__ if _resolved_module is not None else f"trl.trainer.{trainer_file}"@@ -3829,7 +3383,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):if trainer_file == "grpo_trainer":_patch_resume_from_checkpoint_memory(patched_trainer)- # Patch Trainerexec(f"trl.{RLTrainer_name} = created_module.Unsloth{RLTrainer_name}",locals(),@@ -3846,7 +3399,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):globals(),)- # Patch Configexec(f"trl.{RLConfig_name} = created_module.Unsloth{RLConfig_name}",locals(),@@ -3863,13 +3415,18 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):globals(),)_displaced_config = None+ # TRL 1.0.0+ wraps generation in: with torch.no_grad(), disable_gradient_checkpointing(self.model, ...): The+ # toggle only suppresses a cosmetic PyTorch warning; under no_grad it has no functional effect. But on exit+ # it calls gradient_checkpointing_enable(), overwriting Unsloth's custom "unsloth" wrapper -- for Gemma-4+ # this corrupts forward numerics and blows GRPO KL divergence up to ~10^12 at step 1. Replacing the context+ # manager with a no-op preserves Unsloth's wrapper. trl < 1.0.0 (no disable_gradient_checkpointing): early+ # return. trl >= 1.0.0: noop is correct; only loss is the cosmetic warning.try:config_module_name = trainer_file.replace("_trainer", "_config")config_module = importlib.import_module(f"trl.trainer.{config_module_name}")if hasattr(config_module, RLConfig_name):- # Remember what this attribute held: on the TRL releases that put a- # thin wrapper here it is a class of its own, and its instances need- # the same pickling fallback the pristine class gets.+ # Remember what this attribute held: on the TRL releases that put a thin wrapper here it is a class+ # of its own, and its instances need the same pickling fallback the pristine class gets._displaced_config = getattr(config_module, RLConfig_name)setattr(config_module,@@ -3915,38 +3472,31 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importinit = inspect.getsource(RLTrainer.__init__)old_init = init- # Remove brackets in comments since it interferes ie (...)+ # Remove brackets in comments, replacing (...) with [...], since they interfere.comments = re.findall(r"\#[^\n]{1,}\n", init)bracketed_comments = [x for x in comments if "(" in x or ")" in x]- # Replace with [...] insteadfor bracketed_comment in bracketed_comments:init = init.replace(bracketed_comment,bracketed_comment.replace("(", "[").replace(")", "]"),)- # Remove peft_configinit = init.replace("elif peft_config is None:", "elif False:")init = init.replace("elif peft_config is not None:", "elif False:")init = init.replace("if peft_config is None:", "if False:")init = init.replace("if peft_config is not None:", "if False:")init = init.replace("get_peft_model(model, peft_config)", "model")- # New TRL 0.20.0init = init.replace("if peft_config is not None or (is_peft_available() and isinstance(model, PeftModel)):","if False:",)- # New TRL 0.20.0init = init.replace("model = self._prepare_peft_model(model, peft_config, args)\n", "pass\n")- # TRL 0.22.0+ uses prepare_peft_model as a standalone function+ # TRL 0.22.0+ uses prepare_peft_model as a standalone function.init = init.replace("model = prepare_peft_model(model, peft_config, args)", "pass")- # Skip add_adapter("ref") for reference model computation- # Unsloth: We comment out the "ref" adapter creation because:- # 1. We want to use the original BASE MODEL as the reference model, not the SFT/LoRA model- # 2. PEFT doesn't allow multiple adapters when target_parameters is used (MoE models)- # When "ref" is not in peft_config, GRPO/RLOO fallback uses disable_adapter()- # which gives the base model logits - exactly what we want+ # Skip add_adapter("ref"): the BASE model is the wanted reference, and PEFT forbids multiple+ # adapters under target_parameters (MoE). Without "ref", GRPO/RLOO falls back to+ # disable_adapter(), which is exactly the base model logits.add_adapter_block_pattern = (r"([ \t]*)" # Capture leading indentationr"if\s+is_peft_available\(\)\s+and\s+is_peft_model\(model\)\s+and\s+args\.beta\s*!=\s*0\.0\s*:"@@ -3960,11 +3510,10 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importindent = match.group(1)lines = full_match.split("\n")commented_lines = []- # Add explanation comment firstcommented_lines.append(f"{indent}# Unsloth: Commented out - use base model as reference, not SFT/LoRA model")- # Comment out each line - insert # after leading whitespace to preserve indentation+ # Comment out each line by inserting # after the leading whitespace, to preserve indentation.for line in lines:if line.strip():stripped = line.lstrip()@@ -3976,9 +3525,8 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importinit = re.sub(add_adapter_block_pattern, comment_out_block, init, flags = re.DOTALL)- # Set use_vllm if not setif "args.use_vllm" in init and "model" in init and "args" in init:- # .*? matches first match. .+? matches final match.+ # .*? matches the first match, .+? the final one.replacer = re.findall(r"def __init__\(.*?\).*?\:\n",init,@@ -3995,16 +3543,13 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import+ " " * 16+ "args.use_vllm = True\n")- # " " * 16 + "args.vllm_importance_sampling_correction = True\n" + \- # " " * 16 + "args.vllm_importance_sampling_cap = 2.0\n"if "grpo" in trainer_file and trl_version >= Version("0.18.0"):- # If model has vllm_engine, then use vllm in colocate mode. Donot wait for server+ # If the model has a vllm_engine, use vllm in colocate mode and do not wait for a server.vllm_setter += " " * 12 + "args.vllm_mode='colocate'\n"if trl_version >= Version("0.23.0"):- # Align TRL sleep mode with the engine's actual enable_sleep_mode- # (the vision standby gate may have disabled it); fall back to the- # standby env var when the engine cannot be introspected.+ # Align TRL sleep mode with the engine's actual enable_sleep_mode (the vision standby gate may have+ # disabled it); fall back to the standby env var when the engine cannot be introspected.vllm_setter += (" " * 12+ "_unsloth_esm = getattr(getattr(getattr(getattr(model.vllm_engine, 'llm_engine', None), 'vllm_config', None), 'model_config', None), 'enable_sleep_mode', None)\n"@@ -4016,8 +3561,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importinit = init.replace(replacer, replacer + vllm_setter)- # breakpoint()-vllm_part = re.findall(r"(\n[\s]{8}" r"if (self|args)\.use_vllm\:.*?" r"\n[\s]{8}" "else:\n)",init,@@ -4026,15 +3569,11 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importif len(vllm_part) == 1:vllm_part, args = vllm_part[0][0], vllm_part[0][1]- # Strip all commentsnew_vllm_part = re.sub(r"^\s*\#[^\n]*\n?", "", vllm_part, flags = re.MULTILINE) # to also remove whole comment line instead of just starting at #- new_vllm_part = re.sub(- r"\s*\#.*$", "", new_vllm_part, flags = re.MULTILINE- ) # remove comments that occur after code+ new_vllm_part = re.sub(r"\s*\#.*$", "", new_vllm_part, flags = re.MULTILINE)- # Get SamplingParamssampling_params = re.findall(r"\n[\s]{4,}(self\.[^\s]{1,}[\s]{0,}\=[\s]{0,}SamplingParams\(.+?\))",new_vllm_part,@@ -4043,14 +3582,13 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importif len(sampling_params) == 1:sampling_params = sampling_params[0]- # Fix guided_decodingsampling_params = sampling_params.replace("guided_decoding=guided_decoding,","guided_decoding="'GuidedDecodingParams(backend="outlines", regex=args.vllm_guided_decoding_regex) ''if getattr(args, "vllm_guided_decoding_regex", None) is not None else None,',)- # Replace with our vLLM engine when sharing weights+ # Replace with our vLLM engine when sharing weights.sampling_params = (" " * 12+ "if getattr(getattr(model, 'vllm_engine', None), 'shared_weights', False): "@@ -4059,7 +3597,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import+ sampling_params)- # count the indentation of last line of sampling_params.splitted_sampling_params = sampling_params.split("\n")if len(splitted_sampling_params) >= 2:last_line = splitted_sampling_params[-1]@@ -4067,9 +3604,8 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importlast_prev_indentation = len(last_prev_line) - len(last_prev_line.lstrip())last_indentation = len(last_line) - len(last_line.lstrip())- # Add extra arguments to SamplingParamsextra = "**getattr(getattr(args, 'vllm_sampling_params', vLLMSamplingParams()), '_set_kwargs', {})"- # Backwards replace+ # Backwards replace.to_replace = (",\n"+ " " * last_prev_indentation@@ -4079,7 +3615,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import+ ")")sampling_params = to_replace.join(sampling_params.rsplit(")", 1))- # Strip multiple commassampling_params = re.sub(r"[\,][\s]{0,}\,", ",", sampling_params)new_vllm_part = (@@ -4087,8 +3622,8 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import)if trl_version >= Version("0.18.0"):- # Guard LLM init - use existing vLLM engine when sharing weights,- # otherwise keep the original LLM() creation for sync/reload path+ # Guard LLM init: use the existing vLLM engine when sharing weights, otherwise keep the original+ # LLM() creation for the sync/reload path.vllm_llm_init_pattern = r"(?P<indent>[ \t]*)self\.llm\s*=\s*LLM\(.*?\)*\)\s*?\n(?!,)"def guard_llm_init(match):@@ -4110,7 +3645,7 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importinit = init.replace(vllm_part, new_vllm_part)- # Search for vLLM calling in all child functions+ # Search for vLLM calls in all child functions.functions = dir(RLTrainer)RLTrainer_source = inspect.getsource(RLTrainer)functions = [x for x in functions if f"def {x}" in RLTrainer_source]@@ -4136,7 +3671,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importcontinueoriginal_source = source- # Check for functionfor edit_function in edit_functions:source = edit_function(function, source)@@ -4153,28 +3687,25 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importsource,)- # llm_model = self.llm.llm_engine.model_executor.driver_worker.model_runner.modelsource = re.sub(r"(\n[\s]{4,}).+?model_executor\.driver_worker.+?\n",r"\n\1pass\n",source,)- # llm_model.load_weights(model.state_dict().items())source = re.sub(r"(\n[\s]{4,}).+?load_weights\(.+?\n",r"\n\1pass\n",source,)- # .state_dict()source = re.sub(r"\.state_dict\(\)",r"",source,)- # Replace self.llm.generate and self.llm.chat with lora_request (only when sharing weights)+ # Replace self.llm.generate and self.llm.chat with lora_request, only when sharing weights.if "CUDA_VISIBLE_DEVICES" in os.environ:lora_name = (trainer_file@@ -4192,19 +3723,14 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import+ r" else None)",source,)- # All these are to fix multiple commas before lora_request (in case the original code ends with something like ",)")- # https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py#L1388 for eg has such an ending+ # Fix multiple commas before lora_request, in case the original code ends with ",)" as trl's+ # grpo_trainer.py#L1388 does.source = re.sub(r"\,[\s]{1,}\,[\s]{0,}lora_request", ", lora_request", source)source = re.sub(r"[\s]{1,}\,[\s]{0,}lora_request", ", lora_request", source)source = re.sub(r"[\,]{1,}[\s]{0,}lora_request", ", lora_request", source)- # Prefer using unsloth's sampling params and fallback to trl's if not found- # We'll enable this later separately when combining both this and GRPOConfig params- # source = re.sub(- # r"sampling_params\s*=\s*sampling_params",- # r"sampling_params = getattr(self.args, 'vllm_sampling_params', sampling_params)",- # source- # )- # Fix later versions of SamplingParams via grpo_update_SamplingParams+ # Prefer Unsloth's sampling params and fall back to trl's; to be enabled once both these and+ # GRPOConfig params are combined.+ # Fix later versions of SamplingParams via grpo_update_SamplingParams.source = source.replace("sampling_params = SamplingParams(**generation_kwargs)","sampling_params = SamplingParams("@@ -4215,11 +3741,9 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import")",)- # Skip if no changes doneif source == original_source:continue- # Find all importsimports += [x for x in all_imports if not x.startswith("_") and x in source]changed[function] = (@@ -4227,10 +3751,8 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importsource,)- # Import all functionsimports = list(set(imports))- # Patch all functionsfor function in changed:old, new = changed[function]RLTrainer_source = RLTrainer_source.replace(old, new)@@ -4242,7 +3764,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, importdef patch_trl_rl_trainers():- # Patch all TRL modules if they have vLLM or PEFTimport trl.trainerall_trainers = dir(trl.trainer)@@ -4258,17 +3779,9 @@ def patch_trl_rl_trainers():def patch_trl_disable_gradient_checkpointing():- # TRL 1.0.0+ wraps generation in:- # with torch.no_grad(), disable_gradient_checkpointing(self.model, ...):- # The toggle only suppresses a cosmetic PyTorch warning; under no_grad it- # has no functional effect. But on exit it calls- # gradient_checkpointing_enable(), overwriting Unsloth's custom- # "unsloth" wrapper -- for Gemma-4 this corrupts forward numerics and- # blows GRPO KL divergence up to ~10^12 at step 1.- #- # Replacing the context manager with a no-op preserves Unsloth's wrapper.- # trl < 1.0.0 (no disable_gradient_checkpointing): early return.- # trl >= 1.0.0: noop is correct; only loss is the cosmetic warning.+ # TRL 1.0.0+ wraps generation in disable_gradient_checkpointing(), which on exit calls+ # gradient_checkpointing_enable() and overwrites Unsloth's "unsloth" wrapper: Gemma-4 forward+ # numerics corrupt and GRPO KL hits ~1e12 at step 1. A no-op CM keeps the wrapper.try:import trl.models.utils as _tmuexcept ImportError:@@ -4290,10 +3803,8 @@ def patch_trl_disable_gradient_checkpointing():_tmu.disable_gradient_checkpointing = _noop_disable_gradient_checkpointing- # Also rebind any trl.* module that already imported the symbol by- # reference (cached at import time). Walk sys.modules dynamically so this- # catches every trainer doing- # `from ...models.utils import disable_gradient_checkpointing`.+ # Also rebind any trl.* module that imported the symbol by reference at import time, walking+ # sys.modules so every `from ...models.utils import disable_gradient_checkpointing` is caught.for _mod_name, _mod in list(sys.modules.items()):if _mod is None or not _mod_name.startswith("trl."):continue@@ -4324,14 +3835,13 @@ def patch_trl_disable_gradient_checkpointing():def patch_trl_openenv():for function in RL_ADDITIONAL_FUNCTIONS["openenv"]:logger.info(f"Unsloth: Patching trl openenv with function: {function.__name__}")- function() # Call the function to apply the patch+ function()returndef patch_trl_vllm_generation():- # trl moved vllm stuff to trl/generation/vllm_generation.py- # We need to min_p patch it to not instantiate another vLLM instance if we already have one with fast_inference- # Find the instance of self.llm = LLM(..) (multiline) and wrap it around an if clause+ # trl moved vllm code to trl/generation/vllm_generation.py; patch it so it does not build a+ # second vLLM instance when fast_inference has one: wrap the multiline `self.llm = LLM(..)`.for function in RL_ADDITIONAL_FUNCTIONS["vllm_generation"]:logger.info(f"Unsloth: Patching trl VLLMGeneration with function: {function.__name__}")function()@@ -4341,16 +3851,12 @@ def patch_trl_vllm_generation():def PatchFastRL(algorithm = None, FastLanguageModel = None):if FastLanguageModel is not None:PatchRL(FastLanguageModel)- # Under UNSLOTH_ALLOW_CPU=1 (CPU-only CI), skip TRL trainer rewriting so- # downstream `inspect.getsource(trl.SFTTrainer)` drift detectors see the- # pristine upstream class, not the compiled Unsloth* wrappers.+ # Under UNSLOTH_ALLOW_CPU=1 (CPU-only CI), skip TRL trainer rewriting so downstream+ # inspect.getsource(trl.SFTTrainer) drift detectors see the pristine upstream class.if os.environ.get("UNSLOTH_ALLOW_CPU", "0") == "1":return- # Install the disable_gradient_checkpointing noop BEFORE- # patch_trl_rl_trainers, which imports extra trl.* submodules; any module- # imported after the sys.modules walk would keep the original broken- # binding. Installing first ensures the canonical symbol is replaced before- # those submodules bind it.+ # Install the disable_gradient_checkpointing noop BEFORE patch_trl_rl_trainers, which imports+ # more trl.* submodules: anything imported after the sys.modules walk keeps the old binding.patch_trl_disable_gradient_checkpointing()patch_trl_rl_trainers()patch_trl_openenv()diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 11ebaaca1c4..3ec736431a4 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -65,7 +65,7 @@ UNSLOTH_GRPO_SEQ_PACKING_ON = os.environ.get("UNSLOTH_GRPO_SEQ_PACKING", "1").lo"no","off",)-# Packing needs zoo#840's masked-column guard in grpo_compute_loss (installed zoo is fixed per-process).+# Packing needs zoo#840's masked-column guard in grpo_compute_loss (the installed zoo is fixed per-process).try:UNSLOTH_ZOO_HAS_MASKED_COL_GUARD = "torch.where(_keep, new" in inspect.getsource(RL_REPLACEMENTS["grpo_compute_loss"]@@ -137,7 +137,6 @@ except Exception:trl_version = Version("0.0.0")-# Check untrained tokensdef sft_trainer_fix_untrained_tokens(call_args, extra_args):if "model" in call_args and "train_dataset" in call_args:fix_tokenizer = (@@ -155,9 +154,8 @@ def sft_trainer_fix_untrained_tokens(call_args, extra_args):RL_EXTRA_ARGS["sft_trainer"].append(sft_trainer_fix_untrained_tokens)-# Fix top_k for GRPO vLLM.-# https://github.com/huggingface/trl/pull/4695 with this change trl added top_k in GRPOConfig and defaults to 0-# We don't want that since vllm's all include top_k is -1 and 0 returns an error on SamplingParams creation.+# huggingface/trl#4695 added top_k to GRPOConfig defaulting to 0, but vLLM's include-all top_k+# is -1 and 0 errors on SamplingParams creation.def grpo_config_fix_vllm_top_k(old_RLTrainer_source, old_RLConfig_source):return "if use_vllm and (top_k is None or top_k == 0): top_k = -1\n"@@ -497,21 +495,17 @@ def _require_replace(return function.replace(old, new, count)-# The one line every unsloth_zoo revision of sft_prepare_dataset ends its worker-# count on, as a regex so a renamed right-hand side still matches and the-# indentation is carried into the replacement. `git log -S` on the Zoo says this-# line last changed in Aug 2025 (#257) and has survived 24 later commits to-# dataset_utils.py, while the block around it was rewritten three times in 2026-# alone (#473, #553, #733) -- which is why it is the fallback rather than the-# primary anchor.+# The one line every unsloth_zoo sft_prepare_dataset ends its worker count on, as a regex so a+# renamed right-hand side still matches and the indentation carries over. Unchanged since Aug+# 2025 (#257) while the block around it was rewritten repeatedly, hence a fallback anchor._ZOO_MAP_NUM_PROC_ASSIGNMENT = re.compile(r"^(?P<indent>[ \t]*)map_kwargs\[\"num_proc\"\][ \t]*=[ \t]*[^\n]+$",flags = re.MULTILINE,)-# The Zoo seeds its truncation length from args.max_length and only falls through-# to args.max_seq_length when that is 0. rl.py now hands TRL >= 1.0.0 a None-# there, so normalise it or the fall-through is skipped and nothing truncates.+# The Zoo seeds its truncation length from args.max_length and falls through to+# args.max_seq_length only when that is 0; rl.py hands TRL >= 1.0.0 a None, so normalise it or+# nothing truncates._ZOO_MAX_LENGTH_SEED = re.compile(r"^(?P<indent>[ \t]*)max_seq_length[ \t]*=[ \t]*getattr\(args,[ \t]*[\"']max_length[\"'],[ \t]*0\)[ \t]*$",flags = re.MULTILINE,@@ -560,14 +554,10 @@ def _replace_or_fallback(`consequence` says what that absence does, since the warning below speaks onlyabout the worker count."""- # Already done upstream, and checked FIRST. An edit is only missing if its- # RESULT is missing, and a Zoo that adopts the replacement itself is the- # forward case this has to survive. Ordering matters twice over: `old` is a- # prefix of `new` for the max_length seed, so the wide anchor below matches- # the already-normalized line and appended a second `or 0` to it, while a- # differently spelled upstream form matches no anchor at all and used to- # raise under `required = True` -- failing every SFT trainer over behaviour- # that is already present.+ # Already done upstream, and checked FIRST: an edit is missing only if its RESULT is, and a Zoo+ # that adopts the replacement is the forward case. Order matters: `old` is a prefix of `new`+ # for the max_length seed, so the wide anchor matched the normalized line and appended a+ # second `or 0`.if _same_source(new) in _same_source(function):return functionif old in function:@@ -611,13 +601,10 @@ def sft_trainer_prepare_dataset(function_name, function):flags = re.MULTILINE | re.DOTALL,)if matched:- # Use fast version!function = inspect.getsource(fast_sft_prepare_dataset)- # why: anchor the wrapped-packing setup on the function signature -- a- # structural anchor that always exists -- not the unsloth_zoo license comment,- # which is only lower-bounded and a newer Zoo may move or drop. Anchoring there- # let the setup silently no-op while edits below referenced its variables,- # NameError-ing every SFT dataset prep. Fail loudly if the signature is missing.+ # Anchor the wrapped-packing setup on the function signature, which always exists, not the+ # unsloth_zoo license comment, which a newer Zoo may move: anchoring there let the setup+ # silently no-op while later edits used its variables, NameError-ing every SFT dataset prep.function, _n_setup = re.subn(r"(def sft_prepare_dataset\s*\(.*?\)\s*(?:->[^:\n]*)?:[ \t]*\n)",lambda match: match.group(1) + _WRAPPED_PACKING_SETUP,@@ -637,19 +624,16 @@ def sft_trainer_prepare_dataset(function_name, function):fallback_pattern = _ZOO_MAX_LENGTH_SEED,fallback_new = r'\g<indent>max_seq_length = getattr(args, "max_length", 0) or 0',where = "sft_prepare_dataset max_length seed",- # Not optional, unlike the worker count this helper was written for.- # The generated trainer clears `args.max_length` for padding-free, so- # an unrewritten seed reads that `None` instead of the `0` that makes- # the guard below fall through to `max_seq_length`: raw datasets stop- # being truncated, or `None > 0` raises. Both anchors missing means- # the neighbouring _require_replace edits have almost certainly gone- # too, so this raises where they do rather than one call later.+ # Not optional, unlike the worker count this helper was written for: the generated trainer+ # clears args.max_length for padding-free, so an unrewritten seed reads None instead of the 0+ # that falls through to max_seq_length. Both anchors missing means the neighbouring+ # _require_replace edits have gone too.required = True,consequence = ", so `max_length` would not be enforced for raw ""datasets under padding-free batching",)- # why: route each edit through _require_replace so a drifted anchor fails- # loudly instead of leaving a dangling reference to the setup variables.+ # Route each edit through _require_replace so a drifted anchor fails loudly instead of leaving+ # a dangling reference to the setup variables.function = _require_replace(function,"truncation = do_truncation,",@@ -662,9 +646,9 @@ def sft_trainer_prepare_dataset(function_name, function):"if do_truncation and not _unsloth_wrapped_packing and max_seq_length > 0:",where = "sft_prepare_dataset truncation guard",)- # why: reuse the guarded _unsloth_pack_has_strategy from the setup instead of- # re-calling _inspect.signature(pack_dataset) here -- the setup wraps that call- # in try/except, so a non-introspectable pack_dataset must not crash here.+ # Reuse the guarded _unsloth_pack_has_strategy from the setup rather than re-calling+ # _inspect.signature(pack_dataset): the setup wraps that in try/except, so a+ # non-introspectable pack_dataset must not crash here.function = _require_replace(function,"""dataset = pack_dataset(@@ -683,14 +667,10 @@ def sft_trainer_prepare_dataset(function_name, function):)""",where = "sft_prepare_dataset pack_dataset call",)- # why: the map() call site -- not the config -- is where the worker- # count must be made safe. The Zoo copy asks stdlib `multiprocessing`- # for a start method datasets takes from `multiprocess`, and its- # low-memory branch yields 1, which still builds a Pool(1) on- # datasets >= 4.1. The shared helper caps a config None instead of- # the Zoo's uncapped `cpu_count + 4 -> 64`. Imported from the Zoo so- # generated source never imports back into its generator;- # unsloth.dataset_num_proc is only the fallback for an older Zoo.+ # The map() call site, not the config, is where the worker count is made safe: the Zoo copy+ # asks stdlib multiprocessing for a start method datasets takes from multiprocess, and its+ # low-memory branch yields 1, still a Pool(1) on datasets >= 4.1. Imported from the Zoo so+ # generated source never imports back into its generator.function = _replace_or_fallback(function,"""if not isinstance(dataset, IterableDataset):@@ -716,17 +696,10 @@ def sft_trainer_prepare_dataset(function_name, function):map_kwargs["num_proc"] = _unsloth_get_dataset_num_proc(getattr(args, "dataset_num_proc", None))""",- # This block is the likeliest anchor in the file to drift -- the- # Zoo rewrote it three times in 2026 -- but its absence is not- # harmless, so it cannot simply be optional: the config layer- # encodes serial as `1` for this site to turn back into None, and- # an un-rewritten Zoo hands that `1` to Dataset.map. The fallback- # keys on the assignment the block ends with, which has not- # changed since Aug 2025, and reads args rather than the Zoo's- # own local so it is the same computation the block anchor above- # installs; the Zoo's now-dead sizing runs and is discarded.- # Hard-failing instead would break every install on a newer Zoo,- # which is a far larger blast radius than one worker.+ # The likeliest anchor here to drift, but its absence is not harmless: the config layer encodes+ # serial as 1 for this site to turn back into None, and an un-rewritten Zoo hands that 1 to+ # Dataset.map. The fallback keys on the block's closing assignment, unchanged since Aug 2025.+ # Hard-failing instead would break every install on a newer Zoo.fallback_pattern = _ZOO_MAP_NUM_PROC_ASSIGNMENT,fallback_new = r"""\g<indent>try:\g<indent> from unsloth_zoo.dataset_num_proc import get_dataset_num_proc as _unsloth_get_dataset_num_proc@@ -737,12 +710,9 @@ def sft_trainer_prepare_dataset(function_name, function):\g<indent>)""",where = "sft_prepare_dataset dataset_num_proc selection",)- # why: datasets never reads the child's exit status, so every worker- # death -- OOM kill, segfault, real exception -- flattens into "One- # of the subprocesses has abruptly died during map operation". Wrap- # both tokenizing map() calls (count = 2: the prompt-completion and- # plain-text branches) so the user gets the worker count, start- # method and implied memory instead.+ # datasets never reads the child's exit status, so every worker death flattens into "One of the+ # subprocesses has abruptly died during map operation". Wrap both tokenizing map() calls so the+ # user gets the worker count, start method and implied memory.function = _require_replace(function,""" with _w.catch_warnings():@@ -755,11 +725,9 @@ def sft_trainer_prepare_dataset(function_name, function):_w.filterwarnings("ignore", message=".*couldn't be hashed properly.*")""",count = 2,where = "sft_prepare_dataset tokenizing map() calls",- # required = False, like the selection above: this only improves the- # message a dead worker produces, and a diagnostic must not be able- # to fail a training run because a Zoo release moved the line it- # anchors on. test_zoo_sft_prepare_dataset_anchor_has_not_drifted is- # what notices, in CI rather than in someone's run.+ # required = False, like the selection above: this only improves a dead worker's message, and a+ # diagnostic must not fail a run because a Zoo release moved its anchor.+ # test_zoo_sft_prepare_dataset_anchor_has_not_drifted notices in CI instead.required = False,)function = function.split("\n")@@ -800,7 +768,7 @@ def sft_trainer_prepare_dataset(function_name, function):check_text = "\n".join(" " * 8 + x for x in check_text)check_text = check_text.rstrip() + "\n"- # .*? matches first match. .+? matches final match.+ # .*? matches the first match, .+? the final one.replacer = re.findall(r"def " + function_name + r"\(.*?\).*?\:\n",function,@@ -823,8 +791,7 @@ def sft_trainer_prepare_dataset(function_name, function):RL_FUNCTIONS["sft_trainer"].append(sft_trainer_prepare_dataset)-# Ignore mean_token_accuracy since it needs logits-# We override it directly with our version+# Ignore mean_token_accuracy since it needs logits; it is overridden with our version.def sft_trainer_compute_loss(function_name, function):if function_name != "compute_loss":return function@@ -851,8 +818,8 @@ def sft_trainer_compute_loss(function_name, function):RL_FUNCTIONS["sft_trainer"].append(sft_trainer_compute_loss)-# Route ORPO/CPO row tokenization through the underlying text tokenizer when the-# processing class is a multimodal processor; CPO reuses this code (#4952).+# Route ORPO/CPO row tokenization through the underlying text tokenizer when the processing+# class is a multimodal processor; CPO reuses this code (#4952).def orpo_trainer_text_tokenizer(function_name, function):if function_name == "build_tokenized_answer":function = re.sub(@@ -898,11 +865,9 @@ RL_FUNCTIONS["orpo_trainer"].append(orpo_trainer_text_tokenizer)RL_FUNCTIONS["cpo_trainer"].append(orpo_trainer_text_tokenizer)-# Resolve `processing_class.pad_token_id` through the underlying tokenizer when-# a multimodal processor is supplied (processors lack `pad_token_id`). Without-# this, ORPO/CPOTrainer.__init__ raises AttributeError on-# `DPODataCollatorWithPadding(pad_token_id=processing_class.pad_token_id, ...)`-# and on `self.padding_value = ... else processing_class.pad_token_id`.+# Resolve processing_class.pad_token_id through the inner tokenizer when a multimodal processor+# is supplied: processors lack pad_token_id, so ORPO/CPOTrainer.__init__ raises AttributeError+# in the collator and padding_value._PAD_FALLBACK = ("(getattr(processing_class, 'pad_token_id', None) ""if getattr(processing_class, 'pad_token_id', None) is not None "@@ -913,12 +878,9 @@ _PAD_FALLBACK = (def orpo_trainer_processor_pad_token(function_name, function):if function_name != "__init__":return function- # Multimodal processors (e.g. Gemma3/Gemma4 Processor) expose pad_token /- # eos_token on `.tokenizer`, not on the processor itself. TRL 1.x CPO/ORPO- # __init__ defaults `processing_class.pad_token` from `.eos_token` before- # tokenizing, which AttributeErrors on such a processor. Route the default- # through the inner tokenizer. Older TRL lacks this block, so the sub is a- # no-op there and only the pad_token_id fallback below applies.+ # Multimodal processors expose pad_token / eos_token on .tokenizer, not on themselves, and TRL+ # 1.x CPO/ORPO __init__ defaults pad_token from eos_token before tokenizing. Older TRL lacks+ # this block, so the sub is a no-op there.function = re.sub(r"(?m)^([ \t]*)if processing_class\.pad_token is None:\n"r"\1[ \t]+processing_class\.pad_token\s*=\s*processing_class\.eos_token\n",@@ -937,8 +899,8 @@ RL_FUNCTIONS["orpo_trainer"].append(orpo_trainer_processor_pad_token)RL_FUNCTIONS["cpo_trainer"].append(orpo_trainer_processor_pad_token)-# Fix bare pop("push_to_hub_token") in compiled SFT/IterativeSFT trainer __init__-# On transformers 5.0+, to_dict() no longer includes push_to_hub_token, so bare pop KeyErrors+# Fix the bare pop("push_to_hub_token") in the compiled SFT/IterativeSFT __init__: on+# transformers 5.0+ to_dict() no longer includes it, so a bare pop KeyErrors.def sft_trainer_push_to_hub_token(function_name, function):if function_name != "__init__":return function@@ -950,7 +912,7 @@ def sft_trainer_push_to_hub_token(function_name, function):RL_FUNCTIONS["sft_trainer"].append(sft_trainer_push_to_hub_token)-# Autocast precision for GRPO+# Autocast precision for GRPO.def _unsloth_grpo_autocast(self):"""Decide the GRPO autocast once and latch it on the trainer.@@ -963,25 +925,25 @@ def _unsloth_grpo_autocast(self):use_bf16 = getattr(args, "bf16", None)use_fp16 = getattr(args, "fp16", None)if not isinstance(precision, str):- # transformers < 5 has no args.mixed_precision, but rl.py sets the- # fp16 / bf16 flags on this same args for every branch it takes.+ # transformers < 5 has no args.mixed_precision, but rl.py sets the fp16 / bf16 flags on this+ # same args for every branch it takes.if isinstance(use_bf16, bool) and isinstance(use_fp16, bool):precision = "bf16" if use_bf16 else ("fp16" if use_fp16 else "no")else:precision = os.environ.get("ACCELERATE_MIXED_PRECISION", "fp16")self._autocast_dtype = torch.float16 if precision == "fp16" else torch.bfloat16- # "no" is a real value: full finetuning and an explicit float32 load- # both set it, and reading it as bfloat16 raises on a T4 or V100.+ # "no" is a real value: full finetuning and an explicit float32 load both set it, and reading+ # it as bfloat16 raises on a T4 or V100.self._autocast_enabled = precision != "no"self._autocast_force_float32 = False- # Stamped by from_pretrained: UNSLOTH_FORCE_FLOAT32 is process wide, so a- # model loaded after this trainer was built would answer for it here.+ # Stamped by from_pretrained: UNSLOTH_FORCE_FLOAT32 is process wide, so a model loaded after+ # this trainer was built would answer for it here.forced = getattr(getattr(self, "model", None), "_unsloth_forced_float32", None)if forced is None:forced = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"if forced and precision != "bf16":- # Gemma3 / gpt-oss set "no" but still want float16 autocast. A trainer- # already on bf16 keeps it: float16 is what the forced list avoids.+ # Gemma3 / gpt-oss set "no" but still want float16 autocast; a trainer already on bf16 keeps+ # it, since float16 is what the forced list avoids.self._autocast_dtype = torch.float16self._autocast_enabled = Trueself._autocast_force_float32 = True@@ -995,8 +957,8 @@ def _unsloth_grpo_autocast_kwargs(self, device_type = "cuda"):if not getattr(self, "_autocast_force_float32", False) and torch.is_autocast_enabled(device_type):- # Already inside an autocast: inherit its dtype by omitting the key, since- # autocast passes whatever it gets straight to set_autocast_dtype.+ # Already inside an autocast: inherit its dtype by omitting the key, since autocast passes+ # whatever it gets to set_autocast_dtype.return {"enabled": enabled}return {"enabled": enabled, "dtype": dtype}@@ -1005,8 +967,7 @@ def grpo_trainer__prepare_inputs(function_name, function):if function_name != "_prepare_inputs":return function- # Latched on the trainer, so a second trainer's __init__ cannot change this- # trainer's autocast mid run.+ # Latched on the trainer, so a second trainer's __init__ cannot change this trainer's autocast mid run.function = function.replace("with torch.inference_mode():","with torch.inference_mode(), "@@ -1022,8 +983,8 @@ def grpo_trainer__prepare_inputs(function_name, function):RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__prepare_inputs)-# Guard reload_weights and sync_weights - skip when fast inference LoRA shares weights with vLLM-# https://github.com/huggingface/trl/commit/7856d3b1f6518601732f489883b341bb6dd36434#diff-964e6fd373aa93037604064cb2b822d7f8e2735e33f791065acf2c4c3552d393R1168-R1169+# Guard reload_weights and sync_weights: skip when fast inference LoRA shares weights with+# vLLM (huggingface/trl commit 7856d3b).def _guard_vllm_sync_reload_for_shared_weights(function):# Guard reload_weights - only call when not sharing weights with vLLMreload_weights_pattern = re.compile(@@ -1065,22 +1026,10 @@ def grpo_trainer__generate_single_turn(function_name, function):function = _guard_vllm_sync_reload_for_shared_weights(function)- # TRL 0.24.0-0.25.1 truncation regression fix- #- # TRL 0.22.2-0.23.1 used smart truncation via truncate_with_protected_tokens():- # - Tokenizes first without truncation- # - Then truncates keeping the RIGHTMOST tokens (preserves assistant turn)- # - Protects special tokens (image_token, vision_start/end) from removal- #- # TRL 0.24.0-0.25.1 removed this and passed kwargs directly to the tokenizer:- # max_length=self.max_prompt_length, truncation=True, add_special_tokens=False- # This causes issues because tokenizer truncation doesn't protect special tokens- # and may not preserve the end of the prompt properly.- #- # TRL 0.26.2+ removed these kwargs entirely (no tokenizer-level truncation).- #- # Fix: Remove these kwargs so TRL 0.24.0-0.25.1 behaves like 0.26.2+ (no truncation).- # This is a no-op for versions that don't have these kwargs (0.22.2-0.23.1, 0.26.2+).+ # TRL 0.24.0-0.25.1 truncation regression: 0.22.2-0.23.1 used truncate_with_protected_tokens+ # (tokenize, keep the RIGHTMOST tokens, protect vision tokens), 0.24.0-0.25.1 passed+ # max_length/truncation to the tokenizer, which protects nothing, and 0.26.2+ removed those+ # kwargs. Dropping them makes 0.24.0-0.25.1 behave like 0.26.2+; a no-op elsewhere.for pattern in [r'["\']?max_length["\']?\s*[:=]\s*self\.max_prompt_length\s*,\s*\n?',r'["\']?truncation["\']?\s*[:=]\s*True\s*,\s*\n?',@@ -1119,7 +1068,8 @@ def grpo_trainer__generate(function_name, function):RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__generate)-# Fix incorrect special tokens handling and truncation in older TRL versions+# Older TRL mishandles special tokens: 0.19.0 passed skip_special_tokens = True where it+# should be False.def grpo_trainer__generate_and_score_completions(function_name, function):if function_name != "_generate_and_score_completions":return function@@ -1130,10 +1080,10 @@ def grpo_trainer__generate_and_score_completions(function_name, function):"prompt_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False",)- # Left pad prompt before calculation old and ref hidden states+ # Left pad the prompt before calculating old and ref hidden states.line_to_replace = 'batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size'- # The new multi-line string that will replace the line above+ # The new multi-line string that replaces the line above.replacement_lines = """max_left_pad = Nonebatch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size@@ -1165,7 +1115,6 @@ def grpo_trainer__generate_and_score_completions(function_name, function):if self.args.gradient_accumulation_steps % generate_every != 0 or (self.use_vllm):"""- # Use re.sub() to perform the replacementfunction, num_replacements = pattern_to_find.subn(replacement_text, function)pattern_to_find = re.compile(@@ -1176,8 +1125,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function):re.DOTALL | re.MULTILINE,)- # sanitize_logprob is injected as a module-level function via RLTrainer_replacement- # template in rl.py (from RL_REPLACEMENTS), so just reference it directly here.+ # sanitize_logprob is injected as a module-level function by the RLTrainer_replacement template+ # in rl.py, so reference it directly.replacement_text = (r"\1all_logprobs = [\n"r"\1 [sanitize_logprob(next(iter(logprob.values()))) for logprob in output.logprobs]\n"@@ -1188,7 +1137,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function):function, num_replacements = pattern_to_find.subn(replacement_text, function)- # Always between max_prompt_length and use_vllm+ # Always between max_prompt_length and use_vllm.found = re.findall(r"\n(([ ]{8,})if self\.max_prompt_length is not None:.*?\2if self\.use_vllm:)",function,@@ -1226,9 +1175,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function):if self.use_vllm:"""function = function.replace(replace_part, new_replacement)- # Important note: we disable TRL's importance sampling logic- # It is disabled because the LLM path moves left padding to the right.- # We must adjust the vLLM sampling_logprob tensor in Unsloth to account for this.+ # TRL's importance sampling is disabled because the LLM path moves left padding to the right,+ # so Unsloth adjusts the vLLM sampling_logprob tensor itself.string_to_find = "if self.use_vllm and self.vllm_importance_sampling_correction:"replacement_string = "if False and self.use_vllm and self.vllm_importance_sampling_correction:"@@ -1250,8 +1198,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function):function = function.replace(string_to_find, replacement_string)- # TRL 0.24.0+ extracts prompts = [x["prompt"] for x in inputs], losing metadata- # like reasoning_effort. Inject code to store per-sample chat_template_kwargs on self.+ # TRL 0.24.0+ extracts prompts = [x["prompt"] for x in inputs], losing metadata like+ # reasoning_effort, so inject code storing per-sample chat_template_kwargs on self._metadata_extraction = ("\n"" # Unsloth: Extract per-sample chat_template_kwargs before metadata is lost\n"@@ -1267,7 +1215,6 @@ def grpo_trainer__generate_and_score_completions(function_name, function):" _kw_[_k_] = _inp_[_k_]\n"" self._unsloth_batch_chat_kwargs.append(_kw_)\n")- # Insert after: prompts = [x["prompt"] for x in inputs]_target_line = 'prompts = [x["prompt"] for x in inputs]'if _target_line in function:function = function.replace(@@ -1275,7 +1222,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function):_target_line + _metadata_extraction,)- # This path is for TRL 0.24.0 images is a variable exclusive to this version+ # This path is for TRL 0.24.0: `images` is a variable exclusive to that version.string_to_find = """ if images is not None:output["num_images"] = num_images"""@@ -1292,7 +1239,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function):function = function.replace(string_to_find, replacement_string)if trl_version >= Version("0.24.0"):- # We replace the call using 'completions' with one using 'completions_text'+ # Replace the call using 'completions' with one using 'completions_text'.string_to_find = " rewards_per_func = self._calculate_rewards(inputs, prompts, completions, completion_ids_list)"replacement_string = (" if images is not None:\n"@@ -1315,14 +1262,13 @@ def grpo_trainer__generate_and_score_completions(function_name, function):)if "wake_up()" not in function:- # Sleep functionality has been added to trl in v0.23.0. We do not want to redo this.- # https://github.com/huggingface/trl/commit/edbe8234bc7e528f72ac76607de9d3e4753e2709+ # Sleep functionality was added to trl in v0.23.0 (commit edbe823), so do not redo it.pattern = re.compile(r".*self\.llm\.generate\(.*\).*", re.MULTILINE)matches = list(pattern.finditer(function))patched = function- # Generally there's only one match. But this is just to make sure we don't miss any.+ # Generally there is only one match; the loop is to make sure none are missed.for match in reversed(matches):line = match.group(0)indent_match = re.match(r"(\s*)", line)@@ -1449,7 +1395,6 @@ def grpo_trainer_fix_maybe_apply_chat_template(function_name, function):RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_fix_maybe_apply_chat_template)-# Remove _move_model_to_vllmdef grpo_trainer__move_model_to_vllm(function_name, function):if function_name != "_move_model_to_vllm":return function@@ -1464,7 +1409,7 @@ def grpo_trainer__move_model_to_vllm(function_name, function):RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__move_model_to_vllm)-# Edit _get_per_token_logps to handle mixed precision+# Edit _get_per_token_logps to handle mixed precision.def grpo_trainer__get_per_token_logps(function_name, function):if function_name != "_get_per_token_logps":return function@@ -1479,7 +1424,6 @@ def grpo_trainer__get_per_token_logps(function_name, function):):if True: # os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0':return None # Unsloth efficient GRPO- # Otherwise, calculate normally:_unsloth_grpo_autocast(self)os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"@@ -1488,30 +1432,15 @@ def grpo_trainer__get_per_token_logps(function_name, function):dtype = self._autocast_dtype,enabled = getattr(self, "_autocast_enabled", True),):- # We add 1 to `logits_to_keep` because the last logits of the sequence is later excluded+ # logits_to_keep gets 1 added because the last logit of the sequence is excluded later.logits = model(input_ids = input_ids,attention_mask = attention_mask,logits_to_keep = logits_to_keep + 1,).logits- # logits = logits[:, :-1, :] # (B, L-1, V), exclude the last logit: it corresponds to the next token predreturn logits- # input_ids = input_ids[:, -logits_to_keep:]- # For transformers<=4.48, logits_to_keep argument isn't supported, so here we drop logits ourselves.- # See https://github.com/huggingface/trl/issues/2770- # logits = logits[:, -logits_to_keep:]- # return logits- # See https://huggingface.co/blog/the_n_implementation_details_of_rlhf_with_ppo#policy-training-implementation-details- # logits = logits / self.temperature- # logps = selective_log_softmax(logits, input_ids)-- # row_indices, col_indices = torch.where(logps < -20)-- # # Method 1: Check if tensors have elements- # if len(row_indices) > 0 and len(col_indices) > 0:- # breakpoint() # Breakpoint triggered here- # print("Found high values!")- # return logps # compute logprobs for the input tokens+ # transformers <= 4.48 does not support logits_to_keep, so drop the logits here; see+ # huggingface/trl#2770.function = inspect.getsource(_get_per_token_logps)return function@@ -1524,7 +1453,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):if function_name != "_get_per_token_logps_and_entropies":return function- # Just copy over from _get_per_token_logps replacement function above. For now this returns None anyway+ # Copied from the _get_per_token_logps replacement above; for now this returns None anyway.def _get_per_token_logps_and_entropies(self,model,@@ -1537,9 +1466,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):*args,**kwargs,):- # All Unsloth code here in this function is licensed under AGPL3- # if True: # os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0':- # return None, None # logps, entropies Unsloth efficient GRPO+ # All Unsloth code in this function is licensed under AGPL3.if compute_efficient:return None, Noneelse:@@ -1556,7 +1483,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):kwargs.get("image_sizes", None),)num_images = kwargs.get("num_images", None)- # Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models+ # Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models.token_type_ids = kwargs.get("token_type_ids", None)mm_token_type_ids = kwargs.get("mm_token_type_ids", None)if mm_token_type_ids is not None or image_grid_thw is not None:@@ -1568,8 +1495,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):lm_head = self.model.get_output_embeddings().weight- # Size on the dtype the forward actually runs in: with autocast off that- # is the model's own dtype, float32 or bfloat16 depending on the load.+ # Size on the dtype the forward actually runs in: with autocast off that is the model's own dtype.forward_dtype = (self._autocast_dtype if getattr(self, "_autocast_enabled", True) else lm_head.dtype)@@ -1624,8 +1550,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):rows_per_image = image_grid_thw.prod(dim = -1)rows_per_sample = torch.split(rows_per_image, num_images)rows_per_sample = torch.stack([s.sum() for s in rows_per_sample])- # why: cum_rows is indexed via .item() inside the per-chunk loop;- # keeping it on CPU avoids per-iteration GPU->CPU sync.+ # cum_rows is indexed via .item() inside the per-chunk loop, so keeping it on CPU avoids a+ # per-iteration GPU->CPU sync.cum_rows = torch.cat([torch.tensor([0], device = rows_per_sample.device),@@ -1660,7 +1586,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):mm_token_type_ids_chunks = []current_pixel_idx = 0- # TRL 0.23.0 batching logic+ # TRL 0.23.0 batching logic.for start in range(0, total_samples, batch_size):end = min(start + batch_size, total_samples)@@ -1725,8 +1651,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):temperature = self.temperaturemodel_config = _unsloth_get_model_config(model)if detect_logit_transforms is not None:- # model_config, not model: under DDP/Accelerate `model` is a wrapper that- # does not forward `.config`, so the helper would report zeros.+ # model_config, not model: under DDP/Accelerate `model` is a wrapper that does not forward+ # .config, so the helper would report zeros._transforms = detect_logit_transforms(model_config)logit_softcapping = _transforms["logit_softcapping"]logit_scale_multiply = _transforms["logit_scale_multiply"]@@ -1752,27 +1678,24 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):)os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"+ # Sequence packing (default on; UNSLOTH_GRPO_SEQ_PACKING=0 disables): one varlen [1, sum L]+ # forward replaces the padded [B, Lmax] loop and fixes the left-pad RoPE error. Self-verified+ # against the per-row forward, re-checked as T grows, and falls back if a backend ignores+ # packed_seq_lengths.# ---- Sequence packing (default-on; disable with UNSLOTH_GRPO_SEQ_PACKING=0) ----- # One varlen [1, sum L] forward replaces the padded [B, Lmax] loop (also fixes the- # left-pad RoPE error). Self-verified against the per-row forward, re-checked as T- # grows; falls back if a backend ignores packed_seq_lengths.logprobs = None- # ---- PrefixGrouper (GRPO shared-prompt dedup; default ON, exact + self-verified) ----- # G completions per prompt share the prefix; the packed path forwards it G times,- # PrefixGrouper stores it once (FlexAttention shared-prefix mask), cutting the trunk- # forward from G*(P+R) to P+G*R tokens. Gated by UNSLOTH_GRPO_PREFIX_GROUPER (needs- # seq-packing), tok_r auto-gate, and first-use self-verify vs the packed path- # (mismatch => fall back + mark unsafe), so a mask/isolation regression cannot ship- # silently. When off / ungrouped / unverified, the packed path below runs as before.+ # PrefixGrouper (GRPO shared-prompt dedup, default ON): G completions share the prompt, so+ # storing it once behind a FlexAttention shared-prefix mask cuts the trunk forward from+ # G*(P+R) to P+G*R tokens. Gated by UNSLOTH_GRPO_PREFIX_GROUPER, a tok_r auto-gate and a+ # first-use self-verify, so a mask/isolation regression cannot ship silently._pg_result = None_pg_use = False_pg_skip_pk = False # once a shape is PG-verified, skip the full-row forward_pg_forward_fn = None # deferred PG forward (runs at the verify site below)_pg_num_gen = getattr(self, "num_generations", None)- # Env gate hoisted to module level (mirrored via RL_PRE_ITEMS). Skip PG under vLLM- # (fast_inference=True): the rollout dominates the step, so PG saves little and its- # first-use self-verify is net overhead.+ # Env gate hoisted to module level (mirrored via RL_PRE_ITEMS). Skip PG under vLLM: the rollout+ # dominates the step, so PG saves little and its self-verify is net overhead._pg_engage = (UNSLOTH_GRPO_PREFIX_GROUPER_ONand not getattr(self, "use_vllm", False)@@ -1780,10 +1703,9 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):)if _pg_engage:try:- # Skip softcap models (the flex kernel never applies attn_logit_softcapping)- # and hybrid SSM / MoE models: only the threaded attention forwards get the- # shared-prefix isolation, so a Mamba or MoE decoder that does not forward- # prefix_seg_info would leak suffixes across completions. PG also rides on+ # Skip softcap models (the flex kernel never applies attn_logit_softcapping) and hybrid SSM /+ # MoE models: only the threaded attention forwards get shared-prefix isolation, so a decoder+ # that does not forward prefix_seg_info leaks suffixes across completions. PG also rides on# sequence packing, so it needs the same zoo masked-column guard._pg_cfg = getattr(unwrapped_model, "config", None)_pg_engage = (@@ -1795,8 +1717,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):and _pg_num_gen is not Noneand _pg_num_gen >= 2and not getattr(_pg_cfg, "attn_logit_softcapping", None)- # normal backends apply config.attention_dropout in training; the flex- # path is deterministic, so skip PG when it is set.+ # Normal backends apply config.attention_dropout in training; the flex path is deterministic,+ # so skip PG when it is set.and not getattr(_pg_cfg, "attention_dropout", 0)and not any(getattr(_pg_cfg, _pg_a, None) is not None@@ -1816,7 +1738,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):if _pg_engage:try:_pg_pad = self.processing_class.pad_token_id- # cap the PG span (P+max(R)) at the sliding window, like the packed _pk_sw guard.+ # Cap the PG span (P+max(R)) at the sliding window, like the packed _pk_sw guard._pg_sw = getattr(getattr(unwrapped_model, "config", None), "sliding_window", None)@@ -1869,14 +1791,14 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):)_pg_hidden = None # release before any verify forwarddevice_synchronize()- # clip to the loss window [B, logits_to_keep+max_left_pad]+ # Clip to the loss window [B, logits_to_keep+max_left_pad]._pg_w = logits_to_keep + max_left_padif _pg_r.shape[1] > _pg_w:_pg_r = _pg_r[:, -_pg_w:]return _pg_r- # trust only within the verified envelope: re-verify when T or the- # longest segment grows, like the packed path+ # Trust only within the verified envelope: re-verify when T or the longest segment grows, like+ # the packed path._pg_T = int(_pg_layout.flat_ids.shape[1])_pg_maxseg = int(_pg_layout.position_ids.max()) + 1_pg_env = (@@ -1885,25 +1807,23 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):if (not _pg_verify_on()) or (_pg_env is not None and _pg_T <= _pg_env[0] and _pg_maxseg <= _pg_env[1]):- # trusted shape: run PG now and skip the full-row forward below+ # Trusted shape: run PG now and skip the full-row forward below._pg_result = _pg_run_forward()_pg_use = True_pg_skip_pk = Trueelse:- # unverified shape: defer the forward until the packed reference- # exists (verify site below), so a declined packed path never wastes- # a whole-batch PG forward+ # Unverified shape: defer the forward until the packed reference exists, so a+ # declined packed+ # path never wastes a whole-batch PG forward._pg_forward_fn = _pg_run_forwardexcept Exception as _pg_err:_pg_result = None_pg_use = False_pg_skip_pk = False_pg_forward_fn = None- # A FlexAttention/Triton compile failure or OOM here is GPU-wide, not- # layout-specific, so retrying the same PG forward every step just re-pays- # the failure. Persistently disable PG (mirrors the seq-packing handler- # setting _unsloth_seq_packing_nograd_ok = False); the packed/padded path- # below still produces the exact result.+ # A FlexAttention/Triton compile failure or OOM here is GPU-wide, not layout-specific, so+ # retrying every step just re-pays it. Disable PG persistently; the packed/padded path below+ # still gives the exact result.unwrapped_model._unsloth_prefix_grouper_nograd_disabled = Trueif isinstance(_pg_err, torch.cuda.OutOfMemoryError):torch.cuda.empty_cache()@@ -1914,11 +1834,10 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):flush = True,)- # ---- Sequence packing (default-on; disable with UNSLOTH_GRPO_SEQ_PACKING=0) ----- # One varlen [1, sum L] block-diagonal forward replaces the padded [B, Lmax] loop- # (exact per-row result; also fixes the padded path's left-pad RoPE error).- # Self-verified vs the per-row forward, re-checked as T grows; falls back if a- # backend ignores packed_seq_lengths. lm_head runs on completion positions only.+ # Sequence packing (default on; UNSLOTH_GRPO_SEQ_PACKING=0 disables): one varlen+ # block-diagonal forward replaces the padded loop exactly and fixes its left-pad RoPE error.+ # Self-verified, re-checked as T grows, falls back if a backend ignores packed_seq_lengths, and+ # lm_head runs on completion positions only._pk_result = None_pk_use = False_pk_enabled = UNSLOTH_GRPO_SEQ_PACKING_ON@@ -1944,22 +1863,22 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):_pk_L = input_ids.shape[1]_pk_W = logits_to_keep + max_left_pad_pk_maxseg = max(_pk_nz_cpu) if _pk_nz_cpu else 0- # sliding-window models lose the per-sequence local window in a packed stream+ # Sliding-window models lose the per-sequence local window in a packed stream._pk_sw = getattr(getattr(unwrapped_model, "config", None), "sliding_window", None)_pk_sw_ok = not (isinstance(_pk_sw, int) and _pk_sw > 0 and _pk_maxseg > _pk_sw)- # per-row completion mask (same as the loss); prompt-only rows count as inactive+ # Per-row completion mask (same as the loss); prompt-only rows count as inactive._pk_cmask = create_completion_attention_mask(input_ids[:, -_pk_W:], left_pad_tokens_per_prompt, max_left_pad, _pk_pad)_pk_active = int(_pk_cmask.any(dim = 1).sum())- # skip the packed forward entirely at known-unsafe lengths (avoids a wasted pass / OOM)+ # Skip the packed forward entirely at known-unsafe lengths, avoiding a wasted pass or OOM._pk_unsafe = getattr(unwrapped_model, "_unsloth_seq_packing_nograd_unsafe_T", None)- # cap the flattened forward at one padded [batch_size, seq_len] mini-batch's- # token budget; anything larger uses the chunked padded loop+ # Cap the flattened forward at one padded [batch_size, seq_len] mini-batch's token budget;+ # anything larger uses the chunked padded loop._pk_cap = batch_size * seq_lenif (_pk_T >= 2@@ -1976,7 +1895,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):as_tuple = False) # [T, 2] = (row, col), row-major_pk_within = _pk_nz_idx[1:, 0] == _pk_nz_idx[:-1, 0] # [T-1]- # per-row completion start after left-packing (matches create_completion_attention_mask)+ # Per-row completion start after left-packing, matching create_completion_attention_mask._pk_cstart = (_pk_L - logits_to_keep) - left_pad_tokens_per_prompt # [rows]_pk_ctgt = (_pk_nz_idx[1:, 1] >= _pk_cstart[_pk_nz_idx[1:, 0]]) & _pk_withinwith _get_inference_mode_context_manager(model):@@ -1985,7 +1904,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):dtype = self._autocast_dtype,enabled = getattr(self, "_autocast_enabled", True),):- # use_cache=False: a KV cache silently disables varlen packing+ # use_cache=False: a KV cache silently disables varlen packing._pk_hidden = unwrapped_model(input_ids = _pk_flat,position_ids = _pk_pos,@@ -1996,7 +1915,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):).logits_pk_out = _pk_hidden[0, :-1, :][_pk_ctgt].unsqueeze(0)_pk_ids = _pk_flat[0, 1:][_pk_ctgt].unsqueeze(0)- # Guard: check if model returned hidden states or logits+ # Hidden states or logits? Logits mean the forward already applied scaling/softcapping.if _unsloth_grpo_returns_hidden_states(unwrapped_model, _pk_out, lm_head):@@ -2011,16 +1930,17 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):temperature,)[0]else:- # Model returned logits directly - scaling/softcapping already applied by model forward+ # Model returned logits directly - scaling/softcapping already applied by+ # model forward_pk_sel = chunked_selective_log_softmax(_pk_out,_pk_ids,temperature,_pk_chunks,)[0]- # GPT-OSS offload race guard (matches the padded loop)+ # GPT-OSS offload race guard, matching the padded loop.device_synchronize()- # scatter each logprob back to its (row, col) so [:, -_pk_W:] matches padded+ # Scatter each logprob back to its (row, col) so [:, -_pk_W:] matches the padded path._pk_tgt = (_pk_nz_idx[1:, 0] * _pk_L + _pk_nz_idx[1:, 1])[_pk_ctgt]_pk_result = (torch.zeros(@@ -2031,15 +1951,15 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):.index_put((_pk_tgt,), _pk_sel.to(torch.float32)).view(total_rows, _pk_L)[:, -_pk_W:])- # re-verify when T or the longest segment grows past what was verified- # (a LongRoPE cache switch can change the result)+ # Re-verify when T or the longest segment grows past the verified envelope; a LongRoPE cache+ # switch can change the result._pk_vT = int(getattr(unwrapped_model, "_unsloth_seq_packing_nograd_verified_T", 0))_pk_vS = int(getattr(unwrapped_model, "_unsloth_seq_packing_nograd_verified_seg", 0))- # debug: hand-edit this condition to force re-verify every step+ # Debug: hand-edit this condition to force re-verify every step.if _pk_ok is True and _pk_T <= _pk_vT and _pk_maxseg <= _pk_vS:_pk_use = True # already verified for this shapeelse:@@ -2066,7 +1986,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):use_cache = False,).logits_pk_rout = _pk_rh[:, :-1, :]- # Guard: check if model returned hidden states or logits+ # Hidden states or logits? Logits mean the forward already applied+ # scaling/softcapping.if _unsloth_grpo_returns_hidden_states(unwrapped_model, _pk_rout, lm_head):@@ -2081,7 +2002,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):temperature,)[0]else:- # Model returned logits directly - scaling/softcapping already applied by model forward+ # Model returned logits directly - scaling/softcapping already+ # applied by model forward_pk_rsel = chunked_selective_log_softmax(_pk_rout,_pk_real[:, 1:],@@ -2096,7 +2018,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):_pk_rkeep].to(torch.float32)device_synchronize()- # compare over the loss-mask region only+ # Compare over the loss-mask region only._pk_cm = _pk_cmask.float()_pk_diff = float(((_pk_result - _pk_ref).abs() * _pk_cm).max())if UNSLOTH_ENABLE_LOGGING:@@ -2104,11 +2026,12 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):f"[Unsloth] GRPO seq-packing (no-grad) verify: T={_pk_T} maxseg={_pk_maxseg} packed-vs-perrow max|d|={_pk_diff:.4f}",flush = True,)- # kernel-noise floor ~0.25; cross-sample contamination is >= 2.4+ # Kernel-noise floor is ~0.25; cross-sample contamination is >= 2.4.if _pk_diff < 7e-1:unwrapped_model._unsloth_seq_packing_nograd_ok = True- # widen the trusted shape only when >= 2 completion rows exercised- # cross-sample packing; single-row passes prove nothing+ # Widen the trusted shape only when at least 2 completion rows exercised cross-+ # sample packing;+ # a single row proves nothing.if _pk_active >= 2:unwrapped_model._unsloth_seq_packing_nograd_verified_T = max(_pk_vT, _pk_T@@ -2121,10 +2044,10 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):else:_pk_use = Falseif _pk_diff >= 1.5:- # contamination (attention ignores the packed mask): disable packing+ # Contamination (attention ignores the packed mask): disable packing.unwrapped_model._unsloth_seq_packing_nograd_ok = Falseelse:- # likely a length boundary (LongRoPE): mark unsafe, keep smaller shapes+ # Likely a length boundary (LongRoPE): mark unsafe, keep smaller shapes.unwrapped_model._unsloth_seq_packing_nograd_unsafe_T = (_pk_T if _pk_unsafe is None else min(_pk_unsafe, _pk_T))@@ -2134,7 +2057,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):flush = True,)except Exception as _pk_err:- # any failure: drop intermediates, use the padded loop, do not retry+ # Any failure: drop intermediates, use the padded loop, do not retry._pk_hidden = None_pk_sel = None_pk_result = None@@ -2147,14 +2070,14 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):f"[Unsloth] GRPO sequence-packing (no-grad) disabled (fell back to padded): {_pk_err!r}",flush = True,)+ # PrefixGrouper first-use self-verify (no-grad): compare the untrusted PG result to the packed+ # result over the completion mask. Below tol_ok trust the structure, at or above TOL_KILL mark+ # it unsafe forever, borderline falls back for this shape.# ---- PrefixGrouper first-use self-verify (no-grad) ----- # Compare the untrusted PG result to the full-row packed result (itself verified vs- # per-row) over the completion mask: < tol_ok -> trust the structure; >= TOL_KILL ->- # unsafe forever; borderline -> fall back this shape.if _pg_forward_fn is not None and not _pg_use:if _pk_use and _pk_result is not None:try:- # deferred PG forward, run only now that the packed reference exists+ # Deferred PG forward, run only now that the packed reference exists._pg_result = _pg_forward_fn()_pg_W2 = logits_to_keep + max_left_pad_pg_cm = create_completion_attention_mask(@@ -2208,7 +2131,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):f"[Unsloth] GRPO PrefixGrouper (no-grad) verify failed (fell back to packed): {_pg_err3!r}",flush = True,)- # else: no packed reference (packing off/failed) -> cannot verify; fall back.+ # No packed reference (packing off or failed) means this cannot be verified, so fall back.if _pg_use and _pg_result is not None:logprobs = _pg_result # PrefixGrouper verified/trusted -> skip the loop@@ -2253,7 +2176,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):)logits_chunk = outputs.logits- del outputs # free hidden_states before chunked log-softmax+ del outputscompletion_input_ids_chunk = input_ids_chunk[:, -(logits_to_keep + max_left_pad) :@@ -2262,7 +2185,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function)::, -(logits_to_keep + max_left_pad + 1) :, :]logits_chunk = logits_chunk[:, :-1, :]- # Guard: check if model returned hidden states or logits+ # Hidden states or logits? Logits mean the forward already applied scaling/softcapping.if _unsloth_grpo_returns_hidden_states(unwrapped_model, logits_chunk, lm_head):@@ -2277,7 +2200,10 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):temperature = temperature,)else:- # Model returned logits directly - scaling/softcapping already applied by model forward+ # Model returned logits directly - scaling/softcapping already applied by model+ # forward+ # Model returned logits directly - scaling/softcapping already applied by model+ # forwardlogprobs_chunk = chunked_selective_log_softmax(logits_chunk,completion_input_ids_chunk,@@ -2285,8 +2211,9 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):input_ids_chunk.shape[0] * multiplier,)else:- # Essentially, for VLMs we do not go via the optimized path in models/,- # so we don't encounter the Flash Attn left-padding issue.+ # VLMs do not take the optimized path in models/, so they never hit the Flash Attn+ # left-padding+ # issue.outputs = unwrapped_model(input_ids = input_ids_chunk,attention_mask = attention_mask_chunk,@@ -2299,11 +2226,11 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):)logits_chunk = outputs.logits- del outputs # free hidden_states before chunked log-softmax+ del outputslogits_chunk = logits_chunk[:, :-1, :]completion_input_ids_chunk = input_ids_chunk[:, -logits_to_keep:]- # Guard: check if model returned hidden states or logits+ # Hidden states or logits? Logits mean the forward already applied scaling/softcapping.if _unsloth_grpo_returns_hidden_states(unwrapped_model, logits_chunk, lm_head):@@ -2318,14 +2245,12 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):temperature = temperature,)else:- # Model returned logits directly - scaling/softcapping already applied by model forwardlogprobs_chunk = chunked_selective_log_softmax(logits_chunk,completion_input_ids_chunk,temperature,)- # This is needed to avoid race conditions with GPT OSS offload_embbed=True- # However, it seems that this line does not slow down or disrupt models.+ # Avoids a race with GPT OSS offload_embbed=True; it does not appear to slow models down.device_synchronize()all_logprobs_list.append(logprobs_chunk)if logprobs is None: # padded fallback when packing was not used@@ -2334,35 +2259,18 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):entropies = Noneos.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0"- # aux loss is unused: it is off by default (router_aux_loss_coef set to 0 in models/rl.py)- # and explicit opt-in is rejected at trainer init, so this is always None (kept in the- # return for TRL >= 1.7.0's 3-tuple contract).+ # aux loss is unused: off by default (router_aux_loss_coef = 0 in models/rl.py) and explicit+ # opt-in is rejected at trainer init, so it is always None. Kept for TRL >= 1.7.0's 3-tuple.aux_loss = Nonereturn logprobs.detach(), entropies, aux_loss # logps, entropies, aux_loss- # input_ids = input_ids[:, -logits_to_keep:]- # For transformers<=4.48, logits_to_keep argument isn't supported, so here we drop logits ourselves.- # See https://github.com/huggingface/trl/issues/2770- # logits = logits[:, -logits_to_keep:]- # return logits- # See https://huggingface.co/blog/the_n_implementation_details_of_rlhf_with_ppo#policy-training-implementation-details- # logits = logits / self.temperature- # logps = selective_log_softmax(logits, input_ids)-- # row_indices, col_indices = torch.where(logps < -20)-- # # Method 1: Check if tensors have elements- # if len(row_indices) > 0 and len(col_indices) > 0:- # breakpoint() # Breakpoint triggered here- # print("Found high values!")- # return logps # compute logprobs for the input tokens+ # transformers <= 4.48 does not support logits_to_keep, so drop the logits here; see+ # huggingface/trl#2770.function = inspect.getsource(_get_per_token_logps_and_entropies)if trl_version < Version("1.7.0"):- # TRL < 1.7.0 unpacks (logps, entropies) at every call site; TRL >= 1.7.0- # always unpacks (logps, entropies, aux_loss). Drop the aux_loss element so- # the return arity matches the installed TRL. Regex tolerates comment /- # whitespace drift on the return line; fail loud if the anchor ever stops- # matching rather than silently shipping a 3-tuple to older TRL.+ # TRL < 1.7.0 unpacks (logps, entropies) while TRL >= 1.7.0 unpacks (logps, entropies,+ # aux_loss), so drop the third element to match. The regex tolerates comment/whitespace drift+ # and fails loud rather than ship a 3-tuple to older TRL.new_function, n = re.subn(r"return (logprobs\.detach\(\), entropies), aux_loss[^\n]*",r"return \1 # logps, entropies",@@ -2489,7 +2397,7 @@ def _unsloth_grpo_hidden_states_signal(model):getattr(candidate, "_unsloth_grpo_hidden_states_forward_wrapped", False)for candidate in candidates):- # the wrapper honours the flag unless it recorded that this call could not+ # The wrapper honours the flag unless it recorded that this call could not.if any(hasattr(candidate, "_unsloth_grpo_hidden_states_degraded") for candidate in candidates):@@ -2497,8 +2405,8 @@ def _unsloth_grpo_hidden_states_signal(model):getattr(candidate, "_unsloth_grpo_hidden_states_degraded", False)for candidate in candidates)- # an `unsloth/models/rl.py` predating the per-call attribute only ever set- # the warn-once flag; it is the best signal such a wrapper offers+ # An unsloth/models/rl.py predating the per-call attribute set only the warn-once flag; it is+ # the best signal such a wrapper offers.return not any(getattr(candidate, "_unsloth_grpo_hidden_states_warning_issued", False)for candidate in candidates@@ -2513,10 +2421,9 @@ _GRPO_HIDDEN_STATES_WIDTH_DISPATCH = re.compile(flags = re.MULTILINE,)-# Deliberately loose: any branch header that decides something off an ``lm_head``-# dimension. Used only to count how many decisions the strict pattern above was-# supposed to rewrite, so that a zoo which respells *some* of them is rejected-# rather than half-patched.+# Deliberately loose: any branch header deciding something off an lm_head dimension. Used only+# to count how many the strict pattern should have rewritten, so a zoo respelling SOME of them+# is rejected rather than half-patched._GRPO_HIDDEN_STATES_WIDTH_DISPATCH_CANDIDATE = re.compile(r"^[ \t]*(?:el)?if[ \t]+[^\n]*\blm_head\.shape\[[^\]\n]+\][^\n]*:[ \t]*$",flags = re.MULTILINE,@@ -2599,7 +2506,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(grpo_compute_loss_slow)RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(grpo_update_SamplingParams))RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_get_inference_mode_context_manager))# inspect.getsource inlines function bodies but not module imports, so constants the inlined-# grpo functions reference (e.g. UNSLOTH_ENABLE_LOGGING) must be redefined in the generated cache.+# grpo functions use (UNSLOTH_ENABLE_LOGGING) must be redefined in the generated cache.RL_PRE_ITEMS["grpo_trainer"].append("import os as _unsloth_os\n""UNSLOTH_ENABLE_LOGGING = _unsloth_os.environ.get('UNSLOTH_ENABLE_LOGGING', '0') in ('1', 'True', 'true')\n"@@ -2626,8 +2533,8 @@ RL_PRE_ITEMS["grpo_trainer"].append(" except Exception:\n"" UNSLOTH_GRPO_PREFIX_GROUPER_ON = False\n")-# getsource inlines the grpo function bodies but not this file's imports, so the-# generated cache needs its own guarded import or detect_logit_transforms is a NameError.+# getsource inlines the grpo bodies but not this file's imports, so the generated cache needs+# its own guarded import or detect_logit_transforms is a NameError.RL_PRE_ITEMS["grpo_trainer"].append("try:\n"" from unsloth_zoo.device_map_planner import detect_logit_transforms\n"@@ -2636,7 +2543,7 @@ RL_PRE_ITEMS["grpo_trainer"].append()-# Edit _get_per_token_logps to handle mixed precision+# Edit _get_per_token_logps to handle mixed precision.def grpo_trainer_compute_loss(function_name, function):if function_name != "compute_loss":return function@@ -2650,7 +2557,6 @@ def grpo_trainer_compute_loss(function_name, function):):if return_outputs:raise ValueError("The GRPOTrainer does not support returning outputs")- # Compute the per-token log probabilities for the modelprompt_ids, prompt_mask = inputs["prompt_ids"], inputs["prompt_mask"]completion_ids, completion_mask = (@@ -2666,14 +2572,14 @@ def grpo_trainer_compute_loss(function_name, function):inputs.get("image_sizes", None),)num_images = inputs.get("num_images", None)- # Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models+ # Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models.token_type_ids = inputs.get("token_type_ids", None)mm_token_type_ids = inputs.get("mm_token_type_ids", None)num_items_in_batch = inputs.get("num_items_in_batch", None)sampling_per_token_logps = inputs.get("sampling_per_token_logps", None)tool_mask = inputs.get("tool_mask", None)- # Missing when evaluate() runs standalone; eval does not accumulate, so- # fall back to 1 to avoid underreporting eval_loss (#2464).+ # Missing when evaluate() runs standalone; eval does not accumulate, so fall back to 1 rather+ # than underreport eval_loss (#2464).current_gradient_accumulation_steps = getattr(self, "current_gradient_accumulation_steps", 1)@@ -2689,7 +2595,6 @@ def grpo_trainer_compute_loss(function_name, function):mm_token_type_ids,completion_ids = completion_ids,)- # attention_mask = Nonelogits_to_keep = completion_ids.size(1) # we only need to compute the logits for the completion tokens@@ -2723,30 +2628,19 @@ def grpo_trainer_compute_loss(function_name, function):per_token_logps = get_logps_func(model, input_ids, attention_mask, logits_to_keep, compute_efficient = True)- # Compute the KL divergence between the model and the reference model- # _prepare_inputs doesn't return reference log probs anymore. We need to calculate it ourselves.- # https://github.com/huggingface/trl/blob/05bc43e960396581e458195b8388efe6b82cae1f/trl/trainer/grpo_trainer.py#L1328- # if self.beta != 0.0:- # with torch.inference_mode(), model.disable_adapter():- # ref_per_token_logps = per_token_logps = get_logps_func(model, input_ids, attention_mask, logits_to_keep)- # else:- # ref_per_token_logps = None+ # KL divergence between model and reference: _prepare_inputs no longer returns reference log+ # probs. See trl grpo_trainer.py#L1328.ref_logps = inputs.get("ref_per_token_logps", None)- # per_token_kl = torch.exp(ref_per_token_logps - per_token_logps) - (ref_per_token_logps - per_token_logps) - 1- # x - x.detach() allows for preserving gradients from x+ # x - x.detach() preserves gradients from x.advantages = inputs["advantages"]- # per_token_loss = torch.exp(per_token_logps - per_token_logps.detach()) * advantages.unsqueeze(1)- # per_token_loss = -(per_token_loss - self.beta * per_token_kl)- # loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean()old_logps = inputs.get("old_per_token_logps", None)input_ids = input_ids[:, -logits_to_keep:]- # Get logit softcapping and logit scalemodel_config = _unsloth_get_model_config(model)- # The old and reference logps come from _get_per_token_logps_and_entropies and the- # gradient logps from here, so both must read the transforms the same way or the- # importance ratio compares two different policies.+ # The old and reference logps come from _get_per_token_logps_and_entropies and the gradient+ # logps from here, so both must read the transforms alike or the importance ratio compares two+ # different policies.if detect_logit_transforms is not None:# model_config, not model: see _get_per_token_logps_and_entropies._transforms = detect_logit_transforms(model_config)@@ -2904,7 +2798,7 @@ def grpo_trainer_compute_loss(function_name, function):**_grpo_accumulated_loss_kwargs,)else:- # to ensure backwards compatibility with trl 0.15.2 and maybe even 0.17+ # For backwards compatibility with trl 0.15.2 and maybe 0.17.loss, completion_length, mean_kl, coef_1, completion_mask = grpo_accumulated_loss(trainer = self,input_ids = _input_ids,@@ -3001,7 +2895,6 @@ def grpo_trainer_compute_loss(function_name, function):advantages = advantages.unsqueeze(1)if self.loss_type in ["grpo", "bnpo", "dr_grpo", "dapo"]:- # Compute the clipped probability ratiosis_low_clipped = (coef_1 < 1 - self.epsilon_low) & (advantages < 0)is_high_clipped = (coef_1 > 1 + self.epsilon_high) & (advantages > 0)is_region_clipped = is_low_clipped | is_high_clipped@@ -3037,15 +2930,13 @@ def grpo_trainer_compute_loss(function_name, function):RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_compute_loss)-# Fix KTO shape mismatch when Unsloth model forward truncates input_ids-# but labels aren't truncated. TRL 0.27.2+ _process_tokens only truncates-# completions, not prompts -- so prompts exceeding max_seq_length cause the-# model to produce shorter logits than the labels expect.+# KTO shape mismatch: the Unsloth forward truncates input_ids while labels are untouched, and+# TRL 0.27.2+ _process_tokens truncates only completions, so an over-length prompt makes the+# model emit shorter logits than the labels expect.def kto_trainer_get_batch_logps(function_name, function):if function_name != "get_batch_logps":return function- # The raise is inside an if block inside the method, so we need- # to preserve the exact indentation of the raise statement.+ # The raise sits inside an if inside the method, so its exact indentation must be preserved.old = 'raise ValueError("Logits (batch and sequence length dim) and labels must have the same shape.")'new = ("# Unsloth: auto-truncate to shorter sequence length (model may have truncated input_ids)\n"@@ -3060,10 +2951,10 @@ def kto_trainer_get_batch_logps(function_name, function):RL_FUNCTIONS["kto_trainer"].append(kto_trainer_get_batch_logps)-# TRL 1.x dropped KTOTrainer.get_batch_logps and moved the log-prob math into-# _compute_logps / compute_ref_log_probs / _compute_kl_logps, which call-# selective_log_softmax on completion-only tokens. Same truncation hazard as-# above, so clamp logits/ids/mask to the shorter seq length (no-op when equal).+# TRL 1.x dropped KTOTrainer.get_batch_logps and moved the math into _compute_logps /+# compute_ref_log_probs / _compute_kl_logps, which call selective_log_softmax on+# completion-only tokens. Same truncation hazard, so clamp logits/ids/mask to the shorter+# length (a no-op when equal)._KTO_COMPLETION_RE = re.compile(r"(?P<ws>[ \t]*)shift_logits = completion_logits\[:, :-1, :\]\.contiguous\(\)\n"r"(?P=ws)per_token_logps = selective_log_softmax\(\s*shift_logits,\s*"@@ -3117,11 +3008,10 @@ def kto_trainer_align_completion_logps(function_name, function):RL_FUNCTIONS["kto_trainer"].append(kto_trainer_align_completion_logps)-# https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py#L356-# TRL warns if batch size is not a multiple of num_generations -> fix this.+# TRL warns if batch size is not a multiple of num_generations; see trl grpo_trainer.py#L356.def grpo_trainer_fix_batch_size(RLTrainer_source, RLConfig_source):if "divisible by the number of generations" not in RLTrainer_source:- # in later trl versions this doesn't exist anymore+ # In later trl versions this no longer exists.return ""if "num_generations" not in RLConfig_source:return ""@@ -3139,12 +3029,11 @@ def grpo_trainer_fix_batch_size(RLTrainer_source, RLConfig_source):RL_CONFIG_CHANGES["grpo_trainer"].append(grpo_trainer_fix_batch_size)-# Add other reward function namesdef grpo_trainer_metrics(RLTrainer_source, RLConfig_source):if "reward_funcs" not in RLTrainer_source:return ""- # For new TRL we have /mean and /std+ # New TRL has /mean and /std.use_mean = "rewards/{reward_func_name}/mean" in RLTrainer_sourceuse_std = "rewards/{reward_func_name}/std" in RLTrainer_sourceif not use_mean:@@ -3173,16 +3062,10 @@ RL_METRICS_CHANGES["grpo_trainer"].append(grpo_trainer_metrics)def openenv_vllm_reload_weights():- # This function patches the trl openenv generate_rollout_completions function to:- # 1. Guard the reload_weights call (skip when sharing weights with vLLM)- # 2. Fix wake_up call to be compatible with unsloth (remove tags to wake everything)- #- # The issue: TRL's wake_up(tags=["kv_cache"]) only wakes kv_cache, leaving is_sleeping=True- # at the executor level. This causes unsloth's patched generate to try waking up again,- # resulting in double create_and_map on already-mapped handles.- #- # The fix: Use wake_up() with no tags, which wakes everything. Unsloth's patched- # CuMemAllocator.wake_up skips weights anyway, so this is safe.+ # Patch trl's openenv generate_rollout_completions to guard reload_weights when sharing weights+ # with vLLM, and to call wake_up() untagged: TRL's wake_up(tags=["kv_cache"]) leaves+ # is_sleeping=True at the executor, so Unsloth's generate wakes again and double create_and_maps+ # mapped handles. Unsloth's CuMemAllocator.wake_up skips weights anyway.if importlib.util.find_spec("trl") is None:returnif Version(importlib_version("trl")) < Version("0.26.0"):@@ -3198,7 +3081,7 @@ def openenv_vllm_reload_weights():)return- # trl 0.28 changed the function name yet again! Thanks trl :)+ # trl 0.28 changed the function name again.patch_target_name = "_generate_rollout_completions_colocate"if hasattr(openenv_utils, patch_target_name):patch_target = getattr(openenv_utils, patch_target_name)@@ -3207,15 +3090,9 @@ def openenv_vllm_reload_weights():patch_target_name = "generate_rollout_completions"patch_target = getattr(openenv_utils, patch_target_name)- # TRL 0.29.1+ ships some openenv helpers as compiled bytecode without- # accessible source on disk; inspect.getsource raises OSError("could- # not get source code") in that case. Skip the source-rewrite patch- # rather than crash. The unmodified TRL openenv path will run, which- # means the duplicate `collective_rpc("reload_weights")` is NOT- # stripped (line 1800 below) and `wake_up(tags=["kv_cache"])` is NOT- # retagged to `wake_up()` (line 1804). Users who do not use openenv- # GRPO are unaffected; openenv GRPO users on this TRL build may see- # redundant reload_weights calls or partial wake_up behavior.+ # TRL 0.29.1+ ships some openenv helpers as bytecode with no source, so inspect.getsource+ # raises OSError; skip the rewrite rather than crash. The unmodified path keeps the duplicate+ # reload_weights and the tagged wake_up, so openenv GRPO users may see redundant reloads.try:src = inspect.getsource(patch_target)except OSError as e:@@ -3246,20 +3123,20 @@ def openenv_vllm_reload_weights():src = reload_weights_pattern.sub(replace_reload_weights, src)- # Change wake_up(tags=["kv_cache"]) to wake_up() - wake everything to set is_sleeping=False- # This prevents double wake_up issues. Unsloth's allocator skips weights anyway.+ # wake_up() with no tags wakes everything and sets is_sleeping=False, preventing a double+ # wake_up; Unsloth's allocator skips weights anyway.src = re.sub(r"\.wake_up\(tags=\[.*?\]\)", ".wake_up()", src)if original_src == src:logger.warning("Unsloth: Warning - regex did not match, patch may have failed")return- # Execute and explicitly assign to module+ # Execute and explicitly assign to the module.local_ns = {}exec(compile(src, "<unsloth>", "exec"), openenv_utils.__dict__, local_ns)patched_func = local_ns[patch_target_name]- # Patch the target function in utils; if dispatcher was patched also update parent module alias.+ # Patch the target function in utils; if the dispatcher was patched, also update the parent module alias.setattr(openenv_utils, patch_target_name, patched_func)if patch_target_name == "generate_rollout_completions":openenv.generate_rollout_completions = patched_func@@ -3270,11 +3147,8 @@ RL_ADDITIONAL_FUNCTIONS["openenv"].append(openenv_vllm_reload_weights)def vllm_generation_init_patch():- # trl moved vllm stuff to trl/generation/vllm_generation.py- # We need to patch it to not instantiate another vLLM instance if we already have one with fast_inference- # Edit the TRL source directly and install the patched function in the TRL module.- # https://github.com/huggingface/trl/commit/0eb66d8f2fc63b3d00d8dbc18f99c3f48750bd16- # This exists in trl versions 0.28.0 and above+ # trl moved vllm code to trl/generation/vllm_generation.py (commit 0eb66d8, 0.28.0+), which+ # must be patched so it does not build a second vLLM instance when fast_inference has one.if importlib.util.find_spec("trl") is None:return@@ -3318,7 +3192,6 @@ def vllm_generation_init_patch():setattr(vllm_generation.VLLMGeneration, method_name, local_ns[method_name])return True- # Patch init to remove vLLM.LLM instantiationdef patch_init_vllm(src):pattern = re.compile(r"(?P<llm_block>^(?P<indent>[ \t]*)self\.llm\s*=\s*LLM\s*\(\n(?:.*\n)*?^(?P=indent)\))",@@ -3345,9 +3218,7 @@ def vllm_generation_init_patch():)return patched_src- # has some sync_weights or reload rpc calls.- # we patched the grpo_trainer to strip them for prev versions- # Ref: grpo_trainer__generate_single_turn above around L270-280+ # Newer versions have sync_weights or reload rpc calls; earlier ones are stripped in the patched grpo_trainer above.def patch_sync_weights(src):pattern = re.compile(r"^(?P<def_line>def sync_weights\(self\):\n)(?P<body>(?:.*\n)*)",@@ -3356,8 +3227,8 @@ def vllm_generation_init_patch():def replace_sync_weights(match):body = match.group("body")- # Chain getattr so server mode (where self.llm is not set) does- # not raise AttributeError before the default kicks in.+ # Chain getattr so server mode, where self.llm is unset, does not raise AttributeError before+ # the default kicks in.guard = (" if getattr(getattr(self, 'llm', None), 'shared_weights', False) or ""getattr(self, 'unsloth_fast_inference_lora', False):\n"@@ -3382,38 +3253,17 @@ def vllm_generation_init_patch():)return patched_src- # `generate` is deliberately NOT source-patched.- #- # It used to be, with two regexes: one anchored on a- # `self.llm.collective_rpc("reload_weights")` line, and one injecting- # `lora_request=...` into `self.llm.generate(...)`. Both anchors live inside a TRL- # method body, and both have already drifted:- # * TRL >= 1.x deleted the `collective_rpc("reload_weights")` call from `generate`- # entirely and calls `self.sync_weights()` instead- # (https://github.com/vllm-project/vllm/issues/29341). The old anchor matched 0- # times, raised, and - because the lora injection ran *after* it in the same- # function - the adapter stopped being passed to vLLM at all. Since `_init_vllm`- # and `sync_weights` had already been installed by then, weights were no longer- # synced into vLLM either, so GRPO rollouts were silently sampled from the BASE- # model with no error raised.- # * `(self\.llm\.generate\([^\)]+)\)` is not paren-balanced, so it also mis-edits any- # call whose arguments contain nested parentheses or span several lines.- #- # So intercept on the vLLM engine instead of on TRL's method body. `self.llm` is the- # vLLM `LLM` object in colocate mode in every TRL release that has `VLLMGeneration`,- # and `LLM.generate` / `LLM.chat` / `LLM.collective_rpc` are public, stable vLLM APIs.- # Checked against every vLLM release from 0.11.0 to 0.27.1: all three exist on `LLM` at- # each one, `generate` and `chat` both take `lora_request`, `LLM` has no `__slots__` and- # no `__setattr__`/`__getattr__` hook (so the instance override below always takes), and- # nothing here is decorated. TRL's method body is free to move around;- # whatever shape it takes, it has to reach the engine through those calls. The- # override is scoped to the dynamic extent of one `VLLMGeneration.generate` call and- # is undone in a `finally`, so `model.fast_generate` and any other user of the same- # engine are unaffected.+ # `generate` is deliberately NOT source-patched. Its two anchors drifted: TRL >= 1.x deleted+ # the collective_rpc("reload_weights") call for self.sync_weights(), so the anchor matched 0+ # times, raised, and took the lora injection with it -- GRPO rollouts came from the BASE model.+ # The generate regex is also not paren-balanced and mis-edits multi-line calls.+ # Intercept on the vLLM engine instead of TRL's method body: self.llm is the vLLM LLM object in+ # colocate mode in every release with VLLMGeneration, and generate / chat / collective_rpc are+ # public stable APIs, checked across vLLM 0.11.0-0.27.1. The override is scoped to one+ # VLLMGeneration.generate call and undone in a finally._UNSLOTH_GENERATE_WRAPPED = "_unsloth_vllm_generation_lora_wrapped"- # Mirror the per-device naming in rl.py so two ranks on one node do not race on the- # same adapter directory.+ # Mirror the per-device naming in rl.py so two ranks on one node do not race on the same adapter directory.lora_name = "vllm_gen_lora"if "CUDA_VISIBLE_DEVICES" in os.environ:lora_name += "_" + os.environ.get("CUDA_VISIBLE_DEVICES", "0").replace(",", "")@@ -3432,7 +3282,7 @@ def vllm_generation_init_patch():self, "unsloth_fast_inference_lora", False)if llm is None or not sharing:- # Server mode, or a vLLM engine TRL created itself -> upstream behaviour.+ # Server mode, or a vLLM engine TRL created itself: keep upstream behaviour.return original_generate(self, *args, **kwargs)load_lora = getattr(self, "_unsloth_load_lora", None)@@ -3471,15 +3321,15 @@ def vllm_generation_init_patch():try:positional = inspect.signature(bound).bind_partial(*args).argumentsexcept (TypeError, ValueError):- # Unintrospectable callable (C extension, odd wrapper): the keyword- # check above is all we have, and injecting is the safe default.+ # Unintrospectable callable (C extension, odd wrapper): the keyword check above is all we have,+ # and injecting is the safe default.return Falsereturn "lora_request" in positionaldef wrap_generation_call(bound):def unsloth_generation_call(*args, **kwargs):- # vLLM needs the adapter handed to it explicitly: the shared engine- # holds the BASE weights, and sync_weights is a no-op when sharing.+ # vLLM needs the adapter handed to it explicitly: the shared engine holds the BASE weights, and+ # sync_weights is a no-op when sharing.if load_lora is not None and not caller_already_bound_lora(bound, args, kwargs):kwargs["lora_request"] = load_lora(lora_name, load_tensors = True)return bound(*args, **kwargs)@@ -3488,8 +3338,8 @@ def vllm_generation_init_patch():def wrap_collective_rpc(bound):def unsloth_collective_rpc(method, *args, **kwargs):- # The engine already shares the live training weights, so- # reload_weights would pull the ORIGINAL checkpoint back off disk.+ # The engine already shares the live training weights, so reload_weights would pull the+ # ORIGINAL checkpoint back off disk.if method == "reload_weights":return Nonereturn bound(method, *args, **kwargs)@@ -3516,17 +3366,16 @@ def vllm_generation_init_patch():original_generate, "__qualname__", "VLLMGeneration.generate")generate.__doc__ = getattr(original_generate, "__doc__", None)- # inspect.getsource / inspect.signature unwrap this, so drift detectors and any- # other source-reading patch still see TRL's own `generate`.+ # inspect.getsource / inspect.signature unwrap this, so drift detectors and other source-reading+ # patches still see TRL's own generate.generate.__wrapped__ = original_generatesetattr(generate, _UNSLOTH_GENERATE_WRAPPED, True)vllm_generation.VLLMGeneration.generate = generatereturn True- # Snapshot before patching: a HALF-patched VLLMGeneration is worse than an unpatched- # one. `_init_vllm` + `sync_weights` without the generate-side adapter injection means- # no weight sync AND no LoRA, i.e. rollouts from the base model with no error. If any- # one of the three fails, put all three back.+ # Snapshot before patching: a HALF-patched VLLMGeneration is worse than none, since _init_vllm+ # plus sync_weights without the generate-side adapter injection means no weight sync AND no+ # LoRA, i.e. base-model rollouts with no error. If one of the three fails, restore all three.method_names = ("_init_vllm", "sync_weights", "generate")originals = {name: getattr(vllm_generation.VLLMGeneration, name, None) for name in method_names}try:diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.pyindex 0a1bea85a27..7c1d4d16813 100644--- a/unsloth/models/sentence_transformer.py+++ b/unsloth/models/sentence_transformer.py@@ -79,7 +79,6 @@ def _save_pretrained_torchao(if hasattr(inner_model, "_orig_mod"):inner_model = inner_model._orig_mod- # merge LoRA firstif hasattr(inner_model, "merge_and_unload"):inner_model = inner_model.merge_and_unload()@@ -106,9 +105,9 @@ def _save_pretrained_torchao(def patch_unsloth_save():original_causal = transformers.AutoModelForCausalLMoriginal_rmtree = shutil.rmtree- # unsloth_save_pretrained_torchao expects AutoModelForCausalLM+ # unsloth_save_pretrained_torchao expects AutoModelForCausalLM.transformers.AutoModelForCausalLM = transformers.AutoModel- # prevent unsloth from deleting the unquantized model directory+ # Prevent unsloth from deleting the unquantized model directory.shutil.rmtree = lambda *args, **kwargs: Nonetry:yield@@ -126,7 +125,7 @@ def _save_pretrained_torchao(token = token,)- # avoid `0_Transformer-torchao`, it was either this or fix modules.json+ # Avoid `0_Transformer-torchao`; it was either this or fix modules.json.torchao_dir = transformer_dir + "-torchao"if os.path.exists(torchao_dir):if not os.path.exists(transformer_dir):@@ -157,7 +156,6 @@ def _save_pretrained_torchao(pass-# Thanks Etherl:def _save_pretrained_gguf(self,save_directory,@@ -177,10 +175,9 @@ def _save_pretrained_gguf(Saves the SentenceTransformer model to GGUF format by saving the inner transformer model,converting it, and placing the resulting GGUF files in the save directory."""- # 1. Save standard SentenceTransformer structure (configs, modules.json, etc.)self.save_pretrained(save_directory)- # 2. Extract inner transformer model (PEFT merge handled by the gguf saver)+ # Extract the inner transformer model; PEFT merge is handled by the gguf saver.inner_model = self[0].auto_modelif hasattr(inner_model, "_orig_mod"):inner_model = inner_model._orig_mod@@ -199,36 +196,34 @@ def _save_pretrained_gguf(except:pass- # Unsloth saves + converts here; absolute for later commonpath comparison+ # Unsloth saves and converts here; absolute for the later commonpath comparison.transformer_dir = os.path.join(save_directory, transformer_path)transformer_dir = os.path.abspath(transformer_dir)if tokenizer is None:tokenizer = self.tokenizer- # 4. Call Unsloth's GGUF saver on the inner model targeting the transformer subdirectory- # No rmtree guard here: the merge cleanup that deletes save_directory is gated on- # push_to_hub, which is forced False below.+ # No rmtree guard here: the merge cleanup that deletes save_directory is gated on push_to_hub, which+ # is forced False below.result = unsloth_save_pretrained_gguf(inner_model,save_directory = transformer_dir,tokenizer = tokenizer,quantization_method = quantization_method,first_conversion = first_conversion,- push_to_hub = False, # Force local first to move files+ push_to_hub = False,token = token,max_shard_size = max_shard_size,temporary_location = temporary_location,maximum_memory_usage = maximum_memory_usage,- # transformer_dir is the ST's own 0_Transformer module (written in step 1, uploaded- # in step 7), not a throwaway: reclaiming it would hand back a folder that no longer- # loads as a SentenceTransformer, so a short disk fails loudly instead.+ # transformer_dir is the ST's own 0_Transformer module, not a throwaway: reclaiming it would hand+ # back a folder that no longer loads as a SentenceTransformer, so a short disk fails loudly.merge_is_disposable = False,imatrix_file = imatrix_file,gguf_shard_size = gguf_shard_size,)- # 5. Move GGUF files from the subdirectory (0_Transformer) to the root save_directory+ # Move GGUF files from the 0_Transformer subdirectory to the root save_directory.gguf_files = result.get("gguf_files", [])new_gguf_locations = []@@ -243,26 +238,22 @@ def _save_pretrained_gguf(try:is_subpath = os.path.commonpath([abs_gguf_file, transformer_dir]) == transformer_direxcept ValueError:- # Windows different-drive commonpath raises+ # Windows different-drive commonpath raises.is_subpath = Falseif is_subpath:- # Move the GGUF out of transformer_dir to rootshutil.move(gguf_file, dest_path)new_gguf_locations.append(dest_path)else:- # Elsewhere: move to root if not already thereif os.path.abspath(dest_path) != abs_gguf_file:shutil.move(gguf_file, dest_path)new_gguf_locations.append(dest_path)result["gguf_files"] = new_gguf_locations- # 6. Add brandingtry:FastSentenceTransformer._add_unsloth_branding(save_directory)- # Add GGUF details to READMEreadme_path = os.path.join(save_directory, "README.md")if os.path.exists(readme_path):with open(readme_path, "a", encoding = "utf-8") as f:@@ -273,13 +264,12 @@ def _save_pretrained_gguf(except:pass- # 7. Handle Push to Hub if requestedif push_to_hub:if token is None:token = get_token()api = HfApi(token = token)- repo_id = save_directory # Assuming save_directory is the repo name if pushing+ repo_id = save_directoryprint(f"Unsloth: Uploading to {repo_id}...")try:@@ -391,7 +381,6 @@ def _push_to_hub_gguf(except Exception as e:print(f"Unsloth Warning: Could not create repo: {e}")- # Convert locally in a temp dir, then uploadwith tempfile.TemporaryDirectory(prefix = "unsloth_st_gguf_") as temp_dir:print(f"Unsloth: Converting SentenceTransformer to GGUF format...")@@ -417,7 +406,6 @@ def _push_to_hub_gguf(print(f"Unsloth: Uploading GGUF to https://huggingface.co/{full_repo_id}...")- # Upload GGUF filesfor file_location in gguf_files:if os.path.exists(file_location):filename = os.path.basename(file_location)@@ -433,7 +421,6 @@ def _push_to_hub_gguf(revision = revision,)- # Upload Modelfile if existsif modelfile_location and os.path.exists(modelfile_location):print(" Uploading Ollama Modelfile...")api.upload_file(@@ -446,7 +433,6 @@ def _push_to_hub_gguf(revision = revision,)- # Upload config.json if existsconfig_path = os.path.join(temp_dir, "config.json")if os.path.exists(config_path):print(" Uploading config.json...")@@ -460,7 +446,6 @@ def _push_to_hub_gguf(revision = revision,)- # Create and upload READMEgguf_basenames = [os.path.basename(f) for f in gguf_files if os.path.exists(f)]readme_content = f"""---tags:@@ -509,7 +494,6 @@ This sentence-transformers model was finetuned and converted to GGUF format usinrevision = revision,)- # Add tagsall_tags = ["gguf", "llama-cpp", "unsloth", "sentence-transformers"]if is_vlm:all_tags.append("vision-language-model")@@ -593,7 +577,6 @@ class FastSentenceTransformer(FastModel):if module.get("type", "") == "sentence_transformers.models.Pooling":pooling_path = module.get("path", "")if pooling_path:- # find config.json for the pooling moduleif os.path.exists(model_name) and os.path.exists(os.path.join(model_name, pooling_path, "config.json")):@@ -613,8 +596,7 @@ class FastSentenceTransformer(FastModel):if pooling_config_path:with open(pooling_config_path, "r", encoding = "utf-8") as f:pooling_config = json.load(f)- # from here:- # https://github.com/huggingface/sentence-transformers/blob/main/sentence_transformers/models/Pooling.py#L43+ # From sentence-transformers models/Pooling.py#L43pooling_map = {"pooling_mode_cls_token": "cls","pooling_mode_mean_tokens": "mean",@@ -635,7 +617,7 @@ class FastSentenceTransformer(FastModel):)return "mean"- # should prolly be done upstream instead of this hackfest here+ # Should probably be done upstream instead of here.@staticmethoddef _patch_mpnet_v4():"""Patch MPNetModel for gradient checkpointing (transformers 4)."""@@ -655,8 +637,10 @@ class FastSentenceTransformer(FastModel):modeling_mpnet.MPNetModel._set_gradient_checkpointing = _set_gradient_checkpointing- # patch MPNetEncoder.forward for checkpointing; based on:- # https://github.com/huggingface/transformers/blob/v4.57.3/src/transformers/models/mpnet/modeling_mpnet.py#L321+ # Patch MPNetEncoder.forward for checkpointing; based on transformers v4.57.3+ # models/mpnet/modeling_mpnet.py#L321+ # based on: https://github.com/huggingface/transformers/blob/v4.57.3/src/transformers/models/distilbert/+ # modeling_distilbert.py#L666def forward(self,hidden_states: torch.Tensor,@@ -689,7 +673,7 @@ class FastSentenceTransformer(FastModel):attention_mask,head_mask[i] if head_mask is not None else None,position_bias,- use_reentrant = True, # fix for torch 2.9+ use_reentrant = True,)else:layer_outputs = layer_module(@@ -740,8 +724,8 @@ class FastSentenceTransformer(FastModel):modeling_mpnet.MPNetModel._set_gradient_checkpointing = _set_gradient_checkpointing- # patch MPNetEncoder.forward for checkpointing; based on:- # https://github.com/huggingface/transformers/blob/v5.0.0rc1/src/transformers/models/mpnet/modeling_mpnet.py#L284+ # Patch MPNetEncoder.forward for checkpointing; based on transformers v5.0.0rc1+ # models/mpnet/modeling_mpnet.py#L284def forward(self,hidden_states: torch.Tensor,@@ -805,11 +789,10 @@ class FastSentenceTransformer(FastModel):@staticmethoddef _patch_distilbert_v4():- # change kwargs to positional args to be compatible with peft_utils+ # Change kwargs to positional args to be compatible with peft_utils; based on transformers v4.57.3+ # models/distilbert/modeling_distilbert.py#L666"""Patch DistilBertModel.forward to use positional args (transformers 4)."""- # based on:- # https://github.com/huggingface/transformers/blob/v4.57.3/src/transformers/models/distilbert/modeling_distilbert.py#L666def forward(self,input_ids: Optional[torch.Tensor] = None,@@ -867,7 +850,7 @@ class FastSentenceTransformer(FastModel):attention_mask = _prepare_4d_attention_mask_for_sdpa(attention_mask, embeddings.dtype, tgt_len = input_shape[1])- # patch here, change kwargs to positional args:+ # patch here: unsloth gradient checkpointing hook needs positional argumentsreturn self.transformer(embeddings,attention_mask,@@ -898,8 +881,7 @@ class FastSentenceTransformer(FastModel):@staticmethoddef _patch_distilbert_v5():"""Patch DistilBertModel.forward to use positional args (transformers 5)."""- # based on:- # https://github.com/huggingface/transformers/blob/v5.0.0rc1/src/transformers/models/distilbert/modeling_distilbert.py#L386+ # Based on transformers v5.0.0rc1 models/distilbert/modeling_distilbert.py#L386from transformers.masking_utils import create_bidirectional_maskdef forward(@@ -921,7 +903,7 @@ class FastSentenceTransformer(FastModel):attention_mask = attention_mask,)- # patch here: unsloth gradient checkpointing hook needs positional arguments+ # Unsloth's gradient checkpointing hook needs positional arguments.return self.transformer(embeddings,attention_mask,@@ -962,11 +944,10 @@ class FastSentenceTransformer(FastModel):with open(readme_path, "r", encoding = "utf-8") as f:content = f.read()- # add unsloth tag to frontmatterif "---\ntags:\n" in content:content = content.replace("---\ntags:\n", "---\ntags:\n- unsloth\n")else:- # tags exist but not at the start: append via regex+ # Tags exist but not at the start: append via regex.pattern = r"(^tags:\s*\n)"if re.search(pattern, content, re.MULTILINE):content = re.sub(pattern, r"\1- unsloth\n", content, count = 1, flags = re.MULTILINE)@@ -1026,11 +1007,8 @@ class FastSentenceTransformer(FastModel):"""Helper to create and configure a Transformer module."""from sentence_transformers.models import Transformer- # Prevents loading the model a second time and redirects AutoProcessor/- # AutoTokenizer so Transformer.__init__ picks up our pre-fixed tokenizer.- # On sentence-transformers >=5.4 `tokenizer` is a read-only @property- # backed by `self.processor`, so a post-init assignment raises; the- # constructor redirect sets self.processor correctly instead.+ # Prevents loading the model a second time and redirects AutoProcessor/AutoTokenizer so+ # Transformer.__init__ picks up our pre-fixed tokenizer.from transformers import AutoProcessor, AutoTokenizerdef is_requested_model_name(args, kwargs):@@ -1087,7 +1065,7 @@ class FastSentenceTransformer(FastModel):return original_processor_from_pretrained(*args, **kwargs)try:- # Temporarily redirect Auto* loading to return our pre-loaded objects+ # Temporarily redirect Auto* loading to return our pre-loaded objects.AutoModel.from_pretrained = return_existing_modelAutoProcessor.from_pretrained = return_existing_processorAutoTokenizer.from_pretrained = return_existing_tokenizer@@ -1109,12 +1087,9 @@ class FastSentenceTransformer(FastModel):elif "tokenizer_args" in transformer_init_params:transformer_kwargs["tokenizer_args"] = trust_remote_code_kwargs.copy()- # Build via Transformer.load so the saved modality_config is honored: plain- # Transformer(...) makes ST 5.x infer a "message" modality for chat-template- # models (e.g. Qwen3-Embedding), chat-wrapping inputs and degrading embeddings- # (#6881). Only use .load when it resolves a Hub id (accepts the kwargs or- # **kwargs); legacy ST 3.x/4.x load(input_path) is local-only with no modality- # bug, so fall back to the constructor.+ # Build via Transformer.load so the saved modality_config is honored: a plain Transformer(...)+ # makes ST 5.x infer a "message" modality for chat-template models, chat-wrapping inputs and+ # degrading embeddings (#6881). Only use .load for a Hub id; ST 3.x/4.x load() is local-only.transformer_module = Nonetransformer_load = getattr(Transformer, "load", None)has_modules_json = (@@ -1139,8 +1114,9 @@ class FastSentenceTransformer(FastModel):"trust_remote_code": trust_remote_code,**transformer_kwargs,}- # Resolve config/tokenizer from the module's saved subfolder- # (modules.json "path"), like stock ST; "" (root) is a no-op.+ # Resolve config/tokenizer from the module's saved subfolder (modules.json "path"),+ # like stock ST;+ # "" (root) is a no-op.if module_subfolder:load_kwargs["subfolder"] = module_subfolderif not accepts_var_kw:@@ -1154,14 +1130,13 @@ class FastSentenceTransformer(FastModel):AutoProcessor.from_pretrained = original_processor_from_pretrainedAutoTokenizer.from_pretrained = original_tokenizer_from_pretrained- # On sentence-transformers >=5.4 `tokenizer` is a read-only property backed- # by `self.processor` (already wired via the redirect above). On older- # versions it's a regular attribute and the explicit assignment is required.+ # On sentence-transformers >= 5.4 `tokenizer` is a read-only property backed by self.processor;+ # on older versions it is a regular attribute and this assignment is required.if not isinstance(getattr(type(transformer_module), "tokenizer", None), property):transformer_module.tokenizer = tokenizertransformer_module.do_lower_case = getattr(tokenizer, "do_lower_case", False)- # sentence-transformers only passes along known keys to model.forward+ # sentence-transformers only passes along known keys to model.forward.preinit_model_forward_params = getattr(transformer_module, "model_forward_params", set())model_forward_params = list(inspect.signature(model.forward).parameters)transformer_module.model_forward_params = set(model_forward_params) | {@@ -1172,12 +1147,11 @@ class FastSentenceTransformer(FastModel):"return_dict",}if preinit_model_forward_params is None:- # ST 6 uses None to allow arbitrary **kwargs+ # ST 6 uses None to allow arbitrary **kwargs.transformer_module.model_forward_params = Noneelse:transformer_module.model_forward_params |= preinit_model_forward_params- # determine max_seq_length if not providedif max_seq_length is None:if hasattr(model, "config") and hasattr(model.config, "max_position_embeddings"):max_seq_length = model.config.max_position_embeddings@@ -1272,7 +1246,6 @@ class FastSentenceTransformer(FastModel):)modules[name] = transformer_moduleelse:- # load other modules (Pooling, Normalize, etc.)module_path = module_config["path"]if os.path.isdir(model_name):load_path = os.path.join(model_name, module_path)@@ -1298,7 +1271,7 @@ class FastSentenceTransformer(FastModel):return modules, False- # fallback if no modules.json (non sentence-transformers models)+ # Fallback if there is no modules.json (non sentence-transformers models).print("Unsloth: No modules.json found, falling back to [Transformer, Pooling, Normalize]. This may or may not work.")@@ -1327,7 +1300,7 @@ class FastSentenceTransformer(FastModel):return modules, True- # Encoder model types that benefit from native torch.compile instead of Unsloth patching+ # Encoder model types that benefit from native torch.compile instead of Unsloth patching.ENCODER_MODEL_TYPES = {"mpnet","bert",@@ -1372,9 +1345,8 @@ class FastSentenceTransformer(FastModel):params_m = params / 1e6- # Empirical formula based on benchmarks with batch_size=2, grad_accum=4- # Small models: high fixed overhead, lower speedup- # Large models: warmup scales but speedup is significant+ # Empirical formula from benchmarks with batch_size=2, grad_accum=4: small models carry high fixed+ # overhead and a lower speedup, large models scale warmup but gain significantly.if params_m < 50:estimated_warmup = 35 + params_m * 0.3base_speedup = 1.35@@ -1385,7 +1357,6 @@ class FastSentenceTransformer(FastModel):estimated_warmup = 15 + params_m * 0.04base_speedup = 1.60- # Estimate time per step (ms) and time savednaive_ms = 50 + params_m * 1.0compiled_ms = naive_ms / base_speeduptime_saved_per_step_s = (naive_ms - compiled_ms) / 1000@@ -1395,11 +1366,10 @@ class FastSentenceTransformer(FastModel):else:breakeven = float("inf")- # Return threshold with 1.2x safety marginthreshold = breakeven * 1.2- # Optional adjustment based on expected work per step.- # This uses only pre-run information (batch size, grad accum, seq length).+ # Optional adjustment based on expected work per step, using only pre-run information (batch size,+ # grad accum, seq length).generic_scale = 1.0fast_scale = 1.0if batch_size is not None or grad_accum is not None or max_seq_length is not None:@@ -1461,11 +1431,8 @@ class FastSentenceTransformer(FastModel):model[0].model = compiledelse:model[0].auto_model = compiled- # Fix for accelerate unwrap_model bug:- # When SentenceTransformer contains a compiled inner model,- # accelerate checks has_compiled_regions() which returns True,- # then tries to access model.__dict__["_orig_mod"] which fails.- # This workaround sets _orig_mod to satisfy accelerate.+ # Works around an accelerate unwrap_model bug: with a compiled inner model accelerate sees+ # has_compiled_regions() True and then fails on model.__dict__["_orig_mod"], so set it.model.__dict__["_orig_mod"] = modelelse:model = torch.compile(model, mode = mode)@@ -1508,17 +1475,15 @@ class FastSentenceTransformer(FastModel):"Run `pip install sentence-transformers` to install it.")- # The other leaf loaders resolve the "unsloth" sentinel by planning; this one- # declines. `st_device` below hands `device_map` to `SentenceTransformer(device=)`,- # which ends in `self.to(device)`: the sentinel raises there, and that same `.to()`- # would pull any split model back onto one card. The env-var opt-in is resolved too,- # or `UNSLOTH_AUTO_DEVICE_MAP=1` asks for a plan without ever naming the sentinel.+ # The other leaf loaders resolve the "unsloth" sentinel by planning; this one declines. st_device+ # below hands device_map to SentenceTransformer(device=), which ends in self.to(device): the+ # sentinel raises there, and that same .to() would pull a split model back onto one card. The+ # env-var opt-in is resolved too, or UNSLOTH_AUTO_DEVICE_MAP=1 asks for a plan without naming the+ # sentinel.device_map = requested_device_map(device_map)- # Always "sequential", never the asked-for name's own declined value: the `st_device`- # blocks below normalise only dicts, "auto" and "sequential", so "balanced" would- # reach `SentenceTransformer(device = "balanced")` and then `.to("balanced")`, which- # is not a torch device. There is nothing to shard here in any case.- # `isinstance` first: a caller's explicit dict is unhashable, so `in` alone raises.+ # Always "sequential", never the asked-for name's own declined value: the st_device blocks+ # normalise only dicts, "auto" and "sequential", so "balanced" would reach .to("balanced").+ # isinstance first, since a caller's explicit dict is unhashable and `in` alone raises.if isinstance(device_map, str) and device_map in _PLANNED_DEVICE_MAPS:print("Unsloth: Not planning a device map; SentenceTransformer moves the assembled "@@ -1526,8 +1491,8 @@ class FastSentenceTransformer(FastModel):)device_map = "sequential"- # Validate the load modes BEFORE the prefetch so a bad config fails without downloading weights.- # Guard on not for_inference: that branch below never used these flags.+ # Validate the load modes BEFORE the prefetch so a bad config fails without downloading weights;+ # guarded on not for_inference, since that branch never used these flags.if not for_inference:# sanity check, thanks Etherl:if full_finetuning and (load_in_4bit or load_in_8bit):@@ -1547,9 +1512,8 @@ class FastSentenceTransformer(FastModel):"If you want 8bit finetuning, set both `load_in_16bit = False` and `load_in_8bit = True`")- # Prefetch so the ST load below is a cache hit. weights_at_root stays False (ST component- # weights live in per-module subfolders). Resolve the same cache the load uses: HF cache_dir,- # else cache_folder, else SENTENCE_TRANSFORMERS_HOME, else default -- a wrong cache misses the warm.+ # Prefetch so the ST load below is a cache hit. weights_at_root stays False, since ST component+ # weights live in per-module subfolders._st_prefetched = maybe_prefetch_hf_snapshot(model_name,token = token,@@ -1558,14 +1522,14 @@ class FastSentenceTransformer(FastModel):or kwargs.get("cache_folder")or os.environ.get("SENTENCE_TRANSFORMERS_HOME"),local_files_only = kwargs.get("local_files_only", False),- # Forward force_download so the refresh happens in the killable child, then clear it so the- # in-process ST load reuses the warm cache instead of re-downloading over unguarded Xet.+ # Forward force_download so the refresh happens in the killable child, then clear it so the in-+ # process ST load reuses the warm cache instead of re-downloading over unguarded Xet.force_download = kwargs.get("force_download", False),)if _st_prefetched and kwargs.get("force_download", False):kwargs["force_download"] = False- # if for_inference == True, skip Unsloth optimizations to avoid torch compile issues+ # With for_inference == True, skip Unsloth optimizations to avoid torch compile issues.if for_inference:st_device = device_mapif isinstance(st_device, dict) or (@@ -1573,7 +1537,7 @@ class FastSentenceTransformer(FastModel):):st_device = None- # Propagate dtype to model_kwargs (else inference defaults to float32)+ # Propagate dtype to model_kwargs, else inference defaults to float32.model_kwargs = kwargs.get("model_kwargs", {})model_kwargs["dtype"] = dtype if dtype is not None else "auto"@@ -1595,8 +1559,8 @@ class FastSentenceTransformer(FastModel):if k in kwargs:st_kwargs[k] = kwargs[k]- # ST takes cache_folder, not cache_dir: map cache_dir onto it so this load hits the warm- # (None lets ST honor SENTENCE_TRANSFORMERS_HOME, matching the prefetch).+ # ST takes cache_folder, not cache_dir: map cache_dir onto it so this load hits the warm (None lets+ # ST honor SENTENCE_TRANSFORMERS_HOME, matching the prefetch)._st_cache = kwargs.get("cache_dir") or kwargs.get("cache_folder")if _st_cache is not None:st_kwargs["cache_folder"] = _st_cache@@ -1619,20 +1583,16 @@ class FastSentenceTransformer(FastModel):except:pass- # Fast encoder path: Use native torch.compile for encoder models (6x speedup)- # This bypasses Unsloth's auto-compiler which adds @torch.compiler.disable decorators- # that interfere with torch.compile and cause runtime errors for encoder models.- # NOTE: The old Unsloth path is BROKEN for encoder models with torch 2.9+ due to- # conflicting @torch.compile and @torch.compiler.disable decorators.- # Set UNSLOTH_COMPILE_DISABLE=1 to disable torch.compile and use the old path.+ # Fast encoder path: native torch.compile for encoder models (6x speedup), bypassing Unsloth's+ # auto-compiler, whose @torch.compiler.disable decorators error for encoders on torch 2.9+.+ # Set UNSLOTH_COMPILE_DISABLE=1 for the old path.is_encoder_model = model_type.lower() in FastSentenceTransformer.ENCODER_MODEL_TYPESuse_fast_encoder = os.environ.get("UNSLOTH_COMPILE_DISABLE", "0") != "1"if use_fast_encoder and is_encoder_model:- # torch.compile mode: "default" is safest for PEFT/LoRA training- # Note: "reduce-overhead" uses CUDA Graphs which is incompatible with PEFT+ # torch.compile mode "default" is safest for PEFT/LoRA training; "reduce-overhead" uses CUDA Graphs,+ # which are incompatible with PEFT.compile_mode = "default"- # Determine dtype - handle float16 machines that don't support bfloat16if dtype is None:if load_in_16bit:dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16@@ -1642,14 +1602,12 @@ class FastSentenceTransformer(FastModel):print("Unsloth: Device does not support bfloat16. Using float16 instead.")dtype = torch.float16- # Determine devicest_device = device_mapif isinstance(st_device, dict) or (isinstance(st_device, str) and st_device in ["auto", "sequential"]):st_device = "cuda"- # Build model_kwargs for SentenceTransformermodel_kwargs = {"torch_dtype": dtype}encoder_attn_impl = resolve_encoder_attention_implementation(@@ -1662,7 +1620,6 @@ class FastSentenceTransformer(FastModel):if encoder_attn_impl is not None:model_kwargs["attn_implementation"] = encoder_attn_impl- # Print optimization statussdpa_str = " + SDPA" if supports_sdpa else ""if load_in_4bit:print(@@ -1673,7 +1630,7 @@ class FastSentenceTransformer(FastModel):f"Unsloth: Using fast encoder path for {model_type} (torch.compile{sdpa_str})")- # Handle 4-bit quantization via BitsAndBytesConfig+ # Handle 4-bit quantization via BitsAndBytesConfig; when quantizing, device must be handled by accelerate.if load_in_4bit:from transformers import BitsAndBytesConfig@@ -1687,12 +1644,12 @@ class FastSentenceTransformer(FastModel):# When using quantization, device must be handled by acceleratest_device = None- # Handle gradient checkpointing - warn user it conflicts with torch.compile+ # Gradient checkpointing conflicts with torch.compile, so warn the user._use_gc = use_gradient_checkpointingif _use_gc and _use_gc != False:print("Unsloth Warning: Gradient checkpointing is incompatible with torch.compile.")print("Disabling torch.compile to enable gradient checkpointing.")- compile_mode = None # Disable compilation+ compile_mode = Noneis_mpnet = "mpnet" == model_type.lower()@@ -1701,8 +1658,8 @@ class FastSentenceTransformer(FastModel):elif is_mpnet:FastSentenceTransformer._patch_mpnet_v5()- # ST takes cache_folder, not cache_dir: map cache_dir onto it so this load hits the warm- # (None lets ST honor SENTENCE_TRANSFORMERS_HOME, matching the prefetch).+ # ST takes cache_folder, not cache_dir: map cache_dir onto it so this load hits the warm (None lets+ # ST honor SENTENCE_TRANSFORMERS_HOME, matching the prefetch).st_model = SentenceTransformer(model_name,device = st_device,@@ -1713,7 +1670,6 @@ class FastSentenceTransformer(FastModel):cache_folder = kwargs.get("cache_dir") or kwargs.get("cache_folder"),)- # Store metadata for get_peft_modelst_model._unsloth_fast_encoder = Truest_model._compile_mode = compile_modest_model._dtype = dtype@@ -1723,7 +1679,6 @@ class FastSentenceTransformer(FastModel):st_model[0], getattr(st_model[0].auto_model, "config", None))- # Add save methodsdef _save_pretrained_merged(self,save_directory,@@ -1731,15 +1686,13 @@ class FastSentenceTransformer(FastModel):save_method = "merged_16bit",**save_kwargs,):- # `tokenizer` and `save_method` are positional to match- # FastLanguageModel.save_pretrained_merged(dir, tokenizer,- # save_method = ...), which is how the docs and notebooks call- # it. Keyword callers behave exactly as before.+ # tokenizer and save_method are positional to match+ # FastLanguageModel.save_pretrained_merged(dir, tokenizer, save_method = ...), which is how the+ # docs and notebooks call it; keyword callers are unaffected.save_method = _normalize_save_method(save_method)if save_method not in ("merged_16bit", None):- # Refused before anything is written: this path merges and- # unloads unconditionally, so accepting "lora" would write- # full weights for a request to write adapters.+ # Refused before anything is written: this path merges and unloads unconditionally, so accepting+ # "lora" would write full weights for a request to write adapters.raise NotImplementedError(f"Unsloth: save_method = {save_method!r} is not "f"supported for this SentenceTransformer; only "@@ -1752,7 +1705,6 @@ class FastSentenceTransformer(FastModel):save_kwargs.pop("tokenizer", None)if hasattr(self[0], "auto_model"):inner = self[0].auto_model- # Handle compiled modelif hasattr(inner, "_orig_mod"):inner = inner._orig_modif hasattr(inner, "merge_and_unload"):@@ -1800,7 +1752,7 @@ class FastSentenceTransformer(FastModel):return st_model- # Warn if using 4-bit with encoder (slow due to dequantization overhead)+ # Warn if using 4-bit with an encoder: it is slow due to dequantization overhead.if is_encoder_model and load_in_4bit:print("Unsloth Warning: 4-bit quantization adds ~2.3x overhead for encoder models.")print("Consider using load_in_16bit=True for better performance.")@@ -1813,15 +1765,14 @@ class FastSentenceTransformer(FastModel):if supported:kwargs["add_pooling_layer"] = False- # fp8 is not supported, force it off+ # fp8 is not supported, so force it off.fp8 = kwargs.pop("load_in_fp8", None)if fp8:logging.info("Unsloth: Disabling fp8 for model")load_in_fp8 = False- # this is a fix for Snowflake/snowflake-arctic-embed-l-v2.0- # it has pooler weights which we don't care about for training,- # however unsloth throws an exception if "UNSLOTH_WARN_UNINITIALIZED" == 1 and it sees unused weights+ # Fix for Snowflake/snowflake-arctic-embed-l-v2.0: it has pooler weights irrelevant to training, but+ # unsloth throws when UNSLOTH_WARN_UNINITIALIZED == 1 and it sees unused weights.old_environ = os.environ.get("UNSLOTH_WARN_UNINITIALIZED", "1")os.environ["UNSLOTH_WARN_UNINITIALIZED"] = "0"@@ -1837,9 +1788,9 @@ class FastSentenceTransformer(FastModel):elif is_mpnet:FastSentenceTransformer._patch_mpnet_v5()- # No modules.json -> force 16-bit: saving is custom for these models and- # 4-bit would need dequant in save_pretrained_merged, not worth it.- # Resolve the warmed cache: hf_hub_download ignores SENTENCE_TRANSFORMERS_HOME, so pass it as cache_dir.+ # No modules.json means forcing 16-bit: saving is custom for these models and 4-bit would need+ # dequant in save_pretrained_merged. Resolve the warmed cache: hf_hub_download ignores+ # SENTENCE_TRANSFORMERS_HOME, so pass it as cache_dir.has_modules_json = (FastSentenceTransformer._module_path(model_name,@@ -1847,6 +1798,7 @@ class FastSentenceTransformer(FastModel):cache_dir = kwargs.get("cache_dir")or kwargs.get("cache_folder")or os.environ.get("SENTENCE_TRANSFORMERS_HOME"),+ # Same revision as the weight load so modules hit the warm (None = default branch).revision = revision,)is not None@@ -1860,20 +1812,15 @@ class FastSentenceTransformer(FastModel):load_in_4bit = Falseload_in_16bit = True- # The fallback FastModel load reads HF cache_dir, not ST's cache_folder/SENTENCE_TRANSFORMERS_HOME.- # Point it at the warmed cache, but only when no explicit cache_dir was passed (which wins).+ # The fallback FastModel load reads HF cache_dir, not ST's cache_folder/SENTENCE_TRANSFORMERS_HOME,+ # so point it at the warmed cache, but only when no explicit cache_dir was passed._st_cache_dir = kwargs.get("cache_folder") or os.environ.get("SENTENCE_TRANSFORMERS_HOME")if _st_cache_dir is not None and "cache_dir" not in kwargs:kwargs["cache_dir"] = _st_cache_dir- # The decline above only spends the sentinel on our copy. Strip the marker before- # the nested load, or FastModel reads it as "nobody chose this" and re-upgrades it- # under UNSLOTH_AUTO_DEVICE_MAP, splitting a model `st_device` then pulls back onto- # one card. Only the marker -- an explicit dict placement must arrive as a dict.- #- # A plain value rather than pinning the env var around the call: os.environ is- # process-wide, so that pin reached unrelated loads on other threads, and two- # overlapping sentence loads could restore it out of order.+ # The decline above only spends the sentinel on our copy: strip the marker before the nested load,+ # or FastModel reads it as "nobody chose this" and re-upgrades it under UNSLOTH_AUTO_DEVICE_MAP.+ # A plain value rather than pinning the env var, since os.environ is process-wide.device_map = unmarked_device_map(device_map)try:model, tokenizer = FastModel.from_pretrained(@@ -1916,11 +1863,11 @@ class FastSentenceTransformer(FastModel):max_seq_length,pooling_mode,trust_remote_code = trust_remote_code,- # Same resolved cache as above so the fallback module loads hit the warm, not Xet.+ # Same resolved cache as above so the fallback module loads hit the warm, not Xet, and the same+ # revision as the weight load (None = default branch).cache_dir = kwargs.get("cache_dir")or kwargs.get("cache_folder")or os.environ.get("SENTENCE_TRANSFORMERS_HOME"),- # Same revision as the weight load so modules hit the warm (None = default branch).revision = revision,)@@ -1940,15 +1887,12 @@ class FastSentenceTransformer(FastModel):save_method = "merged_16bit",**kwargs,):- # Positional to match FastLanguageModel.save_pretrained_merged;- # see the note on the other definition above. This path forwards to- # that merge, which understands every save_method.+ # Positional to match FastLanguageModel.save_pretrained_merged; see the note on the other definition+ # above. This path forwards to that merge, which understands every save_method.save_method = _normalize_save_method(save_method)if self.no_modules and save_method not in ("merged_16bit", None):- # The no_modules branch below merges and unloads unconditionally- # and drops save_method, so accepting "lora" here would return- # full weights for a request to write adapters. Refuse before- # writing, as the fast-encoder definition above does.+ # The no_modules branch below merges and unloads unconditionally and drops save_method, so accepting+ # "lora" here would return full weights for a request to write adapters.raise NotImplementedError(f"Unsloth: save_method = {save_method!r} is not supported "f"for this SentenceTransformer: no modules.json was found, "@@ -1957,16 +1901,16 @@ class FastSentenceTransformer(FastModel):)if save_method is not None:kwargs.setdefault("save_method", save_method)- # check which adapter files exist before save_pretrainedadapter_files = ["adapter_model.safetensors", "adapter_config.json"]existing_before = {f for f in adapter_files if os.path.exists(os.path.join(save_directory, f))}+ # sentence-transformers config and modules are only saved if save_pretrained is called.# sentence-transformers config and modules only get saved if we call save_pretrainedself.save_pretrained(save_directory)- # remove LoRA adapters only if they were created by save_pretrained (not pre-existing)+ # Remove LoRA adapters only if they were created by save_pretrained, not pre-existing.for file in adapter_files:if file not in existing_before:try:@@ -1979,10 +1923,10 @@ class FastSentenceTransformer(FastModel):else:kwargs.pop("tokenizer", None)if self.no_modules:- # fallback for non-sentence-transformers models+ # Fallback for non-sentence-transformers models.print("Unsloth: No modules detected. Using standard merge_and_unload for saving...")safe_kwargs = kwargs.copy()- # filter out Unsloth-specific args that are not in huggingface's save_pretrained+ # Filter out Unsloth-specific args that are not in huggingface's save_pretrained.unsloth_args = ["save_method","temporary_location",@@ -2000,7 +1944,6 @@ class FastSentenceTransformer(FastModel):save_directory, tokenizer = tokenizer, **kwargs)- # add Unsloth branding to the generated READMEtry:FastSentenceTransformer._add_unsloth_branding(save_directory)except Exception as e:@@ -2084,7 +2027,7 @@ class FastSentenceTransformer(FastModel):print("Setting task_type to FEATURE_EXTRACTION")if isinstance(model, SentenceTransformer):- # Check if this is a fast encoder model (uses torch.compile instead of Unsloth patching)+ # Check whether this is a fast encoder model, which uses torch.compile instead of Unsloth patching.is_fast_encoder = getattr(model, "_unsloth_fast_encoder", False)if is_fast_encoder:@@ -2092,17 +2035,14 @@ class FastSentenceTransformer(FastModel):transformer_module = model[0]inner_model = transformer_module.auto_model- # Check if model is quantized (4-bit/8-bit)is_quantized = (getattr(inner_model, "is_quantized", False)or getattr(inner_model.config, "quantization_config", None) is not None)- # Track if gradient checkpointing was actually enabledgc_enabled = False- # this is needed when from_pretrained was called without gradient- # checkpointing but get_peft_model requests it+ # Needed when from_pretrained was called without gradient checkpointing but get_peft_model requests it.if use_gradient_checkpointing and use_gradient_checkpointing != False:import transformersfrom packaging.version import Version@@ -2115,7 +2055,6 @@ class FastSentenceTransformer(FastModel):elif model_type == "mpnet":FastSentenceTransformer._patch_mpnet_v5()- # Prepare for k-bit training if quantizedif is_quantized:from ._utils import prepare_model_for_kbit_training_gc_for_kbit = (@@ -2130,7 +2069,7 @@ class FastSentenceTransformer(FastModel):gc_enabled = bool(_gc_for_kbit)except ValueError as e:if "does not support gradient checkpointing" in str(e):- # Model doesn't support gradient checkpointing, disable it+ # The model does not support gradient checkpointing, so disable it.print(f"Unsloth Warning: {inner_model.__class__.__name__} does not support gradient checkpointing. Skipping.")@@ -2144,7 +2083,8 @@ class FastSentenceTransformer(FastModel):else:raise- # Enable gradient checkpointing if requested (only for non-quantized, since prepare_model handles it)+ # Enable gradient checkpointing if requested, only for non-quantized models, since prepare_model+ # handles the rest.elif use_gradient_checkpointing and use_gradient_checkpointing != False:if hasattr(inner_model, "gradient_checkpointing_enable"):try:@@ -2157,7 +2097,6 @@ class FastSentenceTransformer(FastModel):f"Unsloth Warning: {inner_model.__class__.__name__} does not support gradient checkpointing. Skipping.")- # Create LoRA configlora_config = LoraConfig(r = r,lora_alpha = lora_alpha,@@ -2168,16 +2107,15 @@ class FastSentenceTransformer(FastModel):task_type = kwargs.get("task_type", "FEATURE_EXTRACTION"),)- # Apply PEFT directly (not through FastModel)peft_model = peft_get_peft_model(inner_model, lora_config)- # Apply QAT if specifiedqat_scheme = kwargs.get("qat_scheme", None)if qat_scheme is not None:from ._utils import _prepare_model_for_qatpeft_model = _prepare_model_for_qat(peft_model, qat_scheme)- # Determine compile mode (only if not using gradient checkpointing)+ # Determine the compile mode, only if not using gradient checkpointing, and re-enable torch.compile+ # when gradient checkpointing was requested but could not be enabled.compile_mode = getattr(model, "_compile_mode", "default")# Re-enable torch.compile if gradient checkpointing was requested but couldn't be enabledif compile_mode is None and not gc_enabled:@@ -2186,9 +2124,8 @@ class FastSentenceTransformer(FastModel):"Unsloth: Re-enabling torch.compile since gradient checkpointing is not supported")- # Re-assign the peft model back to the transformer module.- # On sentence-transformers >=5.4 `auto_model` is a read-only property- # backed by `self.model`, so write to the backing attribute there.+ # Re-assign the peft model back to the transformer module; on sentence-transformers >= 5.4+ # auto_model is a read-only property backed by self.model, so write to the backing attribute there.if isinstance(getattr(type(transformer_module), "auto_model", None), property):transformer_module.model = peft_modelelse:@@ -2197,14 +2134,12 @@ class FastSentenceTransformer(FastModel):transformer_module, getattr(inner_model, "config", None))- # Store compile info for auto-compile at trainer time- # torch.compile is deferred until training starts so we can check max_steps+ # torch.compile is deferred until training starts so max_steps can be checked.if compile_mode is not None:model._compile_mode = compile_modemodel._compile_threshold = FastSentenceTransformer._estimate_compile_threshold(model)- # Flag to indicate compile has not been applied yetmodel._compile_pending = Trueprint(f"Unsloth: torch.compile will be applied automatically if max_steps > {model._compile_threshold}"@@ -2216,7 +2151,6 @@ class FastSentenceTransformer(FastModel):return model- # Original path for non-fast-encoder modelstransformer_module = model[0]inner_model = transformer_module.auto_model@@ -2240,9 +2174,8 @@ class FastSentenceTransformer(FastModel):**kwargs,)- # re-assign the peft model back to the transformer module.- # On sentence-transformers >=5.4 `auto_model` is a read-only property- # backed by `self.model`, so write to the backing attribute there.+ # Re-assign the peft model back to the transformer module; on sentence-transformers >= 5.4+ # auto_model is a read-only property backed by self.model, so write to the backing attribute there.if isinstance(getattr(type(transformer_module), "auto_model", None), property):transformer_module.model = peft_modelelse:@@ -2294,7 +2227,6 @@ def _patch_sentence_transformer_trainer():@wraps(_original_init)def _patched_init(self, *args, **kwargs):- # Extract model and training_argsmodel = kwargs.get("model") or (args[0] if args else None)training_args = kwargs.get("args") or (args[1] if len(args) > 1 else None)@@ -2307,7 +2239,7 @@ def _patch_sentence_transformer_trainer():max_steps = getattr(training_args, "max_steps", -1)compile_mode = getattr(model, "_compile_mode", "default")- # Re-estimate threshold now that training args are available+ # Re-estimate the threshold now that training args are available.batch_size = getattr(training_args, "per_device_train_batch_size", None)grad_accum = getattr(training_args, "gradient_accumulation_steps", None)max_seq_length = getattr(model, "max_seq_length", None)@@ -2340,7 +2272,6 @@ def _patch_sentence_transformer_trainer():)model._compile_pending = False- # Call original __init___original_init(self, *args, **kwargs)# Disable mixed precision when FORCE_FLOAT32 is active (matches rl.py behavior)@@ -2362,6 +2293,8 @@ def _patch_sentence_transformer_trainer():def _patch_st_trainer_load_from_checkpoint():+ # Parameterless modules (Pooling, Normalize) make next(module.parameters()) raise StopIteration; route+ # through the SentenceTransformer's device property instead.try:from sentence_transformers import SentenceTransformerTrainerexcept ImportError:@@ -2462,9 +2395,8 @@ def _patch_st_trainer_load_from_checkpoint():fresh = module_cls.load(module_dir)if not isinstance(fresh, module_cls):raise RuntimeError(f"Unsloth: Module {idx} reload returned wrong type.")- # Parameterless modules (Pooling, Normalize) make- # next(module.parameters()) raise StopIteration; route through- # the SentenceTransformer's device property instead.+ # Parameterless modules (Pooling, Normalize) make next(module.parameters()) raise StopIteration, so+ # route through the SentenceTransformer's device property instead.try:fresh.to(self.model.device)except AttributeError:diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 2296e61f8bd..bbfd6d8cfc5 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -104,7 +104,7 @@ import contextlibtry:from huggingface_hub.utils import get_tokenexcept:- # Old HF Hub versions <= 0.0.25+ # Old HF Hub versions <= 0.0.25.from huggingface_hub.utils._token import get_tokenfrom ..device_type import (get_device_stats,@@ -170,16 +170,16 @@ def _repair_dispatch_hooks(model):except ImportError:return 0- # dispatch passes the model's skip keys, and sets io_same_device on the ROOT- # only; setting it on a submodule copies every result back and forth.+ # dispatch sets io_same_device on the ROOT only; setting it on a submodule copies every result+ # back and forth.skip_keys = getattr(model, "_skip_keys_device_placement", None)io_same_device = not hasattr(model, "_hf_hook")targets = {name: device for name, device in device_map.items() if name}- # A coarse map covers an endpoint through an ANCESTOR and dispatch hooks it- # anyway, so iterating keys alone misses the rebuilt one. Endpoints only:- # repairing every covered descendant would hook thousands that never lost one.+ # A coarse map covers an endpoint through an ANCESTOR and dispatch hooks it anyway, so iterating+ # keys alone misses the rebuilt one. Endpoints only: repairing every covered descendant would+ # hook thousands that never lost one.for get in ("get_input_embeddings", "get_output_embeddings"):try:replaced = getattr(model, get, lambda: None)()@@ -212,7 +212,7 @@ def _repair_dispatch_hooks(model):if isinstance(device, int) and not isinstance(device, bool)else device)- # `.type`, not `==`: a map may carry `torch.device("cpu", 0)`.+ # .type, not ==: a map may carry torch.device("cpu", 0).device_kind = (execution_device.typeif isinstance(execution_device, torch.device)@@ -306,8 +306,8 @@ def _attach_bnb_multidevice_hooks("""if fast_inference:return- # Before the bnb gate: the rebuild happens whatever the quantization. Not under- # offload, where `_embedding_dispatch_device` READS this hook to place the ids.+ # Before the bnb gate: the rebuild happens whatever the quantization. Not under offload, where+ # _embedding_dispatch_device READS this hook to place the ids.if not offload_embedding:repaired = _repair_dispatch_hooks(model)if repaired:@@ -367,7 +367,7 @@ def _attach_bnb_multidevice_hooks(_stripped.append((param, key, param.__dict__.pop(key)))try:- # CUDA -> int index, non-CUDA -> type string ("cpu", "meta").+ # CUDA gets an int index, non-CUDA a type string ("cpu", "meta").device_map_int = {k: (v.index if v.type == "cuda" else v.type) if isinstance(v, torch.device) else vfor k, v in inferred_map.items()@@ -412,7 +412,7 @@ NUM_LOGITS_TO_KEEP = dict()def _unsloth_generate_accepts_kwarg(model, key):- # True if the top level accepts this generate kwarg (some models expose it on an inner forward only).+ # True if the top level accepts this generate kwarg; some models expose it on an inner forward only.try:model_args = set(inspect.signature(model.prepare_inputs_for_generation).parameters)except (TypeError, ValueError, AttributeError):@@ -427,8 +427,8 @@ def _unsloth_generate_accepts_kwarg(model, key):def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_device):# Lookup runs on the weight's current device (CPU when offloaded); the output returns to the- # decoder device read live from output_embeddings (lm_head, untied here) so it tracks- # model.to() moves. A meta (disk-offloaded) or missing lm_head falls back to return_device.+ # decoder device read live from output_embeddings, so it tracks model.to() moves. A meta or+ # missing lm_head falls back to return_device.if embed_tokens is None:return Falseif getattr(embed_tokens, "_unsloth_offload_hooks_installed", False):@@ -465,7 +465,7 @@ def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_devdef _embeddings_are_tied(input_embeddings, output_embeddings):- # Tied lm_head reuses this weight; offloading to CPU would strand the output projection.+ # A tied lm_head reuses this weight, so offloading to CPU would strand the output projection.if input_embeddings is None or output_embeddings is None:return Falsew_in = getattr(input_embeddings, "weight", None)@@ -485,14 +485,14 @@ def _offload_embedding_unsupported_platform():def _embedding_dispatch_device(input_embeddings):- # accelerate's hook wraps forward and re-sends the ids to the device it recorded, after- # our offload pre-hook already sent them to the CPU weight. Returns that device, or None.+ # accelerate's hook re-sends the ids to the device it recorded, after our offload pre-hook+ # already sent them to the CPU weight. Returns that device, or None.hook = getattr(input_embeddings, "_hf_hook", None)return None if hook is None else getattr(hook, "execution_device", None)-# Big in absolute terms and big on this card, since every lookup then crosses PCIe. 2.5 GiB-# is 16% of a 16 GB T4 and worth moving, 3% of an 80 GB card and not.+# Big in absolute terms and big on this card, since every lookup then crosses PCIe: 2.5 GiB is+# 16% of a 16 GB T4 and worth moving, 3% of an 80 GB card and not._OFFLOAD_EMBEDDING_MIN_BYTES = 2**30_OFFLOAD_EMBEDDING_MIN_FRACTION = 0.05@@ -555,9 +555,9 @@ def _resolve_offload_embedding(model, offload_embedding):if _embedding_dispatch_device(in_embed) is not None:return _decline("this model is dispatched across devices, which overrides the offload.")if is_distributed():- # The offload leaves embed_tokens on the CPU while the rest of the rank stays on- # CUDA. Under full finetuning it is trainable, and DDP with device_ids refuses a- # module whose trainable parameters span both, so the run dies before step one.+ # The offload leaves embed_tokens on the CPU while the rest of the rank stays on CUDA; under+ # full finetuning it is trainable, and DDP with device_ids refuses a module whose trainable+ # parameters span both.return _decline("a distributed launch cannot wrap a model split across CPU and GPU.")return _embedding_is_worth_offloading(in_embed) if automatic else True@@ -702,7 +702,7 @@ def _clear_generation_caches(model):del module._flex_attention_cacheexcept:pass- # Solves AttributeError: 'SlidingWindowLayer' object has no attribute 'max_batch_size'+ # Solves AttributeError: 'SlidingWindowLayer' object has no attribute 'max_batch_size'.if hasattr(module, "_cache") and "cache_utils" in str(module._cache.__class__):try:del module._cache@@ -720,7 +720,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):elif "input_features" in kwargs:input_ids = kwargs["input_features"]elif "inputs_embeds" in kwargs:- # canonical HF name for embedding inputs (e.g. multimodal generate)+ # Canonical HF name for embedding inputs (e.g. multimodal generate).input_ids = kwargs["inputs_embeds"]elif "input_embeds" in kwargs:input_ids = kwargs["input_embeds"]@@ -748,13 +748,12 @@ def unsloth_base_fast_generate(self, *args, **kwargs):is_vlm = is_vlm or hasattr(self.config, "vision_config")arch = self.config.architectures[0]- # Remove token_type_ids - WRONG for Gemma 3 since bidirectional attention+ # Removing token_type_ids is WRONG for Gemma 3, which uses bidirectional attention.if hasattr(self, "generate") and hasattr(self, "forward"):# did not combine with below since self might not have modelkeys = inspect.signature(self.forward).parameters.keys()if "token_type_ids" not in keys:kwargs.pop("token_type_ids", None)- # kwargs.pop("token_type_ids", None)# Vision processors emit mm_token_type_ids that generate() rejects (Qwen3-VL); unlike# logits_to_keep it is an incoming kwarg, so drop it when generate does not accept it.@@ -763,7 +762,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):):kwargs.pop("mm_token_type_ids", None)- # VLMs do not allow logits_to_keep+ # VLMs do not allow logits_to_keep.global NUM_LOGITS_TO_KEEPif arch not in NUM_LOGITS_TO_KEEP:m = self@@ -790,7 +789,6 @@ def unsloth_base_fast_generate(self, *args, **kwargs):kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)- # Get pixel values for VLMstry:kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype)except:@@ -800,8 +798,8 @@ def unsloth_base_fast_generate(self, *args, **kwargs):except:pass- # Mixed precision autocast. Stamped by from_pretrained: UNSLOTH_FORCE_FLOAT32 is- # process wide, so a later load would otherwise decide this model's rollouts.+ # Mixed precision autocast. Stamped by from_pretrained: UNSLOTH_FORCE_FLOAT32 is process wide,+ # so a later load would otherwise decide this model's rollouts.forced_float32 = getattr(self, "_unsloth_forced_float32", None)if forced_float32 is None:forced_float32 = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"@@ -809,10 +807,9 @@ def unsloth_base_fast_generate(self, *args, **kwargs):autocaster = torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = torch.float16)dtype = torch.float16else:- # CUDA autocast does not validate its dtype the way the CPU/XPU/MPS paths do,- # so autocast(dtype = torch.float32) enters ENABLED rather than turning into a- # no-op, and the compiled graph then returns inf/NaN logits. A float32 model has- # nothing to autocast to; disable instead of asking for a dtype that is not one.+ # CUDA autocast does not validate its dtype like the CPU/XPU/MPS paths, so+ # autocast(dtype=torch.float32) enters ENABLED instead of becoming a no-op and the compiled+ # graph returns inf/NaN logits. A float32 model has nothing to autocast to, so disable.autocaster = torch.autocast(device_type = DEVICE_TYPE_TORCH,dtype = dtype,@@ -821,13 +818,13 @@ def unsloth_base_fast_generate(self, *args, **kwargs):# Prepare LoRA# state_dict = convert_lora_modules(self, dtype = dtype)- # FlashAttention breaks on the forced static cache below (unfilled slots stay- # unmasked while decoding), so delegate after normalization but before it.+ # FlashAttention breaks on the forced static cache below (unfilled slots stay unmasked while+ # decoding), so delegate after normalization but before it._clear_generation_caches(self)if _uses_flash_attention_for_generation(self.config):- # Pin the literal "dynamic": None is merged back to the model default, and a- # static cache still arrives via kwargs / the caller's generation_config (TRL).- # The kwarg wins (update runs last); skip it when the caller passed a cache.+ # Pin the literal "dynamic": None is merged back to the model default, and a static cache still+ # arrives via kwargs or the caller's generation_config (TRL). The kwarg wins, since update runs+ # last; skip it when the caller passed a cache.if kwargs.get("past_key_values") is None:kwargs["cache_implementation"] = "dynamic"try:@@ -836,7 +833,6 @@ def unsloth_base_fast_generate(self, *args, **kwargs):finally:_clear_generation_caches(self)- # Set compile dynamic shapestorch._dynamo.mark_static(input_ids, 0)torch._dynamo.mark_dynamic(input_ids, 1)if "attention_mask" in kwargs:@@ -846,16 +842,15 @@ def unsloth_base_fast_generate(self, *args, **kwargs):torch._dynamo.mark_static(kwargs["token_type_ids"], 0)torch._dynamo.mark_dynamic(kwargs["token_type_ids"], 1)- # Fix generation_config- # Use hybrid if sliding window seen, otherwise try static+ # Fix generation_config: use hybrid if a sliding window was seen, otherwise try static.cache_implementation = getattr(self.config, "cache_implementation", None)if getattr(self, "_supports_static_cache", getattr(self, "_can_compile_fullgraph", True)):if os.environ.get("UNSLOTH_DISABLE_STATIC_GENERATION", "0") == "0":+ # Should work in latest transformers!cache_implementation = "static"elif Version(transformers_version) < Version("4.56.0.dev0"):cache_implementation = Noneelse:- # Should work in latest transformers!cache_implementation = "static"else:cache_implementation = None@@ -870,7 +865,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):cache_implementation = "hybrid"else:cache_implementation = "static"- # [TODO] Unsure why static fails+ # Static fails for reasons unknown.if do_bfloat16_mixed_precision:cache_implementation = None@@ -889,11 +884,10 @@ def unsloth_base_fast_generate(self, *args, **kwargs):finally:_clear_generation_caches(self)- # FastBaseModel.for_training(self)return output-# Offline helpers live in loader_utils.py (shared canonical source).+# Offline helpers live in loader_utils.py, the shared canonical source.from .loader_utils import (_get_effective_local_files_only,_hub_repo_or_local_path,@@ -947,7 +941,6 @@ def _construct_vlm_processor_fallback(local_files_only = local_files_only,revision = revision,)- # Load image processorimage_processor = AutoImageProcessor.from_pretrained(load_path,token = token,@@ -956,8 +949,8 @@ def _construct_vlm_processor_fallback(local_files_only = local_files_only,revision = revision,)- # Load tokenizer via PreTrainedTokenizerFast (bypasses tokenizer_class check).- # Resolve the cached snapshot first so transformers does not call model_info (#7481).+ # Load the tokenizer via PreTrainedTokenizerFast, bypassing the tokenizer_class check and+ # resolving the cached snapshot first so transformers does not call model_info (#7481).tok = _load_pretrained_tokenizer_fast(tokenizer_name,padding_side = "left",@@ -967,16 +960,16 @@ def _construct_vlm_processor_fallback(local_files_only = local_files_only,revision = revision,)- # Read tokenizer_config.json for special tokens: prefer the local file (offline- # / local checkpoint dir), else hf_hub_download with local_files_only forwarded.+ # Read tokenizer_config.json for special tokens: prefer the local file (offline / local+ # checkpoint dir), else hf_hub_download with local_files_only forwarded.try:import json as _jsontok_config = None_local_cfg = os.path.join(tokenizer_name, "tokenizer_config.json")if os.path.isdir(tokenizer_name):- # Local dir: read directly. A missing file raises a clear FileNotFoundError- # rather than letting hf_hub_download treat the path as a repo id.+ # Local dir: read directly, so a missing file raises a clear FileNotFoundError rather than+ # letting hf_hub_download treat the path as a repo id.if os.path.exists(_local_cfg):with open(_local_cfg, "r", encoding = "utf-8") as f:tok_config = _json.load(f)@@ -996,7 +989,6 @@ def _construct_vlm_processor_fallback()with open(config_path, "r", encoding = "utf-8") as f:tok_config = _json.load(f)- # Set model-specific special tokens and their IDsfor key in ("image_token","image_start_token",@@ -1013,11 +1005,10 @@ def _construct_vlm_processor_fallback(except Exception as _e:_fb_err = _e # remember (non-fatal here); surfaced only if no processor is built- # Find the processor class - try model_type first, then top-level config model_type+ # Find the processor class by model_type first: it may be a sub-model type ("lfm2" not+ # "lfm2_vl"), so fall back to the top-level config.model_type, which often has the mapping.proc_class_name = PROCESSOR_MAPPING_NAMES.get(model_type)if proc_class_name is None:- # model_type might be a sub-model type (e.g. "lfm2" instead of "lfm2_vl").- # Try the top-level config.model_type which often has the processor mapping.try:config = AutoConfig.from_pretrained(load_path,@@ -1036,7 +1027,6 @@ def _construct_vlm_processor_fallback(proc_class = getattr(transformers, proc_class_name, None)if proc_class is not None:processor = proc_class(image_processor = image_processor, tokenizer = tok)- # Copy chat_template from tokenizer to processor if neededif not getattr(processor, "chat_template", None) and getattr(tok, "chat_template", None):@@ -1102,7 +1092,6 @@ class FastBaseModel:auto_config = None,offload_embedding = OFFLOAD_EMBEDDING_AUTO,float32_mixed_precision = None, # Forces float32 mixed precision- # vLLM parametersfast_inference = False,gpu_memory_utilization = 0.5,float8_kv_cache = False,@@ -1112,13 +1101,11 @@ class FastBaseModel:unsloth_vllm_standby = False,load_in_fp8 = False, # fp8 LoRA (True, False, 'block')text_only = False,- # True when the caller already swapped a multimodal config for its text sub-config,- # so `auto_config` no longer describes the repo. Set by loader.py, and by the block- # below on the direct-call path.+ # True when the caller already swapped a multimodal config for its text sub-config, so+ # auto_config no longer describes the repo. Set by loader.py and by the block below.text_only_decoder = False,- # True when `auto_config` came from the caller. It cannot be inferred here:- # FastModel pops `config` out of kwargs before this sees them, so by now it looks- # exactly like one we resolved ourselves.+ # True when auto_config came from the caller. It cannot be inferred here: FastModel pops config+ # out of kwargs before this sees them, so it looks exactly like one we resolved ourselves.auto_config_from_caller = False,**kwargs,):@@ -1126,8 +1113,8 @@ class FastBaseModel:if auto_config is None and user_config is not None:auto_config = user_config- # Offline snapshot for the loads below; not popped, so the weight load still- # reads local_files_only from **kwargs. See _get_effective_local_files_only.+ # Offline snapshot for the loads below; not popped, so the weight load still reads+ # local_files_only from **kwargs. See _get_effective_local_files_only.local_files_only = _get_effective_local_files_only(kwargs)if unsloth_vllm_standby and os.environ.get("UNSLOTH_VLLM_STANDBY", "0") != "1":@@ -1142,13 +1129,12 @@ class FastBaseModel:if os.environ.get("UNSLOTH_MODEL_NAME", "") == "":os.environ["UNSLOTH_MODEL_NAME"] = model_name.lower()- # Read from kwargs, not the signature: the weight load below forwards **kwargs, so- # binding it would drop it there.+ # Read from kwargs, not the signature: the weight load below forwards **kwargs, so binding it+ # would drop it there._revision = kwargs.get("revision")_tokenizer_revision_arg = kwargs.pop("tokenizer_revision", None)if _revision is not None and fast_inference and is_vLLM_available():- # vLLM fetches the default branch, so pinning only the config and tokenizer- # would mix two refs in one model.+ # vLLM fetches the default branch, so pinning only the config and tokenizer would mix two refs in one model.logger.warning_once(f"Unsloth: Ignoring revision = `{_revision}` since vLLM loads weights from ""the default branch. Use `fast_inference = False` to load a pinned revision."@@ -1158,8 +1144,8 @@ class FastBaseModel:kwargs.pop("revision", None)_revision_repo = model_name # The repo the pin names, captured before any remap.- # Resolve text-only before the is_vlm / vLLM checks so is_vlm stays consistent;- # skip the vision tower only for families with their own text decoder (Gemma 3). #5816+ # Resolve text-only before the is_vlm / vLLM checks so is_vlm stays consistent; skip the vision+ # tower only for families with their own text decoder (Gemma 3) (#5816).if text_only and auto_config is None:auto_config = AutoConfig.from_pretrained(model_name,@@ -1187,12 +1173,10 @@ class FastBaseModel:# Pure text model requested text-only with a VLM auto class.auto_model = AutoModelForCausalLMis_vlm = auto_model in [AutoModelForVision2Seq, AutoModelForImageTextToText]- # A repo-code VLM may register only AutoModel / AutoModelForCausalLM (e.g.- # DeepSeek-OCR, Nemotron-VL), so auto_model is not a VLM class even though the- # config is a vision model. Keep is_vlm (auto-class derived) for processor- # selection below -- these repos ship no AutoProcessor -- but treat the model as a- # VLM on the vLLM path so a vision_config model is never silently loaded/converted- # as text-only. text_only resolves the tower away first, so honour that here.+ # A repo-code VLM may register only AutoModel / AutoModelForCausalLM (DeepSeek-OCR,+ # Nemotron-VL), so auto_model is not a VLM class though the config is a vision model. Keep+ # is_vlm for processor selection, but treat it as a VLM on the vLLM path so a vision_config+ # model is never silently loaded as text-only.is_vlm_config = is_vlm or (not text_only and hasattr(auto_config, "vision_config"))is_whisper = whisper_language is not None and whisper_task is not Noneauto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer@@ -1213,10 +1197,8 @@ class FastBaseModel:)if any(arch in VLLM_NON_LORA_VLM for arch in model_types):- # mllama is still only in vllm v0 https://arc.net/l/quote/llwkfgmu- # https://docs.vllm.ai/en/stable/models/supported_models.html#text-generation_1- # vLLM V0 does not support LoRA on multi modal models.- # TODO: Update this once vLLM V1 supports Llama 3.2 aka mllama+ # mllama is still only in vllm v0, and vLLM V0 does not support LoRA on multimodal+ # models. TODO: revisit once vLLM V1 supports Llama 3.2 (mllama).vllm_enable_lora = Falseos.environ["UNSLOTH_USE_NEW_MODEL"] = "1"@@ -1249,7 +1231,6 @@ class FastBaseModel:print(statistics)- # Warn about fast transfersif "HF_HUB_ENABLE_HF_TRANSFER" in os.environ:old_hf_transfer = os.environ["HF_HUB_ENABLE_HF_TRANSFER"]if old_hf_transfer in ("False", "false"):@@ -1265,14 +1246,14 @@ class FastBaseModel:if old_hf_transfer != "0":os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"- # For debugging - we use a download counter to see if environments are not breaking or if HF is down+ # A download counter, to see whether environments are breaking or HF is down.get_statistics(kwargs.get("local_files_only", False))- # The base + tokenizer prefetch runs AFTER the load-mode validation below, so an invalid+ # The base plus tokenizer prefetch runs AFTER the load-mode validation below, so an invalid# load_in_* combination fails without first downloading a snapshot.- # Whether float32 was asked for rather than arrived at by upcasting. Only an- # explicit request may suppress the V100/T4 float16 autocast (see #4082).+ # Whether float32 was asked for rather than arrived at by upcasting: only an explicit request+ # may suppress the V100/T4 float16 autocast (#4082).user_float32 = _requested_float32(dtype)if dtype is None:dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16@@ -1296,15 +1277,15 @@ class FastBaseModel:# Check for custom data-typescustom_datatype = Nonecorrect_dtype = None- # `custom_datatype` has to stay None when unset: the consumer below tests- # `is not None` and would walk every module of every model to `exec("")`.+ # custom_datatype has to stay None when unset: the consumer below tests `is not None` and would+ # walk every module of every model to exec("")._raw_custom_dtype, _code_is_trusted = trusted_custom_dtype()if _raw_custom_dtype != "":assert _raw_custom_dtype.count(";") >= 4checker, _dtype, _bnb_compute_dtype, _custom_datatype, execute_code = (_raw_custom_dtype.split(";", 4))- # Allow custom dtypes on all runs+ # Allow custom dtypes on all runs, or only on float16 datatypes.allow_all_runs = checker == "all"# Allow only on float16 datatypesallow_float16_runs = (checker == "float16" or checker == "torch.float16") and (@@ -1319,11 +1300,10 @@ class FastBaseModel:if _resolved_bnb_compute_dtype is not None:bnb_compute_dtype = _resolved_bnb_compute_dtypecorrect_dtype = bnb_compute_dtype- # The last two fields are code, run only when this process set the- # variable; an inherited environment picks dtypes but injects nothing.+ # The last two fields are code, run only when this process set the variable; an inherited+ # environment picks dtypes but injects nothing.if _code_is_trusted:custom_datatype = _custom_datatype- # Execute code as wellif len(execute_code.strip()) != 0:exec(execute_code)else:@@ -1345,15 +1325,14 @@ class FastBaseModel:revision = _revision,)model_class = resolve_model_class(auto_model, auto_config)- # Check if using forced float32 - we load it in bfloat16, then cast to float16!- # Resolved here rather than at the load, because attention resolution below and the- # device-map planner both have to size the same dtype the load will really use.+ # Forced float32 loads in bfloat16 then casts to float16. Resolved here, not at the load,+ # because attention resolution and the device-map planner both size the same dtype.torch_dtype = dtypeif do_forced_float32:torch_dtype = torch.bfloat16- # What attention actually runs in, not the checkpoint load dtype: the- # UNSLOTH_FORCE_CUSTOM_DTYPE families (csm, falcon_h1, nemotron_h) load float32 to keep- # Mamba precision, then cast projections back to correct_dtype (float16), so flash stays.+ # What attention actually runs in, not the load dtype: the UNSLOTH_FORCE_CUSTOM_DTYPE families+ # (csm, falcon_h1, nemotron_h) load float32 for Mamba precision then cast projections back to+ # correct_dtype, so flash stays.attn_dtype = correct_dtype if correct_dtype is not None else torch_dtypeattn_impl = resolve_attention_implementation(model_class,@@ -1363,15 +1342,15 @@ class FastBaseModel:dtype = attn_dtype,)- # Handle FP8 models: get_model_name has already redirected this to BF16 sibling if the model ships with- # FP8 weights. We just need to update it here for sanity.+ # FP8 models: get_model_name already redirected to the BF16 sibling if the model ships FP8+ # weights, so just update it here for sanity.auto_config.model_name = model_namekwargs["attn_implementation"] = attn_implbnb_config = Noneuser_quantization_config = kwargs.get("quantization_config", None)- # Check if model already has a non-bitsandbytes quantization config (e.g. compressed-tensors/NVFP4)+ # Check whether the model already has a non-bitsandbytes quantization config (compressed-tensors/NVFP4).from .loader_utils import (check_and_disable_bitsandbytes_loading,sync_unsloth_model_name_bnb_flags,@@ -1380,8 +1359,8 @@ class FastBaseModel:load_in_4bit, load_in_8bit, _ = check_and_disable_bitsandbytes_loading(auto_config, load_in_4bit = load_in_4bit, load_in_8bit = load_in_8bit)- # Correct UNSLOTH_MODEL_NAME's bnb tokens now that the effective bnb state is known- # (the per-load env was built before remap/disable). gpt-oss only; no-op otherwise.+ # Correct UNSLOTH_MODEL_NAME's bnb tokens now the effective bnb state is known (the per-load+ # env was built before remap/disable). gpt-oss only.sync_unsloth_model_name_bnb_flags(load_in_4bit, load_in_8bit)if full_finetuning and (load_in_4bit or load_in_8bit):@@ -1392,26 +1371,25 @@ class FastBaseModel:load_in_8bit = Falseload_in_16bit = False- # text_only loads the decoder alone, but the planner gets only `model_name` and- # rebuilds the whole VLM: it budgets a vision tower this load never creates and- # names `model.language_model.layers.0` where the standalone decoder has- # `model.layers.0`, so the first decoder weight ends up with no device set.+ # text_only loads the decoder alone, but the planner gets only model_name and rebuilds the whole+ # VLM: it budgets a vision tower this load never creates and names+ # model.language_model.layers.0 where the standalone decoder has model.layers.0._planner_skip_reason = ("text_only loads a decoder the repo config does not describe"if text_only_decoderelse None)- # Same failure from the other direction: `num_labels` (or an explicit `auto_model`)- # loads a task head whose `score` replaces the planned `lm_head`, and dispatch- # refuses a map with no `score.weight`.+ # Same failure from the other direction: num_labels (or an explicit auto_model) loads a task+ # head whose `score` replaces the planned lm_head, and dispatch refuses a map with no+ # score.weight.if _planner_skip_reason is None:_planner_skip_reason = planner_class_mismatch_reason(model_class,planner_model_class(auto_config, trust_remote_code = trust_remote_code),)- # The comparison above returns None for a concrete `PreTrainedModel` subclass:- # `resolve_model_class` reads `auto_model._model_mapping`, which only Auto classes- # have. Unknown is not compatible, so decline rather than plan the repo's class.+ # The comparison above returns None for a concrete PreTrainedModel subclass, since+ # resolve_model_class reads auto_model._model_mapping, which only Auto classes have. Unknown is+ # not compatible, so decline.if (_planner_skip_reason is Noneand model_class is None@@ -1421,17 +1399,16 @@ class FastBaseModel:"an explicit model class has no auto mapping, so the plan cannot be ""checked against the class the load builds")- # The weights load against the caller's config; the planner rebuilds the repo's from- # a name. Same class, another `num_hidden_layers` or `vocab_size`, and the map omits- # blocks or under-budgets weights. Only the caller knows, so do not guess.+ # The weights load against the caller's config while the planner rebuilds the repo's from a+ # name, so the same class with another num_hidden_layers or vocab_size gives a map that omits+ # blocks or under-budgets weights.if _planner_skip_reason is None and (user_config is not None or auto_config_from_caller):_planner_skip_reason = ("a caller-supplied config may not describe the repo the planner rebuilds")- # A no-op unless the caller asked for "unsloth" (or set UNSLOTH_AUTO_DEVICE_MAP);- # an already-planned map comes back unchanged, so a direct FastBaseModel call- # behaves the same as going through FastModel.+ # A no-op unless the caller asked for "unsloth" (or set UNSLOTH_AUTO_DEVICE_MAP); an+ # already-planned map comes back unchanged.device_map = resolve_unsloth_device_map(requested_device_map(device_map),model_name,@@ -1443,14 +1420,14 @@ class FastBaseModel:token = token,trust_remote_code = trust_remote_code,**planner_hub_kwargs(kwargs),- # The pin the config and weights below use; the default branch would size a- # different checkpoint than the one being loaded.+ # The pin the config and weights below use; the default branch would size a different+ # checkpoint than the one being loaded.revision = _revision,- # The dtype the load below is given: `from_pretrained` overrides config.json,- # so planning the checkpoint's own mis-sizes by 2x whenever it changed.+ # The dtype the load below is given: from_pretrained overrides config.json, so planning the+ # checkpoint's own mis-sizes by 2x whenever it changed.**add_dtype_kwargs(torch_dtype),- # A caller-supplied config overrides the flags: loader.py clears them when it- # forwards one, so the flags alone would size a 4bit load at full precision.+ # A caller-supplied config overrides the flags: loader.py clears them when it forwards one, so+ # the flags alone would size a 4bit load at full precision.**planner_quantization_kwargs(load_in_4bit = load_in_4bit,load_in_8bit = load_in_8bit,@@ -1467,9 +1444,9 @@ class FastBaseModel:"Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!")- # Prefetch the repo (killable child) so the in-process load below is a cache hit. vLLM owns the- # weight download only when actually available; if fast_inference was requested but vLLM is- # missing, the load falls through in-process, so weights must still be warmed here.+ # Prefetch the repo (killable child) so the in-process load is a cache hit. vLLM owns the+ # download only when actually available; if it was requested but is missing, the load falls+ # through in-process, so warm the weights here._vllm_owns_weights = fast_inference and is_vLLM_available()_prefetched = maybe_prefetch_hf_snapshot(model_name,@@ -1483,19 +1460,19 @@ class FastBaseModel:use_safetensors = kwargs.get("use_safetensors"),from_tf = kwargs.get("from_tf", False),from_flax = kwargs.get("from_flax", False),- # Bare load reads only ROOT weights; skip subdir weights. Ignored when a subfolder is set.+ # A bare load reads only ROOT weights, so skip subdir weights; ignored when a subfolder is set.weights_at_root = True,variant = kwargs.get("variant"), # forward so the warm keeps the variant .bingguf_file = kwargs.get("gguf_file"), # forward so the warm fetches the GGUF (else ignored))- # Child did the forced download; clear the flag so the load reuses the warm cache.+ # The child did the forced download; clear the flag so the load reuses the warm cache.if _prefetched and kwargs.get("force_download", False):kwargs["force_download"] = False- # Warm a SEPARATE tokenizer repo only (model_name is covered above). Not model_name here: this- # runs before fast_inference_setup may remap the repo, so it would warm the wrong one.+ # Warm a SEPARATE tokenizer repo only. Not model_name: this runs before fast_inference_setup+ # may remap the repo, so it would warm the wrong one._tokenizer_repo = (tokenizer_name if (isinstance(tokenizer_name, str) and tokenizer_name) else model_name)@@ -1515,9 +1492,8 @@ class FastBaseModel:)_skip_modules = SKIP_QUANTIZATION_MODULES.copy()- # Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers.- # Mamba fused kernels pass out_proj.weight directly to F.linear,- # which fails with quantized Params4bit. Skip out_proj from quantization.+ # Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers, whose fused kernels pass+ # out_proj.weight straight to F.linear and fail with quantized Params4bit, so skip out_proj.if any(mt == "nemotron_h" for mt in (model_types or [])):_skip_modules.append("out_proj")@@ -1563,13 +1539,13 @@ class FastBaseModel:else:os.environ["UNSLOTH_ENABLE_FULL_FINETUNING"] = "0"- # Fix AttributeError: 'BitsAndBytesConfig' object has no attribute 'get_loading_attributes'+ # Fix AttributeError: 'BitsAndBytesConfig' object has no attribute 'get_loading_attributes'.if bnb_config is not None and not hasattr(bnb_config, "get_loading_attributes"):bnb_config.get_loading_attributes = lambda *args, **kwargs: {}- # Cannot be None, since HF now checks for the config+ # Cannot be None, since HF now checks for the config.if load_in_4bit or load_in_8bit:- # Ignore load_in_4bit / load_in_8bit for MXFP4 - best to get config file+ # Ignore load_in_4bit / load_in_8bit for MXFP4; better to get the config file.if "gpt-oss-20b" in model_name.lower() or "gpt-oss-120b" in model_name.lower():passelse:@@ -1591,7 +1567,7 @@ class FastBaseModel:quantization_config = auto_config.quantization_configquant_method = quantization_config["quant_method"]- # Sometimes bitsandbytes_4bit + bitsandbytes_8bit is provided+ # Sometimes bitsandbytes_4bit + bitsandbytes_8bit is provided.if (quant_method == "bitsandbytes"and "bitsandbytes" not in AUTO_QUANTIZATION_CONFIG_MAPPING@@ -1605,14 +1581,13 @@ class FastBaseModel:quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]quantizer_kwargs = {}if quant_method == "compressed-tensors":- # Ignore thesepasselse:- # We cannot dequantize since gpt-oss-20b MXFP4 will now be gpt-oss-20b-BF16+ # Cannot dequantize, since gpt-oss-20b MXFP4 would become gpt-oss-20b-BF16.if load_in_16bit and "dequantize" in inspect.signature(quantizer).parameters:quantizer_kwargs["dequantize"] = Truetry:- # Sometimes this fails so we wrap it in a try except+ # Sometimes this fails, so wrap it in try/except.quantization_config = quantizer.from_dict(quantization_config, **quantizer_kwargs)@@ -1650,10 +1625,10 @@ class FastBaseModel:)offload_embedding = Falseif not fast_inference:- # Prevent load_in_fp8 from being forwarded into HF internal model loading+ # Prevent load_in_fp8 from being forwarded into HF internal model loading.load_in_fp8 = kwargs.pop("load_in_fp8", None)- # Transformers 5.x @strict config classes reject unexpected kwargs.- # Move config-level attributes onto the config object directly.+ # Transformers 5.x @strict config classes reject unexpected kwargs, so move config-level+ # attributes onto the config object directly._num_labels = kwargs.pop("num_labels", None)if _num_labels is not None:set_task_config_attr(model_config, "num_labels", _num_labels)@@ -1668,15 +1643,11 @@ class FastBaseModel:model_name,config = model_config,device_map = device_map,- # torch_dtype = torch_dtype, # Transformers removed torch_dtype- # quantization_config = bnb_config,token = token,trust_remote_code = trust_remote_code,- # attn_implementation = attn_implementation,**kwargs,)- # Must precede _attach_bnb_multidevice_hooks: it returns early- # while offload_embedding is True.+ # Must precede _attach_bnb_multidevice_hooks: it returns early while offload_embedding is True.offload_embedding = _resolve_offload_embedding(model,offload_embedding,@@ -1720,7 +1691,7 @@ class FastBaseModel:# Device-safe embedding offload._install_offload_embedding_hooks(embed_tokens, out_embed, _embed_device)- # Must free GPU memory otherwise will not free!+ # GPU memory must be freed explicitly or it will not be freed.clean_gpu_cache()gc.collect()else:@@ -1778,10 +1749,8 @@ class FastBaseModel:if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs:load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]- # Load vLLM firstllm = load_vllm(**load_vllm_kwargs)- # Convert to HF format_, quant_state_dict = get_vllm_state_dict(llm,config = model_config,@@ -1802,7 +1771,6 @@ class FastBaseModel:finally:raise_handler.remove()- # Return old flagos.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer# Check float32 norm weights@@ -1814,26 +1782,24 @@ class FastBaseModel:or "layer_norm" in name) and hasattr(module, "weight"):module._pre_set_compute_dtype = torch.float32- # Edit data-typesif custom_datatype is not None:with torch.no_grad():for jj, (name, module) in enumerate(model.named_modules()):exec(custom_datatype)- # Clear deleted GPU itemsfor _ in range(3):gc.collect()clean_gpu_cache()- # Counteract saved tokenizers+ # Counteract saved tokenizers.tokenizer_name = model_name if tokenizer_name is None else tokenizer_name- # The loader resolves this (caller's adapter repo vs resolved base model); the- # fallback below covers a direct call.+ # The loader resolves this (caller's adapter repo vs resolved base model); the fallback below+ # covers a direct call._tokenizer_revision = _tokenizer_revision_argif _tokenizer_revision is None and tokenizer_name == _revision_repo:_tokenizer_revision = _revision- # On the vLLM path the tokenizer warm was deferred (fast_inference_setup may remap model_name).- # Warm the now-final tokenizer repo so the load below hits the cache (a cached/local repo is a no-op).+ # On the vLLM path the tokenizer warm was deferred, since fast_inference_setup may remap+ # model_name, so warm the now-final tokenizer repo (cached or local is a no-op).if _vllm_owns_weights and isinstance(tokenizer_name, str) and tokenizer_name:maybe_prefetch_hf_snapshot(tokenizer_name,@@ -1845,7 +1811,7 @@ class FastBaseModel:tokenizer_only = True,)- # Fix _Unsloth_Patched_ prefix in local config files from old saves (issue #4085)+ # Fix the _Unsloth_Patched_ prefix in local config files from old saves (#4085).if os.path.isdir(tokenizer_name):import json as _jsonfor _cfg_name in (@@ -1867,8 +1833,8 @@ class FastBaseModel:except Exception:pass- # Functional load chain (AutoProcessor -> get_auto_processor -> manual VLM- # fallback); offline is already forced upstream. Surfaces the error for the retry.+ # Functional load chain (AutoProcessor -> get_auto_processor -> manual VLM fallback); offline is+ # already forced upstream, and the error is surfaced for the retry.def _acquire_processor(lfo):_err = None # underlying load failure (used by the entry-point retry)if (whisper_language and whisper_task) or auto_model.__name__.endswith(@@ -1916,8 +1882,8 @@ class FastBaseModel:# Swallow so the manual fallback / entry-point retry can run._tok = None- # Build the processor manually if it failed to load or silently degraded to- # a text-only tokenizer (no image_processor) for a VLM (issue #4085).+ # Build the processor manually if it failed to load or silently degraded to a text-only+ # tokenizer (no image_processor) for a VLM (#4085)._processor_is_degraded = (is_vlm and _tok is not None and not hasattr(_tok, "image_processor"))@@ -1937,26 +1903,25 @@ class FastBaseModel:if _fallback is not None:_tok = _fallbackelif _err is None or (_fb_err is not None and _is_offline_related_error(_fb_err)):- # Prefer a network fallback error over a permanent primary one so the- # offline retry still fires.+ # Prefer a network fallback error over a permanent primary one so the offline retry still fires._err = _fb_errreturn _tok, _errdef _is_degraded_vlm(_t):- # VLM that loaded only a text-only tokenizer (no image_processor).+ # A VLM that loaded only a text-only tokenizer (no image_processor).return is_vlm and _t is not None and not hasattr(_t, "image_processor")tokenizer, _primary_err = _acquire_processor(local_files_only)- # Online network failure/degrade: raise so @_offline_aware_load retries from cache.- # Permanent / missing-file errors propagate; when already offline keep what we got.+ # Online network failure or degrade: raise so @_offline_aware_load retries from cache.+ # Permanent / missing-file errors propagate, and when already offline keep what we got.if ((tokenizer is None or _is_degraded_vlm(tokenizer))and not local_files_onlyand _is_offline_related_error(_primary_err)):raise _primary_err- # Missing torchvision silently degrades a VLM processor to text-only; surface the- # real cause instead of a later collator error (#4202), incl. on a silent degrade.+ # Missing torchvision silently degrades a VLM processor to text-only; surface the real cause+ # instead of a later collator error (#4202).if is_vlm and (tokenizer is None or not hasattr(tokenizer, "image_processor")):if _missing_torchvision_error(_primary_err):raise ImportError(@@ -1970,8 +1935,8 @@ class FastBaseModel:f"Unsloth: Warning - VLM processor fallback returned None for model_type={model_type_arch}",file = sys.stderr,)- # Backwards compat: if processor has no chat_template (e.g. old saves without- # chat_template.jinja) but the inner tokenizer does, copy it to the processor.+ # Backwards compat: if the processor has no chat_template (old saves without chat_template.jinja)+ # but the inner tokenizer does, copy it to the processor.if (hasattr(tokenizer, "tokenizer")and getattr(tokenizer, "chat_template", None) is None@@ -1981,9 +1946,7 @@ class FastBaseModel:if hasattr(tokenizer, "tokenizer"):__tokenizer = tokenizer.tokenizer- # Add padding side as well__tokenizer.padding_side = "left"- # Check bos, eos, pad tokensif hasattr(__tokenizer, "bos_token"):tokenizer.bos_token = __tokenizer.bos_tokentokenizer.bos_token_id = __tokenizer.bos_token_id@@ -2006,7 +1969,7 @@ class FastBaseModel:try:model, tokenizer = patch_tokenizer(model, tokenizer)except Exception as _patch_err:- # Some VLM processors (e.g. ERNIE VL) fail patching; fall back to AutoTokenizer.+ # Some VLM processors (ERNIE VL) fail patching; fall back to AutoTokenizer.try:from transformers import AutoTokenizer as _AutoTokenizer@@ -2020,24 +1983,24 @@ class FastBaseModel:revision = _tokenizer_revision,)model, _fallback_tok = patch_tokenizer(model, _fallback_tok)- # Re-attach as processor wrapper if original was a processor+ # Re-attach as a processor wrapper if the original was a processor.if hasattr(tokenizer, "image_processor"):tokenizer.tokenizer = _fallback_tokelse:tokenizer = _fallback_tokexcept Exception as _fb_err:- # Online network failure: propagate for the offline retry; else raise the patch error.+ # Online network failure: propagate for the offline retry, else raise the patch error.if not local_files_only and _is_offline_related_error(_fb_err):raiseraise _patch_errmodel = post_patch_loss_function(model)- # Log Unsloth version for future fastpaths for inference+ # Log the Unsloth version for future inference fastpaths.if hasattr(model, "config"):model.config.update({"unsloth_version": __version__})patch_saving_functions(model, vision = True)if tokenizer is None:- # Last resort: AutoTokenizer, then PreTrainedTokenizerFast (raise on network failure to retry).+ # Last resort: AutoTokenizer, then PreTrainedTokenizerFast, raising on network failure to retry.def _last_resort_tokenizer(lfo):from transformers import AutoTokenizer as _AutoTokenizerload_path = _hub_repo_or_local_path(@@ -2087,19 +2050,18 @@ class FastBaseModel:) from _last_resort_errpatch_saving_functions(tokenizer, vision = True)- # Fix gradient accumulation. See issue #4982.+ # Fix gradient accumulation; see #4982.from transformers.trainer import Trainerapply_accepts_loss_kwargs_fix(model)patch_gradient_accumulation_fix(Trainer)- # Save tokenizer for inference purposes- tokenizer.padding_side = "left" # Force inference+ tokenizer.padding_side = "left"if hasattr(tokenizer, "tokenizer"):- tokenizer.tokenizer.padding_side = "left" # Force inference- # Audio feature extractors must stay right padded: left (a text setting,- # forwarded by from_pretrained) shifts Whisper mels and desyncs Gemma 4- # audio token counts (crash on transformers < 5.10).+ tokenizer.tokenizer.padding_side = "left"+ # Audio feature extractors must stay right padded: left (a text setting, forwarded by+ # from_pretrained) shifts Whisper mels and desyncs Gemma 4 audio token counts, crashing on+ # transformers < 5.10.feature_extractor = getattr(tokenizer, "feature_extractor", None)if (feature_extractor is not None@@ -2112,14 +2074,12 @@ class FastBaseModel:m._saved_temp_tokenizer = tokenizerm = m.modelm.max_seq_length = max_seq_length- # Save to modules as wellfor module in model.modules():module.max_seq_length = max_seq_lengthm._saved_temp_tokenizer = tokenizer# Prevent Transformers Trainer from auto-wrapping Unsloth LoRA models in DP._mark_unsloth_disable_data_parallel(model, disable = not full_finetuning)- # Patch generateif os.environ.get("UNSLOTH_DISABLE_FAST_GENERATION", "0") == "0" and hasattr(model, "generate"):@@ -2128,7 +2088,6 @@ class FastBaseModel:unsloth_base_fast_generate.__doc__ = model._old_generate.__doc__model.generate = types.MethodType(unsloth_base_fast_generate, model)model._unsloth_trust_remote_code = trust_remote_code- # Post patchesmodel = FastBaseModel.post_patch_model(model,use_gradient_checkpointing = use_gradient_checkpointing,@@ -2137,22 +2096,18 @@ class FastBaseModel:tokenizer = tokenizer,float32_mixed_precision = float32_mixed_precision,)- # Clear deleted GPU itemsfor _ in range(3):gc.collect()clean_gpu_cache()- # Saving restores sentencepiece assets from the repo name alone, which carries no- # branch. Stamped here, not per processor branch, so a fallback cannot lose it.+ # Saving restores sentencepiece assets from the repo name alone, which carries no branch.+ # Stamped here, not per processor branch, so a fallback cannot lose it._mark_loaded_revision(tokenizer, _tokenizer_revision)model = _mark_forced_float32(model, do_forced_float32)model = _mark_full_finetuning(model, full_finetuning)- # LAST, for the reason the llama loader repairs last: the attach above- # runs during the load, and `patch_model_and_tokenizer` further down- # REPLACES the embedding and lm_head with freshly built modules, which- # carry the weights over but not the `_hf_hook`. A model this loader- # split across cards would otherwise still meet the original- # cross-device `index_select`. Idempotent, so the earlier attach stands.+ # LAST, like the llama loader: patch_model_and_tokenizer below REPLACES the embedding and+ # lm_head with fresh modules carrying the weights but not the _hf_hook, so a model split across+ # cards would still meet the original cross-device index_select. Idempotent.if not fast_inference and not offload_embedding:try:_repaired = _repair_dispatch_hooks(model)@@ -2200,8 +2155,8 @@ class FastBaseModel:):if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")- # Full finetuning still compiles, so a stray pre-train forward can poison the- # cache; install the detector here too (it is idempotent).+ # Full finetuning still compiles, so a stray pre-train forward can poison the cache; install the+ # detector here too (idempotent)._unsloth_install_pretrain_detector(model)return modeltransformers_set_seed(random_state)@@ -2214,9 +2169,8 @@ class FastBaseModel:if isinstance(model, PeftModelForCausalLM):raise RuntimeError("Unsloth: You already added LoRA adapters to your model!")- # Remember whether the CALLER explicitly opted into audio. "all-linear" turns- # the flag on implicitly below, but an old unsloth_zoo that cannot do audio- # must not make a plain all-linear (text/vision) run fail.+ # Remember whether the CALLER explicitly opted into audio: "all-linear" turns the flag on+ # implicitly below, but an old unsloth_zoo without audio must not fail a plain all-linear run._audio_explicitly_requested = bool(finetune_audio_layers)if target_modules == "all-linear":finetune_vision_layers = True@@ -2224,11 +2178,9 @@ class FastBaseModel:finetune_attention_modules = Truefinetune_mlp_modules = Truefinetune_audio_layers = True- # Older unsloth_zoo (before get_peft_regex gained finetune_audio_layers) does- # not accept the kwarg. Pass it only when supported; if the caller EXPLICITLY- # asked for audio but it is unsupported, fail loudly rather than silently- # training a language-only adapter. (all-linear's implicit opt-in degrades- # gracefully instead of raising.)+ # Older unsloth_zoo (before get_peft_regex gained finetune_audio_layers) rejects the kwarg, so+ # pass it only when supported; an EXPLICIT audio request fails loudly rather than silently+ # training a language-only adapter, while all-linear's implicit opt-in degrades gracefully.if "finetune_audio_layers" in inspect.signature(get_peft_regex).parameters:_audio_kwargs = {"finetune_audio_layers": finetune_audio_layers}elif _audio_explicitly_requested:@@ -2238,19 +2190,13 @@ class FastBaseModel:)else:_audio_kwargs = {}- # Remember the caller's ORIGINAL explicit leaf list for MoE expert- # detection. When an explicit list is routed through get_peft_regex for- # family scoping below, the generated regex carries get_peft_regex's full- # "mlp|feed_forward|ffn|dense" component block even when the caller named- # only attention leaves (q/k/v/o_proj). Keying expert detection on that- # regex would train the experts for an attention-only request. The- # original list carries the true leaf intent, so use it for MoE detection;- # only the auto (None / "all-linear") path relies on the regex, whose mlp- # block is the sole remaining MLP-intent signal on fused-expert models.+ # Remember the caller's ORIGINAL explicit leaf list for MoE expert detection: routing it through+ # get_peft_regex adds the full "mlp|feed_forward|ffn|dense" block even for attention-only leaves,+ # so keying on that regex would train the experts. Only the auto path relies on the regex._moe_detect_target = target_modules if type(target_modules) in (list, tuple) else None- # get_peft_regex drops these (no attention/MLP ancestor) and LoRA on them never- # trains, so redirect before scoping, matching FastLanguageModel.+ # get_peft_regex drops these (no attention/MLP ancestor) and LoRA on them never trains, so+ # redirect before scoping, matching FastLanguageModel.target_modules, modules_to_save, _moved = _redirect_embedding_targets(target_modules,modules_to_save,@@ -2287,11 +2233,9 @@ class FastBaseModel:)else:assert type(target_modules) in (list, tuple, str)- # Route an explicit list through get_peft_regex when the caller scoped a- # layer family (one of the finetune_* below is off) OR opted into audio (so- # the new audio/embedder branches are considered). finetune_audio_layers is- # a POSITIVE term here: using `not finetune_audio_layers` would -- since it- # defaults False -- force every explicit list through the filter.+ # Route an explicit list through get_peft_regex when the caller scoped a layer family or opted+ # into audio, so the new audio/embedder branches are considered. finetune_audio_layers is a+ # POSITIVE term: negating it would force every explicit list through the filter._scoping = (not finetune_vision_layersor not finetune_language_layers@@ -2321,42 +2265,33 @@ class FastBaseModel:and hasattr(model.vllm_engine.llm_engine, "vllm_config")and getattr(model.vllm_engine.llm_engine.vllm_config, "lora_config", None) is None):- # If vLLM is being used but lora is not enabled, throw an error- # Ref https://github.com/vllm-project/vllm/blob/51ba839555a5d122eadd91e9c16463ac288f5fa1/vllm/v1/engine/processor.py#L148-L151+ # If vLLM is used but lora is not enabled, throw an error; see vllm v1/engine/processor.py#L148-L151.raise RuntimeError("Unsloth: LoRA is not enabled for this model!")if finetune_vision_layers:- # vLLM does not support LoRA on vision layers- # https://github.com/vllm-project/vllm/blob/main/vllm/lora/models.py#L471-L477- # TODO: Update this once vLLM V1 supports LoRA on vision layers (possibly not happening)+ # vLLM does not support LoRA on vision layers; see vllm lora/models.py#L471-L477.raise RuntimeError("Unsloth: Finetuning vision layers is not supported for fast_inference. Only text layers are supported!")if model.config.model_type in VLLM_NON_LORA_VLM:- # mllama is still only in vllm v0 https://arc.net/l/quote/llwkfgmu- # https://docs.vllm.ai/en/stable/models/supported_models.html#text-generation_1- # vLLM V0 does not support LoRA on multi modal models.- # TODO: Update this once vLLM V1 supports Llama 3.2 aka mllama+ # mllama is still only in vllm v0, and vLLM V0 does not support LoRA on multimodal+ # models. TODO: revisit once vLLM V1 supports Llama 3.2 (mllama).raise RuntimeError("Unsloth: LoRA finetuning for Llama 3.2 aka mllama models is not supported with fast_inference!")- # Clear deleted GPU itemsfor _ in range(3):gc.collect()clean_gpu_cache()max_seq_length = model.max_seq_length- # If we pass loftq_config = None we will get an error+ # Passing loftq_config = None gives an error.loftq_config = validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, model)- # Auto-detect MoE models and populate target_parameters for expert layers.- # Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an- # attention-only request does not train experts via get_peft_regex's mlp block,- # but only when MLP and language families are both still in scope. If the caller- # scoped MLP or language OFF (finetune_mlp_modules / finetune_language_layers- # False), the scoped regex already dropped the experts, so honor it instead of- # re-introducing the original list's gate/up/down leaves.+ # Auto-detect MoE models and populate target_parameters for expert layers. Prefer the caller's+ # ORIGINAL explicit leaf list over the scoped regex so an attention-only request does not train+ # experts, but only while MLP and language families are both in scope: with finetune_mlp_modules+ # or finetune_language_layers False the scoped regex already dropped the experts.if target_parameters is None:_moe_targets = _select_moe_detection_targets(_moe_detect_target,@@ -2366,9 +2301,8 @@ class FastBaseModel:)target_parameters = get_moe_target_parameters(model, _moe_targets)- # Per-expert Linear expert layouts (e.g. gpt-oss bnb-4bit) target experts via- # target_modules, not fused Parameters. Extend either form PEFT accepts: a leaf- # list (explicit) or a regex string (auto / all-linear / scoped). No-op otherwise.+ # Per-expert Linear layouts (gpt-oss bnb-4bit) target experts via target_modules, not fused+ # Parameters. Extend either form PEFT accepts: a leaf list, or a regex string._moe_module_detect = _select_moe_detection_targets(_moe_detect_target,target_modules,@@ -2401,7 +2335,6 @@ class FastBaseModel:n = max(1, min(int(finetune_last_n_layers), _total_layers))layers_to_transform = list(range(_total_layers - n, _total_layers))- # Get only allowed parameters for LoraConfiglocal_variables = {**locals(),**kwargs,@@ -2415,14 +2348,15 @@ class FastBaseModel:model,use_gradient_checkpointing = use_gradient_checkpointing,)- # Gemma4 ClippableLinear wraps nn.Linear -- PEFT can't inject LoRA on it directly.- # Monkey-patch PEFT to target the inner .linear child instead.+ # Gemma4 ClippableLinear wraps nn.Linear and PEFT cannot inject LoRA on it directly, so patch+ # PEFT to target the inner .linear child._clippable_linear_cls = Nonetry:from transformers.models.gemma4.modeling_gemma4 import (Gemma4ClippableLinear as _clippable_linear_cls,)except ImportError:+ # Respect whatever it was set beforepassif _clippable_linear_cls is not None:from peft.tuners.lora.model import LoraModel as _LoraModel@@ -2465,8 +2399,8 @@ class FastBaseModel:model = _get_peft_model(model, lora_config)- # PEFT may have wrapped an endpoint this load repaired; the hook stays on- # `base_layer` and the adapter branch reads the caller's tensor.+ # PEFT may have wrapped an endpoint this load repaired; the hook stays on base_layer and the+ # adapter branch reads the caller's tensor.try:_lifted = _lift_endpoint_hooks_onto_adapters(model)if _lifted:@@ -2477,16 +2411,12 @@ class FastBaseModel:except Exception as _exc:logger.warning_once(f"Unsloth: could not lift adapter hooks: {_exc}")- # Restore original PEFT methodif _clippable_linear_cls is not None:_LoraModel._create_and_replace = _original_car- # Apply QAT + LoRA if specifiedif qat_scheme is not None:print("Unsloth: Applying QAT to mitigate quantization degradation")model = _prepare_model_for_qat(model, qat_scheme)- # Fix LoraConfig.auto_mapping is Nonefix_lora_auto_mapping(model)- # Enable gradients on modules which are trainablerequires_grad_for_gradient_checkpointing(model)trust_remote_code = getattr(model, "_unsloth_trust_remote_code", False)model = FastBaseModel.post_patch_model(@@ -2495,17 +2425,14 @@ class FastBaseModel:trust_remote_code = trust_remote_code,)model.max_seq_length = max_seq_length- # Save to modules as wellfor module in model.modules():module.max_seq_length = max_seq_length- # Clear deleted GPU itemsfor _ in range(3):gc.collect()clean_gpu_cache()patch_saving_functions(model, vision = True)patch_peft_fast_inference(model)- # Add for_inference and for_trainingmodel.for_training = functools.partial(FastBaseModel.for_training, model)model.for_inference = functools.partial(FastBaseModel.for_inference, model)m = model@@ -2513,8 +2440,8 @@ class FastBaseModel:m.for_training = functools.partial(FastBaseModel.for_training, m)m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model- # Detect a stray pre-train forward so train() can drop the torch.compile- # graph cache it would otherwise poison (see prepare_for_training_mode).+ # Detect a stray pre-train forward so train() can drop the torch.compile graph cache it would+ # otherwise poison (see prepare_for_training_mode)._unsloth_install_pretrain_detector(model)model = _exclude_rope_inv_freq_from_ddp(model)return model@@ -2531,7 +2458,7 @@ class FastBaseModel:full_finetuning = os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1"if type(float32_mixed_precision) is bool:- # Respect whatever it was set before+ # Respect whatever it was set to before.passelse:float32_mixed_precision = True@@ -2539,8 +2466,8 @@ class FastBaseModel:# Use bfloat16 precision for full finetuningfloat32_mixed_precision = False- # VLMs can hit DDP "marked ready twice" with re-entrant checkpointing.- # See: https://github.com/unslothai/unsloth/issues/3713.+ # VLMs can hit DDP "marked ready twice" with re-entrant checkpointing (#3713), so under DDP skip+ # the offloaded/re-entrant checkpoint patch and default native checkpoint to non-reentrant.use_reentrant = not is_distributed()if not use_reentrant:# Under DDP, avoid the offloaded/re-entrant checkpoint patch.@@ -2567,18 +2494,14 @@ class FastBaseModel:float32_mixed_precision = float32_mixed_precision,patch_modules_to_save = True,)- # Persist the configured GC mode so the trainer restores it verbatim.- # for_inference() clears the module flags (GRPO does this every generation- # step), and a plain TrainingArguments defaults gradient_checkpointing=False,- # which would otherwise silently disable this setting at train time (#4735).+ # Persist the configured GC mode so the trainer restores it verbatim: for_inference() clears the+ # module flags every GRPO generation step, and TrainingArguments defaults+ # gradient_checkpointing=False, which would silently disable it at train time (#4735).model._unsloth_gradient_checkpointing = use_gradient_checkpointing- # Gemma3N audio conformer processes variable-length audio tensors- # that cause stride mismatches in AOT autograd compiled backward- # when non-reentrant checkpointing is used. The notebook or TRL- # may override gradient_checkpointing_kwargs with use_reentrant=False- # after this point, so we intercept gradient_checkpointing_enable- # to always force use_reentrant=True for Gemma3N.+ # The Gemma3N audio conformer's variable-length tensors cause stride mismatches in the AOT+ # autograd compiled backward under non-reentrant checkpointing, and TRL may override+ # gradient_checkpointing_kwargs later, so intercept the enable and force use_reentrant=True._model_type = getattr(getattr(model, "config", None), "model_type", "") or ""if "gemma3n" in _model_type.lower() or "gemma4" in _model_type.lower():_original_gc_enable = model.gradient_checkpointing_enable@@ -2600,7 +2523,6 @@ class FastBaseModel:raise RuntimeError("Unsloth: Unsuccessfully patched inner_training_loop")patch_saving_functions(model, vision = True)- # Patch tokenizer to pad to the leftm = modelwhile hasattr(m, "model"):if hasattr(m, "_saved_temp_tokenizer"):@@ -2613,11 +2535,9 @@ class FastBaseModel:# Prevent Transformers Trainer from auto-wrapping Unsloth LoRA models in DP._mark_unsloth_disable_data_parallel(model, disable = not full_finetuning)- # Clear deleted GPU itemsfor _ in range(3):gc.collect()clean_gpu_cache()- # Add for_inference and for_trainingmodel.for_training = functools.partial(FastBaseModel.for_training, model)model.for_inference = functools.partial(FastBaseModel.for_inference, model)m = model@@ -2625,10 +2545,9 @@ class FastBaseModel:m.for_training = functools.partial(FastBaseModel.for_training, m)m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model- # Set weight[padding_idx] = 0 for embeddings that are NOT tied with the- # lm_head. When weights are tied, zeroing the padding row also zeros- # the corresponding lm_head row, forcing logit = 0 for the pad token.- # Only do this if tokenizer is defined since eos_token == pad_token sometimes!+ # Set weight[padding_idx] = 0 only for embeddings NOT tied with the lm_head: when tied, zeroing+ # the padding row zeros the lm_head row and forces logit = 0 for the pad token. Only when the+ # tokenizer is defined, since eos_token == pad_token sometimes.pad_token_id = getattr(tokenizer, "pad_token_id", None)lm_head = getattr(model, "lm_head", None)lm_head_weight = getattr(lm_head, "weight", None) if lm_head is not None else None@@ -2644,7 +2563,6 @@ class FastBaseModel:module.padding_idx == pad_token_idand module.padding_idx < module.weight.shape[0]):- # Skip if tied to lm_headif (lm_head_weight is not Noneand module.weight.data_ptr() == lm_head_weight.data_ptr()@@ -2665,10 +2583,8 @@ class FastBaseModel:m.gradient_checkpointing = Falseif hasattr(m, "training"):m.training = False- # Pad tokenizer to the leftif hasattr(m, "_saved_temp_tokenizer"):m._saved_temp_tokenizer.padding_side = "left"- # Set a flag for generation!m._flag_for_generation = Truem = model@@ -2678,12 +2594,12 @@ class FastBaseModel:_for_inference(m)model.eval() # to turn off training on modules deeper in+ # Since transformers 4.53, this must be turned off explicitly.# Since transformers 4.53, must turn off explicitlyfor module in model.modules():if hasattr(module, "gradient_checkpointing"):module.gradient_checkpointing = False- # Also disable training for embeddings for NEFTuneif hasattr(model, "get_input_embeddings"):embeddings = model.get_input_embeddings()if hasattr(embeddings, "training"):@@ -2692,19 +2608,20 @@ class FastBaseModel:embeddings = model.get_output_embeddings()if hasattr(embeddings, "training"):embeddings.training = False- # Restore use_cache values that prepare_model_for_training disabled- # for gradient checkpointing (older unsloth_zoo has no restore helper)+ # Restore use_cache values that prepare_model_for_training disabled for gradient checkpointing+ # (older unsloth_zoo has no restore helper).+ # Restore use_cache values that prepare_model_for_training disabled for gradient checkpointing (older+ # unsloth_zoo has no restore helper)try:from unsloth_zoo.training_utils import restore_use_cacherestore_use_cache(model)except ImportError:pass- # Must disable returning hidden states in the case for GRPO+ # Returning hidden states must be disabled for GRPO, and returning logits enabled.os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0"- # Must enable returning logitsos.environ["UNSLOTH_RETURN_LOGITS"] = "1"- # Turn off skip guards and set stance to default+ # Turn off skip guards and set stance to default.if torch_compiler_set_stance is not None:torch_compiler_set_stance(stance = "default", skip_guard_eval_unsafe = False)return model@@ -2716,7 +2633,6 @@ class FastBaseModel:"Unsloth: I think you're passing a tokenizer, not the model to for_training!")- # Delete all fast inference lorasfor param in model.parameters():if hasattr(param, "_fast_lora"):del param._fast_lora@@ -2726,13 +2642,11 @@ class FastBaseModel:m.gradient_checkpointing = use_gradient_checkpointingif hasattr(m, "training"):m.training = True- # Pad tokenizer to the leftif hasattr(m, "_saved_temp_tokenizer"):m._saved_temp_tokenizer.padding_side = "right"- # Set a flag for generation!if hasattr(m, "_flag_for_generation"):try:- # A PEFT wrapper delegates the read but owns nothing to delete+ # A PEFT wrapper delegates the read but owns nothing to delete.del m._flag_for_generationexcept AttributeError:pass@@ -2744,12 +2658,12 @@ class FastBaseModel:_for_training(m)model.train() # to turn on training on modules deeper in+ # Since transformers 4.53, this must be turned on explicitly.# Since transformers 4.53, must turn on explicitlyfor module in model.modules():if hasattr(module, "gradient_checkpointing"):module.gradient_checkpointing = use_gradient_checkpointing- # Also re-enable training for embeddings for NEFTuneif hasattr(model, "get_input_embeddings"):embeddings = model.get_input_embeddings()if hasattr(embeddings, "training"):@@ -2758,21 +2672,24 @@ class FastBaseModel:embeddings = model.get_output_embeddings()if hasattr(embeddings, "training"):embeddings.training = True- # Re-disable use_cache if prepare_model_for_training had disabled it- # and for_inference restored it (record only exists after a disable)+ # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored+ # it; the record only exists after a disable.+ # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored it+ # (record only exists after a disable)if (use_gradient_checkpointingand getattr(model, "_unsloth_use_cache_originals", None) is not None):+ # Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts); see llama.py.try:from unsloth_zoo.training_utils import disable_use_cachedisable_use_cache(model)except ImportError:pass- # Can re-enable not returning logits+ # Not returning logits can be re-enabled.os.environ["UNSLOTH_RETURN_LOGITS"] = "0"- # Turn off skip guards and set stance to default+ # Turn off skip guards and set stance to default.if torch_compiler_set_stance is not None:torch_compiler_set_stance(stance = "default", skip_guard_eval_unsafe = False)return model@@ -2799,7 +2716,7 @@ def _iter_message_lists(example, column):def _local_path_from_video_value(video_path):- # data: URIs are inline payloads, not files, and contain no "://"+ # data: URIs are inline payloads, not files, and contain no "://".if video_path.startswith("data:"):return Noneif "://" not in video_path:@@ -2810,7 +2727,7 @@ def _local_path_from_video_value(video_path):from urllib.request import url2pathnameparsed = urlparse(video_path)- # RFC 8089: only an empty authority or "localhost" is the local machine+ # RFC 8089: only an empty authority or "localhost" is the local machine.if parsed.netloc and parsed.netloc != "localhost":return Nonepath = url2pathname(parsed.path)@@ -2855,8 +2772,8 @@ def check_dataset_for_missing_videos(passmissing = []- # Report each missing path once; only confirmed-existing paths enter- # `checked`, so retries after an error re-check previously missing files.+ # Report each missing path once; only confirmed-existing paths enter `checked`, so retries after+ # an error re-check previously missing files.seen_missing = set()if checked is None:checked = set()@@ -2900,7 +2817,7 @@ def check_dataset_for_missing_videos(return missing-# Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts); see llama.py.+# Auto-enable grouped-GEMM MoE (transformers < 5 ModuleList experts); see llama.py.try:from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moeFastBaseModel.from_pretrained = staticmethod(diff --git a/unsloth/ollama_template_mappers.py b/unsloth/ollama_template_mappers.pyindex 065165d5d79..f457caeaff9 100644--- a/unsloth/ollama_template_mappers.py+++ b/unsloth/ollama_template_mappers.py@@ -20,6 +20,7 @@ __all__ = [OLLAMA_TEMPLATES = {}+# =========================================== Unslothunsloth_ollama = '''@@ -36,6 +37,7 @@ SYSTEM """You are a helpful assistant to the user"""OLLAMA_TEMPLATES["unsloth"] = unsloth_ollama+# =========================================== Zephyrzephyr_ollama = '''@@ -72,10 +74,8 @@ PARAMETER min_p 0.1OLLAMA_TEMPLATES["chatml"] = chatml_ollama+# Ollama templates from ollama.com/library/mistral (v0.1 blob 22e1b2e8dc2f, v0.2 blob e6836092461f).# =========================================== Mistral-1-# Ollama from https://www.ollama.com/library/mistral-# Mistral v0.1 https://ollama.com/library/mistral:v0.1/blobs/22e1b2e8dc2f-# Mistral v0.2 https://ollama.com/library/mistral:v0.2/blobs/e6836092461fmistral_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """[INST] {{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} [/INST]"""@@ -83,8 +83,7 @@ PARAMETER stop "[INST]"PARAMETER stop "[/INST]"'''-# mistral:v0.3 https://ollama.com/library/mistral:v0.3/blobs/1ff5b64b61b9-# mistral-large https://ollama.com/library/mistral-large:latest/blobs/96adabcf2c08+# mistral:v0.3 blob 1ff5b64b61b9; mistral-large blob 96adabcf2c08.mistral_v03_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{- if .Messages }}@@ -439,8 +438,8 @@ OLLAMA_TEMPLATES["magistral"] = magistral_ollamaOLLAMA_TEMPLATES["codestral"] = codestral_ollama-# =========================================== Llama-2# Ollama from https://www.ollama.com/library/llama3+# =========================================== Llama-2llama_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """[INST] <<SYS>>{{ .System }}<</SYS>>@@ -453,8 +452,8 @@ PARAMETER min_p 0.1OLLAMA_TEMPLATES["llama"] = llama_ollama-# =========================================== Vicuna# Ollama from https://www.ollama.com/library/vicuna+# =========================================== Vicunavicuna_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}USER: {{ .Prompt }} {{ end }}ASSISTANT: {{ .Response }} {__EOS_TOKEN__}"""@@ -501,8 +500,8 @@ SYSTEM """Below are some instructions that describe some tasks. Write responsesOLLAMA_TEMPLATES["alpaca"] = alpaca_ollama-# =========================================== Gemma# Ollama from https://www.ollama.com/library/gemma+# =========================================== Gemmagemma_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """<start_of_turn>user@@ -540,9 +539,8 @@ PARAMETER min_p 0.1OLLAMA_TEMPLATES["gemma_chatml"] = gemma_chatml_ollama+# Same as Gemma 1, but with sliding window attention; ollama.com/library/gemma2 blob 6522ca797f47.# =========================================== Gemma 2-# Same as Gemma 1, but with sliding window attention!-# https://ollama.com/library/gemma2/blobs/6522ca797f47gemma2_ollama = gemma_ollama + "PARAMETER num_ctx 4096\n"OLLAMA_TEMPLATES["gemma2"] = gemma2_ollama@@ -550,8 +548,8 @@ OLLAMA_TEMPLATES["gemma2"] = gemma2_ollamagemma2_chatml_ollama = gemma_chatml_ollama + "PARAMETER num_ctx 4096\n"OLLAMA_TEMPLATES["gemma2_chatml"] = gemma2_chatml_ollama-# =========================================== Llama-3# Ollama from https://www.ollama.com/library/llama3+# =========================================== Llama-3llama3_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>@@ -573,8 +571,8 @@ OLLAMA_TEMPLATES["llama-3"] = llama3_ollamaOLLAMA_TEMPLATES["llama3"] = llama3_ollama-# =========================================== Phi-3# Ollama from https://www.ollama.com/library/phi3+# =========================================== Phi-3phi3_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{ if .System }}<|system|>@@ -595,7 +593,6 @@ OLLAMA_TEMPLATES["phi-3"] = phi3_ollamaOLLAMA_TEMPLATES["phi-35"] = OLLAMA_TEMPLATES["phi-3"]OLLAMA_TEMPLATES["phi-3.5"] = OLLAMA_TEMPLATES["phi-3"]-# =========================================== Llama-3.1"""No trimming in Llama 3.1 Instruct!Also an extra newline for Cutting Knowledge Date@@ -794,8 +791,8 @@ OLLAMA_TEMPLATES["llama-32-vision"] = llama_32_vision_ollamafor version in ("llama-3.2", "llama-3.3", "llama-32", "llama-33"):OLLAMA_TEMPLATES[version] = OLLAMA_TEMPLATES["llama-3.1"]+# tinyllama-chat from ollama.com/library/tinyllama blob af0ddbdaaa26.# =========================================== tinyllama-# tinyllama-chat https://ollama.com/library/tinyllama:latest/blobs/af0ddbdaaa26tinyllama_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """<|system|>@@ -813,9 +810,8 @@ SYSTEM """You are a helpful AI assistant."""OLLAMA_TEMPLATES["tinyllama"] = tinyllama_ollama+# Qwen2 blob 77c91b422cc9; Qwen2.5 from ollama.com/library/qwen2.5 blob eb4402837c78.# =========================================== Qwen 2/2.5-# Qwen2 https://ollama.com/library/qwen2:latest/blobs/77c91b422cc9-# Qwen2.5 from https://ollama.com/library/qwen2.5/blobs/eb4402837c78qwen25_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{- if .Messages }}@@ -987,7 +983,7 @@ _phi4_ollama_template = ("<|im_start|><|assistant|><|im_sep|>{{ .Response }}<|im_end|>")-# Ollama from https://www.ollama.com/library/phi4 is different+# The ollama.com/library/phi4 template is different.phi_4_ollama = f'''FROM {{__FILE_LOCATION__}}TEMPLATE """{_phi4_ollama_template}"""@@ -1057,8 +1053,8 @@ OLLAMA_TEMPLATES["phi-4-mini"] = phi_4_mini_ollamaOLLAMA_TEMPLATES["phi-4-mini-reasoning"] = phi_4_mini_reasoning_ollama-# =========================================== Gemma-3# Ollama from https://ollama.com/library/gemma3/blobs/e0a42594d802+# =========================================== Gemma-3gemma3_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{- range $i, $_ := .Messages }}@@ -1111,7 +1107,6 @@ OLLAMA_TEMPLATES["gemma3"] = gemma3_ollamaOLLAMA_TEMPLATES["gemma3-270m"] = gemma3_270m_ollama-# =========================================== Qwen-3# Ollama template for Qwen-3 (see https://ollama.com/library/qwen3/blobs/eb4402837c78)qwen3_ollama = '''FROM {__FILE_LOCATION__}@@ -1179,8 +1174,8 @@ OLLAMA_TEMPLATES["qwen-3"] = qwen3_ollamaOLLAMA_TEMPLATES["qwen3"] = qwen3_ollama-# =========================================== Gemma-3n# Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802+# =========================================== Gemma-3ngemma3n_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{- range $i, $_ := .Messages }}@@ -1214,9 +1209,10 @@ TEMPLATE """{{- range $i, $_ := .Messages }}OLLAMA_TEMPLATES["gemma-4"] = gemma4_ollamaOLLAMA_TEMPLATES["gemma4"] = gemma4_ollama-# =========================================== GPT-OSS# Ollama from https://ollama.com/library/gpt-oss:latest/blobs/fa6710a93d78+# =========================================== GPT-OSS+gptoss_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.@@ -1401,7 +1397,6 @@ OLLAMA_TEMPLATES["gpt-oss"] = gptoss_ollamaOLLAMA_TEMPLATES["gptoss"] = gptoss_ollama-# =========================================== Qwen3# Ollama from https://ollama.com/library/qwen3/blobs/53e4ea15e8f5qwen3_ollama = '''@@ -1464,10 +1459,11 @@ OLLAMA_TEMPLATES["qwen3-instruct"] = qwen3_ollamaOLLAMA_TEMPLATES["qwen3-thinking"] = qwen3_ollama-# =========================================== Starling-LM# Ollama from https://ollama.com/library/starling-lm:7b/blobs/4b21bfc435b4+# =========================================== Starling-LM+starling_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{ if .System }}GPT4 Correct System: {{ .System }}<|end_of_turn|>@@ -1484,10 +1480,11 @@ PARAMETER min_p 0.1OLLAMA_TEMPLATES["starling"] = starling_ollama-# =========================================== Yi-chat# Ollama from https://ollama.com/library/yi:34b-chat/blobs/62fbfd9ed093+# =========================================== Yi-chat+yi_chat_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{ if .System }}<|im_start|>system@@ -1500,9 +1497,10 @@ TEMPLATE """{{ if .System }}<|im_start|>systemOLLAMA_TEMPLATES["yi-chat"] = yi_chat_ollama-# =========================================== Granite# Ollama from https://ollama.com/library/granite3.2:latest/blobs/3e7ca51acd6e+# =========================================== Granite+granite_32_ollama = '''FROM {__FILE_LOCATION__}TEMPLATE """{{- /*@@ -2219,7 +2217,6 @@ for key, values in OLLAMA_TEMPLATE_TO_MODEL_MAPPER.items():for value in values:MODEL_TO_OLLAMA_TEMPLATE_MAPPER[value] = key- # Get lowercasedlowered_key = key.lower()for value in values:MODEL_TO_OLLAMA_TEMPLATE_MAPPER[value.lower()] = lowered_keydiff --git a/unsloth/optimizers/q_galore_adamw.py b/unsloth/optimizers/q_galore_adamw.pyindex 9acd597f614..d3083aff8e9 100644--- a/unsloth/optimizers/q_galore_adamw.py+++ b/unsloth/optimizers/q_galore_adamw.py@@ -133,7 +133,6 @@ class QGaLoreAdamW8bit(Optimizer2State):has_weight_quant = self._has_weight_quant(p, group)- # --- Dequantize weight if INT8 ---if has_weight_quant:if p._q_scales is not None:float_weight = _dequantize(@@ -143,9 +142,7 @@ class QGaLoreAdamW8bit(Optimizer2State):p._q_shape,)p.data = float_weight- # else: first step, weights are still float — skip dequantize- # --- GaLore projection ---if "rank" in group:if "projector" not in state:state["projector"] = GaLoreProjector(@@ -161,8 +158,7 @@ class QGaLoreAdamW8bit(Optimizer2State):queue_size = group.get("queue_size", 5),)- # Temporarily disable weight decay for GaLore params- # (we apply it manually after project-back)+ # Temporarily disable weight decay for GaLore params; it is applied manually after project-back.if "weight_decay" in group and group["weight_decay"] > 0:group["_wd_saved"] = group["weight_decay"]group["weight_decay"] = 0@@ -174,19 +170,17 @@ class QGaLoreAdamW8bit(Optimizer2State):p.data = torch.zeros_like(grad, dtype = p.data.dtype, device = p.data.device)p.grad = grad- # --- 8-bit Adam update ---if "state1" not in state:self.init_state(group, p, gindex, pindex)self.prefetch_state(p)self.update_step(group, p, gindex, pindex)- # --- GaLore project-back ---if "rank" in group:- # p.data now holds the weight update in low-rank space+ # p.data now holds the weight update in low-rank space.p.data = p._saved_data.add_(state["projector"].project_back(p.data))- # Re-apply decoupled weight decay using pre-update weights+ # Re-apply decoupled weight decay using pre-update weights.if "_wd_saved" in group:p.data.add_(p.data,@@ -197,7 +191,6 @@ class QGaLoreAdamW8bit(Optimizer2State):del p._saved_data- # --- Re-quantize weight to INT8 ---if has_weight_quant:float_data = p.datastochastic = group.get("stochastic_round", True)@@ -208,9 +201,8 @@ class QGaLoreAdamW8bit(Optimizer2State):p._q_scales = scalesp._q_zeros = zerosp._q_shape = shape- # Scalar placeholder to free float memory; the forward- # pre-hook (install_weight_quant_hooks) dequantizes before- # the next forward pass.+ # Scalar placeholder to free float memory; the forward pre-hook (install_weight_quant_hooks)+ # dequantizes before the next forward pass.p.data = torch.empty(1, dtype = p.data.dtype, device = p.data.device)state["step"] += 1@@ -251,9 +243,8 @@ class QGaLoreAdamW8bit(Optimizer2State):for name, p in model.named_parameters():if id(p) in weight_quant_params:- # Store metadata without converting weights to uint8; the first- # step() quantizes after the update. Dummy scales/zeros keep- # _has_weight_quant() True on the first step.+ # Store metadata without converting weights to uint8; the first step() quantizes after the update,+ # and dummy scales/zeros keep _has_weight_quant() True on that first step.p._q_scales = Nonep._q_zeros = Nonep._q_shape = p.data.shape@@ -355,8 +346,8 @@ def make_q_galore_param_groups(if not param.requires_grad:continue- # Match target module names; exclude 1-D params (biases, norms) since- # GaLoreProjector.project requires 2-D gradients.+ # Match target module names and exclude 1-D params (biases, norms), since GaLoreProjector.project+ # requires 2-D gradients.name_parts = name.split(".")is_galore = param.dim() >= 2 and any(t in name_parts for t in targets)diff --git a/unsloth/optimizers/q_galore_projector.py b/unsloth/optimizers/q_galore_projector.pyindex 3fba75e5b39..cc1ef9ea9ea 100644--- a/unsloth/optimizers/q_galore_projector.py+++ b/unsloth/optimizers/q_galore_projector.py@@ -111,8 +111,6 @@ class GaLoreProjector:self.ortho_matrix_shape = None# ------------------------------------------------------------------- # Public API- # ------------------------------------------------------------------def project(self, full_rank_grad: torch.Tensor, step: int) -> torch.Tensor:"""Project a full-rank gradient into the low-rank subspace.@@ -131,7 +129,8 @@ class GaLoreProjector:assert self.proj_type == "std", "Only proj_type='std' is supported."if full_rank_grad.shape[0] >= full_rank_grad.shape[1]:- # "tall" matrix → right projection (grad @ Q^T)+ # A "tall" matrix takes the right projection (grad @ Q^T), a "wide" one the left (Q^T @ grad).+ # "wide" matrix → left projection (Q^T @ grad)if self.ortho_matrix is None or step % self.update_proj_gap == 0:float_ortho = self._compute_orthogonal(full_rank_grad,@@ -144,7 +143,6 @@ class GaLoreProjector:self._ortho_float_cache = self._load_ortho()low_rank_grad = torch.matmul(full_rank_grad, self._ortho_float_cache.t())else:- # "wide" matrix → left projection (Q^T @ grad)if self.ortho_matrix is None or step % self.update_proj_gap == 0:float_ortho = self._compute_orthogonal(full_rank_grad,@@ -181,8 +179,6 @@ class GaLoreProjector:return full_rank_grad * self.scale# ------------------------------------------------------------------- # SVD- # ------------------------------------------------------------------@staticmethoddef _compute_orthogonal(weights: torch.Tensor, rank: int, side: str) -> torch.Tensor:@@ -209,8 +205,8 @@ class GaLoreProjector:U, s, Vh = torch.linalg.svd(matrix, full_matrices = False)result = Vh[:rank, :] if side == "right" else U[:, :rank]else:- # Oversampling p=10 per Halko et al. 2009 (arXiv:0909.4061)- # recommendation of p=5..10 for large low-rank matrices.+ # Oversampling p=10, per Halko et al. 2009 (arXiv:0909.4061), which recommends p=5..10 for large+ # low-rank matrices.q = min(rank + 10, min(m, n))U, s, V = torch.svd_lowrank(matrix, q = q, niter = 2)result = V[:, :rank].t() if side == "right" else U[:, :rank]@@ -220,8 +216,6 @@ class GaLoreProjector:return result# ------------------------------------------------------------------- # Adaptive scheduling- # ------------------------------------------------------------------def _update_adaptive_schedule(self, float_ortho: torch.Tensor, side: str) -> None:"""Track subspace stability and increase ``update_proj_gap`` if stable."""@@ -246,8 +240,6 @@ class GaLoreProjector:self.past_ortho_vector = current_vector.clone()# ------------------------------------------------------------------- # Quantized projection matrix storage- # ------------------------------------------------------------------def _store_ortho(self, float_ortho: torch.Tensor) -> None:"""Store the orthogonal matrix, optionally quantized."""@@ -276,11 +268,6 @@ class GaLoreProjector:return self.ortho_matrix-# ======================================================================-# Quantization utilities (shared with the optimizer)-# ======================================================================--@torch.no_grad()def _quantize(w: torch.Tensor,@@ -318,7 +305,7 @@ def _dequantize(w: torch.Tensor, scales: torch.Tensor, zeros: torch.Tensor, original_shape: tuple) -> torch.Tensor:"""Dequantize from uint8 back to float."""- # Infer group size: scales has shape (n_groups, 1), so n_groups = scales.shape[0]+ # Infer the group size: scales has shape (n_groups, 1), so n_groups = scales.shape[0].total = w.numel()n_groups = scales.shape[0] if scales.dim() > 1 else scales.numel()group_size = total // n_groups if n_groups > 0 else totaldiff --git a/unsloth/registry/_deepseek.py b/unsloth/registry/_deepseek.pyindex 618453fb82f..ff3fc9607b9 100644--- a/unsloth/registry/_deepseek.py+++ b/unsloth/registry/_deepseek.py@@ -24,7 +24,6 @@ class DeepseekR1ModelInfo(ModelInfo):return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key)-# Deepseek V3 Model MetaDeepseekV3Meta = ModelMeta(org = "deepseek-ai",base_name = "DeepSeek",@@ -80,7 +79,6 @@ DeepseekR1DistillLlamaMeta = ModelMeta(quant_types = {"8": [QuantType.UNSLOTH, QuantType.GGUF], "70": [QuantType.GGUF]},)-# Deepseek R1 Distill Qwen Model MetaDeepseekR1DistillQwenMeta = ModelMeta(org = "deepseek-ai",base_name = "DeepSeek-R1-Distill",@@ -164,7 +162,7 @@ def _list_deepseek_r1_distill_models():for model in models:model_id = model.idmodel_name = model_id.split("/")[-1]- # parse out only the version+ # Parse out only the version.version = model_name.removeprefix("DeepSeek-R1-Distill-")distill_models.append(version)@@ -184,7 +182,3 @@ if __name__ == "__main__":print(f"\u2718 {model_id}")else:print(f"\u2713 {model_id}")- # distill_models = _list_deepseek_r1_distill_models()- # for model in sorted(distill_models):- # if "qwen" in model.lower():- # print(model)diff --git a/unsloth/registry/_gemma.py b/unsloth/registry/_gemma.pyindex 8b43dc35665..08b40dbd0e6 100644--- a/unsloth/registry/_gemma.py+++ b/unsloth/registry/_gemma.py@@ -11,7 +11,6 @@ class GemmaModelInfo(ModelInfo):return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key)-# Gemma3 Base Model MetaGemmaMeta3Base = ModelMeta(org = "google",base_name = "gemma",@@ -23,7 +22,6 @@ GemmaMeta3Base = ModelMeta(quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH],)-# Gemma3 Instruct Model MetaGemmaMeta3Instruct = ModelMeta(org = "google",base_name = "gemma",diff --git a/unsloth/registry/_phi.py b/unsloth/registry/_phi.pyindex 17d8f40a903..da33d64fcf5 100644--- a/unsloth/registry/_phi.py+++ b/unsloth/registry/_phi.py@@ -11,7 +11,6 @@ class PhiModelInfo(ModelInfo):return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key)-# Phi Model MetaPhiMeta4 = ModelMeta(org = "microsoft",base_name = "phi",@@ -23,7 +22,6 @@ PhiMeta4 = ModelMeta(quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH],)-# Phi Instruct Model MetaPhiInstructMeta4 = ModelMeta(org = "microsoft",base_name = "phi",diff --git a/unsloth/registry/_qwen.py b/unsloth/registry/_qwen.pyindex 8041ff21030..d754c633a96 100644--- a/unsloth/registry/_qwen.py+++ b/unsloth/registry/_qwen.py@@ -33,7 +33,6 @@ class QwenQVQPreviewModelInfo(ModelInfo):return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key)-# Qwen2.5 Model MetaQwen_2_5_Meta = ModelMeta(org = "Qwen",base_name = "Qwen",@@ -45,7 +44,6 @@ Qwen_2_5_Meta = ModelMeta(quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH],)-# Qwen2.5 VL Model MetaQwen_2_5_VLMeta = ModelMeta(org = "Qwen",base_name = "Qwen",@@ -57,7 +55,6 @@ Qwen_2_5_VLMeta = ModelMeta(quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH],)-# Qwen QwQ Model MetaQwenQwQMeta = ModelMeta(org = "Qwen",base_name = "QwQ",@@ -69,10 +66,8 @@ QwenQwQMeta = ModelMeta(quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH, QuantType.GGUF],)-# Qwen QVQ Preview Model Meta-# No QuantType.NONE: the unquantized mirror unsloth/QVQ-72B-Preview was removed-# from the Hub (only unsloth/QVQ-72B-Preview-bnb-4bit remains). The upstream-# Qwen/QVQ-72B-Preview is still registered via include_original_model.+# No QuantType.NONE: the unquantized mirror unsloth/QVQ-72B-Preview was removed from the Hub,+# leaving only the bnb-4bit build; upstream Qwen/QVQ-72B-Preview is still registered.QwenQVQPreviewMeta = ModelMeta(org = "Qwen",base_name = "QVQ",diff --git a/unsloth/registry/registry.py b/unsloth/registry/registry.pyindex 220dde99447..e853daf939c 100644--- a/unsloth/registry/registry.py+++ b/unsloth/registry/registry.py@@ -16,7 +16,6 @@ class QuantType(Enum):BF16 = "bf16" # only for Deepseek V3-# Tags for Hugging Face model pathsBNB_QUANTIZED_TAG = "bnb-4bit"UNSLOTH_DYNAMIC_QUANT_TAG = "unsloth" + "-" + BNB_QUANTIZED_TAGGGUF_TAG = "GGUF"@@ -31,7 +30,7 @@ QUANT_TAG_MAP = {}-# NOTE: models registered with org="unsloth" and QUANT_TYPE.NONE are aliases of QUANT_TYPE.UNSLOTH+# Models registered with org="unsloth" and QUANT_TYPE.NONE are aliases of QUANT_TYPE.UNSLOTH.@dataclassclass ModelInfo:org: str@@ -164,13 +163,13 @@ def _register_models(model_meta: ModelMeta, include_original_model: bool = Falsefor size in model_sizes:for instruct_tag in instruct_tags:- # Handle quant types per model sizeif isinstance(quant_types, dict):_quant_types = quant_types[size]else:_quant_types = quant_typesfor quant_type in _quant_types:- # NOTE: models registered with org="unsloth" and QUANT_TYPE.NONE are aliases of QUANT_TYPE.UNSLOTH+ # NOTE: models registered with org="unsloth" and QUANT_TYPE.NONE are aliases of+ # QUANT_TYPE.UNSLOTH_org = "unsloth" # quantized versions of the original modelregister_model(model_info_cls = model_info_cls,diff --git a/unsloth/save.py b/unsloth/save.pyindex 2c401d6b0ef..d3da157a764 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+# Defensive imports: added in unsloth-zoo PR #526 and may not exist on older versions.from unsloth_zoo.utils import Versionfrom importlib.metadata import version as importlib_versionfrom unsloth_zoo.hf_utils import dtype_from_config, HAS_TORCH_DTYPE@@ -25,16 +26,14 @@ from unsloth_zoo.llama_cpp import (_download_convert_hf_to_gguf,)-# H4: Defensive imports -- these were added in unsloth-zoo PR #526-# and may not exist on older versionstry:from unsloth_zoo.llama_cpp import LLAMA_CPP_DEFAULT_DIR, IS_WINDOWSexcept ImportError:import sysIS_WINDOWS = sys.platform == "win32"LLAMA_CPP_DEFAULT_DIR = "llama.cpp"-# Without bnb, peft stops exporting its 4bit LoRA layer too. Both names only feed-# isinstance checks, so placeholders nothing can match are exact stand-ins.+# Without bnb, peft stops exporting its 4bit LoRA layer too. Both names only feed isinstance checks, so+# placeholders nothing can match are exact stand-ins.try:from bitsandbytes.nn import Linear4bit as Bnb_Linear4bitfrom peft.tuners.lora import Linear4bit as Peft_Linear4bit@@ -84,7 +83,7 @@ except:try:from huggingface_hub.utils import get_tokenexcept:- # For older versions of huggingface_hub+ # For older versions of huggingface_hub.from huggingface_hub.utils._token import get_tokenfrom pathlib import Pathfrom peft import PeftModelForCausalLM, PeftModel@@ -97,18 +96,16 @@ __all__ = ["create_huggingface_repo",]-# llama.cpp specific targets - all takes 90s. Below takes 60s+# llama.cpp specific targets: all takes 90s, the below 60s.LLAMA_CPP_TARGETS = ["llama-quantize","llama-cli","llama-server",]-# Check environments. `is_kaggle_environment` needs a real Kaggle kernel, not-# merely a KAGGLE_* variable: KAGGLE_USERNAME / KAGGLE_KEY / KAGGLE_CONFIG_DIR-# are what the Kaggle CLI reads on an ordinary laptop, and every one of the-# Kaggle branches below (rewriting save paths to /tmp, deleting the cached base-# model) is wrong to take there.+# is_kaggle_environment needs a real kernel, not a KAGGLE_* variable: the Kaggle CLI reads those on an+# ordinary laptop, and the branches below (/tmp save paths, deleting the cached base model) are all wrong+# there.from .disk_utils import (KAGGLE_TMP,is_colab_environment,@@ -123,7 +120,6 @@ from .disk_utils import (IS_COLAB_ENVIRONMENT = is_colab_environment()IS_KAGGLE_ENVIRONMENT = is_kaggle_environment()-# WeightsLLAMA_WEIGHTS = ("self_attn.q_proj","self_attn.k_proj",@@ -142,8 +138,7 @@ LLAMA_LAYERNORMS = ("self_attn.k_norm",)-# https://github.com/ggerganov/llama.cpp/blob/master/examples/quantize/quantize.cpp#L19-# From https://mlabonne.github.io/blog/posts/Quantize_Llama_2_models_using_ggml.html+# See llama.cpp examples/quantize/quantize.cpp#L19 and mlabonne.github.io Quantize_Llama_2_models_using_ggml.ALLOWED_QUANTS = {"not_quantized": "Recommended. Fast conversion. Slow inference, big files.","fast_quantized": "Recommended. Fast conversion. OK inference, OK file size.",@@ -171,8 +166,8 @@ ALLOWED_QUANTS = {"q3_k_xs": "3-bit extra small quantization",}-# IQ (importance-matrix) quants. llama.cpp refuses these without an imatrix, so they are only-# accepted when imatrix_file=... is supplied to save_pretrained_gguf / push_to_hub_gguf.+# IQ quants need an importance matrix; llama.cpp refuses them without one, so they are only accepted when+# imatrix_file=... is supplied.IMATRIX_QUANTS = {"iq1_s": "1.56 bpw. Smallest, lowest quality. Needs an imatrix.","iq1_m": "1.75 bpw. Very small. Needs an imatrix.",@@ -207,14 +202,10 @@ def has_curl():CURL_FLAG = "-DLLAMA_CURL=ON" if has_curl() else "-DLLAMA_CURL=OFF"-# FP8/FP4 compressed export via llm-compressor (for vLLM).-# save_method alias -> (llm-compressor scheme, needs_calibration, output dir suffix).-# alias -> (llm-compressor scheme, needs_calibration, output-dir suffix). needs_calibration is-# True only for schemes with static activation scales (FP8 static, NVFP4); everything else is-# weight-only or dynamic-activation and runs data-free. Unsupported schemes in the installed-# compressed-tensors (e.g. MXFP8 on older stacks) are gated by _scheme_is_available at runtime.+# llm-compressor FP8/FP4 export for vLLM: alias -> (scheme, needs_calibration, dir suffix). needs_calibration+# only for static activation scales (FP8 static, NVFP4); the rest run data-free. Schemes absent from the+# installed compressed-tensors are gated at runtime.COMPRESSED_EXPORT_SCHEMES = {- # FP8"fp8": ("FP8_DYNAMIC", False, "fp8"),"fp8_dynamic": ("FP8_DYNAMIC", False, "fp8"),"dynamic_fp8": ("FP8_DYNAMIC", False, "fp8"),@@ -223,7 +214,6 @@ COMPRESSED_EXPORT_SCHEMES = {"static_fp8": ("FP8", True, "fp8-static"),"fp8_block": ("FP8_BLOCK", False, "fp8-block"),"block_fp8": ("FP8_BLOCK", False, "fp8-block"),- # INT8 / INT-weight"int8": ("INT8", False, "int8"),"w8a8": ("W8A8", False, "w8a8"),"w8a8_int8": ("W8A8", False, "w8a8"),@@ -235,14 +225,12 @@ COMPRESSED_EXPORT_SCHEMES = {"w4a16_asym": ("W4A16_ASYM", False, "w4a16-asym"),"w4a8": ("W4A8", False, "w4a8"),"w4afp8": ("W4AFP8", False, "w4afp8"),- # MXFP (microscaling)"mxfp8": ("MXFP8", False, "mxfp8"),"w8a8_mxfp8": ("MXFP8", False, "mxfp8"),"mxfp4": ("MXFP4", False, "mxfp4"),"w4a4_mxfp4": ("MXFP4", False, "mxfp4"),"mxfp4a16": ("MXFP4A16", False, "mxfp4a16"),"w4a16_mxfp4": ("MXFP4A16", False, "mxfp4a16"),- # NVFP4"nvfp4": ("NVFP4", True, "nvfp4"),"w4a4_nvfp4": ("NVFP4", True, "nvfp4"),"nvfp4a16": ("NVFP4A16", False, "nvfp4a16"),@@ -250,8 +238,8 @@ COMPRESSED_EXPORT_SCHEMES = {}-# torchao "portable" quant export: device-agnostic FP8 / INT8, no NVIDIA GPU needed.-# alias -> (kind, sibling suffix). FP8 saves to safetensors, INT8 to .bin; both load in vLLM.+# torchao portable export: device-agnostic FP8 / INT8, no NVIDIA GPU needed. alias -> (kind, sibling suffix);+# FP8 writes safetensors, INT8 .bin, both load in vLLM.TORCHAO_EXPORT_SCHEMES = {"torchao_fp8": ("fp8", "torchao-fp8"),"torchao_int8": ("int8", "torchao-int8"),@@ -287,8 +275,8 @@ def _loaded_via_remote_code(obj):if node is None or id(node) in seen:continueseen.add(id(node))- # __module__ can be None/absent on some dynamically created or C-extension classes;- # treat anything non-string as "not remote code" rather than crashing the export.+ # __module__ can be None on dynamically created or C-extension classes; treat a non-string as "not+ # remote code" rather than crashing the export.module = getattr(type(node), "__module__", None)if isinstance(module, str) and module.startswith("transformers_modules"):return True@@ -297,8 +285,8 @@ def _loaded_via_remote_code(obj):queue.append(node.get_base_model())except Exception:pass- # PEFT / trainer wrappers hold the real model in base_model / model; a built-in- # ProcessorMixin holds its (possibly custom-code) components as attributes.+ # PEFT / trainer wrappers hold the real model in base_model / model; a ProcessorMixin holds its+ # (possibly custom-code) components as attributes.for attr in ("base_model","model",@@ -344,7 +332,7 @@ def _is_cmake_only_llama_cpp(llama_cpp_dir: str = "llama.cpp") -> bool:"""makefile_path = os.path.join(llama_cpp_dir, "Makefile")if not os.path.exists(makefile_path):- # No Makefile: only CMake-only if a real CMake project is present+ # No Makefile: only CMake-only if a real CMake project is present.return os.path.exists(os.path.join(llama_cpp_dir, "CMakeLists.txt"))try:with open(makefile_path, "r", encoding = "utf-8", errors = "ignore") as f:@@ -382,8 +370,8 @@ def _quantize_q2_k_l(print_output: bool = True,imatrix = None,):- # "Q2_K_L" is an Unsloth preset, not a native llama.cpp ftype: q2_k with- # output/token-embedding tensors kept at q8_0 for higher precision.+ # "Q2_K_L" is an Unsloth preset, not a native llama.cpp ftype: q2_k with output and token embeddings kept+ # at q8_0.command = [str(quantizer_location),*(["--imatrix", str(imatrix)] if imatrix else []),@@ -496,7 +484,7 @@ def _has_tokenizer_model(tokenizer, token = None):if cache_key in _TOKENIZER_MODEL_CACHE:return _TOKENIZER_MODEL_CACHE[cache_key]- # Hub repo id: probe local cache before model_info (issue #7481).+ # Hub repo id: probe the local cache before model_info (#7481).cache_dir = _tokenizer_cache_dir(tokenizer) or os.environ.get("HF_HUB_CACHE")if not cache_dir:hf_home = os.environ.get("HF_HOME")@@ -617,8 +605,7 @@ def _free_cached_model(model):from huggingface_hub import scan_cache_dircached_repos = list(scan_cache_dir().repos)- # Go through every cached repo, and delete the one that matches the model we want to save.- # Can save 4GB of disk space - useful for Kaggle systems.+ # Delete the cached repo matching the model being saved: saves 4GB of disk, useful on Kaggle.for cached_repo in cached_repos:if cached_repo.repo_id == model.config._name_or_path:remove_cache_commit = list(cached_repo.revisions)[0].commit_hash@@ -637,7 +624,6 @@ def _free_cached_model(model):def _merge_lora(layer, name):bias = getattr(layer, "bias", None)if isinstance(layer, (Bnb_Linear4bit, Peft_Linear4bit, Peft_Linear)):- # Is LoRA so we need to merge!W, quant_state, A, B, s, bias = get_lora_parameters_bias(layer)if quant_state is not None:dtype = quant_state.dtype if type(quant_state) is not list else quant_state[2]@@ -645,14 +631,9 @@ def _merge_lora(layer, name):else:dtype = W.dtypeW = W.to(torch.float32).t()- # W = W.t()if A is not None:- # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32)))- # W += sABW.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s)- # W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s)- # if not torch.isfinite(W).all():maximum_element = torch.max(W.min().abs(), W.max())if not torch.isfinite(maximum_element).item():raise ValueError(f"Unsloth: Merge failed.\n{name} has some elements = infinity.")@@ -663,14 +644,11 @@ def _merge_lora(layer, name):def fast_save_pickle(shard, name):- # Use this if # CPUs is <= 2print(f"Unsloth: Saving {name}...")torch.save(shard,name,- # HIGHEST_PROTOCOL seems to not work with Pytorch!- # pickle_module = pickle,- # pickle_protocol = pickle.HIGHEST_PROTOCOL,+ # HIGHEST_PROTOCOL seems not to work with Pytorch.)return@@ -851,7 +829,6 @@ def unsloth_save_model(safe_serialization: bool = True,variant: Optional[str] = None,save_peft_format: bool = True,- # Push to hubuse_temp_dir: Optional[bool] = None,commit_message: Optional[str] = "Trained with Unsloth",private: Optional[bool] = None,@@ -859,7 +836,6 @@ def unsloth_save_model(revision: str = None,commit_description: str = "Upload model trained with Unsloth 2x faster",tags: List[str] = None,- # Our functionstemporary_location: str = "_unsloth_temporary_saved_buffers",maximum_memory_usage: float = 0.9,datasets: Optional[List[str]] = None,@@ -902,7 +878,6 @@ def unsloth_save_model():del save_pretrained_settings[deletion]- # First check for a token!if push_to_hub:from huggingface_hub import whoamitry:@@ -914,7 +889,6 @@ def unsloth_save_model(assert maximum_memory_usage > 0 and maximum_memory_usage <= 0.95- # Clean memory up firstfor _ in range(3):torch.cuda.empty_cache()gc.collect()@@ -932,7 +906,6 @@ def unsloth_save_model(print("Unsloth: Merging 4bit and LoRA weights to 4bit...")print("This might take 5 minutes...")- # Counteract no LoRA adapters!if hasattr(model, "merge_and_unload"):model = model.merge_and_unload()print("Done.")@@ -960,7 +933,6 @@ def unsloth_save_model(elif save_method == "merged_4bit":print("Unsloth: Saving 4bit Bitsandbytes model. Please wait...")- # Update model tag_ = upload_to_huggingface(model,save_directory,@@ -987,7 +959,6 @@ def unsloth_save_model(tags = tags,)if tokenizer is not None:- # Set padding side to left for inference_tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizerold_padding_side = _tokenizer.padding_side_tokenizer.padding_side = "left"@@ -1006,14 +977,12 @@ def unsloth_save_model(tags = tags,)- # Revert back padding side_tokenizer.padding_side = old_padding_sideif hasattr(model, "config"):print(f"Saved {save_method} model to https://huggingface.co/" + save_directory)return save_directory, None- # Tokenizer has different saving argumentstokenizer_save_settings = {"save_directory": save_pretrained_settings["save_directory"],"legacy_format": None,@@ -1023,7 +992,6 @@ def unsloth_save_model("token": save_pretrained_settings["token"],}- # Check if PEFT Model or not - if yes, 3 levels. If not 2 levels.from peft import PeftModelForCausalLMif isinstance(model, PeftModelForCausalLM):@@ -1031,16 +999,13 @@ def unsloth_save_model(else:internal_model = model- # Cannot be converted properly!if ((save_method == "merged_4bit")or (save_method == "lora")or (not hasattr(model, "model") or not hasattr(internal_model.model, "layers"))):- # Do general saving- # Edit save_pretrained_settings- # [TODO] _create_repo has errors due to **kwargs getting accepted- # commit_description does not seem to work?+ # _create_repo has errors due to **kwargs getting accepted, and commit_description does not seem to+ # work.what_to_delete = (("use_temp_dir",@@ -1068,7 +1033,6 @@ def unsloth_save_model(])- # Update model tagif push_to_hub:_ = upload_to_huggingface(model,@@ -1085,14 +1049,12 @@ def unsloth_save_model(if tokenizer is not None:print("Unsloth: Saving tokenizer...", end = "")- # Set padding side to left for inference_tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizerold_padding_side = _tokenizer.padding_side_tokenizer.padding_side = "left"tokenizer.save_pretrained(**tokenizer_save_settings)- # Revert back padding side_tokenizer.padding_side = old_padding_sideprint(" Done.")@@ -1103,7 +1065,7 @@ def unsloth_save_model(if save_method != "lora":print(" This might take 10 minutes for Llama-7b...", end = "")- # [TODO] Is this correct?+ # Correctness unverified.if save_method == "lora":save_pretrained_settings["selected_adapters"] = None@@ -1115,10 +1077,9 @@ def unsloth_save_model(print(" Done.")return save_directory, None- # If push_to_hub, we must remove the .../ part of a repo+ # With push_to_hub the ".../" part of a repo must be removed; the +1 solves absolute path issues.username = Noneif push_to_hub and "/" in save_directory:- # +1 solves absolute path issuesnew_save_directory = save_directoryusername = new_save_directory[: new_save_directory.find("/")]new_save_directory = new_save_directory[new_save_directory.find("/") + 1 :]@@ -1142,7 +1103,6 @@ def unsloth_save_model(print("Unsloth: Merging 4bit and LoRA weights to 16bit...")- # Determine max RAM usage minus shardingmax_ram = psutil.virtual_memory().availablesharded_ram_usage = 5 * 1024 * 1024 * 1024if type(max_shard_size) is str:@@ -1155,7 +1115,6 @@ def unsloth_save_model(elif type(max_shard_size) is int:sharded_ram_usage = max_shard_size- # Switch to our fast saving modules if it's a slow PC!n_cpus = psutil.cpu_count(logical = False)if n_cpus is None:n_cpus = psutil.cpu_count()@@ -1177,7 +1136,6 @@ def unsloth_save_model(save_pretrained_settings["safe_serialization"] = safe_serializationsave_pretrained_settings["save_function"] = save_function- # Only safe_serialization uses more RAMif safe_serialization:max_ram -= sharded_ram_usageelse:@@ -1190,24 +1148,20 @@ def unsloth_save_model(f"{round(psutil.virtual_memory().total / 1024 / 1024 / 1024, 2)} RAM for saving.")- # Move temporary_location to /tmp in Kaggleif IS_KAGGLE_ENVIRONMENT:temporary_location = os.path.join(KAGGLE_TMP, temporary_location)- # Max directory for disk savingif not os.path.exists(temporary_location):os.makedirs(temporary_location)- # Check if Kaggle or Colab, since only 20GB of Disk space allowed.+ # Kaggle and Colab allow only 20GB of disk, so free up 4GB of space.if IS_KAGGLE_ENVIRONMENT or IS_COLAB_ENVIRONMENT:- # We free up 4GB of spacelogger.warning_once("Unsloth: Kaggle/Colab has limited disk space. We need to delete the downloaded\n""model which will save 4-16GB of disk space, allowing you to save on Kaggle/Colab.")_free_cached_model(internal_model)- # HF also uses a OrderedDictfrom collections import OrderedDictstate_dict = OrderedDict()@@ -1219,13 +1173,12 @@ def unsloth_save_model(elif torch_dtype == "bfloat16":torch_dtype = torch.bfloat16- # Check modules to save float32 dtypestate_dict["model.embed_tokens.weight"] = internal_model.model.embed_tokens.weight.data.to(torch_dtype)- # A merged tensor lives on the GPU of its source layer, so budget against W's own- # device, not GPU0, else a sharded model OOMs GPU1+ while only GPU0 is checked.+ # A merged tensor lives on the GPU of its source layer, so budget against W's own device: a GPU0-only+ # check OOMs GPU1+ on a sharded model._max_vram_by_device = {}def _device_vram_budget(dev):@@ -1248,7 +1201,6 @@ def unsloth_save_model(name = f"model.layers.{j}.{item}.weight"W, bias = _merge_lora(proj, name)- # Bias termif bias is not None:state_dict[f"model.layers.{j}.{item}.bias"] = bias@@ -1262,14 +1214,8 @@ def unsloth_save_model(elif W.device.type != "cuda":# Already off-GPU: keeping it costs no VRAMstate_dict[name] = W- # [TODO] Saving to RAM seems to leak memory???- # elif (max_ram - W.nbytes) > 0:- # # Save to CPU memory- # logger.warning_once(f"We will save to RAM and not VRAM now.")- # state_dict[name] = W.to("cpu", non_blocking = True, copy = True)- # max_ram = max(max_ram - W.nbytes, 0)+ # Saving to RAM seems to leak memory.else:- # Save to Disklogger.warning_once("\nWe will save to Disk and not RAM now.")filename = os.path.join(temporary_location, f"{name}.pt")torch.save(@@ -1278,39 +1224,33 @@ def unsloth_save_model(pickle_module = pickle,pickle_protocol = pickle.HIGHEST_PROTOCOL,)- # weights_only = True weirdly fails?+ # weights_only = True weirdly fails.state_dict[name] = torch.load(filename, map_location = "cpu", mmap = True, weights_only = False)for item in LLAMA_LAYERNORMS:try:- # Skip for Gemma 2state_dict[f"model.layers.{j}.{item}.weight"] = eval(f"layer.{item}.weight.data")except:continuestate_dict["model.norm.weight"] = internal_model.model.norm.weight.data- # Check for modules_to_save float32 dtype- # Check for tied weightsif (internal_model.model.embed_tokens.weight.data_ptr()!= internal_model.lm_head.weight.data_ptr()):state_dict["lm_head.weight"] = internal_model.lm_head.weight.data.to(torch_dtype)- # All tensors MUST be type torch.Tensor and not torch.nn.parameter.Parameter+ # All tensors MUST be torch.Tensor and not torch.nn.parameter.Parameter.for key, value in state_dict.items():if hasattr(value, "data"):state_dict[key] = value = value.dataif type(value) is not torch.Tensor:logger.warning_once(f"Unsloth: {key} is not a Tensor but a {type(value)}.")- # Edit save_pretrained_settings- # [TODO] _create_repo has errors due to **kwargs getting acceptedsave_pretrained_settings["state_dict"] = state_dict- # commit_description does not seem to work?what_to_delete = (("use_temp_dir",@@ -1338,7 +1278,6 @@ def unsloth_save_model(])- # Update model tagif push_to_hub:_ = upload_to_huggingface(model,@@ -1352,7 +1291,6 @@ def unsloth_save_model(datasets = datasets,)- # First check if we're pushing to an organization!save_directory = save_pretrained_settings["save_directory"]if save_pretrained_settings["push_to_hub"]:@@ -1364,18 +1302,14 @@ def unsloth_save_model(else:actual_username = username- # Check if pushing to an organizationif save_pretrained_settings["push_to_hub"] and (username != actual_username):print(f"Unsloth: Saving to organization with address {new_save_directory}")- # We upload everything at the end!tokenizer_save_settings["push_to_hub"] = Falsetokenizer_save_settings["save_directory"] = new_save_directory- # Save tokenizerif tokenizer is not None:print("Unsloth: Saving tokenizer...", end = "")- # Set padding side to left for inference_tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizerold_padding_side = _tokenizer.padding_side_tokenizer.padding_side = "left"@@ -1387,14 +1321,12 @@ def unsloth_save_model(filename_prefix = tokenizer_save_settings.get("filename_prefix"),)- # Revert back padding side_tokenizer.padding_side = old_padding_sideprint(" Done.")else:print()- # Since merged, edit quantization_configold_config = model.confignew_config = model.config.to_dict()if "quantization_config" in new_config:@@ -1406,20 +1338,14 @@ def unsloth_save_model(original_model.config = new_configmodel.config = new_config- # Save!- # [TODO] --> is this correct?- # save_pretrained_settings["selected_adapters"] = None-- # Check if pushing to an organizationif save_pretrained_settings["push_to_hub"] and (username != actual_username):print(f"Unsloth: Saving to organization with address {new_save_directory}")- # Pushing to organization: .save_pretrained doesn't work, so save- # locally first then upload manually.+ # Pushing to an organization: .save_pretrained does not work, so save locally first and upload+ # manually.save_pretrained_settings["save_directory"] = new_save_directorysave_pretrained_settings["push_to_hub"] = Falseinternal_model.save_pretrained(**save_pretrained_settings)- # Now manually go through each file and upload them manually!filenames = os.listdir(new_save_directory)hf_api = HfApi(token = save_pretrained_settings["token"])@@ -1436,7 +1362,6 @@ def unsloth_save_model(else:internal_model.save_pretrained(**save_pretrained_settings)- # Revert config backoriginal_model = modelwhile hasattr(original_model, "model"):original_model = original_model.model@@ -1461,8 +1386,6 @@ def unsloth_save_model(torch.cuda.empty_cache()gc.collect()- # Remove temporary location-shutil.rmtree(temporary_location, ignore_errors = True)for _ in range(3):@@ -1485,15 +1408,12 @@ def install_llama_cpp_clone_non_blocking():def install_llama_cpp_make_non_blocking():- # https://github.com/ggerganov/llama.cpp/issues/7062- # Weirdly GPU conversion for GGUF breaks??- # env = { **os.environ, "LLAMA_CUDA": "1", }+ # GPU conversion for GGUF weirdly breaks; see ggerganov/llama.cpp#7062.- # Skip the make-clean probe on CMake-only checkouts (its error output is misleading)+ # Skip the make-clean probe on CMake-only checkouts, whose error output is misleading.IS_CMAKE = _is_cmake_only_llama_cpp("llama.cpp")if not IS_CMAKE:- # Confirm make still works, silentlytry:result = subprocess.run(["make", "clean", "-C", "llama.cpp"],@@ -1502,16 +1422,13 @@ def install_llama_cpp_make_non_blocking():)IS_CMAKE = result.returncode != 0except FileNotFoundError:- # No make executable; use CMakeIS_CMAKE = Trueif not IS_CMAKE:- # Uses old MAKEn_jobs = max(int((psutil.cpu_count() or 1) * 1.5), 1)full_command = ["make", "all", "-j" + str(n_jobs), "-C", "llama.cpp"]else:- # Uses new CMAKE- n_jobs = max(int(psutil.cpu_count() or 1), 1) # Use less CPUs since 1.5x faster+ n_jobs = max(int(psutil.cpu_count() or 1), 1)check = os.system(f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}")@@ -1520,7 +1437,6 @@ def install_llama_cpp_make_non_blocking():raise RuntimeError(f"*** Unsloth: Failed compiling llama.cpp using os.system(...) with error {check}. Please report this ASAP!")- # f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",full_command = ["cmake","--build",@@ -1531,9 +1447,6 @@ def install_llama_cpp_make_non_blocking():"--clean-first","--target",] + LLAMA_CPP_TARGETS- # https://github.com/ggerganov/llama.cpp/issues/7062- # Weirdly GPU conversion for GGUF breaks??- # run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)run_installer = subprocess.Popen(full_command, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)@@ -1542,24 +1455,20 @@ def install_llama_cpp_make_non_blocking():def install_python_non_blocking(packages = []):full_command = ["pip", "install"] + packages+ # GPU conversion for GGUF weirdly breaks; see ggerganov/llama.cpp#7062.run_installer = subprocess.Popen(full_command, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)return run_installer-# Bound the first-use auto-install so no unvetted release is pulled: not an inflated "0.999.0", nor-# a crafted higher in-range patch like "0.12.999" from a mirror. Cap to the exact vetted patch and-# bump deliberately. Floor 0.6.0 keeps torch>=2.4 resolvable (0.7+ need torch>=2.7; torch pinned below).+# Cap the first-use auto-install at the exact vetted patch, so no inflated "0.999.0" or in-range "0.12.999"+# from a mirror is pulled. Floor 0.6.0 keeps torch>=2.4 resolvable (0.7+ need torch>=2.7)._LLM_COMPRESSOR_SPEC = "llmcompressor>=0.6.0,<=0.12.0"-# Highest transformers release llm-compressor 0.10.x/0.12.x can run against (its metadata pins-# transformers<=4.57.6). Models that require a newer-transformers sidecar (e.g. Qwen3.5 needs-# transformers 5.3.0) cannot be quantized by llm-compressor at all: it imports-# transformers.modeling_utils.TORCH_INIT_FUNCTIONS, which was removed in transformers 5.x, so the-# compressed-export subprocess dies with a cryptic ImportError AFTER the expensive 16bit merge.-# Detect that up front and fail fast with an actionable message. Bump this in lockstep with a-# llm-compressor release that supports newer transformers.+# Highest transformers llm-compressor 0.10.x/0.12.x runs against (it pins <=4.57.6). It imports+# TORCH_INIT_FUNCTIONS, removed in transformers 5.x, so a newer-transformers model dies with a cryptic+# ImportError AFTER the expensive merge. Bump with llm-compressor._LLM_COMPRESSOR_MAX_TRANSFORMERS = "4.57.6"@@ -1587,11 +1496,8 @@ def _transformers_exceeds_llm_compressor_ceiling(transformers_version = None):return False, str(transformers_version)-# A caller (e.g. Unsloth Studio) can enable FP8/FP4 export of newer-transformers models (Qwen3.5,-# Gemma-4, ...) by provisioning a dedicated llm-compressor-main "shadow" (transformers>=5.9 layered-# over the existing torch) and pointing us at its sys.path entry via this env var. When set, the-# quantization subprocess uses it instead of the workspace llm-compressor and the ceiling fail-fast-# is bypassed.+# A caller (Unsloth Studio) can point this at an llm-compressor-main shadow (transformers>=5.9 over the+# existing torch); the subprocess then uses it and the ceiling check is bypassed._COMPRESSED_QUANTIZE_PYTHONPATH_ENV = "UNSLOTH_COMPRESSED_QUANTIZE_PYTHONPATH"@@ -1650,8 +1556,8 @@ def install_llm_compressor():except Exception:pass- # Prefer pip, but fall back to uv when this interpreter has no pip seeded (common in- # uv-created / relocatable venvs), so the export does not hard-fail with "No module named pip".+ # Prefer pip, falling back to uv when the interpreter has no pip seeded (uv-created or relocatable venvs),+ # so the export does not die on "No module named pip".import importlib.utilif importlib.util.find_spec("pip") is not None:@@ -1732,8 +1638,7 @@ def try_execute(commands, force_complete = False):def install_llama_cpp_old(version = -10):- # Download the 10th latest release since the latest might be broken!- # FALLBACK mechanism+ # Download the 10th latest release since the latest might be broken; this is the fallback mechanism.releases = subprocess.check_output(["git", "ls-remote", "--tags", "https://github.com/ggerganov/llama.cpp.git"])@@ -1745,7 +1650,6 @@ def install_llama_cpp_old(version = -10):latest = releases[-1]version = releases[version].split(" ")[0]- # Check if the llama.cpp existsif os.path.exists("llama.cpp"):print("**[WARNING]** You have a llama.cpp directory which is broken.\n"@@ -1760,19 +1664,15 @@ def install_llama_cpp_old(version = -10):shutil.rmtree("llama.cpp", ignore_errors = True)- # Clone a specific commit- # Also don't use the GPU!commands = ["git clone --recursive https://github.com/ggerganov/llama.cpp",f"cd llama.cpp && git reset --hard {version} && git clean -df",]try_execute(commands)- # Detect CMake-only build system before trying makeuse_cmake = _is_cmake_only_llama_cpp("llama.cpp")if not use_cmake:- # Try using MAKEcommands = ["make clean -C llama.cpp",f"make all -j{(psutil.cpu_count() or 1) * 2} -C llama.cpp",@@ -1780,7 +1680,6 @@ def install_llama_cpp_old(version = -10):use_cmake = try_execute(commands) == "CMAKE"if use_cmake:- # Use CMAKEcommands = [f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}",f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1) * 2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",@@ -1789,7 +1688,6 @@ def install_llama_cpp_old(version = -10):]try_execute(commands)- # Check if successfulif not (os.path.exists("llama.cpp/llama-quantize.exe")or os.path.exists("llama.cpp/llama-quantize")@@ -1806,9 +1704,7 @@ def install_llama_cpp_old(version = -10):def install_llama_cpp_blocking(use_cuda = False):- # https://github.com/ggerganov/llama.cpp/issues/7062- # Weirdly GPU conversion for GGUF breaks??- # use_cuda = "LLAMA_CUDA=1" if use_cuda else ""+ # GPU conversion for GGUF weirdly breaks; see ggerganov/llama.cpp#7062.commands = ["git clone --recursive https://github.com/ggerganov/llama.cpp",@@ -1818,21 +1714,16 @@ def install_llama_cpp_blocking(use_cuda = False):returntry_execute(commands)- # Detect CMake-only build system before trying makeuse_cmake = _is_cmake_only_llama_cpp("llama.cpp")if not use_cmake:commands = ["make clean -C llama.cpp",- # https://github.com/ggerganov/llama.cpp/issues/7062- # Weirdly GPU conversion for GGUF breaks??- # f"{use_cuda} make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp",f"make all -j{(psutil.cpu_count() or 1) * 2} -C llama.cpp",]use_cmake = try_execute(commands) == "CMAKE"if use_cmake:- # Use CMAKEcommands = [f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}",f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1) * 2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",@@ -1843,13 +1734,11 @@ def install_llama_cpp_blocking(use_cuda = False):def get_executable(executables):- # Get system locations (System Path).split(system separator)system_directories = os.environ.get("PATH").split(os.pathsep)for directory in system_directories:for executable in executables:path = os.path.join(directory, executable)- # Check if the executable exists and is executableif os.path.exists(path) and os.access(path, os.X_OK):return pathreturn None@@ -2105,17 +1994,15 @@ def save_to_gguf(`gguf_shard_size` applies to final f32, f16 and bf16 outputs. Quantized outputsremain single-file. None preserves the historical 50GB converter limit."""- # print_output True only if UNSLOTH_ENABLE_LOGGING=1+ # print_output is True only if UNSLOTH_ENABLE_LOGGING=1.if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1":print_output = Trueelse:print_output = False- # Validate model dtypeassert model_dtype == "float16" or model_dtype == "bfloat16"model_dtype = "f16" if model_dtype == "float16" else "bf16"- # Convert quantization_method to listif isinstance(quantization_method, list):passelif isinstance(quantization_method, str):@@ -2127,7 +2014,6 @@ def save_to_gguf(else:raise TypeError("Unsloth: quantization_method can only be a string or a list of strings")- # Check if bfloat16 is supportedif model_dtype == "bf16" and not torch.cuda.is_bf16_supported():logger.warning("Unsloth: Cannot convert to bf16 GGUF since your computer doesn't support it.\n"@@ -2137,8 +2023,8 @@ def save_to_gguf(has_imatrix = imatrix is not None and str(imatrix) != ""if has_imatrix:- # quantize_gguf gained the imatrix kwarg in a recent unsloth_zoo; fail fast (before the- # expensive conversion) if the installed version cannot apply it, rather than dropping it.+ # quantize_gguf gained the imatrix kwarg in a recent unsloth_zoo; fail fast before the expensive+ # conversion rather than silently dropping it.import inspectif "imatrix" not in inspect.signature(quantize_gguf).parameters:raise RuntimeError(@@ -2146,7 +2032,6 @@ def save_to_gguf("Please upgrade it: uv pip install --upgrade unsloth_zoo")- # Map quant methodsnew_quantization_methods = []for quant_method in quantization_method:if quant_method == "not_quantized":@@ -2177,11 +2062,10 @@ def save_to_gguf(new_quantization_methods.append(quant_method)quantization_method = new_quantization_methods- # Determine optimal first_conversionif is_gpt_oss:print("Unsloth: GPT-OSS model detected - using special conversion settings")- first_conversion = "None" # No quantization for GPT-OSS- # Only keep one conversion method since GPT-OSS doesn't quantize+ first_conversion = "None"+ # GPT-OSS does not quantize, so keep only one conversion method.quantization_method = ["None"]elif first_conversion is None:first_conversion = _choose_first_conversion(@@ -2190,7 +2074,6 @@ def save_to_gguf(has_imatrix = has_imatrix,)- # Check bfloat16 support again for first_conversionif first_conversion == "bf16" and not torch.cuda.is_bf16_supported():logger.warning("Unsloth: Switching bf16 to f16 due to hardware limitations")first_conversion = "f16"@@ -2203,7 +2086,6 @@ def save_to_gguf(quantization_method,is_vlm,)- # Print conversion infoneeds_quantize_pass = any(m != first_conversion for m in quantization_method)if needs_quantize_pass:second_step = f"[2] Converting GGUF {first_conversion_dtype} to {quantization_method} might take 10 minutes each."@@ -2220,31 +2102,28 @@ def save_to_gguf()print(print_info)- # Step 1: Ensure llama.cpp is installedtry:quantizer_location, converter_location = check_llama_cpp()print("Unsloth: llama.cpp found in the system. Skipping installation.")except:print("Unsloth: Installing llama.cpp. This might take 3 minutes...")if IS_KAGGLE_ENVIRONMENT:- # Kaggle: no CUDA support due to environment limitationsquantizer_location, converter_location = install_llama_cpp(gpu_support = False, print_output = print_output)else:+ # Kaggle: no CUDA support due to environment limitations.quantizer_location, converter_location = install_llama_cpp(- gpu_support = False, # GGUF conversion doesn't need CUDA+ gpu_support = False,print_output = print_output,)- # Step 2: Download and patch converter scriptprint("Unsloth: Preparing converter script...")with use_local_gguf():converter_path, supported_text_archs, supported_vision_archs = (_download_convert_hf_to_gguf())- # Step 3: Initial GGUF conversionprint(f"Unsloth: [1] Converting model into {first_conversion_dtype} GGUF format.")print(f"This might take 3 minutes...")@@ -2261,14 +2140,11 @@ def save_to_gguf(max_shard_size = converter_shard_size,print_output = print_output,)- # update is_vlm switchis_vlm = is_vlm_update- # Check conversion successfor file in initial_files:if not os.path.exists(file):- # Gated like the outer handler: the disk advice is only right when- # the disk is actually the problem, and a converter that has no- # llama.cpp support fails here with plenty of space free.+ # Gated like the outer handler: disk advice is only right when disk is the problem, and a+ # converter with no llama.cpp support fails here with space to spare.if IS_KAGGLE_ENVIRONMENT and _gguf_failure_looks_like_disk(RuntimeError(f"Conversion produced no output at {file}"),os.path.dirname(file) or None,@@ -2284,7 +2160,6 @@ def save_to_gguf(f"Unsloth: Conversion failed for {file}\nPlease check disk space and try again.")- # Move initial GGUF files into a dedicated _gguf directoryif gguf_directory is None:gguf_directory = f"{model_directory}_gguf"else:@@ -2310,10 +2185,8 @@ def save_to_gguf(print(f"Unsloth: Initial conversion completed! Files: {initial_files}")- # Step 4: Additional quantizations using llama-quantizeall_saved_locations = initial_files.copy()- # Get CPU count for quantizationn_cpus = psutil.cpu_count()if n_cpus is None:n_cpus = 1@@ -2322,20 +2195,14 @@ def save_to_gguf(if not is_gpt_oss:base_gguf = initial_files[0]- # Deduplicate while keeping order; methods equal to the base conversion already- # exist on disk and need no quantize pass.+ # Deduplicate while keeping order; a method equal to the base conversion is already on disk.methods_to_quantize = [m for m in dict.fromkeys(quantization_method) if m != first_conversion]- # The merge is not read again -- llama-quantize reads the GGUF, already- # written and moved -- and on a disk holding merge, base GGUF and quants- # at once those bytes are what runs it out. Only fires when the room is- # not there, so an export that would have worked keeps the merge.- # Nemotron-3-Nano-30B-A3B on a 132GB disk: 63GB merge + 60GB BF16 GGUF +- # ~18GB Q4_K_M = 141GB, and llama-quantize died at tensor 88 of 401 with- # `basic_ios::clear: iostream error`, its own `fout.exceptions(failbit)`- # firing on a write with nowhere to go.+ # The merge is not read again, and on a tight disk its bytes are what run the export out.+ # Nemotron-3-Nano-30B-A3B on 132GB: 63GB merge + 60GB GGUF + 18GB Q4_K_M = 141GB, and llama-quantize+ # died mid-write. Only fires when the room is not there._free_merge_if_disk_is_tight(model_directory,gguf_directory,@@ -2361,9 +2228,8 @@ def save_to_gguf(imatrix = imatrix,)else:- # Use unsloth-zoo's standard quantization for all other methods. Only pass- # imatrix when set so older unsloth_zoo (no imatrix kwarg) still works for- # plain quants; an imatrix that cannot be applied was rejected above.+ # Standard unsloth-zoo quantization for everything else. Pass imatrix only when set, so an+ # older zoo still handles plain quants; an inapplicable imatrix was rejected above.quant_kwargs = dict(input_gguf = base_gguf,output_gguf = output_location,@@ -2377,13 +2243,8 @@ def save_to_gguf(quant_kwargs["n_threads"] = n_threadsreturn quantize_gguf(**quant_kwargs)except Exception as e:- # What this pass was going to write, so "no room" is judged- # against the output rather than a constant. Priced as a lower- # bound, not the reclamation estimate: that one is deliberately- # generous, and here a size that comes out high calls a disk full- # that had the room and hides the rebuild advice below. Best- # effort -- an unreadable base, or a type with no known width,- # just falls back to the fixed floor.+ # Judge "no room" against what this pass writes, not a constant. Priced as a lower bound, not+ # the generous reclamation estimate: a high guess calls a disk full that had room.try:_ratio = _gguf_output_size_ratio(quant_method,@@ -2404,9 +2265,8 @@ def save_to_gguf()except OSError:_needed = None- # Same gate as above: a broken quantizer with 19GB free is not- # a disk problem, and the outer handler cannot undo an- # explanation already baked into this message.+ # Same gate as above: a broken quantizer with 19GB free is not a disk problem, and the outer+ # handler cannot undo an explanation already baked into this message.if IS_KAGGLE_ENVIRONMENT and _gguf_failure_looks_like_disk(e,gguf_directory,@@ -2430,10 +2290,8 @@ def save_to_gguf(needed_bytes = _needed,partial_output = output_location,):- # Kaggle is not the only place a disk fills. The rebuild- # advice below is only correct when the quantizer is the- # problem; on a full disk it is a long compile that fixes- # nothing, so say what actually happened instead.+ # Kaggle is not the only place a disk fills, and the rebuild advice below only fits a broken+ # quantizer; on a full disk it is a long compile that fixes nothing.try:_free_gb = shutil.disk_usage(gguf_directory).free / 1024**3_where = f" ({_free_gb:.1f}GB free at {gguf_directory})"@@ -2477,18 +2335,15 @@ def save_to_gguf(for m in methods_to_quantizeif os.path.exists(os.path.join(gguf_directory, f"{model_name}.{m.upper()}.gguf"))}- # Each llama-quantize pass loads the whole base GGUF into RAM, so only run two at- # once when the host has headroom for two copies, else a multi-quant export that- # fit sequentially could OOM.+ # Each llama-quantize pass loads the whole base GGUF into RAM, so run two at once only with headroom+ # for two copies, else a multi-quant export that fit sequentially OOMs.try:base_bytes = sum(os.path.getsize(f) for f in initial_files if not _is_gguf_companion(f))mem_ok = psutil.virtual_memory().available >= int(2.5 * base_bytes)except Exception:mem_ok = False- # Independent llama-quantize runs on the same base GGUF can overlap. Kept at 2- # workers; run sequentially when streaming logs (UNSLOTH_ENABLE_LOGGING), on- # Kaggle/Colab, when RAM is tight, or when the kill switch (0/false/no/off/empty)- # is set.+ # Independent llama-quantize runs can overlap, capped at 2 workers. Sequential when streaming logs, on+ # Kaggle/Colab, when RAM is tight, or when the kill switch is set._parallel_flag = os.environ.get("UNSLOTH_PARALLEL_GGUF_QUANTS", "1").strip().lower()parallel_quants = (len(methods_to_quantize) > 1@@ -2515,14 +2370,13 @@ def save_to_gguf(for i, method in enumerate(methods_to_quantize)}done, pending = wait(future_to_idx, return_when = FIRST_EXCEPTION)- # Do not start queued passes after a failure (avoid filling the disk).+ # Do not start queued passes after a failure, to avoid filling the disk.for fut in pending:fut.cancel()first_exc = next((f.exception() for f in done if f.exception() is not None), None)if first_exc is not None:- # Remove only outputs this run newly created; a file that pre-dated the- # run (or a canceled pass that never wrote) is left intact, so a rerun- # never deletes a prior artifact. Base kept for retry.+ # Remove only outputs this run created: a pre-existing file, or a canceled pass that never+ # wrote, stays, so a rerun never deletes a prior artifact. The base is kept for retry.wait(future_to_idx)for method in methods_to_quantize:if method in preexisting_outputs:@@ -2554,11 +2408,11 @@ def save_to_gguf(all_saved_locations.remove(f)Path(f).unlink(missing_ok = True)- # flip the list to get [text_model, mmproj] order. for text models stays the same.+ # Flip the list to get [text_model, mmproj] order; text models stay the same.all_saved_locations.reverse()- # When the base format is preserved, move base files (incl. shards) away from- # list boundaries so example commands ([0]=model, [-1]=mmproj) stay correct.+ # When the base format is preserved, move base files (and shards) off the list boundaries so+ # example commands ([0]=model, [-1]=mmproj) stay correct.if want_full_precision and len(all_saved_locations) > len(base_files) + 1:for f in base_files:if f in all_saved_locations:@@ -2626,10 +2480,9 @@ def unsloth_save_pretrained_merged("You can do it separately via `tokenizer.save_pretrained(...)`")- # Kaggle's working directory is ~20GB while the overlay at /tmp on the same- # kernel is measured in terabytes. Relative paths under /kaggle/working that- # do not fit move there; absolute paths, hub pushes and every non-Kaggle- # machine are untouched.+ # Kaggle's working directory is ~20GB while /tmp on the same kernel is terabytes, so relative paths under+ # /kaggle/working that do not fit move there. Absolute paths, hub pushes and non-Kaggle machines are+ # untouched._forwards_state_dict, _writes_model_verbatim = _merge_writer_disposition(self, save_method)save_directory = _preflight_merge_disk(self,@@ -2639,14 +2492,11 @@ def unsloth_save_pretrained_merged(state_dict = state_dict,forwards_state_dict = _forwards_state_dict,writes_model_verbatim = _writes_model_verbatim,- # No `writer_runs_merge_guard`: this entrypoint's writer for a plain- # merge is `unsloth_save_model`, which merges and writes the shards- # itself and never reaches `merge_and_overwrite_lora`. A compressed- # export from here does go through `unsloth_generic_save`, and the- # preflight recognises that from the method alone.+ # No writer_runs_merge_guard: a plain merge here is written by unsloth_save_model, which never reaches+ # merge_and_overwrite_lora. A compressed export does go through unsloth_generic_save, which the+ # preflight recognises from the method alone.)- # FP8 / FP4 compressed-tensors export (llm-compressor) -> handled separately._compressed = _normalize_compressed_method(save_method)if _compressed is not None:scheme, needs_calibration, suffix = _compressed@@ -2663,7 +2513,6 @@ def unsloth_save_pretrained_merged(calibration_dataset = calibration_dataset,num_calibration_samples = num_calibration_samples,max_seq_length = max_seq_length,- # Forward standard save kwargs to the 16bit merge.state_dict = state_dict,save_function = save_function,max_shard_size = max_shard_size,@@ -2679,7 +2528,6 @@ def unsloth_save_pretrained_merged(gc.collect()return- # torchao portable FP8/INT8 export (no NVIDIA GPU) -> separate path._torchao = _normalize_torchao_method(save_method)if _torchao is not None:kind, suffix = _torchao@@ -2692,7 +2540,6 @@ def unsloth_save_pretrained_merged(push_to_hub = push_to_hub,token = token,is_main_process = is_main_process,- # Forward standard save kwargs to the 16bit merge.state_dict = state_dict,save_function = save_function,max_shard_size = max_shard_size,@@ -2761,7 +2608,6 @@ def unsloth_push_to_hub_merged("You can do it separately via `tokenizer.push_to_hub(...)`")- # FP8 / FP4 compressed-tensors export (llm-compressor) -> handled separately._compressed = _normalize_compressed_method(save_method)if _compressed is not None:scheme, needs_calibration, suffix = _compressed@@ -2782,7 +2628,6 @@ def unsloth_push_to_hub_merged(calibration_dataset = calibration_dataset,num_calibration_samples = num_calibration_samples,max_seq_length = max_seq_length,- # Forward standard save kwargs to the 16bit merge.use_temp_dir = use_temp_dir,max_shard_size = max_shard_size,safe_serialization = safe_serialization,@@ -2795,7 +2640,6 @@ def unsloth_push_to_hub_merged(gc.collect()return- # torchao portable FP8/INT8 export (no NVIDIA GPU) -> separate path._torchao = _normalize_torchao_method(save_method)if _torchao is not None:kind, suffix = _torchao@@ -2813,7 +2657,6 @@ def unsloth_push_to_hub_merged(commit_description = commit_description,create_pr = create_pr,revision = revision,- # Forward standard save kwargs to the 16bit merge.use_temp_dir = use_temp_dir,max_shard_size = max_shard_size,safe_serialization = safe_serialization,@@ -2906,7 +2749,6 @@ def create_huggingface_repo(private = private,)- # Create model cardfrom huggingface_hub import ModelCardcontent = MODEL_CARD.format(@@ -2921,7 +2763,7 @@ def create_huggingface_repo(card.data.datasets = datasetscard.push_to_hub(save_directory, token = token)except:- # Repo already exists — update datasets metadata separately+ # Repo already exists, so update datasets metadata separately.if datasets:try:from huggingface_hub import metadata_update@@ -2959,7 +2801,6 @@ def upload_to_huggingface(private = private,)- # Create model cardfrom huggingface_hub import ModelCardcontent = MODEL_CARD.format(@@ -2974,7 +2815,6 @@ def upload_to_huggingface(card.data.datasets = datasetscard.push_to_hub(save_directory, token = token)except:- # Repo already exists — update datasets metadata separatelyif datasets:try:from huggingface_hub import metadata_update@@ -2985,7 +2825,6 @@ def upload_to_huggingface()if file_location is not None:- # Now upload filehf_api = HfApi(token = token)if "/" in file_location:@@ -2993,7 +2832,6 @@ def upload_to_huggingface(else:uploaded_location = file_location- # find ftevent file from tensorboard and upload itimport globftevent_files = glob.glob("*out.tfevents*", recursive = True)@@ -3019,7 +2857,6 @@ def upload_to_huggingface(commit_message = "(Trained with Unsloth)",)- # We also upload a config.json fileif create_config:import json@@ -3037,7 +2874,6 @@ def upload_to_huggingface(def fix_tokenizer_bos_token(tokenizer):- # Check if BOS added already, then warnfix_bos_token = Falsechat_template = getattr(tokenizer, "chat_template", None)@@ -3053,11 +2889,9 @@ def fix_tokenizer_bos_token(tokenizer):"Unsloth: ##### Your chat template has a BOS token. We shall remove it temporarily.")- # Remove {{bos_token}}new_chat_template = re.sub(r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\}[\s]{0,}\}", "", chat_template)- # Remove {{bos_token +new_chat_template = re.sub(r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\+[\s]{0,}","",@@ -3086,7 +2920,7 @@ def create_ollama_modelfile(tokenizer, base_model_name, model_location):f"Unsloth: No Ollama template mapping found for model '{base_model_name}'. Skipping Ollama Modelfile")return None- tokenizer._ollama_modelfile = ollama_modelfile # This comes from the unpacking above+ tokenizer._ollama_modelfile = ollama_modelfilemodelfile = ollama_modelfileFILE_LOCATION_REPLACER = "⚫@✅#🦥__FILE_LOCATION__⚡@🦥#⛵"@@ -3094,8 +2928,7 @@ def create_ollama_modelfile(tokenizer, base_model_name, model_location):LEFT_BRACKET_REPLACER = "⚫@✅#🦥"RIGHT_BRACKET_REPLACER = "⚡@🦥#⛵"- # Fixes https://github.com/unslothai/unsloth/issues/1087- # We must convert all {'s and }'s but keep {__FILE_LOCATION__} intact+ # Convert all {'s and }'s but keep {__FILE_LOCATION__} intact, reverted below (#1087).modelfile = (modelfile.replace("{__FILE_LOCATION__}", FILE_LOCATION_REPLACER).replace("{__EOS_TOKEN__}", EOS_TOKEN_REPLACER)@@ -3103,7 +2936,6 @@ def create_ollama_modelfile(tokenizer, base_model_name, model_location):.replace("}", RIGHT_BRACKET_REPLACER))- # Revert {__FILE_LOCATION__} backmodelfile = modelfile.replace(FILE_LOCATION_REPLACER, "{__FILE_LOCATION__}").replace(EOS_TOKEN_REPLACER, "{__EOS_TOKEN__}")@@ -3252,12 +3084,9 @@ def _hub_cache_prewarm_disabled(disable):os.environ[key] = previous-# unsloth_zoo's own merge guard compares the save against `int(free * 0.95)`-# (`saving_utils.merge_and_overwrite_lora`), and `model_16bit_bytes` counts-# tensors only: config.json, the tokenizer files and the safetensors index are-# written on top of them. Ask the redirect for that same effective figure, or a-# working directory that measures "just big enough" keeps the export and the-# merge then refuses it outright instead of using /tmp.+# The zoo's merge guard compares against int(free * 0.95), and model_16bit_bytes counts tensors only, with+# config, tokenizer and index on top. Ask the redirect for that same effective figure, else a "just big+# enough" directory is kept and the merge then refuses._MERGE_FREE_SPACE_RESERVE = 0.95# torchao weight-only fp8 / int8: one byte per quantized weight.@@ -3429,11 +3258,9 @@ def _full_model_checkpoint_bytes(model, state_dict = None):which is how the 20GB Kaggle working directory fills instead."""try:- # `is not None`, matching `unsloth_generic_save`, which forwards the- # dict on exactly that test. An explicitly empty one therefore reaches- # `save_pretrained` and writes no tensors at all, so sizing the resident- # model there would price tens of gigabytes for a save that writes none- # and move it off persistent Kaggle storage for nothing.+ # `is not None`, matching unsloth_generic_save, which forwards the dict on that test. An empty dict+ # reaches save_pretrained and writes no tensors, so sizing the resident model would price tens of+ # gigabytes for a save that writes none.if state_dict is not None:total = 0for tensor in state_dict.values():@@ -3586,10 +3413,8 @@ def _warn_if_torchao_staging_filesystem_is_short(destination, staging_bytes):if _same_filesystem(staging_directory, destination):returnfree = free_bytes(staging_directory)- # The staging merge is written by `merge_and_overwrite_lora`, which- # refuses it unless `free * 0.95` covers it. Comparing against the- # bare size leaves the 5% band where the merge predictably dies and- # this says nothing, which is the one thing it exists to prevent.+ # The staging merge is written by merge_and_overwrite_lora, which refuses it unless free * 0.95 covers+ # it, so the bare size leaves a 5% band where the merge dies silently.needed = math.ceil(staging_bytes / _MERGE_FREE_SPACE_RESERVE)if free is None or free >= needed:return@@ -3673,12 +3498,12 @@ def _merge_writer_disposition(model, save_method):return True, Falseif method != "merged_16bit":return False, False- # A PeftModel in the generic fallback writes ADAPTERS and not a checkpoint,- # so it keeps the sizing it has rather than being priced a full model.+ # A PeftModel in the generic fallback writes ADAPTERS, not a checkpoint, so it keeps its own sizing rather+ # than being priced a full model.if isinstance(model, (PeftModel, PeftModelForCausalLM)):return False, Falsetry:- # The same predicate `unsloth_save_model` dispatches on, and no other.+ # The same predicate unsloth_save_model dispatches on, and no other.takes_generic_fallback = not hasattr(model, "model") or not hasattr(getattr(model, "model", None), "layers")@@ -3734,56 +3559,37 @@ def _preflight_merge_disk("""if push_to_hub:return save_directory- # `unsloth_save_model` normalizes spaces before it dispatches, so- # "merged 16bit" is the same full merge as "merged_16bit" and has to be- # measured as one. `_normalize_compressed_method` normalizes the same way.+ # unsloth_save_model normalizes spaces before dispatching, so "merged 16bit" is the same full merge as+ # "merged_16bit" and must be measured as one.method = str(save_method).lower().strip().replace("-", "_").replace(" ", "_")try:- # Every compressed export keeps the 16-bit merge at `save_directory`- # and writes a quantized sibling next to it, so all of them belong- # here, not just mxfp4.+ # Every compressed export keeps the 16-bit merge at save_directory and writes a quantized sibling+ # beside it, so all of them belong here, not just mxfp4.compressed = _normalize_compressed_method(method)except Exception:# An unsupported near-miss name raises later, where the message is.return save_directory- # The torchao portable exports are the same shape as the compressed ones- # from here: a quantized sibling at `save_directory + "-<suffix>"`.+ # The torchao portable exports have the same shape: a quantized sibling at save_directory + "-<suffix>".torchao = _normalize_torchao_method(method)- # `lora` on a model that has no adapter writes the WHOLE model instead:- # both `unsloth_generic_save` and `unsloth_save_model` fall back to- # `save_pretrained` there, so a full fine-tune asked for "lora" fills- # /kaggle/working exactly like a merge. A real PeftModel writes adapters- # only and is still skipped.+ # "lora" on a model with no adapter writes the WHOLE model: both save paths fall back to save_pretrained,+ # so a full fine-tune asked for "lora" fills /kaggle/working like a merge. A real PeftModel writes+ # adapters only and is still skipped.is_peft = isinstance(model, (PeftModel, PeftModelForCausalLM))full_model_lora = method == "lora" and not is_peft- # `unsloth_generic_save` only reaches for `model.state_dict()` when it was- # given none, and a model with no adapter is saved from whatever dictionary- # it ends up holding. A PeftModel goes to `merge_and_overwrite_lora`, which- # takes no state dict at all, so only the non-PEFT case follows it.+ # unsloth_generic_save reaches for model.state_dict() only when given none, so a model with no adapter is+ # saved from whatever dict it holds; a PeftModel goes to merge_and_overwrite_lora, which takes no state dict+ # at all.supplied_dict = state_dict if (forwards_state_dict and not is_peft) else None- # Same reason the dict is only followed for a non-PEFT model: a PeftModel- # never reaches the fallback that would copy one verbatim.+ # Same reason the dict is only followed for a non-PEFT model: a PeftModel never reaches the fallback that+ # would copy one verbatim.verbatim = bool(writes_model_verbatim) and not is_peftif compressed is None and torchao is None and method != "merged_16bit" and not full_model_lora:return save_directorytry:- # A merge writes 2 bytes per logical parameter and no GGUF at all, so- # this is `model_16bit_bytes` and not the GGUF export estimate: that- # one always prices an intermediate GGUF conversion this never does.- # The full-model fallback casts nothing, so it is sized from the- # tensors' own dtype instead.- #- # A 16-bit save of a model with no adapter writes the dictionary it was- # handed, cast, and not the resident parameters: an empty or partial one- # writes correspondingly less, and one carrying more than the model does- # writes more. Sizing the model there redirects a nearly empty save off- # persistent Kaggle storage, or leaves a larger one to fill the working- # filesystem.- #- # A compressed or torchao export writes that same 16-bit checkpoint- # through the same `unsloth_generic_save`, as its staging (torchao) or- # kept (compressed) merge, so the dictionary decides its size too and- # not only a literal `merged_16bit` request.+ # A merge writes 2 bytes per parameter and no GGUF, so this is model_16bit_bytes, not the GGUF+ # estimate, which always prices an intermediate conversion. The no-adapter save writes the dictionary+ # it was handed, cast, so an empty one writes almost nothing and sizing the model would relocate it+ # off persistent Kaggle storage.if full_model_lora or verbatim:need = _full_model_checkpoint_bytes(model, state_dict)elif supplied_dict is not None:@@ -3792,45 +3598,26 @@ def _preflight_merge_disk(need = model_16bit_bytes(model)if need <= 0:return save_directory- # What the torchao staging directory costs on whatever filesystem- # `tempfile` resolves to. Zero for every other export, which stages- # nothing.+ # What the torchao staging directory costs on whatever filesystem tempfile resolves to; zero for every+ # other export, which stages nothing.staging = 0- # The lexical suffix the quantized sibling is written under, so the- # filesystem holding it can be measured separately below.+ # The lexical suffix the quantized sibling is written under, so its filesystem can be measured+ # separately below.sibling_suffix = ""sibling_bytes = 0- # What `merge_and_overwrite_lora` will be asked to write HERE, which is- # the only part its `free * 0.95` guard measures. Split out from `need`- # because the reserve belongs on this and on nothing else: charging it- # around the whole estimate relocates an export that fits, and on- # Kaggle that means moving it to a /tmp the kernel does not keep.- # The guard belongs to `merge_and_overwrite_lora` and to nothing else,- # and only `unsloth_generic_save`'s PEFT branch calls it. Every other- # writer that lands a 16-bit checkpoint here is a bare- # `save_pretrained`: the generic architecture fallback, the full-model- # `lora` fallback, `unsloth_save_model`'s own merge, and- # `unsloth_generic_save`'s no-adapter branch, which casts the dict and- # writes it directly. None of them reserves anything, so charging them- # 1/0.95 refuses a filesystem their writer would have accepted.- #- # Sizing is decided separately, above: a compressed export really is- # cast to two bytes by `unsloth_generic_save`, so it keeps that sizing- # whether or not there is an adapter to merge.+ # What merge_and_overwrite_lora writes HERE, the only part its free * 0.95 guard measures. Split out+ # from `need` because the reserve belongs on this alone: charging it around the whole estimate+ # relocates an export that fits. Every other writer here is a bare save_pretrained and reserves+ # nothing.guard_runs_here = is_peft and (compressed is not None or bool(writer_runs_merge_guard))merge_here = need if guard_runs_here else 0if torchao is not None:- # `_unsloth_save_torchao` merges into a `tempfile.mkdtemp` staging- # directory rather than `save_directory`, so the only thing landing- # here is the 8-bit sibling, and no merge guard runs against this- # filesystem at all.+ # _unsloth_save_torchao merges into a tempfile.mkdtemp staging dir, so only the 8-bit sibling lands+ # here and no merge guard runs against this filesystem.staging = needmerge_here = 0- # No ignore list: `_unsloth_save_torchao` quantizes with a bare- # `Float8WeightOnlyConfig()` / `Int8WeightOnlyConfig()`. Charging it- # the compressed recipe's exclusions would over-count, and- # over-counting relocates an export that fits into a /tmp Kaggle- # does not keep as notebook output.+ # No ignore list: _unsloth_save_torchao quantizes with a bare Float8/Int8WeightOnlyConfig(), and+ # charging it the compressed recipe's exclusions would over-count and relocate an export that fits.need = _quantized_sibling_bytes(model, need, _TORCHAO_SIBLING_WEIGHT_BITS)sibling_suffix = torchao[1]sibling_bytes = need@@ -3839,16 +3626,15 @@ def _preflight_merge_disk(model,need,_compressed_scheme_weight_bits(compressed[0]),- # Everything the recipe refuses to quantize is copied across at- # 16 bits: the vision tower, linear attention, MTP, MoE gates.+ # Everything the recipe refuses to quantize is copied at 16 bits: vision tower, linear+ # attention, MTP, MoE gates._compressed_ignore_patterns(model),)sibling_suffix = compressed[2]need += sibling_bytes- # The reserve raises the ask for the merge alone, and the rest is added- # at face value. `max` rather than a sum of the two halves: the sibling- # coexists with the merge, so the peak is the whole estimate, and it- # must also clear the merge's own reserved figure.+ # The reserve raises the ask for the merge alone; the rest is added at face value. max, not a sum: the+ # sibling coexists with the merge, so the peak is the whole estimate and it must also clear the+ # merge's reserved figure.need = max(need, math.ceil(merge_here / _MERGE_FREE_SPACE_RESERVE))new_directory, message = kaggle_tmp_redirect(save_directory,@@ -4173,9 +3959,8 @@ def _gguf_source_dtype(model):if not torch.cuda.is_bf16_supported():return "f16"except Exception:- # `save_to_gguf` calls this unguarded, so a raise here means no- # export happens at all and the figure never gets used. "f16" is- # the same width either way; only the name has to match.+ # save_to_gguf calls this unguarded, so a raise means no export at all and the figure is never used.+ # "f16" is the same width either way; only the name has to match.return "f16"return dtype@@ -4233,27 +4018,17 @@ def _preflight_gguf_disk(first_conversion = _choose_first_conversion(methods, model_dtype, has_imatrix = has_imatrix)- # `save_to_gguf` drops a bf16 initial conversion to f16 on hardware- # with no bf16, AFTER it has resolved one, so the drop has to be- # applied after both branches above and not only inside the resolver.- # `_gguf_source_dtype` covers the dtype this is TOLD; it cannot cover- # a `first_conversion` the caller passed, nor the single direct-convert- # method `_choose_first_conversion` returns unchanged.- #- # It is the requested-output rule that makes this cost a checkpoint:- # the estimate omits an output EQUAL to the initial conversion, so- # `["bf16"]` at first_conversion "bf16" is priced as one 16-bit file,- # while a T4 writes an f16 intermediate AND a separate bf16 output.- # 15.3GB unaccounted for on Qwen3-8B, the same figure and the same- # mechanism as the dtype case.+ # save_to_gguf drops a bf16 initial conversion to f16 on hardware without bf16 AFTER resolving one, so+ # the drop must be applied after both branches, not only in the resolver. The estimate omits an output+ # equal to the initial conversion, so ["bf16"] at first_conversion "bf16" was priced as one file while+ # a T4 writes f16 plus bf16: 15.3GB missing on Qwen3-8B.if first_conversion == "bf16":try:if not torch.cuda.is_bf16_supported():first_conversion = "f16"except Exception:- # `save_to_gguf` asks unguarded, so a raise here means no- # export runs and this figure is never used. f16 is the wider- # of the two readings, and never the narrower.+ # save_to_gguf asks unguarded, so a raise means no export runs and this is never used. f16 is+ # the wider of the two readings, never the narrower.first_conversion = "f16"need = estimate_gguf_export_bytes(model = model,@@ -4261,11 +4036,9 @@ def _preflight_gguf_disk(first_conversion = first_conversion,needs_merge = needs_merge,)- # The pre-warm only happens on the merge path, and only when it has- # not already been switched off. Kaggle and Colab return before it- # ever runs (`_prewarm_base_model_hub_cache`), so pricing a cache copy- # there would send an export that fits in /kaggle/working to /tmp,- # which is not kept as notebook output.+ # The pre-warm runs only on the merge path and only while enabled. Kaggle and Colab return before it,+ # so pricing a cache copy there sends an export that fits in /kaggle/working to a /tmp that is not+ # kept as notebook output.prewarm_possible = (needs_mergeand not IS_KAGGLE_ENVIRONMENT@@ -4284,19 +4057,15 @@ def _preflight_gguf_disk(if prewarm_possibleelse need)- # The estimate prices the checkpoint at two bytes per parameter. The- # non-PEFT fallback writes it at the model's own dtype, so an fp32- # model needs the difference on top. Zero for every 16-bit model.+ # The estimate prices the checkpoint at 2 bytes per parameter, but the non-PEFT fallback writes the+ # model's own dtype, so an fp32 model needs the difference. Zero for a 16-bit model.if need > 0 and needs_merge:extra = _fallback_checkpoint_extra_bytes(model)need += extraneed_with_cache += extra- # The same estimate without the checkpoint: what lands in the `_gguf`- # sibling directory, which is the intermediate conversion plus every- # quant. Only used when that sibling turns out to be on a different,- # smaller filesystem than `save_directory`. Its own try, so an- # estimator that cannot answer this leaves the main guard standing- # rather than turning the whole preflight off.+ # The same estimate without the checkpoint: the `_gguf` sibling's intermediate plus every quant. Used+ # only when that sibling sits on a smaller filesystem. Its own try, so an estimator that cannot answer+ # leaves the main guard standing.try:need_sibling = estimate_gguf_export_bytes(model = model,@@ -4306,12 +4075,8 @@ def _preflight_gguf_disk()except Exception:need_sibling = 0- # The first of the two phases a disposable merge really peaks at: the- # merge and the intermediate GGUF on disk together, before any quant- # is written. `quantization_methods = ()` still prices the- # intermediate, which is what the conversion has to write before it- # can be read. Its own try for the same reason as the sibling: an- # estimator that cannot answer leaves the aggregate standing.+ # The first phase a disposable merge peaks at: merge plus intermediate GGUF, before any quant.+ # quantization_methods = () still prices the intermediate, which must exist to be read.try:need_merge_phase = estimate_gguf_export_bytes(model = model,@@ -4321,12 +4086,10 @@ def _preflight_gguf_disk() + (extra if need > 0 and needs_merge else 0)except Exception:need_merge_phase = 0- # Reclamation only helps when a quantize pass actually follows it, and- # `save_to_gguf` skips a method equal to the initial conversion because- # that file is already on disk. Same rule, same list.+ # Reclamation only helps when a quantize pass follows, and save_to_gguf skips a method equal to the+ # initial conversion, whose file is already on disk.has_quantize_pass = bool([m for m in dict.fromkeys(methods) if m != first_conversion])- # The intermediate conversion on its own, for the filesystem it is- # written to before the move. Same shape of `try` as the two above.+ # The intermediate conversion on its own, for the filesystem it is written to before the move.try:need_conversion = estimate_gguf_export_bytes(model = model,@@ -4337,20 +4100,29 @@ def _preflight_gguf_disk(except Exception:need_conversion = 0except Exception:- # Sizing is best effort. A failure here must not stop an export that- # would otherwise have worked.+ # Sizing is best effort: a failure here must not stop an export that would otherwise have worked.return save_directory, Trueif need <= 0:return save_directory, True- # The redirect has to be asked for the same figure the refusal below reads.- # `need_with_cache` is the aggregate, and a disposable merge is reclaimed- # before the quants are written, so an export whose phased peak fits in- # /kaggle/working was relocated to a /tmp Kaggle does not keep as notebook- # output. Same predicates as the branch that lowers `need_here`, read- # against the directory as it stands BEFORE any move, and it can only ever- # lower the ask: nothing that used to stay put is relocated now.+ # Ask the redirect for the same figure the refusal reads. need_with_cache is the aggregate, but a+ # disposable merge is reclaimed before the quants, so a phased peak that fits in /kaggle/working was+ # relocated. Read before any move, and it can only lower the ask.phased_need = 0+ # The intermediate conversion is written to the process CWD, not into `save_directory` and not into the+ # `_gguf` sibling, and only afterwards moved. When that CWD is on its own filesystem, nothing above has+ # measured the disk the largest staging artefact actually lands on: on Kaggle it is the 20GB working+ # directory the redirect just moved the export away from.+ # The intermediate conversion is written to the working directory and only moved to the sibling afterwards,+ # so when that working directory is on THIS filesystem the checkpoint and the conversion are on it together.+ # Charging each of them alone lets a 60GB checkpoint and a 60GB conversion both pass on 100GB, and then it+ # fills. Only the split branch needs this: on one filesystem the aggregate already counts them both, and a+ # conversion sharing the sibling's disk is counted in `need_sibling`. `max` and not `+`, because the two+ # figures are two PHASES and not two artefacts. The merge runs first and its guard wants the checkpoint over+ # 0.95 with nothing else written yet; the conversion is written after, against the unreserved checkpoint.+ # Summing them charges the reserve on top of a conversion that does not exist while the guard runs: a 60GB+ # merge and a 60GB conversion on 122GB free clear both phases (63.2GB, then 120GB) and the sum asks 123.2GB+ # and refuses.if (merge_is_disposableand needs_merge@@ -4372,18 +4144,10 @@ def _preflight_gguf_disk(need_bytes = redirect_need,what = "GGUF export",)- # Asked the aggregate because THIS directory already holds weights the- # reclamation may not touch, and declined. The relocation writes the merge- # into a directory of the export's own, where it can be reclaimed, so the- # move is worth a second ask at the phased peak: 63GB merge, 60GB- # intermediate, 18GB Q4_K_M asks 141GB, a 130GB /tmp declines it, and the- # 123GB peak it would really have peaked at there fits.- #- # Gated on this directory already being too small for the aggregate the- # refusal below will read, which is what makes the lower ask safe. A lower- # ask can otherwise CANCEL a move: /kaggle/working with room for the peak- # but not the aggregate keeps an export that has no reclamation here, and- # then the refusal reads the aggregate anyway.+ # Asked the aggregate and declined, because this directory already holds weights reclamation may not+ # touch. After the move the merge is reclaimable, so re-ask at the phased peak: 141GB aggregate declines a+ # 130GB /tmp, but the real 123GB peak fits. Gated on this directory being too small for the aggregate,+ # else a lower ask could cancel the move for nothing.if message is None and 0 < phased_need < redirect_need:free_before_move = free_bytes(save_directory)if free_before_move is not None and free_before_move < need:@@ -4398,17 +4162,9 @@ def _preflight_gguf_disk(free = free_bytes(save_directory)- # The quants and the intermediate GGUF are written to a SIBLING of- # `save_directory`, so they live on the parent's filesystem. That is the- # same disk `free` just measured unless `save_directory` is itself a mount- # point or a symlink onto another one. When it is, each filesystem is- # charged only for what is written to it: the checkpoint here, the- # conversion and the quants there. Charging the aggregate to either one- # refuses a split export whose every artefact has room.- #- # One predicate, computed once, so the two halves cannot disagree about- # whether the storage is split, and an unmeasurable path leaves every- # decision below exactly as it was.+ # The quants and intermediate GGUF go to a SIBLING of save_directory, on the parent's filesystem, which is+ # the disk just measured unless save_directory is a mount or symlink. When split, charge each filesystem+ # only what it holds. One predicate, computed once, so the two halves cannot disagree.gguf_directory = _gguf_output_directory(save_directory)gguf_free = free_bytes(gguf_directory)separate_storage = (@@ -4416,34 +4172,24 @@ def _preflight_gguf_disk(and gguf_free is not Noneand _on_separate_filesystems(save_directory, gguf_directory))- # Resolved before the split is priced, because where the conversion lands- # decides which filesystem it is charged to.+ # Resolved before the split is priced, because where the conversion lands decides which filesystem it is+ # charged to.conversion_directory = _gguf_conversion_directory(_gguf_model_input_directory(model, save_directory))- # Cleared where the base-model cache turns out to share a filesystem that- # has room for what the export writes there but not for a cached base as- # well: dropping the optional half beats failing the export. The message- # travels with the flag because more than one filesystem can now set it,- # and each has to name the one it measured.+ # Cleared when the cache shares a filesystem with room for the export but not a cached base too: dropping+ # the optional half beats failing. The message travels with the flag, since more than one filesystem can+ # set it.sibling_prewarm_ok = Trueprewarm_drop_message = None- # Resolved once, above the split, because the cache is not necessarily on- # either of the two filesystems the split is about: the conversion's own- # filesystem below asks the same question of the same path. Zero whenever- # the pre-warm cannot run, which is what leaves every branch below inert.+ # Resolved once, above the split, since the cache need not be on either filesystem. Zero whenever the pre-+ # warm cannot run, which leaves every branch below inert.cache_extra = max(0, need_with_cache - need)cache_directory = _hub_cache_directory() if cache_extra > 0 else None- # The cache copy lands wherever the cache is, which is not necessarily the- # filesystem holding `save_directory` - `HF_HOME` on a data volume is the- # ordinary layout on a machine with more than one disk. Charging it here- # anyway drops the pre-warm on a disk that had room for everything written- # to it, and the next export downloads the whole base again, which is the- # exact re-download the pre-warm exists to stop. It only ever LOWERS the- # figure, and only the pre-warm decision reads it, so no export that fit- # before is refused now. Unresolvable leaves it charged here, where it was- # charged before. The split branch asks the same of the sibling below.+ # The cache lands wherever HF_HOME points, not necessarily save_directory's disk. Charging it here drops+ # the pre-warm on a disk that had room, and the next export re-downloads the base. It only LOWERS the+ # figure and only the pre-warm reads it.cache_here = cache_extraif (cache_extra > 0@@ -4454,33 +4200,15 @@ def _preflight_gguf_disk(need_here = needneed_here_with_cache = need + cache_hereif separate_storage:- # `need_sibling` is the same estimate without the checkpoint, so the- # difference is the checkpoint and nothing else. An estimator that- # could not size the sibling leaves it 0, and then this is `need`:- # the previous behaviour, not a relaxed one.- #- # Nothing else, though, includes the slack: both terms carry the- # estimator's flat `DISK_SLACK_BYTES` and the subtraction cancels it,- # leaving two bytes per parameter exactly. `merge_and_overwrite_lora`- # refuses that merge unless `free * 0.95` covers it, so the same- # reserve `_preflight_merge_disk` already applies has to be applied- # here too, or this passes an export the merge kills seconds later.- # The aggregate branch needs no such thing: it charges the quants as- # well, which is already more than the checkpoint plus its reserve.+ # need_sibling is this estimate without the checkpoint, so the difference is the checkpoint alone; an+ # unsizable sibling leaves it 0, giving the previous behaviour. Both terms carry DISK_SLACK_BYTES and+ # it cancels, so merge_and_overwrite_lora's free * 0.95 reserve must be re-applied or this passes an+ # export the merge kills seconds later.checkpoint_here = max(0, need - need_sibling)need_here = checkpoint_here- # The cache copy is not part of the merge and is not what the zoo- # guard measures, so it rides on top of the checkpoint rather than- # being reserved itself.- #- # It also does not necessarily land HERE. `save_directory` being a- # mount onto another disk is the whole premise of this branch, and- # then `~/.cache` and the lexical `_gguf` sibling both stay on the- # parent disk. Charging the checkpoint's filesystem for bytes written- # to the sibling's drops a pre-warm that had room and, worse, lets the- # sibling check accept `need_sibling` alone on a filesystem the base- # model is about to be downloaded onto. `cache_here` above has already- # answered the first half; the sibling is the half only this branch has.+ # The cache copy is not part of the merge and not what the zoo guard measures, so it rides on top of+ # the checkpoint. It need not land HERE either: with save_directory mounted elsewhere, ~/.cache and+ # the _gguf sibling stay on the parent disk.cache_sibling = 0if (cache_extra > 0@@ -4491,45 +4219,20 @@ def _preflight_gguf_disk(need_here_with_cache = checkpoint_here + cache_herewrites_a_lora_merge = isinstance(model, (PeftModel, PeftModelForCausalLM))if writes_a_lora_merge and needs_merge and need_sibling > 0 and checkpoint_here > 0:- # Four conditions, each removing a way to charge for a guard that- # will not run. Only a PEFT model reaches `merge_and_overwrite_lora`- # and its `free * 0.95`: a non-PEFT model with no reusable local- # checkpoint still needs `needs_merge`, because the GGUF path has- # to write one, but it is written by a bare `self.save_pretrained`- # that reserves nothing, so charging the reserve there refuses a- # filesystem with room for the checkpoint the writer would have- # accepted. `needs_merge = False` writes no merge at all. A sibling- # the estimator could not size leaves `checkpoint_here` equal to- # the aggregate, which is a fallback figure and not a checkpoint- # to reserve against. And `min(need, ...)` keeps the promise the- # split made when it was introduced: it may cancel a redirect,- # never cause a refusal the aggregate would have allowed.+ # Four conditions, each dropping a charge for a guard that will not run: only a PEFT model reaches+ # merge_and_overwrite_lora's free * 0.95, a bare self.save_pretrained reserves nothing,+ # needs_merge = False writes no merge, and an unsizable sibling leaves a fallback figure.+ # min(need, ...) keeps the split's promise: cancel a redirect, never cause a refusal.reserved = min(need, math.ceil(checkpoint_here / _MERGE_FREE_SPACE_RESERVE))need_here = max(need_here, reserved)- # The cache copy is written BEFORE the merge and is still there when- # the guard runs, so it comes off the free space the guard measures.- # It rides on top of the reserved figure rather than beside it: a- # 16GB checkpoint with a 14GB cache on 30.5GB free passes a `max`- # asking 30GB, and then the merge sees 16.5GB and refuses 16GB under- # its own 5%. Added instead, this band drops the pre-warm, which is- # the optional half, rather than failing the export.+ # The cache copy is written BEFORE the merge and is still there when the guard runs, so it comes+ # off the free space the guard sees. Riding on top of the reserved figure rather than beside it:+ # 16GB checkpoint + 14GB cache on 30.5GB passes a max of 30GB and then the merge refuses under its+ # own 5%. Added instead, this drops the pre-warm, not the export.need_here_with_cache = max(need_here_with_cache, reserved + cache_here)- # The intermediate conversion is written to the working directory and- # only moved to the sibling afterwards, so when that working directory- # is on THIS filesystem the checkpoint and the conversion are on it- # together. Charging each of them alone lets a 60GB checkpoint and a- # 60GB conversion both pass on 100GB, and then it fills. Only the- # split branch needs this: on one filesystem the aggregate already- # counts them both, and a conversion sharing the sibling's disk is- # counted in `need_sibling`.- #- # `max` and not `+`, because the two figures are two PHASES and not two- # artefacts. The merge runs first and its guard wants the checkpoint- # over 0.95 with nothing else written yet; the conversion is written- # after, against the unreserved checkpoint. Summing them charges the- # reserve on top of a conversion that does not exist while the guard- # runs: a 60GB merge and a 60GB conversion on 122GB free clear both- # phases (63.2GB, then 120GB) and the sum asks 123.2GB and refuses.+ # The intermediate conversion lands in the working directory before the move, so on THIS filesystem+ # the checkpoint and conversion coexist and charging each alone lets 60GB + 60GB pass on 100GB. max,+ # not +: they are two phases, and summing them refuses 122GB free that clears both.if (need_conversion > 0and conversion_directory is not None@@ -4539,11 +4242,9 @@ def _preflight_gguf_disk(need_here_with_cache = max(need_here_with_cache, checkpoint_here + need_conversion + cache_here)- # Not gated on the sibling being the TIGHTER of the two any more.- # Now that the checkpoint is charged only its own portion, a sibling- # with more free space than `save_directory` and still less than- # `need_sibling` has to be refused here, because the aggregate- # comparison that used to catch it no longer exists.+ # No longer gated on the sibling being the TIGHTER of the two: now that the checkpoint is charged only+ # its own portion, a sibling roomier than save_directory but under need_sibling must be refused here,+ # since the aggregate comparison is gone.if gguf_free < need_sibling:raise RuntimeError(f"Unsloth: Not enough disk space to convert to GGUF.\n"@@ -4572,49 +4273,30 @@ def _preflight_gguf_disk(and need_sibling > 0and _merge_reclamation_is_possible(save_directory)):- # `_free_merge_if_disk_is_tight` deletes this export's own merge once- # the intermediate GGUF exists and before the first quantize pass, so- # the three artefacts never coexist: the peak is the larger of "merge- # plus intermediate" and "intermediate plus every quant", not their- # sum. Nemotron-3-Nano-30B-A3B is the case in front of it - 63GB merge,- # 60GB BF16 intermediate, 18GB Q4_K_M - which peaks at 123GB on a 132GB- # disk and which the aggregate 141GB refuses.- #- # Only in the single-filesystem branch, because the reclamation itself- # declines when the merge and the GGUF output are on different devices:- # freeing bytes on one filesystem does nothing for a quantize pass- # writing to another.+ # _free_merge_if_disk_is_tight deletes this export's merge once the intermediate exists, so the three+ # artefacts never coexist and the peak is the larger of the two phases, not their sum:+ # Nemotron-3-Nano-30B-A3B peaks at 123GB where the 141GB aggregate refuses a 132GB disk. Single+ # filesystem only, since reclamation declines across devices.peak = max(need_merge_phase, need_sibling)if peak < need:- # Can only ever lower the figure, so no input that head allowed is- # refused here. The cache copy is not what gets reclaimed, so it- # rides on top of the peak exactly as it rode on top of the sum.+ # Can only lower the figure, so nothing the head allowed is refused here. The cache copy is not+ # reclaimed, so it rides on top of the peak as it did on the sum.need_here = peakneed_here_with_cache = peak + cache_here- # The intermediate conversion is written to the process CWD, not into- # `save_directory` and not into the `_gguf` sibling, and only afterwards- # moved. When that CWD is on its own filesystem, nothing above has measured- # the disk the largest staging artefact actually lands on: on Kaggle it is- # the 20GB working directory the redirect just moved the export away from.+ # The intermediate conversion is written to the process CWD and only then moved, so when that CWD is its+ # own filesystem nothing above has measured the disk the largest staging artefact lands on -- on Kaggle,+ # the 20GB working directory the redirect just left.if (need_conversion > 0and conversion_directory is not Noneand _on_separate_filesystems(conversion_directory, gguf_directory)):conversion_free = free_bytes(conversion_directory)- # The pre-warm runs BEFORE the merge and leaves the base model in the- # Hugging Face cache for the rest of the export, so when the cache- # shares this filesystem the cached base and the intermediate are on- # it together. Nothing above has charged that: `cache_here` and- # `cache_sibling` only ever place the cache on the checkpoint's- # filesystem or the sibling's, and this is neither.- #- # The pre-warmer's own gate does not cover it either. It asks for two- # base copies free on the cache filesystem, and an `f32` conversion is- # two base copies on its own: 38.1GB free clears the pre-warm's 30.5GB- # threshold and this 30.5GB conversion check, then holds a 15.3GB- # cached base while writing 30.5GB, and runs out 7.6GB short.+ # The pre-warm leaves the base model cached for the rest of the export, so on a shared filesystem the+ # cached base and the intermediate coexist, which cache_here and cache_sibling never charge. The pre-+ # warmer's own gate asks for two base copies, and an f32 conversion is two copies on its own: 38.1GB+ # free clears both checks and runs out 7.6GB short.cache_with_conversion = (cache_extraif cache_directory is not None@@ -4641,9 +4323,8 @@ def _preflight_gguf_disk(and cache_with_conversion > 0and conversion_free < need_conversion + cache_with_conversion):- # Only reached once the conversion alone has cleared the raise- # above, so this can never turn a refusal into a pass: it drops- # the optional half of an export that otherwise fits.+ # Only reached once the conversion alone cleared the raise above, so this cannot turn a refusal into+ # a pass: it drops the optional half of an export that otherwise fits.sibling_prewarm_ok = Falseprewarm_drop_message = (f"Unsloth: Skipping the Hugging Face cache pre-warm - the Hugging Face "@@ -4833,7 +4514,7 @@ def unsloth_save_pretrained_gguf(save_directory = os.path.normpath(os.fspath(save_directory))gguf_shard_size = _resolve_gguf_shard_size(gguf_shard_size)- # save_method="lora" exports the adapter itself as a GGUF LoRA (not a merged model).+ # save_method="lora" exports the adapter itself as a GGUF LoRA, not a merged model.if save_method is not None and str(save_method).lower() == "lora":if not is_main_process:return None@@ -4857,8 +4538,8 @@ def unsloth_save_pretrained_gguf(_outtype = "f16"return _unsloth_save_lora_gguf(self, tokenizer, save_directory, outtype = _outtype)- # base_model_name keeps the full id for create_ollama_modelfile's mapper lookup;- # only the filename stem is trimmed.+ # base_model_name keeps the full id for create_ollama_modelfile's mapper lookup; only the filename stem is+ # trimmed.base_model_name = getattr(getattr(self, "config", None), "_name_or_path", None)try:base_model_name = get_model_name(base_model_name, load_in_4bit = False)@@ -4866,23 +4547,20 @@ def unsloth_save_pretrained_gguf(passmodel_name = _model_basename(base_model_name)- # Check if push_to_hub is requestedif push_to_hub:raise ValueError("Unsloth: Please use .push_to_hub_gguf() instead of .save_pretrained_gguf() with push_to_hub=True")- # Step 1: Check if this is a VLM (Vision-Language Model) and check if gpt-ossis_vlm = _is_vlm(self)is_processor = is_vlm and isinstance(tokenizer, ProcessorMixin)is_gpt_oss = _is_gpt_oss(self)- # Step 1b: Will this fit? Ask before the merge, not after. Runs here rather- # than lower down because `arguments` below snapshots locals(), so a- # redirected `save_directory` has to be in place first. gpt-oss takes the- # mxfp4 route instead of the merge/convert/quantize one this sizes.+ # Will this fit? Ask before the merge. Runs here because `arguments` below snapshots locals(), so a+ # redirected save_directory must be in place first. gpt-oss takes the mxfp4 route, not the+ # merge/convert/quantize one this sizes._gguf_prewarm_ok = Trueif not is_gpt_oss:save_directory, _gguf_prewarm_ok = _preflight_gguf_disk(@@ -4890,24 +4568,20 @@ def unsloth_save_pretrained_gguf(save_directory = save_directory,quantization_method = quantization_method,first_conversion = first_conversion,- # Resolved here rather than left at the default, because the- # default says "f16" and the export asks the config.+ # Resolved rather than left at the default, which says "f16" while the export asks the config.model_dtype = _gguf_source_dtype(self),has_imatrix = _imatrix_is_enabled(imatrix_file),needs_merge = _gguf_writes_16bit_checkpoint(self),- # The same flag `save_to_gguf` reclaims on. Where a non-PEFT model- # reuses its own checkpoint the flag is cleared below, and there- # `_gguf_writes_16bit_checkpoint` is already False on the same- # condition, so the two cannot disagree.+ # The same flag save_to_gguf reclaims on. Where a non-PEFT model reuses its own checkpoint the+ # flag is cleared below on the same condition, so the two cannot disagree.merge_is_disposable = merge_is_disposable,)- # Step 2: Prepare arguments for model savingarguments = dict(locals())arguments["model"] = selfarguments["tokenizer"] = tokenizerarguments["push_to_hub"] = False # We handle upload ourselves- # GPT-OSS needs mxfp4 save method+ # GPT-OSS needs the mxfp4 save method.if is_gpt_oss:if quantization_method is not None:_qm = (@@ -4939,30 +4613,22 @@ def unsloth_save_pretrained_gguf(del arguments["merge_is_disposable"] # decides reclamation, not how the merge is writtendel arguments["gguf_shard_size"] # only used by the gguf converter- # Preserve the requested output before reusing a non-PEFT checkpoint as input.- # Same definition the preflight sized, so the disk it measured is the one- # these files land on.+ # Preserve the requested output before reusing a non-PEFT checkpoint as input. Same definition the+ # preflight sized, so it measured the disk these files land on.gguf_directory = _gguf_output_directory(save_directory)- # Step 3: Fix tokenizer BOS token if neededif is_processor:fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer.tokenizer)else:fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer)- # Resolve the importance matrix (download upstream / validate path / rename *.gguf_file) up- # front, so a bad path or an unavailable upstream imatrix fails before the expensive 16-bit- # merge, and a failed auto-resolution never reaches the IQ-quant gate.+ # Resolve the imatrix (download, validate, rename *.gguf_file) up front, so a bad path or an unavailable+ # upstream fails before the expensive merge and never reaches the IQ-quant gate.imatrix_path = _resolve_imatrix_file(self, imatrix_file, token, save_directory)- # Who owns what, settled before a byte is written. Reclamation may only take- # files this export produced, and that is not a question the directory can- # answer afterwards: a caller reusing an output directory can already hold a- # finished sharded save, index and every shard it names, self-consistent and- # entirely present. transformers removes neither (its stale sweep does not- # match `model.safetensors.index`, and it only prunes shards under the stem it- # is writing), so after the merge that set is indistinguishable from one this- # run wrote. Recorded here instead, where the difference still exists.+ # Settle ownership before a byte is written: reclamation may only take files this export made, and+ # afterwards the directory cannot say, since a reused output directory can already hold a finished sharded+ # save transformers neither removes nor distinguishes from ours.try:preexisting_weights = frozenset(os.listdir(save_directory))except FileNotFoundError:@@ -4972,16 +4638,13 @@ def unsloth_save_pretrained_gguf(# Provenance unreadable. Proving nothing, the reclamation takes nothing.preexisting_weights = None- # Step 4: Save/merge model to 16-bit formatis_peft_model = isinstance(self, PeftModelForCausalLM) or isinstance(self, PeftModel)- # The flag holds for both branches that write the weights themselves; the middle- # branch reuses the user's own checkpoint instead and clears it, whatever the- # caller asked for.+ # The flag holds for both branches that write the weights; the middle branch reuses the user's own+ # checkpoint and clears it, whatever the caller asked for.if is_peft_model:print(f"Unsloth: Merging model weights to {'mxfp4' if is_gpt_oss else '16-bit'} format...")try:- # Call unsloth_generic_save directly (it's in the same file)with _hub_cache_prewarm_disabled(not _gguf_prewarm_ok):unsloth_generic_save(**arguments)@@ -4991,26 +4654,28 @@ def unsloth_save_pretrained_gguf(f"{_offloaded_parameter_hint(self)}") from eelse:- # Non-PEFT model: checkpoint files already exist; point save_to_gguf- # at the original path instead of re-saving to a temp subdir.+ # Non-PEFT model: the checkpoint already exists, so point save_to_gguf at the original path instead of+ # re-saving into a temp subdir.original_path = getattr(self.config, "_name_or_path", None)if original_path and os.path.isdir(original_path):print(f"Unsloth: Model is not a PEFT model. Using existing checkpoint at {original_path}")save_directory = original_path- # The user's own checkpoint, not an intermediate: without this an- # ordinary save_pretrained_gguf on a tight disk would delete the- # model it was handed.+ # The user's own checkpoint, not an intermediate: without this a save_pretrained_gguf on a tight+ # disk would delete the model it was handed.merge_is_disposable = False- # Persist tokenizer fixes (e.g. BOS token stripping) to disk- # so the GGUF converter picks up the corrected chat template.+ # Persist tokenizer fixes (BOS stripping) so the GGUF converter reads the corrected template.+ # Persist tokenizer fixes (e.g. BOS token stripping) to disk so the GGUF converter picks up the+ # corrected chat template.if tokenizer is not None:tokenizer.save_pretrained(save_directory)else:- # Fallback: save the in-memory model to save_directoryprint("Unsloth: Model is not a PEFT model. Saving directly without LoRA merge...")os.makedirs(save_directory, exist_ok = True)+ # `gguf_directory` can point anywhere, and freeing bytes on one filesystem does nothing for a+ # quantize pass writing to another: without this the merge could be deleted for a destination it+ # cannot help, data gone and the export still out of space.try:self.save_pretrained(save_directory)if tokenizer is not None:@@ -5024,18 +4689,15 @@ def unsloth_save_pretrained_gguf(if is_processor:tokenizer = tokenizer.tokenizer- # Use old chat template if the bos is removedif fix_bos_token:tokenizer.chat_template = old_chat_template- # Step 6: Clean up memoryfor _ in range(3):import gcgc.collect()if torch.cuda.is_available():torch.cuda.empty_cache()- # Step 7: Get model dtype and typetry:model_dtype = dtype_from_config(self.config)model_type = self.config.model_type@@ -5048,18 +4710,13 @@ def unsloth_save_pretrained_gguf(else:raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")except Exception as e:- # Fallback if dtype_from_config failsprint(f"Unsloth: Could not determine dtype ({e}), defaulting to float16")model_dtype = "float16"- # Step 8: Convert to GGUF formatprint("Unsloth: Converting to GGUF format...")- # Convert quantization_method to list if string- # Use old style quantization_methodquantization_methods = []if quantization_method is not None:- # Convert quantization_method to listif isinstance(quantization_method, list):passelif isinstance(quantization_method, str):@@ -5129,7 +4786,6 @@ def unsloth_save_pretrained_gguf(else:raise RuntimeError(f"Unsloth: GGUF conversion failed: {_describe_exception(e)}") from e- # Step 9: Create Ollama modelfilemodelfile_location = Noneollama_success = Falseif all_file_locations:@@ -5150,7 +4806,6 @@ def unsloth_save_pretrained_gguf(except Exception as e:print(f"Warning: Could not create Ollama modelfile: {e}")- # Step 10: Show BOS token warning if applicableif fix_bos_token:logger.warning("Unsloth: ##### The current model auto adds a BOS token.\n"@@ -5181,7 +4836,6 @@ def unsloth_save_pretrained_gguf(f"Unsloth: convert model to ollama format by running - ollama create model_name -f {modelfile_location}")- # Return a dict with all needed info for push_to_hubreturn {"save_directory": save_directory,"gguf_directory": gguf_directory,@@ -5203,13 +4857,12 @@ _DISK_FULL_PATTERNS = ("write failed: no space",)-# Kaggle allows 20GB. Below this headroom a failed conversion is plausibly-# about space; above it, blaming disk sends the user nowhere useful.+# Kaggle allows 20GB. Below this headroom a failed conversion is plausibly about space; above it, blaming disk+# sends the user nowhere useful._DISK_HEADROOM_BYTES = 2 * 1024**3-# A SIGKILLed child is 128 + 9 to a shell, so llama-quantize (run through-# `shell = True`) surfaces as "returned non-zero exit status 137" with no-# signal named anywhere in the text.+# A SIGKILLed child is 128 + 9 to a shell, so llama-quantize under shell = True surfaces as "returned non-zero+# exit status 137" with no signal named._OOM_KILL_PATTERNS = ("sigkill","exit status 137",@@ -5300,10 +4953,9 @@ def _gguf_failure_looks_like_disk(return Trueif getattr(exc, "errno", None) == 28:return True- # The output directory holds the file, so the first path that answers decides- # and the working directory is only a fallback. A roomy output disk called- # full because some unrelated filesystem is short would blame the disk for a- # quantizer failure and hide the advice that would have fixed it.+ # The output directory holds the file, so it decides and the working directory is only a fallback: calling+ # a roomy output disk full because an unrelated filesystem is short hides the advice that would have fixed+ # the quantizer failure.threshold = needed_bytes if needed_bytes and needed_bytes > 0 else _DISK_HEADROOM_BYTESwritten, written_device = 0, Noneif partial_output:@@ -5322,8 +4974,8 @@ def _gguf_failure_looks_like_disk(# Never let the diagnostic be the thing that raises.continueif written:- # Only on the filesystem that actually holds the partial file --- # bytes on one device are not room on another.+ # Only on the filesystem that actually holds the partial file: bytes on one device are not room on+ # another.try:if os.stat(path).st_dev == written_device:free += written@@ -5333,14 +4985,13 @@ def _gguf_failure_looks_like_disk(return False-# The full-precision output types. Everything else llama-quantize writes is-# quantized and carries its nominal width as the leading digit of the name-# (`q4_k_m`, `iq3_xxs`).+# The full-precision output types; everything else llama-quantize writes is quantized and carries its nominal+# width as the leading digit of the name._GGUF_BITS_PER_WEIGHT = {"f32": 32.0, "f16": 16.0, "bf16": 16.0}-# k- and i-quants store block scales and mins beside the weights, so a type is-# wider than its name. llama.cpp's published 7B sizes keep that overhead under a-# bit a weight (Q4_K_M near 4.5, Q6_K near 6.6), so 1.5 bounds every type here.+# k- and i-quants store block scales and mins beside the weights, so a type is wider than its+# name; llama.cpp's 7B sizes keep that under a bit per weight (Q4_K_M near 4.5, Q6_K near 6.6),+# so 1.5 bounds every type._QUANT_OVERHEAD_BITS = 1.5_QUANT_NOMINAL_BITS = re.compile(r"^i?q(\d+)")@@ -5385,38 +5036,25 @@ def _gguf_output_size_ratio(base = _gguf_type_bits(first_conversion) or 16.0target = _gguf_type_bits(quant_method)if target is None:- # Unrecognised: charge a whole copy of the base, as every method used to- # get -- or, for a diagnosis, admit the size is unknown.+ # Unrecognised: charge a whole copy of the base, as every method used to, or for a diagnosis admit the+ # size is unknown.return 1.0 if upper_bound else Noneif upper_bound and str(quant_method).lower() not in _GGUF_BITS_PER_WEIGHT:target += _QUANT_OVERHEAD_BITSreturn target / base-# The names a disposable merge is written under, and only those. Both writers of-# one produce safetensors: the PEFT branch goes through unsloth_zoo's-# `merge_and_overwrite_lora`, which rewrites the base safetensors shards (and the-# `consolidated.safetensors` some repos ship), and the non-PEFT fallback calls-# `save_pretrained` with no arguments, so transformers' `SAFE_WEIGHTS_NAME`.-#-# Neither the stem nor the extension is decoration. transformers clears stale-# shards only when both hold (`modeling_utils`:-# `filename.startswith(weights_no_suffix)` and `r"(.*?)-\d{5}-of-\d{5}"`), so a-# shard set under another stem, and a `pytorch_model.bin` from an earlier save in-# the other serialization, are files `save_pretrained` neither writes nor removes.-# This helper deletes permanently, so it leaves both alone. A merge under a name-# this misses is still reclaimed whenever it wrote an index, because the index-# names its own shards.+# The names a disposable merge is written under, and only those: both writers emit safetensors. Neither stem+# nor extension is decoration, since transformers clears stale shards only when both match, so another stem or+# an older .bin is a file save_pretrained never writes or removes -- and this helper deletes permanently._MERGE_WEIGHT_NAME = re.compile(r"^(model|consolidated)(-\d{5}-of-\d{5})?\.safetensors$")-# The index `save_pretrained` writes for a sharded save. Safetensors only, for-# the same reason the matcher is: a `pytorch_model.bin.index.json` belongs to an-# earlier save in the other serialization, which transformers leaves in place,-# and reading it would hand its shards to the deletion.+# The index save_pretrained writes for a sharded save. Safetensors only, like the matcher: a+# pytorch_model.bin.index.json belongs to an earlier save transformers leaves in place, and reading it would+# hand its shards to the deletion._WEIGHT_INDEX_NAMES = ("model.safetensors.index.json",)-# One shard of a sharded save, under any stem. Only ever applied to names an-# index already listed -- on its own this is far too wide, and dropping it from-# the matcher above is exactly what stopped a user's `backup-00001-of-00002`-# being read as the merge.+# One shard of a sharded save, under any stem. Applied only to names an index already listed: alone it is far+# too wide, and dropping it from the matcher is what stopped a user's backup-00001-of-00002 being read as the+# merge._INDEX_SHARD_NAME = re.compile(r"^(?P<stem>.+)-(?P<part>\d{5})-of-(?P<total>\d{5})\.safetensors$")@@ -5444,14 +5082,9 @@ def _merge_weight_files(model_directory, names):continueif _is_one_whole_shard_set(listed, names):indexed.update(listed)- # The index goes with the shards it named. `names` is already- # filtered to what this export wrote, so reaching here means this- # export wrote this index too, and leaving it behind is not neutral:- # it points at files that are about to be deleted, and on the next- # export into the same directory the provenance snapshot calls that- # leftover preexisting and filters it out, taking with it the only- # way a shard set under a stem `_MERGE_WEIGHT_NAME` misses is ever- # found again.+ # The index goes with the shards it named, and `names` is already filtered to this export's files.+ # Leaving it behind points at deleted files, and the next export's snapshot calls it preexisting,+ # losing the only way a shard set under a missed stem is found again.spent_indexes.add(index_name)return sorted(n for n in names if n in indexed or n in spent_indexes or _MERGE_WEIGHT_NAME.match(n)@@ -5487,8 +5120,7 @@ def _is_one_whole_shard_set(listed, names):totals.add(shard.group("total"))if len(stems) != 1 or len(totals) != 1:return False- # `of-000NN` states the count outright, so a listing missing shards -- or- # naming extra ones -- is not the set it claims to be.+ # "of-000NN" states the count, so a listing missing or adding shards is not the set it claims.if len(listed) != int(totals.pop()):return Falsereturn listed <= set(names)@@ -5519,10 +5151,8 @@ def _free_merge_if_disk_is_tight("""if not merge_is_disposable:return 0- # The other half of the safety story. `merge_is_disposable` says the export- # wrote this directory; this says which files in it the export wrote. A- # caller that cannot say gets no reclamation rather than a guess, because- # the guess is permanent.+ # merge_is_disposable says the export wrote this directory; this says which files it wrote. A caller that+ # cannot say gets no reclamation rather than a guess, since the guess is permanent.if preexisting_weights is None:return 0if not model_directory or not os.path.isdir(model_directory):@@ -5531,8 +5161,8 @@ def _free_merge_if_disk_is_tight(if not quant_methods:return 0try:- # llama-quantize copies companions rather than quantizing them, so they- # are excluded from the output and memory estimates.+ # llama-quantize copies companions rather than quantizing them, so they are excluded from the output+ # and memory estimates.base_bytes = sum(os.path.getsize(f)for f in initial_files@@ -5544,18 +5174,16 @@ def _free_merge_if_disk_is_tight(return 0target_directory = gguf_directory or model_directory- # `gguf_directory` can point anywhere, and freeing bytes on one filesystem- # does nothing for a quantize pass writing to another: without this the merge- # could be deleted for a destination it cannot help, data gone and the export- # still out of space.+ # gguf_directory can point anywhere, and freeing bytes on one filesystem does nothing for a pass writing+ # to another: the merge would be deleted for a destination it cannot help.try:if os.stat(model_directory).st_dev != os.stat(target_directory).st_dev:return 0except OSError:return 0- # Every output stays on disk, so the passes add up. Overestimating costs a- # deletion that was not strictly required; underestimating costs the export.+ # Every output stays on disk, so the passes add up. Overestimating costs an unnecessary deletion;+ # underestimating costs the export.needed = (base_bytes * sum(_gguf_output_size_ratio(m, first_conversion) for m in quant_methods)+ _DISK_HEADROOM_BYTES@@ -5573,12 +5201,9 @@ def _free_merge_if_disk_is_tight(except OSError:# Unreadable is no reason to fail an export that no longer needs it.return 0- # Anything that was already here is the caller's, whatever it is named. This- # is what stops a reused output directory losing a finished sharded save, or- # a `consolidated.safetensors` the merge never wrote: both are answers the- # names alone cannot give, since they are exactly the names a merge uses.- # It also drops a stale index out of the reading below, so the shards it- # names are never inherited.+ # Anything already here is the caller's, whatever it is named: that is what stops a reused directory+ # losing a finished sharded save or a consolidated.safetensors. It also drops a stale index from the+ # reading below, so its shards are never inherited.names = [name for name in names if name not in preexisting_weights]for name in _merge_weight_files(model_directory, names):path = os.path.join(model_directory, name)@@ -5661,7 +5286,7 @@ def unsloth_push_to_hub_gguf(if not is_main_process:return None- # save_method="lora" exports the adapter itself as a GGUF LoRA (not a merged model).+ # save_method="lora" exports the adapter itself as a GGUF LoRA, not a merged model.if save_method is not None and str(save_method).lower() == "lora":_qm = quantization_methodif isinstance(_qm, (list, tuple)) and len(_qm) == 1:@@ -5690,7 +5315,6 @@ def unsloth_push_to_hub_gguf(revision = revision,)- # Step 1: Determine save directorymodel_name = repo_id.split("/")[-1] if "/" in repo_id else repo_idif use_temp_dir or use_temp_dir is None:@@ -5700,14 +5324,12 @@ def unsloth_push_to_hub_gguf(save_directory = temp_dircleanup_temp = Trueelse:- save_directory = model_name # Use model name, not repo_id+ save_directory = model_namecleanup_temp = False- # Step 2: Call save_pretrained_gguf to do the conversionprint(f"Unsloth: Converting model to GGUF format...")try:- # Call save_pretrained_gguf - it returns all the info we needresult = unsloth_save_pretrained_gguf(self = self,save_directory = save_directory,@@ -5725,7 +5347,6 @@ def unsloth_push_to_hub_gguf(gguf_shard_size = gguf_shard_size,)- # Extract resultsall_file_locations = result["gguf_files"]modelfile_location = result["modelfile_location"]want_full_precision = result["want_full_precision"]@@ -5742,7 +5363,6 @@ def unsloth_push_to_hub_gguf(passraise RuntimeError(f"Failed to convert model to GGUF: {_describe_exception(e)}") from e- # Step 3: Upload to HuggingFace Hubprint("Unsloth: Uploading GGUF to Huggingface Hub...")try:@@ -5750,14 +5370,12 @@ def unsloth_push_to_hub_gguf(api = HfApi(token = token)- # Get full repo idif "/" not in repo_id:username = api.whoami()["name"]full_repo_id = f"{username}/{repo_id}"else:full_repo_id = repo_id- # Create repoapi.create_repo(repo_id = full_repo_id,repo_type = "model",@@ -5765,12 +5383,9 @@ def unsloth_push_to_hub_gguf(exist_ok = True,)- # Upload GGUF filesfor file_location in all_file_locations:original_name = os.path.basename(file_location)- # Replace temp directory name with proper model nameif cleanup_temp and "unsloth_gguf_" in original_name:- # Extract the quantization part (e.g., ".Q8_0.gguf" or ".Q8_0-mmproj.gguf")quant_suffix = (original_name.split(".", 1)[1] if "." in original_name else original_name)@@ -5791,7 +5406,6 @@ def unsloth_push_to_hub_gguf(revision = revision,)- # Upload config.json if existsconfig_path = os.path.join(actual_save_directory, "config.json")if os.path.exists(config_path):print("Uploading config.json...")@@ -5805,7 +5419,6 @@ def unsloth_push_to_hub_gguf(revision = revision,)- # Upload Modelfile if existsif modelfile_location and os.path.exists(modelfile_location):print("Uploading Ollama Modelfile...")api.upload_file(@@ -5818,7 +5431,6 @@ def unsloth_push_to_hub_gguf(revision = revision,)- # Create and upload READMEreadme_content = f"""---tags:- gguf@@ -5838,7 +5450,6 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi## Available Model files:"""for file in all_file_locations:- # Fix filename in README toooriginal_name = os.path.basename(file)if cleanup_temp and "unsloth_gguf_" in original_name:quant_suffix = (@@ -5849,7 +5460,6 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://giproper_name = original_name.replace(os.path.basename(save_directory), model_name)readme_content += f"- `{proper_name}`\n"- # Special note for VLM with Modelfileif is_vlm and modelfile_location:readme_content += "\n## ⚠️ Ollama Note for Vision Models\n"readme_content += "**Important:** Ollama currently does not support separate mmproj files for vision models.\n\n"@@ -5889,7 +5499,6 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://giprint(f"Unsloth: Successfully uploaded GGUF to https://huggingface.co/{full_repo_id}")- # Add tagsif tags is None:tags = []tags.extend(["gguf", "llama-cpp", "unsloth"])@@ -5918,7 +5527,6 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://giraise RuntimeError(f"Failed to upload to Hugging Face Hub: {_describe_exception(e)}") from efinally:- # Clean up temporary directoryif cleanup_temp:print("Unsloth: Cleaning up temporary files...")for d in [save_directory, f"{save_directory}_gguf"]:@@ -5931,12 +5539,9 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gireturn full_repo_id-# Corrected function to save LoRA to a custom directorydef save_lora_to_custom_dir(model, tokenizer, save_directory):- # Create the custom directory if it doesn't existos.makedirs(save_directory, exist_ok = True)- # Call the unsloth_save_model function with the custom directoryunsloth_save_model(model,tokenizer,@@ -5973,8 +5578,8 @@ def _lora_base_model_id(model):return os.fspath(base) if base else ""-# Upstream Unsloth GGUF repos ship a calibration imatrix under one of these names; the GGUF-format-# one is suffixed .gguf_file so the Hub does not list it as a model GGUF (renamed to .gguf locally).+# Upstream Unsloth GGUF repos ship a calibration imatrix under one of these names; the GGUF one is suffixed+# .gguf_file so the Hub does not list it as a model, and renamed locally._IMATRIX_UPSTREAM_NAMES = ("imatrix_unsloth.dat", "imatrix_unsloth.gguf_file")@@ -5997,8 +5602,8 @@ def _gguf_repo_candidates(model):passif not name:continue- # The upstream imatrix lives in unsloth/<base>-GGUF, so map any org (e.g. meta-llama/...)- # onto the unsloth org; keep an already-formed -GGUF id as-is.+ # The upstream imatrix lives in unsloth/<base>-GGUF, so map any org onto unsloth; an already-formed+ # -GGUF id is kept as-is.repo = name if name.endswith("-GGUF") else f"unsloth/{name.split('/')[-1]}-GGUF"if repo not in candidates:candidates.append(repo)@@ -6037,8 +5642,8 @@ def _resolve_imatrix_file(model, imatrix_file, token, dest_dir):f"(got {type(imatrix_file).__name__}).")- # imatrix_file=True: auto-resolve from the upstream Unsloth GGUF repo. HfApi is the module-level- # import (save.py top); hf_hub_download is imported here as it is not needed elsewhere.+ # imatrix_file=True auto-resolves from the upstream Unsloth GGUF repo; hf_hub_download is imported here+ # since nothing else needs it.from huggingface_hub import hf_hub_downloadif token is None:@@ -6096,7 +5701,7 @@ def _unsloth_save_lora_gguf(if token is None:token = get_token()- # Resolve the dequantized base id (the adapter usually references a 4bit repo).+ # Resolve the dequantized base id, since the adapter usually references a 4bit repo.base_model_id = _lora_base_model_id(model)if not base_model_id:raise RuntimeError(@@ -6109,6 +5714,8 @@ def _unsloth_save_lora_gguf(passmodel_name = _model_basename(base_model_id)+ # Save the adapter: an isolated temp dir for a hub push, else save_directory itself, wrapped so it is always+ # cleaned up.# Save the adapter; for a hub push use an isolated temp dir, else save_directory itself.if push_to_hub:lora_dir = tempfile.mkdtemp(prefix = "unsloth-lora-gguf-")@@ -6120,12 +5727,13 @@ def _unsloth_save_lora_gguf(try:save_lora_to_custom_dir(model, tokenizer, lora_dir)- # Ensure a full llama.cpp checkout (ships convert_lora_to_gguf.py) and locate the converter.+ # Ensure a full llama.cpp checkout, which ships convert_lora_to_gguf.py: a prebuilt install or a+ # reused CWD copy carries binaries but not the script.install_llama_cpp(just_clone_repo = True)converter = os.path.join(LLAMA_CPP_DEFAULT_DIR, "convert_lora_to_gguf.py")if not os.path.exists(converter):- # A prebuilt llama.cpp install (or a reused CWD copy) carries binaries but not the- # converter script, so force a dedicated source checkout that ships it.+ # A prebuilt llama.cpp install (or a reused CWD copy) carries binaries but not the converter script,+ # so force a dedicated source checkout that ships it.source_dir = os.path.join(os.path.dirname(os.path.normpath(LLAMA_CPP_DEFAULT_DIR)), "llama.cpp-source")@@ -6144,8 +5752,8 @@ def _unsloth_save_lora_gguf(cmd += ["--base", base_model_id]else:cmd += ["--base-model-id", base_model_id]- # Only pass --trust-remote-code when the loaded model actually came from custom code (the- # approved load decision), not merely because its config carries an auto_map entry.+ # Pass --trust-remote-code only when the model really came from custom code (the approved load+ # decision), not merely because its config carries an auto_map entry.if _loaded_via_remote_code(model):cmd.append("--trust-remote-code")@@ -6156,6 +5764,8 @@ def _unsloth_save_lora_gguf(env["HUGGING_FACE_HUB_TOKEN"] = tokenprint(f"Unsloth: Converting LoRA adapter at '{lora_dir}' to GGUF -> '{out_gguf}'")+ # Resolve the cache from the live env like the merge, not huggingface_hub's frozen constants: a runtime+ # cache redirect (read-only default, Unsloth) would else miss (#6890).try:with subprocess.Popen(cmd,@@ -6257,9 +5867,8 @@ from .models.loader_utils import (_tokenizer_wants_local_only,)-# Imported lazily at the two call sites below: a zoo older than the one that made-# its own bitsandbytes import optional would otherwise break `import unsloth` on a-# host without bnb, which is the whole point of the guards above.+# Imported lazily at the two call sites: an older zoo, before its bitsandbytes import became optional, would+# otherwise break `import unsloth` on a host without bnb.from unsloth_zoo.llama_cpp import (install_llama_cpp,convert_to_gguf as _convert_to_gguf,@@ -6320,8 +5929,8 @@ def _prewarm_base_model_hub_cache()if not model_name or is_local_path:return- # Mirror the merge: an FP8 base with a 16bit sibling merges onto the sibling, so- # pre-warm the sibling (what the merge downloads), not the FP8 repo (#6890).+ # Mirror the merge: an FP8 base with a 16bit sibling merges onto the sibling, so pre-warm the sibling+ # rather than the FP8 repo (#6890).if base_is_quantized and quant_type == "fp8" and save_method == "merged_16bit":try:from unsloth_zoo.saving_utils import _resolve_fp8_16bit_sibling@@ -6339,8 +5948,8 @@ def _prewarm_base_model_hub_cache(from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download- # Resolve the cache from the live env like the merge, not huggingface_hub's frozen- # constants: a runtime cache redirect (read-only default, Unsloth) would else miss (#6890).+ # Resolve the cache from the live env like the merge, not huggingface_hub's frozen constants, or a+ # runtime cache redirect misses (#6890).try:from unsloth_zoo.hf_cache import _active_caches_hub_cache = _active_caches()[1]@@ -6348,8 +5957,8 @@ def _prewarm_base_model_hub_cache(except Exception:hub_cache_dir = None- # Mirror the zoo's shard listing (drop consolidated.safetensors when proper- # shards coexist) so the cached set is a superset of what the merge looks up.+ # Mirror the zoo's shard listing (drop consolidated.safetensors when real shards coexist) so the+ # cached set is a superset of what the merge looks up.shard_names = []total_size_in_bytes = 0for x in HfFileSystem(token = token).ls(model_name, detail = True):@@ -6373,9 +5982,8 @@ def _prewarm_base_model_hub_cache(except Exception:pass- # Mirror the merge's index filter (download path only): some repos ship leftover shards- # the index omits; keep only indexed ones, else the disk gate over-counts and we fetch- # unused shards.+ # Mirror the merge's index filter on the download path: some repos ship shards the index omits, so+ # keep only indexed ones or the disk gate over-counts.if len(shard_names) > 1:try:import json as _json@@ -6398,7 +6006,7 @@ def _prewarm_base_model_hub_cache(passtotal_size_in_bytes = sum(size for _, size in shard_names)- # The cache copy is extra disk on top of the merge working copy; need room for both.+ # The cache copy is extra disk on top of the merge working copy; there must be room for both.from huggingface_hub import constants as _hf_constants# abspath so a relative HF_HUB_CACHE walks up to an existing root, not "".@@ -6460,10 +6068,8 @@ def save_to_gguf_generic(if not os.path.exists(os.path.join("llama.cpp", "unsloth_convert_hf_to_gguf.py")):install_llama_cpp(just_clone_repo = True)- # Use old style quantization_methodnew_quantization_methods = []if quantization_method is not None:- # Convert quantization_method to listif isinstance(quantization_method, list):passelif isinstance(quantization_method, str):@@ -6490,7 +6096,6 @@ def save_to_gguf_generic(new_quantization_methods.append(quant_method.lower())else:new_quantization_methods.append(quantization_type.lower())- # Check if wrong methodfor quant_method in new_quantization_methods:if quant_method not in ALLOWED_QUANTS.keys():error = f"Unsloth: Quant method = [{quant_method}] not supported. Choose from below:\n"@@ -6498,8 +6103,6 @@ def save_to_gguf_generic(error += f"[{key}] => {value}\n"raise RuntimeError(error)- # Go through all types and save individually - somewhat inefficient- # since we save F16 / BF16 multiple timesfor quantization_type in new_quantization_methods:metadata = _convert_to_gguf(save_directory,@@ -6546,7 +6149,6 @@ def unsloth_generic_save(safe_serialization: bool = True,variant: Optional[str] = None,save_peft_format: bool = True,- # Push to hubuse_temp_dir: Optional[bool] = None,commit_message: Optional[str] = "Trained with Unsloth",private: Optional[bool] = None,@@ -6554,7 +6156,6 @@ def unsloth_generic_save(revision: str = None,commit_description: str = "Upload model trained with Unsloth 2x faster",tags: List[str] = None,- # Our functionstemporary_location: str = "_unsloth_temporary_saved_buffers",maximum_memory_usage: float = 0.9,datasets: Optional[List[str]] = None,@@ -6575,8 +6176,8 @@ def unsloth_generic_save(elif save_method == "merged_4bit_forced":save_method = "merged_4bit"- # Full-finetuned models (no LoRA) have no adapters to merge, so fall back- # to save_pretrained, mirroring the torchao and GGUF save paths.+ # Full-finetuned models have no adapters to merge, so fall back to save_pretrained, mirroring the torchao+ # and GGUF save paths._is_peft = isinstance(model, PeftModel)if not _is_peft:if not is_main_process:@@ -6709,25 +6310,23 @@ def unsloth_generic_save_pretrained_merged("You can do it separately via `tokenizer.save_pretrained(...)`")- # Kaggle's working directory is ~20GB while the overlay at /tmp on the same- # kernel is measured in terabytes. Relative paths under /kaggle/working that- # do not fit move there; absolute paths, hub pushes and every non-Kaggle- # machine are untouched.+ # Kaggle's working directory is ~20GB while /tmp on the same kernel is terabytes, so relative paths under+ # /kaggle/working that do not fit move there. Absolute paths, hub pushes and non-Kaggle machines are+ # untouched.save_directory = _preflight_merge_disk(self,save_directory,save_method,push_to_hub = push_to_hub,state_dict = state_dict,- # `unsloth_generic_save` writes a supplied dictionary rather than the- # resident model when there is no adapter to merge.+ # unsloth_generic_save writes a supplied dictionary rather than the resident model when there is no+ # adapter, and it alone runs merge_and_overwrite_lora when there is one.forwards_state_dict = True,- # And it is the writer that runs `merge_and_overwrite_lora` when there- # IS one, which no other entrypoint here does.+ # And it is the writer that runs `merge_and_overwrite_lora` when there IS one, which no other entrypoint+ # here does.writer_runs_merge_guard = True,)- # FP8 / FP4 compressed-tensors export (llm-compressor) -> handled separately._compressed = _normalize_compressed_method(save_method)if _compressed is not None:scheme, needs_calibration, suffix = _compressed@@ -6744,7 +6343,6 @@ def unsloth_generic_save_pretrained_merged(calibration_dataset = calibration_dataset,num_calibration_samples = num_calibration_samples,max_seq_length = max_seq_length,- # Forward standard save kwargs to the 16bit merge.state_dict = state_dict,save_function = save_function,max_shard_size = max_shard_size,@@ -6760,7 +6358,6 @@ def unsloth_generic_save_pretrained_merged(gc.collect()return- # torchao portable FP8/INT8 export (no NVIDIA GPU) -> separate path._torchao = _normalize_torchao_method(save_method)if _torchao is not None:kind, suffix = _torchao@@ -6773,7 +6370,6 @@ def unsloth_generic_save_pretrained_merged(push_to_hub = push_to_hub,token = token,is_main_process = is_main_process,- # Forward standard save kwargs to the 16bit merge.state_dict = state_dict,save_function = save_function,max_shard_size = max_shard_size,@@ -6840,7 +6436,6 @@ def unsloth_generic_push_to_hub_merged("You can do it separately via `tokenizer.push_to_hub(...)`")- # FP8 / FP4 compressed-tensors export (llm-compressor) -> handled separately._compressed = _normalize_compressed_method(save_method)if _compressed is not None:scheme, needs_calibration, suffix = _compressed@@ -6861,7 +6456,6 @@ def unsloth_generic_push_to_hub_merged(calibration_dataset = calibration_dataset,num_calibration_samples = num_calibration_samples,max_seq_length = max_seq_length,- # Forward standard save kwargs to the 16bit merge.use_temp_dir = use_temp_dir,max_shard_size = max_shard_size,safe_serialization = safe_serialization,@@ -6874,7 +6468,6 @@ def unsloth_generic_push_to_hub_merged(gc.collect()return- # torchao portable FP8/INT8 export (no NVIDIA GPU) -> separate path._torchao = _normalize_torchao_method(save_method)if _torchao is not None:kind, suffix = _torchao@@ -6892,7 +6485,6 @@ def unsloth_generic_push_to_hub_merged(commit_description = commit_description,create_pr = create_pr,revision = revision,- # Forward standard save kwargs to the 16bit merge.use_temp_dir = use_temp_dir,max_shard_size = max_shard_size,safe_serialization = safe_serialization,@@ -6929,9 +6521,7 @@ def _unsloth_save_torchao_with_attached_config(token: Optional[Union[str, bool]] = None,):"""Save a QAT-trained model by converting fake-quantized weights to real quantized weights."""- # Convert QAT fake-quantized weights to real quantized weights_convert_torchao_model(model)- # PEFT models also might come here, so parse itif isinstance(model, PeftModelForCausalLM):_unsloth_save_torchao_with_given_config(model = model,@@ -6943,7 +6533,7 @@ def _unsloth_save_torchao_with_attached_config()return- # TorchAO does not support safe_serialization reliably+ # TorchAO does not support safe_serialization reliably.safe_serialization = Falseif push_to_hub:@@ -6977,7 +6567,6 @@ def _unsloth_save_torchao_with_given_config(torchao_config is not None), "Unsloth: Please specify a torchao_config for post-training quantization!"- # first merge the lora weightsarguments = dict(locals())arguments["push_to_hub"] = False # We save ourselvesarguments["save_method"] = "merged_16bit" # Must be 16bit@@ -7006,7 +6595,6 @@ def _unsloth_save_torchao_with_given_config(else:quantization_config = TorchAoConfig(quant_type = torchao_config)- # Determine if this is a VLMis_vlm = _is_vlm(model)auto_model = AutoModelForImageTextToText if is_vlm else AutoModelForCausalLMauto_processor = AutoProcessor if is_vlm else AutoTokenizer@@ -7015,14 +6603,13 @@ def _unsloth_save_torchao_with_given_config(if isinstance(tokenizer, (PreTrainedTokenizerBase, ProcessorMixin)):tokenizer = patch_saving_functions(tokenizer)- # TorchAO must only use bfloat16 for loading (float16 fails)+ # TorchAO must only use bfloat16 for loading; float16 fails.if HAS_TORCH_DTYPE:kwargs = {"torch_dtype": torch.bfloat16}else:kwargs = {"dtype": torch.bfloat16}- # Else the original stays resident on every GPU while device_map="auto" below- # loads a second copy.+ # Else the original stays resident on every GPU while device_map="auto" below loads a second copy.model_restore = _offload_model_for_quantize_subprocess(model)for _ in range(3):gc.collect()@@ -7031,10 +6618,11 @@ def _unsloth_save_torchao_with_given_config(if hasattr(torch, "xpu") and torch.xpu.is_available():torch.xpu.empty_cache()- # The original stays offloaded until the quantized copy is saved AND released,- # else both are resident at once and the restore OOMs.+ # The original stays offloaded until the quantized copy is saved AND released, else both are resident and+ # the restore OOMs.+ # The original stays offloaded until the quantized copy is saved AND released, else both are resident at+ # once and the restore OOMs.try:- # Reload with quantization appliedquantized_model = auto_model.from_pretrained(save_directory,device_map = "auto",@@ -7044,7 +6632,7 @@ def _unsloth_save_torchao_with_given_config(torchao_save_directory = save_directory + "-torchao"- # TorchAO does not support safe_serialization right now 0.14.0 seems broken!+ # TorchAO does not support safe_serialization right now; 0.14.0 seems broken.safe_serialization = Version(importlib_version("torchao")) > Version("0.14.0")safe_serialization = False@@ -7060,12 +6648,12 @@ def _unsloth_save_torchao_with_given_config(tokenizer.save_pretrained(torchao_save_directory, token = token)finally:- # del here, not at the end of the try: if save_pretrained raises, the copy- # would otherwise still be resident while the original is restored.+ # del here, not at the end of the try: if save_pretrained raises, the copy would still be resident+ # while the original is restored.quantized_model = Nonedel quantized_model- # A failed save leaves a live traceback whose frames still hold the copy, so- # dropping the local alone does not free its VRAM.+ # A failed save leaves a live traceback whose frames hold the copy, so dropping the local alone does+ # not free its VRAM._exc = sys.exc_info()[1]if _exc is not None:traceback.clear_frames(_exc.__traceback__)@@ -7077,7 +6665,6 @@ def _unsloth_save_torchao_with_given_config(torch.xpu.empty_cache()_restore_model_after_quantize_subprocess(model, model_restore)- # Clean up the intermediate unquantized modelif os.path.exists(save_directory):try:shutil.rmtree(save_directory)@@ -7167,18 +6754,16 @@ def _snapshot_dispatch_state(root):for name, mod in root.named_modules()if "_hf_hook" in mod.__dict__]- # remove_duplicate=False: the default hides one half of every tied pair, exactly- # the half that needs re-tying below.+ # remove_duplicate=False: the default hides one half of every tied pair, exactly the half that needs re-+ # tying below.named = list(root.named_parameters(remove_duplicate = False)) + list(root.named_buffers(remove_duplicate = False))places = {name: tensor.device for name, tensor in named}- # Tied weights share one storage, but the CPU round trip repoints every tensor and- # tied_params_map is keyed on the old pointer, so replaying the hooks alone gives- # independent copies: double VRAM, and updates to one no longer reach the other.- # Skip meta tensors: offloaded parameters all sit on meta with pointer 0, which- # would collapse into one fake "tied" group of differently shaped tensors, and- # each side of a tie is already its own meta placeholder so nothing is lost.+ # Tied weights share one storage, but the CPU round trip repoints every tensor while tied_params_map is+ # keyed on the old pointer, so replaying hooks alone yields independent copies: double VRAM and divergent+ # updates. Skip meta tensors, since offloaded parameters all have pointer 0 and would collapse into one+ # fake tied group.groups = {}for name, tensor in named:if tensor.device.type == "meta":@@ -7187,18 +6772,17 @@ def _snapshot_dispatch_state(root):if ptr:groups.setdefault(ptr, []).append(name)ties = [names for names in groups.values() if len(names) > 1]- # Removing a hook restores `forward = _old_forward`, captured before unsloth patched- # the module, so a remove/re-add permanently drops every fused kernel installed after- # the dispatch (measured: apply_lora_mlp_swiglu on all 28 MLPs). It also deletes the- # `to`/`cuda`/... move guards, so record those too.+ # Removing a hook restores forward = _old_forward, captured before unsloth patched the module, so a+ # remove/re-add permanently drops every fused kernel installed after the dispatch (measured on all 28+ # MLPs) and the to/cuda move guards. Record those too.attrs = ("forward", "_old_forward") + tuple(_ACCELERATE_MOVE_GUARDS)saved_attrs = {name: {a: mod.__dict__[a] for a in attrs if a in mod.__dict__}for name, mod in root.named_modules()if any(a in mod.__dict__ for a in attrs)}- # Re-adding a hook runs init_hook -> set_module_tensor_to_device, which builds a fresh- # Parameter and so drops .grad. Snapshot the gradients and reattach them on restore.+ # Re-adding a hook runs init_hook -> set_module_tensor_to_device, building a fresh Parameter and dropping+ # .grad, so snapshot the gradients and reattach on restore.grads = {name: getattr(tensor, "grad", None)for name, tensor in root.named_parameters(remove_duplicate = False)@@ -7271,16 +6855,16 @@ def _restore_dispatch_state(root, snapshot) -> None:for name, hook in hooks:add_hook_to_module(root.get_submodule(name) if name else root, hook)- # Re-adding a hook rewraps whatever `_old_forward` now holds, so put the exact- # callables back, `_old_forward` first.+ # Re-adding a hook rewraps whatever _old_forward now holds, so put the exact callables back, _old_forward+ # first.for name, values in saved_attrs.items():mod = root.get_submodule(name) if name else rootfor attr in ("_old_forward", "forward", *_ACCELERATE_MOVE_GUARDS):if attr in values:mod.__dict__[attr] = values[attr]- # init_hook only re-places tensors the hooked module owns, so anything added after- # the dispatch (the LoRA adapters) is still on CPU.+ # init_hook only re-places tensors the hooked module owns, so anything added after the dispatch (the LoRA+ # adapters) is still on CPU.for mod_name, mod in root.named_modules():for attr in ("_parameters", "_buffers"):store = getattr(mod, attr, None)@@ -7312,8 +6896,8 @@ def _restore_dispatch_state(root, snapshot) -> None:continuefor follower in names[1:]:_share_tensor(root, follower, leader)- # init_hook refilled tied_params_map with the pre-retie tensors, now unreferenced- # by the model but still pinned by the map.+ # init_hook refilled tied_params_map with the pre-retie tensors, now unreferenced by the model but still+ # pinned by the map.if ties:_drop_accelerator_tied_param_cache(snapshot)@@ -7344,9 +6928,8 @@ def _offload_model_for_quantize_subprocess(model):device_map = getattr(model, "hf_device_map", None)if device_map:targets = {str(v).lower() for v in device_map.values()}- # A cpu spill is fine to move, it is already in host RAM. disk/meta is not:- # those parameters are off the model, so .to("cpu") would materialize the- # whole checkpoint into RAM.+ # A cpu spill is safe to move, being in host RAM; disk/meta is not, since those parameters are off+ # the model and .to("cpu") would materialize the whole checkpoint.if not all(t.isdigit() or t.startswith(("cuda", "xpu")) or t == "cpu" for t in targets):return Noneif not any(t.isdigit() or t.startswith(("cuda", "xpu")) for t in targets):@@ -7358,6 +6941,8 @@ def _offload_model_for_quantize_subprocess(model):try:setattr(root, _DISPATCH_SNAPSHOT_ATTR, _snapshot_dispatch_state(root))except Exception as snap_exc:+ # A silent `return None` is indistinguishable from "nothing to move", which hides a real bug+ # behind a merely slower export.# Restore will fall back to re-deriving from the device_map.logger.warning_once(f"Unsloth: could not snapshot the accelerate dispatch "@@ -7367,8 +6952,8 @@ def _offload_model_for_quantize_subprocess(model):try:model.to("cpu")except Exception:- # The move failed after the hooks came off; re-dispatch so the model is- # left usable rather than hookless and half-moved across CPU/GPUs.+ # The move failed after the hooks came off, so re-dispatch and leave the model usable rather+ # than hookless and half-moved across CPU/GPUs._restore_model_after_quantize_subprocess(model, ("dispatch", dict(device_map)))return Nonesnapshot = getattr(root, _DISPATCH_SNAPSHOT_ATTR, None)@@ -7385,8 +6970,8 @@ def _offload_model_for_quantize_subprocess(model):return Nonereturn ("device", device)except Exception as exc:- # A silent `return None` is indistinguishable from "nothing to move", which- # hides a real bug behind a merely slower export.+ # A silent `return None` is indistinguishable from "nothing to move", hiding a real bug behind a merely+ # slower export.logger.warning_once(f"Unsloth: could not free the model's accelerator memory before the quantized "f"export ({type(exc).__name__}: {exc}); continuing with the model resident."@@ -7409,15 +6994,15 @@ def _restore_model_after_quantize_subprocess(model, restore_token) -> None:else:from accelerate import dispatch_model- # skip_keys matters: without it accelerate moves every forward kwarg- # to the executing device, wrong for device-invariant cache tensors.+ # skip_keys matters: without it accelerate moves every forward kwarg to the executing device,+ # wrong for device-invariant cache tensors.dispatch_model(root,device_map = value,skip_keys = getattr(root, "_skip_keys_device_placement", None),)else:- model.to(value) # restore the model to its original device+ model.to(value)except Exception:logger.warning_once("Unsloth: could not restore the model to its original device(s) after the "@@ -7456,21 +7041,19 @@ def _unsloth_save_compressed_tensors(if token is None:token = get_token()- # Only the main process installs deps, merges, quantizes, and uploads (mirrors the non-PEFT- # save path); other ranks return at once so they neither race on dirs nor run pip installs.+ # Only the main process installs deps, merges, quantizes and uploads, mirroring the non-PEFT save path;+ # other ranks return at once rather than race on dirs or run pip installs.if not is_main_process:return None- # 1) Prepare the quantization runtime BEFORE merging, so an unusable config fails fast instead of- # writing a full 16bit checkpoint first. With the llm-compressor-main shadow the subprocess- # validates everything itself, so skip the workspace install / ceiling / scheme checks; without- # it, install the workspace llm-compressor and fail fast past its transformers ceiling.+ # Prepare the quantization runtime BEFORE merging, so an unusable config fails fast instead of writing a+ # full 16bit checkpoint first. Under the llm-compressor-main shadow the subprocess validates itself, so+ # the workspace install / ceiling / scheme checks are skipped._shadow_pythonpath = _compressed_quantize_pythonpath()if _shadow_pythonpath is None:install_llm_compressor()- # llm-compressor cannot run under a newer transformers than its ceiling: the quantization- # subprocess would die with a cryptic ImportError (TORCH_INIT_FUNCTIONS) only AFTER the costly- # 16bit merge. Detect and fail fast with an actionable message instead.+ # llm-compressor cannot run under a newer transformers than its ceiling: the subprocess dies on a+ # cryptic TORCH_INIT_FUNCTIONS ImportError only AFTER the costly merge._exceeds, _tf_ver = _transformers_exceeds_llm_compressor_ceiling()if _exceeds:raise RuntimeError(@@ -7492,8 +7075,8 @@ def _unsloth_save_compressed_tensors("Use save_method in {fp8, mxfp4, nvfp4}, or upgrade transformers + llm-compressor.")- # 2) Pick the local working dir. For a hub push, save_directory is a repo id, so merge and- # quantize inside an isolated temp dir instead of writing ./<repo_id> into the cwd.+ # Pick the local working dir: on a hub push save_directory is a repo id, so merge and quantize inside a+ # temp dir rather than writing ./<repo_id> into the cwd.repo_id, work_tmp, calib_tmp, model_restore = None, None, None, Noneif push_to_hub:repo_id = os.fspath(save_directory)@@ -7504,12 +7087,14 @@ def _unsloth_save_compressed_tensors(local_dir = os.fspath(save_directory)local_dir = local_dir.rstrip("/\\") or local_dir- # Wrap the body so the isolated temp dirs are always cleaned up, even when the merge,- # quantization, validation, or hub upload raises.+ # Wrap the body so the isolated temp dirs are always cleaned up, even when the merge, quantization,+ # validation, or hub upload raises.api = Nonetry:- # Validate Hub access up front (a bad token / denied repo should fail before the expensive- # merge and quantization, matching the normal push path). create_repo is idempotent.+ # Validate Hub access up front, so a bad token or denied repo fails before the expensive merge and+ # quantization; create_repo is idempotent.+ # Validate Hub access up front (a bad token / denied repo should fail before the expensive merge and+ # quantization, matching the normal push path). create_repo is idempotent.if push_to_hub:from huggingface_hub import HfApiapi = HfApi(token = token)@@ -7520,12 +7105,10 @@ def _unsloth_save_compressed_tensors(exist_ok = True,)- # 3) Merge to 16bit at local_dir (kept for local saves) via unsloth_generic_save, so LoRA- # adapters are merged and full-finetuned models written in 16bit consistently. Extra- # save kwargs (state_dict, max_shard_size, ...) flow through merge_kwargs.- # The intermediate 16bit checkpoint is internal staging that the converter subprocess- # reloads with default weight filenames, so never write variant-named shards here; the- # user's variant (if any) is applied to the final compressed checkpoint in the subprocess.+ # Merge to 16bit at local_dir via unsloth_generic_save, so adapters are merged and full-finetuned+ # models written in 16bit alike. The subprocess reloads this staging checkpoint with default weight+ # filenames, so never write variant-named shards here; the user's variant goes on the final compressed+ # checkpoint.variant = merge_kwargs.pop("variant", None)print(f"Unsloth: Merging to 16bit before {scheme} quantization...")merge_args = dict(merge_kwargs)@@ -7542,10 +7125,9 @@ def _unsloth_save_compressed_tensors()unsloth_generic_save(**merge_args)- # 4) Detect VLM from the in-memory model config. A vision/multimodal model exposes a- # vision_config or an explicitly vision-named architecture; a bare- # *ForConditionalGeneration also matches text seq2seq models (T5/BART/Whisper), so it- # is not treated as a VLM on its own.+ # Detect a VLM from the in-memory config: a vision_config or a vision-named architecture. A bare+ # *ForConditionalGeneration also matches text seq2seq (T5/BART/Whisper), so it does not count on its+ # own.is_vlm = Falseif hasattr(model, "config"):archs = getattr(model.config, "architectures", None) or []@@ -7557,26 +7139,26 @@ def _unsloth_save_compressed_tensors("Unsloth: FP8/FP4 compressed export for vision / multimodal models is ""experimental; vision-tower layers may be affected.")- # trust_remote_code must reflect the approved load decision (whether the model / tokenizer- # was actually loaded from custom code), not the config's static auto_map, so a- # built-in-loadable model carrying auto_map cannot run unvetted code in the subprocess.- # Model and tokenizer trust stay separate, like the torchao path: an approved custom- # tokenizer must not enable an unapproved model's code in the subprocess (or vice versa).+ # trust_remote_code must reflect the approved load decision, not the config's static auto_map, so a+ # built-in-loadable model carrying auto_map cannot run unvetted code in the subprocess. Model and+ # tokenizer trust stay separate, as on the torchao path.model_trust = _loaded_via_remote_code(model)tok_trust = _loaded_via_remote_code(tokenizer)- # 5) Marshal the calibration dataset for the subprocess: None -> ultrachat default; a- # str/PathLike is a local save_to_disk dir if it exists else a Hub id; Dataset -> temp.+ # Marshal the calibration dataset for the subprocess: None means the ultrachat default, a str/PathLike+ # is a local save_to_disk dir if it exists else a Hub id, a Dataset goes to temp.calib_kind, calib_value = "none", ""if needs_calibration and calibration_dataset is not None:if isinstance(calibration_dataset, (str, os.PathLike)):calib_value = os.fspath(calibration_dataset)calib_kind = "disk" if os.path.isdir(calib_value) else "hfid"elif hasattr(calibration_dataset, "save_to_disk"):- # Only persist the samples we need, so multi-GB training sets are not fully copied.+ # Only persist the samples needed, so multi-GB training sets are not fully copied.ds_to_save = calibration_dataset- # A DatasetDict's len() is the split count, not rows; pick one split first so the- # row subsample below applies and we do not save every split to the temp dir.+ # A DatasetDict's len() is the split count, not rows, so pick one split first or every split+ # is saved to the temp dir.+ # A DatasetDict's len() is the split count, not rows; pick one split first so the row subsample+ # below applies and we do not save every split to the temp dir.try:from datasets import DatasetDictif isinstance(ds_to_save, DatasetDict):@@ -7611,9 +7193,9 @@ def _unsloth_save_compressed_tensors(f"Unsloth: scheme '{scheme}' is data-free; ignoring calibration_dataset.")- # 6) Quantize in a separate process: importing Unsloth patches transformers attention,- # which breaks the forward llm-compressor runs for calibration. Run the converter by- # file path (not `-m`) so the subprocess stays unpatched, like GGUF -> llama.cpp.+ # Quantize in a separate process: importing Unsloth patches transformers attention, which breaks the+ # forward llm-compressor runs for calibration. Invoke the converter by file path, not `-m`, so the+ # subprocess stays unpatched.out_dir = local_dir + "-" + suffixrunner = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_compressed_quantize.py")cmd = [@@ -7645,8 +7227,8 @@ def _unsloth_save_compressed_tensors(if variant:cmd += ["--variant", variant]- # Free the in-memory model's CUDA memory before the subprocess loads its own copy from- # disk, so the GPUs need not hold both at once. Best-effort, restored in finally.+ # Free the in-memory model's CUDA memory before the subprocess loads its own copy, so the GPUs need+ # not hold both. Best-effort, restored in finally.model_restore = _offload_model_for_quantize_subprocess(model)for _ in range(3):gc.collect()@@ -7659,9 +7241,9 @@ def _unsloth_save_compressed_tensors(env["HF_TOKEN"] = tokenenv["HUGGING_FACE_HUB_TOKEN"] = token- # Clean PYTHONPATH = shadow only. torch still comes from the interpreter's site-packages;- # transformers 5.x + llm-compressor main come from the shadow. Dropping the inherited- # PYTHONPATH removes any parent transformers sidecar so the shadow's is authoritative.+ # Clean PYTHONPATH means shadow only: torch still comes from site-packages while transformers 5.x and+ # llm-compressor main come from the shadow, and dropping the inherited PYTHONPATH removes any parent+ # transformers sidecar.if _shadow_pythonpath is not None:env["PYTHONPATH"] = _shadow_pythonpath@@ -7678,7 +7260,6 @@ def _unsloth_save_compressed_tensors(f"{e.returncode}). See the output above for details.")- # 7) Validate the artifact.cfg_path = os.path.join(out_dir, "config.json")cfg = {}if os.path.exists(cfg_path):@@ -7689,8 +7270,10 @@ def _unsloth_save_compressed_tensors(f"Unsloth: {scheme} export failed - no quantization_config written to {cfg_path}")- # 8) Optional hub upload of the compressed artifact (not the intermediate 16bit one).- # The repo was already created/validated up front, so just upload here.+ # Optional hub upload of the compressed artifact, not the intermediate 16bit one; the repo was created+ # and validated up front.+ # 8) Optional hub upload of the compressed artifact (not the intermediate 16bit one). The repo was+ # already created/validated up front, so just upload here.if push_to_hub:print(f"Unsloth: Uploading {scheme} checkpoint to '{repo_id}' ...")api.upload_folder(@@ -7713,7 +7296,6 @@ def _unsloth_save_compressed_tensors(f"Unsloth: could not update datasets metadata for {repo_id}: {meta_err}")- # 9) Inference hardware note.result = repo_id if push_to_hub else out_dir_print_compressed_hw_note(scheme, result)return result@@ -7753,7 +7335,7 @@ def _unsloth_save_torchao(if token is None:token = get_token()- # Only the main process merges, quantizes, and uploads; other ranks return at once.+ # Only the main process merges, quantizes and uploads; other ranks return at once.if not is_main_process:return None@@ -7774,9 +7356,9 @@ def _unsloth_save_torchao(else:raise RuntimeError(f"Unsloth: unknown torchao export kind '{kind}' (expected fp8/int8).")- # Always merge into an isolated temp staging dir (never save_directory itself), so a co-selected- # 16-bit export written to save_directory is not overwritten or deleted; the torchao output is- # the sibling "<save_directory>-<suffix>" (or the repo id on a hub push).+ # Always merge into an isolated temp staging dir, never save_directory, so a co-selected 16-bit export+ # there is not overwritten. The torchao output is the sibling "<save_directory>-<suffix>", or the repo id+ # on a hub push.repo_id, work_tmp, model_restore = None, None, Nonework_tmp = tempfile.mkdtemp(prefix = "unsloth-torchao-")if push_to_hub:@@ -7800,8 +7382,8 @@ def _unsloth_save_torchao(exist_ok = True,)- # 1) Merge to 16bit at a staging dir (LoRA and base alike). The reload reads default- # weight filenames, so never write variant-named shards here.+ # Merge to 16bit at a staging dir, LoRA and base alike. The reload reads default weight filenames, so+ # never write variant-named shards here.merge_kwargs.pop("variant", None)print(f"Unsloth: Merging to 16bit before torchao {kind} quantization...")merge_args = dict(merge_kwargs)@@ -7818,23 +7400,21 @@ def _unsloth_save_torchao()unsloth_generic_save(**merge_args)- # 2) Detect VLM + reload class. A bare *ForConditionalGeneration also matches text seq2seq- # (T5/BART/Whisper), so key off vision_config / a vision-named architecture only.+ # Detect VLM and reload class: a bare *ForConditionalGeneration also matches text seq2seq, so key off+ # vision_config or a vision-named architecture only.is_vlm = Falseif hasattr(model, "config"):archs = getattr(model.config, "architectures", None) or []is_vlm = hasattr(model.config, "vision_config") or any(x.endswith("ForVisionText2Text") for x in archs)- # trust_remote_code must reflect the approved load decision - whether the in-memory model /- # tokenizer was itself loaded from custom code - not the staged config's auto_map, which an- # attacker can set on a built-in-loadable model to run unvetted code past the consent gate.+ # trust_remote_code must reflect the approved load decision, not the staged config's auto_map, which+ # an attacker can set on a built-in-loadable model to run unvetted code past the gate.model_trust = _loaded_via_remote_code(model)tok_trust = _loaded_via_remote_code(tokenizer)- # Reload with the class that matches the checkpoint: an image-text VLM class (with a- # fallback for older Transformers that lack AutoModelForImageTextToText); the model's own- # architecture class for encoder-decoder seq2seq (T5/BART/Whisper are not causal LMs, and- # AutoModelForCausalLM would fail to load them); otherwise causal-LM.+ # Reload with the class matching the checkpoint: an image-text VLM class (with a fallback for+ # Transformers lacking AutoModelForImageTextToText), the model's own class for encoder-decoder+ # seq2seq, otherwise causal-LM.if is_vlm:try:from transformers import AutoModelForImageTextToText as _reload_model@@ -7855,10 +7435,9 @@ def _unsloth_save_torchao(auto_model = AutoModelForCausalLMauto_processor = AutoProcessor if is_vlm else AutoTokenizer- # 3) Free the in-memory model's accelerator memory before reloading a fresh copy from- # disk, else it sits resident alongside the copy and OOMs a device that fit the- # model once. Covers CUDA and XPU (torchao runs on Intel GPUs too) plus multi-GPU- # dispatched shards, which a plain .to("cpu") cannot move.+ # Free the in-memory model's accelerator memory before reloading from disk, else it sits beside the+ # copy and OOMs a device that fit the model once. Covers CUDA, XPU and multi-GPU dispatched shards,+ # which a plain .to("cpu") cannot move._has_xpu = hasattr(torch, "xpu") and torch.xpu.is_available()model_restore = _offload_model_for_quantize_subprocess(model)for _ in range(3):@@ -7868,8 +7447,8 @@ def _unsloth_save_torchao(if _has_xpu:torch.xpu.empty_cache()- # 4) Reload the staged 16bit checkpoint with torchao applied. bfloat16 is required;- # device_map="auto" falls back to CPU, so this works on any hardware.+ # Reload the staged 16bit checkpoint with torchao applied: bfloat16 is required, and device_map="auto"+ # falls back to CPU, so this works on any hardware.print(f"Unsloth: Quantizing the merged model to torchao {kind}...")dtype_kw = {"torch_dtype": torch.bfloat16} if HAS_TORCH_DTYPE else {"dtype": torch.bfloat16}quantized_model = auto_model.from_pretrained(@@ -7889,7 +7468,6 @@ def _unsloth_save_torchao(if torch.cuda.is_available():torch.cuda.empty_cache()- # 5) Validate the artifact.cfg_path = os.path.join(out_dir, "config.json")cfg = {}if os.path.exists(cfg_path):@@ -7901,7 +7479,7 @@ def _unsloth_save_torchao(f"{cfg_path}")- # 6) Optional hub upload of the quantized artifact (the temp staging is cleaned in finally).+ # Optional hub upload of the quantized artifact; the temp staging is cleaned in finally.if push_to_hub:print(f"Unsloth: Uploading torchao {kind} checkpoint to '{repo_id}' ...")api.upload_folder(@@ -7931,8 +7509,7 @@ def _unsloth_save_torchao()return resultfinally:- # A raise pins the copy in the local and the live traceback, so free both or the- # restore below OOMs.+ # A raise pins the copy in the local and the live traceback, so free both or the restore below OOMs.quantized_model = Nonedel quantized_model_exc = sys.exc_info()[1]@@ -7991,7 +7568,7 @@ def unsloth_save_pretrained_torchao(has_qat_config = hasattr(self, "_torchao_config") and self._torchao_config is not Noneif torchao_config is not None:- # PTQ path: user provided a config, model must NOT have QAT config unless PEFT+ # PTQ path: the user provided a config, so the model must NOT have a QAT config unless PEFT.assert not has_qat_config, ("Unsloth: You passed `torchao_config` but this model was trained with `qat_scheme`. ""For QAT models, do not pass `torchao_config` - the quantization config is already "@@ -8006,7 +7583,7 @@ def unsloth_save_pretrained_torchao(token = token,)else:- # QAT path: no config provided, model must have QAT config+ # QAT path: no config provided, so the model must have a QAT config.assert has_qat_config, ("Unsloth: No `torchao_config` provided and model was not trained with `qat_scheme`. ""Either train with `qat_scheme` parameter, or provide a `torchao_config` for "@@ -8033,7 +7610,6 @@ def patch_saving_functions(model, vision = False):import typesfrom typing import Callable, Optional, Union, List- # And now re add our saving methods!if model.push_to_hub.__name__ == "unsloth_push_to_hub":original_push_to_hub = model.original_push_to_hubelse:@@ -8142,7 +7718,6 @@ def patch_saving_functions(model, vision = False):original_model = modelwhile True:- # Check if push_to_hub exists before accessing its __name__if (hasattr(original_model, "push_to_hub")and original_model.push_to_hub.__name__ != "unsloth_push_to_hub"@@ -8161,10 +7736,8 @@ def patch_saving_functions(model, vision = False):else:break- # Add saving methods to top level modelif not vision:if hasattr(model, "config"):- # Counteract tokenizersmodel.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model)model.save_pretrained_merged = types.MethodType(unsloth_generic_save_pretrained_merged, model@@ -8179,7 +7752,6 @@ def patch_saving_functions(model, vision = False):unsloth_convert_lora_to_ggml_and_save_locally, model)else:- # Vision only 1 optionmodel.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model)model.save_pretrained_merged = types.MethodType(unsloth_generic_save_pretrained_merged, modeldiff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex 1bc5fe30138..d073bf54bcb 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -58,7 +58,7 @@ IGNORED_TOKENIZER_CHECKING = frozenset(IGNORED_TOKENIZER_NAMES = [- # Qwen Coder did not train on tool calling. Math did!+ # Qwen Coder did not train on tool calling; Math did."unsloth/Qwen2.5-Coder-1.5B-Instruct","unsloth/Qwen2.5-Coder-7B-Instruct",]@@ -68,10 +68,8 @@ IGNORED_TOKENIZER_NAMES = frozenset()os.environ["UNSLOTH_IGNORED_TOKENIZER_NAMES"] = "\n".join(IGNORED_TOKENIZER_NAMES)-# Check environments-# A KAGGLE_* variable is not a Kaggle kernel - the Kaggle CLI reads-# KAGGLE_USERNAME / KAGGLE_KEY from the environment on ordinary machines, and-# redirecting their tokenizer cache to /tmp because of it was wrong.+# A KAGGLE_* variable is not a Kaggle kernel: the Kaggle CLI reads KAGGLE_USERNAME / KAGGLE_KEY on+# ordinary machines, and redirecting their tokenizer cache to /tmp because of it was wrong.from .disk_utils import (KAGGLE_TMP,is_colab_environment,@@ -90,13 +88,12 @@ def try_fix_tokenizer(tokenizer, prepend = True):tokenizer_string = converted_tokenizer.to_str()- # Llama does _apple. Sometimes this is wrong!!+ # Llama does _apple; sometimes this is wrong.prepend_text = '{"type":"Prepend","prepend":"▁"},'if not prepend and prepend_text in tokenizer_string:tokenizer_string = tokenizer_string.replace(prepend_text, "", 1)dir_names = dir(tokenizer)- # Get eos_token, bos_token etctoken_names = [x for x in dir_names if x.endswith("_token") and x.count("_") == 1]for token_name in token_names:@@ -107,7 +104,7 @@ def try_fix_tokenizer(tokenizer, prepend = True):if token_id is None:continue- # Locate the token's id mapping in the string+ # Locate the token's id mapping in the string.find_text = f'"id":{token_id},"content":"'find_pos = tokenizer_string.find(find_text)if find_pos == -1:@@ -118,13 +115,12 @@ def try_fix_tokenizer(tokenizer, prepend = True):continuebad_token = tokenizer_string[start:end]- # Check if token is the actual same one - if not, edit it+ # Check whether the token is actually the same one; if not, edit it and replace the vocab section.if bad_token != token:bad_text = f'{find_text}{bad_token}",'good_text = f'{find_text}{token}",'tokenizer_string = tokenizer_string.replace(bad_text, good_text, 1)- # And replace vocab sectionbad_text = f'"{bad_token}":{token_id},'good_text = f'"{token}":{token_id},'tokenizer_string = tokenizer_string.replace(bad_text, good_text, 1)@@ -162,13 +158,11 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_senexcept:FastTokenizer = PreTrainedTokenizerFast- # Get all arguments (bos_token, etc)docs = FastTokenizer.__doc__docs = docs[docs.find("Args:") :]args = re.findall(r"\n[\s]+([^\s]{1,}) \(", docs, flags = re.MULTILINE)args = [x for x in args if not x.endswith("_file")]- # Also some missing maybe!docs = PreTrainedTokenizerFast.__doc__docs = docs[docs.find("Args:") :]args2 = re.findall(r"\n[\s]+([^\s]{1,}) \(", docs, flags = re.MULTILINE)@@ -181,27 +175,24 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_senkwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = True)fast_tokenizer = FastTokenizer(**kwargs)- # Check if they're similar!sorted_slow_tokenizer = get_sorted_dict(slow_tokenizer.get_vocab())sorted_fast_tokenizer = get_sorted_dict(fast_tokenizer.get_vocab())check_vocab = sorted_slow_tokenizer == sorted_fast_tokenizercheck_special = slow_tokenizer.all_special_tokens == fast_tokenizer.all_special_tokens- # Failure so return slow_tokenizer+ # Failure, so return the slow_tokenizer.if not check_vocab or not check_special:return slow_tokenizer- # Now confirm if they matchif not assert_same_tokenization(slow_tokenizer, fast_tokenizer):- # Maybe remove prepending of __apple?+ # Maybe remove the prepending of __apple.kwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = False)fast_tokenizer = FastTokenizer(**kwargs)if not assert_same_tokenization(slow_tokenizer, fast_tokenizer):- # Failure :(return slow_tokenizer- # Also tokenizer.model is missing!+ # tokenizer.model is missing too.name = slow_tokenizer.name_or_path.replace("/", "_")if not os.path.exists(temporary_location):os.makedirs(temporary_location)@@ -209,7 +200,6 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_senslow_tokenizer.save_pretrained(new_location)fast_tokenizer.save_pretrained(new_location)- # Now load it!fast_tokenizer = AutoTokenizer.from_pretrained(new_location)if assert_same_tokenization(slow_tokenizer, fast_tokenizer):return fast_tokenizer@@ -266,7 +256,6 @@ llama_template = (def assert_same_tokenization(slow_tokenizer, fast_tokenizer):- # Get eos_token, bos_token etcif not hasattr(slow_tokenizer, "all_special_tokens"):return Truedir_names = dir(slow_tokenizer)@@ -286,7 +275,6 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):replacement_char = b"\xc3\xaf\xc2\xbf\xc2\xbd".decode("utf-8")all_special_tokens = [x for x in all_special_tokens if x != replacement_char]- # Check if chat template is enabled!check_chat_template1 = Truecheck_chat_template2 = Truecheck_chat_template3 = True@@ -330,7 +318,6 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):"""check_chat_template = check_chat_template1 and check_chat_template2 and check_chat_template3- # Try special tokenstry:string = ("\n".join(all_special_tokens)@@ -341,9 +328,8 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):return check_chat_template and check_special_tokensexcept:- # For eg see https://github.com/unslothai/unsloth/issues/292- # Sometimes tokenizer has weird tokens, causing a combined tokenization to fail.- # [TODO] We temporarily disable this for CodeLlama tokenizers+ # Sometimes a tokenizer has weird tokens, causing a combined tokenization to fail; temporarily+ # disabled for CodeLlama tokenizers (#292).if slow_tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING:return check_chat_templateelse:@@ -356,8 +342,8 @@ def fix_sentencepiece_tokenizer(token_mapping,temporary_location = "_unsloth_sentencepiece_temp",):- # From https://github.com/google/sentencepiece/issues/121- # We need to manually edit the sentencepiece tokenizer!+ # The sentencepiece tokenizer must be edited manually; see google/sentencepiece#121. Only works for+ # SentencePiece <= 3.20.3.try:from transformers.convert_slow_tokenizer import import_protobufsentencepiece_model_pb2 = import_protobuf()@@ -379,11 +365,10 @@ def fix_sentencepiece_tokenizer(if not os.path.exists(temporary_location):os.makedirs(temporary_location)- # Fresh per-call subdir so concurrent/repeated calls can't clobber each other's- # tokenizer.model or leak stale files, without deleting anything the caller owns.+ # Fresh per-call subdir so concurrent or repeated calls cannot clobber each other's tokenizer.model+ # or leak stale files, without deleting anything the caller owns.temporary_location = tempfile.mkdtemp(prefix = "tokenizer_", dir = temporary_location)- # First save the old tokenizerold_tokenizer.save_pretrained(temporary_location)# Only sentencepiece tokenizers write tokenizer.model, so check after the save.@@ -395,10 +380,9 @@ def fix_sentencepiece_tokenizer(tokenizer_file = sentencepiece_model_pb2.ModelProto()tokenizer_file.ParseFromString(open(f"{temporary_location}/tokenizer.model", "rb").read())- # Now save the new tokenizernew_tokenizer.save_pretrained(temporary_location)- # Now correct the old tokenizer's .model file+ # Correct the old tokenizer's .model file.for old_token, new_token in token_mapping.items():ids = old_tokenizer([old_token], add_special_tokens = False).input_idsids = ids[0]@@ -409,7 +393,7 @@ def fix_sentencepiece_tokenizer()continueids = ids[0]- # [TODO] Hack for Starling - try except+ # Hack for Starling: try/except.try:tokenizer_piece = tokenizer_file.pieces[ids]except:@@ -417,11 +401,9 @@ def fix_sentencepiece_tokenizer(assert tokenizer_piece.piece == old_tokentokenizer_piece.piece = new_token- # And now write itwith open(f"{temporary_location}/tokenizer.model", "wb") as file:file.write(tokenizer_file.SerializeToString())- # And load it!from transformers import AutoTokenizertokenizer = AutoTokenizer.from_pretrained(@@ -429,8 +411,8 @@ def fix_sentencepiece_tokenizer(eos_token = new_tokenizer.eos_token,pad_token = new_tokenizer.pad_token,)- # vocab_file points here, so the dir must outlive the tokenizer (a later- # save_pretrained copies the patched tokenizer.model from it); reclaim it on GC.+ # vocab_file points here, so the dir must outlive the tokenizer (a later save_pretrained copies the+ # patched tokenizer.model from it); reclaim it on GC.weakref.finalize(tokenizer, shutil.rmtree, temporary_location, ignore_errors = True)return tokenizer@@ -468,16 +450,14 @@ def fix_sentencepiece_gguf(saved_location):UNUSED = 5BYTE = 6- # Load tokenizer.modeltokenizer_file = sentencepiece_model_pb2.ModelProto()if not os.path.isfile(f"{saved_location}/tokenizer.model"):returntokenizer_file.ParseFromString(open(f"{saved_location}/tokenizer.model", "rb").read())sentence_piece_size = len(tokenizer_file.pieces)- # Build a set of token IDs that are marked as special in tokenizer.json.- # These tokens should use CONTROL type in the sentencepiece model so that- # llama.cpp writes them as CONTROL (type=3) in the GGUF token_type array.+ # Build a set of token IDs marked special in tokenizer.json: these must use CONTROL type in the+ # sentencepiece model so llama.cpp writes them as CONTROL (type=3) in the GGUF token_type array.special_token_ids = set()if os.path.isfile(f"{saved_location}/tokenizer.json"):with open(f"{saved_location}/tokenizer.json", "r", encoding = "utf-8") as f:@@ -487,8 +467,8 @@ def fix_sentencepiece_gguf(saved_location):if entry.get("special", False) and isinstance(token_id, int):special_token_ids.add(token_id)- # Fix existing sentencepiece tokens that are marked as special in tokenizer.json- # but have the wrong type (NORMAL instead of CONTROL) in the sentencepiece model.+ # Fix existing sentencepiece tokens marked special in tokenizer.json but typed NORMAL instead of+ # CONTROL in the sentencepiece model.patched = 0for token_id in special_token_ids:if 0 <= token_id < sentence_piece_size:@@ -502,7 +482,6 @@ def fix_sentencepiece_gguf(saved_location):f"from NORMAL to CONTROL type so llama.cpp / GGUF chat inference works correctly.")- # Load added_tokens_jsonif not os.path.isfile(f"{saved_location}/added_tokens.json"):if patched > 0:with open(f"{saved_location}/tokenizer.model", "wb") as file:@@ -519,7 +498,7 @@ def fix_sentencepiece_gguf(saved_location):added_tokens_json = dict(sorted(added_tokens_json.items(), key = lambda item: item[1]))new_size = sentence_piece_size + len(added_tokens_json)- # Confirm added_tokens_json is correct+ # Confirm added_tokens_json is correct.added_tokens_ids = np.array(list(added_tokens_json.values()))_real_added_tokens_ids = added_tokens_idsif len(added_tokens_ids) < 2:@@ -537,7 +516,7 @@ def fix_sentencepiece_gguf(saved_location):file.write(tokenizer_file.SerializeToString())return- # Edit sentence piece tokens with added_tokens_json+ # Edit sentencepiece tokens with added_tokens_json.logger.warning(f"Unsloth: Extending {saved_location}/tokenizer.model with added_tokens.json.\n"f"Originally tokenizer.model is of size ({sentence_piece_size}).\n"@@ -548,8 +527,7 @@ def fix_sentencepiece_gguf(saved_location):added_token_id = added_tokens_json[added_token_str]new_token.piece = added_token_str.encode("utf-8")new_token.score = -1000.0- # Use CONTROL type for tokens marked as special in tokenizer.json,- # otherwise fall back to USER_DEFINED.+ # Use CONTROL type for tokens marked special in tokenizer.json, otherwise fall back to USER_DEFINED.if added_token_id in special_token_ids:new_token.type = SentencePieceTokenTypes.CONTROLelse:@@ -560,9 +538,6 @@ def fix_sentencepiece_gguf(saved_location):with open(f"{saved_location}/tokenizer.model", "wb") as file:file.write(tokenizer_file.SerializeToString())- # Add padding tokens- # actual_vocab_size = model.config.vocab_size- # padding = actual_vocab_size - len(tokenizer_file.pieces)return@@ -579,17 +554,15 @@ def _load_correct_tokenizer(if IS_COLAB_ENVIRONMENT:cache_dir = cache_direlif IS_KAGGLE_ENVIRONMENT:- # /tmp of Kaggle seems has a 80GB limit!- # Let's utilize them+ # Kaggle's /tmp seems to have an 80GB limit, so use it.cache_dir = os.path.join(KAGGLE_TMP, cache_dir)elif cache_dir == "huggingface_tokenizers_cache":- # This default name is Colab/Kaggle-only; elsewhere use the HF default cache.+ # This default name is Colab/Kaggle-only; elsewhere use the HF default cache, and keep a+ # caller-supplied cache_dir so the tokenizer loads from the prefetch-warmed dir.cache_dir = None- # else: keep a caller-supplied cache_dir so the tokenizer loads from the prefetch-warmed dir instead- # of risking an in-process Hub/Xet transfer.- # Try loading the slow tokenizer. If it fails, then try Fast only- # Mainly to solve Deepseek models with no tokenizer.model file+ # Try the slow tokenizer first and fall back to fast only, mainly to solve Deepseek models with no+ # tokenizer.model file.slow_tokenizer = Nonetry:slow_tokenizer = AutoTokenizer.from_pretrained(@@ -598,7 +571,7 @@ def _load_correct_tokenizer(padding_side = padding_side,token = token,trust_remote_code = trust_remote_code,- # Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373+ # use_fast = False alone is not enough; see twitter.com/danielhanchen/status/1789659394302718373use_fast = False,legacy = False,from_slow = True,@@ -607,11 +580,7 @@ def _load_correct_tokenizer()except:slow_tokenizer = None- # print(- # f"Unsloth: {tokenizer_name} has no tokenizer.model file.\n"\- # "Just informing you about this - this is not a critical error."- # )- # Unsure why this occurs!+ # Reason unknown.if type(slow_tokenizer) is bool:slow_tokenizer = None@@ -639,7 +608,7 @@ def _load_correct_tokenizer(if hasattr(fast_tokenizer, "add_eos_token") and hasattr(slow_tokenizer, "add_eos_token"):fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token- # Confirm if slow and fast are equivalent!+ # Confirm whether slow and fast are equivalent.if assert_same_tokenization(slow_tokenizer, fast_tokenizer):return fast_tokenizerelse:@@ -694,10 +663,10 @@ def load_correct_tokenizer(if fix_tokenizer:_fix_pad_token(tokenizer)- ### 1. Fixup tokenizer's chat_template+ # Fixup the tokenizer's chat_template.old_chat_template = getattr(tokenizer, "chat_template", None)- # Ignore mistral type models since they don't have an add_generation_prompt+ # Ignore mistral type models, which have no add_generation_prompt, and check Llama-2 old style models too.if any(s in str(getattr(tokenizer, "name_or_path", "")).lower() for s in ["mistral", "qwen3guard"]):@@ -722,16 +691,15 @@ def load_correct_tokenizer(passtokenizer.chat_template = chat_template- # Saving restores sentencepiece assets from the repo name alone, which carries no- # branch, so stamp it for the save path. Imported late to break a circular import.+ # Saving restores sentencepiece assets from the repo name alone, which carries no branch, so stamp+ # it for the save path. Imported late to break a circular import.from .models.loader_utils import _mark_loaded_revision_mark_loaded_revision(tokenizer, revision)return tokenizer-# All four Jinja whitespace-control variants of endfor/endif:-# {% endfor %} {%- endfor %} {% endfor -%} {%- endfor -%}+# All four Jinja whitespace-control variants of endfor/endif._RE_ENDFOR = re.compile(r"\{%(-?)\s*endfor\s*(-?)%\}")_RE_ENDIF = re.compile(r"\{%(-?)\s*endif\s*(-?)%\}")_RE_JINJA_COMMENT = re.compile(r"\{#.*?#\}", flags = re.DOTALL)@@ -832,8 +800,8 @@ def _has_add_generation_prompt_block(chat_template):return "if add_generation_prompt" in chat_template and "%}" in chat_templatefor if_node in ast.find_all(jinja2.nodes.If):test = if_node.test- # Reject negated gates: `{% if not add_generation_prompt %}` fires- # when agp=False, so it's not a generation block even if it emits.+ # Reject negated gates: `{% if not add_generation_prompt %}` fires when agp=False, so it is not a+ # generation block even if it emits.if isinstance(test, jinja2.nodes.Not):continue# find_all skips the test root, so check bare Name tests explicitly.@@ -850,9 +818,8 @@ def _has_add_generation_prompt_block(chat_template):return False-# Sentinels for _derive_assistant_prefix_by_render. Diverge at char 0 so-# commonprefix can't absorb them; long random tail makes collision with real-# template literals negligible (see T18).+# Sentinels for _derive_assistant_prefix_by_render: they diverge at char 0 so commonprefix cannot+# absorb them, and the long random tail makes collision with real template literals negligible._RENDER_DIFF_SENTINEL_A = "AAAA_0123456789_UNSLOTH_RENDER_DIFF_SENTINEL"_RENDER_DIFF_SENTINEL_B = "BBBB_0123456789_UNSLOTH_RENDER_DIFF_SENTINEL"_RENDER_DIFF_SENTINEL_C = "CCCC_0123456789_UNSLOTH_RENDER_DIFF_SENTINEL"@@ -889,9 +856,8 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):sent_b_msgs = base_msgs + [{"role": "assistant", "content": _RENDER_DIFF_SENTINEL_B}]sent_c_msgs = base_msgs + [{"role": "user", "content": _RENDER_DIFF_SENTINEL_C}]- # Strip trailing whitespace/comments after the last endfor/endif: they- # appear after the message loop and would break Guard A. The splice in- # `_fix_chat_template` drops them too.+ # Strip trailing whitespace/comments after the last endfor/endif: they appear after the message loop+ # and would break Guard A, and the splice in _fix_chat_template drops them too.probe_template = chat_templateend = _find_end_position(chat_template)if end is not None:@@ -899,8 +865,8 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):if _RE_JINJA_COMMENT.sub("", after).strip() == "":probe_template = chat_template[: end["end"]]- # Sandboxed: probe renders at load time, before user calls- # apply_chat_template. SandboxedEnvironment blocks attribute-chain exploits.+ # Sandboxed: probe renders run at load time, before the user calls apply_chat_template, and+ # SandboxedEnvironment blocks attribute-chain exploits.try:env = SandboxedEnvironment(autoescape = False,@@ -913,9 +879,8 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):except Exception:return None- # Best-effort: alternation-enforcing templates (e.g. Gemma's- # raise_exception) fail on [user, user]; that's a positive signal- # for Guard C, not a probe failure.+ # Best-effort: alternation-enforcing templates (Gemma's raise_exception) fail on [user, user], which+ # is a positive signal for Guard C rather than a probe failure.out_user_c = Nonetry:out_user_c = tmpl.render(messages = sent_c_msgs, add_generation_prompt = False)@@ -940,8 +905,7 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):):return None- # Guard C: reject if a [user, user] render also emits the same prefix- # (role-insensitive template, e.g. `{% set greeting='Hi' %}...`).+ # Guard C: reject if a [user, user] render also emits the same prefix, i.e. a role-insensitive template.if out_user_c is not None and out_user_c.startswith(out_base):tail_c = out_user_c[len(out_base) :]if tail_c.startswith(prefix) and prefix != "":@@ -954,10 +918,8 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):def _fix_chat_template(chat_template, is_sharegpt = False):- # Fast path: already has an {% if add_generation_prompt %} block, nothing- # to do. This catches cases the old string-based check would miss (e.g.- # templates that use {%- if add_generation_prompt -%} with both-side dash,- # or that sneak the block into a nested If/For).+ # Fast path: an {% if add_generation_prompt %} block already exists. This catches cases the old+ # string-based check would miss, such as both-side dashes or a block nested inside an If/For.if _has_add_generation_prompt_block(chat_template):return chat_template@@ -970,9 +932,8 @@ def _fix_chat_template(chat_template, is_sharegpt = False):dash_r = "-" if end["dash_right"] else ""open_tag = lambda body: "{%" + dash_l + " " + body + " " + dash_r + "%}"- # Case 1 (pre-existing base case): template ends with a single trailing- # {{ expr }} that is the generation prefix. Wrap it in an- # {% if add_generation_prompt %} ... {% endif %}.+ # Case 1: the template ends with a single trailing {{ expr }} that is the generation prefix, so wrap+ # it in an {% if add_generation_prompt %} block.if ("{%" + dash_l + " if" not in after_endforand "{%" + dash_l + " set " not in after_endfor@@ -984,18 +945,15 @@ def _fix_chat_template(chat_template, is_sharegpt = False):wrapped = open_tag("if add_generation_prompt") + after_endfor + open_tag("endif")return chat_template[: end["end"]] + wrapped- # Case 2 (GH#4150): template ends at {% endfor %} with only whitespace- # or comments left. Inject an {% if add_generation_prompt %} block with- # the assistant prefix derived by render-diff. The top-level-For gate- # keeps us out of outer-If wrappers (e.g. Qwen3-Guard).+ # Case 2 (#4150): the template ends at {% endfor %} with only whitespace or comments left, so inject+ # an {% if add_generation_prompt %} block with the assistant prefix derived by render-diff.if _RE_JINJA_COMMENT.sub("", after_endfor).strip() == "" and _template_ends_with_toplevel_for(chat_template):- # No redundant "agp not in scrubbed" check: the fast path already- # confirmed no *positive* block, and a mere reference (header- # guard) should still get repaired.+ # No redundant "agp not in scrubbed" check: the fast path already confirmed no POSITIVE block, and a+ # mere reference (header guard) should still get repaired.assistant_prefix = _derive_assistant_prefix_by_render(chat_template, is_sharegpt)- # Dual-probe: dict/list callers don't know the shape up front.+ # Dual-probe: dict/list callers do not know the shape up front.if assistant_prefix is None and not is_sharegpt:assistant_prefix = _derive_assistant_prefix_by_render(chat_template, is_sharegpt = True)if assistant_prefix is None:@@ -1104,7 +1062,7 @@ def _validate_patched_template(tokenizer, patched_template, is_sharegpt):try:tokenizer.chat_template = patched_templateexcept Exception:- return False # read-only tokenizer, skip validation+ return Falsetry:yes = tokenizer.apply_chat_template(msgs,@@ -1123,9 +1081,8 @@ def _validate_patched_template(tokenizer, patched_template, is_sharegpt):tokenizer.chat_template = originalexcept Exception:pass # best-effort restore- # Contract after a successful repair: the two renders differ, and the- # "yes" render is a strict extension of the "no" render (we only- # appended content inside the new add_generation_prompt block).+ # Contract after a successful repair: the two renders differ, and the "yes" render is a strict+ # extension of the "no" render, since content was only appended in the new block.return yes != no and yes.startswith(no)@@ -1135,10 +1092,8 @@ def _repair_string_template(tokenizer, chat_template, is_sharegpt):candidate = _fix_chat_template(chat_template, is_sharegpt = is_sharegpt)if not _has_add_generation_prompt_block(candidate):return None- # Validate with the caller's is_sharegpt first. If that fails, the- # dual-probe in _fix_chat_template may have fallen back to the other- # schema internally -- try validating with the opposite schema before- # giving up.+ # Validate with the caller's is_sharegpt first; if that fails, the dual-probe in+ # _fix_chat_template may have fallen back to the other schema, so try the opposite one.if _validate_patched_template(tokenizer, candidate, is_sharegpt):return candidateif _validate_patched_template(tokenizer, candidate, not is_sharegpt):@@ -1199,13 +1154,14 @@ def _fix_chat_template_for_tokenizer(tokenizer, chat_template):return chat_templateif no != yes:- # Template already responds to the flag; leave as is.+ # The template already responds to the flag; leave it as is.return chat_template- # no == yes: template ignores add_generation_prompt. Try to repair.+ # no == yes: the template ignores add_generation_prompt, so try to repair. It may have the block+ # while the output does not change, the "wasn't provided correctly" case.if _has_add_generation_prompt_block(chat_template):- # Template has the block but it does not change output. This is the- # "wasn't provided correctly" case from the pre-warn code path.+ # Template has the block but it does not change output. This is the "wasn't provided correctly" case+ # from the pre-warn code path.strict = _is_strict_chat_template_mode()msg = _format_chat_template_message(name,@@ -1311,8 +1267,8 @@ def fix_chat_template(tokenizer):if chat_template is None:return None- # Multi-variant dict (e.g. Hermes-3 {default, tool_use}): route each- # variant through the full repair contract via _VariantTokenizerProxy.+ # Multi-variant dict (Hermes-3 {default, tool_use}): route each variant through the full repair+ # contract via _VariantTokenizerProxy.if isinstance(chat_template, dict):fixed = {}for key, tmpl in chat_template.items():@@ -1323,7 +1279,7 @@ def fix_chat_template(tokenizer):fixed[key] = _fix_chat_template_for_tokenizer(proxy, tmpl)return fixed- # List-of-dicts form (older HF multi-template style).+ # List-of-dicts form, the older HF multi-template style.if isinstance(chat_template, list):fixed = []for item in chat_template:@@ -1356,13 +1312,10 @@ def check_tokenizer(_reload = True,cache_dir = None,):- # Checks tokenizer for out of bounds ids.- # Mainly a fix for https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha- # where <sep> had token id=32002.- # See https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha/discussions/25- # Seems like the Fast tokenizer in Rust breaks things!+ # Check the tokenizer for out of bounds ids, mainly for berkeley-nest/Starling-LM-7B-alpha where+ # <sep> had token id=32002 (its discussion 25); the Rust fast tokenizer seems to break things.- # We ignore some of them!+ # Some of them are ignored.if tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING:return tokenizer@@ -1377,7 +1330,6 @@ def check_tokenizer(bad_indices = list(added_tokens_fast.keys())[j:]bad_tokens = list(added_tokens_fast.values())[j:]if not _reload:- # Try removing the tokenadded_tokens = [str(x) for x in tokenizer.added_tokens_decoder.values()]special_tokens = tokenizer.special_tokens_mapimport itertools@@ -1412,14 +1364,13 @@ def check_tokenizer(try_removal.append(token)try_mapper.append(name_token)- # Recheck!can_be_removed = len(try_removal) == len(bad_tokens)if can_be_removed:remove_generic = Truecan_be_removed1 = bad_tokensif can_be_removed:- # Yes it can be fixed!+ # Yes, it can be fixed.for j, bad_token in enumerate(can_be_removed1):remove_id = tokenizer._added_tokens_encoder[bad_token]del tokenizer._added_tokens_decoder[remove_id]@@ -1429,7 +1380,7 @@ def check_tokenizer(# Remove sep token for examplesetattr(tokenizer, try_mapper[j], None)setattr(tokenizer, try_mapper[j] + "_id", None)- # Confirm 1 more time!+ # Confirm one more time.if max(tokenizer.added_tokens_decoder.keys()) < max_embedding_size:logger.warning_once(f"Unsloth loaded a broken tokenizer `{model_name}`, but managed to repair it!\n"@@ -1438,19 +1389,20 @@ def check_tokenizer()return convert_to_fast_tokenizer(tokenizer)- # :( Failure+ # Failure.raise RuntimeError(f"Unsloth tried to load `{model_name}`, but cannot succeed.\n"f"Tokens {bad_tokens} with ids {bad_indices} exceeds the max vocab size of {max_embedding_size}.\n"f"Fix your tokenizer since it'll perform out of bounds memory accesses.")- # Reuse a caller-supplied cache_dir (warmed cache) for the repair reload; else the- # Colab/Kaggle sentinel (HF default elsewhere), as load_correct_tokenizer does.+ # Reuse a caller-supplied cache_dir (the warmed cache) for the repair reload; else the Colab/Kaggle+ # sentinel, as load_correct_tokenizer does.reload_cache_dir = cache_dirif reload_cache_dir is None and (IS_COLAB_ENVIRONMENT or IS_KAGGLE_ENVIRONMENT):reload_cache_dir = "huggingface_tokenizers_cache"+ # Sometimes the slow tokenizer does not work, e.g. Deepseek, but it can also fix things.# Sometimes slow tokenizer does not work like Deepseektry:# Try slow tokenizer which can fix things!@@ -1459,7 +1411,7 @@ def check_tokenizer(model_max_length = model_max_length,padding_side = padding_side,token = token,- # Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373+ # use_fast = False alone is not enough; see twitter.com/danielhanchen/status/1789659394302718373use_fast = False,legacy = False,from_slow = True,@@ -1477,8 +1429,7 @@ def check_tokenizer()breakexcept:- # Tokenizer has out of bounds issues and we can't- # load the slow tokenizer version :(+ # The tokenizer has out of bounds issues and the slow version cannot be loaded.logger.warning_once("Unsloth: Tokenizer is most likely buggy, and Unsloth failed to repair it.\n""It will still work, but beware of out of bounds memory accesses.\n"@@ -1584,18 +1535,14 @@ def patch_sft_trainer_tokenizer():except:returnall_imports = dir(trl.trainer.sft_trainer)- # Make typing names available to the exec'd source bodies. TRL >= 1.x- # type-hints _prepare_dataset / _prepare_non_packed_dataloader with- # `Union[...]` and friends; without these imports in the exec namespace- # those become NameErrors at exec time. Mirrors the pattern used in- # unsloth/models/_utils.py:patch_linear_scaling.+ # Make typing names available to the exec'd source bodies: TRL >= 1.x type-hints _prepare_dataset+ # with Union[...] and friends, which become NameErrors at exec time without these imports.from typing import Union, Optional, List, Any, Callable, Tuple, Dict, Iterator # noqa: F401for (function_name,replacer,) in (- # ("_prepare_non_packed_dataloader", "def tokenize(element):",),("_prepare_non_packed_dataloader",None,@@ -1604,7 +1551,6 @@ def patch_sft_trainer_tokenizer():"_prepare_dataset",None,),- # ("_prepare_packed_dataloader", "if dataset_text_field is not None",),):if not hasattr(sft_trainer, function_name):continue@@ -1645,7 +1591,7 @@ def patch_sft_trainer_tokenizer():check_text = check_text.rstrip() + "\n"if replacer is None:- # .*? matches first match. .+? matches final match.+ # .*? matches the first match, .+? the final one.replacer = re.findall(f"def {function_name}" + r"\(.*?\).*?\:\n",function,@@ -1673,7 +1619,6 @@ def patch_sft_trainer_tokenizer():globals(),)- # Patch train with fix_untrained_tokensfor path_to_trainer in ("sft_trainer.SFTTrainer","dpo_trainer.DPOTrainer",@@ -1720,7 +1665,7 @@ def patch_sft_trainer_tokenizer():"fix_zero_training_loss(self.model, tokenizer, self.train_dataset)\n\n")- # Warn on gradient accumulation steps if it's used+ # Also DPO weirdly tokenizes non numeric columns? Delete them!check_text += ("\n""try:\n"@@ -1736,7 +1681,7 @@ def patch_sft_trainer_tokenizer():"\n\n")- # Add NEFTune since it doesn't seem to work?? We need to manually inject it+ # NEFTune does not seem to work, so inject it manually.check_text += ("\n""if hasattr(self, 'neftune_hook_handle'):\n"@@ -1750,7 +1695,7 @@ def patch_sft_trainer_tokenizer():"\n")- # Also DPO weirdly tokenizes non numeric columns? Delete them!+ # DPO weirdly tokenizes non numeric columns, so delete them.check_text += ("\n""if hasattr(self.train_dataset, 'column_names'):\n"@@ -1776,5 +1721,4 @@ def patch_sft_trainer_tokenizer():)-# Finally patch TRL tokenizer things -> moved to RL-# patch_sft_trainer_tokenizer()+# Finally patch TRL tokenizer things, moved to RL.diff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 824c3e6b46e..7b47d027c16 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -26,7 +26,7 @@ import trlimport inspectfrom trl import SFTTrainer-# why: bypass partially-initialised unsloth ns during _gpu_init load+# Bypass the partially-initialised unsloth namespace during the _gpu_init load.from .models._utils import is_bfloat16_supportedfrom unsloth.utils import (configure_padding_free,@@ -87,9 +87,7 @@ class UnslothVisionDataCollator(_UnslothVisionDataCollatorBase):if formatting_func is None:return super().__call__(examples)- # why: base __call__ would reapply formatting_func; applied above. A- # per-call shallow view shares every other attribute by reference, so a- # concurrent caller can never observe formatting_func blanked on self.+ # The base __call__ would reapply formatting_func, which was applied above.view = copy.copy(self)view.formatting_func = Nonereturn super(UnslothVisionDataCollator, view).__call__(examples)@@ -103,10 +101,9 @@ PADDING_FREE_BLOCKLIST = {"gemma2", # - gemma2: Uses slow_attention_softcapping which has torch.compile issues"gpt_oss", # - gpt_oss: Uses Flex Attention which doesn't handle padding_free correctly}-# Hybrid linear-attention / state-space models (Qwen3.5, Qwen3-Next, ...) carry a-# recurrent gated-delta state plus a causal conv1d that leak across sequence-# boundaries once packing flattens the batch. Detected structurally by-# _is_hybrid_linear_attention_model, not by model name.+# Hybrid linear-attention / state-space models (Qwen3.5, Qwen3-Next) carry a recurrent gated-delta+# state plus a causal conv1d that leak across sequence boundaries once packing flattens the batch.+# Detected structurally by _is_hybrid_linear_attention_model, not by model name.def _should_pack(config) -> bool:@@ -196,7 +193,7 @@ def _cap_is_enforceable_without_padding_free(config, train, evals) -> bool:except Exception:return True # nothing to check against; do not invent a failurepacking = bool(getattr(config, "packing", False))- # TRL's own default: `eval_packing = None` means "whatever `packing` is".+ # TRL's own default: eval_packing = None means "whatever packing is".eval_packing = getattr(config, "eval_packing", None)eval_packing = packing if eval_packing is None else bool(eval_packing)if not packing and not pretokenized_within_cap(train, cap):@@ -345,9 +342,9 @@ def _resolve_string_model_config(model_name, config_arg):from transformers import AutoConfiginit_kwargs = getattr(config_arg, "model_init_kwargs", None) or {}- # why: forward auth + cache args too. Dropping token/use_auth_token made a- # private hybrid fail to load (resolve as None) -> treated as non-hybrid ->- # packing enabled without the shim even though TRL later loads it with the token.+ # Forward auth and cache args too: dropping token/use_auth_token made a private hybrid fail to+ # load (resolving as None), so it was treated as non-hybrid and packing was enabled without the+ # shim.forward = {key: init_kwargs[key]for key in (@@ -361,10 +358,9 @@ def _resolve_string_model_config(model_name, config_arg):)if key in init_kwargs}- # why: TRL merges top-level args.trust_remote_code into the load via setdefault- # before create_model_from_path, so honor it here (model_init_kwargs wins), else- # a remote-code hybrid with SFTConfig(trust_remote_code=True) resolves as None- # and skips the guard.+ # TRL merges top-level args.trust_remote_code into the load via setdefault before+ # create_model_from_path, so honor it here (model_init_kwargs wins), else a remote-code hybrid+ # resolves as None and skips the guard.top_level_trust_remote_code = getattr(config_arg, "trust_remote_code", None)if top_level_trust_remote_code is not None:forward.setdefault("trust_remote_code", top_level_trust_remote_code)@@ -388,7 +384,7 @@ def _chunked_loss_bypasses_forward(config) -> bool:return getattr(config, "loss_type", None) in (None, "chunked_nll")-# Unsloth gradient accumulation fix:+# Unsloth gradient accumulation fix.from transformers import __version__ as transformers_version, ProcessorMixinif Version(transformers_version) > Version("4.45.2"):@@ -501,13 +497,11 @@ def _create_unsloth_optimizer(class UnslothTrainer(SFTTrainer):def create_optimizer(self):- # --- Q-GaLore optimizer ---q_galore_config = getattr(self.args, "q_galore_config", None)if q_galore_config is not None and self.optimizer is None:embedding_lr = getattr(self.args, "embedding_learning_rate", None)return self._create_q_galore_optimizer(q_galore_config, embedding_lr)- # --- Embedding-LR optimizer ---embedding_learning_rate = getattr(self.args, "embedding_learning_rate", None)if embedding_learning_rate is None:return super().create_optimizer()@@ -558,16 +552,16 @@ class UnslothTrainer(SFTTrainer):# --- Split embedding params with custom LR (Fix #2) ---if embedding_lr is not None:- # Fast param->name lookup (O(N) instead of O(N*M))+ # Fast param -> name lookup, O(N) instead of O(N*M).param_to_name = {id(p): name for name, p in self.model.named_parameters()}new_groups = []for group in param_groups:if "rank" in group:- # GaLore group: keep as-is (no embeddings here)+ # GaLore group: keep as-is, since no embeddings are here.new_groups.append(group)continue- # Non-GaLore group: split out embedding params+ # Non-GaLore group: split out embedding params.embed_params = []other_params = []for p in group["params"]:@@ -608,8 +602,8 @@ class UnslothTrainer(SFTTrainer):group_size = config.weight_group_size,stochastic = config.stochastic_round,)- # Pre-hooks dequantize INT8 weights to float before each forward,- # letting the optimizer free float weight memory between steps.+ # Pre-hooks dequantize INT8 weights to float before each forward, letting the optimizer free float+ # weight memory between steps.install_weight_quant_hooks(self.model)n_galore = sum(len(g["params"]) for g in param_groups if "rank" in g)@@ -623,8 +617,7 @@ class UnslothTrainer(SFTTrainer):return self.optimizer-# From `trl>=0.13.0`, they changed how to pass several params to the trainer-# We need to patch to make the transition smooth+# From trl >= 0.13.0 several params are passed to the trainer differently; patch to make the transition smooth.def _resolve_trainer_params(trainer_class, init_fn):"""Resolve the real named parameters for a trainer __init__.@@ -641,7 +634,7 @@ def _resolve_trainer_params(trainer_class, init_fn):if named:return set(params.keys())- # Thin wrapper detected - walk MRO for real signature+ # Thin wrapper detected: walk the MRO for the real signature.for cls in trainer_class.__mro__[1:]:if cls is object:continue@@ -699,15 +692,13 @@ def _ensure_warnings_issued(model):if existing is None:model.warnings_issued = {}else:- # Preserve a non-dict value rather than discard it; trl only ever- # writes one boolean key.+ # Preserve a non-dict value rather than discard it; trl only ever writes one boolean key.try:model.warnings_issued = dict(existing)except Exception:model.warnings_issued = {}except Exception:- # A model refusing the assignment is trl's to report, not ours to turn- # into a different traceback.+ # A model refusing the assignment is trl's to report, not ours to turn into a different traceback.pass@@ -793,7 +784,7 @@ def _backwards_compatible_trainer(trainer_class, config_class):@wraps(original_init)def new_init(self, *args, **kwargs):- # tokenizer is now processing_class+ # tokenizer is now processing_class.trainer_params = _resolve_trainer_params(trainer_class, original_init)if "processing_class" in trainer_params and "tokenizer" in kwargs:@@ -807,7 +798,7 @@ def _backwards_compatible_trainer(trainer_class, config_class):trainer_params.discard("self")trainer_params.discard("args")- # Fields that should be passed to Config init+ # Fields that should be passed to Config init.config_fields = {field.name: field for field in dataclasses.fields(config_class) if field.init}@@ -818,14 +809,14 @@ def _backwards_compatible_trainer(trainer_class, config_class):if hasattr(training_args, name)}- # Params in Config but not in TrainingArguments+ # Params in Config but not in TrainingArguments.from transformers import TrainingArgumentsmoved_params = set(inspect.signature(config_class).parameters.keys()) - set(inspect.signature(TrainingArguments).parameters.keys())- # Separate kwargs into trainer kwargs and config kwargs+ # Separate kwargs into trainer kwargs and config kwargs.trainer_kwargs = {}additional_config_kwargs = {}unknown_kwargs = {}@@ -849,22 +840,20 @@ def _backwards_compatible_trainer(trainer_class, config_class):config_dict.update(additional_config_kwargs)- # Only build the config if the previous init wasn't TrainingArguments:- # reinitialising it would re-trigger mutually-exclusive param checks.- # See https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_config.py#L499-L502+ # Only build the config if the previous init was not TrainingArguments: reinitialising it would re-+ # trigger the mutually-exclusive param checks (trl grpo_config.py#L499-L502).if not isinstance(training_args, TrainingArguments):config = config_class(**config_dict)else:- # Every trl config subclasses TrainingArguments, so this is the- # branch real calls take and config_dict was going nowhere. Set- # the moved values on the caller's config rather than rebuild it.+ # Every trl config subclasses TrainingArguments, so this is the branch real calls take and+ # config_dict was going nowhere: set the moved values on the caller's config rather than rebuild it.config = training_argsfor key, value in additional_config_kwargs.items():# `migrated` keys are already known to be accepted here.if key in config_fields or key in moved_params or key in migrated:setattr(config, key, value)- # Reconstruct kwargs for Trainer+ # Reconstruct kwargs for Trainer.kwargs = trainer_kwargskwargs["args"] = config_ensure_warnings_issued(args[0] if args else kwargs.get("model"))@@ -912,11 +901,8 @@ def _patch_sft_trainer_auto_packing(trl_module):else model)is_hybrid = _is_hybrid_linear_attention_model(hybrid_target)- # Hybrid models corrupt packed batches unless the gated-delta conv + scan- # reset at sequence boundaries. Enable the experimental varlen shim (flag +- # kernels) so packing stays correct, else keep them blocked. A string model- # (patched only after init) and TRL's chunked-loss forward bypass both leave- # the shim off, so hybrid packing falls back to the padded path.+ # Hybrid models corrupt packed batches unless the gated-delta conv and scan reset at sequence+ # boundaries, so enable the experimental varlen shim (flag plus kernels) or keep them blocked.if (is_hybridand not isinstance(model, str)@@ -953,9 +939,7 @@ def _patch_sft_trainer_auto_packing(trl_module):or is_unsupported_modelor is_encoder_decoderor (is_hybrid and not hybrid_varlen_active)- or (- os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"- ) # Disable padding free on forced logits+ or (os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"))requested_pack = bool(getattr(config_arg, "packing", False))if blocked:@@ -979,8 +963,8 @@ def _patch_sft_trainer_auto_packing(trl_module):elif is_unsupported_model:reason = f"unsupported model type(s): {', '.join(model_types)}"elif data_collator is None:- # compute_metrics, preprocess_logits_for_metrics, for_inference() and the- # user can all set it, so name the flag and not a setter.+ # compute_metrics, preprocess_logits_for_metrics, for_inference() and the user can all set it, so+ # name the flag and not a setter.reason = "UNSLOTH_RETURN_LOGITS=1"logger.warning(f"Unsloth: packing=True ignored ({reason}).")@@ -990,7 +974,7 @@ def _patch_sft_trainer_auto_packing(trl_module):packing_active = Truelogger.info("Unsloth: Sample packing enabled for SFTTrainer instance.")- # Resolve padding_free: None (default) = auto-enable unless env-disabled or packing+ # Resolve padding_free: None (default) means auto-enable unless env-disabled or packing.auto_padding_free_active = Falsepadding_free_requested = getattr(config_arg, "padding_free", None) is Trueif not blocked:@@ -1047,11 +1031,9 @@ def _patch_sft_trainer_auto_packing(trl_module):)print(message)- # get_peft_model installs a pre-train forward detector for plain LoRA/vision models,- # but only RL trainers run the reset via prepare_for_training_mode. Wire it into the- # SFT train() path too, else a grad-enabled probe before train() leaves the poisoned- # Dynamo cache in place and the detector hook installed on every training forward.- # (For UnslothSFTTrainer the later prepare_for_training_mode assignment supersedes this.)+ # get_peft_model installs a pre-train forward detector for plain LoRA/vision models, but only RL+ # trainers run the reset via prepare_for_training_mode, so wire it into the SFT train() path too,+ # else a grad-enabled probe before train() leaves the poisoned Dynamo cache in place.if not getattr(self, "_unsloth_train_reset_wrapped", False):try:from unsloth.models._utils import _unsloth_reset_stray_compile_cache@@ -1090,10 +1072,8 @@ def _patch_trl_trainer():trl_configs = set(x[: -len("Config")] for x in trl_classes if x.endswith("Config"))trl_classes = list(trl_trainers & trl_configs)- # Auto-packing wraps first so it lands INSIDE the backwards-compatible one: a- # moved `packing` kwarg must reach the config before packing is decided, else- # the block is undone right after. Guarded so a failure here still leaves the- # pre-0.13 compatibility wrappers installed.+ # Auto-packing wraps first so it lands INSIDE the backwards-compatible wrapper: a moved `packing`+ # kwarg must reach the config before packing is decided, else the block is undone right after.try:_patch_sft_trainer_auto_packing(trl)except Exception as exc:diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex cd3bbd678ad..4607092f181 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -42,13 +42,13 @@ HAS_XFORMERS = xformers is not Nonedef _xformers_runs_on_device() -> bool:"""One tiny attention forward; True iff the xformers kernel actually runs here."""try:- # Pre-Ampere GPUs (sm < 80: Turing/Volta) have no bfloat16 attention kernel- # but run xformers fine in float16, so pick the dtype the device supports.+ # Pre-Ampere GPUs (sm < 80: Turing/Volta) have no bfloat16 attention kernel but run xformers fine in+ # float16, so pick the dtype the device supports.dtype = torch.bfloat16 if SUPPORTS_BFLOAT16 else torch.float16q = torch.zeros((1, 8, 1, 64), device = "cuda", dtype = dtype)attn_bias = xformers.attn_bias.BlockDiagonalCausalMask.from_seqlens([8])xformers_attention(q, q, q, attn_bias = attn_bias)- # Launches are async; synchronize so a deferred kernel failure fails the probe here.+ # Launches are async, so synchronize or a deferred kernel failure escapes this probe.torch.cuda.synchronize()return Trueexcept Exception:@@ -56,30 +56,29 @@ def _xformers_runs_on_device() -> bool:def _xformers_disabled_for_capability(capability, probe = _xformers_runs_on_device) -> bool:- # At sm_120 (RTX 50-series) xformers' cutlass op is capability-rejected (caps at- # sm_90) and its flash-2 op runs only if the build ships an sm_120 kernel, so run- # one real forward to decide. Below sm_120 xformers always works; skip the probe.+ # At sm_120 (RTX 50-series) xformers' cutlass op is capability-rejected (it caps at sm_90) and its+ # flash-2 op runs only if the build ships an sm_120 kernel, so run one real forward to decide;+ # below sm_120 xformers always works.if capability[0] < 12:return Falsereturn not probe()-# FlashAttention always wins in select_attention_backend and nothing downgrades-# flash -> xformers, so when it's installed xformers is never selected: skip the probe.+# FlashAttention always wins in select_attention_backend and nothing downgrades flash to xformers,+# so when it is installed xformers is never selected: skip the probe.if HAS_XFORMERS and not HAS_FLASH_ATTENTION and torch.cuda.is_available():if _xformers_disabled_for_capability(torch.cuda.get_device_capability()):HAS_XFORMERS = False-# On sm_100+ (B200, sm_120) xformers' fp32-capable cutlass op is capability-rejected and-# only its fp16/bf16 flash-2 op runs, so fp32 Q/K/V (DoRA, #1013) must be downcast there;-# below sm_100 cutlass handles fp32 natively. Read once from device 0, like the probe gate.+# On sm_100+ (B200, sm_120) xformers' fp32-capable cutlass op is capability-rejected and only its+# fp16/bf16 flash-2 op runs, so fp32 Q/K/V (DoRA, #1013) must be downcast there; below sm_100+# cutlass handles fp32 natively. Read once from device 0._XFORMERS_FP32_UNSUPPORTED = (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10)SDPA_HAS_GQA = "enable_gqa" in (scaled_dot_product_attention.__doc__ or "")-# PrefixGrouper kernel, resolved once when the env gate is on so PG-off users never load-# torch flex_attention.+# PrefixGrouper kernel, resolved once when the env gate is on so PG-off users never load torch flex_attention._flex_shared_prefix_attention = Noneif os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER", "1").lower() not in ("0", "false", "no", "off"):try:@@ -98,32 +97,11 @@ SDPA = "sdpa"XFORMERS_BLOCK_DIAG_CLS = xformers.attn_bias.BlockDiagonalCausalMask if HAS_XFORMERS else None+# flash-attn 2 varlen backward int32 overflow guard: the varlen BACKWARD kernel allocates+# dq_accum = zeros(total_q + 128 * n_seqs, n_heads, round_up(head_dim, 32)) and indexes it with+# int32, so at 2**31 elements the kernel faults with an illegal memory access and poisons the CUDA+# context. Forward-only never allocates dq_accum, so the guard requires a backward to be possible.# ---- flash-attn 2 varlen backward int32 overflow guard --------------------------------------# The varlen BACKWARD kernel (used by flash-attn 2 directly and by the flash-2 op xFormers-# dispatches a BlockDiagonal* bias to) allocates-#-# dq_accum = zeros(total_q + 128 * n_seqs, n_heads, round_up(head_dim, 32), dtype = fp32)-#-# and indexes it with int32, so once that element count reaches 2**31 the kernel faults with-# "CUDA error: an illegal memory access was encountered". That poisons the CUDA context, so the-# run dies hours in, far from the cause.-#-# Measured on a B200 (bf16, one flattened row, forward + backward) by bisecting the document-# count in a packed row: the last working count and the count at which-# (total_q + 128 * n_seqs) * n_heads * round_up(head_dim, 32) crosses 2**31 agree to within one-# document across every shape tried:-#-# heads head_dim doc_len predicted last OK-# 16 128 1 8128 8129-# 16 96 1 10837 10838-# 16 64 1 16257 16257-# 8 128 1 16257 >= 16257, fails by 16400-# 16 128 2 8065 8060 OK, 8200 fails-# 16 128 4 7944 7944-# 4 128 1 32518 no failure up to 20000 (below the bound)-#-# Forward-only never allocates dq_accum and never faults (20000 documents ran clean), so the-# guard requires a backward to be possible. Plain SDPA is unaffected._INT32_ELEMENTS = 2**31_VARLEN_INT32_GUARD_DISABLED = os.environ.get("UNSLOTH_DISABLE_VARLEN_INT32_GUARD", "0"@@ -223,8 +201,8 @@ class AttentionContext:attention_mask: Optional[Tensor]causal_mask: Optional[Any]sliding_window: Optional[int] = None- # PrefixGrouper: non-None routes Q/K/V through the FlexAttention shared-prefix kernel;- # None leaves every existing construction/behavior unchanged.+ # PrefixGrouper: non-None routes Q/K/V through the FlexAttention shared-prefix kernel; None leaves+ # every existing construction and behavior unchanged.prefix_seg_info: Optional[Any] = None@@ -268,10 +246,9 @@ def resolve_prefix_seg_info(kwargs, past_key_value, attention_mask):return seg-# One dense window mask per (device, shape, window), reused across layers. Every layer of a+# One dense window mask per (device, shape, window), reused across layers: every layer of a# Mistral-style model asks for the identical mask, and at 32K that tensor is 1 GiB with two more-# alive while it is built, so rebuilding it 32 times is how this SDPA fallback OOMs a run that-# xFormers or flash would have carried. Same single-entry-per-device shape as _SDPA_MASK_CACHE.+# alive while it is built, so rebuilding it 32 times OOMs the run._WINDOW_MASK_CACHE: dict = {}@@ -281,8 +258,8 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -entry = _WINDOW_MASK_CACHE.get(device)if entry is not None and entry["params"] == params:return entry["mask"]- # Drop the outgoing mask first. It is dead either way, and holding it while the replacement- # and its temporaries are allocated would make a shape change peak a whole mask higher.+ # Drop the outgoing mask first: it is dead either way, and holding it while the replacement and its+ # temporaries are allocated would make a shape change peak a whole mask higher._WINDOW_MASK_CACHE.pop(device, None)entry = Noneq_pos = torch.arange(k_len - q_len, k_len, device = device)@@ -307,14 +284,12 @@ def run_attention(and SDPA handle packing via a block-diagonal mask."""- # PrefixGrouper shared-prefix attention (GRPO dedup). Q/K/V here are [bsz, H, T, D];- # the kernel takes/returns [1, T, H, D], matching the other backends. The field is- # only set when the env gate is on and grouping succeeded; None keeps every backend- # byte-identical.+ # PrefixGrouper shared-prefix attention (GRPO dedup): Q/K/V here are [bsz, H, T, D] while the kernel+ # takes and returns [1, T, H, D], matching the other backends.if context.prefix_seg_info is not None:flex_shared_prefix_attention = _flex_shared_prefix_attentionif flex_shared_prefix_attention is None:- # gate flipped on after import (or one-time load failed): resolve lazily.+ # Gate flipped on after import, or a one-time load failed: resolve lazily.from ..utils.prefix_grouper_kernel import flex_shared_prefix_attentionscale = None@@ -333,8 +308,8 @@ def run_attention(if backend == FLASH_VARLEN and context.seq_info is None:backend = FLASH_DENSE if HAS_FLASH_ATTENTION else SDPA- # [TODO] Flash/xFormers don't support arbitrary attn masks; with a padding- # mask present (e.g. left-padded generation), fall back to SDPA.+ # Flash/xFormers do not support arbitrary attn masks, so with a padding mask present (left-padded+ # generation) fall back to SDPA.if context.attention_mask is not None and backend in (FLASH_DENSE,FLASH_VARLEN,@@ -342,15 +317,13 @@ def run_attention():backend = SDPA- # Both varlen-capable backends land in the same flash-attn 2 backward kernel, so guard both- # before the int32 overflow aborts the process. Integer arithmetic only, no device sync.+ # Both varlen-capable backends land in the same flash-attn 2 backward kernel, so guard both before+ # the int32 overflow aborts the process. Integer arithmetic only, no device sync.if backend in (FLASH_VARLEN, XFORMERS) and not _VARLEN_INT32_GUARD_DISABLED:- # Both terms are needed. Q/K/V: a frozen hidden state feeding trainable LoRA q/k/v- # still yields a Q that requires grad, which context.requires_grad alone would miss.- # context.requires_grad: gradient checkpointing runs its FIRST forward under no_grad- # and recomputes with grad on, so keying only on the tensors would pick a different- # backend per pass and the backward would see activations the loss never came from.- # The hidden state keeps requires_grad = True across both passes; inference has it False.+ # Both terms are needed. Q/K/V: a frozen hidden state feeding trainable LoRA q/k/v still yields a+ # Q that requires grad. context.requires_grad: gradient checkpointing runs its FIRST forward under+ # no_grad and recomputes with grad on, so keying only on the tensors would pick a different+ # backend per pass.will_backward = context.requires_grad or (torch.is_grad_enabled() and (Q.requires_grad or K.requires_grad or V.requires_grad))@@ -360,9 +333,8 @@ def run_attention(n_seqs = seq_info[0].numel() if seq_info is not None else context.bsztotal_q = context.bsz * context.q_lenif _varlen_backward_overflows_int32(n_seqs, total_q, context.n_heads, context.head_dim):- # SDPA cannot apply logit softcapping, so rerouting a softcapped model (Gemma 2)- # would keep it training on wrong logits and gradients -- worse than the fault- # this guard avoids. Stop and name the ways out instead.+ # SDPA cannot apply logit softcapping, so rerouting a softcapped model (Gemma 2) would keep it+ # training on wrong logits and gradients, worse than the fault this guard avoids.softcap = _configured_softcap(config)if softcap:raise RuntimeError(@@ -399,15 +371,13 @@ def run_attention(kv_seq_len = context.kv_seq_lenrequires_grad = context.requires_gradsliding_window = context.sliding_window- # A non-positive window means "no local attention", not "a window of nothing": a config- # spelling it 0 would otherwise put the mask's lower bound above its causal upper bound- # and hide every position from every other.+ # A non-positive window means "no local attention", not "a window of nothing": a config spelling+ # it 0 would put the mask's lower bound above its causal upper bound.if sliding_window is not None and sliding_window <= 0:sliding_window = None- # DoRA promotes q/k/v_proj outputs to fp32, which FlashAttention rejects (and so does- # the xformers flash-2 op on sm_100+, see _XFORMERS_FP32_UNSUPPORTED), so downcast any- # fp32 Q/K/V to a supported dtype (#1013).+ # DoRA promotes q/k/v_proj outputs to fp32, which FlashAttention rejects (as does the xformers+ # flash-2 op on sm_100+), so downcast any fp32 Q/K/V to a supported dtype (#1013).if (backend in (FLASH_DENSE, FLASH_VARLEN)or (backend == XFORMERS and _XFORMERS_FP32_UNSUPPORTED)@@ -530,7 +500,8 @@ def run_attention(local_mask = local_mask.to(device = Q.device)if local_mask.dim() == 2:- # key padding keep mask: (bsz, k_len), 1/True = real token+ # Key padding keep mask (bsz, k_len), where 1/True is a real token; the tokenizer attention_mask is+ # typically int 0/1.if local_mask.dtype == torch.bool:key_keep = local_maskelse:@@ -545,7 +516,7 @@ def run_attention(if sliding_window is not None:causal_keep &= k_pos[None, :] >= (q_pos[:, None] - (sliding_window - 1))- # (bsz, 1, q_len, k_len) boolean keep mask+ # (bsz, 1, q_len, k_len) boolean keep mask.local_mask = causal_keep[None, None, :, :] & key_keep[:, None, None, :]elif local_mask.dim() == 3:@@ -559,16 +530,14 @@ def run_attention(else:raise ValueError(f"Unsupported SDPA attention_mask rank: {local_mask.dim()}")- # Avoid NaNs from fully-masked rows (common with left padding).+ # Avoid NaNs from fully-masked rows, common with left padding.if local_mask.dtype == torch.bool:no_allowed = ~local_mask.any(dim = -1, keepdim = True) # (bsz,1,q_len,1)local_mask = local_mask | no_allowedif local_mask is None and sliding_window is not None and k_len_local > sliding_window:- # SDPA's is_causal is FULL causal; it has no window. With no padding mask to- # hang the window off, a model whose config declares one attended its whole- # history whenever neither the xformers bias nor flash's window_size was the- # thing running.+ # SDPA's is_causal is FULL causal and has no window, so with no padding mask to hang the window+ # off, a model whose config declares one attended its whole history.local_mask = _windowed_causal_mask(q_len_local, k_len_local, sliding_window, Q.device)@@ -587,8 +556,8 @@ def run_attention(and local_mask.dim() >= 3and local_mask.shape[0] > 1):- # Batched masked inference has shown row-coupled drift with SDPA GQA.- # Fall back to explicit KV expansion for deterministic row-wise behavior.+ # Batched masked inference has shown row-coupled drift with SDPA GQA, so fall back to explicit KV+ # expansion for deterministic row-wise behavior.use_sdpa_gqa = Falseif use_sdpa_gqa:diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex 3262589a6d8..1e460d7791f 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -74,10 +74,9 @@ def move_xformers_attention_bias(attn_bias: Any, device: torch.device):):return attn_bias- # Move the device-bearing metadata instead of the top-level mask. Older- # xFormers versions demote causal masks in their inherited ``to`` method.- # Copies also keep later model shards from rewriting masks retained for- # backward by earlier shards.+ # Move the device-bearing metadata instead of the top-level mask: older xFormers versions demote+ # causal masks in their inherited `to` method, and copies also keep later model shards from+ # rewriting masks retained for backward.moved_bias = copy.copy(attn_bias)moved_seqinfos = {}for name, seqinfo in seqinfos:@@ -214,21 +213,19 @@ def enable_sample_packing(lengths = example.get(sequence_lengths_key)if isinstance(lengths, Iterable):seq_lengths.extend(int(length) for length in lengths)- # Fallback: infer lengths from tokenized inputs when metadata is absent+ # Fallback: infer lengths from tokenized inputs when metadata is absent.if not seq_lengths:for example in examples:ids = example.get("input_ids")if isinstance(ids, Iterable):seq_lengths.append(len(ids))if seq_lengths:- # Boundary labels are NOT masked here. unsloth_zoo's- # _unsloth_get_batch_samples counts num_items_in_batch off this batch and- # discounts the N-1 boundary targets itself, idempotently: it zeroes those- # slots rather than subtracting a constant, so the count is unaffected by- # upstream masking (TRL >= 0.24's labels[position_ids == 0] = -100,- # completion-only masking, assistant_masks). Masking here would be harmless- # to the count; labels are left alone because the guard that needs these- # positions runs in the forward, off packed_seq_lengths.+ # Boundary labels are NOT masked here: unsloth_zoo's _unsloth_get_batch_samples counts+ # num_items_in_batch off this batch and discounts the N-1 boundary targets itself, idempotently,+ # zeroing those slots rather than subtracting a constant, so the count is unaffected by upstream+ # masking (TRL >= 0.24's labels[position_ids == 0] = -100, completion-only masking,+ # assistant_masks). Labels are left alone because the guard that needs these positions runs in the+ # forward, off packed_seq_lengths.batch["packed_seq_lengths"] = torch.tensor(seq_lengths, dtype = torch.int32)if "attention_mask" in batch:batch.pop("attention_mask")@@ -271,9 +268,8 @@ def enable_padding_free_metadata(model, trainer):batch = original_torch_call(examples)if seq_lengths:- # Labels left alone for the same reason as enable_sample_packing:- # num_items_in_batch is counted off this batch, and the zoo's discount of the- # boundary targets is idempotent, so masked slots do not change the count.+ # Labels left alone for the same reason as enable_sample_packing: num_items_in_batch is counted off+ # this batch and the zoo's discount of the boundary targets is idempotent.batch["packed_seq_lengths"] = torch.tensor(seq_lengths,dtype = torch.int32,@@ -284,17 +280,12 @@ def enable_padding_free_metadata(model, trainer):collator._unsloth_padding_free_lengths_wrapped = True-# --- Experimental: correct packing / padding-free for hybrid linear-attention ----# Qwen3.5 / Qwen3-Next mix a gated-delta recurrence with a causal conv1d. Packing-# flattens the batch, and both ops leak state across sequence boundaries unless we-# pass seq_idx (conv) and cu_seqlens (scan). Only the accelerated kernels accept-# these, so we fail closed on the pure-torch fallbacks. Gated behind an env flag.-#-# Overrides only the per-module prefill kernels (causal_conv1d_fn /-# chunk_gated_delta_rule), leaving decode untouched so generation is unaffected.-# Recompute-safe under gradient checkpointing; never fires for cached forwards.-# Feature-detect (never version-detect), fail closed, idempotent, one deduped-# diagnostic when it declines to activate.+# Experimental correct packing / padding-free for hybrid linear-attention: Qwen3.5 / Qwen3-Next+# mix a gated-delta recurrence with a causal conv1d, and packing flattens the batch so both leak+# state across sequence boundaries unless seq_idx (conv) and cu_seqlens (scan) are passed. Only+# the accelerated kernels accept these, so it fails closed on the pure-torch fallbacks, behind an+# env flag, and overrides only the per-module prefill kernels (causal_conv1d_fn /+# chunk_gated_delta_rule), leaving decode untouched._HYBRID_PACKING_ENV_VAR = "UNSLOTH_EXPERIMENTAL_HYBRID_PACKING"_HYBRID_LOGGER = logging.getLogger("unsloth.hybrid_packing")_HYBRID_WARNED: set = set()@@ -447,7 +438,7 @@ def _hybrid_varlen_metadata(kwargs):if total is None:return Nonepsl = kwargs.get("packed_seq_lengths")- if psl is not None and getattr(psl, "numel", lambda: 1)() > 0: # skip empty (no max())+ if psl is not None and getattr(psl, "numel", lambda: 1)() > 0:info = get_packed_info_from_kwargs(kwargs, device)if info is not None:_, cu_seqlens, _ = info@@ -520,9 +511,8 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:module._unsloth_varlen = Nonemodule._unsloth_varlen_wrapped = True- # Refresh the boundary stash on the outermost forward (once per step, outside- # gradient-checkpoint recompute, so it stays valid for recomputed inner- # forwards). Read from both positional and keyword args via the bound signature.+ # Refresh the boundary stash on the outermost forward, once per step and outside gradient-checkpoint+ # recompute so it stays valid for recomputed inner forwards.if not getattr(model, "_unsloth_varlen_forward_wrapped", False):forward_orig = model.forwardtry:@@ -550,12 +540,8 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:module._unsloth_varlen_conv_hit = Falsemodule._unsloth_varlen_scan_hit = Falseout = forward_orig(*args, **kwargs)- # Runtime dispatch handshake: on the first packed forward, confirm BOTH- # boundary kernels ran for EVERY module. seq_idx (conv) and cu_seqlens- # (scan) are both load-bearing, so a partial/absent dispatch (a future- # version no longer routing through self.<kernel>) leaves cross-sequence- # contamination. The batch is already flattened with no padded recovery,- # so abort before loss/backward rather than train on corrupted data.+ # Runtime dispatch handshake: on the first packed forward, confirm BOTH boundary kernels ran for+ # EVERY module.if first_pack:model._unsloth_varlen_handshake_done = Truemissing = [diff --git a/unsloth/utils/prefix_grouper.py b/unsloth/utils/prefix_grouper.pyindex 4e6ff9672ce..ad89d907821 100644--- a/unsloth/utils/prefix_grouper.py+++ b/unsloth/utils/prefix_grouper.py@@ -57,14 +57,12 @@ import torchfrom .prefix_grouper_kernel import build_seg_info_multigroup, PrefixSegInfo-# ----------------------------------------------------------------------------# Env helpers# ---------------------------------------------------------------------------def env_on(name: str, default: str = "0") -> bool:return os.environ.get(name, default).lower() not in ("0", "false", "no", "off")-# One-time env reads; the helpers stay callable since unsloth_zoo imports and calls them.+# One-time env reads; the helpers stay callable, since unsloth_zoo imports and calls them._ENABLED = env_on("UNSLOTH_GRPO_SEQ_PACKING", "1") and env_on("UNSLOTH_GRPO_PREFIX_GROUPER", "1")_VERIFY_ON = env_on("UNSLOTH_GRPO_PREFIX_GROUPER_VERIFY", "1")_TOKR_THRESHOLD = float(os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOKR", "1.3"))@@ -88,8 +86,8 @@ def tol_ok() -> float:return _TOL_OK-# diff >= TOL_KILL = broken mask/isolation -> structure permanently unsafe; between-# tol_ok and TOL_KILL -> fall back for this shape but keep trying others.+# A diff at or above TOL_KILL means a broken mask/isolation, so the structure is permanently unsafe;+# between tol_ok and TOL_KILL, fall back for this shape but keep trying others.TOL_KILL = 1.5@@ -100,7 +98,7 @@ class GroupLayout:flat_ids: torch.Tensor # [1, T] (T == seg.T)position_ids: torch.Tensor # [1, T]prefix_seg_info: PrefixSegInfo- # per completion target token, aligned 1:1:+ # Per completion target token, aligned 1:1.tgt_rows: torch.Tensor # [N] original row indextgt_cols: torch.Tensor # [N] original padded column in that rowtgt_pred: torch.Tensor # [N] flat predicting index (into the T stream)@@ -124,8 +122,8 @@ class GroupLayout:) -> torch.Tensor:"""hidden: [1, T, Hdim] (pre-lm_head hidden states, UNSLOTH_RETURN_HIDDEN_STATES=1).Returns [total_rows, W] float32, byte-compatible with the packed path result."""- # In a sharded model hidden may live on the lm-head device; move the small index- # maps to hidden.device before indexing.+ # In a sharded model hidden may live on the lm-head device, so move the small index maps to+ # hidden.device before indexing.device = hidden.devicepred_h = hidden[0, self.tgt_pred.to(device), :].unsqueeze(0) # [1, N, Hdim]tgt_ids = self.flat_ids[0, self.tgt_flat].to(device).unsqueeze(0) # [1, N]@@ -179,7 +177,7 @@ def _build_groups(ids_cpu, real_cols_cpu, cstart_cpu, num_generations, total_rowcomp_cols_per_row.append(c_cols)if any(len(p) == 0 for p in prompt_toks_per_row):return None- # require BYTE-IDENTICAL prompts across the group (shared-prefix precondition).+ # Require BYTE-IDENTICAL prompts across the group, the shared-prefix precondition.P = len(prompt_toks_per_row[0])if any(len(prompt_toks_per_row[k]) != P for k in range(1, G)):return None@@ -236,12 +234,12 @@ def build_group_layout(device = input_ids.devicetotal_rows, L = input_ids.shapekeep = input_ids != pad_id- # completion start column per row (matches create_completion_attention_mask / _pk_cstart).+ # Completion start column per row, matching create_completion_attention_mask / _pk_cstart.cstart = ((L - logits_to_keep) - left_pad_tokens_per_prompt).to(torch.long)cstart_cpu = cstart.tolist()ids_cpu = input_ids.tolist()- # per-row real (non-pad) columns. GRPO rows are one contiguous real run, so derive- # [first, first+n) on GPU; the O(B*L) scan is only a non-contiguous fallback.+ # Per-row real (non-pad) columns: GRPO rows are one contiguous real run, so derive [first, first+n)+ # on GPU; the O(B*L) scan is only a non-contiguous fallback.n_real = keep.sum(dim = 1)first = torch.argmax(keep.to(torch.int8), dim = 1)ar = torch.arange(L, device = device)@@ -258,7 +256,7 @@ def build_group_layout(if groups is None:return None- # sliding-window guard: a group's PG span is P + max(R); fall back if it exceeds the window.+ # Sliding-window guard: a group's PG span is P + max(R), so fall back if it exceeds the window.if max_segment_cap is not None:for gm in groups:if gm["P"] + max(gm["R_list"]) > max_segment_cap:@@ -286,12 +284,12 @@ def build_group_layout(r0 = gm["prefix_row"]prefix_cols = gm["prefix_cols"] # ORIGINAL real prompt columns (len P) of row0plast = meta["prefix_last_index"] # base + P - 1- # gather the shared prefix once, from row0.+ # Gather the shared prefix once, from row0.flat_src_rows.extend([r0] * P)flat_src_cols.extend(prefix_cols)pos_list.extend(range(P))- # suffixes: every suffix token is a completion-region target (scattered like the- # packed path; completion_mask hides prompt-tail positions).+ # Suffixes: every suffix token is a completion-region target, scattered like the packed path, with+ # completion_mask hiding prompt-tail positions.for i, r in enumerate(rows):cols = gm["suf_cols"][i]r_i = len(cols)@@ -300,7 +298,7 @@ def build_group_layout(flat_src_cols.extend(cols)pos_list.extend(range(P, P + r_i))for j in range(r_i):- # pos 0 is predicted from the prefix's last token; j>=1 from the previous suffix token.+ # Position 0 is predicted from the prefix's last token; j >= 1 from the previous suffix token.pred = plast if j == 0 else (s + j - 1)tgt_rows.append(r)tgt_cols.append(cols[j]) # ORIGINAL padded column in row r@@ -317,9 +315,8 @@ def build_group_layout(max_left_pad = int(left_pad_tokens_per_prompt.max().item()) if total_rows else 0W = logits_to_keep + max_left_pad- # self-verify cache key: the mask/index-map/scatter logic is structural, so key on- # (num_groups, group_sizes), not exact lengths -- GRPO lengths change every step and- # keying on T would re-verify forever ("verify once, then trust", like the packed path).+ # Self-verify cache key: the mask/index-map/scatter logic is structural, so key on (num_groups,+ # group_sizes), not exact lengths.grp_sizes = tuple(sorted(len(gm["R_list"]) for gm in groups))sig = (len(groups), grp_sizes)diff --git a/unsloth/utils/prefix_grouper_kernel.py b/unsloth/utils/prefix_grouper_kernel.pyindex 9a9719b0157..34457a6d74d 100644--- a/unsloth/utils/prefix_grouper_kernel.py+++ b/unsloth/utils/prefix_grouper_kernel.py@@ -60,23 +60,22 @@ from torch.nn.attention.flex_attention import (flex_attention,)-# GRPO feeds many distinct segment lengths; at dynamo's default recompile_limit (8) the-# compiled kernel silently reuses a mismatched specialisation (wrong results). Raise it.+# GRPO feeds many distinct segment lengths, and at dynamo's default recompile_limit (8) the compiled+# kernel silently reuses a mismatched specialisation, giving wrong results.torch._dynamo.config.recompile_limit = max(getattr(torch._dynamo.config, "recompile_limit", 8), 256)torch._dynamo.config.accumulated_recompile_limit = max(getattr(torch._dynamo.config, "accumulated_recompile_limit", 256), 2048)-# Compiled kernels: torch.compile fuses the sparse mask into one kernel. dynamic=True is-# required: T changes almost every GRPO batch and dynamic=False recompiles per T (~14s-# each). T is still padded to a multiple of 128 (_pad_len) for the backward kernel.+# Compiled kernels: torch.compile fuses the sparse mask into one kernel, and dynamic=True is+# required, since T changes almost every GRPO batch and dynamic=False recompiles per T (~14s each)._flex_attention_compiled = torch.compile(flex_attention, dynamic = True)_create_block_mask_compiled = torch.compile(create_block_mask, dynamic = True)-# Flash block sizes by Q dtype (env-overridable). The two disjoint key runs (prefix +-# own-suffix) stress online-softmax accumulation: fp32 needs 32/32 for a ~1e-6 floor;-# bf16 passes parity at 128/64 and is ~5x faster (128/128 OOMs Triton on B200).+# Flash block sizes by Q dtype (env-overridable): the two disjoint key runs stress online-softmax+# accumulation, so fp32 needs 32/32 for a ~1e-6 floor, while bf16 passes parity at 128/64 and is+# ~5x faster (128/128 OOMs Triton on B200)._FP32_BLOCK_M = int(os.environ.get("PG_FLEX_BLOCK_M", "32"))_FP32_BLOCK_N = int(os.environ.get("PG_FLEX_BLOCK_N", "32"))_BF16_BLOCK_M = int(os.environ.get("PG_FLEX_BF16_BLOCK_M", "128"))@@ -93,9 +92,9 @@ def _kernel_options_for_dtype(dtype):# Backward-compat constant (fp32 default)._FLEX_KERNEL_OPTIONS = {"BLOCK_M": _FP32_BLOCK_M, "BLOCK_N": _FP32_BLOCK_N}-# The compiled backward trips an Inductor assertion when T is not a multiple of 128, so-# pad the flat sequence. Pad tokens form a group that attends to / is attended by nothing-# (all-masked rows return 0, not NaN) and are sliced off the output.+# The compiled backward trips an Inductor assertion when T is not a multiple of 128, so pad the+# flat sequence: pad tokens attend to and are attended by nothing (all-masked rows return 0) and+# are sliced off the output._PAD_MULTIPLE = 128_PAD_GROUP = -99 # sentinel group id / suffix id for pad tokens@@ -105,8 +104,6 @@ def _pad_len(T: int) -> int:# ----------------------------------------------------------------------------# Segment metadata-# ---------------------------------------------------------------------------@dataclass@@ -213,7 +210,7 @@ def build_seg_info_multigroup(suffix_counter = 0sig_parts = []for gid, (P, R_list) in enumerate(group_specs):- # prefix+ # Prefix.group_of_list.append(torch.full((P,), gid, dtype = torch.long, device = device))is_prefix_list.append(torch.ones(P, dtype = torch.bool, device = device))suffix_of_list.append(torch.full((P,), -1, dtype = torch.long, device = device))@@ -256,9 +253,10 @@ def build_seg_info_multigroup(return seg, group_meta+# Block-mask builder and cache, keyed on (signature, device): the mask depends only on the per-token+# labels and T, so it is reused across layers and steps.+# ----------------------------------------------------------------------------# Block-mask builder + cache, keyed on (signature, device): the mask depends only on the-# per-token labels and T, so it is reused across layers and steps._BLOCK_MASK_CACHE: Dict[Tuple, BlockMask] = {}@@ -309,12 +307,9 @@ def get_block_mask(return bm# Move labels to the consumer (Q) device: with a sharded model the seg tensors live on- # input_ids.device and would index cross-device. Copies once per (signature, device).- # These copies must also run with inference mode DISABLED (same reason as the mask build):- # when this entry is first built under the no-grad old/ref forward's inference_mode and- # device != seg.device, a .to(device) copy would be an inference tensor that mask_mod- # captures, which then cannot be saved for backward when the grad training forward reuses- # the cached mask.+ # input_ids.device and would index cross-device. These copies must also run with inference mode+ # DISABLED, like the mask build: built under the no-grad forward's inference_mode, a .to(device)+ # copy would be an inference tensor that mask_mod captures and could not be saved for backward.builder = _create_block_mask_compiled if compile_mask else create_block_maskwith torch.inference_mode(False):mask_mod = _make_mask_mod(@@ -364,7 +359,7 @@ def _run_flex(q, k, v, block_mask, enable_gqa, scale, compiled, T, T_pad):kernel_options = _kernel_options_for_dtype(qp.dtype),)else:- # eager path (fp64 parity): dense scores, no kernel_options.+ # Eager path (fp64 parity): dense scores, no kernel_options.out = flex_attention(qp,kp,@@ -377,8 +372,6 @@ def _run_flex(q, k, v, block_mask, enable_gqa, scale, compiled, T, T_pad):# ----------------------------------------------------------------------------# The kernel entry point-# ---------------------------------------------------------------------------def flex_shared_prefix_attention(@@ -422,7 +415,7 @@ def flex_shared_prefix_attention(block_mask = get_block_mask(prefix_seg_info, device, compile_mask = compiled)out = _run_flex(q, k, v, block_mask, enable_gqa, scale, compiled, T, T_pad)- # back to [1, T, n_heads, D]+ # Back to [1, T, n_heads, D].return out.transpose(1, 2).contiguous()