Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Trim comments in studio/backend core services and the unsloth package (#10116)

Daniel Han115339ba0c07
diff --git a/studio/backend/_platform_compat.py b/studio/backend/_platform_compat.py
index 013667d5cae..95f6cacce34 100644
--- a/studio/backend/_platform_compat.py
+++ b/studio/backend/_platform_compat.py
@@ -39,12 +39,12 @@ def _seed_sys_version_cache() -> None:
cleaned = m.group(0) + cleaned[p:]
if cleaned == raw:
- return # Nothing to fix
+ return
try:
result = platform._sys_version(cleaned)
except ValueError:
- return # Still unparsable; don't make things worse
+ return
# Seed the cache so future calls with the raw string skip parsing
cache = getattr(platform, "_sys_version_cache", None)
diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py
index 28b4cadef59..a107b916aad 100644
--- a/studio/backend/auth/authentication.py
+++ b/studio/backend/auth/authentication.py
@@ -187,7 +187,7 @@ class _BearerOrKeyless(HTTPBearer):
# scheme_name pinned so the OpenAPI securitySchemes entry keeps its published name
-security = _BearerOrKeyless(scheme_name = "HTTPBearer") # Reads Authorization: Bearer <token>
+security = _BearerOrKeyless(scheme_name = "HTTPBearer")
def _get_secret_for_subject(subject: str) -> str:
@@ -365,9 +365,7 @@ async def credentials_for_token(
"""
from utils.keyless_api_access import APPROVED_DUMMY_BEARERS, keyless_request_allowed
- # A real token is authoritative and never needs keyless classification. The
- # remaining settings/listener reads use SQLite and DNS, so keep them off the
- # event loop just like the normal credential lookup path.
+ # Settings/listener reads hit SQLite and DNS, so keep them off the event loop.
if token and token not in APPROVED_DUMMY_BEARERS:
return HTTPAuthorizationCredentials(scheme = "Bearer", credentials = token)
eligible = await run_in_threadpool(keyless_request_allowed, request)
@@ -487,7 +485,6 @@ async def _get_current_credential(
token = credentials.credentials
- # --- API key path (sk-unsloth-...) ---
if token.startswith(API_KEY_PREFIX):
verified = await run_in_threadpool(validate_api_key_with_credential, token)
if verified is None:
@@ -498,7 +495,6 @@ async def _get_current_credential(
username, secret = verified
return username, credential_generation(secret)
- # --- JWT path ---
subject = _decode_subject_without_verification(token)
if subject is None:
raise HTTPException(
diff --git a/studio/backend/auth/bootstrap_timeout.py b/studio/backend/auth/bootstrap_timeout.py
index 2653c6e9857..57e24101538 100644
--- a/studio/backend/auth/bootstrap_timeout.py
+++ b/studio/backend/auth/bootstrap_timeout.py
@@ -130,7 +130,7 @@ def enforce_bootstrap_password_deadline(
except Exception:
return False
if not still_default:
- return False # password changed in time -> leave Unsloth running
+ return False
message = (
"\nUnsloth Studio was exposed on the network but its default admin "
diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py
index 2756e334154..fb10ecba168 100644
--- a/studio/backend/auth/storage.py
+++ b/studio/backend/auth/storage.py
@@ -21,12 +21,11 @@ from utils.paths import auth_db_path, ensure_dir
DB_PATH = auth_db_path()
DEFAULT_ADMIN_USERNAME = "unsloth"
-# Single source for the password policy; models/auth.py ChangePasswordRequest
-# and the terminal prompt both enforce it. Keep the unsloth_cli mirror in sync.
+# Single source for the password policy; models/auth.py ChangePasswordRequest and the terminal
+# prompt both enforce it. Keep the unsloth_cli mirror in sync.
MIN_PASSWORD_LENGTH = 8
-# Plaintext bootstrap password file beside auth.db, deleted on first password
-# change so the credential never lingers on disk.
+# Plaintext bootstrap password file, deleted on first password change.
_BOOTSTRAP_PW_PATH = DB_PATH.parent / ".bootstrap_password"
# In-process cache to avoid re-reading the file on every HTML serve.
@@ -81,8 +80,7 @@ def _normalise_bootstrap_file(raw: bytes, password: str) -> None:
if raw != password.encode("utf-8"):
return
- # O_BINARY: without it Windows opens in text mode and turns the LF straight
- # back into CRLF, the bug being fixed.
+ # O_BINARY: Windows text mode turns the LF back into CRLF, the bug being fixed.
fd = os.open(
_BOOTSTRAP_PW_PATH,
os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0),
@@ -103,9 +101,7 @@ def _read_persisted_bootstrap_password() -> Optional[str]:
if not _BOOTSTRAP_PW_PATH.is_file():
return None
- # No caller handles a raise, so an unreadable file has to mean "no bootstrap
- # password", not a dead backend. We write UTF-8, so undecodable bytes are
- # damage whose plaintext is worthless anyway.
+ # An unreadable file must mean "no bootstrap password"; no caller handles a raise.
try:
raw = _BOOTSTRAP_PW_PATH.read_bytes()
password = raw.decode("utf-8").strip()
@@ -114,8 +110,7 @@ def _read_persisted_bootstrap_password() -> Optional[str]:
if not password:
return None
- # Older releases wrote no terminator; best-effort, a read-only auth dir must
- # not fail startup.
+ # Older releases wrote no terminator; a read-only auth dir must not fail startup.
if raw != _bootstrap_file_bytes(password):
try:
_normalise_bootstrap_file(raw, password)
@@ -185,10 +180,7 @@ def clear_bootstrap_password() -> None:
try:
_BOOTSTRAP_PW_PATH.unlink(missing_ok = True)
except OSError as e:
- # Removal failed (Windows AV, read-only auth dir). The hash is already
- # committed, so don't fail the change -- but truncate the file so its
- # stale plaintext can't be re-seeded by generate_bootstrap_password()
- # if auth.db is ever recreated.
+ # Truncate when removal fails: stale plaintext would otherwise be re-seeded if auth.db is recreated.
try:
_BOOTSTRAP_PW_PATH.write_text("", encoding = "utf-8")
cleared = True
@@ -202,8 +194,7 @@ def clear_bootstrap_password() -> None:
"cleared its contents so the old bootstrap password cannot be reused."
)
else:
- # Neither removed nor truncated: stale plaintext is still on disk
- # and would be reused if auth.db is reset. Don't claim otherwise.
+ # Stale plaintext is still on disk and would be reused if auth.db is reset.
message = (
f"Warning: could not delete or clear {_BOOTSTRAP_PW_PATH.name} ({e}); "
"its old bootstrap password is still on disk. Remove it manually to "
@@ -272,21 +263,15 @@ def get_connection() -> sqlite3.Connection:
"""Get a connection to the auth database, creating tables if needed."""
ensure_dir(DB_PATH.parent)
conn = sqlite3.connect(DB_PATH)
- # Keep the auth dir + DB private (they hold the JWT/identity secrets and
- # password hashes); sqlite3.connect would otherwise create the DB 0644 under
- # a 022 umask, letting another OS user read the identity secret and forge proofs.
+ # sqlite3.connect would create the DB 0644 under a 022 umask, exposing identity secrets and password hashes.
for _path, _mode in ((DB_PATH.parent, 0o700), (DB_PATH, 0o600)):
try:
os.chmod(_path, _mode)
except OSError:
pass
conn.row_factory = sqlite3.Row
- # WAL lets token reads run concurrently with refresh-token writes;
- # busy_timeout bounds lock waits. Matches the other Unsloth SQLite stores.
- # Set busy_timeout first: switching journal_mode needs a lock, so if a
- # refresh-token write already holds one, journal_mode=WAL raises SQLITE_BUSY;
- # with busy_timeout already in effect it waits instead of failing and leaving
- # this connection on SQLite's default zero lock wait.
+ # Set busy_timeout before journal_mode=WAL: switching journal mode needs a lock and would otherwise
+ # raise SQLITE_BUSY.
try:
conn.execute("PRAGMA busy_timeout=5000")
conn.execute("PRAGMA journal_mode=WAL")
@@ -357,13 +342,8 @@ def get_connection() -> sqlite3.Connection:
return conn
+# No lock needed: INSERT OR IGNORE is atomic and concurrent populations converge on the same value.
# ── API-key PBKDF2 salt ────────────────────────────────────────────────
-#
-# Module-level cache for the persistent API-key PBKDF2 salt, populated lazily
-# via ``_get_or_create_api_key_pbkdf2_salt``. No lock needed: (a) ``INSERT OR
-# IGNORE`` is atomic at the SQLite layer and (b) concurrent populations
-# converge on the same value, so the worst case is a harmless duplicate read
-# on startup.
_api_key_pbkdf2_salt_cache: Optional[bytes] = None
@@ -385,7 +365,7 @@ def _get_or_create_api_key_pbkdf2_salt() -> bytes:
)
row = cur.fetchone()
if row is None:
- new_value = secrets.token_hex(32) # 32 bytes -> 64 hex chars
+ new_value = secrets.token_hex(32)
conn.execute(
"INSERT OR IGNORE INTO app_secrets (key, value) VALUES (?, ?)",
("api_key_pbkdf2_salt", new_value),
@@ -404,9 +384,7 @@ def _get_or_create_api_key_pbkdf2_salt() -> bytes:
return salt
-# Secret answering the /api/auth/identity challenge (HMAC(secret, nonce)). Lives
-# in this same-user DB so a port squatter or remote/fake server can't forge a
-# proof. Separate from the per-user JWT secret.
+# Identity-challenge secret lives in auth.db so a port squatter cannot forge a proof; separate from the JWT secret.
_IDENTITY_SECRET_DB_KEY = "studio_identity_secret"
_identity_secret_cache: Optional[bytes] = None
@@ -441,9 +419,8 @@ def get_or_create_identity_secret() -> bytes:
return secret
-# Dedicated AES-256 key used to encrypt Unsloth credentials in studio.db.
-# It intentionally lives in auth.db so copying studio.db alone does not expose
-# provider or Hugging Face tokens, and survives password changes/resets.
+# Dedicated AES-256 key: lives in auth.db so copying studio.db alone does not expose provider/HF
+# tokens, and survives password resets.
_CREDENTIAL_ENCRYPTION_KEY_DB_KEY = "credential_encryption_key_v1"
_credential_encryption_key_cache: Optional[bytes] = None
@@ -487,17 +464,14 @@ def compute_identity_proof(nonce: bytes, host: str, port: int) -> str:
real one, e.g. localhost resolving to ::1 while Unsloth is on 127.0.0.1) was
computed for that other endpoint and won't match the one the client dialed."""
try:
- host = ipaddress.ip_address(host).compressed # normalise 127.0.0.1 / ::1 forms
+ host = ipaddress.ip_address(host).compressed
except ValueError:
host = (host or "").lower()
msg = b"|".join([nonce, host.encode(), str(int(port)).encode()])
return hmac.new(get_or_create_identity_secret(), msg, hashlib.sha256).hexdigest()
-# Capability secret for public ``/p`` preview share links. HMAC(secret, ref)
-# turns the deterministic preview ref into an unguessable bearer capability, so a
-# guessed run/checkpoint name can't reach inference. Dedicated (not the per-user
-# JWT secret) so rotating it revokes every shared link without touching logins.
+# Dedicated secret so rotating it revokes every shared preview link without touching logins.
_PREVIEW_LINK_SECRET_DB_KEY = "preview_link_secret"
_preview_link_secret_cache: Optional[bytes] = None
@@ -579,11 +553,8 @@ def _pbkdf2_desktop_secret(raw_secret: str) -> str:
return _pbkdf2_api_key(raw_secret)
-# Memoize the deterministic raw-key -> PBKDF2-hash derivation so the 100k-round
-# KDF runs once per key instead of on every authenticated request. Keyed by a
-# salted HMAC of the key (not the key itself); revocation/expiry are still
-# enforced by the SQLite read on every call, so a cache hit only skips the KDF.
-# Only keys present in the DB are cached, so unknown-key spam can't grow it.
+# Keyed by a salted HMAC, not the key; revocation/expiry are still enforced by the SQLite read, and
+# only known keys are cached.
_api_key_hash_cache: dict[str, str] = {}
# Whether each memoized key was minted internally; set once, since minting decides it.
_api_key_internal_cache: dict[str, bool] = {}
@@ -873,9 +844,8 @@ def consume_refresh_token(token: str) -> Optional[Tuple[str, bool, str]]:
now = datetime.now(timezone.utc).isoformat()
conn = get_connection()
try:
- # One transaction with the delete: an unstamped legacy row has no
- # generation to compare, so reading the credential after committing would
- # hand a reset's new secret to a token issued before it.
+ # One transaction with the delete: an unstamped legacy row has no generation, so a later read could
+ # hand a reset's new secret to an older token.
conn.execute("BEGIN IMMEDIATE")
conn.execute(
"DELETE FROM refresh_tokens WHERE expires_at < ?",
@@ -914,7 +884,6 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]:
token_hash = _hash_token(token)
conn = get_connection()
try:
- # Opportunistically clean up expired tokens
conn.execute(
"DELETE FROM refresh_tokens WHERE expires_at < ?",
(datetime.now(timezone.utc).isoformat(),),
@@ -939,7 +908,6 @@ def verify_refresh_token(token: str) -> Optional[Tuple[str, bool]]:
conn.commit()
return None
- # Check expiry
expires_at = datetime.fromisoformat(row["expires_at"])
if datetime.now(timezone.utc) > expires_at:
conn.execute("DELETE FROM refresh_tokens WHERE id = ?", (row["id"],))
@@ -1031,19 +999,12 @@ def clear_desktop_secret() -> None:
conn.close()
-# ---------------------------------------------------------------------------
-# API key management
# ---------------------------------------------------------------------------
API_KEY_PREFIX = "sk-unsloth-"
-# The ``name`` a workflow mints its internal key under. Unsloth mints internal
-# keys for more than one workflow and they do not carry the same authority, so
-# the name is the only thing that tells them apart after the fact. Deep Research
-# is durable: its hops outlive the session that started them, so they carry a key
-# instead of a JWT and still have to reach the saved connection the run was
-# created with. A data-recipe key is handed to a user-authored recipe subprocess
-# and needs nothing but this host's local /v1, so it must never gain that reach.
+# The name is the only thing distinguishing internal keys by authority: Deep Research keys must
+# reach the saved connection, data-recipe keys only local /v1.
DEEP_RESEARCH_WORKFLOW_KEY_NAME = "deep-research workflow"
@@ -1274,9 +1235,8 @@ def validate_api_key_with_credential(
with _api_key_hash_cache_lock:
if len(_api_key_hash_cache) >= _API_KEY_HASH_CACHE_MAX:
_api_key_hash_cache.clear()
- # is_internal_api_key sizes itself against the hash cache, so clearing one
- # without the other lets the origin cache grow past the bound. Deep Research
- # mints a fresh internal key per model call, so that adds up.
+ # Clear both caches together: the origin cache is sized against the hash cache and would otherwise
+ # grow past its bound.
_api_key_internal_cache.clear()
_api_key_hash_cache[cache_id] = key_hash
if not row["is_active"]:
diff --git a/studio/backend/auth/terminal_prompt.py b/studio/backend/auth/terminal_prompt.py
index 925404f47df..988a857ca46 100644
--- a/studio/backend/auth/terminal_prompt.py
+++ b/studio/backend/auth/terminal_prompt.py
@@ -24,8 +24,7 @@ _CTRL_Z = "\x1a"
_BACKSPACES = ("\x7f", "\x08")
_SUBMITS = ("\r", "\n")
-# Env var that supplies the initial admin password non-interactively (mirror in
-# unsloth_cli/commands/_password_prompt.py). Keep the name in sync.
+# Mirror in unsloth_cli/commands/_password_prompt.py; keep the name in sync.
SUPPLIED_PASSWORD_ENV = "UNSLOTH_STUDIO_PASSWORD"
@@ -33,8 +32,7 @@ def _getch_windows() -> str: # pragma: no cover - exercised via fake on Linux C
import msvcrt
ch = msvcrt.getwch()
- # Function/arrow keys arrive as a two-wchar \x00/\xe0 sequence; consume the
- # second half and report a no-op control char.
+ # Function/arrow keys arrive as a two-wchar \x00/\xe0 sequence; consume the second half.
if ch in ("\x00", "\xe0"):
msvcrt.getwch()
return "\x00"
@@ -69,7 +67,7 @@ class _RestoreTtyOnSignals:
continue
try:
self._previous.append((sig, signal.signal(sig, _restore_and_reraise)))
- except (ValueError, OSError): # non-main thread / unsupported
+ except (ValueError, OSError):
pass
return self
@@ -99,20 +97,18 @@ class _prompt_raw_mode:
try:
import termios
import tty
- except ImportError: # non-POSIX (Windows uses msvcrt, no mode to hold)
+ except ImportError:
return self
try:
fd = sys.stdin.fileno()
old_attrs = termios.tcgetattr(fd)
except (AttributeError, ValueError, OSError, termios.error):
- return self # redirected / captured stdin (tests): nothing to hold
+ return self
self._fd = fd
self._old_attrs = old_attrs
self._signals = _RestoreTtyOnSignals(fd, old_attrs)
self._signals.__enter__()
- # cbreak (not raw) keeps output post-processing while disabling echo/line
- # buffering. It leaves ISIG on, so clear it and surface Ctrl-C as \x03 to
- # the caller loop, which restores the tty itself.
+ # cbreak leaves ISIG on, so clear it and surface Ctrl-C as \x03 for the caller loop to restore the tty.
tty.setcbreak(fd, termios.TCSADRAIN)
new_attrs = termios.tcgetattr(fd)
new_attrs[3] &= ~termios.ISIG
@@ -131,9 +127,7 @@ class _prompt_raw_mode:
def _getch_posix() -> str: # pragma: no cover - needs a real tty
- # Terminal already in cbreak+no-echo for the whole line (_prompt_raw_mode),
- # so just read. Byte-at-a-time incremental decode so a multi-byte UTF-8 char
- # straddling a read boundary isn't dropped.
+ # Byte-at-a-time decode so a multi-byte UTF-8 char straddling a read boundary is not dropped.
import codecs
fd = sys.stdin.fileno()
@@ -141,7 +135,7 @@ def _getch_posix() -> str: # pragma: no cover - needs a real tty
while True:
b = os.read(fd, 1)
if not b:
- return "" # stream EOF; caller raises EOFError
+ return ""
ch = decoder.decode(b)
if ch:
return ch
@@ -164,11 +158,11 @@ def _read_password(prompt: str, *, out: "TextIO | None" = None) -> str:
with _prompt_raw_mode():
while True:
key = _getch()
- if key == "": # stream ended mid-line: abort, don't submit a partial
+ if key == "":
out.write("\n")
out.flush()
raise EOFError
- for ch in key: # a paste can deliver several chars per read
+ for ch in key:
if ch in _SUBMITS:
out.write("\n")
out.flush()
@@ -182,14 +176,14 @@ def _read_password(prompt: str, *, out: "TextIO | None" = None) -> str:
out.write("\n")
out.flush()
raise EOFError
- continue # ignore mid-input
+ continue
if ch in _BACKSPACES:
if chars:
chars.pop()
out.write("\b \b")
out.flush()
continue
- if ch < " ": # other control characters (tab, escape, ...)
+ if ch < " ":
continue
chars.append(ch)
out.write("*")
diff --git a/studio/backend/cloudflare_tunnel.py b/studio/backend/cloudflare_tunnel.py
index a9577559fc8..85265cc2c70 100644
--- a/studio/backend/cloudflare_tunnel.py
+++ b/studio/backend/cloudflare_tunnel.py
@@ -24,25 +24,19 @@ import time
from pathlib import Path
from typing import Callable, Optional, Tuple
-# cloudflared logs the quick-tunnel URL; match only the URL so we do not depend
-# on the surrounding wording, which Cloudflare may change. The negative lookahead
-# drops cloudflared's own API host, which appears in failure lines such as
-# failed to request quick Tunnel: Post "https://api.trycloudflare.com/tunnel"
-# and must never be mistaken for a usable tunnel URL.
+# Match only the URL; the negative lookahead drops cloudflared's own api.trycloudflare.com host from failure lines.
_URL_RE = re.compile(r"https://(?!api\.)[A-Za-z0-9-]+\.trycloudflare\.com")
-# cloudflared logs this once per edge connection it establishes. Until at least
-# one appears the quick-tunnel URL returns Cloudflare error 1033 (HTTP 530), so
-# we wait for it before advertising the URL.
+# Until an edge connection registers, the quick-tunnel URL returns Cloudflare error 1033 (HTTP 530).
_REGISTERED_MARKER = "Registered tunnel connection"
_RELEASE_BASE = "https://github.com/cloudflare/cloudflared/releases/latest/download"
-_READY_TIMEOUT = 15.0 # seconds to wait for the URL + a registered edge connection
-_DOWNLOAD_TIMEOUT = 60 # urlopen timeout for the one-time binary download
+_READY_TIMEOUT = 15.0
+_DOWNLOAD_TIMEOUT = 60
-# A registered edge connection does not mean the hostname resolves yet, so the
-# URL is fetched once before it is advertised.
+# A registered edge connection does not mean the hostname resolves yet, so the URL is fetched once before it is
+# advertised.
_PUBLIC_PROBE_PATH = "/api/health"
_PUBLIC_PROBE_MARKER = "Unsloth UI Backend"
# One deadline for DNS propagation + the health probe, bounding the startup stall.
@@ -50,21 +44,18 @@ _PUBLIC_PROBE_TIMEOUT = 45.0
_PUBLIC_PROBE_ATTEMPT_TIMEOUT = 5.0
_PUBLIC_PROBE_RETRY_DELAY = 1.0
-# Wait for the hostname via DoH first: an early OS lookup negative-caches the
-# NXDOMAIN for up to 30 min.
+# Resolve via DoH first: an early OS lookup negative-caches the NXDOMAIN for up to 30 min.
_DNS_POLL_DELAY = 2.0
# Retry transient DoH failures, but give up fast when DoH is blocked outright.
_DNS_MAX_DOH_ERRORS = 3
_DOH_URL = "https://cloudflare-dns.com/dns-query?name={host}&type=A"
-# The resolver negative-caches a miss of its own, so a query sent before the
-# record can exist blinds the poll for that cache's lifetime. Hold off first.
+# The resolver negative-caches a miss of its own, so a query sent before the record can exist blinds the poll
+# for that cache's lifetime. Hold off first.
_DNS_INITIAL_GRACE = 3.0
# A blinded poll cannot recover, so bound its share of the shared deadline.
_DNS_WAIT_MAX = 20.0
-# Cloudflare's edge routes by TLS SNI, so it serves the tunnel as soon as the
-# connection registers -- before the hostname resolves anywhere. Probing there
-# keeps DNS off the startup path entirely.
+# Cloudflare's edge routes by TLS SNI, so it serves the tunnel before the hostname resolves anywhere.
_EDGE_HOST = "trycloudflare.com"
_EDGE_PROBE_RETRY_DELAY = 0.5
# Bound the wait so the hostname fallback keeps most of the shared deadline.
@@ -147,7 +138,7 @@ def _asset_name() -> Optional[Tuple[str, bool]]:
def _cache_path() -> Optional[Path]:
"""studio_bin_root()/cloudflared(.exe), or None if the studio home is unresolvable."""
try:
- from utils.paths.storage_roots import studio_bin_root # lazy: backend-only import
+ from utils.paths.storage_roots import studio_bin_root
except Exception:
return None
name = "cloudflared.exe" if sys.platform == "win32" else "cloudflared"
@@ -293,8 +284,7 @@ def _edge_addresses() -> list:
return addresses
for info in resolved:
address = info[4][0]
- # macOS reports the A records as IPv4-mapped under AF_INET6. The mapped
- # and bare forms are one frontend, and probing it twice buys nothing.
+ # macOS reports the A records as IPv4-mapped under AF_INET6; the mapped and bare forms are one frontend.
if address.startswith("::ffff:"):
address = address[len("::ffff:") :]
if address not in addresses:
@@ -371,12 +361,14 @@ def verify_public_url(url: str, timeout: float = _PUBLIC_PROBE_TIMEOUT) -> bool:
if host:
if _verify_through_edge(host, deadline):
return True
- # The edge never served the tunnel, so fall back to the hostname and pay
- # the DoH wait that keeps an early OS lookup from caching the miss.
+ # The edge never served the tunnel, so fall back to the hostname and pay the DoH wait that keeps an
+ # early OS lookup from caching the miss.
_wait_for_dns(host, deadline)
probe_url = f"{url.rstrip('/')}{_PUBLIC_PROBE_PATH}"
while True:
+ # Drain cloudflared's output: capture the first trycloudflare URL and the first edge-connection
+ # registration, and keep draining so it never blocks on a full pipe.
try:
req = urllib.request.Request(probe_url, headers = {"User-Agent": "unsloth-studio"})
with urllib.request.urlopen(req, timeout = _PUBLIC_PROBE_ATTEMPT_TIMEOUT) as response:
@@ -422,8 +414,7 @@ class CloudflareTunnel:
self.port = port
self.binary = binary
self.origin_host = origin_host
- # None lets cloudflared pick its default (quic, with its own http2
- # fallback); set to "http2" to force it when quic is blocked.
+ # None lets cloudflared pick quic; set "http2" to force it when quic is blocked.
self.protocol = protocol
self._proc: Optional[subprocess.Popen] = None
self._lock = threading.Lock()
@@ -448,15 +439,13 @@ class CloudflareTunnel:
if self.protocol:
cmd += ["--protocol", self.protocol]
with self._lock:
- # A stop() that landed before us (e.g. a shutdown in the caller's
- # register->start window) marks the tunnel stopped; spawning now would
- # orphan a process nobody owns, so refuse.
+ # Refuse to spawn once a stop() has marked the tunnel stopped: it would orphan a process nobody owns.
if self._stopped:
return
_set_studio_tunnel_runtime_active(self, True)
try:
- # PDEATHSIG binds to the forking thread, so spawning from the
- # settings start worker would kill cloudflared when it returned.
+ # PDEATHSIG binds to the forking thread, so spawning from the settings worker would kill cloudflared
+ # when it returns.
proc = _spawn_child(
lambda: subprocess.Popen(
cmd,
@@ -474,9 +463,8 @@ class CloudflareTunnel:
except Exception:
_set_studio_tunnel_runtime_active(self, False)
raise
- # Adopted before the lock drops: a stop() that got in first would
- # otherwise reap and forget it while nothing was tracked, and this
- # would then record whatever inherited the pid.
+ # Adopt before dropping the lock: a racing stop() would otherwise forget it and this would record
+ # whatever inherited the pid.
_adopt_pid(proc.pid)
self._proc = proc
threading.Thread(
@@ -484,12 +472,11 @@ class CloudflareTunnel:
).start()
def _reader(self, proc: subprocess.Popen) -> None:
- # Drain cloudflared's output: capture the first trycloudflare URL and the
- # first edge-connection registration, and keep draining so it never
- # blocks on a full pipe.
try:
if proc.stdout is not None:
for line in proc.stdout:
+ # stdout closed -> cloudflared has exited. Record why, and unblock any waiters at once
+ # instead of letting them wait out the full timeout.
if self.url is None:
match = _URL_RE.search(line)
if match:
@@ -501,8 +488,6 @@ class CloudflareTunnel:
except Exception:
pass
finally:
- # stdout closed -> cloudflared has exited. Record why, and unblock any
- # waiters at once instead of letting them wait out the full timeout.
if self.url is None:
self.error = "cloudflared exited before emitting a tunnel URL"
elif not self.ready:
@@ -557,8 +542,8 @@ class CloudflareTunnel:
_set_studio_tunnel_runtime_active(self, False)
return True
else:
- # Preserve both the stop handle and the fail-closed trust state when
- # termination could not be confirmed. A later stop can retry.
+ # Preserve both the stop handle and the fail-closed trust state when termination could not be
+ # confirmed. A later stop can retry.
with self._lock:
if self._proc is None:
self._proc = proc
@@ -593,13 +578,12 @@ class CloudflareTunnel:
return running
-# Single serving process per Unsloth launch, so one module-level tunnel handle is
-# enough; the lock guards the start/stop/shutdown races.
+# Single serving process per Unsloth launch, so one module-level tunnel handle is enough; the lock guards the
+# start/stop/shutdown races.
_active_tunnel: Optional[CloudflareTunnel] = None
_active_lock = threading.Lock()
_start_lock = threading.Lock()
-# Latched by stop_studio_tunnel so a shutdown landing *between* a start's retry
-# attempts aborts the loop instead of starting a tunnel nobody will ever stop.
+# Latched by stop_studio_tunnel so a shutdown landing between retries cannot start a tunnel nobody will stop.
_shutdown_requested = False
_tunnel_generation = 0
_tunnel_lifecycle = 0
@@ -873,9 +857,8 @@ def start_studio_tunnel(
or _shutdown_requested
or _active_tunnel is not tunnel
):
- # Stop/shutdown landed while coming up. Detach AND tear
- # down: returning this URL would leave a live public
- # tunnel no later stop_studio_tunnel() can reach.
+ # Detach and tear down: returning this URL would leave a live public tunnel no later
+ # stop_studio_tunnel() can reach.
aborted = True
was_active = False
if _active_tunnel is tunnel:
@@ -922,9 +905,8 @@ def start_studio_tunnel(
elif generation == _tunnel_generation and _active_tunnel is tunnel:
if stopped:
_active_tunnel = None
- # Attempt 1's teardown parked the state at "stopping".
- # Leaving it there for the http2 retry makes
- # stop_studio_tunnel() early-return and stop nothing.
+ # Reset from "stopping" before the http2 retry, or stop_studio_tunnel() early-returns and stops
+ # nothing.
if _tunnel_state == "stopping" and protocol is None:
_tunnel_state = "starting"
else:
@@ -953,11 +935,11 @@ def stop_studio_tunnel(*, admission: Optional[Tuple[int, int]] = None) -> None:
if admission is not None and admission != (_tunnel_lifecycle, _tunnel_generation):
return
if _tunnel_state == "stopping":
+ # Latch so an in-flight start_studio_tunnel won't start a fresh tunnel (e.g. its http2 retry) after
+ # we have already torn down.
_shutdown_requested = True
_tunnel_generation += 1
return
- # Latch so an in-flight start_studio_tunnel won't start a fresh tunnel
- # (e.g. its http2 retry) after we have already torn down.
_shutdown_requested = True
_tunnel_generation += 1
stop_generation = _tunnel_generation
diff --git a/studio/backend/core/__init__.py b/studio/backend/core/__init__.py
index 30ecb27f810..e92cbd1d950 100644
--- a/studio/backend/core/__init__.py
+++ b/studio/backend/core/__init__.py
@@ -12,28 +12,23 @@ torch) before the version-activation code runs.
import sys
from pathlib import Path
-# Add backend dir to sys.path so bare "from utils.*" imports work when core
-# is imported as a package.
+# Add backend dir to sys.path so bare "from utils.*" imports work when core is imported as a package.
_backend_dir = str(Path(__file__).resolve().parent.parent)
if _backend_dir not in sys.path:
sys.path.insert(0, _backend_dir)
__all__ = [
- # Inference
"InferenceBackend",
"get_inference_backend",
- # Training
"get_training_backend",
"TrainingBackend",
"TrainingProgress",
- # Config
"ModelConfig",
"is_vision_model",
"scan_trained_models",
"scan_trained_loras",
"load_model_defaults",
"get_base_model_from_lora",
- # Utils
"format_and_template_dataset",
"normalize_path",
"is_local_path",
@@ -50,7 +45,6 @@ __all__ = [
def __getattr__(name):
- # Inference
if name in ("InferenceBackend", "get_inference_backend"):
from .inference import InferenceBackend, get_inference_backend
@@ -58,7 +52,6 @@ def __getattr__(name):
globals()["get_inference_backend"] = get_inference_backend
return globals()[name]
- # Training
if name in ("TrainingBackend", "get_training_backend", "TrainingProgress"):
from .training import TrainingBackend, get_training_backend, TrainingProgress
@@ -67,7 +60,6 @@ def __getattr__(name):
globals()["TrainingProgress"] = TrainingProgress
return globals()[name]
- # Config (utils.models)
if name in (
"is_vision_model",
"ModelConfig",
@@ -92,7 +84,6 @@ def __getattr__(name):
globals()["get_base_model_from_lora"] = get_base_model_from_lora
return globals()[name]
- # Paths
if name in ("normalize_path", "is_local_path", "is_model_cached"):
from utils.paths import normalize_path, is_local_path, is_model_cached
@@ -101,7 +92,6 @@ def __getattr__(name):
globals()["is_model_cached"] = is_model_cached
return globals()[name]
- # Utils
if name in ("without_hf_auth", "format_error_message"):
from utils.utils import without_hf_auth, format_error_message
@@ -109,7 +99,6 @@ def __getattr__(name):
globals()["format_error_message"] = format_error_message
return globals()[name]
- # Hardware
if name in (
"get_device",
"is_apple_silicon",
@@ -135,7 +124,6 @@ def __getattr__(name):
globals()["DeviceType"] = DeviceType
return globals()[name]
- # Datasets
if name == "format_and_template_dataset":
from utils.datasets import format_and_template_dataset
globals()["format_and_template_dataset"] = format_and_template_dataset
diff --git a/studio/backend/core/_torchao_stub.py b/studio/backend/core/_torchao_stub.py
index 7dd758f854a..a915ec074d8 100644
--- a/studio/backend/core/_torchao_stub.py
+++ b/studio/backend/core/_torchao_stub.py
@@ -28,9 +28,7 @@ from typing import Optional
_STUB_SENTINEL = object()
-# Metaclass for stub types so isinstance(x, StubClass) returns False instead of
-# raising TypeError -- peft's lora/torchao.py does isinstance() against torchao
-# types, which fails if those names resolve to stub modules rather than types.
+# isinstance() against a stub module raises TypeError; peft's lora/torchao.py needs it to return False.
class _StubTypeMeta(type):
def __instancecheck__(cls, instance):
return False
@@ -122,6 +120,7 @@ _HIP_LINE_RE = re.compile(r"^hip\s*(?::[^=]*)?=\s*(.+?)\s*$", re.MULTILINE)
def _version_is_rocm_tagged() -> Optional[bool]:
"""Whether the installed wheel's version carries a rocm tag. None if unreadable."""
+ # Neither on-disk signal was readable, so importing is the only way left.
try:
from importlib.metadata import version
return "rocm" in version("torch").lower()
@@ -177,7 +176,6 @@ def torch_is_rocm() -> bool:
verdict = _installed_torch_is_rocm()
if verdict is not None:
return verdict
- # Neither on-disk signal was readable, so importing is the only way left.
try:
import torch
except Exception:
diff --git a/studio/backend/core/data_recipe/jobs/constants.py b/studio/backend/core/data_recipe/jobs/constants.py
index 0045276e205..b919ea18f23 100644
--- a/studio/backend/core/data_recipe/jobs/constants.py
+++ b/studio/backend/core/data_recipe/jobs/constants.py
@@ -3,7 +3,6 @@
from __future__ import annotations
-# stages parsed from data-designer logs
STAGE_CREATE = "create"
STAGE_PREVIEW = "preview"
STAGE_DAG = "dag"
@@ -25,7 +24,6 @@ USAGE_RESET_STAGES = {
STAGE_PROFILING,
}
-# job event types emitted by worker/manager
EVENT_JOB_ENQUEUED = "job.enqueued"
EVENT_JOB_STARTED = "job.started"
EVENT_JOB_CANCELLING = "job.cancelling"
diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py
index 135c9fccf64..45306e769d8 100644
--- a/studio/backend/core/data_recipe/jobs/manager.py
+++ b/studio/backend/core/data_recipe/jobs/manager.py
@@ -186,7 +186,7 @@ class JobManager:
proc.start()
from utils.process_lifetime import adopt_pid
- adopt_pid(proc.pid) # bind to parent lifetime (Windows job / sweep)
+ adopt_pid(proc.pid)
self._mp_q = mp_q
self._proc = proc
@@ -462,6 +462,7 @@ class JobManager:
def _safe_handle_event(self, job: Job, event: dict) -> None:
"""Apply one event, swallowing any handler error so the pump can't die."""
+ # Worker exited: drain + finalize, guarded so an error can't strand the run "active".
try:
self._handle_event(job, event)
except Exception:
@@ -483,8 +484,7 @@ class JobManager:
try:
event = self._read_queue_with_timeout(mp_q, timeout_sec = 0.25)
except Exception:
- # If a read keeps raising after the worker died, finalize instead
- # of spinning forever; only retry while the worker is still alive.
+ # Only retry while the worker is alive; otherwise finalize instead of spinning forever.
logger.exception("Data-recipe job pump: queue read failed; continuing")
if proc.is_alive():
time.sleep(0.1)
@@ -498,7 +498,6 @@ class JobManager:
if proc.is_alive():
continue
- # Worker exited: drain + finalize, guarded so an error can't strand the run "active".
try:
for e in self._drain_queue(mp_q):
self._safe_handle_event(job, e)
diff --git a/studio/backend/core/data_recipe/jobs/parse.py b/studio/backend/core/data_recipe/jobs/parse.py
index 47f0f936d38..c4e26349b97 100644
--- a/studio/backend/core/data_recipe/jobs/parse.py
+++ b/studio/backend/core/data_recipe/jobs/parse.py
@@ -50,8 +50,7 @@ class ParsedUpdate:
_RE_SAMPLERS = re.compile(
r"Preparing samplers to generate (?P<rows>\d+) records across (?P<cols>\d+) columns"
)
-# Current data-designer phrasing for the same sampling stage (the line above is
-# kept for older builds). No column count is reported here, so only rows is set.
+# Newer data-designer phrasing for the same stage; no column count is reported here.
_RE_SAMPLING_SEED = re.compile(
r"Sampling (?P<rows>\d+) records from .*?seed dataset", re.IGNORECASE
)
diff --git a/studio/backend/core/data_recipe/jobs/types.py b/studio/backend/core/data_recipe/jobs/types.py
index c30606b2d12..8dc693fb0ef 100644
--- a/studio/backend/core/data_recipe/jobs/types.py
+++ b/studio/backend/core/data_recipe/jobs/types.py
@@ -90,8 +90,7 @@ class Job:
progress_columns_total: int | None = None
source_progress_estimated_total: int | None = None
completed_columns: list[str] = field(default_factory = list)
- # Id of the internal sk-unsloth-* API key minted for a local-model workflow.
- # Revoked when the job ends so the key's window matches the run, not its 24h TTL.
+ # Internal sk-unsloth-* key id; revoked when the job ends so its window matches the run, not the 24h TTL.
internal_api_key_id: int | None = None
_current_usage_model: str | None = None
_in_usage_summary: bool = False
diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py
index 79943b3b708..9ffc8282e96 100644
--- a/studio/backend/core/data_recipe/jobs/worker.py
+++ b/studio/backend/core/data_recipe/jobs/worker.py
@@ -82,7 +82,7 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])
"""Subprocess entrypoint. Sends events to `event_queue`."""
import os
- os.environ["PYTHONWARNINGS"] = "ignore" # suppress C-level warnings before imports
+ os.environ["PYTHONWARNINGS"] = "ignore"
import warnings
from loggers.config import LogConfig
@@ -118,8 +118,8 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])
builder = build_config_builder(recipe)
designer = create_data_designer(recipe, artifact_path = str(_ARTIFACT_ROOT))
- # DataDesigner resets root logging in __init__; attach the queue handler
- # to the named loggers directly so parser events survive.
+ # DataDesigner resets root logging in __init__; attach the queue handler to the named loggers so
+ # parser events survive.
handler = _QueueLogHandler(event_queue)
handler.setLevel(logging.INFO)
for logger_name in (
@@ -193,8 +193,7 @@ def run_job_process(*, event_queue, recipe: dict[str, Any], run: dict[str, Any])
def _merge_batches_to_single_parquet(base_dataset_path: Path) -> None:
parquet_dir = base_dataset_path / "parquet-files"
- # Counted, so a companion makes a single-batch job take the merge path and rewrite a
- # dataset that needed none.
+ # Counted, so a companion would make a single-batch job take the merge path and rewrite a dataset that needed none.
parquet_files = drop_appledouble_metadata(sorted(parquet_dir.glob("*.parquet")))
if len(parquet_files) <= 1:
return
diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py
index dc0deafdf5c..e6abd84be96 100644
--- a/studio/backend/core/data_recipe/local_callable_validators.py
+++ b/studio/backend/core/data_recipe/local_callable_validators.py
@@ -186,7 +186,7 @@ def _build_oxc_validation_function(lang: str, validation_mode: str, code_shape:
normalized_code_shape = code_shape if code_shape in _OXC_CODE_SHAPES else "auto"
def _validator(df):
- import pandas as pd # lazy import for local callable runtime
+ import pandas as pd
row_count = int(len(df.index))
if row_count == 0:
@@ -235,8 +235,7 @@ def _run_oxc_batch(
# The wrapper bounds oxlint with this; a kill here would leave the grandchild.
"timeout_ms": int(_OXC_TIMEOUT_S * 1000),
}
- # Resolve a usable Node (system or the isolated install, which is not on the
- # user's PATH); a bare "node" would fail for isolated-Node users.
+ # Resolve a usable Node: a bare "node" fails for isolated-Node users, whose install is not on PATH.
node_executable = resolve_node_executable()
if not node_executable:
return _fallback_results(
diff --git a/studio/backend/core/data_recipe/oxc-validator/validate.mjs b/studio/backend/core/data_recipe/oxc-validator/validate.mjs
index 9eb6565d7dd..d51921fca13 100644
--- a/studio/backend/core/data_recipe/oxc-validator/validate.mjs
+++ b/studio/backend/core/data_recipe/oxc-validator/validate.mjs
@@ -22,11 +22,8 @@ const SNIPPET_PREFIX = "(() => {\n";
const SNIPPET_SUFFIX = "\n})();\nexport {};\n";
const OXLINT_SUPPRESSED_RULES = ["no-unused-vars", "no-new-array"];
// The caller kills only this wrapper, so a wedged oxlint has to die here or it is orphaned.
-// It gets the caller's budget minus what this process spent, on the same monotonic basis.
const OXLINT_DEFAULT_BUDGET_MS = 30_000;
-// Covers the spawn, which precedes this process, plus its own wind-down.
const OXLINT_BUDGET_MARGIN_MS = 2_000;
-// Below this, starting oxlint would only hand the caller's kill an orphan.
const OXLINT_MIN_TIMEOUT_MS = 1_000;
const TOOL_DIR = dirname(fileURLToPath(import.meta.url));
diff --git a/studio/backend/core/data_recipe/service.py b/studio/backend/core/data_recipe/service.py
index 3c2545bf19a..553c102d86b 100644
--- a/studio/backend/core/data_recipe/service.py
+++ b/studio/backend/core/data_recipe/service.py
@@ -181,8 +181,7 @@ def recipe_has_stdio_mcp(recipe: dict[str, Any]) -> bool:
def build_mcp_providers(recipe: dict[str, Any]) -> list:
from data_designer.config.mcp import LocalStdioMCPProvider, MCPProvider # pyright: ignore[reportMissingImports]
- # Same gate as the chat MCP path: stdio providers spawn a local subprocess,
- # so build them only when this host allows it (desktop / explicit opt-in).
+ # Stdio providers spawn a local subprocess, so build them only when this host allows it.
from core.inference.mcp_client import stdio_mcp_enabled
stdio_allowed = stdio_mcp_enabled()
@@ -270,8 +269,8 @@ def build_config_builder(recipe: dict[str, Any]):
specs = oxc_local_callable_specs,
)
- # DataDesignerConfigBuilder.from_config currently skips processors.
- # Re-attach so drop_columns/schema_transform survive the API payload.
+ # DataDesignerConfigBuilder.from_config skips processors; re-attach so drop_columns/schema_transform
+ # survive the API payload.
for processor in recipe_core.get("processors") or []:
if not isinstance(processor, dict):
continue
@@ -292,16 +291,15 @@ def create_data_designer(recipe: dict[str, Any], *, artifact_path: str | None =
from data_designer.interface.data_designer import DataDesigner # pyright: ignore[reportMissingImports]
if artifact_path is None:
- # DataDesigner defaults to cwd/artifacts; packaged Unsloth can run with
- # cwd=/, so keep default callers on Unsloth's writable recipe artifact root.
+ # DataDesigner defaults to cwd/artifacts and packaged Unsloth can run with cwd=/, so pin the
+ # writable recipe artifact root.
artifact_path = str(recipe_datasets_root())
recipe = _strip_frontend_model_config_metadata(recipe)
model_providers = build_model_providers(recipe)
_validate_recipe_runtime_support(recipe, model_providers)
- # DataDesigner requires >=1 model provider even with no LLM columns; stub
- # one so sampler/expression-only recipes run without a real provider.
+ # DataDesigner requires >=1 model provider even with no LLM columns.
if not model_providers:
from data_designer.config.models import ModelProvider # pyright: ignore[reportMissingImports]
model_providers = [
diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py
index cebf669a15c..7ed67132f6b 100644
--- a/studio/backend/core/export/export.py
+++ b/studio/backend/core/export/export.py
@@ -13,12 +13,12 @@ import contextlib
from pathlib import Path
from typing import Optional, Tuple, List
-# unsloth imports torch on non-MLX hosts, so a --no-torch install raises here. Stay importable
-# (null the classes) so exports return a clean "PyTorch is not installed" error.
+# unsloth imports torch, so a --no-torch install raises here; stay importable and return a clean
+# "PyTorch is not installed" error.
try:
from unsloth import FastLanguageModel, FastVisionModel, _IS_MLX
_UNSLOTH_IMPORT_ERROR = None
-except Exception as _unsloth_exc: # ImportError (e.g. missing torch) or a broken native load
+except Exception as _unsloth_exc:
FastLanguageModel = None
FastVisionModel = None
_IS_MLX = False
@@ -47,7 +47,7 @@ if not _IS_MLX:
from peft import PeftModel, PeftModelForCausalLM
from transformers.modeling_utils import PushToHubMixin
import torch
- except Exception as _torch_exc: # ImportError, or a broken native torch load
+ except Exception as _torch_exc:
_TORCH_IMPORT_ERROR = _torch_exc
logger = get_logger(__name__)
@@ -102,8 +102,7 @@ def _multi_gpu_device_map_kwargs() -> dict:
device_map = get_device_map(None)
else:
return {}
- # Every sharding answer `get_device_map` gives; a "balanced"-only whitelist
- # dropped the map the moment CUDA began asking for a planned one.
+ # A "balanced"-only whitelist dropped the map the moment CUDA began asking for a planned one.
if device_map in ("balanced", "unsloth", "unsloth_balanced"):
return {"device_map": device_map}
except Exception as exc:
@@ -259,7 +258,7 @@ def _materialized_imatrix_path(model_dir, imatrix_file):
(see `_is_imatrix`): a basename match would suppress a real output of the same name.
"""
if imatrix_file is True:
- name = "imatrix_unsloth.gguf" # the upstream imatrix_unsloth.gguf_file, renamed
+ name = "imatrix_unsloth.gguf"
elif isinstance(imatrix_file, (str, os.PathLike)):
base = os.path.basename(os.fspath(imatrix_file))
if not base.endswith(".gguf_file"):
@@ -333,7 +332,7 @@ def _hf_offline(timeout = 3):
):
return True
if os.environ.get("UNSLOTH_OFFLINE_PROBE", "1").strip().lower() in {"0", "false", "no", "off"}:
- return False # probe disabled -> assume online; loads still pass local_files_only on env
+ return False
# Shared bounded, proxy-aware probe (also used by the export worker before version activation).
from utils.transformers_version import hf_endpoint_unreachable
@@ -344,7 +343,6 @@ def _hf_offline(timeout = 3):
return False
-# Reuse Unsloth's lock-guarded forced-offline context; no-op fallback if it moves.
try:
from unsloth.models.loader_utils import _force_hf_offline
except Exception:
@@ -488,7 +486,6 @@ class ExportBackend:
checkpoint_path_obj = Path(checkpoint_path)
- # Model identity for type detection
adapter_config = checkpoint_path_obj / "adapter_config.json"
base_model = None
if adapter_config.exists():
@@ -508,9 +505,9 @@ class ExportBackend:
else _device_map_override
)
- # Run the type-detection probes in the forced-offline window (else a gated
- # base 404s); it covers is_vision_model's Hub reads + the transformers-5
- # subprocess, and local_files_only makes detect_audio_type's requests.get skip.
+ # Run the type-detection probes inside the forced-offline window, else a gated base 404s: it
+ # covers is_vision_model's Hub reads, and local_files_only makes detect_audio_type's requests.get
+ # skip.
with _offline_window_if(local_files_only):
self._audio_type = detect_audio_type(
model_id, hf_token = token, local_files_only = local_files_only
@@ -604,7 +601,7 @@ class ExportBackend:
local_files_only = local_files_only,
**_device_map_kw,
)
- tokenizer = processor # vision: processor acts as tokenizer
+ tokenizer = processor
else:
logger.info("Loading as text model...")
@@ -619,15 +616,14 @@ class ExportBackend:
**_device_map_kw,
)
- # Only when we asked for the multi-GPU map: a single-GPU host has no second
- # placement to retry on, so leave its behaviour untouched.
+ # Only for the multi-GPU map: a single-GPU host has no second placement to retry on.
_offloaded = _cpu_offloaded_modules(model) if _device_map_kw else 0
if _device_map_override is None and _offloaded:
del model
raise _CpuSpillRetry(f"{_offloaded} module(s) offloaded to CPU/disk")
if _IS_MLX:
- # MLX doesn't use PeftModel — detect LoRA via adapter_config.json
+ # MLX doesn't use PeftModel - detect LoRA via adapter_config.json
self.is_peft = adapter_config.exists()
else:
self.is_peft = isinstance(model, (PeftModel, PeftModelForCausalLM))
@@ -654,11 +650,8 @@ class ExportBackend:
return True, f"Loaded {model_type} model{peft_info} successfully"
except Exception as e:
- # Sharding is an optimisation, never a requirement. "balanced" budgets from the
- # free memory read BEFORE this process opens a CUDA context on each GPU, so when
- # a training or chat job already owns the others the shard can OOM, or spill to
- # CPU and be refused by bitsandbytes, where the old single-device load succeeded.
- # Fall back once before giving up.
+ # "balanced" budgets from free memory read BEFORE this process opens a CUDA context, so the shard
+ # can OOM when another job owns the other GPUs; fall back once.
if (
_device_map_override is None
and (
@@ -669,8 +662,8 @@ class ExportBackend:
)
and _multi_gpu_device_map_kwargs()
):
- # Retry outside this block: the live traceback pins the half-built model's
- # frames, so an in-block retry inherits the exhausted device.
+ # Retry outside this block: the live traceback pins the half-built model's frames, so an in-block
+ # retry inherits the exhausted device.
retry_reason = str(e)
else:
logger.error(f"Error loading checkpoint: {e}")
@@ -690,9 +683,8 @@ class ExportBackend:
load_in_4bit = load_in_4bit,
trust_remote_code = trust_remote_code,
hf_token = hf_token,
- # Named, not omitted: an omitted map is unsloth's DEFAULT_DEVICE_MAP, which
- # `requested_device_map` upgrades back to the planner, re-running the
- # placement that just failed. A typed "sequential" passes through.
+ # Name the map: an omitted one is unsloth's DEFAULT_DEVICE_MAP, which requested_device_map
+ # upgrades back to the planner, re-running the placement that just failed.
_device_map_override = {"device_map": "sequential"},
)
@@ -744,13 +736,11 @@ class ExportBackend:
if not self.current_model or not self.current_tokenizer:
return False, "No model loaded. Please select a checkpoint first.", None
- # Merged export works for PEFT adapters and non-PEFT Local/HF base models alike
- # (save_pretrained_merged is a no-op merge that just saves the base).
+ # save_pretrained_merged is a no-op merge for non-PEFT base models, so one path covers both.
output_path: Optional[str] = None
- # Quantized formats save to a sibling "<dir>-<suffix>". Two backends: compressed-tensors
- # (llm-compressor, NVIDIA-only) and portable torchao FP8/INT8 (device-agnostic). The alias
- # comes from `compressed_method` (the "all formats" dropdown) or the `format_type` label.
+ # Two backends: compressed-tensors (llm-compressor, NVIDIA-only) and portable torchao FP8/INT8.
+ # The alias comes from compressed_method (the "all formats" dropdown) or the format_type label.
_LABEL_TO_ALIAS = {
"FP8 (compressed-tensors)": "fp8",
"NVFP4 (compressed-tensors)": "nvfp4",
@@ -798,9 +788,8 @@ class ExportBackend:
)
import unsloth.save as _us
- # Prefer the llm-compressor-main shadow (transformers 5.x): it quantizes newer models
- # (Qwen3.5, Gemma-4, ...) the shipped 0.10.x cannot. Route all compressed exports
- # through it when available; else fall back to the workspace 0.10.x path below.
+ # Prefer the llm-compressor-main shadow (transformers 5.x): the shipped 0.10.x cannot quantize newer
+ # models.
_shadow_pp = None
try:
from utils.transformers_version import llmcompressor_shadow_pythonpath
@@ -810,8 +799,7 @@ class ExportBackend:
if _shadow_pp:
os.environ[_us._COMPRESSED_QUANTIZE_PYTHONPATH_ENV] = _shadow_pp
else:
- # No shadow (disabled/offline/failed): the workspace 0.10.x cannot exceed its
- # transformers ceiling, so fail fast for sidecar models; default-tier still works.
+ # The workspace 0.10.x cannot exceed its transformers ceiling, so fail fast for sidecar models.
os.environ.pop(_us._COMPRESSED_QUANTIZE_PYTHONPATH_ENV, None)
_exceeds, _tf_ver = _us._transformers_exceeds_llm_compressor_ceiling()
if _exceeds:
@@ -908,8 +896,8 @@ class ExportBackend:
private = private,
)
elif (is_compressed or is_torchao) and output_path and Path(output_path).is_dir():
- # Already built in output_path; upload it directly instead of re-running the
- # expensive quantization that push_to_hub_merged(save_method=...) would redo.
+ # Upload the artifact already built in output_path; push_to_hub_merged(save_method=...) would
+ # redo the expensive quantization.
hf_api = HfApi(token = hf_token)
repo_id = PushToHubMixin._create_repo(
PushToHubMixin,
@@ -987,8 +975,7 @@ class ExportBackend:
ensure_dir(Path(save_directory))
if _IS_MLX:
- # MLX: save_pretrained_merged handles non-LoRA models too
- # (fuse() is a no-op without LoRA layers)
+ # fuse() is a no-op without LoRA layers, so this handles non-LoRA models too.
self.current_model.save_pretrained_merged(
save_directory,
self.current_tokenizer,
@@ -998,7 +985,6 @@ class ExportBackend:
self.current_model.save_pretrained(save_directory)
self.current_tokenizer.save_pretrained(save_directory)
- # Write export metadata so the Chat page can identify the base model
self._write_export_metadata(save_directory)
logger.info(f"Model saved successfully to {save_directory}")
output_path = str(Path(save_directory).resolve())
@@ -1037,7 +1023,6 @@ class ExportBackend:
private = private,
)
else:
- # Base model name from request or model config
base_model = (
base_model_id or self.current_model.config._name_or_path or "unknown"
)
@@ -1118,8 +1103,7 @@ class ExportBackend:
if not self.current_model or not self.current_tokenizer:
return False, "No model loaded. Please select a checkpoint first.", None
- # Only forward imatrix_file to an unsloth build that accepts it, else older builds raise
- # an unexpected-keyword error even for a plain no-imatrix export.
+ # Older unsloth builds raise on an unexpected imatrix_file kwarg even for a plain no-imatrix export.
if imatrix_file and not _imatrix_export_supported(self.current_model.save_pretrained_gguf):
return (
False,
@@ -1144,8 +1128,8 @@ class ExportBackend:
None,
)
shard_kw = {"gguf_shard_size": gguf_shard_size} if gguf_shard_size is not None else {}
- # Resolution reads a Hub repo, so the local save needs the token too -- kept out of
- # imatrix_kw, which the push below shares and already names token= itself.
+ # Resolution reads a Hub repo, so the local save needs the token; kept out of imatrix_kw, which the
+ # push shares and names token= itself.
local_token_kw = (
{"token": hf_token}
if imatrix_file
@@ -1165,8 +1149,8 @@ class ExportBackend:
quant_methods = ["q4_k_m"]
quant_method = quant_methods if len(quant_methods) > 1 else quant_methods[0]
- # Pin convert_hf_to_gguf.py to setup.sh's tagged llama.cpp ref so it
- # can't drift past the pinned llama-quantize binary's gguf API.
+ # Pin convert_hf_to_gguf.py to setup.sh's llama.cpp ref so it cannot drift past the pinned llama-
+ # quantize gguf API.
global _LLAMA_CPP_SCRIPTS_WARNING_EMITTED
try:
from unsloth_zoo.llama_cpp import (
@@ -1175,8 +1159,8 @@ class ExportBackend:
)
os.environ.setdefault("UNSLOTH_LLAMA_CPP_SCRIPTS_DIR", LLAMA_CPP_DEFAULT_DIR)
except Exception:
- # Not just ImportError: a half-built unsloth_zoo raises RuntimeError or
- # AttributeError, and this pin is an optimisation, not worth failing an export.
+ # Not just ImportError: a half-built unsloth_zoo raises RuntimeError or AttributeError, and this pin
+ # is only an optimisation.
if not _LLAMA_CPP_SCRIPTS_WARNING_EMITTED:
logger.warning(
"Unsloth: installed unsloth_zoo does not honor "
@@ -1202,7 +1186,7 @@ class ExportBackend:
model_tmp_root = tempfile.mkdtemp(prefix = "_tmp_model_", dir = abs_save_dir)
model_tmp_path = Path(model_tmp_root)
_model_tmp = os.path.join(model_tmp_root, "model")
- # Needed by the cleanup below too, so resolve it before anything can raise.
+ # Resolve before anything can raise; the cleanup below needs it too.
imatrix_path = _materialized_imatrix_path(_model_tmp, imatrix_file)
try:
result = self.current_model.save_pretrained_gguf(
@@ -1247,16 +1231,17 @@ class ExportBackend:
"GGUF conversion produced a symlinked Modelfile, "
f"refusing to relocate it: {modelfile}"
)
- # Optional artifact: unsloth generates it best-effort, so a locked or
- # read-only destination must not fail an export whose GGUFs all landed.
+ # Optional artifact: a locked or read-only destination must not fail an export whose GGUFs all
+ # landed.
+ # Optional artifact: unsloth generates it best-effort, so a locked or read-only
+ # destination must not fail an export whose GGUFs all landed.
try:
shutil.move(str(modelfile), os.path.join(abs_save_dir, "Modelfile"))
logger.info(f"Relocated Modelfile → {abs_save_dir}/")
except OSError as exception:
logger.warning(f"Could not relocate the Modelfile: {exception}")
finally:
- # Preserve any GGUF that could not be relocated. The imatrix is an input,
- # so counting it would retain the merged checkpoint on every such export.
+ # The imatrix is an input, so counting it would retain the merged checkpoint on every such export.
unrelocated = []
if model_tmp_path.is_dir():
unrelocated = sorted(
@@ -1272,9 +1257,8 @@ class ExportBackend:
else:
shutil.rmtree(model_tmp_root, ignore_errors = True)
- # iterdir, not glob.glob: glob hides dot-leading names, so an empty
- # model stem's ".Q4_K_M.gguf" got reported as "(none)". This list is the
- # success gate, so a leftover companion would report a run that wrote nothing.
+ # iterdir, not glob.glob: glob hides dot-leading names, so an empty model stem's ".Q4_K_M.gguf"
+ # read as "(none)". This list is the success gate.
final_ggufs = sorted(
str(p)
for p in drop_appledouble_metadata(list(Path(abs_save_dir).iterdir()))
@@ -1287,7 +1271,6 @@ class ExportBackend:
)
if not final_ggufs:
# Reporting success over an empty directory is what hid #7897.
- # The owned temp root is already gone: the finally above drops it.
return (
False,
f"GGUF conversion reported success but wrote no .gguf file to "
@@ -1373,11 +1356,8 @@ class ExportBackend:
"Use the safetensors adapter instead.",
None,
)
- # llama.cpp's convert_lora_to_gguf.py has no concept of DoRA's
- # lora_magnitude_vector tensors: it only reads the standard
- # lora_A/lora_B delta, so exporting a DoRA adapter would silently
- # drop the magnitude rescaling and produce a GGUF LoRA file that
- # loads fine but no longer matches the trained model.
+ # convert_lora_to_gguf.py reads only the standard lora_A/lora_B delta and ignores DoRA's
+ # lora_magnitude_vector, so a DoRA export silently drops magnitude rescaling.
_peft_config = getattr(self.current_model, "peft_config", {}).get("default")
if getattr(_peft_config, "use_dora", False):
return (
@@ -1396,8 +1376,7 @@ class ExportBackend:
f"Choose one of {', '.join(_GGUF_LORA_OUTTYPES)}.",
None,
)
- # getattr so an older build without save_pretrained_gguf returns a clean message
- # instead of an AttributeError (a generic 500).
+ # getattr so an older build without save_pretrained_gguf returns a clean message instead of a generic 500.
_save_gguf_fn = getattr(self.current_model, "save_pretrained_gguf", None)
if _save_gguf_fn is None or not _supports_kwarg(_save_gguf_fn, "save_method"):
return (
@@ -1437,7 +1416,6 @@ class ExportBackend:
"\n ".join(os.path.basename(f) for f in final_ggufs) or "(none)",
)
elif _IS_MLX:
- # MLX: save adapters.safetensors + tokenizer files
self.current_model.save_lora_adapters(save_directory)
self.current_tokenizer.save_pretrained(save_directory)
else:
@@ -1457,8 +1435,7 @@ class ExportBackend:
logger.info(f"Pushing LoRA adapter to Hub: {repo_id}")
if gguf:
- # Upload the locally-built GGUF folder; needs a local save_directory so the
- # conversion is not re-run.
+ # Needs a local save_directory so the conversion is not re-run.
if not (output_path and Path(output_path).is_dir()):
return (
False,
@@ -1499,7 +1476,6 @@ class ExportBackend:
return False, f"Adapter export failed: {str(e)}", None
-# Global export backend instance
_export_backend = None
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py
index e8f258ebe84..d357b1e0998 100644
--- a/studio/backend/core/export/orchestrator.py
+++ b/studio/backend/core/export/orchestrator.py
@@ -58,20 +58,16 @@ class ExportOrchestrator:
self._log_lock = threading.Lock()
# Monotonic seq, never reset, so SSE clients have a stable cursor across clear_logs().
self._log_seq: int = 0
- # _log_seq snapshot at the current run's start; SSE defaults its cursor here so a
- # late-connecting client still sees the full run. Current run has seq > this.
+ # SSE defaults its cursor here so a late-connecting client still sees the full run.
self._run_start_seq: int = 0
# True while an export op runs; SSE ends the stream 1s after this flips False.
self._export_active: bool = False
- # Set by cancel_export(); reset when a new load/export run starts. Lets the
- # caller distinguish a user cancel from a genuine subprocess crash.
+ # Set by cancel_export(); reset when a new load/export run starts. Lets the caller distinguish a
+ # user cancel from a genuine subprocess crash.
self._cancel_requested: bool = False
- # Last finished operation, so a client whose blocking POST was cut off by a
- # Cloudflare tunnel timeout (524 at ~100s, while the op runs for minutes) can
- # poll /api/export/status and still learn the real outcome. Guarded by
- # _op_lock. `_op_seq` is a monotonic counter the client uses as a baseline to
- # tell "my op finished" (seq grew) from a stale previous result.
+ # Kept so a client whose blocking POST was cut by a tunnel 524 can poll /api/export/status; _op_seq
+ # is the monotonic baseline for "my op finished".
self._op_lock = threading.Lock()
self._op_seq: int = 0
self._active_op_kind: Optional[str] = None
@@ -81,8 +77,6 @@ class ExportOrchestrator:
logger.info("ExportOrchestrator initialized (subprocess mode)")
# ------------------------------------------------------------------
- # Live log capture helpers
- # ------------------------------------------------------------------
def _append_log(self, entry: Dict[str, Any]) -> None:
"""Append a worker log line to the buffer, stamped with a monotonic seq."""
@@ -204,22 +198,18 @@ class ExportOrchestrator:
return True
# ------------------------------------------------------------------
- # Subprocess lifecycle
- # ------------------------------------------------------------------
def _spawn_subprocess(self, config: dict) -> None:
"""Spawn a new export subprocess."""
- # Last-resort recheck for spawns outside an active op. Inside an op, _export_active is set and
- # load_checkpoint already rechecked, so a reservation here is an install about to observe
- # is_export_active() and abort; raising would kill this export for an install that never proceeds.
+ # Inside an op a reservation is an install about to abort on is_export_active(), so raising here
+ # would kill the export for an install that never proceeds.
from utils.transformers_version import sidecar_swap_in_progress
from utils.transformers_version import sidecar_swap_kind
_swap_kind = sidecar_swap_kind()
- # Inside an active op an INSTALL reservation is about to abort on the
- # is_export_active check, but a lazy REPAIR has no such check and can be
- # rebuilding the sidecar right now, so it must always refuse the spawn.
+ # An INSTALL reservation aborts on the is_export_active check, but a lazy REPAIR has none and may
+ # be rebuilding the sidecar right now, so always refuse the spawn.
if _swap_kind == "repair" or (_swap_kind is not None and not self._export_active):
from utils.transformers_version import SidecarSwapInProgress
raise SidecarSwapInProgress(
@@ -254,7 +244,7 @@ class ExportOrchestrator:
self._proc.start()
from utils.process_lifetime import adopt_pid
- adopt_pid(self._proc.pid) # bind to parent lifetime (Windows job / sweep)
+ adopt_pid(self._proc.pid)
logger.info("Export subprocess started (pid=%s)", self._proc.pid)
def _shutdown_subprocess(self, timeout: float = 10.0) -> bool:
@@ -298,8 +288,7 @@ class ExportOrchestrator:
pass
if self._proc is not None and self._proc.is_alive():
- # Survived SIGKILL (uninterruptible syscall): keep the handle so callers
- # and the pre-swap guard see a live worker rather than a nulled one.
+ # Survived SIGKILL: keep the handle so callers and the pre-swap guard see a live worker.
logger.error(
"Export subprocess still alive after terminate/kill; "
"preserving its handle for the pre-swap liveness check"
@@ -321,8 +310,6 @@ class ExportOrchestrator:
return self._proc is not None and self._proc.is_alive()
# ------------------------------------------------------------------
- # Queue helpers
- # ------------------------------------------------------------------
def _send_cmd(self, cmd: dict) -> None:
"""Send a command to the subprocess."""
@@ -381,8 +368,8 @@ class ExportOrchestrator:
if rtype == "status":
message = resp.get("message", "")
- # One structured export_progress line per phase (consolidated in the
- # server log, like training/download progress); also shown live.
+ # One structured export_progress line per phase (consolidated in the server log, like
+ # training/download progress); also shown live.
if message:
logger.info("export_progress", phase = message)
self._append_log(
@@ -394,7 +381,7 @@ class ExportOrchestrator:
)
continue
- # Other response types during wait — skip.
+ # Other response types during wait - skip.
logger.debug(
"Skipping response type '%s' while waiting for '%s'",
rtype,
@@ -417,8 +404,6 @@ class ExportOrchestrator:
return events
# ------------------------------------------------------------------
- # Public API — same interface as ExportBackend
- # ------------------------------------------------------------------
def load_checkpoint(
self,
@@ -452,9 +437,8 @@ class ExportOrchestrator:
self._export_active = True
op_success, op_message = False, ""
try:
- # Handshake with the sidecar install route: _export_active is set above, so either this
- # recheck refuses BEFORE tearing down the old worker (keeping the loaded checkpoint), or
- # the install sees is_export_active() and 409s. The spawn-time recheck stays as a last resort.
+ # Handshake with the sidecar install route (see load_checkpoint): either this recheck refuses
+ # before tearing down the old worker, or the install sees is_export_active() and 409s.
from utils.transformers_version import sidecar_swap_in_progress
if sidecar_swap_in_progress():
@@ -467,10 +451,9 @@ class ExportOrchestrator:
# Always kill any existing subprocess and spawn fresh.
if self._ensure_subprocess_alive():
if self._shutdown_subprocess() is False:
- # Survivor still holds GPU memory (a wedged CUDA syscall outliving
- # SIGKILL); its handle is kept so is_worker_alive() and the pre-swap
- # guard still see it. Do not spawn a second worker over it -- fail so
- # the load can retry once it exits.
+ # A survivor still holds GPU memory (a wedged CUDA syscall outliving SIGKILL) and its handle is
+ # kept so is_worker_alive() still sees it, so do not spawn a second worker over it; fail so the
+ # load can retry once it exits.
op_message = (
"The current export worker did not exit and still holds GPU "
"memory; not starting a new checkpoint load over it. Retry shortly."
@@ -483,9 +466,8 @@ class ExportOrchestrator:
try:
self._spawn_subprocess(sub_config)
except Exception:
- # The old worker is already gone; a stale current_checkpoint
- # would make the Export page claim a loaded checkpoint that
- # the next op then fails on with "no subprocess running".
+ # A stale current_checkpoint would make the Export page claim a loaded checkpoint the next op then
+ # fails on.
self.current_checkpoint = None
self.is_vision = False
self.is_peft = False
@@ -637,9 +619,7 @@ class ExportOrchestrator:
self._export_active = True
op_success, op_message, op_output_path = False, "", None
try:
- # Handshake with the sidecar install route (see load_checkpoint): _export_active is set
- # above, so this recheck refuses before the command is sent, or the install sees the active
- # op and 409s. Without it, an install would block in cleanup_memory behind a long export op.
+ # Recheck before sending, else an install blocks in cleanup_memory behind a long export op.
from utils.transformers_version import sidecar_swap_in_progress
if sidecar_swap_in_progress():
@@ -652,8 +632,8 @@ class ExportOrchestrator:
cmd = {"type": "export", "export_type": export_type, **params}
try:
self._send_cmd(cmd)
- # GGUF for 30B+ models can take 30+ min per quant; a multi-quant list runs them
- # all in one op off a single merge, so scale the timeout by the quant count.
+ # GGUF for a 30B+ model can take 30+ min per quant, and a multi-quant list runs every quant in one
+ # op off a single merge, so scale the timeout by quant count.
_qm = params.get("quantization_method")
_n = len(_qm) if isinstance(_qm, (list, tuple)) and _qm else 1
resp = self._wait_response(
@@ -692,7 +672,7 @@ class ExportOrchestrator:
except RuntimeError:
success = False
- # Shut down subprocess after cleanup — no model loaded.
+ # Shut down subprocess after cleanup - no model loaded.
self._shutdown_subprocess()
self.current_checkpoint = None
diff --git a/studio/backend/core/rag/captioner.py b/studio/backend/core/rag/captioner.py
index ccc150c3c76..b6c301c03c2 100644
--- a/studio/backend/core/rag/captioner.py
+++ b/studio/backend/core/rag/captioner.py
@@ -54,7 +54,7 @@ def _collapse_runaway(
for line in text.splitlines():
key = line.strip()
if not key:
- if prev == "": # collapse runs of blank lines to a single separator
+ if prev == "":
continue
prev = ""
out.append("")
diff --git a/studio/backend/core/rag/chunking.py b/studio/backend/core/rag/chunking.py
index c64acb4c60f..cbb4530b1ad 100644
--- a/studio/backend/core/rag/chunking.py
+++ b/studio/backend/core/rag/chunking.py
@@ -36,7 +36,7 @@ def _split(text: str, seps: tuple[str, ...], max_tokens: int, count: TokenCounte
parts = list(text) if sep == "" else text.split(sep)
if len(parts) <= 1:
continue
- if sep: # re-attach the separator
+ if sep:
parts = [p + sep for p in parts[:-1]] + parts[-1:]
out: list[str] = []
for p in parts:
@@ -74,8 +74,8 @@ def _merge(
pt = count(piece)
if buf and buf_tok + pt > max_tokens:
_flush()
- # Bound the carry so carry + this piece fits max_tokens; else a full
- # overlap before a near-max piece overflows the embedder.
+ # Bound the carry so carry + this piece fits max_tokens; else a full overlap before a near-max piece
+ # overflows the embedder.
carry_budget = min(overlap, max(0, max_tokens - pt))
carry, carry_starts, run = [], [], 0
for prev, prev_start in zip(reversed(buf), reversed(buf_starts)):
diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.py
index 169bc5dd010..cac9c4201ff 100644
--- a/studio/backend/core/rag/config.py
+++ b/studio/backend/core/rag/config.py
@@ -10,8 +10,7 @@ import re
DEFAULT_EMBEDDING_MODEL = "unsloth/bge-small-en-v1.5"
EMBEDDING_MODEL = os.environ.get("RAG_EMBEDDING_MODEL", DEFAULT_EMBEDDING_MODEL)
-# Under bge's 512 limit, leaving headroom for the 2 special tokens (else overflow:
-# llama-server 500s, ST truncates). Keep <= embedder_max - ~12.
+# Keep <= embedder_max - ~12: bge's 512 limit plus 2 special tokens (llama-server 500s on overflow, ST truncates).
CHUNK_TOKENS = int(os.environ.get("RAG_CHUNK_TOKENS", "500"))
CHUNK_OVERLAP = int(os.environ.get("RAG_CHUNK_OVERLAP", "64"))
TOP_K_LEXICAL = int(os.environ.get("RAG_TOP_K_LEXICAL", "30"))
@@ -19,80 +18,55 @@ TOP_K_DENSE = int(os.environ.get("RAG_TOP_K_DENSE", "30"))
TOP_K_HYBRID = int(os.environ.get("RAG_TOP_K_HYBRID", "10"))
RRF_K = int(os.environ.get("RAG_RRF_K", "60"))
-# Whole-document context: a thread-attached file under the token budget is injected
-# in full (every chunk, in order) instead of top-K retrieval; above it, use retrieval.
+# Whole-document context: a file under this budget is injected in full instead of top-K retrieval.
THREAD_WHOLE_DOC = os.environ.get("RAG_THREAD_WHOLE_DOC", "1") == "1"
WHOLE_DOC_MAX_TOKENS = int(os.environ.get("RAG_WHOLE_DOC_MAX_TOKENS", "6000"))
-# Conversation archive: turns evicted by the rolling context window go to a per-thread
-# searchable scope and the relevant ones are recalled on the turn that evicted them. Off,
-# evicted turns are simply dropped again, and the recall reserve is not taken. Only applies
-# once the window evicts, which is itself opt-in per request via
-# context_overflow="truncate_oldest".
-#
-# It does NOT turn the rolling window back into what it was before: the compaction headroom
-# and the sticky boundary belong to the window, not to the archive, and have their own knob
-# (ROLLING_COMPACTION_HEADROOM_RATIO). Gating them here instead would make a host without
-# sqlite-vec silently compact differently.
+# Off, evicted turns are simply dropped and the recall reserve is not taken. Only applies once the
+# window evicts, itself opt-in per request via context_overflow="truncate_oldest"; the compaction
+# headroom and sticky boundary belong to the window (ROLLING_COMPACTION_HEADROOM_RATIO), not here.
CONVERSATION_ARCHIVE = os.environ.get("RAG_CONVERSATION_ARCHIVE", "1") == "1"
CONVERSATION_ARCHIVE_TOP_K = int(os.environ.get("RAG_CONVERSATION_ARCHIVE_TOP_K", "4"))
-# Room held back during the fit for the turns recalled straight after it. Sized to
-# CONVERSATION_ARCHIVE_TOP_K * CHUNK_TOKENS with slack for the wrapper text.
+# Sized to CONVERSATION_ARCHIVE_TOP_K * CHUNK_TOKENS with slack for the wrapper text.
CONVERSATION_RECALL_RESERVE_TOKENS = int(
os.environ.get("RAG_CONVERSATION_RECALL_RESERVE_TOKENS", "2048")
)
-# Shape the archive's lexical query: require identifier-like tokens first, drop function
-# words from the fallback. Off restores the plain OR-of-every-token other scopes use.
+# Off restores the plain OR-of-every-token query other scopes use.
CONVERSATION_QUERY_FOCUS = os.environ.get("RAG_CONVERSATION_QUERY_FOCUS", "1") == "1"
-# "chronological" presents recalled turns oldest first, labelled, with later superseding
-# earlier; "relevance" restores the previous rendering. Presentation only: neither changes
-# which turns are selected.
+# Presentation only: neither ordering changes which turns are selected.
CONVERSATION_RECALL_ORDER = os.environ.get("RAG_CONVERSATION_RECALL_ORDER", "chronological")
-# Cosine floor for the automatic recall only, never for a search the model asked for.
-# Default 0.0 (off): a weak match is often still the right turn in one's own conversation.
-# Raise it when the automatic block does more harm than good.
+# Automatic recall only, never a search the model asked for; default 0.0 since a weak match is often
+# still the right turn.
CONVERSATION_FORCED_MIN_SCORE = float(os.environ.get("RAG_CONVERSATION_FORCED_MIN_SCORE", "0.0"))
UPLOAD_EXTS = {".pdf", ".txt", ".md", ".markdown", ".docx", ".html", ".htm"}
-# Reject uploads larger than this, so one pathological file can't drive unbounded parse
-# + vision work at ingest. 0 disables the cap. Default 200 MB.
+# 0 disables the cap; bounds parse + vision work at ingest.
MAX_UPLOAD_BYTES = int(os.environ.get("RAG_MAX_UPLOAD_BYTES", str(200 * 1024 * 1024)))
-# Linked folders use periodic full reconciliation. Metadata-only comparisons keep
-# unchanged passes cheap; caps prevent an accidentally broad folder from becoming
-# an unbounded ingestion queue.
+# Caps prevent an accidentally broad linked folder from becoming an unbounded ingestion queue.
FOLDER_SYNC_INTERVAL_S = float(os.environ.get("RAG_FOLDER_SYNC_INTERVAL_S", "30"))
FOLDER_MAX_FILES = int(os.environ.get("RAG_FOLDER_MAX_FILES", "10000"))
FOLDER_JOB_HISTORY_LIMIT = int(os.environ.get("RAG_FOLDER_JOB_HISTORY_LIMIT", "200"))
-# Extract PDF text as layout-aware Markdown (pymupdf4llm) instead of flat text, so
-# tables, headings and lists survive into chunks and retrieval. Falls back to plain
-# PyMuPDF text when off, when pymupdf4llm is missing, or when extraction fails.
+# Falls back to plain PyMuPDF text when off, when pymupdf4llm is missing, or when extraction fails.
PDF_MARKDOWN = os.environ.get("RAG_PDF_MARKDOWN", "1") == "1"
-# Figure captioning via the loaded vision model: detected figures are transcribed +
-# described so they become searchable. On by default, a no-op without a vision model;
-# the chat's "Describe figures & charts" toggle overrides it per upload.
+# No-op without a vision model; the chat's "Describe figures & charts" toggle overrides it per upload.
CAPTION_IMAGES = os.environ.get("RAG_CAPTION_IMAGES", "1") == "1"
# Total per-document tile budget (figure-bearing pages are tiled, see below).
CAPTION_MAX_IMAGES = int(os.environ.get("RAG_CAPTION_MAX_IMAGES", "24"))
CAPTION_TIMEOUT_S = float(os.environ.get("RAG_CAPTION_TIMEOUT_S", "60"))
-# Larger than a one-line caption since captions transcribe every label. FIGURE_DPI is
-# high enough to keep small box/axis labels legible when tiles are rendered.
+# Captions transcribe every label, and FIGURE_DPI keeps small box/axis labels legible when tiled.
CAPTION_MAX_TOKENS = int(os.environ.get("RAG_CAPTION_MAX_TOKENS", "768"))
FIGURE_DPI = int(os.environ.get("RAG_FIGURE_DPI", "200"))
-# Figure pages are tiled into an overlapping ROWS x COLS grid of high-DPI tiles (plus
-# an optional full page), so small labels and every sub-figure are covered without
-# exact region detection. MAX_PAGES bounds figure pages; MAX_IMAGES bounds total tiles.
+# Overlapping tile grid covers sub-figures and small labels without exact region detection.
FIGURE_TILE_ROWS = int(os.environ.get("RAG_FIGURE_TILE_ROWS", "2"))
FIGURE_TILE_COLS = int(os.environ.get("RAG_FIGURE_TILE_COLS", "2"))
FIGURE_TILE_OVERLAP = float(os.environ.get("RAG_FIGURE_TILE_OVERLAP", "0.12"))
FIGURE_FULLPAGE = os.environ.get("RAG_FIGURE_FULLPAGE", "1") == "1"
CAPTION_MAX_PAGES = int(os.environ.get("RAG_CAPTION_MAX_PAGES", "4"))
-# Scanned-PDF OCR: a page with little extractable text is rendered and transcribed by
-# the vision model so it becomes searchable. Needs a vision model, else skipped (page
-# stays empty). MIN_CHARS is the text length below which a page is treated as scanned.
+# Needs a vision model, else the page stays empty; MIN_CHARS is the text length below which a page counts as scanned.
OCR_SCANNED = os.environ.get("RAG_OCR_SCANNED", "1") == "1"
OCR_MIN_CHARS = int(os.environ.get("RAG_OCR_MIN_CHARS", "16"))
OCR_MAX_PAGES = int(os.environ.get("RAG_OCR_MAX_PAGES", "20"))
@@ -100,15 +74,10 @@ OCR_DPI = int(os.environ.get("RAG_OCR_DPI", "150"))
OCR_TIMEOUT_S = float(os.environ.get("RAG_OCR_TIMEOUT_S", "60"))
OCR_MAX_TOKENS = int(os.environ.get("RAG_OCR_MAX_TOKENS", "2048"))
-# Embedder backend. "auto": sentence-transformers on a CUDA/ROCm GPU (torch fp16
-# wins bulk indexing), else torch-free GGUF llama-server. Switching backends changes
-# the vectors, so the index must be rebuilt.
+# Switching backends changes the vectors, so the index must be rebuilt.
EMBED_BACKEND = os.environ.get("RAG_EMBED_BACKEND", "auto")
-# ``documents.embedding_model`` records the embedder that produced a document's
-# vectors, not just the configured model name. The name alone is not the embedding
-# space: llama-server ignores it and embeds through the GGUF companion, which pools
-# its own way, so one name can mean two spaces on the same machine.
+# The model name alone is not the embedding space: llama-server embeds through the GGUF companion and pools its own way.
EMBEDDING_IDENTITY_TAGS = ("sentence-transformers", "llama-server")
@@ -179,8 +148,7 @@ def _names_gguf(model: str) -> bool:
return "gguf" in re.split(r"[^a-z0-9]+", model.lower())
-# Suffixes unsloth puts on an unquantized re-upload; the GGUF sits on the base
-# name (embeddinggemma-300m-qat-q8_0-unquantized -> embeddinggemma-300m-GGUF).
+# Suffixes unsloth puts on an unquantized re-upload; the GGUF sits on the base name.
_QUANT_SUFFIX_RE = re.compile(r"(?:-qat)?(?:-q\d+_\d+[a-z]*)?-unquantized$", re.I)
@@ -244,27 +212,20 @@ def effective_gguf_repo_for_embedding_model(model: str) -> str:
from utils.embedding_model_settings import get_stored_gguf_repo, remembered_gguf_repo
stored = get_stored_gguf_repo(model)
if stored is None:
- # One stored record, so saving another model takes this one's repo
- # away while a job pinned to it is still ingesting; the derived name
- # would move that job's identity mid-run and split one document set
- # across two tags. The memo is what this process last saw for it.
+ # One stored record, so saving another model would move a pinned job's derived identity mid-run and
+ # split one document set across two tags.
stored = remembered_gguf_repo(model)
except Exception: # noqa: BLE001 - store unavailable: fall back to the convention
stored = None
return stored or gguf_repo_for_embedding_model(model)
-# llama-server backend only. F16 over Q8_0: faster (no per-block dequant for this
-# tiny model) and exact vs fp32, for ~30MB more on disk.
+# F16 over Q8_0: faster (no per-block dequant at this size) and exact, for ~30MB more on disk.
EMBED_GGUF_REPO = os.environ.get("RAG_EMBED_GGUF_REPO", "unsloth/bge-small-en-v1.5-GGUF")
EMBED_GGUF_VARIANT = os.environ.get("RAG_EMBED_GGUF_VARIANT", "F16")
-# Read by BOTH backends, and "auto" means something different to each, because the
-# cost of a GPU is different. llama-server offloads inside its own subprocess, so
-# "auto" there means GPU when there is room. sentence-transformers loads inside the
-# backend process, where the first CUDA allocation pins a primary context for the life
-# of the process (712 MiB on a B200, against 74 MiB of weights), so "auto" there means
-# CPU. Set "gpu" to offload either one; "cpu" keeps both off the GPU.
-EMBED_DEVICE = os.environ.get("RAG_EMBED_DEVICE", "auto") # "auto" | "gpu" | "cpu"
+# "auto" differs per backend: llama-server offloads inside its own subprocess, while
+# sentence-transformers would pin a CUDA primary context (712 MiB on a B200), so it stays on CPU.
+EMBED_DEVICE = os.environ.get("RAG_EMBED_DEVICE", "auto")
def embed_device_preference() -> str:
@@ -297,7 +258,7 @@ def embed_device_requires_gpu() -> bool:
EMBED_HOST = os.environ.get("RAG_EMBED_HOST", "127.0.0.1")
-EMBED_PORT = int(os.environ.get("RAG_EMBED_PORT", "0")) # 0 = auto-pick a free port
+EMBED_PORT = int(os.environ.get("RAG_EMBED_PORT", "0"))
EMBED_BATCH = int(os.environ.get("RAG_EMBED_BATCH", "64"))
EMBED_STARTUP_TIMEOUT_S = float(os.environ.get("RAG_EMBED_STARTUP_TIMEOUT_S", "120"))
EMBED_REQUEST_TIMEOUT_S = float(os.environ.get("RAG_EMBED_REQUEST_TIMEOUT_S", "60"))
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index 9e9103711b1..1ac18958cdd 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -96,9 +96,8 @@ class LlamaServerBackend:
def __init__(self) -> None:
# Lifecycle (spawn/restart/kill) is serialized; HTTP requests are not.
self._lifecycle_lock = threading.Lock()
- # Unload closes the client/process only after every encode/tokenize/probe
- # that already entered has left, and blocks late callers from restarting
- # the backend after it was unpublished.
+ # Unload waits for every encode/tokenize/probe already in flight and blocks late callers from
+ # restarting after unpublish.
self._operation_condition = threading.Condition()
self._active_operations = 0
self._operation_local = threading.local()
@@ -107,24 +106,20 @@ class LlamaServerBackend:
self._port: int | None = None
self._stdout_lines: list[str] = []
self._stdout_thread: threading.Thread | None = None
- # Belongs to whichever model the one subprocess serves, so dim() takes the
- # same _serve_lock the request path does. Reentrant because
- # dim() -> encode() -> _ensure_ready() re-enters it on one thread.
+ # dim() takes the same _serve_lock the request path does; reentrant because dim() -> encode() ->
+ # _ensure_ready() re-enters on one thread.
self._dim: int | None = None
- # One subprocess serves one GGUF, so readiness and the request relying on
- # it must be indivisible: otherwise a swap between them lands A's POST on
- # B's server, storing B's vectors under A's identity. Reentrant because
- # dim() -> encode() -> _post() re-enters on one thread.
+ # One subprocess serves one GGUF, so readiness and the request must be indivisible: a swap between
+ # them lands A's POST on B's server.
self._serve_lock = threading.RLock()
self._model_path: str | None = None
- # Effective GGUF repo the cached path/dim belong to; a Settings change
- # makes it stale, forcing a re-resolve + respawn (see _ensure_ready).
+ # A Settings change makes the cached path/dim stale, forcing a re-resolve and respawn (see _ensure_ready).
self._model_repo: str | None = None
self._binary: str | None = None
self._binary_path_revision: int | None = None
# Sticky after an auto GPU start fails: later spawns stay on CPU.
self._force_cpu = False
- # Pooled client (full URLs per request survive a respawn); trust_env=False skips HTTP(S)_PROXY.
+ # trust_env=False skips HTTP(S)_PROXY; full URLs per request survive a respawn.
self._client = httpx.Client(timeout = config.EMBED_REQUEST_TIMEOUT_S, trust_env = False)
atexit.register(self._shutdown)
@@ -168,8 +163,7 @@ class LlamaServerBackend:
"llama-server. Install llama.cpp or set LLAMA_SERVER_PATH / "
"UNSLOTH_LLAMA_CPP_PATH."
)
- # Before the probe, so the --help check and the spawn both run the real
- # executable rather than an entrypoint that loses DYLD_* to SIP.
+ # Resolve the real executable before the probe: an entrypoint loses DYLD_* to SIP.
binary = _resolve_entrypoint(binary)
self._assert_embedding_support(binary)
self._binary = binary
@@ -230,30 +224,25 @@ class LlamaServerBackend:
from utils.models.model_config import _local_gguf_load_path, colocated_split_shards
from utils.paths import normalize_path
- # Same normalization the sentence-transformers and settings probes already
- # apply, so a drive-letter path is recognized here too. Without it a WSL
- # user's `C:\models\...` GGUF resolved as a filename that cannot exist and
- # went to the Hub, which then reported no weights in `C:\models\...-GGUF`.
+ # Normalize like the sentence-transformers and settings probes, else a WSL user's C:\models\... GGUF
+ # resolves as an impossible filename and goes to the Hub.
p = Path(normalize_path(model)).expanduser()
if p.is_file() and p.suffix.lower() == ".gguf":
return None if is_appledouble_metadata(p) else str(p)
if p.is_dir():
files = [f for f in p.iterdir() if not is_appledouble_metadata(f)]
- # Same pick as a hub listing gets, so a directory and the repo it was
- # downloaded from cannot disagree about which quant the embedder opens.
- # Its name tiebreak is what makes a directory of split shards resolve to
- # shard 1 rather than to whichever one `iterdir` happened to yield first.
+ # Same pick as a hub listing gets, and its name tiebreak resolves a directory of split shards to shard 1.
remaining = list(files)
while remaining:
picked = LlamaServerBackend._pick_gguf(remaining, key = lambda f: f.name)
if picked is None:
break
- # llama-server opens the siblings implicitly, so a torn family is
- # not a usable answer even though shard 1 is sitting right there.
+ # llama-server opens the siblings implicitly, so a torn family is not a usable answer even with
+ # shard 1 present.
if colocated_split_shards(picked)[1]:
- # Enters the family at shard 1 and keeps a complete symlink set
- # intact, the same way the chat loader resolves a local GGUF.
+ # Enters the family at shard 1 and keeps a complete symlink set intact, the same way the
+ # chat loader resolves a local GGUF.
return str(_local_gguf_load_path(picked))
remaining = [f for f in remaining if f != picked]
raise RuntimeError(f"no .gguf file found in local model dir {model!r}")
@@ -334,9 +323,8 @@ class LlamaServerBackend:
for n in names
if name_of(n).lower().endswith(".gguf")
and "mmproj" not in name_of(n).lower()
- # A drafter is a companion, never a model in its own right, so it must be
- # excluded everywhere mmproj is. A cache holding only the companion would
- # otherwise be read as holding the embedder.
+ # A drafter is a companion, so exclude it everywhere mmproj is: a cache holding only the companion
+ # would read as holding the embedder.
and not _is_mtp_drafter(name_of(n))
]
if not usable:
@@ -347,8 +335,7 @@ class LlamaServerBackend:
if require_variant:
return None
match = usable
- # Name breaks a length tie: a directory scan yields no guaranteed order, and among
- # equal-length shard names that would otherwise decide which shard is picked.
+ # Name breaks a length tie: a directory scan yields no guaranteed order.
return sorted(match, key = lambda n: (len(name_of(n)), name_of(n)))[0]
@staticmethod
@@ -366,14 +353,14 @@ class LlamaServerBackend:
from utils.hf_cache_settings import active_hf_hub_cache
from utils.paths import resolve_cached_repo_id_case
- # A repo id typed with different casing than the cache folder is still that repo;
- # missing it here would re-download, or fail outright with the hub unreachable.
+ # A repo id typed with different casing than the cache folder is still that repo; missing it re-
+ # downloads or fails outright offline.
cased = resolve_cached_repo_id_case(repo_id)
repo_dir = Path(active_hf_hub_cache()) / f"models--{cased.replace('/', '--')}"
try:
rev = (repo_dir / "refs" / "main").read_text(encoding = "utf-8").strip()
except (OSError, ValueError):
- return None # not cached here, unreadable, or pinned to a commit
+ return None
# A ref file holds a bare commit hash; a separator would join out of the cache.
if not rev or rev in (".", "..") or "/" in rev or "\\" in rev:
return None
@@ -393,21 +380,16 @@ class LlamaServerBackend:
if snapshot is None:
return None
try:
- # Not a "*.gguf" glob: that is case-sensitive, and quant-per-directory layouts
- # put the file a level down, both of which the hub listing handles.
+ # Not a "*.gguf" glob: that is case-sensitive and misses quant-per-directory layouts, both of which
+ # the hub listing handles.
- # A complete family of sidecars is servable by every test below -- whole, quant
- # matched, opens as a file -- so the retry that drops a torn real set lands on it.
files = [
p for p in snapshot.rglob("*") if p.is_file() and not is_appledouble_metadata(p)
]
except OSError:
return None
- # llama-server opens sibling shards implicitly, so a split set is servable only when
- # it is whole. An unservable winner is dropped and the pick retried rather than
- # failing the lookup, or an incomplete family would shadow a complete one simply by
- # sorting ahead of it. Verified per winner, not per file: a plain file is a complete
- # one-file set, and a whole-cache filter would walk siblings for every shard present.
+ # llama-server opens sibling shards implicitly, so a split set is servable only when whole; drop
+ # an unservable winner and retry, else an incomplete family shadows a complete one.
while files:
pick = LlamaServerBackend._pick_gguf(
files,
@@ -430,20 +412,16 @@ class LlamaServerBackend:
``model_name`` is the model the caller pinned; the live setting would
resolve B's weights for a job still tagging its vectors as A."""
model = model_name or config.effective_embedding_model()
- # Captured once: if the setting changes mid-download, the path must stay
- # tagged with the repo it was resolved FOR, so _current() sees the new
- # setting as stale and respawns instead of serving the old model.
+ # Captured once: the path must stay tagged with the repo it was resolved FOR, so a mid-download
+ # setting change reads as stale and respawns.
desired = config.effective_gguf_repo_for_embedding_model(model)
if self._model_path is not None and self._model_repo == desired:
return self._model_path
local = self._resolve_local_gguf(model)
if local is not None:
return self._adopt_model_path(local, desired)
- # The planned family whenever it is on disk, ahead of the variant lookup.
- # Identity carries the model and repo but not the file, so a preferred
- # variant arriving later would otherwise change the weights under an
- # existing index without changing its tag. Consulted after completion too,
- # which is what makes the record an identity, not a transfer receipt.
+ # Identity carries the model and repo but not the file, so a preferred variant arriving later would
+ # change the weights under an existing index without changing its tag.
planned = self._planned_family_path(model, desired)
if planned is not None:
try:
@@ -455,15 +433,12 @@ class LlamaServerBackend:
# Companion repo, then the naming variants, then the model repo itself.
repo = desired
candidates = list(dict.fromkeys([repo, *config.gguf_repo_candidates(model)]))
- # Only the preferred repo. A file cached under the fallback candidate must not
- # pre-empt a companion repo the hub can still resolve, which is the order the
- # listing follows; offline, the relaxed pass below still reaches both.
+ # Only the preferred repo: a file cached under the fallback must not pre-empt a companion repo the
+ # hub can still resolve.
cached = self._resolve_cached_gguf(desired)
if cached is not None:
- # The configured variant is present in the preferred repo, which is
- # what the picker advertised, so any pending marker has been answered.
- # Retire it here or the model stays cache-only for the life of the
- # install and a later eviction reads as "never downloaded".
+ # Retire the pending marker here, or the model stays cache-only for the life of the install and a
+ # later eviction reads as "never downloaded".
try:
from utils.embedding_model_settings import clear_stored_download_pending
clear_stored_download_pending(model)
@@ -476,29 +451,22 @@ class LlamaServerBackend:
except Exception: # noqa: BLE001 - old/unavailable settings store
download_pending = False
if download_pending:
- # A published repo may not carry the configured quant; the picker
- # deliberately downloaded the complete family the Hub resolver
- # selected, so accept any complete cached quant here. Never fall
- # through to the online loader while the explicit transfer is
- # pending/cancelled.
+ # A published repo may not carry the configured quant, so accept any complete cached quant; never
+ # fall through to the online loader while the transfer is pending or cancelled.
cached = self._resolve_cached_gguf(desired, require_variant = False)
if cached is not None:
- # Reaching here means the configured variant is NOT cached (the
- # strict pass returned if it were), which happens two ways that
- # need opposite answers.
+ # Reaching here means the configured variant is NOT cached (the strict pass returned if it
+ # were), which happens two ways that need opposite answers.
if self._is_planned_family(model, desired, cached):
- # The repo publishes no such variant, so this quant is what
- # the picker advertised and fetched. Retire the marker, or the
- # model stays cache-only and a later eviction refuses to index
- # something that did finish downloading.
+ # The repo publishes no such variant, so this quant is what the picker fetched: retire the marker
+ # here too.
try:
from utils.embedding_model_settings import clear_stored_download_pending
clear_stored_download_pending(model)
except Exception: # noqa: BLE001 - a settings write must not fail a load
pass
- # Otherwise a quant left by an earlier setting, or a record too old
- # to say. Serve it rather than fail the index, but leave the marker:
- # retiring on a stand-in would pin this model to the wrong quant.
+ # Serve a stand-in rather than fail the index, but leave the marker: retiring on it would pin this
+ # model to the wrong quant.
return self._adopt_model_path(cached, desired)
raise RuntimeError(
f"Embedding model {model!r} is not downloaded yet. "
@@ -533,8 +501,7 @@ class LlamaServerBackend:
if not path.is_file():
return None
paths.append(path)
- # Ordered shard 1..N by the planner, so the first is where llama-server
- # enters the family.
+ # Ordered shard 1..N by the planner, so the first is where llama-server enters the family.
return str(paths[0]) if paths else None
except Exception: # noqa: BLE001 - an unreadable record answers nothing
return None
@@ -611,8 +578,8 @@ class LlamaServerBackend:
break
errors.append(f"{candidate!r}: no .gguf files")
if filename is None:
- # Any cached GGUF now beats no embedder. Only where the hub failed to NAME
- # one: a transfer failing on its own (no disk, bad checksum) must surface.
+ # Only where the hub failed to NAME a file: a transfer failing on its own (no disk, bad checksum)
+ # must surface.
for candidate in candidates:
cached = self._resolve_cached_gguf(candidate, require_variant = False)
if cached is not None:
@@ -629,9 +596,8 @@ class LlamaServerBackend:
from utils.hf_cache_settings import active_hf_hub_cache
cache_dir = active_hf_hub_cache()
- # Fetch the whole split family, not just the shard that was picked:
- # `-m shard1` makes llama-server open the siblings itself, so a
- # one-file transfer fails at startup on a repo it just downloaded.
+ # Fetch the whole split family, not just the picked shard: -m shard1 makes llama-server open the
+ # siblings itself.
downloaded = None
for name in family:
fetched = hf_hub_download(
@@ -658,7 +624,7 @@ class LlamaServerBackend:
return True
if dev == "cpu" or self._force_cpu:
return False
- return self._gpu_available() # auto
+ return self._gpu_available()
@staticmethod
def _gpu_available() -> bool:
@@ -670,7 +636,7 @@ class LlamaServerBackend:
from utils.hardware import is_apple_silicon
if is_apple_silicon():
- return True # bundled mac build offloads to Metal
+ return True
from core.inference.llama_cpp import LlamaCppBackend
# [(idx, free_mib)], honors CVD
@@ -691,8 +657,8 @@ class LlamaServerBackend:
return LlamaCppBackend._arch_gate_survivors(binary)
def _build_cmd(self, binary: str, model_path: str, port: int, *, use_gpu: bool) -> list[str]:
- # No --embd-normalize (not in every build; we normalize in Python to match
- # the ST path). --fit off: don't auto-resize ctx/offload to device memory.
+ # No --embd-normalize (absent in some builds; we normalize in Python to match the ST path), and
+ # --fit off so ctx/offload are not auto-resized.
cmd = [
binary,
"-m",
@@ -713,46 +679,36 @@ class LlamaServerBackend:
def _build_env(self, binary: str, *, use_gpu: bool) -> dict[str, str]:
env = child_env_without_native_path_secret()
- env["LLAMA_SET_ROWS"] = "1" # ggml set_rows fast path
+ env["LLAMA_SET_ROWS"] = "1"
if sys.platform == "darwin":
- # _llama_lib_dir, not Path(binary).parent: the managed install puts
- # an entrypoint in front of the real server, and the dylibs sit
- # next to the target, not next to the wrapper. Unconditional,
- # unlike the CUDA branch below: a CPU start loads the same sibling
- # dylibs (#8566).
+ # _llama_lib_dir, not Path(binary).parent: the managed install puts an entrypoint in front of the
+ # real server and the dylibs sit next to the target (#8566).
env = _with_dyld_path(env, _binary_lib_dir(binary))
elif use_gpu:
- # Path(binary).parent, unchanged. Resolving the entrypoint here too
- # would be more correct in principle, but it moves the first
- # LD_LIBRARY_PATH entry for every existing Linux GPU install whose
- # llama-server is a symlink, and this change is not about them.
+ # Left as Path(binary).parent: resolving the entrypoint here would move the first LD_LIBRARY_PATH
+ # entry for every existing Linux GPU install whose llama-server is a symlink.
self._add_linux_cuda_libs(env, str(Path(binary).parent))
_pinned = self._arch_gated_gpu_ids(binary)
if _pinned:
from core.inference.llama_cpp import LlamaCppBackend
- # prefer_rocr: a HIP-only mask still lets HSA enumerate (and die
- # on) the unsupported agent; ROCR drops it at the driver layer.
+ # prefer_rocr: a HIP-only mask still lets HSA enumerate, and die on, the unsupported agent.
LlamaCppBackend._emit_child_gpu_visibility(
env, ",".join(str(i) for i in _pinned), prefer_rocr = True
)
logger.info("pinning the embed server to arch-supported GPU(s) %s", _pinned)
- # Not else: the darwin branch above is about dylibs, so a macOS CPU start
- # still has to blank the devices, exactly as it did before that branch.
+ # Not else: a macOS CPU start still has to blank the devices.
if not use_gpu:
# Blank devices so a CUDA build stays on CPU and reserves no VRAM.
env["CUDA_VISIBLE_DEVICES"] = ""
- # HIP reads CUDA_VISIBLE_DEVICES only when HIP_VISIBLE_DEVICES is unset,
- # so an inherited HIP mask would keep a device (and the ~0.5 GB context it
- # costs) visible to a child we just put on the CPU. Same "-1" sentinel the
- # chat forced-CPU path uses. An inherited ROCR mask is left alone: it hides
- # agents below HIP, so clearing it would expose MORE of them to the HSA
- # enumeration that dies on an uncovered arch (#7624).
+ # HIP reads CUDA_VISIBLE_DEVICES only when HIP_VISIBLE_DEVICES is unset, so an inherited HIP mask
+ # keeps a device (and the ~0.5 GB context it costs) visible to a child we just put on the CPU; an
+ # inherited ROCR mask is left alone, since clearing it exposes MORE agents to the HSA enumeration
+ # that dies on an uncovered arch (#7624).
env["HIP_VISIBLE_DEVICES"] = "-1"
- # LLAMA_ARG_DEVICE / LLAMA_ARG_MAIN_GPU are the env spelling of --device /
- # --main-gpu. Hiding every device while leaving that pick in place makes
- # llama.cpp reject the name that no longer enumerates and exit instead of
- # running on the CPU. Same clear the chat sentinel makes.
+ # LLAMA_ARG_DEVICE / LLAMA_ARG_MAIN_GPU are the env spelling of --device / --main-gpu: hiding
+ # every device while leaving that pick in place makes llama.cpp reject the name that no longer
+ # enumerates and exit instead of running on the CPU.
from core.inference.llama_cpp import LlamaCppBackend
LlamaCppBackend._clear_device_placement_env(env)
@@ -765,7 +721,7 @@ class LlamaServerBackend:
import platform
if sys.platform == "win32":
- return # Windows resolves CUDA via PATH in the inherited env.
+ return
arch = platform.machine()
lib_dirs = [binary_dir]
for pattern in (
@@ -840,8 +796,8 @@ class LlamaServerBackend:
**child_popen_kwargs(),
)
self._process = proc
- # Long-lived, and child_popen_kwargs() is empty on macOS, so the crash
- # record is the only thing that can reap it after a force quit.
+ # child_popen_kwargs() is empty on macOS, so the crash record is the only thing that can reap it
+ # after a force quit.
adopt_pid(proc.pid)
self._port = port
self._stdout_thread = threading.Thread(
@@ -946,8 +902,7 @@ class LlamaServerBackend:
except Exception as e: # noqa: BLE001
logger.warning("error killing llama-server embedder: %s", e)
finally:
- # Only once it is confirmed gone: a survivor must stay recorded so
- # the next startup sweep can reap it.
+ # A survivor must stay recorded so the next startup sweep can reap it.
if proc.poll() is not None:
forget_pid(proc.pid)
self._process = None
@@ -968,9 +923,7 @@ class LlamaServerBackend:
self._client.close()
except Exception: # noqa: BLE001
pass
- # Drop the interpreter-exit hook with the backend it belonged to.
- # Unload/rebuild is an ordinary cycle here (Settings exposes it as a
- # button), and a registry that only ever grows pins every retired
+ # Unload/rebuild is an ordinary cycle here, and a registry that only grows pins every retired
# backend, and its captured stdout tail, for the life of the process.
try:
atexit.unregister(self._shutdown)
@@ -998,8 +951,7 @@ class LlamaServerBackend:
last_exc: Exception | None = None
for attempt in range(2):
with self._serve_lock:
- # Held across readiness AND the request, so a swap can only land
- # between whole requests.
+ # Held across readiness AND the request, so a swap can only land between whole requests.
self._ensure_ready(model_name)
try:
resp = self._client.post(f"{self._base_url}{path}", json = payload)
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index be3a409c807..7473f71958f 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -37,20 +37,17 @@ from . import config
logger = logging.getLogger(__name__)
-# "false" silences the fast tokenizer's fork warning; encode() flips it to "true"
-# only during a batch tokenize (rayon speedup), then restores it.
+# "false" silences the fast tokenizer's fork warning; encode() flips it only during a batch tokenize.
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
_lock = threading.Lock()
-# Serializes encode/tokenize (HF fast tokenizer isn't thread-safe). Separate from
-# _lock so a long encode never blocks a reload.
+# Serializes encode/tokenize (the HF fast tokenizer is not thread-safe); separate from _lock so a
+# long encode never blocks a reload.
_compute_lock = threading.Lock()
_model = None
_name: str | None = None
-# The backend that served this thread's most recent encode. The process embedder can
-# swap between an encode returning and the caller asking what produced the vectors,
-# and the answer has to be the backend that was actually used. See
-# ``encode_with_identity``.
+# The process embedder can swap between an encode returning and the caller asking, so the answer
+# must be the backend actually used. See encode_with_identity.
_served_by = threading.local()
@@ -81,8 +78,7 @@ def _device() -> str:
"""
if config.embed_device_preference() != "gpu":
return "cpu"
- # Still a table lookup, so asking for a GPU on a host without one, or on Apple
- # where this backend has no torch device, lands on CPU rather than on a device
+ # Still a table lookup: asking for a GPU on a host without one lands on CPU rather than on a device
# string torch cannot open.
return _TORCH_DEVICE.get(get_device(), "cpu")
@@ -118,10 +114,8 @@ def _load_device() -> str:
_torchao_stub_done = False
-# Guards the stub's one-shot state and the import that must follow it. Its own
-# lock, not ``_lock``: that one is held across a whole model construction, so
-# borrowing it made a preflight probe wait out someone else's download. Taken
-# innermost, so there is no ordering to get wrong.
+# Its own lock, not _lock: that one is held across a whole model construction, so borrowing it made
+# a preflight probe wait out someone else's download.
_stub_lock = threading.Lock()
@@ -221,8 +215,8 @@ def _guard_model_security(name: str, local_only: bool = False) -> None:
if local_only:
load_subdirs = ()
else:
- # Union audio-model load roots with ST module dirs so a flagged pickle under a
- # Transformer module dir blocks instead of passing as an unreferenced nested shard.
+ # Union the load roots so a flagged pickle under a Transformer module dir blocks instead of passing
+ # as an unreferenced nested shard.
load_subdirs = tuple(
dict.fromkeys(
(*security_load_subdirs(name, token), *_st_module_subdirs(name, token))
@@ -264,18 +258,14 @@ class _CaptureLoadReport(logging.Filter):
"""
_SERIOUS = ("MISSING", "MISMATCH", "CONVERSION")
- # The only UNEXPECTED key worth downgrading is the legacy buffer every BERT-era
- # sentence-transformer ships. Any other discarded weight can genuinely change
- # retrieval quality, so it stays a warning. Matched on the whole key, not as a
- # substring: "encoder.position_ids_projection.weight" is a real discarded weight.
+ # Only the legacy BERT-era buffer is downgraded, matched on the whole key: any other discarded
+ # weight can genuinely change retrieval quality.
_KNOWN_BENIGN_UNEXPECTED = "embeddings.position_ids"
def __init__(self) -> None:
super().__init__()
self.reports: list[str] = []
- # These filters sit on process-global loggers, so a concurrent load on another
- # thread would otherwise have its report swallowed and attributed here. Only
- # capture what the thread that opened the context emits.
+ # These filters sit on process-global loggers, so capture only what the thread that opened the context emits.
self.thread_id = threading.get_ident()
def filter(self, record: logging.LogRecord) -> bool:
@@ -294,10 +284,8 @@ class _CaptureLoadReport(logging.Filter):
for report in self.reports:
if any(tag in report for tag in self._SERIOUS):
return True
- # Row by row over the table only. transformers appends a "Notes:" section
- # explaining each status ("- UNEXPECTED: can be ignored when loading from
- # a different task/architecture"), and reading that as a key row would make
- # every unexpected report serious, including the benign one.
+ # Row by row over the table only: transformers appends a "Notes:" section explaining each status
+ # ("- UNEXPECTED: can be ignored ..."), whose lines would read as serious key rows.
table = report.split("Notes:", 1)[0]
for row in table.splitlines():
if "UNEXPECTED" not in row:
@@ -313,8 +301,8 @@ class _CaptureLoadReport(logging.Filter):
_LOAD_REPORT_LOGGERS = (
"transformers.utils.loading_report",
"transformers.modeling_utils",
- # An adapter-backed embedding model reports through the PEFT integration's own
- # logger, which is not a descendant of either of the above.
+ # An adapter-backed embedding model reports through the PEFT integration's own logger, not a
+ # descendant of either above.
"transformers.integrations.peft",
)
@@ -334,15 +322,10 @@ def _quiet_transformers_load():
log.addFilter(capture)
attached.append(log)
- # The weight-load bar is transformers.utils.logging.tqdm, so disable_progress_bar()
- # reaches it. The "is it on right now" probe has been spelled both ways across
- # versions (is_progress_bar_enabled in 4.x/5.x, are_progress_bars_disabled in
- # some builds), so accept either and skip the restore when neither exists rather
- # than re-enabling a bar the caller had deliberately turned off.
- # transformers' enable_progress_bar() also calls the Hub's enable_progress_bars(),
- # so restoring the transformers flag would clobber a Hub-only disable that someone
- # else installed (unsloth does exactly that in patch_ipykernel_hf_xet for the
- # broken hf-xet/ipykernel pair). Snapshot the Hub state separately and put it back.
+ # The "is it enabled" probe is spelled both ways across transformers versions, so accept either and
+ # skip the restore when neither exists.
+ # transformers' enable_progress_bar() also calls the Hub's, so snapshot and restore the Hub state
+ # separately or a Hub-only disable is clobbered.
hub_bars_off = None
try:
from huggingface_hub.utils import are_progress_bars_disabled
@@ -423,19 +406,15 @@ def _get(model_name: str | None = None):
accelerator for a ~1.5x speedup at negligible accuracy loss, fp32 on CPU."""
global _model, _name
name = model_name or config.effective_embedding_model()
- # Capture offline state once so the gate and the load agree (no window where the gate is
- # skipped as offline but the constructor then reaches the network).
+ # Capture offline state once so the gate and the load agree.
try:
from utils.embedding_model_settings import get_stored_download_pending
download_pending = get_stored_download_pending(name)
except Exception: # noqa: BLE001 - old/unavailable settings store
download_pending = False
offline = hf_env_offline()
- # Two different questions. `local_only` says "load from cache, do not fetch",
- # which a pending transfer also demands. `offline` is whether the Hub is
- # reachable, which is what the security gate needs: told offline while online
- # it applies the fail-closed cached-pickle rule and rejects a .bin-only repo
- # the resolver just accepted and scanned, failing the first index outright.
+ # local_only means "load from cache"; offline is what the security gate needs, and claiming offline
+ # while online rejects a .bin-only repo the resolver just scanned.
local_only = offline or download_pending
with _lock:
if _model is None or _name != name:
@@ -449,82 +428,60 @@ def _get(model_name: str | None = None):
st_kwargs = dict(
device = device,
cache_folder = active_hf_hub_cache(),
- # Keyed on the device we actually load on, not on whether we degraded
- # onto it. fp16 BERT on CPU is slow where it works at all, and several
- # ops raise "not implemented for 'Half'" -- which _SentenceTransformers
- # Backend.encode() catches and answers by swapping the whole process to
- # llama-server, so getting this wrong fails as a silent backend change
- # rather than as an error.
+ # Keyed on the device we load on: fp16 BERT on CPU raises "not implemented for Half", which encode()
+ # answers by swapping the whole process to llama-server.
model_kwargs = dtype_kwargs("float32" if device == "cpu" else "float16"),
)
load_target = name
from utils.paths import is_local_path
from utils.utils import cached_st_source, hf_cache_snapshot_dir
- # The repo AND the directory that supplied the weights, together: a
- # slashless name can match under sentence-transformers/ while a stale
- # literal cache entry is what a second lookup would return, and ST
- # weights alone are satisfied by the first finalized shard of a
- # transfer still in flight.
- #
- # Repo ids only. A local path IS the artifact the resolver accepted,
- # and the picker takes a slashless relative directory, so a folder
- # named all-MiniLM-L6-v2 beside a cached sentence-transformers/ copy
- # would load the Hub's weights under the local path's identity.
+ # The repo AND the directory that supplied the weights, together: ST weights alone are satisfied by
+ # the first finalized shard of a transfer still in flight.
+ # Repo ids only: a local folder named all-MiniLM-L6-v2 would otherwise load the Hub's weights under
+ # the local path's identity.
st_source = None if is_local_path(name) else cached_st_source(name)
if not local_only and st_source is not None:
- # Settings reported this model on-device off the same predicate, so
- # handing SentenceTransformer the repo id here is what lets it reach
- # the Hub for a revision published since, and fetch it during the
- # first index. That is the invisible transfer the picker exists to
- # replace, and it changes the vectors without changing the identity
- # they are tagged with. Load the snapshot that was called cached.
+ # Load the snapshot that was called cached: the repo id lets ST reach the Hub for a newer revision
+ # during the first index, changing the vectors without changing their identity.
load_target = str(st_source[1])
if local_only:
- # ST-specific AND complete: a hybrid repo whose GGUF is cached, or a
- # transfer that has finalized only its first shard, would otherwise
- # retire the marker and be handed to SentenceTransformer.
+ # ST-specific AND complete: a hybrid repo's cached GGUF, or a transfer that finalized only its first
+ # shard, would otherwise retire the marker.
if download_pending and st_source is None:
+ # Defensive: a loadable check and snapshot lookup share no lock, so eviction between them is
+ # still a pending model.
raise EmbeddingModelDownloadRequiredError(
f"Embedding model {name!r} is not downloaded yet. "
"Finish its Settings download before indexing documents."
)
snapshot = st_source[1] if st_source else hf_cache_snapshot_dir(name)
if snapshot is not None:
- # Load from the local snapshot dir: a local path never touches the Hub, so
- # this is offline-safe on ANY sentence-transformers version (even ones
- # predating local_files_only).
+ # A local path never touches the Hub, so this is offline-safe on ANY sentence-transformers
+ # version, even ones predating local_files_only.
load_target = str(snapshot)
elif download_pending:
- # Defensive: a loadable check and snapshot lookup share no
- # lock, so eviction between them is still a pending model.
raise EmbeddingModelDownloadRequiredError(
f"Embedding model {name!r} is not downloaded yet. "
"Finish its Settings download before indexing documents."
)
elif _st_accepts_local_files_only(SentenceTransformer):
st_kwargs["local_files_only"] = True
- # After load_target is settled, so this scans what is about to be
- # deserialized: on the repo id it checked the Hub's current commit while
- # the load opened an older cached one. evaluate_file_security recovers
- # the repo and exact commit from a snapshot path, and fails closed
- # locally when the Hub cannot answer.
+ # Scan after load_target is settled: on the repo id it checked the Hub's current commit while the
+ # load opened an older cached one. evaluate_file_security recovers the repo and exact commit from
+ # a snapshot path.
_guard_model_security(load_target, offline)
with _quiet_transformers_load() as report:
- # The re-emit runs in finally: a load that raises after transformers
- # wrote its report is exactly when a MISSING or MISMATCH line matters,
- # and letting the exception skip the loop would swallow it.
+ # Re-emit in finally: a load that raises after transformers wrote its report is exactly when a
+ # MISSING or MISMATCH line matters.
try:
_model = SentenceTransformer(load_target, **st_kwargs)
finally:
_emit_load_reports(report)
_name = name
if download_pending:
- # Only once the model is actually constructed. Retiring it before
- # the constructor let an unsupported architecture or a device that
- # cannot initialize fall through to llama-server with no marker
- # left, so the fallback was free to fetch the GGUF companion
- # during the first index: the unapproved transfer this replaces.
+ # Retire only once the model is constructed: retiring earlier let a failed construction fall through
+ # to llama-server with no marker, freeing the fallback to fetch the GGUF companion.
try:
from utils.embedding_model_settings import clear_stored_download_pending
clear_stored_download_pending(name)
@@ -558,9 +515,8 @@ def _st_encode(
"""ST encode -> (N, dim) float32. Serialized (fast-tokenizer borrow check),
under inference_mode when torch is present, with rayon enabled for the call."""
with _compute_lock:
- # Admission and model lookup are one lease. If lookup happened first,
- # unload could clear the globals and return while this call retained a
- # strong local reference and had not begun inference yet.
+ # Admission and model lookup are one lease: lookup first would let unload clear the globals while
+ # this call still held a strong reference.
model = _get(model_name)
os.environ["TOKENIZERS_PARALLELISM"] = "true"
try:
@@ -630,10 +586,10 @@ class _SentenceTransformersBackend:
try:
return _st_encode(texts, model_name = model_name, normalize = normalize)
except (UnsafeEmbeddingModelError, EmbeddingModelDownloadRequiredError):
- raise # policy failures must hard-fail, not change backend/download
+ raise
except Exception as st_err: # noqa: BLE001 - runtime ST/CUDA encode failure
- # ST loaded but this encode blew up; swap the process to the llama-server
- # embedder (so later encodes stay in one space) and retry.
+ # ST loaded but this encode failed: swap the process to llama-server so later encodes stay in one
+ # space, then retry.
fallback = _switch_to_llama_fallback(st_err, model_name)
if fallback is None:
raise
@@ -653,16 +609,10 @@ class _SentenceTransformersBackend:
_backend_lock = threading.Lock()
_backend = None
_backend_key: str | None = None
-# Per model: an ST runtime/load failure pins llama-server for THAT model, so the
-# swap survives the next rebuild while a later safetensors-only selection can
-# still build ST. Keyed by model because one (key, model) pair let a second
-# failing model erase the first one's pin and send a still-running job back to ST.
-# Cleared only by a reset or an explicit unload, both of which are a fresh start.
-#
-# Read WITHOUT _backend_lock throughout: _get_backend holds that lock across a
-# whole model load, and every reader here is a probe the resolver's own budget is
-# supposed to bound. dict.get is atomic, and a pin that lands mid-probe is
-# answered on the next call.
+# Per model, keyed by model: one (key, model) pair let a second failing model erase the first one's
+# pin and send a running job back to ST.
+# Read WITHOUT _backend_lock: _get_backend holds it across a whole model load; dict.get is atomic
+# and a pin landing mid-probe is answered on the next call.
_forced_backends: dict[str, str] = {}
_ST_ALIASES = frozenset({"sentence-transformers", "sentence_transformers", "st"})
@@ -680,10 +630,8 @@ def _resolve_auto() -> str:
is installed."""
from core.inference.llama_cpp import LlamaCppBackend
- # Unfiltered probe on purpose: the winner here runs under PyTorch, so the
- # ROCm arch gate (which asks what the installed llama.cpp prebuilt was built
- # for, #7624) must not apply. A device that prebuilt lacks kernels for is
- # usually still a perfectly good sentence-transformers device.
+ # Unfiltered probe: the winner runs under PyTorch, so the ROCm arch gate for the installed llama.cpp
+ # prebuilt (#7624) must not apply.
if LlamaCppBackend._get_gpu_free_memory():
return "sentence-transformers"
if LlamaCppBackend._find_llama_server_binary():
@@ -722,16 +670,14 @@ def _model_names_gguf_repo(model: str | None) -> bool:
try:
from utils.paths import is_local_path
- # A directory may be named anything: `~/models/my-gguf` holding safetensors
- # is a sentence-transformers model, and only the filesystem can say so.
- # _model_is_local_gguf has already answered for every local id.
+ # A directory may be named anything, so only the filesystem can say that ~/models/my-gguf holding
+ # safetensors is a sentence-transformers model.
if is_local_path(model):
return False
except Exception: # noqa: BLE001 - unparseable path is not a repo id either
return False
- # config's predicate, not a second opinion: gguf_repo_candidates already
- # counts "gguf" as a whole name segment, so owner/GGUF-model is a GGUF repo
- # there and must not be a sentence-transformers one here.
+ # config's predicate, not a second opinion: gguf_repo_candidates already counts "gguf" as a whole
+ # name segment, so owner/GGUF-model is a GGUF repo there and must not be one here.
return config._names_gguf(model.strip().rstrip("/").rsplit("/", 1)[-1])
@@ -742,9 +688,8 @@ def _resolve_auto_for_model(model_name: str | None = None) -> str:
records that choice; the hardware default would send it to llama-server,
which has nothing to open."""
model = model_name or config.effective_embedding_model()
- # Ahead of the stored record: the filesystem was asked, not guessed at, and a
- # stored ST record for a .gguf can only come from a force-save that failed.
- # Only ``auto`` consults this, so an explicit RAG_EMBED_BACKEND still wins.
+ # Ahead of the stored record, since the filesystem was asked rather than guessed at; only auto
+ # consults this, so an explicit RAG_EMBED_BACKEND still wins.
if _model_is_local_gguf(model):
return "llama-server"
try:
@@ -756,13 +701,8 @@ def _resolve_auto_for_model(model_name: str | None = None) -> str:
key = stored.strip().lower()
if key in _ST_ALIASES or key in _LLAMA_ALIASES:
return key
- # Nothing validated, so read the name. _resolve_auto answers
- # "sentence-transformers" on any GPU host, and a GGUF repo resolved as ST is
- # rejected for publishing no safetensors, saveable only over that error --
- # which leaves it pending, and _get() refuses a pending ST model before the
- # llama fallback can run. Below the stored record, though, since a name is
- # only a guess: a repo with a torn GGUF family and usable safetensors has a
- # validated ST plan that overruling would fail as "not downloaded".
+ # Below the stored record, since a name is only a guess: a repo with a torn GGUF family and usable
+ # safetensors has a validated ST plan.
if _model_names_gguf_repo(model):
return "llama-server"
return _resolve_auto()
@@ -779,12 +719,8 @@ def sentence_transformers_runtime_available() -> bool:
"""
try:
_load_device()
- # Not under ``_lock``. Both the resolve GET and the PUT run this before any
- # deadline-bounded Hub call, and ``_get`` holds ``_lock`` across an entire
- # SentenceTransformer construction, download included: sharing it meant
- # opening or saving Settings during a slow first load blocked for as long
- # as that load took, past every deadline the resolver applies to itself.
- # The stub keeps its ordering guarantee under its own lock.
+ # Not under _lock: _get holds it across an entire SentenceTransformer construction, download
+ # included, so sharing it blocked Settings for the length of a slow first load.
_install_torchao_stub_once()
from sentence_transformers import SentenceTransformer
@@ -809,10 +745,8 @@ def resolved_backend_for_model(model_name: str) -> str:
forced = _forced_backends.get(model_name)
key = forced or (_resolve_auto_for_model(model_name) if raw in _AUTO_ALIASES else raw)
if key in _ST_ALIASES and not sentence_transformers_runtime_available():
- # Match _build_st_backend_or_fallback before Settings commits an
- # ST-only pending download. Without a real llama binary ST remains the
- # only possible plan, and its eventual error is more useful than a
- # fabricated GGUF destination.
+ # Without a real llama binary ST is the only possible plan, and its eventual error is more useful
+ # than a fabricated GGUF destination.
if _llama_server_runtime_available():
key = "llama-server"
if key in _LLAMA_ALIASES:
@@ -851,7 +785,7 @@ def _build_st_backend_or_fallback(model_name: str | None = None):
backend.warm(model_name = model_name)
return backend
except (UnsafeEmbeddingModelError, EmbeddingModelDownloadRequiredError):
- raise # policy failures must hard-fail, not change backend/download
+ raise
except Exception as st_err: # noqa: BLE001 - any ST/torch import or load failure
fallback = _try_make_llama_backend()
if fallback is None:
@@ -874,7 +808,7 @@ def _switch_to_llama_fallback(err, model_name: str | None = None):
old = None
with _backend_lock:
if not isinstance(_backend, _SentenceTransformersBackend):
- return _backend # another thread already swapped (or was never ST)
+ return _backend
fallback = _try_make_llama_backend()
if fallback is None:
return None
@@ -887,8 +821,8 @@ def _switch_to_llama_fallback(err, model_name: str | None = None):
old, _backend = _backend, fallback
_forced_backends[failed_model] = "llama-server"
_backend_key = _backend_cache_key(_raw_backend(), "llama-server")
- # The failed ST wrapper is no longer published, but its module-level model
- # would otherwise survive even a later unload of the llama replacement.
+ # The failed ST wrapper is no longer published, but its module-level model would survive even a
+ # later unload of the llama replacement.
_dispose_replaced_backend(old, fallback)
return fallback
@@ -934,9 +868,8 @@ def _dispose_replaced_backend(old, new = None) -> None:
if old is None or old is new:
return
if isinstance(old, _SentenceTransformersBackend):
- # Two ST wrappers share the module-level model. A replacement ST was
- # already warmed against the new name, so clearing it here would discard
- # the model we just selected.
+ # Two ST wrappers share the module-level model and the replacement is already warmed, so clearing it
+ # here would discard the model just selected.
if not isinstance(new, _SentenceTransformersBackend):
_release_st_model()
return
@@ -983,13 +916,11 @@ def _get_backend(model_name: str | None = None):
)
_backend = new
if key in _ST_ALIASES and _is_llama_backend(new):
- # The ST warm probe fell back before producing vectors. Pin that
- # actual backend for this model, but let a different model retry ST.
+ # Pin the backend the warm probe actually fell back to, but let a different model retry ST.
key = "llama-server"
_forced_backends[model] = key
_backend_key = _backend_cache_key(raw, key)
- # A llama shutdown can wait for an in-flight encode, so keep that wait out
- # of the global publication lock. New callers already see ``new``.
+ # A llama shutdown can wait for an in-flight encode, so keep that wait out of the global publication lock.
_dispose_replaced_backend(old, new)
return new
@@ -1013,16 +944,13 @@ def backend_is_loaded(model_name: str | None = None) -> bool:
"""
backend = _backend
if backend is None:
- # No published backend does not mean nothing is loaded: an ST wrapper
- # retired by an unload that lands between _get_backend() and its encode
- # reloads the module-level model with nothing to publish. Answering False
- # stranded those weights, since release_backend returns on the same test.
+ # No published backend does not mean nothing is loaded: answering False stranded module-level
+ # weights, since release_backend returns on the same test.
if model_name is None:
return _model is not None
return _model is not None and _name == model_name
if model_name is None:
- # A llama backend whose process is gone is not resident, whichever model
- # was asked about.
+ # A llama backend whose process is gone is not resident, whichever model was asked about.
if _is_llama_backend(backend):
try:
return bool(backend._process_alive())
@@ -1033,8 +961,8 @@ def backend_is_loaded(model_name: str | None = None) -> bool:
return _model is not None and _name == model_name
if _is_llama_backend(backend):
try:
- # The object keeps _model_repo after the subprocess exits or is
- # reaped, so the repo match alone would call a dead server resident.
+ # The object keeps _model_repo after the subprocess exits, so a repo match alone would call a dead
+ # server resident.
if not backend._process_alive():
return False
return backend._model_repo == config.effective_gguf_repo_for_embedding_model(model_name)
@@ -1051,14 +979,13 @@ def release_backend() -> bool:
retry already covers a server that went away under it."""
global _backend, _backend_key
with _backend_lock:
- # Unload is an explicit fresh start, so a past runtime fallback stops pinning
- # the choice and the saved model picks its backend again.
+ # Unload is an explicit fresh start, so a past runtime fallback stops pinning the choice and the saved
+ # model picks its backend again.
_forced_backends.clear()
backend, _backend, _backend_key = _backend, None, None
if backend is None:
- # Nothing published, but the module-level model can still be there (see
- # backend_is_loaded). Freeing it here is what keeps that leak from being
- # permanent; a no-op when nothing is resident.
+ # Nothing published, but the module-level model can still be there (see backend_is_loaded);
+ # freeing it here is what keeps that leak from being permanent.
return _release_st_model()
_dispose_replaced_backend(backend)
return True
@@ -1083,10 +1010,8 @@ def active_backend_is_llama(model_name: str | None = None) -> bool:
with _backend_lock:
backend = _backend
if backend is not None:
- # A backend exists: report what it ACTUALLY is. A concrete
- # sentence-transformers backend must return False even if the
- # resolver would now pick llama, so its pickle stays gated. If the
- # llama import fails we cannot be llama, so fall to the safe False.
+ # Report what the backend ACTUALLY is: a concrete sentence-transformers backend must return False
+ # even if the resolver would now pick llama, so its pickle stays gated.
try:
from .embed_llama_server import LlamaServerBackend
except Exception: # noqa: BLE001 - llama plumbing import must never block
@@ -1176,7 +1101,7 @@ def encode_with_identity(
vectors = encode(texts, model_name = model_name, normalize = normalize)
served = getattr(_served_by, "backend", None)
name = model_name or config.effective_embedding_model()
- if served is None: # a stubbed encode never reached a backend
+ if served is None:
return vectors, embedding_identity(name)
return vectors, _identity(_is_llama_backend(served), name)
@@ -1241,8 +1166,7 @@ def token_counter(model_name: str | None = None) -> Callable[[str], int]:
):
raise
with counter_lock:
- # Another counting thread may already have replaced the retired
- # counter while this one was leaving it.
+ # Another counting thread may already have replaced the retired counter.
if state[0] is served_backend:
replacement = _get_backend(model_name)
if replacement is served_backend:
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.py
index fe8d35fab53..13229b81dae 100644
--- a/studio/backend/core/rag/folder_sync.py
+++ b/studio/backend/core/rag/folder_sync.py
@@ -575,8 +575,7 @@ def _retire_scope_rows(
"INSERT OR IGNORE INTO linked_folder_retired_scopes(scope, retired_at) VALUES(?, ?)",
(scope, _now()),
)
- # the ownership check and this write cannot share a transaction across two databases,
- # so a folder another process linked after that check keeps its own state
+ # the ownership check and this write cannot share a transaction across two databases
bound = "" if linked_before is None else " AND created_at<=?"
params = () if linked_before is None else (linked_before,)
if folders_exist:
@@ -739,8 +738,8 @@ def reconcile_retired_scopes(project_exists) -> dict[str, list[str]]:
if not project_id:
continue
try:
- # under the lock create_folder and upload admission take, and rechecked inside
- # it: a project recreated with the same id must not have its new folders retired
+ # rechecked inside the lock create_folder takes: a project recreated with the same id must not have
+ # its new folders retired
with _scope_lock(scope):
checked_at = _now()
if project_exists(project_id):
@@ -766,8 +765,7 @@ def reconcile_retired_scopes(project_exists) -> dict[str, list[str]]:
else:
continue
if owner_exists:
- # the id came back, so the scope has an owner again and its retirement,
- # which only ever meant "ownerless", must not keep gating that owner
+ # retirement only ever meant "ownerless", so a returned id must not keep gating its owner
if unretire_scope(scope):
restored.append(scope)
elif delete_retired_scope(scope):
@@ -979,23 +977,22 @@ def _scan(
continue
if os.path.splitext(entry.name)[1].lower() not in config.UPLOAD_EXTS:
continue
- # Finder metadata carries the document's extension, and a text parser reads it
- # with errors="replace", so it would be embedded and cited as a real chunk.
+ # Finder metadata carries the document's extension, so a text parser would embed and cite it as a
+ # real chunk.
if is_appledouble_metadata(Path(full)):
continue
st = entry.stat(follow_symlinks = False)
from_path = False
if st.st_ino in (None, 0):
- # Windows builds DirEntry from FindFirstFileW, which carries no file index, so
- # DirEntry.stat leaves st_dev/st_ino at 0. os.lstat opens a handle and fills
- # both; without it the stored identity is (0, 0) forever and a replacement of
- # identical size and mtime never reaches the reconciliation work list.
+ # Windows DirEntry.stat leaves st_dev/st_ino at 0 (FindFirstFileW carries no file index); os.lstat
+ # fills both, else the stored identity is (0, 0) forever.
try:
st = os.lstat(full)
from_path = True
except OSError:
- # A file that vanished mid-scan must still reach _snapshot as a failure
- # rather than abort the scan, which is never authoritative for deletion.
+ # A file that vanished mid-scan must reach _snapshot as a failure: a scan is never
+ # authoritative for
+ # deletion.
pass
rel = os.path.relpath(full, root).replace(os.sep, "/")
found[rel] = {
@@ -1004,8 +1001,8 @@ def _scan(
"mtime_ns": st.st_mtime_ns,
"device": st.st_dev,
"inode": st.st_ino,
- # A recovered identity is comparable to the next scan's, but not to os.fstat's:
- # shared-folder and WebDAV drivers report different ids for the two call paths.
+ # A recovered identity is comparable to the next scan's but not to os.fstat's: shared-folder and
+ # WebDAV drivers report different ids per call path.
"identity_from_path": from_path,
}
if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:
@@ -1039,10 +1036,8 @@ def _snapshot(root: str, metadata: dict) -> str:
metadata["inode"],
)
actual = (before.st_size, before.st_mtime_ns, before.st_dev, before.st_ino)
- # Only an identity os.fstat can reproduce is comparable here: os.scandir reports none at all
- # on Windows, and the os.lstat _scan falls back to disagrees with os.fstat on file systems
- # without stable file ids. Size and mtime still gate those, and the post-copy check below is
- # fstat to fstat either way.
+ # Only an identity os.fstat can reproduce is comparable: os.scandir reports none on Windows, and the
+ # lstat fallback disagrees with fstat on file systems without stable ids.
usable = metadata["inode"] not in (None, 0) and not metadata.get("identity_from_path")
compared = 4 if usable else 2
if actual[:compared] != expected[:compared]:
@@ -1210,8 +1205,8 @@ def _install_mapping(
raise
finally:
conn.close()
- # Only after the commit: a rollback here restores a live searchable document,
- # unlike the delete paths where the surviving row is the retry queue.
+ # Only after the commit: a rollback here restores a live searchable document, unlike the delete
+ # paths where the surviving row is the retry queue.
for stored_path in replaced:
_remove_snapshot(stored_path)
@@ -1257,8 +1252,8 @@ def _delete_mapping(folder_id: str, rel: str) -> None:
raise
finally:
conn.close()
- # After the commit, as in _install_mapping: a rollback here restores a live
- # document, and an auto_sync=0 folder may not reconcile again for a long time.
+ # After the commit: a rollback restores a live document, and an auto_sync=0 folder may not reconcile
+ # again for a long time.
_remove_snapshot(row["stored_path"])
@@ -1607,9 +1602,8 @@ def _queue_successor(job_id: str) -> None:
def reconcile_folder(job_id: str) -> None:
"""Reconcile and persist a terminal folder state for every unexpected failure."""
if _claim_job(job_id) is None:
- # The caller may already hold an activated claim (the queue claims before
- # dispatching); drop it here or the heartbeat renews it for the process
- # lifetime and an unlink waits on a job that will never run.
+ # Drop an already-activated claim here, or the heartbeat renews it for the process lifetime and an
+ # unlink waits on a job that will never run.
job_leases.release(job_leases.FOLDER_SYNC, job_id)
return
lease_lost = False
@@ -1725,8 +1719,8 @@ def _worker(stop_event: threading.Event | None = None, project_exists = None) ->
try:
job = _next_job()
except Exception:
- # Writer-lock contention must not retire the only worker: the
- # initialization and periodic paths already back off and retry.
+ # Writer-lock contention must not retire the only worker; the initialization and periodic paths
+ # retry.
logger.warning("linked-folder queue selection failed", exc_info = True)
stop_event.wait(1.0)
continue
@@ -1818,8 +1812,8 @@ def start_auto_sync(
if not rag_db.rag_available():
return False
except sqlite3.OperationalError:
- # The worker retries initialization, so transient database contention must
- # not turn a one-shot startup preflight into a process-lifetime outage.
+ # The worker retries initialization, so transient database contention must not turn a startup
+ # preflight into a process-lifetime outage.
pass
with _thread_lock:
retired = _thread if _thread is not None and _thread.is_alive() else None
diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.py
index cfd7c2cb337..cd846ff88d9 100644
--- a/studio/backend/core/rag/ingestion.py
+++ b/studio/backend/core/rag/ingestion.py
@@ -21,16 +21,15 @@ from . import captioner, chunking, config, embeddings, job_leases, parsers, stor
logger = logging.getLogger(__name__)
-# Per-job event queues, drained by job_events; ``None`` ends the stream.
+# Per-job event queues, drained by job_events; None ends the stream.
_jobs: dict[str, "queue.Queue"] = {}
_workers: dict[str, threading.Thread] = {}
_jobs_lock = threading.Lock()
-_EMBED_BATCH = 64 # bounds peak memory
+_EMBED_BATCH = 64
_SHA256_HEX_RE = re.compile(r"^[0-9a-f]{64}$")
-# Poll with a timeout so the generator wakes periodically to detect a gone
-# client or a terminal job whose worker died without the None sentinel.
+# Poll with a timeout so the generator notices a gone client or a worker that died without the None sentinel.
_SSE_POLL_SECONDS = 1.0
_TERMINAL_JOB_STATUSES = {"completed", "failed", "cancelled"}
@@ -240,14 +239,13 @@ def _run(
caption_on = config.CAPTION_IMAGES if caption is None else caption
# Skip all figure work (PDF rasterization included) without a vision model.
if caption_on and is_pdf and captioner.vision_endpoint() is not None:
- # Tile figure pages, transcribe+describe each tile, then merge/dedup/splice
- # into the page text so small labels and every sub-figure are captured.
+ # Tile figure pages, transcribe+describe each tile, then merge/dedup/splice into the page text so
+ # small labels and every sub-figure are captured.
try:
fig_pages = parsers.pages_with_figures(
stored_path,
max_pages = config.CAPTION_MAX_PAGES,
- # Skip only pages OCR actually transcribed (it covers them whole); a
- # scanned figure page past the OCR cap or with empty OCR still tiles.
+ # Skip only pages OCR actually transcribed; a scanned figure page past the OCR cap still tiles.
exclude_pages = ocred,
)
tiles = (
@@ -281,8 +279,8 @@ def _run(
count = count,
)
if not chunks:
- # An empty parse still completes the document and retires the one it replaces, so it
- # needs the same guard as the chunk write below.
+ # An empty parse still completes the document and retires the one it replaces, so it needs the same
+ # guard as the chunk write.
if _abort_if_document_deleted(conn, job_id, document_id):
return
# inside the write transaction the guard opened, as _progress does
@@ -295,8 +293,8 @@ def _run(
return
_progress(conn, job_id, "embedding", 0.5)
- # An ST encode failure swaps the process to llama-server, so the embedder that
- # produced these vectors is only known once they exist.
+ # An ST encode failure swaps the process to llama-server, so the embedder that produced these
+ # vectors is only known once they exist.
vectors, identity = _embed_all([c.text for c in chunks], model_name)
store.set_document_embedding_model(conn, document_id, identity)
@@ -314,8 +312,8 @@ def _run(
if _abort_if_document_deleted(conn, job_id, document_id):
return
store.add_chunks(conn, scope, document_id, chunks, vectors, regions)
- # add_chunks commits, which releases the lock taken above, so retake it before reporting
- # success: a delete landing in that gap must not be recorded as a completed ingestion.
+ # add_chunks commits and releases the lock, so retake it: a delete landing in that gap must not be
+ # recorded as a completed ingestion.
if _abort_if_document_deleted(conn, job_id, document_id):
return
store.set_document_status(conn, document_id, "completed", num_chunks = len(chunks))
@@ -382,27 +380,20 @@ def start_ingestion(
sha = content_hash or _sha256_file(stored_path)
conn = rag_db.get_connection()
try:
- # Named before the transaction opens, because naming the embedder on a fresh
- # process is slow work that touches no database: it searches for the
- # llama-server binary, runs nvidia-smi under a ten second timeout, and on a
- # host without it imports torch. Under BEGIN IMMEDIATE that is a RESERVED
- # lock held for all of it, and connections wait only busy_timeout (5s) for
- # one, so a concurrent ingest or job heartbeat fails with "database is
- # locked" instead of queueing.
+ # Name the embedder before BEGIN IMMEDIATE: it runs nvidia-smi and may import torch, and holding a
+ # RESERVED lock that long fails concurrent writers with "database is locked".
effective_model = model_name or config.effective_embedding_model()
effective_identity = embeddings.embedding_identity(effective_model)
- # Serialize admission with durable scope retirement across backend
- # processes. The job lease is committed in the same transaction as the
- # document, so cleanup never observes an unowned in-flight document.
+ # The job lease is committed in the same transaction as the document, so cleanup never observes an
+ # unowned in-flight document.
conn.execute("BEGIN IMMEDIATE")
if conn.execute(
"SELECT 1 FROM linked_folder_retired_scopes WHERE scope=?", (scope,)
).fetchone():
conn.rollback()
raise RuntimeError("Owning scope is being deleted")
- # (old_document_id, old_stored_path) replaced by this upload; deleted by
- # the worker only after the replacement completes, so a failed re-index
- # never destroys the still-searchable original.
+ # (old_document_id, old_stored_path) replaced by this upload; deleted by the worker only after the
+ # replacement completes, so a failed re-index never destroys the still-searchable original.
replaces: tuple[str, str | None] | None = None
existing = store.document_by_hash(conn, scope, sha) if dedupe else None
if existing is not None:
@@ -410,10 +401,9 @@ def start_ingestion(
empty_completed = (
doc is not None and doc.get("status") == "completed" and not doc.get("num_chunks")
)
- # Vectors from a different embedder are stale; re-uploading must
- # re-index, not dedupe. NULL (legacy rows) is assumed current. Only
- # completed rows are replaceable: a pending/running duplicate has a
- # live worker whose writes must not land on a deleted document.
+ # Vectors from a different embedder are stale, so re-uploading must re-index; NULL (legacy rows)
+ # is assumed current. Only completed rows are replaceable, since a running duplicate's writes must
+ # not land on a deleted document.
stale_model = (
doc is not None
and doc.get("status") == "completed"
@@ -422,9 +412,7 @@ def start_ingestion(
)
)
if empty_completed or stale_model:
- # A prior ingest of identical bytes yielded zero chunks (e.g. a scanned
- # PDF uploaded before a vision model loaded), or was embedded with a
- # different model. Re-ingest, don't dedupe.
+ # Zero chunks previously, or a different embedder: re-ingest rather than dedupe.
replaces = (existing, doc.get("stored_path"))
else:
job_id = _new_job(conn, existing, scope, status = "completed", progress = 1.0)
@@ -480,9 +468,8 @@ def start_ingestion(
return document_id, job_id
worker = threading.Thread(
target = _run,
- # effective_model (not the raw model_name) pins the embedder for the
- # whole job: a Settings change mid-ingestion must not switch tokenizer
- # or embedder between batches of one document.
+ # effective_model, not the raw model_name, pins the embedder for the whole job: a Settings change
+ # mid-ingestion must not switch tokenizer or embedder between batches.
args = args,
daemon = True,
)
@@ -630,11 +617,8 @@ def job_events(job_id: str):
try:
row = get_job_status(job_id)
except Exception: # noqa: BLE001
- # A transient status read (e.g. the DB momentarily locked) must
- # not abort the stream: routes/rag.py would turn the raised
- # exception into a terminal {type: error} frame and the UI would
- # drop a document whose worker is still running. Heartbeat and
- # retry on the next poll instead.
+ # A transient status read must not abort the stream: routes/rag.py would turn it into a terminal
+ # error frame and the UI would drop a document whose worker is still running.
logger.warning(
"job_events status read failed for %s; continuing", job_id, exc_info = True
)
@@ -651,19 +635,14 @@ def job_events(job_id: str):
break
yield event
finally:
- # Drop the queue once nothing more will be emitted into it: either a
- # terminal exit, or a disconnect after the job already finished (the UI
- # stops on the terminal event, before [DONE], so terminal is still False
- # here -- _run writes the terminal DB status before emitting it). Keep it
- # only while the worker is still running, so an early disconnect can
- # reconnect and resume its events.
+ # Keep the queue while the worker runs so an early disconnect can reconnect and resume; terminal is
+ # still False at disconnect, since _run writes the DB status before emitting it.
if not terminal:
try:
row = get_job_status(job_id)
terminal = row is None or row.get("status") in _TERMINAL_JOB_STATUSES
except Exception: # noqa: BLE001
- # Can't confirm terminality (transient DB error) -- keep the queue so
- # a reconnect can resume rather than orphaning a live worker's events.
+ # Cannot confirm terminality, so keep the queue rather than orphan a live worker's events.
terminal = False
if terminal:
with _jobs_lock:
diff --git a/studio/backend/core/rag/locators.py b/studio/backend/core/rag/locators.py
index 9331bb15ac1..d1a00dc9576 100644
--- a/studio/backend/core/rag/locators.py
+++ b/studio/backend/core/rag/locators.py
@@ -16,8 +16,7 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any
-# Anchor: up to MAX interior words from the chunk's start, shrunk toward MIN
-# to recover a unique match.
+# Anchor: up to MAX interior words from the chunk's start, shrunk toward MIN to recover a unique match.
MAX_ANCHOR_WORDS = 12
MIN_ANCHOR_WORDS = 4
@@ -95,7 +94,7 @@ def _rects_from_words(page_words: list, indices: list[int], pw: float, ph: float
for j in indices:
w = page_words[j]
x0, y0, x1, y1 = float(w[0]), float(w[1]), float(w[2]), float(w[3])
- key = (w[5], w[6]) # block, line
+ key = (w[5], w[6])
box = lines.get(key)
if box is None:
lines[key] = [x0, y0, x1, y1]
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.py
index 0b42906b859..caf62b708b7 100644
--- a/studio/backend/core/rag/parsers.py
+++ b/studio/backend/core/rag/parsers.py
@@ -70,11 +70,9 @@ def _html(raw: str) -> list[Page]:
return [_page("\n".join(parser.out), 1)]
-# pymupdf4llm rebuilds text from positioned glyphs, which mangles complex-shaping
-# scripts (RTL Arabic/Hebrew emerge as shaped Presentation Forms, Indic matras drop to
-# U+FFFD) and can silently drop most of a heavy-RTL page. When Markdown trips these
-# signals we fall back to PyMuPDF's logical-order get_text(). Thresholds mirror the chat
-# extractor guard (unslothai/unsloth#5351 review).
+# pymupdf4llm rebuilds text from positioned glyphs and mangles complex-shaping scripts (RTL forms,
+# Indic matras to U+FFFD), so fall back to PyMuPDF's logical-order get_text(); thresholds mirror
+# unslothai/unsloth#5351.
_SHAPED_PRESENTATION_FORMS = re.compile("[\ufb1d-\ufdff\ufe70-\ufefc]")
_PDF_FALLBACK_MIN_BAD_GLYPHS = 5
_PDF_FALLBACK_BAD_GLYPH_RATIO = 0.0005
@@ -130,7 +128,7 @@ def _pdf(
want_images: bool,
max_pages: int | None = None,
) -> tuple[list[Page], list[ParsedImage], int]:
- import fitz # PyMuPDF
+ import fitz
pages: list[Page] = []
images: list[ParsedImage] = []
@@ -152,9 +150,9 @@ def _pdf(
page = doc[page_number]
plain = page.get_text("text") or ""
candidate = md[i] if md else ""
- # Prefer layout-aware Markdown (keeps tables/headings legible for retrieval),
- # but drop to PyMuPDF's logical-order text when Markdown is off/empty or when
- # pymupdf4llm mangled it (RTL/Indic) or dropped most of the page.
+ # Prefer layout-aware Markdown (keeps tables/headings legible for retrieval), but drop to PyMuPDF's
+ # logical-order text when Markdown is off/empty or when pymupdf4llm mangled it (RTL/Indic) or
+ # dropped most of the page.
if (
candidate
and not _markdown_corrupted(candidate)
@@ -297,7 +295,7 @@ def render_pdf_figure_tiles(
wanted = [int(n) for n in page_numbers]
if not wanted:
return []
- rows, cols = max(1, int(rows)), max(1, int(cols)) # never divide by zero
+ rows, cols = max(1, int(rows)), max(1, int(cols))
try:
import pymupdf
except Exception:
@@ -386,19 +384,19 @@ def _docx_table_rows(table) -> list[str]:
from docx.text.paragraph import Paragraph
rows: list[str] = []
- seen: set = set() # <w:tc> already emitted; dedups merges spanning columns or rows
+ seen: set = set()
for row in table.rows:
cells: list[str] = [""] * getattr(row, "grid_cols_before", 0)
- trailing: list[str] = [] # nested rows + any post-nested text, kept in order
+ trailing: list[str] = []
for cell in row.cells:
- # A merged cell shares one <w:tc> across the columns and rows it spans:
- # emit its text once, then placeholders, so columns and rows stay aligned.
+ # A merged cell shares one <w:tc> across its span: emit its text once, then placeholders, so columns
+ # and rows stay aligned.
if cell._tc in seen:
cells.append("")
continue
seen.add(cell._tc)
- # Paragraph text before the first nested table is the aligned field; the
- # nested table and anything after it flatten below the row, in order.
+ # Paragraph text before the first nested table is the aligned field; the nested table and anything
+ # after it flatten below the row.
field: list[str] = []
after_table = False
for item in cell.iter_inner_content():
@@ -406,7 +404,7 @@ def _docx_table_rows(table) -> list[str]:
after_table = True
trailing.extend(_docx_table_rows(item))
elif isinstance(item, Paragraph):
- text = " ".join(item.text.split()) # collapse in-cell newlines
+ text = " ".join(item.text.split())
if text:
(trailing if after_table else field).append(text)
cells.append(" ".join(field)) # empty cells kept so columns line up
diff --git a/studio/backend/core/rag/retrieval.py b/studio/backend/core/rag/retrieval.py
index 505e6ecb7d2..b8210c209e9 100644
--- a/studio/backend/core/rag/retrieval.py
+++ b/studio/backend/core/rag/retrieval.py
@@ -58,8 +58,8 @@ def retrieve_dense(
) -> list[Hit]:
k = k or config.TOP_K_DENSE
effective = model_name or config.effective_embedding_model()
- # The identity comes from the encode, so it names the backend that actually served
- # this query even if a concurrent ST failure swapped the process meanwhile.
+ # The identity comes from the encode, so it names the backend that served this query even if a
+ # concurrent ST failure swapped the process meanwhile.
vectors, identity = embeddings.encode_with_identity(
[query], model_name = effective, normalize = True
)
diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.py
index a1360df20a2..c26e0160807 100644
--- a/studio/backend/core/rag/store.py
+++ b/studio/backend/core/rag/store.py
@@ -64,8 +64,7 @@ def _now() -> str:
_TOKEN = re.compile(r"\w+", re.UNICODE)
-# Straight and curly double quotes, single quotes and backticks: how a user names a word
-# instead of using it. Non-greedy and single-line so an unclosed quote spans nothing.
+# Quotes mark a word being named rather than used; non-greedy and single-line so an unclosed quote spans nothing.
_QUOTED = re.compile(r"\"([^\"\n]+)\"|\u201c([^\u201d\n]+)\u201d|'([^'\n]+)'|`([^`\n]+)`")
@@ -76,11 +75,9 @@ def _match_query(query: str) -> str:
return " OR ".join(f'"{t}"' for t in toks)
-# A closed list of function words, so behaviour is identical on every install. Nothing
-# here can carry the subject of a question.
-#
-# `no` and `not` are deliberately NOT here: they carry the whole difference in "what did I
-# say not to delete?", where dropping them leaves only terms BM25 floors at 1e-6.
+# A closed list of function words, so behaviour is identical on every install. no and not are
+# deliberately NOT here: they carry the whole difference in "what did I say not to delete?",
+# where dropping them leaves only terms BM25 floors at 1e-6.
_ARCHIVE_STOPWORDS = frozenset(
"""
a about all am an and any are as at be been being but by can could did do does doing
@@ -90,11 +87,8 @@ was we were what when where which who why will with would you your
""".split()
)
-# Identifier-ish: a token containing a digit (ZQXVARA123, 9134) or an underscore, or one
-# written in capitals and long enough not to be an "I" or an "OK". These are the tokens a
-# person uses when they mean one specific thing. Containing a digit, not mixing one with a
-# letter: a purely numeric subject ("the current value of 9134") otherwise has no shape at
-# all once the capitals rule needs contrast.
+# Identifier-ish tokens are how a person names one specific thing; a digit alone counts, since a
+# purely numeric subject has no other shape.
_HAS_DIGIT = re.compile(r"\d", re.UNICODE)
_HAS_LETTER = re.compile(r"[^\W\d_]", re.UNICODE)
@@ -115,9 +109,7 @@ def _is_identifier(token: str, raw_tokens: frozenset[str]) -> bool:
if "_" in token:
return True
if _HAS_DIGIT.search(token):
- # A bare number needs LENGTH to be a name, the same bar the capitals rule carries.
- # Without it "answer in 2 sentences" filters the archive on "2". A token mixing
- # letters and digits is a name at any length.
+ # A bare number needs LENGTH to be a name, else "answer in 2 sentences" filters the archive on "2".
return bool(_HAS_LETTER.search(token)) or len(token) >= 3
return len(token) >= 3 and token.upper() in raw_tokens
@@ -158,15 +150,13 @@ def conversation_match_queries(query: str) -> list[str]:
tokens = list(dict.fromkeys(_TOKEN.findall(query.lower())))
if not tokens:
return []
- # The capitals rule needs CONTRAST: in an all-caps line every word passes it, so the
- # filter ORs in "what" and "the" and filters nothing. Shape still decides, so
- # ZQXVARA123 is an identifier either way; only shouted prose changes.
+ # Identifier-ish: a token containing a digit (ZQXVARA123, 9134) or an underscore, or one in
+ # capitals and long enough not to be an "I" or an "OK". The capitals rule needs CONTRAST: in an
+ # all-caps line every word passes it and the filter filters nothing.
raw_tokens = frozenset() if query == query.upper() else frozenset(_TOKEN.findall(query))
identifiers = [t for t in tokens if _is_identifier(t, raw_tokens)]
- # A QUOTED word is the subject whatever the stopword list thinks: `What did I say about
- # "this"?` otherwise reduces to '"say"' and an archived `Use this endpoint` is
- # unreachable. Quoted tokens stay out of `identifiers`, so this widens only the
- # permissive pass.
+ # A QUOTED word is the subject whatever the stopword list thinks; quoted tokens stay out of
+ # identifiers, so this widens only the permissive pass.
quoted = frozenset(
token
for match in _QUOTED.findall(query.lower())
@@ -568,25 +558,18 @@ def search_lexical(
if not scopes:
return []
placeholders = ",".join("?" * len(scopes))
- # One snapshot for the gate and the read: WAL pins it at the transaction's first
- # read, so a scope retired in between cannot land rows in a result the gate already
- # decided to run unfiltered. A caller's own transaction is used instead.
+ # One snapshot for the gate and the read: WAL pins it at the transaction's first read, so a scope
+ # retired in between cannot land rows in a result the gate decided to run unfiltered.
own_read_txn = not conn.in_transaction
+ # Read-only, but it has to end: an open snapshot blocks WAL checkpointing.
if own_read_txn:
conn.execute("BEGIN")
try:
- # The filtered form joins chunks and documents and runs both subqueries for every
- # matched row BEFORE the LIMIT, so it costs more the commoner the query terms are.
- # With nothing linked that work is provably wasted (linked_folder_rows_exist).
+ # The filtered form runs both subqueries for every matched row BEFORE the LIMIT, and with nothing
+ # linked that work is provably wasted (linked_folder_rows_exist).
if oldest_first:
- # The mirror of `newest_first`, for the same reason: rowid ordering is
- # scrambled by a re-embed, which can push the oldest turn out of the window.
- # NULLs first, since a row archived before the column existed is the oldest.
- # Then `created_at`, then the chunk id: on a LEGACY archive every ordinal is
- # NULL, so every term after the score was constant and both halves of the
- # two-ended fetch returned the same arbitrary subset, which `_both_ends` then
- # deduplicated -- the 256-candidate strategy reaching one end twice and the
- # later revisions not at all.
+ # Order by archive ordinal, not rowid, which a re-embed scrambles; NULLs first as oldest, then
+ # created_at and chunk id, else on a legacy archive both halves return the same subset.
sql = (
f"SELECT chunks_fts.chunk_id, bm25(chunks_fts) AS s FROM chunks_fts "
f"JOIN chunks c ON c.id=chunks_fts.chunk_id "
@@ -596,9 +579,7 @@ def search_lexical(
f"d.created_at ASC, chunks_fts.chunk_id ASC LIMIT ?"
)
elif newest_first:
- # Ordered by archive ordinal, not rowid: rowid is insertion order, and a
- # re-embed reinserts a chunk, so a rowid DESC window returned ordinals
- # 70, 193, 116, ... and missed the newest turn. NULLs sort last, as oldest.
+ # rowid is insertion order and a re-embed reinserts a chunk, so a rowid DESC window missed the newest turn.
sql = (
f"SELECT chunks_fts.chunk_id, bm25(chunks_fts) AS s FROM chunks_fts "
f"JOIN chunks c ON c.id=chunks_fts.chunk_id "
@@ -627,7 +608,6 @@ def search_lexical(
)
rows = conn.execute(sql, (mq, *scopes, k)).fetchall()
finally:
- # Read-only, but it has to end: an open snapshot blocks WAL checkpointing.
if own_read_txn:
conn.commit()
# bm25() is negative (more negative = better); flip to higher-is-better.
@@ -655,15 +635,12 @@ def search_dense(
return []
dim = rag_db.vec_table_dim(conn)
if dim is not None and dim != len(vector):
- # Embedding model switched widths and nothing re-indexed yet; the stale
- # table cannot answer new-model queries (vec0 errors on the MATCH).
+ # The stale table cannot answer new-model queries: vec0 errors on the MATCH.
return []
- # The pre-tag spelling of the same request, kept acceptable so an existing index
- # keeps answering after an upgrade.
+ # The pre-tag spelling of the same request, kept acceptable so an existing index keeps answering after an upgrade.
untagged = config.embedding_identity_model(embedding_model) or embedding_model
- # dict.fromkeys keeps the caller's order and collapses a scope named twice, which
- # is now load-bearing: widening carries per-scope state, and a repeat would both
- # multiply that scope's fetch twice per round and emit its hits twice into the merge.
+ # dict.fromkeys collapses a scope named twice: a repeat would multiply that scope's fetch and emit
+ # its hits twice into the merge.
scopes = list(
dict.fromkeys(
s
@@ -673,16 +650,10 @@ def search_dense(
).fetchone()
)
)
- # Over-fetch when filtering so stale-model hits don't starve the top-k. Their
- # distances come from another space, so they can fill every fetched slot while
- # compatible chunks sit further down the KNN list: widen until k of them survive
- # the filter or the scope has nothing left to give.
- #
- # Per scope, not across the merge. vec0 constrains its partition key by equality,
- # so each scope is its own KNN list with its own stale prefix; a project scope
- # buried under another embedder's vectors would otherwise stop widening the
- # moment the thread scope handed over k weak hits, and the merge would then rank
- # a stronger project chunk it never fetched.
+ # Stale-model hits come from another space and can fill every fetched slot, so widen until k
+ # compatible ones survive the filter.
+ # Per scope, not across the merge: vec0 constrains its partition key by equality, so each scope has
+ # its own stale prefix.
kept: dict[str, list[tuple[str, float]]] = {}
fetches = dict.fromkeys(scopes, max(k * 3, k + 10))
pending = list(scopes)
@@ -710,8 +681,7 @@ def search_dense(
return out[:k]
-# Widening is bounded: past this many nearest neighbours per scope the scope is
-# effectively another embedder's, and a re-upload is the answer, not a longer scan.
+# Past this many nearest neighbours the scope is effectively another embedder's, and a re-upload is the answer.
_MAX_DENSE_FETCH = 4096
# One id per bound parameter, kept under the oldest SQLITE_MAX_VARIABLE_NUMBER.
_ID_BATCH = 900
diff --git a/studio/backend/core/rag/tool.py b/studio/backend/core/rag/tool.py
index 671ea3847ef..ff614285780 100644
--- a/studio/backend/core/rag/tool.py
+++ b/studio/backend/core/rag/tool.py
@@ -147,9 +147,8 @@ def format_conversation_recall(rows, hits) -> tuple[str, list[dict]]:
"page": None,
"text": text,
"turn": int(ordinal) + 1 if ordinal is not None else None,
- # Carried so a merge of two searches can reorder one long turn's pieces;
- # nothing renders them. `createdAt` is the tie-breaker `_conversation_order`
- # needs, since pre-ordinal rows have `turn` None and ordinals are not UNIQUE.
+ # createdAt is the tie-breaker _conversation_order needs: pre-ordinal rows have turn None and
+ # ordinals are not UNIQUE.
"chunkIndex": _row_value(r, "chunk_index"),
"createdAt": _row_value(r, "created_at"),
"score": round(float(h.score), 4) if h.score is not None else None,
@@ -326,15 +325,15 @@ def whole_document_context(
chunks, or the total exceeds ``max_tokens``."""
if not scope_thread_id:
return None
- # A non-positive budget means "never inject" (disable whole-doc via
- # RAG_THREAD_WHOLE_DOC=0), not "inject the whole corpus unbounded".
+ # A non-positive budget means "never inject" (disable whole-doc via RAG_THREAD_WHOLE_DOC=0), not
+ # "inject the whole corpus unbounded".
if max_tokens <= 0:
return None
scope = thread_scope(scope_thread_id)
conn = rag_db.get_connection()
try:
- # Cheap budget pre-check (SUM, no text hydration): reject an oversized attachment
- # before loading the whole corpus; all_chunks_for_scope runs only once it fits.
+ # Cheap SUM pre-check so an oversized attachment is rejected before the whole corpus is hydrated;
+ # all_chunks_for_scope runs only once it fits.
if scope_token_estimate(conn, scope) > max_tokens:
return None
rows = all_chunks_for_scope(conn, scope)
diff --git a/studio/backend/core/rag/web_rank.py b/studio/backend/core/rag/web_rank.py
index 94a2161f481..77cf4a81832 100644
--- a/studio/backend/core/rag/web_rank.py
+++ b/studio/backend/core/rag/web_rank.py
@@ -95,10 +95,8 @@ def retrieve_web_chunks(
)
if not chunks:
continue
- # The identity has to come from the encode that produced these vectors: a
- # concurrent ST failure swaps the process embedder, and reading it after
- # the fact would label this page with a space its vectors were never in,
- # which the hybrid query below then searches instead of filtering out.
+ # The identity must come from the encode that produced these vectors: a concurrent ST failure
+ # swaps the process embedder, and reading it after the fact labels the page with the wrong space.
vectors, identity = embeddings.encode_with_identity(
[chunk.text for chunk in chunks], model_name = model, normalize = True
)
diff --git a/studio/backend/core/research/citations.py b/studio/backend/core/research/citations.py
index 517afa97eb3..560dac0c343 100644
--- a/studio/backend/core/research/citations.py
+++ b/studio/backend/core/research/citations.py
@@ -15,9 +15,8 @@ import re
from core.research.redaction import _escape_link_destination
-# Unrolled rather than the equivalent (?:[^\[\]]+|\[[^\[\]]*\])* : that alternation backtracks
-# catastrophically on an unterminated "[Document:" (ordinary malformed model output), and this
-# runs on the event loop, so one bad report would stall all of Unsloth.
+# Unrolled rather than (?:[^\[\]]+|\[[^\[\]]*\])* : that alternation backtracks catastrophically on
+# an unterminated "[Document:", and this runs on the event loop.
_DOCUMENT_CITATION = re.compile(r"\[Document:[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]")
_MARKDOWN_LINK_START = re.compile(r"\[([^\]\n]+)\]\((https?://)")
_SOURCES_HEADING = re.compile(
@@ -195,9 +194,8 @@ def _allowed_document_citations(sources: list[dict]) -> set[str]:
def _validate_report_document_sources(report: str, sources: list[dict]) -> str:
allowed = _allowed_document_citations(sources)
- # Tokenize valid citations first so a ``]`` inside a filename (e.g.
- # ``budget [final].pdf``) does not truncate them, then strip any remaining
- # (invalid) document citations and restore the valid ones.
+ # Tokenize valid citations first so a "]" inside a filename ("budget [final].pdf") does not
+ # truncate them, then strip the invalid ones and restore the valid.
placeholders: dict[str, str] = {}
for index, citation in enumerate(sorted(allowed, key = len, reverse = True)):
if citation in report:
diff --git a/studio/backend/core/research/parsing.py b/studio/backend/core/research/parsing.py
index 98f994142f4..0b6a9d5a368 100644
--- a/studio/backend/core/research/parsing.py
+++ b/studio/backend/core/research/parsing.py
@@ -21,8 +21,8 @@ from core.research.redaction import _sanitize_public_query
_STREAMED_TITLE = re.compile(r'"title"\s*:\s*"((?:[^"\\]|\\.)*)"')
# One more than the plan-step cap, since the plan's own title matches too.
_MAX_PREVIEW_LABELS = 31
-# Allows the container marker of a list item or block quote before the fence ("- ```"), else the
-# open is missed and a marker quoted inside is mistaken for the real boundary.
+# Allow a list-item or block-quote container marker before the fence, else the open is missed and a
+# marker quoted inside is taken for the real boundary.
_MARKDOWN_FENCE = re.compile(r"^ {0,3}(?:(?:[-*+]|\d{1,9}[.)])[ \t]+|>[ \t]?)*(`{3,}|~{3,})")
@@ -126,8 +126,7 @@ def _normalize_synthesis_audit(
item.get("documentCitations"),
allowed_document_citations,
)
- # A claim is supported only when the audit maps it to web or document evidence
- # gathered in this run.
+ # A claim is supported only when the audit maps it to web or document evidence gathered in this run.
if claim and (urls or document_citations):
supported_claims.append(
{
@@ -301,13 +300,12 @@ def _report_after_boundary(text: str, boundary: str) -> str | None:
fence_char = token[0]
fence_length = len(token)
continue
- # CommonMark measures indentation in columns with a four-column tab stop, so one tab opens
- # an indented code block just as four spaces do.
+ # CommonMark measures indentation in columns with a four-column tab stop, so one tab opens an
+ # indented code block just as four spaces do.
prefix = content[: len(content) - len(content.lstrip(" \t"))]
indentation = len(prefix.expandtabs(4))
- # The prompt shows the marker in backticks, so models echo it that way. splitlines
- # breaks on \x0b\x0c\x1c-\x1e\x85

, which rstrip("\r\n") leaves behind,
- # so strip every whitespace form rather than let a stray one hide the boundary.
+ # splitlines breaks on whitespace forms rstrip("\r\n") leaves behind, so strip every one rather than
+ # let a stray one hide the boundary.
if indentation <= 3 and content[len(prefix) :].strip().strip("`").strip() == boundary:
boundary_line = index
if boundary_line is None:
diff --git a/studio/backend/core/research/redaction.py b/studio/backend/core/research/redaction.py
index 9b235d114f2..d7c1f011c29 100644
--- a/studio/backend/core/research/redaction.py
+++ b/studio/backend/core/research/redaction.py
@@ -15,8 +15,7 @@ import ipaddress
import re
-# Wrapper delimiters used in the decision/synthesis prompts. Any occurrence inside
-# untrusted evidence is escaped so gathered content cannot close a block early.
+# Any occurrence inside untrusted evidence is escaped so gathered content cannot close a prompt block early.
_PROMPT_DELIMITER_TAGS = re.compile(
r"</?\s*(?:untrusted_web_evidence|untrusted_evidence|source_catalog"
r"|document_source_catalog|conversation_context_json|research_question"
@@ -25,9 +24,8 @@ _PROMPT_DELIMITER_TAGS = re.compile(
r"|untrusted_synthesis_audit_json|synthesis_audit_json)\s*>",
re.IGNORECASE,
)
-# The synthesis boundary marker. Gathered pages are quoted back into the report, so an unescaped
-# copy could move the boundary and truncate the report to whatever the page put after it. Spelled
-# out to avoid importing prompts; the hardening test pins it against _REPORT_BOUNDARY_MARKER.
+# An unescaped copy could move the synthesis boundary and truncate the report; spelled out to avoid
+# importing prompts, and pinned by the hardening test.
_REPORT_BOUNDARY_TAG = re.compile(r"<!--\s*UNSLOTH_FINAL_REPORT\s*-->")
_QUERY_CREDENTIAL = re.compile(
r"""(?ix)(?<![A-Za-z0-9])(?:api[\s_-]?key|access[\s_-]?(?:key|token)
@@ -55,8 +53,7 @@ _QUERY_CREDENTIAL_SUFFIXES = (
"token",
)
_QUERY_PUBLIC_ASSIGNMENT_SUFFIXES = ("designtoken", "cancellationtoken")
-# Bearer authorization tokens carry no key=value label, so the credential pattern above misses
-# them; the length floor keeps ordinary prose ("bearer of bad news") from matching.
+# Bearer tokens carry no key=value label; the length floor keeps ordinary prose ("bearer of bad news") from matching.
_QUERY_BEARER = re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{8,}")
_QUERY_EMAIL = re.compile(r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b")
_QUERY_PRIVATE_ID = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
@@ -67,8 +64,8 @@ _QUERY_OPAQUE_TOKEN = re.compile(
r"|hf_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{20,}"
r"|AKIA[A-Z0-9]{16})\b"
)
-# International (+CC ...) or NANP-formatted phone numbers. Requires separators or a
-# leading ``+`` so bare numeric research terms are not redacted.
+# International (+CC) or NANP-formatted phone numbers: requires separators or a leading "+" so
+# bare numeric research terms are not redacted.
_QUERY_PHONE = re.compile(
r"(?<!\w)\+\d[\d\s().-]{7,17}\d(?!\w)|(?<!\w)\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}(?!\w)"
)
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py
index 16c4bf665cb..05ba64b36a6 100644
--- a/studio/backend/core/research_runs.py
+++ b/studio/backend/core/research_runs.py
@@ -74,35 +74,24 @@ _MAX_CONTEXT_CHARS = 12_000
_MAX_CONTEXT_MESSAGE_CHARS = 4_000
_MAX_SYNTHESIS_EVIDENCE_CHARS = 32_000
# The synthesis prompt must fit the loaded context or it is silently truncated and the report
-# degenerates (echoes the evidence tail). The context box accepts anything from 128 up, so the
-# budget adapts: the reserve covers the generated report and every trimmable section is measured
-# against what the untrimmable scaffolding leaves. Unknown context keeps the full cap.
+# degenerates into an echo of the evidence tail. The context box accepts anything from 128 up, so
+# the budget adapts; unknown context keeps the full cap.
_MIN_SYNTHESIS_EVIDENCE_CHARS = 1_500
-# Trimming the question or the evidence to nothing produces a confidently empty report, so each
-# keeps a floor: overflow on a tiny context is recoverable, an empty prompt is not.
+# Each section keeps a floor: overflow on a tiny context is recoverable, an empty prompt is not.
_MIN_QUESTION_CHARS = 800
_SYNTHESIS_EVIDENCE_CHARS_PER_TOKEN = 3.0
_SYNTHESIS_CONTEXT_RESERVE_TOKENS = 4_096
-# Below this loaded context the prompt scaffolding alone fills the window and the grounded
-# report degenerates, so grounding is skipped (snippet-only) for smaller loads.
+# Below this loaded context the prompt scaffolding alone fills the window, so grounding is skipped.
_AUTO_SCRAPE_MIN_CONTEXT_TOKENS = 8_192
-# Optionally ground synthesis in page text: the top results are ingested into an ephemeral RAG
-# scope (deleted after, so the user's knowledge base is untouched) and hybrid-retrieved into
-# <chunk> evidence. OFF by default, opt in via UNSLOTH_RESEARCH_AUTO_SCRAPE=1: benchmarking
-# showed no reliable factoid-accuracy gain over snippets on a local model (snippets usually
-# already carry the fact) while adding latency. Gated per run by budgets["maxAutoScrape"]
-# (absent/0 means no scrape, so existing runs keep legacy behavior). Safe only with the context
-# gate in _research and the adaptive budget in _synthesis_evidence_budget; without them, denser
-# evidence overflows a small context.
+# OFF by default (UNSLOTH_RESEARCH_AUTO_SCRAPE=1): benchmarking showed no reliable accuracy gain
+# over snippets while adding latency, and it is safe only with the context gate in _research.
_AUTO_SCRAPE_TOP_K = 3
_AUTO_SCRAPE_TOTAL_CHARS = 6_000
_WEB_RAG_TOP_N = 6
_WEB_RAG_MIN_SCORE = 0.30
-# Poll interval while a run waits for a local model to be (re)loaded, and the detail
-# routes.inference returns when nothing is loaded (its 400 is transient, not a bad request).
+# routes.inference's 400 when nothing is loaded is transient, not a bad request.
_MODEL_WAIT_POLL_SECONDS = 2.0
-# Each wait is bounded by modelTimeoutSeconds, but a model that keeps disappearing would
-# otherwise re-send forever, so cap how many times one call may wait.
+# A model that keeps disappearing would re-send forever, so cap how many times one call may wait.
_MAX_MODEL_WAITS = 3
_NO_MODEL_LOADED_DETAIL = "No model loaded"
# routes.inference reports the same unloaded state this way when auto-switch finds no local match.
@@ -116,23 +105,20 @@ _NAMED_MODEL_WAIT_SECONDS = 60.0
# Transport keepalives prevent HTTP read timeouts without proving that a model is progressing.
_MODEL_FIRST_OUTPUT_TIMEOUT_SECONDS = 120.0
_MODEL_OUTPUT_IDLE_TIMEOUT_SECONDS = 120.0
-# Cancellation is cooperative, so bound the wait for a cancelled iterator to unwind;
-# otherwise a stuck one holds a timed-out call open for the rest of the wall clock.
+# Cancellation is cooperative, so bound the unwind; a stuck iterator holds a timed-out call open for
+# the rest of the wall clock.
_STREAM_CLEANUP_TIMEOUT_SECONDS = 5.0
# The SSE comment routes/inference.py sends while queued, not while the backend is silent.
_ADMISSION_WAIT_COMMENT = ": admission-wait"
_ADMISSION_DONE_COMMENT = ": admission-done"
# Queue notices arrive on the configured heartbeat, so allow for a few missed ones.
_ADMISSION_HEARTBEAT_MISSES = 3
-# Model-loading budget when the request budget is unlimited, and its ceiling for any budget:
-# the poll loop stays bounded however long generation itself may run.
+# Also the ceiling for any budget, so the poll loop stays bounded however long generation itself runs.
_DEFAULT_MODEL_TIMEOUT_SECONDS = 900.0
_MAX_MODEL_WAIT_BUDGET_SECONDS = 3600.0
-# Bounds a provider's Retry-After when the run has no wall clock of its own: past the hourly
-# windows providers reset on, short of parking a run and its lease on one mistaken header.
+# Past the hourly windows providers reset on, short of parking a run and its lease on one mistaken header.
_MAX_RATE_LIMIT_WAIT_SECONDS = 3600.0
-# Lifetime of the internal key one model call authenticates with. Retry waits are bounded by
-# what is left of it: a key that expires mid-backoff fails auth without reaching the provider.
+# Retry waits measure against this: a key that expires mid-backoff fails auth without reaching the provider.
_MODEL_CALL_KEY_LIFETIME_SECONDS = 2 * 60 * 60
# Headroom so the named stall guards expire before HTTPX's own read timeout does.
_STREAM_READ_TIMEOUT_MARGIN_SECONDS = 30.0
@@ -185,14 +171,13 @@ def _auto_scrape_default() -> int:
return 0
-# Nav menus, language sidebars, and percent-encoded link lists are not evidence and derail
-# retrieval; drop link-dominated and encoded-URL lines.
+# Nav menus, language sidebars and percent-encoded link lists are not evidence and derail retrieval.
_MD_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)")
_PERCENT_ESCAPE = re.compile(r"%[0-9A-Fa-f]{2}")
_LIST_PREFIX = re.compile(r"^(?:[\*\-\+•]|\d+[.)])\s")
_BLANK_RUN = re.compile(r"\n{3,}")
-# Bare tracking/redirect URLs arrive as one unbroken token (prose never has an 80-char word);
-# not evidence, and a small model will latch onto and echo it.
+# Bare tracking URLs arrive as one unbroken token (prose never has an 80-char word) and a small
+# model will latch onto and echo it.
_LONG_TOKEN = re.compile(r"\S{80,}")
@@ -250,11 +235,8 @@ def _safe_error(exc: BaseException) -> str:
return "Local model request timed out"
if isinstance(exc, httpx.HTTPStatusError):
return f"Local model request failed with HTTP {exc.response.status_code}"
- # str() on a stream error is deliberately the server's own text, so that the
- # token-count regex in routes/inference.py still matches it. Deep Research has no
- # such regex, so reading str() here showed the raw "Context size has been exceeded."
- # and dropped the Model settings hint from an oversize refusal: the very case this
- # exception was introduced to explain.
+ # str() must stay the server's own text so routes/inference.py's token-count regex still matches;
+ # reading it here dropped the Model settings hint from an oversize refusal.
friendly = getattr(exc, "friendly", None)
text = friendly if isinstance(friendly, str) and friendly else str(exc)
text = text.replace("\n", " ").strip()
@@ -329,6 +311,7 @@ def _peek_inference_backend() -> Any:
"""
from core.inference import get_inference_backend
+ # Native / transformers: the orchestrator the API layer reads (not the subprocess singleton).
try:
from core.inference.orchestrator import get_inference_backend as _real
from core.inference.orchestrator import peek_inference_backend
@@ -346,7 +329,6 @@ def _loaded_context_length() -> int | None:
orchestrator) so grounding sizes evidence to the same context the API layer serves. The ML
backends live in a worker subprocess, so the core.inference.inference singleton is unpopulated
here and importing it pulls in the ML stack; read the orchestrator the routes use instead."""
- # GGUF / llama.cpp keeps context on its own backend (checked first, like the API layer).
try:
from routes.inference import get_llama_cpp_backend
llama = get_llama_cpp_backend()
@@ -356,7 +338,6 @@ def _loaded_context_length() -> int | None:
return ctx
except Exception:
logger.debug("research.context_probe_llama_failed", exc_info = True)
- # Native / transformers: the orchestrator the API layer reads (not the subprocess singleton).
try:
backend = _peek_inference_backend()
name = getattr(backend, "active_model_name", None)
@@ -567,8 +548,8 @@ def _rate_limit_wait(requested: float, remaining: float, headroom: float) -> flo
The delay is the provider's, not a share of the model-load budget, so it is bounded by what is
left of the call minus the room the re-send needs; coming back early only spends an attempt on
the same refusal. The standing ceiling covers a run with no wall clock at all."""
- # Never reserve all of what is left: a call whose wall clock is no larger than its
- # first-output budget reserves the whole of it, and every wait collapses to zero.
+ # Never reserve all of what is left: a call whose wall clock equals its first-output budget would
+ # collapse every wait to zero.
headroom = min(headroom, remaining / 2)
return max(0.0, min(requested, _MAX_RATE_LIMIT_WAIT_SECONDS, remaining - headroom))
@@ -732,9 +713,8 @@ def _bounded_synthesis_evidence(
return "(none)"
if max_chars <= 0:
return ""
- # Split the budget evenly across every note so a small context still keeps a slice of every
- # research step. A per-note floor would let the earliest notes consume the whole budget and
- # the final slice would drop later steps entirely.
+ # Split evenly across notes: a per-note floor would let the earliest notes consume the whole budget
+ # and drop later steps entirely.
separator = "\n\n"
available = max(0, max_chars - len(separator) * (len(notes) - 1))
base, remainder = divmod(available, len(notes))
@@ -960,12 +940,12 @@ class ResearchSupervisor:
try:
await self._task
except asyncio.CancelledError:
+ # Polling is intentionally sufficient for one local process; requests never own tasks.
pass
finally:
await asyncio.to_thread(db.release_worker_leases, self.worker_id)
def wake(self) -> None:
- # Polling is intentionally sufficient for one local process; requests never own tasks.
pass
def cancel(self, run_id: str) -> None:
@@ -1054,9 +1034,7 @@ class ResearchSupervisor:
)
if not pages:
return "", []
- # Reuse Unsloth's knowledge-base RAG pipeline (ingest -> hybrid retrieve -> <chunk>
- # render) over an ephemeral scope; runs off the event loop since embedding and the
- # sqlite/vec index work are CPU/GPU bound.
+ # Runs off the event loop, since embedding and the sqlite/vec index work are CPU/GPU bound.
from core.rag import web_rank
section, _sources = await asyncio.to_thread(
@@ -1128,10 +1106,8 @@ class ResearchSupervisor:
except asyncio.CancelledError:
raise
except sqlite3.OperationalError as exc:
- # Losing the writer lock is normal for polling, not a fault, and produced
- # six tracebacks in one session. Only that case is quietened. Neither may
- # re-raise: that escapes the while loop and stops the supervisor for the
- # life of the process, and the sibling `except Exception` cannot catch it.
+ # Losing the writer lock is normal for polling, not a fault; neither branch may re-raise, since that
+ # escapes the while loop and stops the supervisor for the life of the process.
if is_sqlite_busy_error(exc):
logger.warning("research.supervisor_db_busy: %s", exc)
else:
@@ -1171,8 +1147,8 @@ class ResearchSupervisor:
model, a llama.cpp update, a name that no longer resolves) needs a user action that no
wait can outlast, so surfacing the refusal beats burning the whole budget first."""
loop = asyncio.get_running_loop()
- # Share the model budget across the allowed waits: spending all of it on one lets the
- # enclosing wall clock fire first, burying the real refusal under a timeout.
+ # Share the model budget across the allowed waits: spending it all on one lets the enclosing wall
+ # clock fire first and bury the real refusal.
budget = _model_wait_budget(run)
if max_seconds is not None:
budget = min(budget, max_seconds)
@@ -1194,8 +1170,8 @@ class ResearchSupervisor:
"""
run_id = run["id"]
step = _retry_after_seconds(response) or _MODEL_SWITCH_RETRY_SECONDS
- # Same budget share as _wait_for_local_model: one wait must leave room for the others and
- # for the refusal, or the enclosing wall clock fires first and reports a timeout instead.
+ # Same budget share as _wait_for_local_model: one wait must leave room for the others and for the
+ # refusal, or the enclosing wall clock fires first and reports a timeout instead.
remaining = min(step * waits, _NAMED_MODEL_WAIT_SECONDS, _model_wait_budget(run))
logger.info("research.waiting_for_model_switch run_id=%s seconds=%.0f", run_id, remaining)
while remaining > 0:
@@ -1252,11 +1228,11 @@ class ResearchSupervisor:
await asyncio.wait({task}, timeout = _STREAM_CLEANUP_TIMEOUT_SECONDS)
except asyncio.CancelledError:
# Must keep propagating, but the child outlives this frame, so hand it over first.
+ # Bound expired but the task lives on: absorb its outcome when it cooperates.
self._absorb_when_done(run_id, task, what)
raise
if not task.done():
logger.warning("research.%s_cleanup_timed_out run_id=%s", what, run_id)
- # Bound expired but the task lives on: absorb its outcome when it cooperates.
self._absorb_when_done(run_id, task, what)
return
try:
@@ -1301,8 +1277,8 @@ class ResearchSupervisor:
discarded = True
await self._discard_task(run_id, line_task, "stream_iterator")
await self._check_active(run_id)
- # A line that arrived during the wait is earned; recomputing the
- # deadline first would let an expiry in the same turn discard it.
+ # A line that arrived during the wait is earned; recomputing the deadline first would let an expiry
+ # in the same turn discard it.
if line_task.done():
break
timeout = wait_timeout()
@@ -1336,9 +1312,8 @@ class ResearchSupervisor:
token, key = await asyncio.to_thread(
auth_storage.create_api_key,
username = run["ownerSubject"],
- # The name is load-bearing, not a label: the external-provider route
- # scopes its saved-credential exception to exactly this workflow, so
- # the two sides must not drift apart.
+ # The name is load-bearing: the external-provider route scopes its saved-credential exception to
+ # exactly this workflow.
name = auth_storage.DEEP_RESEARCH_WORKFLOW_KEY_NAME,
expires_at = expires,
internal = True,
@@ -1350,21 +1325,18 @@ class ResearchSupervisor:
"messages": messages,
"stream": True,
"stream_options": {"include_usage": True},
- # Keep every model hop in this durable run on one isolated Codex
- # prompt-cache session rather than sharing the transport fallback.
+ # Keep every model hop in this durable run on one isolated Codex prompt-cache session rather than
+ # sharing the transport fallback.
"thread_id": f"research:{run['id']}",
- # Gathered page text lands in these prompts and research never reads tool calls
- # back, so this hop must stay out of the tool loop. Both opt-outs are needed:
- # --enable-tools overrides a per-request enable_tools, and an omitted
+ # Both opt-outs are needed: --enable-tools overrides a per-request enable_tools, and an omitted
# enabled_tools resolves to every built-in, python and terminal included.
"tool_choice": "none",
"enabled_tools": [],
"temperature": inference.get("temperature", 0.2),
}
- # Route the hop to whichever saved connection the run was created with.
- # The route's _sanitize_config already refused anything but an enabled
- # saved connection of a studio-tools-capable provider type.
+ # The route's _sanitize_config already refused anything but an enabled saved connection of a studio-
+ # tools-capable provider type.
if inference.get("providerType"):
payload.update(
{
@@ -1465,9 +1437,8 @@ class ResearchSupervisor:
)
if model_timeout > 0:
first_output_budget = min(first_output_budget, model_timeout)
- # Unlimited only drops the total wall clock; connect and headers stay bounded. This
- # bound also caps the silence between queue notices, which no wall clock covers, so
- # it has to clear the heartbeat those notices are paced by.
+ # Unlimited only drops the total wall clock; this bound also caps the silence between queue notices,
+ # so it has to clear the heartbeat they are paced by.
admission_gap_budget = max(
first_output_budget,
_MODEL_OUTPUT_IDLE_TIMEOUT_SECONDS,
@@ -1479,8 +1450,8 @@ class ResearchSupervisor:
if model_timeout
else httpx.Timeout(
first_output_budget,
- # Strictly looser than the guards above, so a stall is reported by name
- # rather than as a message-less HTTPX ReadTimeout.
+ # Strictly looser than the guards above, so a stall is reported by name rather than as a message-
+ # less HTTPX ReadTimeout.
read = admission_gap_budget + _STREAM_READ_TIMEOUT_MARGIN_SECONDS,
)
)
@@ -1497,8 +1468,7 @@ class ResearchSupervisor:
)
call_started = loop.time()
- # No backoff below may outlast this call's wall clock or the key the re-send
- # authenticates with, so all of them measure against whichever ends first.
+ # No backoff below may outlast this call's wall clock or the key the re-send authenticates with.
retry_deadline = key_minted + _MODEL_CALL_KEY_LIFETIME_SECONDS
if model_timeout:
retry_deadline = min(retry_deadline, call_started + model_timeout)
@@ -1508,6 +1478,7 @@ class ResearchSupervisor:
):
response: httpx.Response | None = None
send_task: asyncio.Task | None = None
+ # A retry builds a fresh task, so the guard starts over with it.
send_discarded = False
model_waits = 0
attempt = 0
@@ -1526,7 +1497,6 @@ class ResearchSupervisor:
)
try:
send_task = asyncio.create_task(client.send(request, stream = True))
- # A retry builds a fresh task, so the guard starts over with it.
send_discarded = False
while not send_task.done():
await asyncio.wait({send_task}, timeout = 0.2)
@@ -1539,8 +1509,7 @@ class ResearchSupervisor:
response.raise_for_status()
first_output_deadline = loop.time() + first_output_budget
except (httpx.TransportError, httpx.HTTPStatusError) as exc:
- # Only reachable before a body byte is touched (the stream is consumed
- # after this loop), so a re-send cannot duplicate report text.
+ # Only reachable before a body byte is touched, so a re-send cannot duplicate report text.
unloaded = (
await _model_unloaded(exc.response)
if isinstance(exc, httpx.HTTPStatusError)
@@ -1568,8 +1537,8 @@ class ResearchSupervisor:
if unloaded == "switching":
await self._wait_for_model_switch(run, exc.response, model_waits)
elif unloaded:
- # Nothing loaded (restart, eject): wait for a model to come back,
- # without spending a transport attempt.
+ # Nothing loaded (restart, eject): wait for a model to come back, without
+ # spending a transport attempt.
if not await self._wait_for_local_model(
run,
_NAMED_MODEL_WAIT_SECONDS if unloaded == "named" else None,
@@ -1578,8 +1547,7 @@ class ResearchSupervisor:
else:
delay = 2**attempt
if rate_limited:
- # This runs to minutes, so re-read the run while it
- # waits; the backoff below never outlasts one poll.
+ # This runs to minutes, so re-read the run while it waits.
await self._wait_out_rate_limit(
run,
_retry_after_seconds(exc.response) or delay,
@@ -1592,9 +1560,9 @@ class ResearchSupervisor:
# re-check the lease and cancellation before re-sending.
await self._check_active(run["id"])
continue
- # A proxied provider 429 arrives as a 200 whose first line is the
- # refusal, so the status cannot see it. No body byte is used yet, so a
- # re-send cannot duplicate report text.
+ # A proxied provider 429 arrives as a 200 whose first line is the refusal, so the
+ # status cannot see
+ # it; no body byte is used yet.
stream = self._iter_stream_lines(run["id"], response, semantic_deadline)
head = await _peek_stream_head(stream)
throttled = _stream_rate_limit_delay(head)
@@ -1613,12 +1581,12 @@ class ResearchSupervisor:
if self._cancel_event(run["id"]).is_set():
await self._check_active(run["id"])
if not line.startswith("data:"):
- # Queueing has no timeout by design, so it is not charged: suspend
- # for it, and start the budget when the slot is granted. A plain
- # ": keep-alive" means a silent backend, which is what we bound.
+ # Queueing has no timeout by design, so suspend for it and start the budget when the slot is
+ # granted.
if line.startswith(_ADMISSION_WAIT_COMMENT):
- # Unlimited has no wall clock behind this, so bound the gap
- # between queue notices; each notice refreshes it.
+ # Unlimited has no wall clock behind this, so bound the gap between queue
+ # notices; each notice
+ # refreshes it.
first_output_deadline = (
None if model_timeout else loop.time() + admission_gap_budget
)
@@ -1630,13 +1598,16 @@ class ResearchSupervisor:
break
if not data:
continue
+ # Arming research in the composer is the approval, so the plan is queued as it is stored
+ # rather than parked for a second confirmation.
+ # revoked before the phase event, so a cancel there cannot leak a live key.
try:
chunk = json.loads(data)
_stream_error = stream_error_from_chunk(chunk)
if _stream_error is not None:
- # The server's own text names the cause and, for a context
- # refusal, both token counts. Flattening it to a fixed
- # string left the user with nothing to act on.
+ # The server's own text names the cause and both token counts; flattening it to
+ # a fixed string left
+ # the user nothing to act on.
raise _stream_error
normalized_usage = _normalize_completion_usage(
chunk.get("usage") if isinstance(chunk, dict) else None
@@ -1692,8 +1663,9 @@ class ResearchSupervisor:
try:
await response.aclose()
except Exception:
- # Closing a broken stream is best-effort and must not replace the
- # generation result or the timeout/error that caused teardown.
+ # Closing a broken stream is best-effort and must not replace the generation result
+ # or the error
+ # that caused teardown.
logger.warning(
"research.stream_cleanup_failed run_id=%s",
run["id"],
@@ -1772,8 +1744,8 @@ class ResearchSupervisor:
logger.debug("research.phase_event_failed run_id=%s", run_id, exc_info = True)
async def _process(self, run: dict) -> None:
- # The attempt this worker claimed. Everything it writes after a terminal status is
- # only its to write while the run is still on that attempt.
+ # Everything this worker writes after a terminal status is only its to write while the run is still
+ # on that attempt.
attempt = int(run.get("retryCount") or 0)
cancel_event = self._cancel_event(run["id"])
if await asyncio.to_thread(db.is_cancel_requested, run["id"]):
@@ -1851,8 +1823,8 @@ class ResearchSupervisor:
renewed = await asyncio.to_thread(db.heartbeat, run_id, self.worker_id)
except Exception:
logger.warning("research.heartbeat_failed run_id=%s", run_id, exc_info = True)
- # A busy SQLite writer is not proof that ownership was lost.
- # Retry briefly, but stop well before the 120-second lease expires.
+ # A busy SQLite writer is not proof that ownership was lost; retry briefly, but stop well before the
+ # 120-second lease expires.
consecutive_errors += 1
if consecutive_errors >= 10:
self._lost_leases.add(run_id)
@@ -1880,9 +1852,8 @@ class ResearchSupervisor:
_planner_system_prompt(max_steps, run["config"].get("websitePolicy")),
run["config"],
)
- # Same whole-prompt budget as the decision and synthesis paths. The question is budgeted
- # before the history, but it is unbounded on its own (a pasted document arrives here
- # verbatim) and would otherwise overflow before planning.
+ # The question is budgeted before the history but is unbounded on its own (a pasted document arrives
+ # verbatim) and would overflow before planning.
planning_total = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)
planning_question = question[
: max(
@@ -1936,8 +1907,8 @@ class ResearchSupervisor:
await self._check_active(run["id"])
raise
run.update(result)
- # The structured inline card renders the plan; no second markdown copy below it.
+ # The structured inline card renders the plan; no second markdown copy below it.
async def _research(self, run: dict) -> None:
resuming = run.get("claimedFromStatus") == "running"
fresh = await asyncio.to_thread(db.get_run, run["id"])
@@ -1952,8 +1923,7 @@ class ResearchSupervisor:
tool_timeout = int(budgets["toolTimeoutSeconds"])
# Absent for runs created before auto-scrape: default 0 keeps their behavior unchanged.
max_auto_scrape = int(budgets.get("maxAutoScrape", 0))
- # On a tiny context the prompt overhead alone fills the window and the grounded report
- # degenerates, so fall back to snippet-only.
+ # On a tiny context the prompt overhead alone fills the window, so fall back to snippet-only.
if max_auto_scrape > 0:
loaded_ctx = _loaded_context_length()
if loaded_ctx is not None and loaded_ctx < _AUTO_SCRAPE_MIN_CONTEXT_TOKENS:
@@ -2023,9 +1993,8 @@ class ResearchSupervisor:
)
for source in document_sources
}
- # Mirrors the live loop: evidence must hold only chunks that made it into the
- # catalog, else the validator strips citations to the rest and synthesis is left
- # building claims on uncataloged document text.
+ # Evidence must hold only chunks that reached the catalog, else the validator strips citations to
+ # the rest and synthesis builds claims on uncataloged text.
accepted_rag_sources = []
for source in restored_rag_sources:
source_key = str(
@@ -2079,9 +2048,8 @@ class ResearchSupervisor:
_AGENT_SYSTEM_PROMPT + (f"\n\n{policy_prompt}" if policy_prompt else ""),
run["config"],
)
- # Same whole-prompt budget as synthesis: a fixed 60k evidence tail is many times a
- # small context, and this runs every step, so an overflow here kills the run long
- # before it can synthesize what it already gathered.
+ # A fixed 60k evidence tail is many times a small context and this runs every step, so an overflow
+ # here kills the run before it can synthesize.
decision_total = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)
decision_question, decision_plan_json = _fit_decision_inputs(
question,
@@ -2089,8 +2057,8 @@ class ResearchSupervisor:
len(decision_system),
decision_total,
)
- # The catalog is unbounded too (maxSources entries, snippets up to 4000 chars), so it
- # is fitted before the sections that depend on what it leaves.
+ # The catalog is unbounded too (maxSources entries, snippets up to 4000 chars), so it is fitted
+ # before the sections that depend on what it leaves.
decision_catalog = _fit_source_catalog(
source_catalog,
_trimmable_budget(
@@ -2197,9 +2165,8 @@ class ResearchSupervisor:
if action is None:
break
argument = action["query"]
- # Persist model-derived state only after the associated action is final. Seed
- # fallbacks intentionally carry no state, so rejected decisions cannot leak stale
- # notes into the executed step, resume state, or synthesis.
+ # Persist model-derived state only after the action is final, so rejected decisions cannot leak
+ # stale notes into the executed step, resume state, or synthesis.
next_state = _normalize_research_state(action.get("researchState"))
if next_state:
research_state = next_state
@@ -2296,10 +2263,8 @@ class ResearchSupervisor:
for source in accepted_rag_sources
)
elif rag_sources:
- # Every chunk was refused by the source cap, so none has a catalog entry and the
- # validator would strip any citation to it: drop the evidence rather than let
- # synthesis build claims on it. Gated on rag_sources so a text-only KB reply
- # ("No documents are attached to this chat.") still passes through.
+ # Chunks refused by the source cap have no catalog entry and the validator would strip every
+ # citation to them; gated on rag_sources so a text-only KB reply still passes through.
rag_result = ""
rag_sources = accepted_rag_sources
step_sources = []
@@ -2349,8 +2314,8 @@ class ResearchSupervisor:
fetched_urls.update(scraped_urls)
await self._check_active(run["id"])
if scraped_section:
- # Additive, not replace: see _merge_scraped_evidence for why
- # replacing the snippets regressed accuracy.
+ # Additive, not replace: see _merge_scraped_evidence for why replacing the snippets regressed
+ # accuracy.
result = _merge_scraped_evidence(result, scraped_section)
note = (
f"### {action['title']} ({action['action']})\n"
@@ -2419,8 +2384,8 @@ class ResearchSupervisor:
f" Chunk ID: {source.get('chunkId') or '(unknown)'}"
for index, source in enumerate(document_sources, 1)
)
- # Budget each synthesis call as a whole. Model-derived JSON shares the evidence budget,
- # and conversation history receives only the space left after the fixed prompt scaffold.
+ # Model-derived JSON shares the evidence budget, and conversation history receives only what the
+ # fixed scaffold leaves.
total_budget = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)
plan_json = json.dumps(run["plan"], ensure_ascii = False)
audit_system = _system_prompt_with_instructions(
@@ -2667,8 +2632,7 @@ class ResearchSupervisor:
reasoning = await asyncio.to_thread(db.get_reasoning_text, run["id"])
if synthesis_reasoning and synthesis_reasoning not in reasoning:
reasoning += synthesis_reasoning
- # Renew ownership before synchronizing the discoverable chat message.
- # A restarted worker can safely overwrite this same message.
+ # Renew ownership before syncing the discoverable chat message; a restarted worker can safely overwrite it.
renewed = await asyncio.to_thread(db.heartbeat, run["id"], self.worker_id)
if not renewed:
await self._check_active(run["id"])
diff --git a/studio/backend/core/tool_healing.py b/studio/backend/core/tool_healing.py
index 59790eb6d0d..c57d9c0872a 100644
--- a/studio/backend/core/tool_healing.py
+++ b/studio/backend/core/tool_healing.py
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-#
# Bracket-tag, rehearsal, and thinking-block-strip logic adapted from forge
# (https://github.com/antoinezambelli/forge), Copyright (c) 2025-2026
# Antoine Zambelli, used under the MIT License.
@@ -32,17 +31,15 @@ import re
# One nesting level in the strip regexes; deeper may leak markup (still parsed).
_BRACKETED_JSON_ONE_LEVEL = r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"
-# Rehearsal ``name[ARGS]{..}`` strips; group 1 = name for tool-list gating. Closed =
-# complete body, tail = truncated; ``(?<!\[CALL_ID\])`` keeps the v11 call-id from reading as a name.
+# group 1 = name for tool-list gating; the (?<!\[CALL_ID\]) lookbehind keeps the v11 call-id from reading as a name.
_REHEARSAL_CLOSED_STRIP_RE = re.compile(
r"(?<!\[CALL_ID\])\b([\w-]+)\[ARGS\]\s*" + _BRACKETED_JSON_ONE_LEVEL, re.DOTALL
)
_REHEARSAL_TAIL_STRIP_RE = re.compile(r"(?<!\[CALL_ID\])\b([\w-]+)\[ARGS\]\s*(?:\{.*)?$", re.DOTALL)
-# Tool-XML strip patterns; hyphen in the name class covers dashed MCP names.
-# Closed-pair patterns are named so _PAT_REQUIRED_TOKEN can skip a doomed lazy rescan when
-# the close token is absent: an unguarded ``<tag>.*?</tag>`` rescans to EOF from every opener
-# (quadratic on a stream of unclosed openers). Also reused by the quote-aware Gemma pre-pass.
+# Hyphen in the name class covers dashed MCP names.
+# Closed-pair patterns are named so _PAT_REQUIRED_TOKEN can skip a doomed lazy rescan: an unguarded
+# <tag>.*?</tag> rescans to EOF from every opener (quadratic).
_TC_JSON_CLOSED_PAT = re.compile(r"<tool_call>.*?</tool_call>", re.DOTALL)
_TC_GEMMA_CLOSED_PAT = re.compile(r"<\|tool_call>.*?<tool_call\|>", re.DOTALL)
_TC_FUNC_CLOSED_PAT = re.compile(r"<function=[\w-]+>.*?</function>", re.DOTALL)
@@ -61,9 +58,8 @@ _TOOL_CLOSED_PATS = [
# Drop the bare v11 [/TOOL_CALLS] closer the balanced scan leaves behind.
re.compile(r"\[/TOOL_CALLS\]"),
]
-# Bare open markers strip a partial call mid-stream; the rehearsal tail needs `{` or EOF
-# so prose ``foo[ARGS]`` survives. The XML open-tail forms reach EOF and are reused by
-# _tool_call_markup_spans (a think tag in an unclosed call's args stays argument data).
+# The rehearsal tail needs "{" or EOF so prose foo[ARGS] survives; the XML open-tail forms reach EOF
+# and are reused by _tool_call_markup_spans.
_TOOL_OPEN_XML_TAIL_PATS = [
re.compile(r"<tool_call>.*$", re.DOTALL),
re.compile(r"<\|tool_call>.*$", re.DOTALL),
@@ -81,21 +77,19 @@ _TOOL_ALL_PATS = (
# Rehearsal strips (name in group 1); name-gated via ``enabled_tool_names``, strip-all when None.
_REHEARSAL_STRIP_PATS = frozenset({_REHEARSAL_CLOSED_STRIP_RE, _REHEARSAL_TAIL_STRIP_RE})
-# Stripped before the quote-aware Gemma helper so a Gemma opener quoted in argument
-# data cannot make the helper truncate the block and its tail.
+# Stripped before the quote-aware Gemma helper so a Gemma opener quoted in argument data cannot
+# truncate the block and its tail.
_TOOL_CLOSED_BLOCK_PATS = [_TC_JSON_CLOSED_PAT, _TC_FUNC_CLOSED_PAT]
-# A lazy closed-pair pattern whose close token is absent would rescan to EOF from every
-# opener; skip that doomed (quadratic) pass. Shared by both strip helpers.
+# A lazy closed-pair pattern whose close token is absent would rescan to EOF from every opener; skip
+# that quadratic pass.
_PAT_REQUIRED_TOKEN = {
- # A tuple marks a required CLOSER. Besides skipping the pass when it is absent, callers
- # bound the regex at the LAST one: nothing matches past it, and a tail full of openers
- # would retry a lazy pattern at each one (quadratic). The bound is unconditional --
- # any gap test for "are there openers after it" is itself the quadratic case.
+ # A tuple marks a required CLOSER, and callers bound the regex at the LAST one unconditionally: any
+ # gap test for openers past it is itself the quadratic case.
_TC_JSON_CLOSED_PAT: ("</tool_call>", "<tool_call>"),
_TC_GEMMA_CLOSED_PAT: ("<tool_call|>", "<|tool_call>"),
_TC_FUNC_CLOSED_PAT: ("</function>", "<function="),
- # Literal in both rehearsal patterns: skips the sub AND its _code_spans scan on the
- # common answer that has no rehearsal at all.
+ # Literal in both rehearsal patterns: skips the sub AND its _code_spans scan on the common answer that has
+ # no rehearsal at all.
_REHEARSAL_CLOSED_STRIP_RE: "[ARGS]",
_REHEARSAL_TAIL_STRIP_RE: "[ARGS]",
}
@@ -172,8 +166,8 @@ def apply_tool_strip_patterns(
elif required is not None and required not in text:
continue
if pat in _REHEARSAL_STRIP_PATS:
- # Same two gates the parser applies in _iter_bracket_spans, so a rehearsal that
- # is not promoted to a call stays visible instead of vanishing from the answer.
+ # Same two gates the parser applies in _iter_bracket_spans, so an unpromoted rehearsal stays visible
+ # instead of vanishing from the answer.
spans = _code_spans(text)
_t = text
text = pat.sub(lambda m: _rehearsal_strip(m, pat, _t, spans, enabled_tool_names), text)
@@ -182,9 +176,8 @@ def apply_tool_strip_patterns(
return text
-# Pre-compiled patterns for tool-call XML parsing.
-# <|content_invoke_tool_json|> is TML Inkling's native call marker; its JSON uses
-# an ``args`` key and the block closes with <|end_message|>.
+# <|content_invoke_tool_json|> is TML Inkling's native call marker: its JSON uses an "args" key and
+# the block closes with <|end_message|>.
_TC_JSON_START_RE = re.compile(r"(?:<tool_call>|<\|content_invoke_tool_json\|>)\s*\{")
_TC_GEMMA_START_RE = re.compile(r"<\|tool_call>\s*call\s*:\s*([\w.\-]+)\s*\{")
_TC_FUNC_START_RE = re.compile(r"<function=([\w-]+)>\s*")
@@ -197,16 +190,11 @@ _TC_PARAM_CLOSE_RE = re.compile(r"\s*</parameter>\s*$")
_GEMMA_QUOTE = '<|"|>'
_PARAM_CLOSE_TAG = "</parameter>"
_FUNC_CLOSE_TAG = "</function>"
-# A bare (unquoted) Gemma value ends at `}` or at a comma that begins the next
-# `key:` pair. A comma NOT followed by a key token is part of the value (e.g.
-# `location:New York, NY`), so it must not terminate the value. The key token
-# must be identifier-shaped (start with a letter or underscore); a comma
-# followed by digits-then-colon is value text such as a timestamp or ratio
-# (`meet at 10:00, 11:00 tomorrow`), not a new key.
+# A comma NOT followed by an identifier-shaped key token is part of a bare Gemma value (location:New
+# York, NY; meet at 10:00, 11:00).
_GEMMA_NEXT_KEY_RE = re.compile(r"\s*[A-Za-z_][\w.\-]*\s*:")
-# A candidate starting inside a think block is a rehearsal (block kept so literal tags in
-# real args survive); ``$`` accepts an unclosed block mid-stream.
+# A candidate starting inside a think block is a rehearsal; "$" accepts an unclosed block mid-stream.
_THINK_TAG_RE = re.compile(r"<think>.*?(?:</think>|$)|\[THINK\].*?(?:\[/THINK\]|$)", re.DOTALL)
# Bare open/close markers for prefilled-reasoning turns (template opens <think> in the prompt).
_THINK_OPEN_RE = re.compile(r"<think>|\[THINK\]")
@@ -215,27 +203,20 @@ _THINK_CLOSE_RE = re.compile(r"</think>|\[/THINK\]")
# Mistral canonical array: [TOOL_CALLS] + JSON list of {"name","arguments"} objects.
_MISTRAL_ARRAY_RE = re.compile(r"\[TOOL_CALLS\]\s*(?=\[)")
-# Mistral name form + v11 [ARGS]/[CALL_ID] shapes; [CALL_ID] is metadata, not the name,
-# and hyphens keep dashed MCP names whole.
+# [CALL_ID] is metadata, not the name, and hyphens keep dashed MCP names whole.
_MISTRAL_BRACKET_RE = re.compile(
r"\[TOOL_CALLS\]\s*([\w-]+)(?:\[CALL_ID\][\w-]+)?(?:\[ARGS\])?\s*(?=\{)"
)
-# Rehearsal ``name[ARGS]{json}`` (no [TOOL_CALLS]); the lookbehind keeps the v11 call-id
-# from being taken as the function name.
+# The lookbehind keeps the v11 call-id from being taken as the function name.
_REHEARSAL_RE = re.compile(r"(?<!\[CALL_ID\])\b([\w-]+)\[ARGS\]\s*(?=\{)")
-# Markdown code: a fenced block (closing fence optional so a streaming block counts) or
-# an inline span. Gates the markerless rehearsal form only, so quoting the syntax as
-# documentation stays text instead of executing. ``>`` container prefixes are allowed so
-# a fence inside a block quote counts. A backtick fence's info string cannot itself contain
-# backticks, so ```` ```a``` ```` opening a line is an inline span and not a fence running
-# to EOF; the closer tolerates a CR so a CRLF block still closes. Both would otherwise
-# swallow the rest of the message and drop a real call after it. Inline runs are enumerated by length (double before
-# single) rather than backreferenced: ``code`` must be one span and not two empty pairs
-# around live markup, but a ``(`+)..\1`` form backtracks over every candidate length and
-# turned 21 KB of unmatched runs into a 1.8s scan. A lone backtick is valid content inside
-# a doubled span, so only a run of two closes it.
+# Gates the markerless rehearsal form only, so quoting the syntax as documentation stays text
+# instead of executing. A fence's info string cannot contain backticks, so a triple-backtick run
+# opening a line is an inline span and not a fence running to EOF; the closer tolerates a CR so a
+# CRLF block still closes. Inline runs are enumerated by length rather than backreferenced: a
+# (`+)..\1 form backtracks over every candidate length and turned 21 KB of unmatched runs into a
+# 1.8s stall.
_CODE_SPAN_RE = re.compile(
r"^[ \t]*(?:>[ \t]*)*(?:```+[^`\n]*|~~~+[^\n]*)$"
r".*?(?:^[ \t]*(?:>[ \t]*)*(?:```+|~~~+)[ \t\r]*$|\Z)"
@@ -451,7 +432,7 @@ def _iter_bracket_spans(
last_brace_close = text.rfind("}")
last_array_close = max(text.rfind("]"), last_brace_close)
cursor = start
- code_spans: list | None = None # computed on the first rehearsal candidate only
+ code_spans: list | None = None
while cursor < n:
for kind, rx in specs:
m = nexts[kind]
@@ -463,9 +444,9 @@ def _iter_bracket_spans(
kind, m = min(live, key = lambda km: km[1].start())
last_close = last_array_close if kind == "array" else last_brace_close
if m.end() > last_close:
- # This and every later candidate of the same kind lacks its closing character.
# Keep other formats live, but skip a balanced scan to EOF per doomed opener.
nexts[kind] = None
+ # Truncated body: skip and keep scanning; the caller's catch-all strips the tail.
cursor = m.end()
continue
if kind == "array":
@@ -473,7 +454,6 @@ def _iter_bracket_spans(
else:
end = _balanced_json_span(text, m.end())
if end is None:
- # Truncated body: skip and keep scanning; the caller's catch-all strips the tail.
cursor = m.end()
continue
if (
@@ -481,6 +461,7 @@ def _iter_bracket_spans(
and enabled_tool_names is not None
and m.group(1) not in enabled_tool_names
):
+ # Quoted syntax, not a call: advance past its body without yielding.
# Inactive-name rehearsal is prose: advance past its body without yielding.
cursor = end + 1
continue
@@ -488,7 +469,6 @@ def _iter_bracket_spans(
if code_spans is None:
code_spans = _code_spans(text)
if _in_code(code_spans, m.start()):
- # Quoted syntax, not a call: advance past its body without yielding.
cursor = end + 1
continue
yield (m.start(), end + 1, kind, m)
@@ -621,15 +601,14 @@ def _quote_gemma_object_keys(src: str) -> str:
parts.append(src[i:colon_pos])
parts.append(":")
i = colon_pos + 1
- # Gemma may emit bare string values ({unit:celsius}); quote them so
- # json.loads succeeds. JSON scalars/objects/arrays/quoted stay as-is.
+ # Gemma may emit bare string values ({unit:celsius}); quote them so json.loads succeeds.
ws = i
while i < len(src) and src[i].isspace():
i += 1
parts.append(src[ws:i])
if i < len(src) and src[i] == "[":
- # Array value: quote bare string elements (e.g. labels:[bug,ui])
- # so json.loads succeeds instead of dropping the call.
+ # Array value: quote bare string elements (e.g. labels:[bug,ui]) so json.loads succeeds instead
+ # of dropping the call.
arr_end = _balanced_bracket_end(src, i)
if arr_end is None:
parts.append(src[i:])
@@ -639,9 +618,7 @@ def _quote_gemma_object_keys(src: str) -> str:
i = arr_end + 1
elif i < len(src) and src[i] not in '"{':
v_start = i
- # Consume the bare value up to `}` or a comma that starts the
- # next key:value pair; a comma inside the value (e.g.
- # `New York, NY`) does not terminate it.
+ # A comma inside the value (New York, NY) does not terminate it.
while i < len(src):
if src[i] == "}":
break
@@ -653,7 +630,7 @@ def _quote_gemma_object_keys(src: str) -> str:
json.loads(raw.strip())
parts.append(raw)
except (json.JSONDecodeError, ValueError):
- # Quote bare value; empty ({k:}) becomes "" so json.loads sees {"k":""} not invalid {"k":}.
+ # Empty ({k:}) becomes "" so json.loads sees {"k":""} and not invalid {"k":}.
parts.append(json.dumps(raw.strip()))
else:
parts.append(src[key_start:i])
@@ -688,9 +665,7 @@ def _inside_open_parameter(
if param_start_re is None:
param_start_re = _TC_PARAM_START_RE
last_param_start = -1
- # Both supported vocabularies begin with ``<param``, so search backwards to the nearest
- # candidate and validate it with the regex instead of rescanning the whole prefix per
- # marker. Unknown caller patterns keep the general finditer path.
+ # Search backwards to the nearest <param candidate instead of rescanning the whole prefix per marker.
reverse_param_search = (
param_start_re is _TC_PARAM_START_RE
or param_start_re.pattern.startswith(r"<(?:parameter|param)")
@@ -708,24 +683,21 @@ def _inside_open_parameter(
last_param_start = match.start()
if last_param_start < 0:
return False
- # The parameter's OWN close tag decides: if it closes after ``pos`` the position is
- # argument data (even across literal function closes); an unclosed one falls back to func close.
- # A closer at/before ``pos`` settles it. Bounded, so an absent alternate spelling
- # does not scan the whole suffix on every later function marker.
+ # The parameter's OWN close tag decides: one closing after pos makes the position argument data.
+ # Bounded, so an absent alternate spelling does not scan the whole suffix.
for tag in param_closers:
found = content.find(tag, last_param_start, min(len(content), pos + len(tag)))
if 0 <= found <= pos:
return False
- # Try the closer matching the opener spelling first. Cross-spelling closes are still
- # accepted; the normal form just needs a shorter scan.
+ # Try the closer matching the opener spelling first. Cross-spelling closes are still accepted; the normal
+ # form just needs a shorter scan.
if content.startswith("<param", last_param_start) and not content.startswith(
"<parameter", last_param_start
):
param_closers = tuple(reversed(param_closers))
if any(content.find(tag, pos + 1) >= 0 for tag in param_closers):
return True
- # With no parameter close anywhere, only a function close at/before ``pos`` proves the
- # candidate left the parameter. A later close and no close both mean still inside.
+ # With no parameter close anywhere, a later close and no close both mean still inside.
for tag in func_closers:
found = content.find(tag, last_param_start, min(len(content), pos + len(tag)))
if 0 <= found <= pos:
@@ -773,14 +745,14 @@ def _marker_coverage(content: str, markers) -> list[tuple[int, int]]:
for _s, be in brace_regions:
max_end = max(max_end, be)
brace_max_ends.append(max_end)
- events = [] # (position, order) with order 0 = braces-done, 1 = close marker
+ events = []
for idx, (_start, brace_end, _kind, _m) in enumerate(markers):
if brace_end >= 0:
events.append((brace_end, 0, _kind, idx))
for kind, close_re in (("json", _TC_END_TAG_RE), ("gemma", _TC_GEMMA_END_TAG_RE)):
for cm in close_re.finditer(content):
- # A close inside another call's balanced braces is quoted data; it
- # must not pop an earlier close-less marker and swallow a sibling.
+ # A close inside another call's balanced braces is quoted data; it must not pop an earlier close-
+ # less marker and swallow a sibling.
if brace_starts:
region_idx = bisect.bisect_left(brace_starts, cm.start()) - 1
inside_braces = region_idx >= 0 and cm.start() < brace_max_ends[region_idx]
@@ -794,9 +766,9 @@ def _marker_coverage(content: str, markers) -> list[tuple[int, int]]:
close_end_for: dict[int, int] = {}
for _pos, order, kind, payload in events:
if order == 0:
- waiting[kind].append(payload) # marker index, now awaiting its close
+ waiting[kind].append(payload)
elif waiting[kind]:
- close_end_for[waiting[kind].pop()] = payload # innermost open marker closes here
+ close_end_for[waiting[kind].pop()] = payload
coverage = []
for idx, (start, brace_end, _kind, _m) in enumerate(markers):
if brace_end < 0:
@@ -869,8 +841,7 @@ def parse_tool_calls_from_text(
in ``content`` (including its close tag when present), so a caller can
remove exactly the parsed markup and keep every other byte intact.
"""
- # Candidates starting inside a think block are rehearsals, skipped; blocks are kept, and a
- # think marker opening inside a call is argument data (excluded from spans).
+ # Candidates starting inside a think block are rehearsals; a think marker opening inside a call is argument data.
_think_spans = _think_spans_outside_tool_markup(content)
_think_starts = [s for s, _e in _think_spans]
@@ -881,10 +852,8 @@ def parse_tool_calls_from_text(
tool_calls: list[dict] = []
call_spans: list[tuple] = []
- # Collect JSON/Gemma markers; _marker_coverage decides nesting so a marker inside
- # another call's coverage (even one that failed to parse) is data, not executed. A
- # marker opening inside a think block is a rehearsal and is skipped.
- parsed_items = [] # (start, span_end, name, arguments) in document order
+ # A marker inside another call's coverage, even one that failed to parse, is data and is not executed.
+ parsed_items = []
markers = [mk for mk in _build_markers(content) if not _in_think(mk[0])]
coverage = _marker_coverage(content, markers)
covered_until = -1
@@ -894,7 +863,7 @@ def parse_tool_calls_from_text(
if covered_by_earlier_marker:
continue
if brace_end < 0:
- continue # unclosed: not parseable; the fallback still excludes its XML
+ continue
if not allow_incomplete:
tail = content[brace_end + 1 :].lstrip()
close_re = _TC_END_TAG_RE if kind == "json" else _TC_GEMMA_END_TAG_RE
@@ -904,8 +873,8 @@ def parse_tool_calls_from_text(
if kind == "json":
obj = json.loads(content[m.end() - 1 : brace_end + 1])
name = obj.get("name", "")
- # Accept ``parameters`` alias for ``arguments`` (Llama-3.2 drift inside
- # Hermes) and ``args`` (TML Inkling native calls).
+ # Accept "parameters" (Llama-3.2 drift inside Hermes) and "args" (TML Inkling) as aliases for
+ # "arguments".
arguments = obj.get("arguments")
if arguments is None:
arguments = obj.get("parameters")
@@ -913,9 +882,9 @@ def parse_tool_calls_from_text(
arguments = obj.get("args", {})
if isinstance(arguments, dict):
arguments = json.dumps(arguments)
- # Inkling echoes the bare tool name (and a role opener) before the
- # marker: <|message_model|>NAME<|content_invoke_tool_json|>{...}.
- # Fold that echo into the markup span so promotion removes it too.
+ # Inkling echoes the bare tool NAME (and a role opener) before the marker:
+ # <|message_model|>NAME<|content_invoke_tool_json|>{...}. Fold that echo into the markup span so
+ # promotion removes it too.
if name and content.startswith("<|content_invoke_tool_json|>", start):
pre = content[:start]
if pre.endswith(name):
@@ -1019,8 +988,8 @@ def parse_tool_calls_from_text(
)
call_spans.append((start, span_end))
- # Patterns 3+4: Mistral [TOOL_CALLS] and bare rehearsal via one balanced scan in document
- # order, so a Mistral call and a rehearsal in one message both parse.
+ # Patterns 3+4: Mistral [TOOL_CALLS] and bare rehearsal via one balanced scan in document order, so a
+ # Mistral call and a rehearsal in one message both parse.
if not tool_calls:
for start, end, kind, m in _iter_bracket_spans(
content, enabled_tool_names = enabled_tool_names
@@ -1031,13 +1000,13 @@ def parse_tool_calls_from_text(
closer = re.match(r"\s*\[/TOOL_CALLS\]", content[end:])
region_end = end + closer.end() if closer else end
if kind == "array":
- # Decode elements individually (comma-tolerant): one json.loads of the whole
- # body rejects the comma-less multi-call arrays Mistral/Ollama templates emit.
+ # Decode elements individually: one json.loads of the whole body rejects the comma-less multi-call
+ # arrays Mistral/Ollama templates emit.
payload, item_ends = _decode_array_items(content, m.end(), end)
if not payload:
continue
- # Tile the region so every byte belongs to exactly one span; a with_spans consumer
- # keeps skipped bytes visible and strips promoted markup exactly once.
+ # Tile the region so every byte belongs to exactly one span and promoted markup is stripped exactly
+ # once.
tile_start = start
last_span_idx = -1
for item_idx, item in enumerate(payload):
@@ -1051,9 +1020,8 @@ def parse_tool_calls_from_text(
except (json.JSONDecodeError, ValueError):
pass
if not isinstance(args, (dict, str)):
- # ``"arguments": null`` (or any non-object scalar) becomes {} like the
- # <tool_call> path, not the string "null" auto-heal would mangle to
- # a bogus {"query":"null"}.
+ # A non-object scalar becomes {}, not the string "null" that auto-heal would mangle into
+ # {"query":"null"}.
args = {}
tool_calls.append(
{
@@ -1061,9 +1029,9 @@ def parse_tool_calls_from_text(
"type": "function",
"function": {
"name": item.get("name", ""),
- # A bare scalar string stays raw (like the <tool_call> path);
- # json.dumps would double-encode it so the arg healer wraps
- # "weather" with its literal quotes.
+ # A bare scalar string stays raw; json.dumps would double-encode it so the arg
+ # healer wraps it with
+ # literal quotes.
"arguments": args if isinstance(args, str) else json.dumps(args),
},
}
@@ -1121,8 +1089,7 @@ def _tool_call_markup_spans(text: str) -> list[tuple[int, int]]:
stripped WITH the call, not kept as a reasoning block. Covers closed XML/bracket
calls and an unclosed XML call (run via allow_incomplete); without the open-ended
span the unclosed call's markup would leak after execution."""
- # Skip a lazy closed-pair pattern whose close token is absent: its finditer would rescan
- # to EOF from every opener (quadratic on a stream of unclosed openers).
+ # Skip a lazy closed-pair pattern whose close token is absent: its finditer rescans to EOF from every opener.
spans = []
for pat in _TOOL_CLOSED_PATS:
required = _PAT_REQUIRED_TOKEN.get(pat)
@@ -1155,8 +1122,7 @@ def _think_spans_outside_tool_markup(text: str) -> list[tuple[int, int]]:
a greedy unclosed <think> past the call is still that call's argument data."""
think_spans = [m.span() for m in _THINK_TAG_RE.finditer(text)]
call_spans = _tool_call_markup_spans(text)
- # Prefilled reasoning: the template opens <think> in the prompt, so add a leading span
- # (0..close) to skip calls rehearsed there; guarded so a stray close in a normal answer is safe.
+ # Prefilled reasoning: the template opens <think> in the prompt, so a leading span skips calls rehearsed there.
close = _THINK_CLOSE_RE.search(text)
if close is not None:
opener = _THINK_OPEN_RE.search(text)
@@ -1178,8 +1144,8 @@ def strip_outside_think(text: str, strip_segment) -> str:
blocks, preserving the blocks verbatim (tool-looking text inside is rehearsal).
``is_last`` is True only after the final block, so trailing-tail patterns apply
only there. Shared by every strip path so they stay consistent."""
- # A think marker opening inside a complete call is argument text; excluding it lets the
- # stripper see the whole call. START-tested, so an unclosed match stays argument data.
+ # A think marker opening inside a complete call is argument text; START-tested, so an unclosed match
+ # stays argument data.
think_spans = _think_spans_outside_tool_markup(text)
if not think_spans:
return strip_segment(text, True)
@@ -1205,14 +1171,13 @@ def _strip_gemma_native_spans(text: str, *, final: bool) -> str:
continue
brace_end = _balanced_brace_end(text, match.end() - 1, gemma_quotes = True)
if brace_end is None:
- # Unbalanced: nothing completes from here on. Drop the rest if final,
- # else keep it; stop either way (rescanning would be quadratic).
+ # Unbalanced: nothing completes from here on, so stop either way (rescanning would be quadratic).
if final:
out.append(text[cursor:start])
cursor = len(text)
break
- # Junk between } and <tool_call|> is malformed-call markup: strip through
- # the close, keep text after it. No close anywhere means stop (linear).
+ # Junk between } and the close marker is malformed-call markup: strip through the close, and no
+ # close anywhere means stop.
close = _TC_GEMMA_END_TAG_RE.search(text, brace_end + 1)
if close is None:
if final:
@@ -1294,9 +1259,8 @@ def _strip_markup_segment(
final: bool,
enabled_tool_names = None,
) -> str:
- # Bracket-tag calls (Mistral/rehearsal) first via balanced scan (any nesting depth,
- # rehearsal name-gated); then the quote-aware Gemma-native passes so a literal
- # <tool_call|> in an argument cannot truncate a block; finally the regex XML/tail sweeps.
+ # Bracket-tag balanced scan first, then the quote-aware Gemma passes so a literal marker in an
+ # argument cannot truncate a block, then the regex sweeps.
text = _strip_bracket_tag_calls(text, enabled_tool_names = enabled_tool_names)
text = _strip_closed_blocks_outside_gemma(text)
text = _strip_gemma_native_spans(text, final = final)
diff --git a/studio/backend/core/training/dataset_bounds.py b/studio/backend/core/training/dataset_bounds.py
index 1a231893741..a3921e49492 100644
--- a/studio/backend/core/training/dataset_bounds.py
+++ b/studio/backend/core/training/dataset_bounds.py
@@ -18,45 +18,34 @@ import re
import tempfile
from typing import Any, Optional
-# Deliberately loose: rows are consumed by things that never produce a step (the
-# eval split carved off the train set, rows train_on_responses_only drops when the
-# response template is missing). Running short only means re-reading the subset,
-# but that repeats rows, and 4x is still orders of magnitude under the datasets
-# this exists for.
+# Deliberately loose: rows are consumed by things that never produce a step (the eval split, rows
+# train_on_responses_only drops when the response template is missing), and running short only
+# re-reads the subset; 4x is still orders of magnitude under the datasets this exists for.
MAX_STEPS_ROW_SLACK = 4
# Below this a subset is small enough to skew a run for no meaningful saving.
MIN_MAX_STEPS_ROWS = 1024
-# Launchers that advertise a data-parallel process count. Read as env because this
-# module is torch-free and the bound is computed before any process group exists, so
-# torch.distributed cannot be asked: at bound time it is almost never initialised.
-# The MPI names routes/inference.py reads, plus the torchrun ones it does not, and a
-# looser test: it pairs each size with its rank partner because it refuses requests;
-# sizing a row bound only needs a bare size. Annotated per variable, several of which
-# are widely miscited.
+# Read as env because this module is torch-free and the bound is computed before any process group
+# exists, so torch.distributed cannot be asked. A bare size is enough here, unlike
+# routes/inference.py, which pairs each size with its rank partner because it refuses requests.
WORLD_SIZE_ENV_VARS = (
"WORLD_SIZE", # torchrun, accelerate launch, deepspeed. NOT set by any MPI
"LOCAL_WORLD_SIZE", # torchrun's --nproc-per-node; it sets WORLD_SIZE too
"MLX_WORLD_SIZE", # only mlx.launch's NCCL backend, which is CUDA-only
- "OMPI_COMM_WORLD_SIZE", # Open MPI / prterun, alongside OMPI_COMM_WORLD_RANK
+ "OMPI_COMM_WORLD_SIZE",
"PMI_SIZE", # MPICH and Intel MPI via Hydra; srun only under --mpi=pmi2
"PMIX_SIZE", # nothing sets this: PMIx answers job size through PMIx_Get
"MPI_WORLD_SIZE", # likewise undocumented in every MPI checked
"MV2_COMM_WORLD_SIZE", # MVAPICH2, and only under its mpirun_rsh launcher
)
-# An mlx.launch world size that is not a number in the env: of its five backends only
-# NCCL (CUDA) exports MLX_WORLD_SIZE; ring and JACCL export a path to a JSON file whose
-# outer list has one entry per rank, and that length is the world size. Without these
-# two an mlx.launch reads as one process, and under-counting recycles rows -- so the
-# Apple path, the only one MLX training runs on, would keep the bug this fixes.
+# Of mlx.launch's five backends only NCCL exports MLX_WORLD_SIZE; ring and JACCL export a path to
+# a JSON file whose outer list has one entry per rank.
WORLD_SIZE_ENV_FILES = (
"MLX_HOSTFILE", # ring backend: a path to [["ip:port", ...], ...], one per rank
"MLX_IBV_DEVICES", # jaccl backend: a path to the N x N RDMA matrix, one row per rank
)
-# A hostfile is a few hundred bytes per rank. Read a bounded prefix so a wrong path
-# (an env var pointed at something enormous) cannot pull a file into memory; a
-# truncated read fails to parse as JSON and is discarded, which is the safe answer.
+# Read a bounded prefix so a wrong path cannot pull an enormous file into memory; a truncated read
+# fails to parse as JSON and is discarded.
MAX_WORLD_SIZE_FILE_BYTES = 1 << 20
-# Written into a run's output directory at its first start; read back on resume.
# Its absence is the signal that a checkpoint predates the bound.
ROW_BOUND_MARKER_FILE = "unsloth_row_bound.json"
# transformers writes checkpoint-<global_step> and nothing else under that prefix.
@@ -111,10 +100,8 @@ def world_size_from_rank_files(environ: Any = None) -> int:
if value.lstrip()[:1] in ("[", "{"):
payload = json.loads(value[:MAX_WORLD_SIZE_FILE_BYTES])
elif os.path.isfile(value):
- # Binary, so the cap really is bytes: a text read() counts
- # CHARACTERS, so 4-byte codepoints would pull 4x the cap off disk.
- # json.loads takes bytes; non-UTF-8 raises UnicodeDecodeError (a
- # ValueError), caught below.
+ # Binary, so the cap really is bytes: a text read() counts CHARACTERS, and json.loads takes bytes
+ # anyway.
with open(value, "rb") as handle:
payload = json.loads(handle.read(MAX_WORLD_SIZE_FILE_BYTES))
else:
diff --git a/studio/backend/core/training/diffusion_checkpoint.py b/studio/backend/core/training/diffusion_checkpoint.py
index 2e540aec8c6..5326b743a47 100644
--- a/studio/backend/core/training/diffusion_checkpoint.py
+++ b/studio/backend/core/training/diffusion_checkpoint.py
@@ -59,8 +59,8 @@ CHECKPOINT_FORMAT = "unsloth-diffusion-checkpoint"
CHECKPOINT_VERSION = 1
CHECKPOINT_PREFIX = "checkpoint-"
-# Hidden, and does NOT start with "checkpoint-", so neither glob("checkpoint-*") nor a user's
-# file browser mistakes a half-written bundle for a real one.
+# Hidden, and not prefixed "checkpoint-", so neither glob("checkpoint-*") nor a file browser
+# mistakes a half-written bundle for a real one.
_STAGING_PREFIX = ".tmp-checkpoint-"
TRAINER_STATE_FILENAME = "trainer_state.json"
@@ -70,8 +70,7 @@ OPTIMIZER_FILENAME = "optimizer.pt"
SCHEDULER_FILENAME = "scheduler.pt"
RNG_FILENAME = "rng_state.pt"
-# How many checkpoint-<N> bundles to keep in a run's output dir. A LoRA bundle is a few MB, but
-# a 10000-step run at save_steps=50 would otherwise leave 200 of them.
+# A 10000-step run at save_steps=50 would otherwise leave 200 bundles.
DEFAULT_SAVE_TOTAL_LIMIT = 2
@@ -79,8 +78,8 @@ class ResumeError(ValueError):
"""A resume request that cannot be honoured. The message is shown to the user."""
-# ── identity ──────────────────────────────────────────────────────────────────
# Human labels for the identity fields that HARD-REJECT a resume, in report order.
+# ── identity ──────────────────────────────────────────────────────────────────
_IDENTITY_LABELS: tuple[tuple[str, str], ...] = (
("kind", "training type"),
("family", "model family"),
@@ -92,58 +91,44 @@ _IDENTITY_LABELS: tuple[tuple[str, str], ...] = (
("lora_alpha", "LoRA alpha"),
("lora_dropout", "LoRA dropout"),
("cfg_dropout", "caption dropout"),
- # The trajectory. Each of these is read from the INCOMING config by the trainer while the
- # optimizer moments and the scheduler position come from the bundle, so changing one
- # continues a run whose objective or learning-rate curve is no longer the one those moments
- # were produced under -- reported as a clean continue.
+ # Read from the INCOMING config while the moments and scheduler position come from the bundle, so
+ # changing one continues a trajectory those moments were not produced under.
("flow_shift", "timestep shift"),
("weighting_scheme", "loss weighting scheme"),
("snr_gamma", "min-SNR gamma"),
("lr_scheduler", "learning-rate schedule"),
("lr_warmup_steps", "learning-rate warmup"),
- # The input stream. Both trainers build the latent cache and the crop/flip variant plan
- # from these BEFORE restore_resume_state puts the RNG back, so changing one continues the
- # old optimizer and sampler against a different sequence of images.
+ # The latent cache and crop/flip plan are built from these BEFORE restore_resume_state puts the RNG
+ # back, so changing one continues the old sampler against a different image sequence.
("seed", "random seed"),
("cache_latents", "latent caching"),
("cache_mode", "latent cache path"),
- # The mode the LOOP actually ran in, which the request does not settle:
- # UNSLOTH_DIFFUSION_NO_LATENT_CACHE turns it off, and an over-budget cache falls back to
- # encoding in-loop. The two paths draw their crops and flips from different streams
- # (variant_rng versus the loop rng), so a restored RNG state does not reproduce the run.
+ # The mode the LOOP actually ran in: UNSLOTH_DIFFUSION_NO_LATENT_CACHE turns it off and an
+ # over-budget cache falls back to encoding in-loop, and the two paths draw crops and flips from
+ # different streams (variant_rng versus the loop rng), so a restored RNG state does not reproduce
+ # the run.
("cache_variants", "cached crop variants"),
("center_crop", "centre cropping"),
("random_flip", "random flipping"),
- # The matmul kernels. _apply_perf_flags routes CUDA matmuls through TF32 or strict fp32
- # from this, so restoring moments produced under one into a run using the other continues
- # the trajectory at a different numeric precision -- and off is the documented strict
- # reproducibility mode, which a resume that silently defaults it back on undoes.
+ # TF32 versus strict fp32 continues the trajectory at a different numeric precision, and off is the
+ # documented strict-reproducibility mode.
("enable_tf32", "TF32 matmuls"),
- # The update shape. The batch and accumulation counts decide how many samples each
- # optimizer step consumes, and the clip norm is applied to the restored moments on the
- # very next update, so all three change the trajectory the checkpoint was produced on.
+ # Batch, accumulation and clip norm all change the trajectory the restored moments were produced on.
("train_batch_size", "batch size"),
("gradient_accumulation_steps", "gradient accumulation"),
("max_grad_norm", "gradient clipping"),
- # Recorded from the beginning, but only for the UI until now. Both trainers resize and
- # crop the incoming images to it before the restored sampler and moments see anything, so
- # a checkpoint trained at 1024 could report a clean resume and finish at 768.
+ # Both trainers resize and crop to this before the restored sampler and moments see anything, so a
+ # bundle trained at 1024 could report a clean resume and finish at 768.
("resolution", "training resolution"),
("precision", "mixed precision"),
("base_precision", "base precision"),
- # What the frozen base was ACTUALLY converted to. fp8 and mxfp8 conversion can fail on the
- # host and both fall back to bf16, so a bundle requested as fp8 records fp8 while its
- # moments were produced against bf16 linears. Resuming that on a host where the conversion
- # succeeds continues those moments against an fp8 base and reports a clean resume.
+ # What the frozen base was ACTUALLY converted to: fp8/mxfp8 conversion can fall back to bf16, so
+ # recording the request reports a clean resume across different numerics.
("base_precision_effective", "resolved base precision"),
- # The EMA coefficient. restore_resume_state loads the old shadow tensors and update count
- # while the trainer builds LoRAEMA from the INCOMING decay, so every later update applies a
- # new coefficient to an average produced under the old one and the exported EMA adapter is a
- # hybrid -- reported as a faithful continuation.
+ # LoRAEMA is built from the INCOMING decay while the shadow tensors are restored, so a changed
+ # coefficient makes the exported EMA adapter a hybrid.
("ema_decay", "EMA decay"),
)
-# Fields whose value can legitimately be unknown on one side (an uncached Hub repo has no
-# resolvable revision; the start route knows the dataset only after its own discovery pass).
# Unknown on either side means "cannot tell", which must not be reported as a mismatch.
_OPTIONAL_IDENTITY_FIELDS = frozenset(
{
@@ -151,8 +136,8 @@ _OPTIONAL_IDENTITY_FIELDS = frozenset(
"dataset_fingerprint",
"lora_dropout",
"cfg_dropout",
- # Absent only in a manifest written before these were recorded, which reads as
- # "cannot tell" rather than as a mismatch.
+ # Absent only in a manifest written before these were recorded, which reads as "cannot tell" rather than
+ # as a mismatch.
"ema_decay",
"flow_shift",
"weighting_scheme",
@@ -169,9 +154,8 @@ _OPTIONAL_IDENTITY_FIELDS = frozenset(
"train_batch_size",
"gradient_accumulation_steps",
"max_grad_norm",
- # Known only inside the trainer, and only for the DiT path. Unknown on either side
- # reads as "cannot tell", which is what keeps the route's pre-eviction preflight and
- # the SDXL trainer unaffected.
+ # Known only inside the DiT trainer; unknown reads as "cannot tell", which keeps the route preflight
+ # and the SDXL trainer unaffected.
"base_precision_effective",
}
)
@@ -225,30 +209,24 @@ class CheckpointIdentity:
base_revision: Optional[str] = None
dataset_fingerprint: Optional[str] = None
lora_dropout: Optional[float] = None
- # Caption (classifier-free-guidance) dropout. The DiT loop asks rng.random() once per
- # sample when it is above zero and swaps in the empty prompt on a hit, so changing it
- # across a resume diverges the restored RNG stream on the very next step AND changes the
- # objective, while the run reports a clean continue. A bundle from before this field
- # existed reads None, which the optional rule skips rather than reporting as a mismatch.
+ # The DiT loop draws rng.random() once per sample while this is above zero, so a change diverges the
+ # restored RNG stream on the next step and changes the objective.
cfg_dropout: Optional[float] = None
- # Trajectory-defining knobs, all optional for the same reason: a bundle from before they
- # were recorded reads unknown, not mismatched.
+ # Trajectory-defining knobs, all optional for the same reason: a bundle from before they were recorded reads
+ # unknown, not mismatched.
flow_shift: Optional[str] = None
weighting_scheme: Optional[str] = None
- # Text, not a float: None is the documented way to DISABLE min-SNR, so a float field could
- # not tell "trained with it off" from "written before the field existed" -- and the optional
- # rule would skip the comparison for both. "off" is the disabled value; None is absence.
+ # Text, not a float: None is the documented way to DISABLE min-SNR, so a float field could not tell
+ # "trained with it off" from "written before the field existed".
snr_gamma: Optional[str] = None
lr_scheduler: Optional[str] = None
lr_warmup_steps: Optional[int] = None
seed: Optional[int] = None
- # Booleans as text ("on"/"off") for the same reason snr_gamma is: False is a real value and
- # None has to stay reserved for a manifest that predates the field.
+ # Booleans as text ("on"/"off") for the same reason snr_gamma is: False is a real value and None has to stay
+ # reserved for a manifest that predates the field.
cache_latents: Optional[str] = None
- # Resolved, not requested, and known only inside the trainer -- the start route leaves it
- # None, which the optional rule reads as "cannot tell" so the pre-eviction preflight is
- # unaffected. The trainer's own preflight, which runs with the real value, is where a
- # mismatched cache path is caught.
+ # Resolved, not requested, and left None by the start route so the pre-eviction preflight is
+ # unaffected; the trainer's own preflight catches a mismatched cache path.
cache_mode: Optional[str] = None
cache_variants: Optional[int] = None
center_crop: Optional[str] = None
@@ -258,17 +236,14 @@ class CheckpointIdentity:
gradient_accumulation_steps: Optional[int] = None
# Text, so 0.0 (clipping disabled) is a value and None stays "not recorded".
max_grad_norm: Optional[str] = None
- # Text for the same reason: 0.0 means EMA is off, which is a real setting, and None has to
- # stay reserved for a manifest that predates the field.
+ # Text for the same reason: 0.0 means EMA is off, which is a real setting, and None has to stay reserved for
+ # a manifest that predates the field.
ema_decay: Optional[str] = None
- # Post-conversion, set by the DiT trainer once it knows whether fp8/mxfp8 took. None
- # everywhere else, which the optional rule reads as "cannot tell".
+ # Post-conversion, set by the DiT trainer once it knows whether fp8/mxfp8 took. None everywhere else, which
+ # the optional rule reads as "cannot tell".
base_precision_effective: Optional[str] = None
- # WHICH repo ``base_revision`` was read from, which is not always ``base_model``: a gated base
- # is fetched from its byte-identical ungated mirror, and the two repos have different commit
- # SHAs for the same weights. Not a mismatch reason of its own -- it only says whether the two
- # revisions are comparable at all. None means a bundle written before mirrors existed, whose
- # revision came from the canonical base, so it reads as ``base_model`` below.
+ # WHICH repo base_revision was read from: a gated base is fetched from its byte-identical ungated
+ # mirror and the two repos carry different SHAs, so this only says whether they are comparable.
base_revision_repo: Optional[str] = None
def as_dict(self) -> dict[str, Any]:
@@ -282,10 +257,8 @@ class CheckpointIdentity:
"lora_target_modules": list(self.lora_target_modules),
"lora_rank": int(self.lora_rank),
"lora_alpha": int(self.lora_alpha),
- # Both trainers pass this to the LoRA constructor, so it changes the stochastic
- # forward pass the restored optimizer moments were produced against. Written as a
- # rounded float: a manifest from before this field existed reads as None, which the
- # optional-field rule below skips rather than reporting as a mismatch.
+ # Passed to the LoRA constructor, so it changes the stochastic forward pass the restored optimizer
+ # moments were produced against.
"lora_dropout": self.lora_dropout,
"cfg_dropout": self.cfg_dropout,
"flow_shift": self.flow_shift,
@@ -307,10 +280,8 @@ class CheckpointIdentity:
"base_precision_effective": self.base_precision_effective,
"precision": self.precision,
"base_precision": self.base_precision,
- # A hard identity field, not just a label: both trainers resize and crop the
- # incoming images to it before the restored sampler and moments see anything, so a
- # bundle trained at 1024 could report a clean resume and finish at 768. Loadable
- # tensors are not a faithful resume.
+ # A hard identity field: both trainers crop the images to this before the restored sampler sees
+ # anything, so loadable tensors are not a faithful resume.
"resolution": int(self.resolution),
}
@@ -371,19 +342,16 @@ class CheckpointIdentity:
mine, theirs = self.as_dict(), other.as_dict()
for field, label in _IDENTITY_LABELS:
a, b = mine.get(field), theirs.get(field)
- # None / "" is "cannot tell". NOT falsiness: a lora_dropout of 0.0 is a real value
- # and the commonest one, so truthiness would skip exactly the comparison that
- # matters -- 0.0 against 0.15.
+ # None / "" is "cannot tell", NOT falsiness: a lora_dropout of 0.0 is a real value and the
+ # commonest one, so truthiness would skip exactly the 0.0-against-0.15 comparison that matters.
if field in _OPTIONAL_IDENTITY_FIELDS and (a in (None, "") or b in (None, "")):
continue
if field == "base_revision" and not (
_revision_is_comparable(a) and _revision_is_comparable(b)
):
continue
- # Two commit SHAs are only comparable when they name the same repo. A gated base is
- # fetched from its byte-identical ungated mirror, so the same weights carry a
- # different SHA depending on which repo the host happened to pull from, and comparing
- # across the two reports a revision change that did not happen.
+ # Two commit SHAs are comparable only when they name the same repo: a gated base fetched from its
+ # ungated mirror carries a different SHA for identical weights.
if field == "base_revision" and _revision_repo(self) != _revision_repo(other):
continue
if a == b:
@@ -490,10 +458,8 @@ def _content_probe(path: Any) -> str:
with open(path, "rb") as handle:
digest.update(handle.read(_PROBE_BYTES))
if size > _PROBE_BYTES:
- # Every byte past the head is covered, either as the tail sample or, for a file
- # under 2x the probe, in full. The old `> 2 * _PROBE_BYTES` gate left a band
- # (64 KiB to 128 KiB, which is most JPEGs) reading its first 64 KiB and nothing
- # else, so a same-length replacement sharing a head went unnoticed.
+ # Every byte past the head is covered: the old "> 2 * _PROBE_BYTES" gate left 64-128 KiB files
+ # reading only their first 64 KiB, so a same-length replacement sharing a head went unnoticed.
handle.seek(max(_PROBE_BYTES, size - _PROBE_BYTES))
digest.update(handle.read(_PROBE_BYTES))
# ValueError: open() rejects an embedded NUL rather than raising OSError.
@@ -513,9 +479,8 @@ def _resolve_lora_targets(cfg: Any) -> tuple[str, ...]:
configured = tuple(cfg.lora_target_modules)
if str(getattr(cfg, "resolved_family", "") or "").strip().lower() == "sdxl":
return configured
- # Deliberately NOT wrapped in a try/except: quietly falling back to the generic tuple would
- # make the route compute a different fingerprint from the trainer, its preflight would pass,
- # and the mismatch would surface in the child AFTER the GPU residents were freed.
+ # Deliberately NOT wrapped: falling back to the generic tuple would make the route fingerprint
+ # differ from the trainer's, and the mismatch would surface only after the residents were freed.
from core.training.diffusion_dit_trainer import _SPECS, _select_lora_targets
spec = _SPECS.get(cfg.resolved_family)
@@ -573,9 +538,8 @@ def with_resolved_revision(identity: "CheckpointIdentity", base_model: Any) -> "
resolved = source_revision(base_model)
if not _revision_is_comparable(resolved) or resolved == identity.base_revision:
return identity
- # The repo travels with the revision: callers pass the repo they FETCHED from, which for a
- # gated base is the ungated mirror, and a SHA without the repo that produced it cannot be
- # compared against one from the other repo.
+ # The repo travels with the revision: a SHA without the repo that produced it cannot be compared
+ # against one from the other repo.
return replace(identity, base_revision = resolved, base_revision_repo = str(base_model or ""))
@@ -597,8 +561,8 @@ def identity_for_config(
from core.training.diffusion_train_extras import source_revision
targets = tuple(resolved_targets) if resolved_targets else _resolve_lora_targets(cfg)
- # ``base_model`` stays canonical everywhere else in the identity; only the revision pair
- # below follows the repo the weights are pulled from.
+ # ``base_model`` stays canonical everywhere else in the identity; only the revision pair below follows the
+ # repo the weights are pulled from.
fetch_base_model = str(getattr(cfg, "fetch_base_model", None) or cfg.base_model or "")
return CheckpointIdentity(
family = str(getattr(cfg, "resolved_family", "") or ""),
@@ -608,8 +572,8 @@ def identity_for_config(
lora_alpha = int(cfg.lora_alpha if cfg.lora_alpha is not None else cfg.lora_rank),
lora_dropout = round(float(getattr(cfg, "lora_dropout", 0.0) or 0.0), 6),
cfg_dropout = round(float(getattr(cfg, "cfg_dropout", 0.0) or 0.0), 6),
- # flow_shift is float | "auto" | None, so it is recorded as text: "auto" and the number
- # it resolves to are different runs, and comparing them as floats would lose that.
+ # flow_shift is float | "auto" | None, so it is recorded as text: "auto" and the number it
+ # resolves to are different runs, and comparing them as floats would lose that.
flow_shift = str(getattr(cfg, "flow_shift", None)),
weighting_scheme = str(getattr(cfg, "weighting_scheme", "") or "none"),
snr_gamma = _snr_gamma_key(getattr(cfg, "snr_gamma", None)),
@@ -625,20 +589,15 @@ def identity_for_config(
gradient_accumulation_steps = int(getattr(cfg, "gradient_accumulation_steps", 0) or 0),
max_grad_norm = f"{round(float(getattr(cfg, 'max_grad_norm', 0.0) or 0.0), 6)}",
ema_decay = f"{round(float(getattr(cfg, 'ema_decay', 0.0) or 0.0), 6)}",
- # The EFFECTIVE precision, not the request: a pre-Ampere card resolves bf16 to fp16, and
- # recording the request let an fp16 bundle resume in bf16 on a newer card -- restored
- # moments continuing under different frozen-base numerics, reported as a clean resume.
+ # The EFFECTIVE precision, not the request: a pre-Ampere card resolves bf16 to fp16, so recording
+ # the request let an fp16 bundle resume in bf16 on a newer card.
precision = effective_mixed_precision(cfg),
base_precision = str(getattr(cfg, "base_precision", "") or ""),
resolution = int(cfg.resolution),
kind = kind,
- # The revision of the repo actually FETCHED, recorded together with which repo that was.
- # Reading the canonical base instead loses the check entirely: the mirror is selected
- # precisely BECAUSE the canonical repo is not cached, so it has no local ref and every
- # mirror-backed bundle would record "unresolved", which mismatch_reason skips. Pairing
- # the revision with its repo keeps the comparison wherever it is meaningful (same repo
- # on both sides, including every pre-mirror bundle) and drops it only across a
- # canonical/mirror pair, whose SHAs differ for byte-identical weights.
+ # Record the revision of the repo actually FETCHED: the mirror is chosen precisely because the
+ # canonical repo is not cached, so reading the canonical one records "unresolved", which
+ # mismatch_reason then skips.
base_revision = source_revision(fetch_base_model),
base_revision_repo = fetch_base_model,
dataset_fingerprint = dataset_fingerprint(dataset_pairs) if dataset_pairs else None,
@@ -663,6 +622,8 @@ def capture_rng_state(streams: Optional[dict[str, Any]] = None) -> dict[str, Any
try:
payload["streams"][str(name)] = _random_state_to_json(stream.getstate())
except Exception: # noqa: BLE001 -- a stream we cannot read simply is not restored
+ # Could not hand it back. Leaving it on disk still beats deleting the only copy of another run's
+ # last resumable state.
continue
try:
import numpy as np
@@ -686,11 +647,9 @@ def capture_rng_state(streams: Optional[dict[str, Any]] = None) -> dict[str, Any
for i, state in enumerate(torch.cuda.get_rng_state_all()):
tensors[f"torch_cuda_{i}"] = state
except Exception: # noqa: BLE001 -- one device erroring loses the whole capture
- # ALL OR NOTHING on a CUDA host. Keeping the CPU half made the result non-empty,
- # so the write-time check accepted it and the preflight (which only requires
- # torch_cpu) offered it -- and the restore then left the CUDA generator at its
- # freshly seeded position, silently changing every latent, noise and timestep
- # draw from the resumed step on. An empty result fails the write instead.
+ # ALL OR NOTHING on a CUDA host: keeping the CPU half made the result non-empty, so the preflight
+ # (which only requires torch_cpu) offered it and the restore then left the CUDA generator freshly
+ # seeded, silently changing every latent and timestep draw.
tensors = {}
except Exception: # noqa: BLE001 -- torch RNG capture is best-effort
tensors = {}
@@ -745,20 +704,10 @@ def restore_rng_state(
if cpu is not None:
torch.set_rng_state(cpu.cpu().to(torch.uint8))
if torch.cuda.is_available():
- # Per device, not set_rng_state_all. That call needs one state per visible
- # device, so covering it with an all-or-nothing guard meant a bundle written
- # with fewer devices visible than the resume sees restored NOTHING -- including
- # cuda:0, the only device training touches. The whole per-step noise stream
- # (randn_like for the latent and the noise, randint for the timestep) then
- # restarted at the wrong offset while every other part of the resume looked
- # correct, so the run finished healthy and silently wrong: measured on a real
- # B200 SDXL run, 192/192 LoRA tensors differed from the uninterrupted control
- # and the first step after the resume was off by 0.024. The exposure is
- # ordinary -- the trainer is a spawned child inheriting CUDA_VISIBLE_DEVICES,
- # which Unsloth masks elsewhere, so a mask change between the run and the Resume
- # click was enough. Restoring device by device covers whatever the bundle
- # holds and leaves the rest alone, and is identical to the old behaviour when
- # the counts match (verified bit-exact against the same control).
+ # Per device, not set_rng_state_all: that needs one state per visible device, so a bundle written
+ # with fewer devices visible restored NOTHING, cuda:0 included, and the per-step noise stream
+ # (randn_like for latent and noise, randint for the timestep) restarted at the wrong offset while
+ # the run looked healthy.
for i in range(torch.cuda.device_count()):
state = tensors.get(f"torch_cuda_{i}")
if state is None:
@@ -834,28 +783,21 @@ def save_checkpoint(
if step < 0:
raise ValueError("checkpoint step must be >= 0")
if not adapter_state:
- # An empty safetensors file has no keys, so the bundle would fail its own validation and
- # read as "no checkpoint" -- a silent un-resumable run. Fail loudly at write time instead.
+ # An empty safetensors file has no keys, so the bundle would fail its own validation and read as "no
+ # checkpoint"; fail loudly at write time instead.
raise ValueError("refusing to write a checkpoint with no adapter tensors")
root = Path(output_dir).expanduser()
root.mkdir(parents = True, exist_ok = True)
doomed: list[Path] = []
if discard_existing:
- # Recorded now, deleted AFTER the new bundle is promoted. Clearing first means a kill or
- # an I/O failure mid-write leaves the directory with no checkpoint at all -- and any
- # earlier run sharing the folder has irreversibly lost its own, which is the opposite of
- # what this writer promises.
+ # Deleted only AFTER the new bundle is promoted: clearing first leaves the directory with no
+ # checkpoint at all if the write dies.
doomed = list_checkpoints(root)
else:
- # Re-reaching a step that already has a VALID bundle happens one way SAFELY: the run
- # resumed at N and stopped before N+1 completed, so its state is byte-identical to what
- # is already there, and keeping it avoids the one destructive branch in _promote
- # (swapping a good bundle out to make room), where a kill would leave the slot empty.
- #
- # It has to be THAT bundle, though. Resuming checkpoint-10 in a folder that also holds
- # checkpoint-15 and stopping at 15 is the same arithmetic and a completely different
- # situation: the freshly trained optimizer, scheduler, sampler and RNG would be dropped
- # on the floor and checkpoint_saved would name a bundle from an earlier run.
+ # Re-reaching a step that already has a valid bundle is safe only when the state is byte-identical,
+ # and keeping it avoids _promote's one destructive branch.
+ # It has to be THAT bundle: resuming checkpoint-10 in a folder that also holds checkpoint-15 and
+ # stopping at 15 would drop the freshly trained state and name an earlier run's bundle.
existing = root / f"{CHECKPOINT_PREFIX}{step}"
if (
source_checkpoint is not None
@@ -875,15 +817,13 @@ def save_checkpoint(
_save_tensors(save_file, ema_state, staging / EMA_FILENAME)
files["ema"] = EMA_FILENAME
if optimizer is not None:
- # torch.save, not safetensors: an optimizer state dict is a nested structure of
- # tensors AND scalars (bitsandbytes AdamW8bit additionally carries uint8 moments
- # plus their quantization maps), which safetensors cannot express.
+ # torch.save, not safetensors: optimizer state is nested tensors AND scalars (AdamW8bit adds uint8
+ # moments plus quantization maps), which safetensors cannot express.
_torch_save(torch, optimizer.state_dict(), staging / OPTIMIZER_FILENAME)
files["optimizer"] = OPTIMIZER_FILENAME
optimizer_class = optimizer_key(optimizer)
- # `adapter_state` is trainable_state_dict(model), i.e. named_parameters() order --
- # the same traversal the trainers build their param list from, so its key order IS
- # the optimizer's positional order.
+ # adapter_state is trainable_state_dict(model), i.e. named_parameters() order, the same traversal
+ # the trainers build their param list from, so its key order IS the optimizer's positional order.
optimizer_param_names = [str(name) for name in adapter_state]
if lr_scheduler is not None:
_torch_save(torch, lr_scheduler.state_dict(), staging / SCHEDULER_FILENAME)
@@ -893,12 +833,9 @@ def save_checkpoint(
rng_json = rng.get("json")
rng_tensors = rng.get("tensors") or {}
if not rng_tensors:
- # capture_rng_state never raises -- it returns what it managed to read -- so a
- # torch generator it could not snapshot produced a bundle with no rng file at
- # all. save_checkpoint returned happily, the service emitted checkpoint_saved,
- # and the history advertised a resumable stop that _assert_required_state then
- # refuses the moment the user clicks Resume. Fail the WRITE instead, which the
- # caller reports as resume_blocked_reason.
+ # capture_rng_state never raises, so a generator it could not snapshot produced a bundle with no
+ # rng file that _assert_required_state refuses on Resume; fail the WRITE instead, which the caller
+ # reports as resume_blocked_reason.
raise RuntimeError(
"the run's random-number state could not be captured, so this checkpoint "
"would not be resumable"
@@ -912,42 +849,28 @@ def save_checkpoint(
"kind": identity.kind,
"global_step": step,
"target_steps": int(target_steps),
- # Checkpoints are only ever taken on an optimizer-step boundary, so the
- # gradient-accumulation position is always clean. Recorded explicitly so a future
- # mid-accumulation checkpoint is a value change, not a format change.
+ # Checkpoints are only taken on an optimizer-step boundary; recorded explicitly so a future mid-
+ # accumulation checkpoint is a value change, not a format change.
"micro_step": 0,
"created_at": time.time(),
"identity": identity.as_dict(),
"sampler": sampler_state or None,
"rng": rng_json,
"ema_updates": int(ema_updates),
- # Which optimizer wrote the moments. bitsandbytes AdamW8bit stores "state1"/"state2",
- # torch AdamW stores "exp_avg"/"exp_avg_sq", and the trainers pick between them from
- # the host (bnb present, fused kernel available, UNSLOTH_DIFFUSION_FP32_OPTIM). Both
- # accept the other's state_dict and then KeyError on the first step, so the resume
- # has to compare this instead.
+ # bitsandbytes AdamW8bit stores state1/state2 and torch AdamW exp_avg/exp_avg_sq, and the trainers
+ # pick between them from the host (bnb present, fused kernel available,
+ # UNSLOTH_DIFFUSION_FP32_OPTIM); each loads the other's state_dict and then KeyErrors on the first
+ # step, so the resume compares this.
"optimizer_class": optimizer_class,
- # The trainable parameter names in the order the optimizer holds them. Its state is
- # keyed by POSITION, so a PEFT/diffusers upgrade that changes traversal order while
- # keeping the same names would rebind every Adam moment to a different tensor --
- # and many LoRA projections share a shape, so it loads cleanly and silently corrupts
- # the continued trajectory. Additive: an older bundle has no list and skips the
- # check rather than failing it.
+ # Optimizer state is keyed by POSITION, so a traversal-order change rebinds every Adam moment to a
+ # different same-shaped tensor and silently corrupts the continuation. Additive, so an older
+ # bundle skips the check.
"optimizer_param_names": optimizer_param_names,
- # Loop bookkeeping that is not state to restore into an object (running loss, wall
- # clock). Nested rather than merged, so a caller can never shadow a reserved key.
+ # Nested rather than merged, so a caller can never shadow a reserved key.
"progress": dict(progress or {}),
"files": files,
- # Byte size per listed file, so read_checkpoint can catch a truncation the
- # header parse structurally cannot. _valid_state_file walks a torch zip's pickle
- # to STOP and requires one non-empty data/ member; truncating the tensor storages
- # themselves leaves both intact, and torch.load then returns UNINITIALIZED memory
- # -- measured at +/-1e22 and non-finite -- which a resume feeds straight into the
- # optimizer as Adam moments. The run diverges on its first step while reporting a
- # clean resume. This is free to write and free to check.
- #
- # Additive on purpose: an older bundle has no sizes and skips the check rather
- # than failing it, and an older build ignores the key, so no version bump.
+ # Byte sizes catch a truncation the header parse structurally cannot: torch.load then returns
+ # UNINITIALIZED memory (measured +/-1e22 and non-finite) that a resume feeds in as Adam moments.
"file_sizes": _file_sizes(staging, files),
}
# LAST: the manifest is the completion marker, so nothing may be written after it.
@@ -958,16 +881,12 @@ def save_checkpoint(
shutil.rmtree(staging, ignore_errors = True)
raise
for stale in doomed:
- # `final` can be one of these when the new step reuses an existing bundle's name;
# _promote already replaced it, so removing it here would delete the new checkpoint.
if stale != final:
shutil.rmtree(stale, ignore_errors = True)
_prune_staging(root)
- # The source bundle is pinned as well as the one just written. A resumed run writes into
- # the directory it resumed FROM, so with keep=2 a run that resumes checkpoint-10 and saves
- # 20 and 30 prunes 10 -- and "stop without saving" then removes only what this run wrote,
- # leaving the ORIGINAL stopped run with no resume point at all. Its own bundle is not this
- # run's to spend.
+ # Pin the source bundle too: with keep=2 a run that resumes checkpoint-10 and saves 20 and 30 prunes
+ # 10, leaving the original stopped run with no resume point.
prune_checkpoints(
root,
keep = save_total_limit,
@@ -1018,9 +937,8 @@ def _write_text(path: Path, text: str) -> None:
_fsync_file(path)
-# errno values that mean "this platform / filesystem will not flush that handle" rather than
-# "the data did not make it". Windows' _commit needs write access, network and container
-# filesystems return EINVAL/ENOTSUP for fsync, and neither says anything about the write.
+# Windows' _commit needs write access, and network/container filesystems return EINVAL/ENOTSUP for
+# fsync; neither says the data did not make it.
_FSYNC_UNSUPPORTED = frozenset(
code
for code in (
@@ -1050,8 +968,7 @@ def _fsync_file(path: Path) -> None:
the durability guarantee rests on the per-file size check in read_checkpoint instead,
which catches the truncation a lost writeback actually produces."""
try:
- # O_RDWR, not O_RDONLY: Windows' _commit maps to FlushFileBuffers, which needs write
- # access on the handle. We just wrote the file, so we own it.
+ # O_RDWR, not O_RDONLY: Windows' _commit maps to FlushFileBuffers, which needs write access on the handle.
fd = os.open(str(path), os.O_RDWR)
except OSError:
return
@@ -1097,19 +1014,15 @@ def _promote(staging: Path, root: Path, step: int) -> Path:
if final.exists():
displaced = root / f"{_STAGING_PREFIX}replaced-{step}-{uuid.uuid4().hex[:8]}"
os.replace(final, displaced)
- # os.replace does NOT restamp the directory: the inode keeps whatever mtime the bundle
- # was written with, so an old checkpoint moved aside looked long-abandoned the instant
- # it arrived and a concurrent read handed it straight back into the slot we are about to
- # fill. Stamp it with the moment of the swap, which is what the grace is measuring.
+ # os.replace does NOT restamp the directory, so a moved-aside bundle looked long-abandoned the
+ # instant it arrived; stamp it with the moment of the swap.
with contextlib.suppress(OSError):
os.utime(displaced, None)
try:
os.replace(staging, final)
except OSError:
- # The slot is now empty and the only copy of the last resumable state is the one we
- # moved aside, so put it back before the failure propagates. Without this, a resave of
- # an occupied step (a resumed run rewriting checkpoint-15) that fails to promote leaves
- # the run with no checkpoint at all -- strictly worse than the stale one it replaced.
+ # Put the moved-aside copy back before the failure propagates, else a failed resave of an occupied
+ # step leaves the run with no checkpoint at all.
if displaced is not None:
with contextlib.suppress(OSError):
os.replace(displaced, final)
@@ -1118,10 +1031,8 @@ def _promote(staging: Path, root: Path, step: int) -> Path:
return final
-# ``.tmp-checkpoint-stale-<step>-<uuid>`` / ``.tmp-checkpoint-replaced-<step>-<uuid>``: the step
-# is what _promote encodes, so an orphan can be handed back to the slot it came from. ``stale``
-# is a promotion killed mid-swap; ``replaced`` is a bundle a later write displaced, held so the
-# slot can be restored if that write is later discarded.
+# The step is encoded in the name so an orphan can be handed back to its slot: "stale" is a
+# promotion killed mid-swap, "replaced" is a bundle a later write displaced.
_STALE_SLOT = re.compile(r"^(?:stale|replaced)-(\d+)-")
_REPLACED_SLOT = re.compile(r"^replaced-(\d+)-")
@@ -1144,7 +1055,6 @@ def _prune_staging(root: Path) -> None:
return
# Newest first, so a stacked slot gets its immediate predecessor back rather than whichever
- # entry the filesystem happened to list first.
def _written_at(path: Path) -> float:
try:
return path.stat().st_mtime
@@ -1158,14 +1068,8 @@ def _prune_staging(root: Path) -> None:
shutil.rmtree(entry, ignore_errors = True)
-# How long a ``replaced-`` bundle must have been sitting before a READ path will hand it back.
-# _promote leaves the slot empty for the microseconds between the swap-aside and the rename,
-# and a history or detail read landing in that window moved the old bundle back in -- so the
-# writer's own os.replace then failed against a directory that had reappeared, and the periodic
-# or stop-and-save checkpoint was lost. A few seconds covers a window measured in microseconds
-# while still recovering a genuinely crashed promotion on the user's next poll. Only
-# ``replaced-`` entries wait: a ``stale-`` orphan is from an older build and never in flight,
-# and writers pass 0 because they run after their own promotion.
+# A read landing in _promote's microsecond window moved the old bundle back, so the writer's
+# os.replace failed and the checkpoint was lost; only replaced- entries wait, and writers pass 0.
_LIVE_REPLACEMENT_GRACE_SECONDS = 5.0
@@ -1211,8 +1115,7 @@ def _recover_orphaned_slot(root: Path, entry: Path) -> bool:
try:
os.replace(entry, slot)
except OSError:
- # Could not hand it back. Leaving it on disk is still better than deleting the only
- # copy of the run's last resumable state.
+ # Leaving it on disk still beats deleting the only copy of the run's last resumable state.
pass
return True
@@ -1231,11 +1134,8 @@ def _retire_replaced_slots(root: Path, *, restore: bool) -> None:
except OSError:
return
- # NEWEST first per slot. Replacements stack: a run that crashed after displacing
- # checkpoint-15 leaves its copy behind, and a later run displacing the same slot leaves
- # another. Restoring whichever sorted first by NAME (a uuid, so effectively at random)
- # resurrected an older branch's optimizer state instead of the predecessor that was
- # actually in the slot, and later resumes then continued the wrong lineage.
+ # NEWEST first per slot: replacements stack, and restoring whichever sorted first by uuid
+ # resurrected an older branch's optimizer state instead of the actual predecessor.
def _written_at(path: Path) -> float:
try:
return path.stat().st_mtime
@@ -1253,8 +1153,6 @@ def _retire_replaced_slots(root: Path, *, restore: bool) -> None:
restored_slots.add(slot)
continue
except OSError:
- # Could not hand it back. Leaving it on disk still beats deleting the only copy
- # of another run's last resumable state.
continue
shutil.rmtree(entry, ignore_errors = True)
@@ -1300,13 +1198,8 @@ def prune_checkpoints(
back. They are excluded entirely, so the limit governs only what this run wrote."""
if keep <= 0:
return
- # Identity, not pathname, for the same reason the cleanup helpers use it: a run can save at
- # a step whose directory was already there (resume checkpoint-10 in a folder that also holds
- # checkpoint-15, save at 15), and the bundle occupying that path afterwards is THIS run's.
- # Excluding it as pre-existing let the limit be exceeded once per overwritten slot -- an old
- # checkpoint-10 with keep=2 and saves at 10, 20 and 30 retained all three -- which is real
- # disk on a long run. An entry with no identity (an older caller, or a bundle that could not
- # be read) keeps the pathname behaviour, which errs towards keeping.
+ # Identity, not pathname: a run can save at a step whose directory already existed, and excluding
+ # it as pre-existing let the limit be exceeded once per overwritten slot.
kept_from_before: set[Path] = set()
for entry in preexisting or ():
if isinstance(entry, tuple):
@@ -1414,8 +1307,8 @@ def discard_preexisting_checkpoints(
else:
keep[Path(entry)] = _bundle_identity(Path(entry))
for stale in list_checkpoints(root):
- # Identity, not pathname: a bundle this run WROTE OVER one that was here is this run's,
- # and deleting it would throw away the stop checkpoint the user asked for.
+ # Identity, not pathname: a bundle this run wrote OVER one that was here is this run's, and deleting
+ # it would throw away the stop checkpoint the user asked for.
if stale in keep and keep[stale] == _bundle_identity(stale):
shutil.rmtree(stale, ignore_errors = True)
_retire_replaced_slots(root, restore = False)
@@ -1454,11 +1347,8 @@ def clear_own_checkpoints(output_dir: str | os.PathLike[str], preexisting: "Iter
thing the user was trying not to disturb. Bundles are identified by the set captured before
the run's first write, not by step number, because a resume writes lower numbers than the
ones already there."""
- # Keyed by path AND by the bundle's own identity, not by path alone. A resumed run can
- # periodically save at a step whose directory was already there (resume checkpoint-10 in a
- # folder that also holds checkpoint-15, save at 15), which REPLACES that bundle: the
- # original is gone either way, and matching on the name alone then preserved the discarded
- # run's replacement as though it were the bundle it overwrote.
+ # Keyed by path AND by identity: a periodic save can REPLACE a pre-existing bundle, and matching
+ # on the name alone preserved the discarded run's replacement as the bundle it overwrote.
keep: dict[Path, Optional[tuple]] = {}
for entry in preexisting:
# A bare path (an older caller) keeps the pathname-only behaviour for that entry.
@@ -1471,10 +1361,8 @@ def clear_own_checkpoints(output_dir: str | os.PathLike[str], preexisting: "Iter
if stale in keep and keep[stale] == _bundle_identity(stale):
continue
shutil.rmtree(stale, ignore_errors = True)
- # A bundle this run wrote OVER a pre-existing one has just been removed, so its slot is free
- # and the displaced original goes back into it. Without this the discard takes the other
- # run's resume point with it: the replacement is not the bundle it overwrote, so the identity
- # match above cannot keep it, and the original was already gone.
+ # Put the displaced original back: the replacement is not the bundle it overwrote, so the identity
+ # match cannot keep it and the original was already gone.
_retire_replaced_slots(Path(output_dir).expanduser(), restore = True)
@@ -1489,8 +1377,8 @@ def _bundle_identity(path: Path) -> Optional[tuple]:
manifest = json.loads((path / TRAINER_STATE_FILENAME).read_text(encoding = "utf-8"))
except (OSError, ValueError):
return None
- # created_at is the moment THIS bundle's manifest was written -- the completion marker, so
- # it exists on every valid bundle and differs between two writes at the same step.
+ # created_at is when THIS bundle's manifest was written, the completion marker, so it exists on
+ # every valid bundle and differs between two writes at the same step.
return (manifest.get("created_at"), manifest.get("global_step"))
@@ -1558,14 +1446,13 @@ def read_checkpoint(path: str | os.PathLike[str]) -> Optional[dict[str, Any]]:
for role, name in files.items():
if not isinstance(name, str) or not name or Path(name).name != name:
return None
- # Optimizer / scheduler state can be validly tensor-free (SGD without momentum, a
- # constant LR schedule), so only the weight bundles must carry tensors.
+ # Optimizer and scheduler state can validly be tensor-free (SGD without momentum, a constant LR
+ # schedule), so only the weight bundles must carry tensors.
if not _valid_state_file(directory / name, require_tensor = role in ("adapter", "ema")):
return None
expected_size = sizes.get(role)
if isinstance(expected_size, int) and not isinstance(expected_size, bool):
- # Absent on a bundle written before sizes were recorded, which skips the check
- # rather than failing it -- an old checkpoint stays resumable.
+ # Absent on a bundle written before sizes were recorded, which skips the check rather than failing it.
try:
if (directory / name).stat().st_size != expected_size:
return None
@@ -1610,11 +1497,9 @@ def latest_valid_checkpoint(
its own. The identity gate cannot catch that -- same family, same base, same dataset, same
LoRA shape -- so the earlier run resumes the later run's optimizer moments, LR position
and RNG under its own config, and records the wrong lineage while doing it."""
- # Repair first. A promotion killed between the swap-aside and the rename leaves the run's
- # only bundle under the hidden stale name, and _prune_staging fixes that -- but it runs
- # after a later successful save, which a user who cannot resume will never reach. So the
- # read path that decides "there is nothing to resume" hands the orphan back to its slot
- # before it answers. Idempotent, and it only ever touches an EMPTY slot.
+ # A promotion killed mid-swap leaves the only bundle under the hidden stale name, and
+ # _prune_staging runs only after a later save a stuck user never reaches. Idempotent, and it only
+ # ever touches an EMPTY slot.
_recover_orphaned_slots(Path(output_dir).expanduser())
for candidate in list_checkpoints(output_dir):
manifest = read_checkpoint(candidate)
@@ -1632,8 +1517,7 @@ def latest_valid_checkpoint(
created = float(manifest.get("created_at") or 0.0)
except (TypeError, ValueError):
created = 0.0
- # A bundle with no usable created_at (0.0) is not fenced out by the upper bound:
- # it predates the field, and refusing it would make an old run unresumable.
+ # A bundle with created_at 0.0 predates the field, so the upper bound must not fence it out.
if created and created > float(not_after):
continue
# An extra gate for callers whose answer PINS the bundle it names.
@@ -1718,10 +1602,8 @@ def _source_checkpoint_bundle(
# Not the bundle this run resumed: something replaced the slot after the fact.
if not written or abs(written - float(source_created_at)) > 1e-6:
return None
- # Same gate the directory scan applies, for the same reason: this path is pinned back to the
- # start route verbatim, and read_checkpoint is only a header scan. A source whose required
- # state or real torch.load has gone since would be advertised as resumable and refused on
- # the first click.
+ # Same gate the directory scan applies: this path is pinned back verbatim and read_checkpoint is
+ # only a header scan, so a source that lost required state would be advertised as resumable.
if not _fully_loadable(candidate, manifest):
return None
return (candidate, manifest)
@@ -1758,9 +1640,8 @@ def describe_resume_state(
try:
root = Path(str(output_dir)).expanduser()
if not root.is_dir():
- # A resume into a NEW output dir that died before its first save never created it,
- # and the bundle it was validated against is still sitting in the source dir. The
- # fallback below is exactly for that, so it has to be reachable from here.
+ # A resume into a NEW output dir that died before its first save never created it, so the fallback
+ # below has to be reachable from here.
recovered = _source_checkpoint_bundle(source_checkpoint, source_created_at)
if recovered is None:
return {
@@ -1773,18 +1654,13 @@ def describe_resume_state(
root,
not_before = started_at,
not_after = ended_at,
- # The action this answer drives sends the exact path back, which the preflight
- # then treats as explicit and cannot scan past -- so the bundle named here has
- # to be one that will actually load, not merely one whose header parses.
+ # The exact path is sent back and treated as explicit, so the bundle named here has to be one that
+ # will actually load, not merely one whose header parses.
usable = _fully_loadable,
)
if found is None and source_checkpoint:
- # This run RESUMED and then ended before writing a bundle of its own -- an OOM on
- # the first restored step is the usual way. The source it was validated against is
- # still on disk and still correct to continue from, but it predates started_at, so
- # the fence above excludes it and the run reads as unresumable when retrying is the
- # obvious thing to do. Read it directly rather than widening the fence, which exists
- # to stop an unrelated earlier run's bundles being offered.
+ # A run that resumed and died before its first save has a source bundle predating started_at: read
+ # it directly rather than widening the fence, which exists to hide unrelated earlier runs.
found = _source_checkpoint_bundle(source_checkpoint, source_created_at)
except OSError:
return blank
@@ -1792,8 +1668,8 @@ def describe_resume_state(
if not list_checkpoints(root):
reason = "This run saved no resume checkpoint, so it cannot be continued."
elif started_at is not None and latest_valid_checkpoint(root) is not None:
- # Bundles are there, but every one predates this run: they belong to an earlier run
- # that trained into the same folder. Saying "corrupt" here would be a lie.
+ # Every bundle predates this run and belongs to an earlier run that trained into the same folder, so
+ # "corrupt" would be a lie.
reason = (
"This run saved no resume checkpoint of its own; the checkpoints in its folder "
"were left by an earlier run of the same adapter."
@@ -1803,8 +1679,7 @@ def describe_resume_state(
return {**blank, "resume_blocked_reason": reason}
path, manifest = found
step = int(manifest.get("global_step") or 0)
- # The target the resume would run to. The record's own total_steps wins (it is what the UI
- # replays); the manifest's copy covers a record written before that field existed.
+ # The record's own total_steps wins; the manifest's copy covers a record written before that field existed.
target = int(total_steps or manifest.get("target_steps") or 0)
if target and step >= target:
return {
@@ -1840,37 +1715,35 @@ def resolve_resume_dir(path_value: str) -> Path:
except ValueError as error:
message = str(error)
if not message.startswith("Resume checkpoint"):
- # The containment resolver raises an internal "path escapes root: <abs> -> <abs>"
- # that quotes server paths at the user; replace it with the same wording the LLM
- # resume flow shows.
+ # The containment resolver's message quotes server paths at the user; replace it with the resume
+ # flow's wording.
message = "Resume checkpoint must be inside Unsloth outputs."
raise ResumeError(message) from error
- # A name that cleans away to nothing (".", "outputs", "./.") lands on the outputs ROOT, where
- # the scan would sweep checkpoint dirs across unrelated runs. Same guard the start route
- # applies to output_dir.
+ # A name that cleans away to nothing (".", "outputs", "./.") lands on the outputs ROOT, where the
+ # scan would sweep checkpoint dirs across unrelated runs. Same guard the start route applies to
+ # output_dir.
if resolved.resolve(strict = False) == outputs_root().resolve(strict = False):
+ # Named like a bundle, is not one, and holds no bundles either: the original message is the accurate
+ # one.
raise ResumeError(
f"'{path_value}' is the outputs folder itself, not a training run inside it."
)
return resolved
-# What restore_resume_state() refuses to continue without. Kept here rather than inferred from
-# the manifest, because the manifest is exactly what a truncated or hand-edited bundle gets
-# wrong: _assert_loadable only opens the roles the bundle happens to LIST, so a checkpoint that
-# lists an adapter and nothing else passed the route preflight, the resident GPU model was
-# evicted, and the child then died on "This checkpoint carries no optimizer state".
-# The random.Random streams both trainers construct and hand to capture_rng_state. Named here
-# because the preflight has to know what a complete bundle carries.
+# Kept here rather than inferred from the manifest: _assert_loadable only opens the roles the
+# bundle LISTS, so an adapter-only checkpoint passed the route preflight and the child then died
+# after the resident GPU model was evicted.
+# The random.Random streams both trainers hand to capture_rng_state; the preflight has to know what
+# a complete bundle carries.
_TRAINER_RNG_STREAMS: tuple[str, ...] = ("loop", "variant")
_REQUIRED_STATE: tuple[tuple[str, str], ...] = (
("adapter", "the trained LoRA weights"),
("optimizer", "the optimizer moments"),
("scheduler", "the learning-rate schedule position"),
- # Both image trainers draw the latent, the noise and the timestep from torch immediately
- # after resume, so a bundle with no RNG file continues a different random stream while
- # restore_rng_state leaves the generator at its fresh seed and says nothing.
+ # Both image trainers draw the latent, noise and timestep from torch immediately after resume, so a
+ # bundle with no RNG file continues a different stream and says nothing.
("rng", "the random-number generator state"),
)
@@ -1884,20 +1757,14 @@ def _assert_required_state(path: Path, manifest: dict[str, Any]) -> None:
for role, label in _REQUIRED_STATE
if not isinstance(listed.get(role), str) or not listed.get(role)
]
- # The sampler's permutation lives in the manifest rather than a file. Required for an
- # image bundle, where both trainers always supply a sampler and the child refuses without
- # it -- after the route has evicted the resident models. Left optional for any other kind,
- # whose trainer may legitimately have none.
+ # Required for an image bundle, where both trainers always supply a sampler and the child refuses
+ # without it; optional for any other kind.
if str(manifest.get("kind") or "image") == "image" and not isinstance(
manifest.get("sampler"), dict
):
missing.append("the dataset sampler position")
- # The torch generator lives in the rng FILE; the two random.Random streams the trainers own
- # (the loop's index/crop draws and the latent-cache variant picks) live in the manifest
- # beside it. restore_rng_state is per-part best-effort, so a bundle that lists the file but
- # has lost either stream restores torch and silently leaves those two at their fresh seeds:
- # crop and flip selection diverges on the first step, and the sampler draws a different
- # permutation once its saved cycle ends. Both trainers always write both.
+ # restore_rng_state is per-part best-effort, so a bundle that lists the rng file but lost either
+ # random.Random stream leaves crop/flip selection and the sampler permutation freshly seeded.
rng_manifest = manifest.get("rng")
saved_streams = rng_manifest.get("streams") if isinstance(rng_manifest, dict) else None
if not isinstance(saved_streams, dict) or not all(
@@ -1978,8 +1845,8 @@ def _assert_loadable(path: Path, manifest: dict[str, Any]) -> None:
if role == "rng" and not (
isinstance(state, dict) and state.get("torch_cpu") is not None
):
- # An rng file with no torch state restores nothing torch draws from, which
- # is every latent, noise and timestep the loop asks for.
+ # An rng file with no torch state restores nothing torch draws from, which is every latent, noise
+ # and timestep the loop asks for.
raise ResumeError(
f"'{path.name}' carries no torch random-number state, so the run "
"would continue on a different random stream. Resume from an earlier "
@@ -2006,14 +1873,9 @@ def preflight_resume(
``identity`` may leave ``dataset_fingerprint`` unset on the first (pre-discovery) pass;
that comparison is then skipped and re-run once the images are known."""
root = resolve_resume_dir(path_value)
- # An explicit checkpoint-<N> directory resumes exactly that step; a run directory takes
- # its newest complete bundle.
- #
- # The name alone does not settle which it is. An adapter can legitimately be called
- # "checkpoint-2026", and its output directory then matches the bundle pattern while
- # containing checkpoint-11 rather than a trainer_state.json of its own -- read as an
- # explicit bundle it is simply rejected, with the real checkpoint sitting inside it. So the
- # explicit branch is taken only when the path IS a valid bundle.
+ # The name alone does not settle it: an adapter can legitimately be called "checkpoint-2026", so
+ # its output directory matches the bundle pattern while holding no trainer_state.json of its own.
+ # The explicit branch is taken only when the path IS a valid bundle.
explicit = read_checkpoint(root) if checkpoint_step(root) >= 0 else None
candidates: list[tuple[Path, dict]]
if explicit is not None:
@@ -2021,8 +1883,6 @@ def preflight_resume(
else:
candidates = iter_valid_checkpoints(root)
if not candidates and checkpoint_step(root) >= 0:
- # Named like a bundle, is not one, and holds no bundles either: the original
- # message is the accurate one.
raise ResumeError(
f"'{root.name}' is not a complete training checkpoint (it is missing files, "
"or was left behind by an interrupted save)."
@@ -2032,20 +1892,15 @@ def preflight_resume(
"No complete training checkpoint was found for this run, so there is nothing to "
"resume from. Start a new run instead."
)
- # Newest first, and a failure here is not the end of the search when a DIRECTORY was given:
- # the header scan that produced this list is deliberately cheap, so the newest bundle can
- # fail torch.load or turn out to be missing required state while the retained older one is
- # perfectly good. Stopping at the first failure made the retention policy pointless. An
- # EXPLICIT bundle has no alternatives, so its first failure is still the answer.
+ # Newest first, and the cheap header scan means the newest bundle can still fail torch.load while a
+ # retained older one is good; stopping at the first failure made retention pointless.
first_error: Optional[ResumeError] = None
for path, manifest in candidates:
try:
return _validated_resume(path, manifest, identity, target_steps)
except ResumeError as exc:
- # "Already at the target" is an answer, not a damaged bundle. Falling past it
- # walked back to checkpoint-400 of a run that finished at 500 and retrained
- # completed work -- the exact rollback the fence exists to prevent. Only a bundle
- # that cannot be USED is a reason to try the retained one behind it.
+ # "Already at the target" is an answer, not a damaged bundle: falling past it walked back to an
+ # earlier checkpoint and retrained completed work.
if getattr(exc, "terminal", False):
raise
if first_error is None:
@@ -2080,8 +1935,8 @@ def _validated_resume(
_assert_loadable(path, manifest)
step = int(manifest.get("global_step") or 0)
if target_steps and step >= int(target_steps):
- # Terminal: the newest bundle having nothing left to train says nothing is WRONG with
- # it, so the directory scan must stop here rather than offering the retained older one.
+ # Terminal: nothing is WRONG with the newest bundle, so the scan must stop rather than offer the
+ # retained older one.
raise _terminal(
ResumeError(
f"This checkpoint is already at step {step} of {int(target_steps)}, so there "
diff --git a/studio/backend/core/training/diffusion_clip_formats.py b/studio/backend/core/training/diffusion_clip_formats.py
index f87c7d6a9ab..10d44862406 100644
--- a/studio/backend/core/training/diffusion_clip_formats.py
+++ b/studio/backend/core/training/diffusion_clip_formats.py
@@ -17,8 +17,7 @@ not count is a dataset that uploads fine and then trains on nothing.
from __future__ import annotations
-# Containers the video decode path (PyAV) opens. Lowercase, with the leading dot, so a
-# ``Path.suffix.lower()`` can be tested against the set directly.
+# Lowercase, with the leading dot, so a Path.suffix.lower() can be tested against the set directly.
CLIP_EXTS = frozenset({".mp4", ".mov", ".mkv", ".webm", ".m4v", ".avi"})
__all__ = ["CLIP_EXTS"]
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 010ac0c9ab0..4a1c91f4848 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -83,7 +83,8 @@ from core.training.diffusion_train_extras import (
save_ema_adapter,
)
-# Per-family LoRA target modules (attention projections). FLUX / Qwen double-stream blocks also carry added-kv projections; Z-Image is single-stream.
+# Per-family LoRA target modules (attention projections). FLUX / Qwen double-stream blocks also
+# carry added-kv projections; Z-Image is single-stream.
_FLUX_TARGETS = (
"to_q",
"to_k",
@@ -96,7 +97,8 @@ _FLUX_TARGETS = (
)
_QWEN_TARGETS = _FLUX_TARGETS
_ZIMAGE_TARGETS = ("to_q", "to_k", "to_v", "to_out.0")
-# The Krea 2 authors' recommended defaults (their DreamBooth script): attention + SwiGLU + text-fusion projector + conditioning embedders. For long runs they suggest narrowing to attention.
+# The Krea 2 authors' recommended defaults (their DreamBooth script): attention + SwiGLU + text-
+# fusion projector + conditioning embedders. For long runs they suggest narrowing to attention.
_KREA2_TARGETS = (
"img_in",
"final_layer.linear",
@@ -138,23 +140,26 @@ class _FamilySpec:
lora_targets: tuple[str, ...]
# bf16 only (Z-Image overflows fp16 and its RoPE/embedder run in fp32).
force_bf16: bool
- # Approximate dense-bf16 transformer weight size for the family's DEFAULT base, used by
- # base_precision="auto" to pick a mode that fits free VRAM. A family covering more than one
+ # Used by base_precision="auto" to pick a mode that fits free VRAM; a family covering more than one
# size (flux.2-klein is 4B and 9B) must not be sized off this alone -- see _dense_bf16_gb.
dense_bf16_gb: float
- # Phased load so the multi-GB transformer never coexists with the text encoders + VAE: ``load_conditioners`` builds the
- # pipeline WITHOUT it and returns (pipe, vae); ``load_transformer`` then loads it alone once the conditioners are freed.
+ # Phased load so the multi-GB transformer never coexists with the text encoders + VAE:
+ # load_conditioners builds the pipeline WITHOUT it and returns (pipe, vae), then load_transformer
+ # loads it alone once the conditioners are freed.
load_conditioners: Callable[..., tuple[Any, Any]]
load_transformer: Callable[..., Any]
# Encode a list of captions -> a per-caption tuple of CPU tensors (the family's embeds).
encode_prompts: Callable[..., list[tuple]]
# Encode a pixel tensor [B,3,H,W] in [-1,1] -> latents (family-normalised, on device).
encode_latents: Callable[..., Any]
- # Encode pixels to (A, B) affine posterior params so a per-step sample is A + B * randn. B is None for a deterministic family.
+ # Encode pixels to (A, B) affine posterior params so a per-step sample is A + B * randn. B is None
+ # for a deterministic family.
encode_latent_stats: Callable[..., tuple]
- # Collate per-caption embed tuples into one batched tuple on device. ``pad_to`` pins a fixed text length for variable-length embeds (compile).
+ # Collate per-caption embed tuples into one batched tuple on device. ``pad_to`` pins a fixed text
+ # length for variable-length embeds (compile).
collate: Callable[..., tuple]
- # One transformer forward: (transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype) -> model_pred aligned with target = noise - latents.
+ # One transformer forward: (transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype)
+ # -> model_pred aligned with target = noise - latents.
forward: Callable[..., Any]
# Save the LoRA in diffusers format via the family pipeline's save_lora_weights.
save: Callable[..., None]
@@ -195,7 +200,8 @@ def _training_sigma_table(scheduler, flow_shift):
mu = float(getattr(sc, "max_shift", None) or math.log(3.0))
shifted = scheduler.time_shift(mu, 1.0, sigmas)
if getattr(sc, "shift_terminal", None):
- # The stretch scales off the table's own final sigma, so it must see the full descending schedule, never a batch.
+ # The stretch scales off the table's own final sigma, so it must see the full descending schedule,
+ # never a batch.
shifted = scheduler.stretch_shift_to_terminal(shifted)
return shifted
s = float(flow_shift)
@@ -248,7 +254,7 @@ def _encoders_to_device(pipe, device) -> None:
try:
enc.to(device)
except (ValueError, RuntimeError, NotImplementedError):
- pass # already-placed 4-bit encoder / non-movable module
+ pass
def _bnb_4bit_config():
@@ -258,12 +264,14 @@ def _bnb_4bit_config():
load_in_4bit = True,
bnb_4bit_quant_type = "nf4",
bnb_4bit_compute_dtype = torch.bfloat16,
- # Double quantization compresses the per-block absmax scales with a second 8-bit pass: ~0.4 bits/param off the frozen base at no fidelity cost, which matters most on the 20B+ DiTs.
+ # Double quantization compresses the per-block absmax scales with a second 8-bit pass: ~0.4 bits/param
+ # off the frozen base at no fidelity cost, which matters most on the 20B+ DiTs.
bnb_4bit_use_double_quant = True,
)
-# Kept as a module name for existing callers/tests; the heuristic itself moved to diffusion_train_common so config validation can use it without importing this module.
+# The heuristic moved to diffusion_train_common so config validation can use it without importing
+# this module; the name is kept for existing callers.
_repo_is_prequantized = repo_is_prequantized
@@ -320,7 +328,8 @@ def _load_dit_transformer(transformer_cls, cfg, device, base_precision):
transformer = transformer.to(device)
return transformer
- # Dense load for bf16 / fp8 / mxfp8 / int8. int8 quantizes AFTER the LoRA attaches: quantizing first makes peft dispatch TorchaoLoraLinear, whose peft-0.18 constructor rejects the torchao-0.16 config API.
+ # int8 quantizes AFTER the LoRA attaches: quantizing first makes peft dispatch TorchaoLoraLinear,
+ # whose peft-0.18 constructor rejects the torchao-0.16 config API.
return transformer_cls.from_pretrained(
cfg.base_model,
subfolder = "transformer",
@@ -360,9 +369,8 @@ def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:
512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family)
),
)
- # After quantize_, as the helper requires: it reparents the Linears it wraps. Not
- # best-effort -- a raise means the base is quantized but not safely runnable, which is the
- # one state worse than either end.
+ # After quantize_, as the helper requires (it reparents the Linears it wraps). Not best-effort: a
+ # raise means the base is quantized but not safely runnable.
apply_small_m_padding(transformer, "int8", family)
@@ -428,7 +436,8 @@ def _mx_module_filter(mod, fqn: str) -> bool:
return False
if fqn.endswith("proj_out") or ".proj_out." in fqn:
return False
- # Skip biased linears: the torchao 0.17 MX path swaps the weight for a wrapper that drops the bias, changing the output the LoRA regresses against.
+ # Skip biased linears: the torchao 0.17 MX path swaps the weight for a wrapper that drops the bias,
+ # changing the output the LoRA regresses against.
if getattr(mod, "bias", None) is not None:
return False
return mod.in_features % 32 == 0 and mod.out_features % 32 == 0
@@ -550,22 +559,25 @@ def _resolve_base_precision(cfg, spec, device) -> str:
"sm_89+, mxfp8 sm_100+) and this is a ROCm/AMD GPU. Use base_precision='nf4', "
"'bf16', or 'auto'."
)
- # int8 has no runtime fallback, so an explicit int8 against a missing torchao (or the Windows-ROCm stub) would leave the
- # transformer dense with compile disabled. The auto pick and /info gate on a FUNCTIONAL torchao; do the same here.
+ # int8 has no runtime fallback, so gate on a FUNCTIONAL torchao here as the auto pick and /info do;
+ # find_spec is satisfied by the Windows-ROCm stub.
if mode == "int8" and not has_functional_torchao():
raise ValueError(
"base_precision='int8' needs a functional torchao install; this host's "
"torchao is missing or the non-functional Windows-ROCm stub. Use "
"base_precision='nf4', 'bf16', or 'auto'."
)
- # The stub answers torchao.float8 / torchao.prototype.mx_formats with a no-op that reports success, so the run would report fp8 while training bf16.
- # Keyed on the stub, not has_functional_torchao(): that probes int8's symbols, and a real-but-partial torchao must still reach the arch checks below.
+ # The stub answers torchao.float8 / torchao.prototype.mx_formats with a no-op that reports success,
+ # so the run would report fp8 while training bf16.
+ # Keyed on the stub, not has_functional_torchao(): that probes int8's symbols, and a real-but-
+ # partial torchao must still reach the arch checks below.
if mode in ("fp8", "mxfp8") and is_stubbed("torchao"):
raise ValueError(
f"base_precision={mode!r} is not available on this host: torchao is the "
"non-functional Windows-ROCm stub. Use base_precision='nf4', 'bf16', or 'auto'."
)
- # mxfp8 needs Blackwell (sm100+): its MX GEMM raises at the first training step, after a full dense load. Re-check here to fail fast for a stale client.
+ # mxfp8 needs Blackwell (sm100+): its MX GEMM raises at the first training step, after a full dense
+ # load. Re-check here to fail fast for a stale client.
if mode == "mxfp8" and device == "cuda":
try:
import torch
@@ -578,22 +590,22 @@ def _resolve_base_precision(cfg, spec, device) -> str:
"Use base_precision='bf16', 'int8', 'nf4', or 'auto'."
)
return mode
- # auto may only resolve to the dense modes when the run uses bf16 compute, mirroring the normalized() rule for explicit dense modes; otherwise stay on the nf4 floor.
+ # auto may only resolve to the dense modes when the run uses bf16 compute, mirroring the
+ # normalized() rule for explicit dense modes; otherwise stay on the nf4 floor.
if getattr(cfg, "mixed_precision", "bf16") != "bf16":
return "nf4"
prequant = repo_is_prequantized(cfg.base_model)
free_gb = None
capability = None
has_fp8 = False
- # int8 has no runtime fallback, so gate the auto pick on a FUNCTIONAL torchao: find_spec("torchao") is satisfied by the Windows-ROCm stub whose quantize_ is a no-op.
# Auto must not select a torchao mode on ROCm.
has_torchao = has_functional_torchao() and not torch_is_rocm()
if device == "cuda":
try:
import torch
- # Windows ROCm over-reports free VRAM (#8403), which would pick a
- # dense precision the card cannot actually hold.
+ # Windows ROCm over-reports free VRAM (#8403), which would pick a dense precision the card cannot
+ # actually hold.
from utils.hardware import trusted_mem_get_info
free_gb = trusted_mem_get_info()[0] / 1e9
@@ -667,14 +679,16 @@ def _flux_collate(
):
import torch
- # FLUX embeds are fixed-length, so a plain cat batches them; text_ids are shared position ids, identical across prompts.
+ # FLUX embeds are fixed-length, so a plain cat batches them; text_ids are shared position ids,
+ # identical across prompts.
pe = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)
pooled = torch.cat([e[1] for e in entries]).to(device = device, dtype = weight_dtype)
text_ids = entries[0][2].to(device = device, dtype = torch.float32)
return (pe, pooled, text_ids)
-# Per-run cache of the step-invariant FLUX conditioning tensors (RoPE image ids + guidance vector): their shapes are fixed once resolution/batch are. Cleared at run start.
+# Per-run cache of the step-invariant FLUX conditioning tensors: their shapes are fixed once
+# resolution and batch are. Cleared at run start.
_FLUX_STATIC: dict[tuple, tuple] = {}
@@ -685,7 +699,8 @@ def _flux_static_inputs(bsz, h, w, device):
key = (bsz, h, w, str(device))
hit = _FLUX_STATIC.get(key)
if hit is None:
- # Position ids drive RoPE and are indices, not activations, so keep them float32 regardless of the training dtype.
+ # Position ids drive RoPE and are indices, not activations, so keep them float32 regardless of the
+ # training dtype.
img_ids = FluxPipeline._prepare_latent_image_ids(bsz, h // 2, w // 2, device, torch.float32)
guidance = torch.full((bsz,), 1.0, device = device, dtype = torch.float32)
hit = _FLUX_STATIC[key] = (img_ids, guidance)
@@ -728,7 +743,8 @@ def _qwen_load_conditioners(cfg, device, weight_dtype):
def _qwen_load_transformer(cfg, device, weight_dtype, base_precision):
- # The prequant default ships the transformer 4-bit and loads trainable as-is under nf4; the dense modes need the 20B Qwen/Qwen-Image base.
+ # The prequant default ships the transformer 4-bit and loads trainable as-is under nf4; the dense
+ # modes need the 20B Qwen/Qwen-Image base.
from diffusers import QwenImageTransformer2DModel
return _load_dit_transformer(QwenImageTransformer2DModel, cfg, device, base_precision)
@@ -762,10 +778,11 @@ def _qwen_latent_affine(vae, ref):
def _qwen_encode_latents(vae, pixel_values):
import torch
- # AutoencoderKLQwenImage is a 3D (video) VAE: add a temporal dim, encode, and normalise by the per-channel latents_mean / latents_std.
- px = pixel_values.to(torch.float32).unsqueeze(2) # [B,3,1,H,W]
+ # AutoencoderKLQwenImage is a 3D (video) VAE: add a temporal dim, encode, and normalise by the per-
+ # channel latents_mean / latents_std.
+ px = pixel_values.to(torch.float32).unsqueeze(2)
with torch.no_grad():
- lat = vae.encode(px).latent_dist.sample() # [B,16,1,h,w]
+ lat = vae.encode(px).latent_dist.sample()
mean, std = _qwen_latent_affine(vae, lat)
return (lat - mean) / std
@@ -789,7 +806,8 @@ def _qwen_collate(
import torch
import torch.nn.functional as F
- # Qwen embeds are variable-length: pad to the batch max (or a pinned ``pad_to`` bucket under compile) and batch the validity mask with them.
+ # Qwen embeds are variable-length: pad to the batch max (or a pinned ``pad_to`` bucket under
+ # compile) and batch the validity mask with them.
seqs = [e[0].shape[1] for e in entries]
target = max(pad_to or 0, max(seqs))
pes, masks = [], []
@@ -804,7 +822,8 @@ def _qwen_collate(
masks.append(mask)
pe_b = torch.cat(pes).to(device = device, dtype = weight_dtype)
mask_b = torch.cat(masks).to(device)
- # A single unpadded sample keeps the legacy None mask (identical math; avoids a behaviour delta for existing single-image runs).
+ # A single unpadded sample keeps the legacy None mask (identical math; avoids a behaviour delta for
+ # existing single-image runs).
if len(entries) == 1 and entries[0][1] is None and target == seqs[0]:
mask_b = None
return (pe_b, mask_b)
@@ -816,7 +835,8 @@ def _qwen_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, devi
pe, mask = embeds_batch
bsz, c, f, h, w = noisy.shape
packed = QwenImagePipeline._pack_latents(noisy, bsz, c, h, w)
- # Each batch entry is a LIST of one (frame, h/2, w/2) tuple: the transformer indexes sample[0] / sample[1:] per entry, so a flat list breaks it.
+ # Each batch entry is a LIST of one (frame, h/2, w/2) tuple: the transformer indexes sample[0] /
+ # sample[1:] per entry, so a flat list breaks it.
img_shapes = [[(1, h // 2, w // 2)]] * bsz
pred = transformer(
hidden_states = packed,
@@ -845,7 +865,8 @@ def _zimage_load_conditioners(cfg, device, weight_dtype):
def _zimage_load_transformer(cfg, device, weight_dtype, base_precision):
- # Prequant default loads 4-bit as-is under nf4; the dense modes use the bf16 Tongyi-MAI base. Z-Image is bf16 only (its RoPE/embedder run fp32; fp16 overflows).
+ # Prequant default loads 4-bit as-is under nf4; the dense modes use the bf16 Tongyi-MAI base.
+ # Z-Image is bf16 only (its RoPE/embedder run fp32; fp16 overflows).
from diffusers import ZImageTransformer2DModel
return _load_dit_transformer(ZImageTransformer2DModel, cfg, device, base_precision)
@@ -877,7 +898,8 @@ def _zimage_encode_latents(vae, pixel_values):
def _zimage_encode_latent_stats(vae, pixel_values):
- # Z-Image trains from the posterior mode (deterministic), so the cached entry is the final latent: B is None and the loop skips the per-step draw.
+ # Z-Image trains from the posterior mode (deterministic), so the cached entry is the final latent: B
+ # is None and the loop skips the per-step draw.
return _zimage_encode_latents(vae, pixel_values), None
@@ -895,7 +917,8 @@ def _zimage_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, de
import torch
(caps,) = embeds_batch
- # List I/O: one [C,1,H,W] latent + one [seq,2560] caption per sample. The timestep convention is REVERSED ((1000 - t) / 1000) and the prediction is NEGATED.
+ # List I/O: one [C,1,H,W] latent + one [seq,2560] caption per sample. The timestep convention is
+ # REVERSED ((1000 - t) / 1000) and the prediction is NEGATED.
x_list = list(noisy.unsqueeze(2).unbind(dim = 0))
t_norm = (1000 - timesteps) / 1000
out = transformer(x_list, t_norm, list(caps), return_dict = False)[0]
@@ -913,7 +936,8 @@ def _zimage_save(pipe_cls, out_dir, transformer_lora_layers):
# ── Krea 2 ────────────────────────────────────────────────────────────────────
def _krea2_load_conditioners(cfg, device, weight_dtype):
- # The krea repo ships transformers-5.x style configs the pinned 4.x line cannot parse, so the conditioning pipeline is assembled per-component (diffusion_krea2.py).
+ # The krea repo ships transformers-5.x style configs the pinned 4.x line cannot parse, so the
+ # conditioning pipeline is assembled per-component (diffusion_krea2.py).
import torch
from core.inference.diffusion_krea2 import load_krea2_pipeline
@@ -925,7 +949,8 @@ def _krea2_load_conditioners(cfg, device, weight_dtype):
def _krea2_load_transformer(cfg, device, weight_dtype, base_precision):
- # The transformer subfolder is diffusers-format. No prequant repo yet, so nf4 quantizes the 12B transformer on the fly.
+ # The transformer subfolder is diffusers-format. No prequant repo yet, so nf4 quantizes the 12B
+ # transformer on the fly.
from diffusers import Krea2Transformer2DModel
return _load_dit_transformer(Krea2Transformer2DModel, cfg, device, base_precision)
@@ -937,7 +962,8 @@ def _krea2_encode_prompts(pipe, captions, device):
out = []
with torch.no_grad():
for cap in captions:
- # encode_prompt pads/truncates to max_sequence_length, so every embed is [1, 512, num_text_layers, 2560] with a [1, 512] mask (static shapes; padding sits before the assistant suffix, as at training).
+ # encode_prompt pads/truncates to max_sequence_length, so every embed is
+ # [1, 512, num_text_layers, 2560] with a [1, 512] mask (static shapes, padding before the suffix).
pe, mask = pipe.encode_prompt(
prompt = cap,
device = device,
@@ -948,7 +974,8 @@ def _krea2_encode_prompts(pipe, captions, device):
return out
-# Krea 2 conditions on the Qwen-Image VAE with the same per-channel latents_mean / latents_std normalisation, so latent encoding is shared.
+# Krea 2 conditions on the Qwen-Image VAE with the same per-channel latents_mean / latents_std normalisation, so
+# latent encoding is shared.
_krea2_encode_latents = _qwen_encode_latents
_krea2_encode_latent_stats = _qwen_encode_latent_stats
@@ -971,7 +998,8 @@ def _krea2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, dev
from diffusers import Krea2Pipeline
pe, mask = embeds_batch
- # [B,16,1,H,W] to [B, (H/2)*(W/2), 64] 2x2 patches. Krea2Pipeline._pack_latents is an instance method (it reads self.patch_size), so the packing is inlined like the reference script.
+ # Krea2Pipeline._pack_latents is an instance method (it reads self.patch_size), so the packing is
+ # inlined like the reference script.
bsz, c, _f, h, w = noisy.shape
packed = noisy.reshape(bsz, c, h // 2, 2, w // 2, 2)
packed = packed.permute(0, 2, 4, 1, 3, 5).reshape(bsz, (h // 2) * (w // 2), c * 4)
@@ -999,9 +1027,10 @@ def _krea2_save(pipe_cls, out_dir, transformer_lora_layers):
)
+# Both variants share Flux2Transformer2DModel and the upstream packing/forward conventions,
+# differing only in the conditioning stack (dev: Mistral-3-Small; Klein: Qwen3) and size. Latents
+# train patchified and batch-norm-normalised, from the posterior MODE (deterministic).
# ── FLUX.2 (dev + Klein) ──────────────────────────────────────────────────────
-# Both variants share Flux2Transformer2DModel and the upstream DreamBooth packing/forward conventions, differing only in the conditioning
-# stack (dev: Mistral-3-Small; Klein: Qwen3) and size. Latents train patchified and batch-norm-normalised, from the posterior MODE (deterministic).
_FLUX2_COMMON_TARGETS = (
# Double-stream blocks: separate q/k/v plus the ModuleList out proj.
"to_k",
@@ -1011,17 +1040,16 @@ _FLUX2_COMMON_TARGETS = (
# Single-stream blocks use one fused qkv+mlp input projection.
"to_qkv_mlp_proj",
)
-# Their output projection is a plain Linear called ``to_out``. A bare suffix also matches the
-# double-stream ModuleList, which PEFT cannot wrap, so the upstream trainers name each single block
-# explicitly: 24 possible blocks for Klein and 48 for dev. Missing high indexes are harmless on the
-# smaller Klein-4B layout, which has 20 single blocks.
+# A bare "to_out" suffix also matches the double-stream ModuleList, which PEFT cannot wrap, so each
+# single block is named explicitly; missing high indexes are harmless on the Klein-4B layout.
_FLUX2_KLEIN_TARGETS = _FLUX2_COMMON_TARGETS + tuple(
f"single_transformer_blocks.{i}.attn.to_out" for i in range(24)
)
_FLUX2_DEV_TARGETS = _FLUX2_COMMON_TARGETS + tuple(
f"single_transformer_blocks.{i}.attn.to_out" for i in range(48)
)
-# The references train dev with its guidance-distillation vector at 3.5 (Klein applies it only when the variant config carries guidance_embeds).
+# The references train dev with its guidance-distillation vector at 3.5 (Klein applies it only when
+# the variant config carries guidance_embeds).
_FLUX2_TRAIN_GUIDANCE = 3.5
@@ -1047,7 +1075,8 @@ def _flux2_encode_prompts(pipe, captions, device):
out = []
with torch.no_grad():
for cap in captions:
- # encode_prompt pads to max_sequence_length, so embeds are fixed-length and text_ids are per-caption [1, txt_len, 4]. The per-class text_encoder_out_layers defaults are left to the pipeline.
+ # encode_prompt pads to max_sequence_length, so embeds are fixed-length and text_ids are per-caption
+ # [1, txt_len, 4]. The per-class text_encoder_out_layers defaults are left to the pipeline.
pe, text_ids = pipe.encode_prompt(
prompt = cap,
device = device,
@@ -1072,7 +1101,8 @@ def _flux2_encode_latents(vae, pixel_values):
def _flux2_encode_latent_stats(vae, pixel_values):
- # FLUX.2 trains from the posterior mode (deterministic): the cached entry is the final latent itself; B is None and the loop skips the per-step sampling draw.
+ # FLUX.2 trains from the posterior mode (deterministic): the cached entry is the final latent
+ # itself; B is None and the loop skips the per-step sampling draw.
return _flux2_encode_latents(vae, pixel_values), None
@@ -1084,13 +1114,14 @@ def _flux2_collate(
):
import torch
- # Fixed-length embeds give a plain concat; text_ids are PER-SAMPLE (unlike FLUX.1's shared grid), so they concat too.
+ # Fixed-length embeds give a plain concat; text_ids are PER-SAMPLE (unlike FLUX.1's shared grid), so
+ # they concat too.
pe = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)
text_ids = torch.cat([e[1] for e in entries]).to(device = device, dtype = torch.float32)
return (pe, text_ids)
-# Step-invariant FLUX.2 position ids, keyed on (batch, patched h, patched w, device): the ids derive only from the latent shape, so rebuilding them every step is allocator churn.
+# The ids derive only from the latent shape, so rebuilding them every step is allocator churn.
_FLUX2_STATIC: dict[tuple, Any] = {}
@@ -1109,7 +1140,8 @@ def _flux2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, dev
from diffusers import Flux2Pipeline
pe, text_ids = embeds_batch
- # ``noisy`` is already in the patchified normalised space, so packing is the [B,C,H,W] to [B, H*W, C] flatten the pipeline uses.
+ # ``noisy`` is already in the patchified normalised space, so packing is the [B,C,H,W] to [B, H*W, C]
+ # flatten the pipeline uses.
packed = Flux2Pipeline._pack_latents(noisy)
img_ids = _flux2_static_img_ids(noisy, device)
guidance = None
@@ -1127,7 +1159,8 @@ def _flux2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, dev
return_dict = False,
)[0]
pred = pred[:, : packed.size(1)]
- # _unpack_latents_with_ids scatters per sample; diffusers 0.39 stacks them itself (its list annotation is stale), other builds hand back the list. Same-resolution batches align either way.
+ # _unpack_latents_with_ids scatters per sample; diffusers 0.39 stacks them itself (its list
+ # annotation is stale), other builds hand back the list. Same-resolution batches align either way.
unpacked = Flux2Pipeline._unpack_latents_with_ids(pred, img_ids)
if isinstance(unpacked, (list, tuple)):
unpacked = torch.stack(unpacked)
@@ -1152,39 +1185,16 @@ def _flux2_klein_save(pipe_cls, out_dir, transformer_lora_layers):
)
+# Milestone one trains from STILL IMAGES: a 1-frame clip is valid LTX-2 input (its VAE compresses
+# time by 8) and style LoRAs converge on 20-50 stills, so this spec reuses the image dataset layer.
+# AUDIOVISUAL: forward REQUIRES audio arguments and diffusers has no video-only escape hatch, so
+# feed a one-token audio stream with isolate_modalities=True and keep audio out of the loss and
+# out of the LoRA targets. The reason _LTX2_TARGETS names its modules in full.
+# Two-stage conditioning: the Gemma3-12B hidden states are ~370 MB per caption, so encode_prompts
+# runs both stages and caches only the small connector output.
+# Video-stream attention only, fully qualified: a bare "to_q" would also match
+# audio_attn1/audio_attn2 and the cross-modality attentions.
# ── LTX-2 (video) ─────────────────────────────────────────────────────────────
-# The first VIDEO family. Milestone one trains from STILL IMAGES: a 1-frame clip is a valid
-# LTX-2 input (its VAE compresses time by 8, so (1 - 1) // 8 + 1 = 1 latent frame), and the
-# community consensus is that style / character LoRAs converge on 20-50 stills while only
-# motion LoRAs need coherent clips. So this spec reuses the shared image dataset layer
-# verbatim and never decodes a video; clip datasets are a follow-up.
-#
-# Two things make LTX-2 unlike the image DiTs:
-#
-# 1. It is AUDIOVISUAL. Every block carries a second, audio-side stream (audio_attn1 /
-# audio_attn2 / audio_ff) plus two cross-modality attentions (audio_to_video_attn,
-# video_to_audio_attn), and ``forward`` takes ``audio_hidden_states`` /
-# ``audio_encoder_hidden_states`` as REQUIRED arguments. Lightricks' own trainer handles a
-# video-only run by passing ``audio=None``, which their transformer short-circuits so the
-# audio and cross-modality branches never execute. diffusers has no such escape hatch, so
-# the equivalent here is: feed a minimal audio token stream, pass
-# ``isolate_modalities=True`` (which turns off the a2v/v2a cross attention for every
-# block, so the audio stream cannot influence the video prediction), keep audio out of the
-# loss, and keep the audio-side modules out of the LoRA targets. The audio stream is one
-# token for a still, so the wasted compute is negligible. This is a deliberate divergence
-# from upstream and is the reason ``_LTX2_TARGETS`` names its modules in full.
-#
-# 2. Conditioning is a TWO-STAGE stack: a Gemma3-12B encoder whose hidden states are
-# [batch, 1024, 3840 * num_layers] (~370 MB per caption in bf16), followed by a small
-# ``connectors`` module that reduces them to the [batch, 1024, 3840] the transformer
-# actually consumes. Caching the raw encoder output would be ~50x larger than caching the
-# connector output for no benefit, so ``encode_prompts`` runs BOTH stages and caches only
-# the connector result, then the loop frees the encoder and the connectors together.
-#
-# Targets: the video-stream attention projections ONLY. Fully qualified on purpose -- a bare
-# "to_q" would also match audio_attn1/audio_attn2/audio_to_video_attn/video_to_audio_attn.
-# This is exactly the split Lightricks document for a video-only run and ship in their
-# video_inpainting / video_outpainting LoRA configs.
_LTX2_TARGETS = (
"attn1.to_q",
"attn1.to_k",
@@ -1196,12 +1206,8 @@ _LTX2_TARGETS = (
"attn2.to_out.0",
)
-# Frame rate the still is placed at on the temporal RoPE axis. LTX-2's rotary temporal
-# coordinate is measured in SECONDS (pixel-frame index / fps), so a 1-frame clip at the
-# family's native 24 fps lands exactly where the first latent frame of a generated 24 fps
-# clip lands -- the position every rendered video contains. Lightricks' preprocessor instead
-# pins images to fps = 1.0, which places them at a temporal coordinate no 24 fps clip ever
-# visits; we take the inference-matching choice and note the divergence.
+# LTX-2's rotary temporal coordinate is in SECONDS, so a 1-frame clip at the native 24 fps lands
+# where a generated clip's first latent frame lands; Lightricks instead pin images to fps=1.0.
_LTX2_TRAIN_FPS = 24.0
@@ -1209,8 +1215,7 @@ def _ltx2_load_conditioners(cfg, device, weight_dtype):
from diffusers import LTX2Pipeline
pipe, vae = _load_pipe_without_transformer(LTX2Pipeline, cfg, device)
- # The connectors are the second half of the conditioning stack and are NOT a text_encoder
- # attribute, so ``_encoders_to_device`` never reaches them; place them explicitly.
+ # The connectors are not a text_encoder attribute, so _encoders_to_device never reaches them.
if getattr(pipe, "connectors", None) is not None:
pipe.connectors.to(device)
return pipe, vae
@@ -1225,8 +1230,8 @@ def _ltx2_encode_prompts(pipe, captions, device):
import torch
_encoders_to_device(pipe, device)
- # The Gemma3 hidden states are per-LAYER stacked ([1, 1024, 3840 * layers]); only the
- # connector output reaches the transformer, so that is what gets cached.
+ # The Gemma3 hidden states are per-LAYER stacked ([1, 1024, 3840 * layers]); only the connector output
+ # reaches the transformer, so that is what gets cached.
out = []
with torch.no_grad():
for cap in captions:
@@ -1237,12 +1242,8 @@ def _ltx2_encode_prompts(pipe, captions, device):
max_sequence_length = 1024,
device = device,
)
- # Read AFTER encode_prompt, exactly where the pipeline reads it. encode_prompt
- # sets ``tokenizer.padding_side = "left"`` itself (Gemma wants left padding for
- # chat-style prompts), so a tokenizer that loaded reporting "right" would have had
- # that stale value baked in here -- and the connectors build the valid-token mask
- # from this, so every caption shorter than the 1024 pad length would be masked on
- # the wrong end and the cached conditioning would not match what inference builds.
+ # Read AFTER encode_prompt, which sets padding_side="left" itself: the connectors build the valid-
+ # token mask from this, so a stale "right" would mask every short caption on the wrong end.
padding_side = getattr(getattr(pipe, "tokenizer", None), "padding_side", "left")
video_emb, audio_emb, conn_mask = pipe.connectors(pe, mask, padding_side = padding_side)
out.append((video_emb.cpu(), audio_emb.cpu(), conn_mask.cpu()))
@@ -1261,8 +1262,8 @@ def _ltx2_latent_affine(vae, ref):
def _ltx2_encode_latents(vae, pixel_values):
import torch
- # A still is a 1-frame clip: [B,3,H,W] -> [B,3,1,H,W]. The VAE compresses 32x spatially
- # and 8x temporally, so a 512px still becomes [B,128,1,16,16].
+ # A still is a 1-frame clip: [B,3,H,W] -> [B,3,1,H,W]. The VAE compresses 32x spatially and 8x
+ # temporally, so a 512px still becomes [B,128,1,16,16].
px = pixel_values.to(torch.float32).unsqueeze(2)
with torch.no_grad():
lat = vae.encode(px).latent_dist.sample()
@@ -1289,7 +1290,6 @@ def _ltx2_collate(
import torch
# encode_prompt pads to max_sequence_length, so every connector embed is [1, 1024, 3840]
- # and a plain concat batches them; ``pad_to`` is moot.
video = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)
audio = torch.cat([e[1] for e in entries]).to(device = device, dtype = weight_dtype)
mask = torch.cat([e[2] for e in entries]).to(device)
@@ -1358,9 +1358,8 @@ def _ltx2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, devi
audio_hidden_states = audio_noisy,
encoder_hidden_states = video_emb,
audio_encoder_hidden_states = audio_emb,
- # LTX-2 conditions on the UNSCALED timestep (its config carries
- # timestep_scale_multiplier = 1000 and the pipeline passes scheduler timesteps
- # through as-is), unlike the FLUX / Qwen families' timestep / 1000.
+ # LTX-2 conditions on the UNSCALED timestep (its config carries timestep_scale_multiplier = 1000
+ # and the pipeline passes scheduler timesteps through as-is), unlike FLUX / Qwen's timestep / 1000.
timestep = timesteps,
sigma = timesteps,
encoder_attention_mask = mask,
@@ -1370,8 +1369,8 @@ def _ltx2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, devi
width = w,
fps = _LTX2_TRAIN_FPS,
audio_num_frames = audio_len,
- # Disable the audio-to-video / video-to-audio cross attention so the placeholder
- # audio stream cannot perturb the video prediction the LoRA is regressing.
+ # Disable the a2v / v2a cross attention so the placeholder audio stream cannot perturb the video
+ # prediction the LoRA is regressing.
isolate_modalities = True,
return_dict = False,
)
@@ -1463,7 +1462,8 @@ _SPECS: dict[str, _FamilySpec] = {
family = "flux.2-dev",
lora_targets = _FLUX2_DEV_TARGETS,
force_bf16 = True,
- # 32B DiT; the Mistral conditioning stack (~46 GB bf16) is loaded, encoded and freed BEFORE this lands on the device (the shared phased load).
+ # 32B DiT; the Mistral conditioning stack (~46 GB bf16) is loaded, encoded and freed BEFORE this lands
+ # on the device (the shared phased load).
dense_bf16_gb = 64.5,
load_conditioners = _flux2_load_conditioners,
load_transformer = _flux2_load_transformer,
@@ -1478,9 +1478,8 @@ _SPECS: dict[str, _FamilySpec] = {
family = "ltx-2",
lora_targets = _LTX2_TARGETS,
force_bf16 = True,
- # 19B audiovisual DiT; the transformer index reports 37.76 GB bf16. The Gemma3-12B
- # conditioning stack (~24 GB resident) is loaded, encoded and freed BEFORE this lands
- # on the device, via the shared phased load.
+ # 19B audiovisual DiT (37.76 GB bf16); the Gemma3-12B conditioning stack is loaded, encoded and
+ # freed BEFORE this lands on the device, via the shared phased load.
dense_bf16_gb = 37.8,
load_conditioners = _ltx2_load_conditioners,
load_transformer = _ltx2_load_transformer,
@@ -1494,7 +1493,8 @@ _SPECS: dict[str, _FamilySpec] = {
}
-# HF repos gating access behind a license acceptance: training needs a token whose account accepted it. Checked by name (no network) so a missing token fails fast with an actionable message.
+# HF repos gating access behind a license acceptance: training needs a token whose account accepted
+# it. Checked by name (no network) so a missing token fails fast with an actionable message.
_GATED_TRAIN_REPOS = frozenset({"black-forest-labs/flux.1-dev", "black-forest-labs/flux.2-dev"})
@@ -1503,8 +1503,8 @@ def _assert_gated_access(base_model: str, hf_token: Optional[str]) -> None:
from core.inference.diffusion_families import _is_local_path
name = str(base_model or "").strip().lower()
- # A local clone named like the vendor repo is weights on disk, not a Hub fetch: no gate
- # applies, and refusing it by name alone is what made that documented layout untrainable.
+ # A local clone named like the vendor repo is weights on disk, not a Hub fetch: refusing it by name
+ # alone is what made that documented layout untrainable.
if _is_local_path(base_model):
return
if name in _GATED_TRAIN_REPOS and not (hf_token and str(hf_token).strip()):
@@ -1637,8 +1637,8 @@ def _build_latent_cache(
pass
a, b = _hold(a), _hold(b)
if not forced and not gated:
- # Size-gate the automatic cache off the first REAL encoded variant: packed 16-channel DiT latents x variants x images
- # can exhaust pinned RAM. Over budget we bail with the VAE resident. ``b`` is None for a deterministic-latent family.
+ # Size-gate the automatic cache off the first REAL encoded variant: packed latents x variants x
+ # images can exhaust pinned RAM.
per_variant = a.numel() * a.element_size()
if b is not None:
per_variant += b.numel() * b.element_size()
@@ -1769,13 +1769,14 @@ def _should_compile(
mode = (cfg.compile_transformer or "auto").strip().lower()
if device != "cuda" or mode == "off":
return False
- # torch.compile cannot trace the torchao int8 subclass in training (inductor rejects the aliased subclass graph outputs), so int8 always runs eager.
+ # torch.compile cannot trace the torchao int8 subclass in training (inductor rejects the aliased
+ # subclass graph outputs), so int8 always runs eager.
if base_precision == "int8":
return False
if mode == "on":
return True
- # auto: regional compile is the whole point of the dense modes (2.6x measured on Z-Image bf16) but is fragile over
- # bitsandbytes 4-bit modules, so it stays off for QLoRA. fp8/mxfp8 are only competitive compiled.
+ # Regional compile is the whole point of the dense modes (2.6x measured on Z-Image bf16) but is
+ # fragile over bitsandbytes 4-bit modules, so it stays off for QLoRA.
return base_precision in ("bf16", "fp8", "mxfp8")
@@ -1813,14 +1814,15 @@ def _maybe_compile_transformer(
try:
dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)
if dynamo_cfg is not None:
- # Heterogeneous-block DiTs (Z-Image: ~11 distinct block shapes) exceed dynamo's default recompile limit of 8; bump it like the inference speed layer does.
+ # Heterogeneous-block DiTs (Z-Image: ~11 distinct block shapes) exceed dynamo's default recompile
+ # limit of 8; bump it like the inference speed layer does.
for attr in ("recompile_limit", "cache_size_limit"):
if hasattr(dynamo_cfg, attr):
setattr(dynamo_cfg, attr, max(getattr(dynamo_cfg, attr) or 0, 64))
if hasattr(dynamo_cfg, "suppress_errors"):
dynamo_cfg.suppress_errors = True
- # dynamic=True matches the inference speed layer's proven default: on torch 2.10 / B200 the dynamic=False specialisation
- # failed with CUBLAS_STATUS_EXECUTION_FAILED. fullgraph only on a dense base: bnb 4-bit layers graph-break by design.
+ # dynamic=True matches the inference speed layer: on torch 2.10 / B200 the dynamic=False
+ # specialisation failed with CUBLAS_STATUS_EXECUTION_FAILED.
fn(fullgraph = not base_is_bnb, dynamic = True)
return True
except Exception as exc: # noqa: BLE001 -- optimisation only, never fatal
@@ -1845,7 +1847,8 @@ def run_dit_lora_training(
if spec is None:
raise ValueError(f"No DiT trainer for family {cfg.resolved_family!r}")
- # DiT families train in bf16, so an explicit fp16 request is refused, not silently upgraded. Validated before the heavy imports so a host without diffusers still sees the real error.
+ # DiT families train in bf16, so an explicit fp16 request is refused, not silently upgraded.
+ # Validated before the heavy imports so a host without diffusers still sees the real error.
if cfg.mixed_precision == "fp16" and spec.force_bf16:
raise ValueError(
f"{spec.family} LoRA training requires bf16: fp16 overflows its fp32 RoPE / "
@@ -1872,8 +1875,8 @@ def run_dit_lora_training(
return True
device = "cuda" if torch.cuda.is_available() else "cpu"
- # The flow-matching + 4-bit path is bf16 throughout (fp32 on a CPU-only box, to keep import/unit tests architecture-agnostic).
- # Fail fast on pre-Ampere CUDA, gating on NATIVE bf16 (capability major >= 8), since is_bf16_supported() counts emulation.
+ # Fail fast on pre-Ampere CUDA, gating on NATIVE bf16 (capability major >= 8), since
+ # is_bf16_supported() counts emulation.
if device == "cuda" and not native_bf16_supported():
raise ValueError(
"This trainer requires a bfloat16-capable GPU (Ampere or newer); "
@@ -1882,19 +1885,17 @@ def run_dit_lora_training(
weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32
_assert_trusted_base_model(cfg.base_model)
- # The repo this run will FETCH, which is what the start route preflights. Checking the
- # canonical id instead would raise here for a gated base that normalization already
- # redirected to its ungated mirror -- after the route had answered 200 and freed the
- # resident models, so the request fails as a dead job rather than as a fast 400.
+ # Check the repo this run will FETCH: the canonical id would raise for a gated base already
+ # redirected to its ungated mirror, after the route answered 200 and freed the residents.
_assert_gated_access(cfg.fetch_base_model or cfg.base_model, cfg.hf_token)
pairs = discover_image_caption_pairs(
cfg.data_dir, instance_prompt = cfg.instance_prompt, caption_column = cfg.caption_column
)
- # Resolve num_epochs into a concrete train_steps now the dataset size is known, and rebind cfg so every downstream read agrees.
+ # Resolve num_epochs into a concrete train_steps now the dataset size is known, and rebind cfg so every
+ # downstream read agrees.
cfg = replace(cfg, train_steps = resolve_train_steps(cfg, len(pairs)), num_epochs = 0)
- # Validate a resume request against this run's identity BEFORE the multi-GB phased load, so
- # a mismatched checkpoint fails in seconds. The identity uses the RESOLVED LoRA targets,
- # which for a DiT family are the spec's, not the generic default the config carries.
+ # Validate a resume against this run's identity BEFORE the multi-GB phased load, using the RESOLVED
+ # LoRA targets rather than the generic default the config carries.
identity = identity_for_config(
cfg,
dataset_pairs = pairs,
@@ -1914,14 +1915,14 @@ def run_dit_lora_training(
lora_path = None,
stopped = True,
steps_run = 0,
- # Same disposition the full path reports. A stop with save=false is a DISCARD
- # however early it lands, and without it the resume fallback offers the source
- # bundle back as though the attempt were still live.
+ # A stop with save=false is a DISCARD however early it lands; without it the resume fallback offers
+ # the source bundle back as though the attempt were still live.
discarded = not save_on_stop,
)
return str(out_dir)
- # TF32 / cudnn.benchmark for the run, restored on the way out (the trainer subprocess is disposable, but restoring keeps in-process callers clean).
+ # TF32 / cudnn.benchmark for the run, restored on the way out (the trainer subprocess is disposable, but
+ # restoring keeps in-process callers clean).
perf_snap = _apply_perf_flags(cfg, device)
try:
return _train_dit(
@@ -1953,9 +1954,8 @@ def _train_dit(
from peft import LoraConfig
from peft.utils import get_peft_model_state_dict
- # Same as the SDXL trainer: diffusers is only in sys.modules from here, and it
- # honours no env var, so its pipeline-loading bars can only be turned off now --
- # before the conditioning cache below loads the VAE and text encoders.
+ # diffusers honours no env var and is only in sys.modules from here, so its pipeline-loading bars
+ # can only be turned off now.
try:
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
@@ -1965,15 +1965,16 @@ def _train_dit(
use_lora_targets = _select_lora_targets(cfg.lora_target_modules, spec.lora_targets)
out_dir = Path(cfg.output_dir).expanduser()
# Load from the byte-identical public mirror selected during normalization, while keeping
- # cfg.base_model canonical for the adapter sidecar, completion event, and resume identity.
- # One fetch config covers the conditioning pipeline, transformer, and scheduler.
+ # cfg.base_model canonical for the adapter sidecar, completion event and resume identity.
fetch_cfg = replace(cfg, base_model = cfg.fetch_base_model or cfg.base_model)
- # Phase 0: the persistent conditioning cache (opt-in via cond_cache_dir). When every planned latent variant AND caption embedding is on disk, the run is "warm": the VAE and text encoders never load.
+ # Phase 0: the persistent conditioning cache (opt-in via cond_cache_dir). When every planned latent variant
+ # AND caption embedding is on disk, the run is "warm": the VAE and text encoders never load.
image_paths = [p for p, _ in pairs]
captions = [c for _, c in pairs]
uniq = sorted(set(captions))
- # CFG dropout swaps a sample's conditioning for the empty prompt, so its embedding must be precomputed alongside the captions (the encoders are freed right after).
+ # CFG dropout swaps a sample's conditioning for the empty prompt, so its embedding must be
+ # precomputed alongside the captions (the encoders are freed right after).
cfg_dropout = float(getattr(cfg, "cfg_dropout", 0.0) or 0.0)
to_encode = uniq + ([""] if cfg_dropout > 0 and "" not in uniq else [])
use_cache = cfg.cache_latents and os.environ.get(
@@ -1982,13 +1983,15 @@ def _train_dit(
pcache = None
if getattr(cfg, "cond_cache_dir", None):
try:
- # Namespace on the CHECKPOINT, not just the family: the keys carry only caption/image content, so one cache dir reused for two checkpoints would train on the other model's embeddings and latent stats.
+ # Namespace on the CHECKPOINT, not just the family: the keys carry only caption/image content, so
+ # one cache dir reused for two checkpoints would train on the other model's embeddings.
from .diffusion_train_extras import source_revision # noqa: PLC0415
namespace = f"{spec.family}_{cfg.base_model}_{source_revision(fetch_cfg.base_model)}"
pcache = PersistentConditioningCache(cfg.cond_cache_dir, namespace, cfg.resolution)
except Exception as exc: # noqa: BLE001 -- the cache is an optimisation, never fatal
_emit(on_event, "warning", message = f"conditioning cache disabled: {exc}")
- # The crop/flip variant plan is seed-deterministic, so persistent keys are stable across runs and the warm check can run before anything loads.
+ # The crop/flip variant plan is seed-deterministic, so persistent keys are stable across runs and
+ # the warm check can run before anything loads.
plan = _plan_cache_variants(
len(image_paths), cfg.cache_variants, cfg.center_crop, cfg.random_flip, cfg.seed
)
@@ -1996,6 +1999,8 @@ def _train_dit(
pipe = None
vae = None
caption_embeds = None
+ # The estimated cache exceeded the host-memory budget; keep the VAE resident and fall through to the in-loop
+ # encode path (latent_cache stays None).
latent_cache = None
if pcache is not None and use_cache:
caption_embeds, latent_cache = _load_warm_conditioning(
@@ -2010,7 +2015,6 @@ def _train_dit(
total = len(image_paths),
)
if caption_embeds is None:
- # Phase 1 (cold): conditioning only. The pipeline loads WITHOUT its transformer, so the encoders + VAE never share VRAM with it. Captions are constant, so their embeddings are precomputed once and the encoders freed.
latent_cache = None
pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)
encoded = _encode_prompts_cached(spec, pipe, to_encode, device, pcache)
@@ -2020,7 +2024,7 @@ def _train_dit(
if device == "cuda":
torch.cuda.empty_cache()
- # Phase 2: the VAE latent cache, then free the VAE too (the cache keeps the posterior affine parameters, so per-step sampling noise is preserved).
+ # The cache keeps the posterior affine parameters, so per-step sampling noise is preserved.
if use_cache:
latent_cache = _build_latent_cache(
spec,
@@ -2035,9 +2039,10 @@ def _train_dit(
plan = plan,
)
if latent_cache is LATENT_CACHE_OVER_BUDGET:
- # The estimated cache exceeded the host-memory budget; keep the VAE resident and fall through to the in-loop encode path (latent_cache stays None).
+ # The estimated cache exceeded the host-memory budget; keep the VAE resident and fall through to the
+ # in-loop encode path (latent_cache stays None).
latent_cache = None
- elif latent_cache is None: # stopped during the cache build; nothing trained yet
+ elif latent_cache is None:
_emit(
on_event,
"complete",
@@ -2045,9 +2050,7 @@ def _train_dit(
lora_path = None,
stopped = True,
steps_run = 0,
- # As above: a discard is a discard however early the stop lands.
- # _save_on_stop is the accessor this function is handed; the flag itself
- # lives in the caller's scope.
+ # A discard is a discard however early the stop lands.
discarded = not _save_on_stop(),
)
return str(out_dir)
@@ -2064,12 +2067,12 @@ def _train_dit(
# Variant picks use their own stream so the training loop index/noise draws stay seed-deterministic.
variant_rng = random.Random(cfg.seed + 1)
- # Phase 3: only now load the transformer, in the resolved base precision (nf4 QLoRA by default; bf16 / int8 / fp8 / mxfp8 are the dense speed modes; "auto" picks from free VRAM).
+ # Phase 3: only now load the transformer, in the resolved base precision (nf4 QLoRA by default; bf16 / int8
+ # / fp8 / mxfp8 are the dense speed modes; "auto" picks from free VRAM).
base_precision = _resolve_base_precision(cfg, spec, device)
transformer = spec.load_transformer(fetch_cfg, device, weight_dtype, base_precision)
base_is_bnb = base_precision == "nf4"
- # Freeze the base; attach the trainable LoRA to the transformer.
transformer.requires_grad_(False)
transformer.add_adapter(
LoraConfig(
@@ -2081,7 +2084,8 @@ def _train_dit(
)
)
if cfg.gradient_checkpointing:
- # Non-reentrant checkpointing: reentrant recompute of a bnb 4-bit LoRA linear can trip an illegal memory access on the larger FLUX transformer, and it is the recommended mode anyway.
+ # Non-reentrant checkpointing: reentrant recompute of a bnb 4-bit LoRA linear can trip an illegal
+ # memory access on the larger FLUX transformer, and it is the recommended mode anyway.
import functools
import torch.utils.checkpoint as _ckpt
transformer.enable_gradient_checkpointing(
@@ -2090,7 +2094,8 @@ def _train_dit(
cast_training_params(transformer, dtype = torch.float32)
lora_params = [p for p in transformer.parameters() if p.requires_grad]
- # int8 / fp8 / mxfp8 convert the frozen base linears AFTER the LoRA attaches, so the adapter modules are excluded and stay high precision.
+ # int8 / fp8 / mxfp8 convert the frozen base linears AFTER the LoRA attaches, so the adapter modules
+ # are excluded and stay high precision.
if base_precision == "int8":
_int8_quantize_base(transformer, cfg.resolved_family)
if base_precision == "fp8" and not _apply_fp8_training(transformer, on_event):
@@ -2098,7 +2103,8 @@ def _train_dit(
if base_precision == "mxfp8" and not _apply_mxfp8_training(transformer, on_event):
base_precision = "bf16"
- # LoRA EMA (opt-in via cfg.ema_decay): shadows ONLY the trainable adapter params, initialised after the precision conversions so they track the final fp32 objects. Exported as a second adapter under ema/.
+ # LoRA EMA (opt-in via cfg.ema_decay): shadows ONLY the trainable adapter params, initialised after
+ # the precision conversions so they track the final fp32 objects.
ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None
compiled = _maybe_compile_transformer(
@@ -2113,7 +2119,7 @@ def _train_dit(
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token
)
- # Timestep-shift + loss-weighting setup (see _training_sigma_table). getattr defaults keep an un-normalized config on the historical behavior.
+ # getattr defaults keep an un-normalized config on the historical behaviour.
flow_shift = getattr(cfg, "flow_shift", None)
if flow_shift is None:
flow_shift = "auto" if spec.family in AUTO_FLOW_SHIFT_FAMILIES else 1.0
@@ -2125,7 +2131,8 @@ def _train_dit(
if str(getattr(cfg, "weighting_scheme", "none") or "none") == "bell"
else None
)
- # The LR schedule advances once per optimizer update, so warmup/decay count optimizer steps; scaling by the accumulation factor would stretch warmup past the run.
+ # The LR schedule advances once per optimizer update, so warmup/decay count optimizer steps; scaling
+ # by the accumulation factor would stretch warmup past the run.
lr_sched = get_scheduler(
cfg.lr_scheduler,
optimizer = optimizer,
@@ -2134,30 +2141,26 @@ def _train_dit(
)
_emit(on_event, "model_load_completed", compiled = compiled, base_precision = base_precision)
- # See the SDXL trainer: the base is on disk now, so its commit can be read and written into
- # every bundle this run saves. The identity built before the load says "unresolved" on the
- # first run of an uncached repo, and an unresolved revision is not comparable, so a later
- # resume could not tell that the repo had moved underneath it.
- # The repo actually fetched, matching identity_for_config: the canonical base is not on disk
- # at all when the mirror was selected, so reading it here would pin "unresolved" forever.
- # with_resolved_revision records which repo the SHA came from, and mismatch_reason only
- # compares two revisions that came from the same one.
+ # Read the commit now the base is on disk: an identity built before the load says "unresolved",
+ # which is not comparable, so a later resume could not tell the repo had moved underneath it.
+ # Record the repo actually fetched: the canonical base is not on disk at all when the mirror was
+ # selected, and mismatch_reason only compares two revisions that came from the same repo.
identity = with_resolved_revision(identity, fetch_cfg.base_model)
# See the SDXL trainer: the cache path the loop actually took, not the one requested.
identity = with_cache_mode(identity, latent_cache is not None)
- # ...and the precision the frozen base ended up in. base_precision is still the request at
- # this point ("auto" resolves here, and a failed fp8/mxfp8 conversion has already fallen
- # back to bf16), so without this a bundle records a base it was never trained against.
+ # base_precision is still the request here ("auto" resolves at load, and a failed fp8/mxfp8
+ # conversion has fallen back to bf16), so without this a bundle records a base it never trained on.
identity = with_resolved_base_precision(identity, base_precision)
transformer.train()
n_images = len(image_paths)
batch_size = cfg.train_batch_size
- # Permutation-cycle index sampler (shared with the SDXL trainer): visits every image once per cycle, so a short run covers the whole dataset. Uses the loop's own rng to stay seed-deterministic.
+ # Permutation-cycle index sampler (shared with the SDXL trainer): visits every image once per cycle,
+ # so a short run covers the whole dataset.
index_sampler = PermutationBatchSampler(n_images, rng)
- # Restore a previous run (adapter, optimizer moments, LR position, EMA shadow, sampler cycle,
- # RNG streams) before the loop, so it picks up at `resumed + 1`. None for a fresh run.
+ # Restore a previous run (adapter, optimizer moments, LR position, EMA shadow, sampler cycle, RNG streams)
+ # before the loop, so it picks up at `resumed + 1`. None for a fresh run.
rng_streams = {"loop": rng, "variant": variant_rng}
restored = restore_resume_state(
cfg,
@@ -2171,20 +2174,16 @@ def _train_dit(
rng_streams = rng_streams,
)
resumed = restored.step if restored is not None else 0
- # Whether this run's source bundle lives in THIS directory. Resuming from
- # directory A into a reused output_dir B leaves B's existing bundles as foreign
- # as they would be for a fresh run -- so the first save has to clear them, or a
- # higher-numbered one survives and a later resume by B picks it over this run.
+ # Resuming from directory A into a reused output_dir B leaves B's existing bundles as foreign as
+ # for a fresh run, so the first save must clear them or a higher-numbered one outranks this run.
resumed_here = bool(resumed) and resumed_into_this_dir(cfg, out_dir)
- # The bundles that were already here when this run started, so a discard below
- # removes only what this run wrote. A resumed run shares its source's directory.
+ # The bundles already here when this run started, so a discard removes only what this run wrote.
preexisting_checkpoints = snapshot_checkpoints(out_dir)
if restored is not None:
was = restored.progress.get("resolved_base_precision")
if was and was != base_precision:
- # Not fatal: the LoRA tensors and their optimizer moments are precision-independent,
- # only the frozen base's numerics change. But base_precision="auto" picks from free
- # VRAM, so this can happen silently and is worth saying out loud.
+ # Not fatal, since only the frozen base's numerics change, but base_precision="auto" picks from free
+ # VRAM, so this can happen silently.
_emit(
on_event,
"warning",
@@ -2204,9 +2203,9 @@ def _train_dit(
done = resumed
def _save_checkpoint(step: int) -> None:
- # Outcome is reported by write_resume_checkpoint's own checkpoint_saved /
- # checkpoint_failed events, so a run that crashes after a save is still known to be
- # resumable (and one whose save failed is still known to be blocked).
+ # write_resume_checkpoint reports its own checkpoint_saved / checkpoint_failed events, so a run
+ # that crashes after a save is still known to be resumable and one whose save failed is still
+ # known to be blocked.
_written, _error = write_resume_checkpoint(
cfg,
step = step,
@@ -2218,23 +2217,18 @@ def _train_dit(
ema = ema,
sampler = index_sampler,
rng_streams = rng_streams,
- # base_precision="auto" resolves from free VRAM at load time, so the identity (which
- # records the REQUESTED mode) cannot tell nf4 from bf16 across two "auto" runs.
- # Recorded here so a resume can at least report the change.
+ # "auto" resolves from free VRAM at load time, so the identity, which records the REQUESTED mode,
+ # cannot tell nf4 from bf16 across two "auto" runs.
progress = {"running_loss": running_loss, "resolved_base_precision": base_precision},
- # NOT discard_existing. A fresh run does own its output dir, but deleting the
- # previous run's bundles at the FIRST periodic save spends them before this run
- # has produced anything: "stop without saving" then removes only what this run
- # wrote, leaves the previous adapter in place, and the run it belonged to is
- # unresumable -- cancelling a retrain destroyed the thing being retrained. The
- # clear happens on the completion path instead, once the new adapter is saved.
+ # NOT discard_existing: deleting the previous run's bundles at the FIRST periodic save spends them
+ # before this run produced anything, so cancelling a retrain destroyed the thing being retrained.
discard_existing = False,
- # And the bundles that were here before this run: a branched resume must not
- # prune the higher-numbered checkpoints it did not write.
+ # A branched resume must not prune the higher-numbered checkpoints it did not write.
preexisting = preexisting_checkpoints,
)
- # bf16 autocast around the forward + loss, matching the diffusers dreambooth scripts: it reconciles the fp32 LoRA params with the bnb 4-bit base matmuls. Without it the 4-bit backward on FLUX dies.
+ # bf16 autocast around the forward + loss, matching the diffusers dreambooth scripts: it reconciles
+ # the fp32 LoRA params with the bnb 4-bit base matmuls.
autocast = (
torch.autocast(device_type = "cuda", dtype = torch.bfloat16)
if device == "cuda"
@@ -2264,7 +2258,8 @@ def _train_dit(
timesteps, t_indices = _sample_timesteps(scheduler, latents.shape[0], device)
sigmas = _gather_sigmas(sigma_table, t_indices, device, weight_dtype, latents.ndim)
if shift_active:
- # The model's timestep conditioning must follow the shifted sigma (timestep = sigma * num_train_timesteps). Gather in fp32 so bf16 rounding never skews it.
+ # The model's timestep conditioning must follow the shifted sigma (timestep = sigma *
+ # num_train_timesteps). Gather in fp32 so bf16 rounding never skews it.
timesteps = (
sigma_table[t_indices].to(device = device, dtype = torch.float32).flatten()
* num_train_ts
@@ -2301,7 +2296,8 @@ def _train_dit(
grad_norm = None
if cfg.max_grad_norm and cfg.max_grad_norm > 0:
- # clip_grad_norm_ returns the total PRE-clip norm: the health signal the UI charts (an exploding norm shows even while the clip caps the update).
+ # clip_grad_norm_ returns the total PRE-clip norm: the health signal the UI charts (an exploding
+ # norm shows even while the clip caps the update).
grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))
optimizer.step()
lr_sched.step()
@@ -2311,8 +2307,7 @@ def _train_dit(
running_loss += step_loss
done = opt_step + 1
now = time.time()
- # Rates count only the steps THIS process ran, so a resumed run does not divide by
- # steps it never executed.
+ # Rates count only the steps THIS process ran, so a resumed run does not divide by steps it never executed.
ran_here = done - resumed
if ran_here == 1:
# Step 1 pays the one-time costs (cudnn autotune, compile warmup), so the reported rate starts after it.
@@ -2338,8 +2333,7 @@ def _train_dit(
peak_memory_gb = peak_gb or None,
)
stop_now = _check_stop()
- # Periodic resume point. Skipped on the final step (a finished run has nothing left to
- # resume) and when stopping, since the stop path writes one at the exact step.
+ # Skipped on the final step and when stopping, since the stop path writes one at the exact step.
if (
not stop_now
and cfg.save_steps
@@ -2356,9 +2350,7 @@ def _train_dit(
ema_path: Optional[str] = None
if not (stopped and not _save_on_stop()):
out_dir.mkdir(parents = True, exist_ok = True)
- # Stopping early is exactly when the run should be resumable, so write the bundle BEFORE
- # the adapter export: if that export then fails, the run still comes back. A completed run
- # has nothing left to resume.
+ # Write the bundle BEFORE the adapter export: if that export fails, the run still comes back.
if stopped and done > 0:
_save_checkpoint(done)
layers = get_peft_model_state_dict(transformer)
@@ -2374,23 +2366,17 @@ def _train_dit(
except Exception as exc: # noqa: BLE001 -- the primary adapter is already saved
_emit(on_event, "warning", message = f"EMA adapter save failed: {exc}")
if not stopped:
- # Same as the LoRA trainer: a completed run has nothing to resume, and the final
- # iteration writes no bundle, so save_steps leaves the run's own checkpoint-400
- # behind for a later resume to roll back to. An earlier run's bundles stay.
+ # A completed run has nothing to resume and the final iteration writes no bundle, so save_steps
+ # would leave this run's own checkpoint behind for a later resume to roll back to.
retire_own_checkpoints(out_dir, preexisting_checkpoints, resumed_here = resumed_here)
elif not resumed_here:
- # Stopped WITH save on a fresh retrain: the stop bundle is a LOWER step than
- # the leftovers of the earlier run in this directory, and resume-by-directory
- # picks the newest by step -- so those would outrank the partial just saved
- # and continue the wrong training. A resumed run keeps what it found.
+ # A stop-with-save on a fresh retrain is a LOWER step than the earlier run's leftovers, and resume-
+ # by-directory picks the newest by step, so those would outrank the partial just saved.
discard_preexisting_checkpoints(out_dir, preexisting_checkpoints)
else:
- # Discarded: the user asked to throw this run away. Before periodic checkpoints
- # existed a discard left nothing behind, because the output directory was only ever
- # created inside this save gate. save_steps now writes bundles as the run goes, so
- # without this a discard leaves up to save_total_limit copies of the optimizer state
- # in a directory the user never got an artifact from -- invisible to every scanner,
- # unresumable (the record is marked discarded), and with no delete path in the UI.
+ # save_steps writes bundles as the run goes, so without this a discard leaves up to
+ # save_total_limit copies of the optimizer state in a directory the user got no artifact from:
+ # invisible to every scanner, unresumable, with no delete path in the UI.
clear_own_checkpoints(out_dir, preexisting_checkpoints)
try:
out_dir.rmdir()
@@ -2411,7 +2397,6 @@ def _train_dit(
wall_seconds = round(time.time() - t_start, 1),
resumed_from_step = resumed or None,
# "Stop without saving" discards the run, so its own periodic checkpoints must not keep
- # offering to continue it.
discarded = bool(stopped and not _save_on_stop()),
)
return str(out_dir)
diff --git a/studio/backend/core/training/diffusion_h3_clips.py b/studio/backend/core/training/diffusion_h3_clips.py
index 7536d6d9ffd..779266e8252 100644
--- a/studio/backend/core/training/diffusion_h3_clips.py
+++ b/studio/backend/core/training/diffusion_h3_clips.py
@@ -24,35 +24,28 @@ from pathlib import Path
from typing import Any, Callable, Optional
from utils.paths.path_utils import drop_appledouble_metadata
+# Checkpoint contracts mirrored from diffusers.modular_pipelines.minimax_h3, which the trainer
+# cannot import; asserted against the live components at load, so a move is caught.
# ── the model's grid ─────────────────────────────────────────────────────────
-#
-# Every constant here is a checkpoint contract, mirrored from
-# ``diffusers.modular_pipelines.minimax_h3`` (which the trainer cannot import without pulling
-# in the whole modular stack, and which a CPU-only test host may not carry at all). The
-# trainer asserts them against the live components at load, so a diffusers release that moves
-# one is caught rather than silently trained through.
H3_FPS = 24
-# The video VAE's ``clip_length`` and ``tokens_chunk_size``: 17 pixel frames per chunk, 5 latent
-# frames kept per chunk, plus a 2-frame head.
+# The video VAE's clip_length and tokens_chunk_size: 17 pixel frames per chunk, 5 latent frames
+# kept per chunk, plus a 2-frame head.
H3_FRAMES_PER_CHUNK = 17
H3_LATENTS_PER_CHUNK = 5
H3_SPATIAL_COMPRESSION = 16
# The transformer's (t, h, w) patch. Only the spatial half is > 1.
H3_PATCH_T, H3_PATCH_H, H3_PATCH_W = 1, 2, 2
-# Both axes have to survive the VAE's 16x spatial compression AND still be a whole number of
-# 2x2 patch rows wide, so the canvas multiple is their product.
+# Both axes have to survive the VAE's 16x spatial compression AND still be a whole number of 2x2
+# patch rows, so the canvas multiple is their product.
H3_CANVAS_MULTIPLE = H3_SPATIAL_COMPRESSION * H3_PATCH_W
H3_AUDIO_SAMPLING_RATE = 32000
H3_AUDIO_LATENTS_PER_SECOND = 40
H3_AUDIO_CHANNELS = 2
-# How much of a clip's audio window may be zero-padded before the clip is refused. 1% of a 5.17s
-# window is ~52ms, which covers the few-millisecond tail a container routinely ends short by
-# while still refusing a stream that is mostly silence.
+# 1% of a 5.17s window is ~52ms, covering the tail a container routinely ends short by while still
+# refusing a mostly-silent stream.
_MAX_AUDIO_PAD_FRACTION = 0.01
-# Peak amplitude at or below which a decoded window counts as silent, on PyAV's "flt" scale where
-# full scale is 1.0. About -80 dBFS: below the noise floor of any real recording, and above the
-# rounding dust an encode/decode round trip can leave on a track that was authored as digital
-# silence, so a genuinely muted soundtrack is caught whether or not it decodes to exact zeros.
+# About -80 dBFS on PyAV's "flt" scale: below the noise floor of any real recording, above the
+# rounding dust an encode/decode round trip leaves on authored digital silence.
_SILENT_AUDIO_PEAK = 1e-4
H3_AUDIO_LATENT_CHANNELS = 32
H3_VIDEO_LATENT_CHANNELS = 24
@@ -67,18 +60,10 @@ H3_MAX_ASPECT_RATIO = 4
H3_PIXEL_MEAN = (0.485, 0.456, 0.406)
H3_PIXEL_STD = (0.229, 0.224, 0.225)
-# Pixel frames one training clip covers. ONE VAE chunk, i.e. the shortest clip the video VAE
-# can encode at all, which is 0.917 s at 24 fps.
-#
-# This is deliberately far below the 5 s floor MiniMax-H3 *generates* at, and the trade is
-# explicit: the packed sequence is quadratic in its own length through the full self-attention,
-# so at the released 768-short-edge canvas a 5 s clip is ~38k rows and a 22-frame clip is ~7k.
-# Training at the native canvas on short clips keeps the SPATIAL statistics -- which is what a
-# style LoRA learns -- exactly on distribution, and only shortens the temporal extent; training
-# a 5 s clip at a canvas small enough to fit would put every spatial statistic off distribution
-# instead. The temporal rotary grid of a 22-frame clip is a strict PREFIX of the grid of a
-# generated one (``_temporal_position_grid`` starts at the same origin with the same spacing),
-# so no row sits at a coordinate the model never visits.
+# ONE VAE chunk, the shortest clip the video VAE can encode at all, which is 0.917 s at 24 fps.
+# Deliberately far below the 5 s floor MiniMax-H3 generates at: the packed sequence is quadratic
+# in its own length, and training short clips at the native canvas keeps the SPATIAL statistics on
+# distribution. A 22-frame clip's temporal rotary grid is a strict PREFIX of a generated one's.
H3_TRAIN_NUM_FRAMES = H3_FRAMES_PER_CHUNK + H3_LATENTS_PER_CHUNK
_VIDEO_EXTS = {".mp4", ".mov", ".mkv", ".webm", ".m4v", ".avi"}
@@ -300,8 +285,7 @@ def decode_clip(
)
stream = container.streams.video[0]
source_fps = float(stream.average_rate or stream.guessed_rate or H3_FPS) or float(H3_FPS)
- # Container duration, in seconds, for the over-long note below. Best effort: an unknown
- # duration simply means no note, never a failed decode.
+ # Best effort: an unknown duration means no note, never a failed decode.
source_duration_s = 0.0
try:
if stream.duration is not None and stream.time_base is not None:
@@ -319,8 +303,8 @@ def decode_clip(
if int(next_target * source_fps / H3_FPS) > source_index:
continue
image = frame.to_image().convert("RGB")
- # Before the crop, not after: the canvas is in display orientation, so cropping the
- # coded frame would trim the wrong pair of edges as well as train it sideways.
+ # Before the crop: the canvas is in display orientation, so cropping the coded frame would trim the
+ # wrong edges as well as train it sideways.
image = apply_display_rotation(image, display_rotation_degrees(frame, stream), Image)
image = _cover_resize(image, width, height, Image)
while (
@@ -390,8 +374,7 @@ def display_rotation_degrees(frame: Any, stream: Any) -> int:
except Exception: # noqa: BLE001 -- a degenerate matrix means "no rotation", not a failure
return 0
theta = int(-round(degrees)) % 360
- # Only the four square turns; anything else cannot be applied without resampling, and no
- # camera writes one.
+ # Only the four square turns; anything else cannot be applied without resampling, and no camera writes one.
return theta if theta in (90, 180, 270) else 0
@@ -437,11 +420,8 @@ def _decode_clip_audio(path: Any, target_samples: int, av: Any, np: Any) -> Any:
chunks = []
have = 0
with av.open(str(path)) as container:
- # Stops at the training window, exactly as the video loop above does. An over-long source
- # is accepted input here -- only the first num_frames are trained and the caller is merely
- # warned -- so decoding the rest of the soundtrack would spend a whole recording's time
- # and memory to build a sub-second sample, and would fail on damage in a region that is
- # never used.
+ # Stops at the training window: only the first num_frames are trained, so decoding the rest of the
+ # soundtrack would spend a whole recording's time and fail on damage in an unused region.
for frame in container.decode(audio = 0):
for resampled in resampler.resample(frame):
block = resampled.to_ndarray().reshape(-1, H3_AUDIO_CHANNELS)
@@ -450,9 +430,8 @@ def _decode_clip_audio(path: Any, target_samples: int, av: Any, np: Any) -> Any:
if have >= target_samples:
break
if have < target_samples:
- # Only when the stream ran out: the resampler holds a partial block back, and that
- # tail is what the pad allowance below is measured against. After an early break
- # there is nothing to flush for -- the window is already full.
+ # Only when the stream ran out: the resampler holds a partial block back and that tail is what the
+ # pad allowance measures; after an early break the window is already full.
for resampled in resampler.resample(None):
chunks.append(resampled.to_ndarray().reshape(-1, H3_AUDIO_CHANNELS))
if not chunks:
@@ -470,11 +449,7 @@ def _decode_clip_audio(path: Any, target_samples: int, av: Any, np: Any) -> Any:
f"soundtrack runs its full length."
)
samples = np.pad(samples, ((0, missing), (0, 0)))
- # A muted track runs the clip's full length, so every check above passes and the window comes
- # back all zeros -- the same target the short-audio refusal exists to keep out, arriving by a
- # route that refusal cannot see. Measured as peak amplitude rather than mean energy so a clip
- # that is merely quiet, or silent for most of its length with one real sound in it, is kept:
- # only a track with nothing above the floor anywhere is turned away.
+ # A muted track passes every check above and comes back all zeros.
if float(np.max(np.abs(samples))) <= _SILENT_AUDIO_PEAK:
raise ValueError(
f"{Path(path).name} has a soundtrack that is silent all the way through. "
diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.py
index d033e25a309..579f230d201 100644
--- a/studio/backend/core/training/diffusion_h3_trainer.py
+++ b/studio/backend/core/training/diffusion_h3_trainer.py
@@ -92,27 +92,15 @@ from core.training.diffusion_train_common import (
)
from core.training.diffusion_train_extras import LoRAEMA, save_ema_adapter
-# The video-stream and audio-stream projections of every block, fully qualified.
-#
-# There is nothing to exclude: H3 has ONE set of block weights serving all three modalities,
-# so this is the whole adapter surface and the audio rows are trained through the same
-# matrices as the video rows. What IS excluded is deliberate:
-# - ``adaln_proj.linear`` projects the timestep embedding into the modulation table. It is
-# 40% of the checkpoint's parameters, but its input is a (num_timesteps, 2688) tensor with
-# two or three rows, so a rank-r adapter there sees a handful of samples per step and
-# learns noise.
-# - ``proj_in`` / ``audio_proj_in`` / ``proj_out`` / ``audio_proj_out`` / ``context_embedder``
-# are the patch and text projections, kept in fp32 by the checkpoint's own
-# ``_keep_in_fp32_modules``; adapting them mixes precisions for no benefit.
-# - the ``token_refiner`` blocks carry ``attn`` and ``ff`` under the SAME leaf names
-# (``token_refiner.refiner_blocks.0.attn.to_q``), so PEFT's suffix rule for a LIST of
-# target names would adapt the text refiner as well as the denoiser stack.
-#
-# That last point is why the targets are a REGEX and not a list: PEFT globs nothing, it either
-# suffix-matches a list or ``re.fullmatch``es a string, so qualifying a list entry with
-# ``transformer_blocks.*.`` matches nothing at all and the adapter silently trains zero
-# parameters (the trainer also refuses an empty adapter, so both halves of that mistake are
-# caught).
+# H3 has ONE set of block weights serving all three modalities, so this is the whole adapter
+# surface; the exclusions below are deliberate. adaln_proj.linear is 40% of the checkpoint's
+# parameters but its input is a (num_timesteps, 2688) tensor with two or three rows, so a rank-r
+# adapter there learns noise. proj_in / audio_proj_in / proj_out / audio_proj_out /
+# context_embedder are the patch and text projections, kept in fp32 by the checkpoint's own
+# _keep_in_fp32_modules; adapting them mixes precisions for no benefit. The token_refiner blocks
+# carry attn and ff under the SAME leaf names (token_refiner.refiner_blocks.0.attn.to_q), which is
+# why the targets are a REGEX and not a list: PEFT suffix-matches a list, so it would adapt the
+# text refiner as well as the denoiser stack.
_H3_TARGET_LEAVES = (
"attn.to_q",
"attn.to_k",
@@ -127,29 +115,24 @@ _H3_TARGETS = (
+ ")"
)
-# Modules whose weights the transformer reads a DTYPE off to align an activation with
-# (``x.to(self.linear.weight.dtype)``). A bitsandbytes ``Params4bit`` reports ``uint8``, so
-# quantizing one of these casts the activation to Byte and the next norm dies with
-# ``"rms_norm" not implemented for 'Byte'``. Keeping them dense is the whole fix; it costs the
-# nf4 saving on ``adaln_proj`` (36.6 GB resident instead of ~17 GB), which is still well under
-# the 66.3 GB dense weight. torchao int8 is unaffected -- its tensor subclass reports the
-# logical bfloat16 -- so ``base_precision="int8"`` quantizes everything.
+# The transformer reads a DTYPE off these weights to align an activation
+# (x.to(self.linear.weight.dtype)), and a bitsandbytes Params4bit reports uint8, so quantizing one
+# casts the activation to Byte and the next norm dies with "rms_norm" not implemented for 'Byte'.
+# Costs the nf4 saving on adaln_proj (36.6 GB instead of ~17 GB); torchao int8 is unaffected,
+# since its subclass reports bfloat16, so base_precision="int8" quantizes everything.
_H3_NF4_SKIP_MODULES = ("context_embedder", "adaln_proj", "norm_out")
# The exponential sigma shifts of the two schedules, from the released scheduler configs.
_H3_VIDEO_SHIFT = 12.0
_H3_AUDIO_SHIFT = 3.0
-# Which Qwen3-VL hidden state conditions the transformer. The last layer is post-norm and is
-# not what the released weights were trained against.
+# The last Qwen3-VL layer is post-norm and is not what the released weights were trained against.
_H3_TEXT_ENCODER_LAYER = 50
-# Components the conditioning phase needs, and the one the training phase needs. Naming them
-# is what keeps the 66 GB transformer off the device while the 63 GiB conditioner is on it.
+# Naming the components is what keeps the 66 GB transformer off the device while the 63 GiB conditioner is on it.
_H3_TEXT_COMPONENTS = ("text_encoder", "tokenizer", "processor")
-# The VAEs load in fp32: both carry modules diffusers refuses to cast (their encoders,
-# decoders and projection heads), so a bf16 load followed by a .to(float32) both warns and
-# leaves the cast half-applied.
+# The VAEs load in fp32: both carry modules diffusers refuses to cast, so a bf16 load followed by
+# .to(float32) warns and leaves the cast half-applied.
_H3_VAE_COMPONENTS = ("vae", "audio_vae")
_H3_CONDITIONING_COMPONENTS = _H3_TEXT_COMPONENTS + _H3_VAE_COMPONENTS
@@ -199,18 +182,14 @@ def _load_conditioners(cfg, device):
from core.inference.diffusion import hub_cache_dir
- # Pinned, like every loader call on the inference side. diffusers resolves an unset
- # cache_dir through huggingface_hub's import-time constant, and the training subprocess is
- # spawned without the cache-environment wrapper, so a cache folder changed mid-session was
- # invisible here: components already sitting in the selected root were missed and roughly
- # 145 GB re-downloaded into the old one.
+ # Pin the cache dir: diffusers resolves an unset one through huggingface_hub's import-time
+ # constant and this subprocess is spawned without the cache-environment wrapper, so components in
+ # the selected root were missed and ~145 GB re-downloaded into the old one.
cache_dir = hub_cache_dir()
pipe = ModularPipeline.from_pretrained(cfg.base_model, token = cfg.hf_token, cache_dir = cache_dir)
- # The token above opens the modular INDEX only. load_components runs a separate
- # from_pretrained per component, against the repos that index names, so it has to carry the
- # token again -- and it swallows a component failure as a logger.warning rather than raising,
- # so an anonymous 401 leaves the attribute unset and the first use dies on None instead of
- # saying the base was gated. The inference H3 loader forwards it for the same reason.
+ # load_components runs a separate from_pretrained per component and swallows a failure as a
+ # warning, so without the token an anonymous 401 leaves the attribute unset and the first use dies
+ # on None instead of naming the gate.
auth = {"token": cfg.hf_token} if cfg.hf_token else {}
pipe.load_components(
names = list(_H3_TEXT_COMPONENTS),
@@ -306,7 +285,7 @@ def _encode_audio_latents(audio_vae, waveform, device) -> Any:
cached for it."""
import torch
- samples = torch.from_numpy(waveform).to(device).unsqueeze(1) # (2, 1, samples)
+ samples = torch.from_numpy(waveform).to(device).unsqueeze(1)
with torch.no_grad():
posterior = audio_vae.encode(samples, return_dict = False)[0]
latents = posterior.mode()
@@ -322,8 +301,8 @@ def _load_transformer(cfg, device, base_precision):
from core.inference.diffusion import hub_cache_dir
- # Same pin as the conditioners above: the denoiser is the 145 GB half, so an unpinned load
- # here is the expensive one to get wrong.
+ # Same pin as the conditioners: the denoiser is the 145 GB half, so an unpinned load here is the
+ # expensive one to get wrong.
cache_dir = hub_cache_dir()
if base_precision == "nf4":
from diffusers import BitsAndBytesConfig as DiffusersBnb
@@ -421,8 +400,8 @@ def _row_timesteps(layout, num_text_tokens: int, t_video: float, t_audio: float,
num_text_tokens,
t_video,
t_audio,
- # No conditioning rows exist in this layout, so the two condition timesteps are
- # unreachable; pass the generated ones so no phantom value enters ``torch.unique``.
+ # No conditioning rows exist in this layout, so pass the generated timesteps and keep a phantom
+ # value out of torch.unique.
t_video,
t_audio,
)
@@ -476,23 +455,14 @@ def run_h3_lora_training(
cfg = config.normalized()
if cfg.resolved_family != "minimax-h3":
raise ValueError(f"This trainer is for minimax-h3, not {cfg.resolved_family!r}.")
- # Every config-only refusal lives in the shared preflight, which the START ROUTE also calls
- # BEFORE it evicts the user's resident GPU models. Raised again here so a direct call to the
- # trainer is refused too, and so the two can never disagree about what H3 accepts.
+ # Every config-only refusal also lives in the shared preflight the START ROUTE calls before
+ # evicting resident GPU models; raised again here so a direct trainer call is refused too.
reason = h3_train_unsupported_reason(cfg)
if reason:
raise ValueError(reason)
- # The two image-LoRA augmentation knobs are fixed for a clip dataset, not honoured: every
- # frame goes through the same centre cover-crop (_cover_resize), and nothing is flipped --
- # a per-frame flip would tear a clip, and a per-clip one has nowhere to live, since the
- # cached tensors carry no variant axis. snr_gamma is the same shape of mismatch on the loss
- # side: the step is a plain unweighted MSE and the field defaults to 5.0, so every default
- # request recorded min-SNR weighting that never ran (weighting_scheme, its sibling, is
- # refused above because it has no default to break). All three SCHEMA defaults disagree with
- # what the loop does, so they are normalised rather than refused: refusing would 422 every
- # default request, and leaving them would have the run record describe a recipe that did not
- # happen. Read off the shared table, which the SERVICE also applies to the config it persists
- # -- normalising only here fixed the run and left the history entry lying about it.
+ # The augmentation and snr_gamma defaults do not match what the clip loop does (one centre
+ # cover-crop, no flips, plain unweighted MSE), so they are normalised rather than refused, which
+ # would 422 every default request. Read off the shared table the SERVICE also applies.
cfg = replace(cfg, **train_recipe_overrides(cfg))
import torch
@@ -521,11 +491,9 @@ def run_h3_lora_training(
)
weight_dtype = torch.bfloat16 if device == "cuda" else torch.float32
- # allow_modular: this loop loads through ModularPipeline.from_pretrained, and a local
- # MiniMax-H3 pipeline carries modular_model_index.json and no model_index.json, so the
- # conventional shape check refused the only local layout the family has. Read off the shared
- # set rather than hard-coded True, because the start route runs this same gate first and has
- # to reach the same verdict about the same directory.
+ # allow_modular: this loop loads through ModularPipeline.from_pretrained, and a local MiniMax-H3
+ # pipeline carries modular_model_index.json and no model_index.json, so the conventional shape
+ # check refused the only local layout the family has.
_assert_trusted_base_model(
cfg.base_model,
allow_modular = (cfg.resolved_family or "").strip().lower() in MODULAR_BASE_FAMILIES,
@@ -570,8 +538,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
to_encode = sorted(set(captions))
+ # Phase 1: the 63 GiB Qwen3-VL conditioner and both VAEs are resident here and nowhere else.
# ── phase 1: conditioning. The 63 GiB Qwen3-VL conditioner and both VAEs are resident
- # here and nowhere else; the 66 GB transformer has not been fetched yet.
pipe = _load_conditioners(cfg, device)
caption_embeds = {cap: _encode_prompt(pipe, cap, device) for cap in to_encode}
_emit(on_event, "preparing", stage = "encode_prompts", done = len(to_encode), total = len(to_encode))
@@ -582,9 +550,9 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
if device == "cuda":
torch.cuda.empty_cache()
+ # One canvas for the run, from the FIRST clip's aspect ratio: every other clip is cover-cropped
+ # onto it, so a mixed-aspect dataset trains on one geometry.
# ── phase 2: the clip cache. One canvas for the run, taken from the FIRST clip's aspect
- # ratio: every other clip is cover-cropped onto it, so a mixed-aspect dataset trains on one
- # geometry rather than reshaping the packed sequence per step.
width, height = _dataset_canvas(clip_paths[0], cfg.resolution)
latent_h, latent_w = height // H3_SPATIAL_COMPRESSION, width // H3_SPATIAL_COMPRESSION
cache: list[tuple[Any, Any, Any]] = []
@@ -594,9 +562,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
num_frames = num_frames,
width = width,
height = height,
- # The window is the clip's opening and the latents are cached once for the run, so
- # a longer source trains only its first seconds while its caption describes the
- # whole thing. Reported rather than left silent.
+ # The window is the clip's opening and the latents are cached once, so a longer source trains only
+ # its first seconds while its caption describes the whole thing.
on_note = lambda message: _emit(on_event, "warning", message = message),
)
video_a, video_b = _encode_video_stats(pipe.vae, frames, device)
@@ -608,12 +575,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
)
entry = (video_a, video_b, audio)
if i == 0 and not _latent_cache_forced():
- # Size-gate off the FIRST real entry, the way the SDXL and DiT caches do, before the
- # rest of the dataset is encoded. Those two answer an over-budget estimate by dropping
- # the cache and encoding per step; H3 cannot, because both VAEs are freed below to
- # make room for the 66 GB transformer. So the honest answer is to say so now, with the
- # numbers, instead of running the whole preparation and being OOM-killed at the end of
- # it with nothing to show.
+ # H3 cannot answer an over-budget estimate by encoding per step, because both VAEs are freed to
+ # make room for the 66 GB transformer, so say so now with the numbers instead of being OOM-killed.
from core.training import diffusion_train_common as _train_common
per_clip = int(sum(t.numel() * t.element_size() for t in entry))
if _latent_cache_over_budget(per_clip, len(clip_paths)):
@@ -652,9 +615,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
)
transformer = _load_transformer(cfg, device, base_precision)
transformer.requires_grad_(False)
- # ``normalized()`` fills lora_target_modules with the generic DEFAULT_LORA_TARGETS when a
- # caller does not set it, so that value means "unset" and the family's own regex wins. Any
- # other explicit tuple is a deliberate override and is passed through as a list.
+ # normalized() fills lora_target_modules with the generic DEFAULT_LORA_TARGETS when unset, so that
+ # value means "unset" and the family's own regex wins.
targets: Any = (
_H3_TARGETS
if tuple(cfg.lora_target_modules) == DEFAULT_LORA_TARGETS
@@ -683,10 +645,10 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
f"would train nothing."
)
if base_precision == "int8":
- # WITH the family. H3's adaln_proj is Linear(2688 -> 96768), so it clears the 512-feature
- # floor and gets quantized, then runs at M = 1 and raises "self.size(0) needs to be
- # greater than 16" on the first step -- after the whole 66.3 GB base has loaded. The
- # family also carries the pad list (context_embedder, token_refiner) the helper applies.
+ # WITH the family: H3's adaln_proj is Linear(2688 -> 96768), so it clears the 512-feature floor
+ # and gets quantized, then runs at M = 1 and raises "self.size(0) needs to be greater than 16"
+ # after the 66.3 GB base has loaded. The family also carries the pad list (context_embedder,
+ # token_refiner) the helper applies.
_int8_quantize_base(transformer, cfg.resolved_family)
ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None
@@ -780,9 +742,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
)
loss_video = F.mse_loss(pred_video[0].float(), target_video.float())
loss_audio = F.mse_loss(pred_audio[0].float(), target_audio.float())
- # Unweighted sum, the model's own objective: the two streams share every
- # matrix the adapter touches, so down-weighting audio would not protect it, it
- # would only stop the loss reporting that it had drifted.
+ # Unweighted sum, the model's own objective: both streams share every matrix the adapter touches, so
+ # down-weighting audio would only stop the loss reporting that it had drifted.
loss = loss_video + loss_audio
(loss / cfg.gradient_accumulation_steps).backward()
step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps
@@ -832,8 +793,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
ema_path: Optional[str] = None
if not (stopped and not _save_on_stop()):
layers = get_peft_model_state_dict(transformer)
- # The trained config, so the alpha survives the round trip rather than being re-derived
- # as the rank. ``default`` is the adapter name add_adapter used.
+ # The trained config, so the alpha survives the round trip rather than being re-derived as the
+ # rank. "default" is the adapter name add_adapter used.
adapter_config = dict(transformer.peft_config["default"].to_dict())
_save_lora(str(out_dir), layers, adapter_config)
lora_path = str(out_dir / DEFAULT_LORA_FILENAME)
@@ -872,10 +833,9 @@ def _dataset_canvas(clip_path: str, short_edge: int) -> tuple[int, int]:
stream = container.streams.video[0]
source_w = int(stream.codec_context.width)
source_h = int(stream.codec_context.height)
- # The coded size is not the displayed size for a rotated clip, and decode_clip rotates
- # its frames, so a portrait phone clip would otherwise get a landscape canvas and be
- # cover-cropped down to it. One frame is decoded because the matrix travels with the
- # frame; the container is open either way.
+ # The coded size is not the displayed size for a rotated clip, so a portrait phone clip would get
+ # a landscape canvas and be cropped down to it; one frame is decoded because the matrix travels
+ # with the frame.
theta = 0
try:
frame = next(container.decode(video = 0), None)
diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.py
index 7be384426cb..44b8e0f5f74 100644
--- a/studio/backend/core/training/diffusion_lora_trainer.py
+++ b/studio/backend/core/training/diffusion_lora_trainer.py
@@ -42,7 +42,8 @@ from dataclasses import replace
from pathlib import Path
from typing import Any, Optional
-# Shared, family-agnostic building blocks. Re-exported so callers/tests that import them from this module keep working.
+# Shared, family-agnostic building blocks. Re-exported so callers/tests that import them from this module keep
+# working.
from core.training.diffusion_train_common import ( # noqa: F401
DEFAULT_LORA_FILENAME,
DEFAULT_LORA_TARGETS,
@@ -121,6 +122,7 @@ def _load_image_tensor(
if random_flip and rng.random() < 0.5:
img = img.transpose(Image.FLIP_LEFT_RIGHT)
# A flip mirrors the crop's left origin, so report the mirrored offset (as diffusers does).
+ # Mirror the crop's left origin (same as _load_image_tensor's random flip).
crop_left = max(0, resized_w - resolution - left)
arr = np.asarray(img, dtype = np.float32) / 255.0
tensor = torch.from_numpy(arr).permute(2, 0, 1) * 2.0 - 1.0
@@ -156,7 +158,6 @@ def _load_image_tensor_planned(
crop_left = left
if flip:
img = img.transpose(Image.FLIP_LEFT_RIGHT)
- # Mirror the crop's left origin (same as _load_image_tensor's random flip).
crop_left = max(0, resized_w - resolution - left)
arr = np.asarray(img, dtype = np.float32) / 255.0
tensor = torch.from_numpy(arr).permute(2, 0, 1) * 2.0 - 1.0
@@ -235,7 +236,8 @@ def _build_sdxl_latent_cache(
a = _hold(dist.mean * vae_scale)
b = _hold(dist.std * vae_scale)
if not forced and not gated:
- # Size-gate the auto cache off the first real variant, before building the rest (it can exhaust pinned RAM). Over budget: bail with the VAE resident.
+ # Size-gate the auto cache off the first real variant: it can exhaust pinned RAM. Over budget, bail
+ # with the VAE resident.
per_variant = a.numel() * a.element_size() + b.numel() * b.element_size()
if _latent_cache_over_budget(per_variant, total_variants):
_emit(
@@ -311,9 +313,8 @@ def run_diffusion_lora_training(
from peft import LoraConfig
from peft.utils import get_peft_model_state_dict
- # Only now is diffusers in sys.modules, and it hard-codes _tqdm_active = True and
- # honours no env var, so this is the earliest point its "Loading pipeline
- # components..." bar can be turned off -- and it comes before the pipeline load below.
+ # diffusers hard-codes _tqdm_active = True and honours no env var, so this is the earliest point its
+ # loading bar can be turned off, and it precedes the pipeline load below.
try:
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
@@ -341,11 +342,12 @@ def run_diffusion_lora_training(
device = "cuda" if torch.cuda.is_available() else "cpu"
precision = cfg.mixed_precision if device == "cuda" else "no"
if precision == "bf16" and device == "cuda" and not native_bf16_supported():
- # Pre-Ampere GPUs have no native bf16 and is_bf16_supported() counts emulation, so use the compute-capability probe and fall back to fp16.
+ # Pre-Ampere GPUs have no native bf16 and is_bf16_supported() counts emulation, so use the compute-
+ # capability probe and fall back to fp16.
precision = "fp16"
weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "no": torch.float32}[precision]
- # TF32 / cudnn.benchmark for the run, restored on the way out. Wraps the whole body so every return restores the backend flags.
+ # Wraps the whole body so every return restores the TF32 / cudnn.benchmark flags.
snap = _apply_perf_flags(cfg, device)
try:
# Preflight the base model against the same trust gate as inference, before any fetch.
@@ -354,10 +356,11 @@ def run_diffusion_lora_training(
pairs = discover_image_caption_pairs(
cfg.data_dir, instance_prompt = cfg.instance_prompt, caption_column = cfg.caption_column
)
- # Resolve num_epochs into a concrete train_steps now the dataset size is known, and rebind cfg so every downstream read agrees.
+ # Resolve num_epochs into a concrete train_steps now the dataset size is known, and rebind cfg so every
+ # downstream read agrees.
cfg = replace(cfg, train_steps = resolve_train_steps(cfg, len(pairs)), num_epochs = 0)
- # Validate a resume request against this run's identity BEFORE the multi-GB base model
- # load, so a mismatched checkpoint fails in seconds instead of after the download.
+ # Validate a resume request against this run's identity BEFORE the multi-GB base model load, so a
+ # mismatch fails in seconds instead of after the download.
identity = identity_for_config(cfg, dataset_pairs = pairs)
if cfg.resume_from_checkpoint:
preflight_resume(
@@ -377,9 +380,8 @@ def run_diffusion_lora_training(
lora_path = None,
stopped = True,
steps_run = 0,
- # Same disposition the full path reports. A stop with save=false is a DISCARD
- # however early it lands, and without it the resume fallback offers the source
- # bundle back as though the attempt were still live.
+ # A stop with save=false is a DISCARD however early it lands; without it the resume fallback offers
+ # the source bundle back as though the attempt were still live.
discarded = not save_on_stop,
)
return str(out_dir)
@@ -414,7 +416,8 @@ def run_diffusion_lora_training(
if weight_dtype != torch.float32:
cast_training_params(unet, dtype = torch.float32)
- # Regionally torch.compile the U-Net repeated blocks via the DiT trainer's never-fatal wrapper (failure falls back to eager).
+ # Regionally torch.compile the U-Net repeated blocks via the DiT trainer's never-fatal wrapper (failure
+ # falls back to eager).
from core.training.diffusion_dit_trainer import _maybe_compile_transformer
compiled = _maybe_compile_transformer(
@@ -423,7 +426,8 @@ def run_diffusion_lora_training(
lora_params = [p for p in unet.parameters() if p.requires_grad]
optimizer = _make_lora_optimizer(lora_params, cfg.learning_rate)
- # The scheduler advances once per optimizer update, so count warmup/decay in optimizer steps; the accumulation factor would stretch warmup.
+ # The scheduler advances once per optimizer update, so count warmup/decay in optimizer steps; the
+ # accumulation factor would stretch warmup.
lr_sched = get_scheduler(
cfg.lr_scheduler,
optimizer = optimizer,
@@ -434,7 +438,8 @@ def run_diffusion_lora_training(
vae_scale = vae.config.scaling_factor
prediction_type = noise_scheduler.config.prediction_type
- # Precompute text embeddings once per unique caption, then free the ~1.5 GB CLIP encoders. Deterministic, so the math is bit-identical to in-loop encoding.
+ # Deterministic, so the precomputed embeddings are bit-identical to in-loop encoding; the ~1.5 GB
+ # CLIP encoders are freed afterwards.
precompute = os.environ.get("UNSLOTH_DIFFUSION_NO_PRECOMPUTE", "") not in ("1", "true")
caption_embeds: dict[str, tuple] = {}
if precompute:
@@ -448,10 +453,11 @@ def run_diffusion_lora_training(
if device == "cuda":
torch.cuda.empty_cache()
- # Precompute the VAE latent cache, then free the VAE: it holds the posterior affine pair so per-step sampling noise is preserved.
+ # The cache holds the posterior affine pair, so per-step sampling noise is preserved once the VAE is freed.
use_cache = cfg.cache_latents and os.environ.get(
"UNSLOTH_DIFFUSION_NO_LATENT_CACHE", ""
) not in ("1", "true")
+ # Over the host-memory budget; keep the VAE resident and encode in-loop.
latent_cache = None
if use_cache:
latent_cache = _build_sdxl_latent_cache(
@@ -465,9 +471,8 @@ def run_diffusion_lora_training(
_check_stop,
)
if latent_cache is LATENT_CACHE_OVER_BUDGET:
- # Over the host-memory budget; keep the VAE resident and encode in-loop.
latent_cache = None
- elif latent_cache is None: # stopped during the cache build; nothing trained yet
+ elif latent_cache is None:
out_dir = Path(cfg.output_dir).expanduser()
_emit(
on_event,
@@ -476,7 +481,7 @@ def run_diffusion_lora_training(
lora_path = None,
stopped = True,
steps_run = 0,
- # As above: a discard is a discard however early the stop lands.
+ # A discard is a discard however early the stop lands.
discarded = not save_on_stop,
)
return str(out_dir)
@@ -490,31 +495,31 @@ def run_diffusion_lora_training(
gc.collect()
if device == "cuda":
torch.cuda.empty_cache()
- # Variant picks use their own stream so the loop index/noise draws stay seed-deterministic whether or not the cache is enabled.
+ # Variant picks use their own stream so the loop index/noise draws stay seed-deterministic whether
+ # or not the cache is enabled.
variant_rng = random.Random(cfg.seed + 1)
_emit(on_event, "model_load_completed", compiled = compiled)
- # The base is on disk now, so its commit can finally be read: the identity above was
- # built before the load and records "unresolved" on the first run of an uncached repo,
- # which no later resume could then enforce. Only ever adds the constraint.
+ # The base is on disk now, so its commit can finally be read: the identity above was built before
+ # the load and records "unresolved" on the first run of an uncached repo.
identity = with_resolved_revision(identity, cfg.base_model)
- # And the cache path the loop actually took, which the request does not settle: the
- # env override and the over-budget fallback both turn it off, and the two paths draw
- # crops and flips from different RNG streams.
+ # The cache path the loop actually took, which the request does not settle: the env override and
+ # the over-budget fallback both turn it off, and the two paths draw crops from different RNGs.
identity = with_cache_mode(identity, latent_cache is not None)
- # Permutation-cycle index sampler: each image is visited once per cycle before any repeat, so a short run does not leave a small dataset partly unseen.
+ # Permutation-cycle index sampler: each image is visited once per cycle before any repeat, so a
+ # short run does not leave a small dataset partly unseen.
index_sampler = PermutationBatchSampler(len(pairs), rng)
def _next_batch() -> tuple[list[int], list[str], list[str]]:
- # Draw the full configured batch, not min(batch, n): the sampler refills across cycles so a small dataset still yields that many indices.
+ # Draw the full configured batch, not min(batch, n): the sampler refills across cycles so a small
+ # dataset still yields that many indices.
idx = index_sampler.next_batch(cfg.train_batch_size)
chosen = [pairs[i] for i in idx]
return idx, [c[0] for c in chosen], [c[1] for c in chosen]
- # Restore a previous run's state (adapter, optimizer moments, LR position, sampler cycle,
- # RNG streams) BEFORE the loop, so `resumed` is the last completed step and the loop
- # picks up at resumed + 1. Zero for a fresh run; a mismatch raises out to the adapter.
+ # Restore a previous run's state BEFORE the loop, so resumed is the last completed step and the loop
+ # picks up at resumed + 1; a mismatch raises out to the adapter.
rng_streams = {"loop": rng, "variant": variant_rng}
restored = restore_resume_state(
cfg,
@@ -527,17 +532,13 @@ def run_diffusion_lora_training(
rng_streams = rng_streams,
)
resumed = restored.step if restored is not None else 0
- # Bound HERE, not at the export below: the checkpoint scan and the periodic saves both
- # need it, and the two earlier assignments live inside early-return branches -- so on
- # every normal run this read came before any binding at all.
+ # Bound HERE: the checkpoint scan and the periodic saves both need it, and the two earlier
+ # assignments live inside early-return branches.
out_dir = Path(cfg.output_dir).expanduser()
- # Whether this run's source bundle lives in THIS directory. Resuming from
- # directory A into a reused output_dir B leaves B's existing bundles as foreign
- # as they would be for a fresh run -- so the first save has to clear them, or a
- # higher-numbered one survives and a later resume by B picks it over this run.
+ # Resuming from directory A into a reused output_dir B leaves B's existing bundles as foreign as
+ # for a fresh run, so the first save must clear them or a higher-numbered one outranks this run.
resumed_here = bool(resumed) and resumed_into_this_dir(cfg, out_dir)
- # The bundles that were already here when this run started, so a discard below
- # removes only what this run wrote. A resumed run shares its source's directory.
+ # The bundles already here when this run started, so a discard removes only what this run wrote.
preexisting_checkpoints = snapshot_checkpoints(out_dir)
unet.train()
@@ -552,9 +553,9 @@ def run_diffusion_lora_training(
done = resumed
def _save_checkpoint(step: int) -> None:
- # Outcome is reported by write_resume_checkpoint's own checkpoint_saved /
- # checkpoint_failed events, so a run that crashes after a save is still known to be
- # resumable (and one whose save failed is still known to be blocked).
+ # write_resume_checkpoint reports its own checkpoint_saved / checkpoint_failed events, so a run
+ # that crashes after a save is still known to be resumable and one whose save failed is still
+ # known to be blocked.
_written, _error = write_resume_checkpoint(
cfg,
step = step,
@@ -566,15 +567,10 @@ def run_diffusion_lora_training(
sampler = index_sampler,
rng_streams = rng_streams,
progress = {"running_loss": running_loss},
- # NOT discard_existing. A fresh run does own its output dir, but deleting the
- # previous run's bundles at the FIRST periodic save spends them before this run
- # has produced anything: "stop without saving" then removes only what this run
- # wrote, leaves the previous adapter in place, and the run it belonged to is
- # unresumable -- cancelling a retrain destroyed the thing being retrained. The
- # clear happens on the completion path instead, once the new adapter is saved.
+ # NOT discard_existing: deleting the previous run's bundles at the FIRST periodic save spends them
+ # before this run produced anything, so cancelling a retrain destroyed the thing being retrained.
discard_existing = False,
- # And the bundles that were here before this run: a branched resume must not
- # prune the higher-numbered checkpoints it did not write.
+ # A branched resume must not prune the higher-numbered checkpoints it did not write.
preexisting = preexisting_checkpoints,
)
@@ -648,7 +644,8 @@ def run_diffusion_lora_training(
step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps
micro += 1
- # max_grad_norm at or below 0 disables clipping (Unsloth sends 0.0); passing 0.0 to clip_grad_norm_ would zero every gradient.
+ # max_grad_norm at or below 0 disables clipping (Unsloth sends 0.0); passing 0.0 to clip_grad_norm_
+ # would zero every gradient.
grad_norm = None
if cfg.max_grad_norm and cfg.max_grad_norm > 0:
# Returned value is the total PRE-clip norm, reported to the UI chart.
@@ -659,8 +656,8 @@ def run_diffusion_lora_training(
running_loss += step_loss
done = opt_step + 1
now = time.time()
- # Rates count only the steps THIS process ran, so a run resumed at step 11 does not
- # divide by 12 steps it never executed.
+ # Rates count only the steps THIS process ran, so a run resumed at step 11 does not divide by 12
+ # steps it never executed.
ran_here = done - resumed
if ran_here == 1:
# Step 1 pays the one-time costs (cudnn autotune, compile warmup), so the rate starts after it.
@@ -690,8 +687,7 @@ def run_diffusion_lora_training(
)
stop_now = _check_stop()
- # Periodic resume point. Skipped on the final step (a finished run has nothing left
- # to resume) and when stopping, since the stop path writes one at the exact step.
+ # Skipped on the final step and when stopping, since the stop path writes one at the exact step.
if (
not stop_now
and cfg.save_steps
@@ -708,9 +704,7 @@ def run_diffusion_lora_training(
catalog_path: Optional[str] = None
if not (stopped and not save_on_stop):
out_dir.mkdir(parents = True, exist_ok = True)
- # Stopping early is exactly when the run should be resumable, so write the bundle
- # BEFORE the adapter export: if that export then fails, the run still comes back.
- # A completed run has nothing left to resume.
+ # Write the bundle BEFORE the adapter export: if that export then fails, the run still comes back.
if stopped and done > 0:
_save_checkpoint(done)
unet_lora = convert_state_dict_to_diffusers(get_peft_model_state_dict(unet))
@@ -725,27 +719,18 @@ def run_diffusion_lora_training(
# ``done`` (the step reached), not cfg.train_steps: a stop at 11/500 must not advertise 500.
catalog_path = _publish_to_lora_catalog(lora_path, cfg, done)
if not stopped:
- # A COMPLETED run has nothing to resume, and the last iteration deliberately
- # writes no bundle -- so with save_steps on, the newest thing left in the
- # directory is checkpoint-400 of a run that finished at 500. The preflight
- # cannot see the run status, so a later resume with a raised target rolled the
- # model, optimizer, scheduler, sampler and RNG back and retrained 401-500.
- # Only this run's own bundles go; an earlier run's stay resumable.
+ # A completed run has nothing to resume and the last iteration writes no bundle, so with
+ # save_steps on the newest thing left is an earlier checkpoint, and a later resume with a raised
+ # target rolled everything back and retrained. Only this run's own bundles go.
retire_own_checkpoints(out_dir, preexisting_checkpoints, resumed_here = resumed_here)
elif not resumed_here:
- # Stopped WITH save on a fresh retrain: the stop bundle is a LOWER step than
- # the leftovers of the earlier run in this directory, and resume-by-directory
- # picks the newest by step -- so those would outrank the partial just saved
- # and continue the wrong training. A resumed run keeps what it found.
+ # A stop-with-save on a fresh retrain is a LOWER step than the earlier run's leftovers, and resume-
+ # by-directory picks the newest by step, so those would outrank the partial just saved.
discard_preexisting_checkpoints(out_dir, preexisting_checkpoints)
else:
- # Discarded: the user asked to throw this run away. Before periodic checkpoints
- # existed a discard left nothing behind, because the output directory was only
- # ever created inside this save gate. save_steps now writes bundles as the run
- # goes, so without this a discard leaves up to save_total_limit copies of the
- # optimizer state in a directory the user never got an artifact from -- invisible
- # to every scanner, unresumable (the record is marked discarded), and with no
- # delete path in the UI.
+ # save_steps writes bundles as the run goes, so without this a discard leaves up to
+ # save_total_limit copies of the optimizer state in a directory the user got no artifact from:
+ # invisible to every scanner, unresumable, with no delete path in the UI.
clear_own_checkpoints(out_dir, preexisting_checkpoints)
try:
out_dir.rmdir()
@@ -763,8 +748,7 @@ def run_diffusion_lora_training(
stopped = stopped,
steps_run = done if cfg.train_steps else 0,
resumed_from_step = resumed or None,
- # "Stop without saving" discards the run, so its own periodic checkpoints must not
- # keep offering to continue it.
+ # A discarded run's own periodic checkpoints must not keep offering to continue it.
discarded = bool(stopped and not save_on_stop),
)
return str(out_dir)
@@ -817,7 +801,8 @@ def run_diffusion_training_process(*, event_queue: Any, stop_queue: Any, config:
return got if saw else False
try:
- # normalized() resolves + validates the family; dispatch through the registry so a DiT family runs its own trainer.
+ # normalized() resolves + validates the family; dispatch through the registry so a DiT family runs
+ # its own trainer.
cfg = _config_from_dict(config).normalized()
trainer = get_trainer(cfg.resolved_family)
trainer(cfg, on_event = on_event, should_stop = should_stop)
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index ed44f1824dd..aa4f81b08bb 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -40,15 +40,17 @@ from core.inference.diffusion_families import (
from core.inference.video_families import detect_video_family, supported_video_family_names
from utils.paths.path_utils import drop_appledouble_metadata
-# The trainers run in a spawned child that imports diffusers itself, so the inference-side install does not carry over. Both import this module first.
+# The trainers run in a spawned child that imports diffusers itself, so the inference-side install
+# does not carry over. Both import this module first.
install_xformers_windows_rocm_stub()
install_torchao_windows_rocm_stub()
-# Default LoRA target modules: the attention projections common to the SDXL U-Net and the DiTs (the diffusers/kohya convention). A family may override this.
+# Default LoRA target modules: the attention projections common to the SDXL U-Net and the DiTs (the
+# diffusers/kohya convention). A family may override this.
DEFAULT_LORA_TARGETS: tuple[str, ...] = ("to_k", "to_q", "to_v", "to_out.0")
-# diffusers SchedulerType names. piecewise_constant is excluded: it alone needs a `step_rules` string the trainers
-# never pass, so accepting it would pass normalized(), free the resident GPU workloads, then crash in the child.
+# piecewise_constant is excluded: it alone needs a step_rules string the trainers never pass, so
+# accepting it would pass normalized(), free the resident GPU workloads, then crash in the child.
_LR_SCHEDULERS: frozenset[str] = frozenset(
{
"linear",
@@ -60,39 +62,29 @@ _LR_SCHEDULERS: frozenset[str] = frozenset(
}
)
-# DiT families whose fp32 RoPE/embedder overflow fp16, so they train in bf16 only. Keep in sync with the DiT trainer's own specs.
+# DiT families whose fp32 RoPE/embedder overflow fp16, so they train in bf16 only. Keep in sync with
+# the DiT trainer's own specs.
_FORCE_BF16_FAMILIES: frozenset[str] = frozenset(
{"qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2", "minimax-h3"}
)
-# VIDEO families (from the separate ``video_families`` registry) Unsloth can train a LoRA on.
-# The video registry has no ``trainable`` flag of its own -- it exists for the inference
-# picker -- so the trainable set lives here, next to the trainers, and every name in it MUST
-# resolve through ``get_trainer``. Not necessarily to the DiT loop: LTX-2 is a
-# ``diffusion_dit_trainer._SPECS`` family, while MiniMax-H3 has its own trainer. A video base
-# outside this set is refused by name in ``resolve_trainable_family`` rather than falling
-# through to a trainer that cannot run it.
+# The video registry has no trainable flag, so the trainable set lives here and every name in it
+# MUST resolve through get_trainer; a video base outside it is refused by name.
TRAINABLE_VIDEO_FAMILIES: frozenset[str] = frozenset({"ltx-2", "minimax-h3"})
-# Families whose ``flow_shift`` default is "auto" (reproduce the family's INFERENCE sigma
-# distribution) rather than the historical identity 1.0. Both schedulers set
-# ``use_dynamic_shifting``, so their static ``shift`` is skipped at init and ``scheduler.sigmas``
-# is the unshifted uniform table; training on it would draw a distribution the model never sees
-# at inference. Qwen-Image pins base_shift = max_shift = log 3 and LTX-2 evaluates its shift at
-# ``max_image_seq_len`` (so mu = max_shift = 2.05 at every resolution): in both cases the
-# inference mu is a constant the "auto" branch can reproduce exactly.
-#
-# MiniMax-H3 is here for the same reason, reached differently: its schedules carry an explicit
-# exponential shift (12.0 video, 3.0 audio, from the released scheduler configs) rather than a
-# dynamic one, and ``run_h3_lora_training`` applies those two whenever ``flow_shift`` is not a
-# number. Without this entry an omitted flow_shift normalized to the identity 1.0, which IS a
-# number, so the trainer took it and every default H3 run trained against an unshifted
-# distribution the sampler never visits.
+# Families whose flow_shift default is "auto" (reproduce the family's INFERENCE sigma
+# distribution) rather than the identity 1.0. Both schedulers set use_dynamic_shifting, so
+# scheduler.sigmas is the unshifted uniform table and training on it draws a distribution the
+# model never sees at inference: Qwen-Image pins base_shift = max_shift = log 3 and LTX-2
+# evaluates its shift at max_image_seq_len (mu = max_shift = 2.05 at every resolution), so the
+# inference mu is a constant "auto" reproduces. MiniMax-H3 instead applies explicit exponential
+# shifts (12.0 video, 3.0 audio) whenever flow_shift is not a number, so without this entry a
+# default H3 run trained unshifted.
AUTO_FLOW_SHIFT_FAMILIES: frozenset[str] = frozenset({"qwen-image", "ltx-2", "minimax-h3"})
-# Video latents are allocated on the family's spatial compression grid, so a training
-# resolution off that grid silently changes the latent geometry (or trips a reshape).
-# LTX-2's VAE compresses 32x spatially, matching its ``resolution_multiple``.
+# Video latents are allocated on the family's spatial compression grid, so a training resolution
+# off that grid silently changes the latent geometry. LTX-2's VAE compresses 32x spatially,
+# matching its resolution_multiple.
_VIDEO_RESOLUTION_MULTIPLE = 32
_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp"}
@@ -100,12 +92,14 @@ _CAPTION_EXTS = (".txt", ".caption")
# diffusers' canonical single-file LoRA name, so load_lora_weights(dir) finds it.
DEFAULT_LORA_FILENAME = "pytorch_lora_weights.safetensors"
-# Architectures Unsloth can neither train nor load: not in the family registry but recognisable by name, so rejecting them by name gives a clear error instead of a mid-run crash.
+# Architectures Unsloth can neither train nor load: not in the family registry but recognisable by
+# name, so rejecting them by name gives a clear error instead of a mid-run crash.
_NON_TRAINABLE_RESIDUAL_TOKENS = frozenset({"sd3", "pixart", "sana", "lumina", "cogview"})
_NON_TRAINABLE_RESIDUAL_PHRASES = ("stable-diffusion-3", "hunyuan-dit")
EventCb = Callable[[dict[str, Any]], None]
-# Returns a falsy value to keep training, or a truthy stop signal: bare True, or a dict that may carry ``save=False`` to cancel without saving a partial adapter.
+# Returns a falsy value to keep training, or a truthy stop signal: bare True, or a dict that may
+# carry ``save=False`` to cancel without saving a partial adapter.
StopCb = Callable[[], Any]
@@ -181,14 +175,8 @@ def _component_only_repos() -> dict[str, tuple[str, str, str]]:
if repo:
bases.add(str(repo).strip().lower())
bases.update(str(r).strip().lower() for r in getattr(fam, "train_base_repos", ()) if r)
- # (scheme, repo) and (base, scheme, repo). The two registries mean DIFFERENT things by
- # these tables, so they cannot be read the same way. An image family's entry is a full
- # quantized PIPELINE mirror, and so a base. A video family's is a hosted pre-quantized
- # DENOISER -- the DiT alone, as the field's own docstring says -- which is a component in
- # exactly the way a pre-cast text encoder is: no model_index.json, no VAE, no scheduler.
- # Treating one as a base is what let it through the training preflight, resolve to its
- # family by name, pass the unsloth/* trust gate, and fail inside from_pretrained only
- # after the resident GPU workloads had been evicted.
+ # An image family's entry is a full quantized PIPELINE mirror, and so a base; a video family's is a
+ # hosted pre-quantized DENOISER, a component.
for table in ("prequant_repos", "prequant_variant_repos"):
for row in getattr(fam, table, ()) or ():
if not row:
@@ -290,8 +278,8 @@ def training_pipeline_import_error(resolved_family: str) -> Optional[str]:
family_probe_class,
)
- # Either registry: a video family is invisible to detect_family, and returning None for one
- # would hand the strict half of the gate back to the spawned child, after the teardown.
+ # A video family is invisible to detect_family, and returning None would hand the strict half of the
+ # gate to the spawned child, after the teardown.
fam = _trainable_family_spec(resolved_family)
if fam is None:
return None
@@ -323,10 +311,8 @@ def resolve_trainable_family(base_model: str, model_family: Optional[str] = None
"""
name = str(base_model or "").strip().lower()
# GGUF weights (a ``.gguf`` file or ``*-GGUF`` repo) are inference-only: training needs the full diffusers pipeline.
- # Exempt a local diffusers checkout that merely has "gguf" in its path, identified by its
- # index file. ``modular_model_index.json`` counts: a MODULAR_BASE_FAMILIES checkout (MiniMax-H3)
- # has that and no ``model_index.json``, so reading only the conventional name refused the one
- # local form the family has, under /models/gguf/..., before model_family could even be read.
+ # modular_model_index.json counts: a MODULAR_BASE_FAMILIES checkout has that and no
+ # model_index.json, so reading only the conventional name refused the one local form the family has.
local = Path(base_model).expanduser() if base_model else None
is_local_diffusers = bool(
local
@@ -339,19 +325,17 @@ def resolve_trainable_family(base_model: str, model_family: Optional[str] = None
f"'{base_model}' is a GGUF checkpoint/repo, which can't be a training base "
f"(training needs the full diffusers model). {_trainable_hint()}"
)
- # A component checkpoint is not a base whatever family the name matches, so this precedes
- # every family branch below (including an explicit model_family override, which names the
- # family but says nothing about what the repo holds).
+ # A component checkpoint is not a base whatever family the name matches, so this precedes every
+ # family branch, including an explicit model_family override.
_refuse_component_only_repo(base_model)
- # Same shape as the component-only refusal: the name resolves to a real, trainable family, but
- # the repo is not something the training loader can open.
+ # Same shape as the component-only refusal: the name resolves to a real, trainable family, but the repo is
+ # not something the training loader can open.
_refuse_ltx23_training_base(base_model)
if model_family and str(model_family).strip():
key = str(model_family).strip().lower()
fam = detect_family("", override = key)
if fam is None:
- # Not an image family: it may still name a VIDEO family, which lives in its own
- # registry. Resolve there before declaring the name unknown.
+ # Not an image family: it may still name a VIDEO family, which lives in its own registry.
vid = detect_video_family("", override = key)
if vid is not None:
if vid.name not in TRAINABLE_VIDEO_FAMILIES:
@@ -375,8 +359,7 @@ def resolve_trainable_family(base_model: str, model_family: Optional[str] = None
_assert_family_pipeline_available(fam)
return fam.name
- # Only once the image registry has declined the name: a video checkpoint. Checked here
- # rather than first so an image repo the picker already claims keeps its existing route.
+ # Checked here rather than first so an image repo the picker already claims keeps its existing route.
vid = detect_video_family(base_model)
if vid is not None:
if vid.name not in TRAINABLE_VIDEO_FAMILIES:
@@ -490,19 +473,19 @@ def get_trainer(family: str) -> Callable[..., str]:
if key in ("flux.1", "qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2"):
from core.training.diffusion_dit_trainer import run_dit_lora_training
return run_dit_lora_training
- # MiniMax-H3 has its own loop: it denoises video and audio jointly over one packed
- # sequence on two coupled schedules, which is outside the DiT trainer's _FamilySpec seams.
+ # MiniMax-H3 denoises video and audio jointly over one packed sequence on two coupled schedules,
+ # which is outside the DiT trainer's _FamilySpec seams.
if key == "minimax-h3":
from core.training.diffusion_h3_trainer import run_h3_lora_training
return run_h3_lora_training
raise ValueError(f"No trainer is registered for family {family!r}.")
-# Per-family training defaults surfaced by the Train UI: starting points, not hard limits. Families absent here fall back to the DiffusionLoraConfig defaults.
+# Per-family training defaults surfaced by the Train UI: starting points, not hard limits. Families
+# absent here fall back to the DiffusionLoraConfig defaults.
FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {
"sdxl": {"lora_rank": 16, "learning_rate": 1e-4, "resolution": 1024},
- # Plain "constant" ignores lr_warmup_steps, so warmup defaults must use a
- # warmup-capable scheduler.
+ # Plain "constant" ignores lr_warmup_steps, so warmup defaults must use a warmup-capable scheduler.
"flux.1": {
"lora_rank": 16,
"learning_rate": 1e-4,
@@ -520,7 +503,8 @@ FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {
"z-image": {"lora_rank": 16, "learning_rate": 1e-4, "resolution": 768},
# The Krea 2 authors' recommended starting point (their DreamBooth defaults): rank/alpha 32, lr 3e-4, 512px.
"krea-2": {"lora_rank": 32, "learning_rate": 3e-4, "resolution": 512},
- # Upstream FLUX.2 DreamBooth references default to rank 16 / lr 1e-4; its uniform timestep draw benefits most from a warmup ramp.
+ # Upstream FLUX.2 DreamBooth references default to rank 16 / lr 1e-4; its uniform timestep draw
+ # benefits most from a warmup ramp.
"flux.2-klein": {
"lora_rank": 16,
"learning_rate": 1e-4,
@@ -535,9 +519,8 @@ FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {
"lr_scheduler": "constant_with_warmup",
"lr_warmup_steps": 20,
},
- # LTX-2, from Lightricks' own ltx-trainer LoRA configs: rank/alpha 32, lr 1e-4. The
- # resolution must be a multiple of 32 (its VAE's spatial compression), and 512 keeps a
- # still at 16x16x1 latents / 256 video tokens, which trains comfortably at batch 1.
+ # From Lightricks' own ltx-trainer LoRA configs; the resolution must be a multiple of 32, its
+ # VAE's spatial compression, and 512 keeps a still at 16x16x1 latents / 256 video tokens.
"ltx-2": {
"lora_rank": 32,
"learning_rate": 1e-4,
@@ -545,11 +528,8 @@ FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {
"lr_scheduler": "constant_with_warmup",
"lr_warmup_steps": 20,
},
- # MiniMax-H3. ``resolution`` is the canvas SHORT EDGE, and 768 is the one the released
- # checkpoint generates on, so a training clip's spatial statistics land exactly on the
- # distribution the sampler works in. rank 16 rather than LTX-2's 32: the adapter is applied
- # to the audio rows as well as the video ones (one shared block stack), so a smaller
- # adapter is the conservative default. Batch size is pinned to 1 by the trainer.
+ # resolution is the canvas SHORT EDGE and 768 is what the released checkpoint generates on;
+ # rank 16 not 32 because the adapter also serves the audio rows through one shared stack.
"minimax-h3": {
"lora_rank": 16,
"learning_rate": 1e-4,
@@ -566,7 +546,8 @@ def train_defaults(family: str) -> dict[str, Any]:
return dict(FAMILY_TRAIN_DEFAULTS.get((family or "").strip().lower(), {}))
-# Display labels + a short VRAM/access note per trainable family, surfaced by the Train UI so users pick a base with realistic expectations.
+# Display labels + a short VRAM/access note per trainable family, surfaced by the Train UI so users pick a base
+# with realistic expectations.
_FAMILY_LABELS = {
"sdxl": "SDXL",
"flux.1": "FLUX.1-dev",
@@ -578,8 +559,7 @@ _FAMILY_LABELS = {
"ltx-2": "LTX-2",
"minimax-h3": "MiniMax-H3",
}
-# Per-family training facts as fields, so the UI can chip them instead of parsing prose.
-# ``params`` is the transformer size (SDXL is not quoted that way), ``note`` is the rest.
+# params is the transformer size (SDXL is not quoted that way); note is the rest.
_FAMILY_TRAIN_SPECS: dict[str, dict[str, Any]] = {
"sdxl": {"params": "", "qlora_vram_gb": 12, "gated": False, "note": "The lightest option."},
"flux.1": {"params": "12B", "qlora_vram_gb": 16, "gated": True, "note": ""},
@@ -598,26 +578,16 @@ _FAMILY_TRAIN_SPECS: dict[str, dict[str, Any]] = {
},
"flux.2-klein": {"params": "4B", "qlora_vram_gb": 10, "gated": False, "note": ""},
"flux.2-dev": {"params": "32B", "qlora_vram_gb": 28, "gated": True, "note": ""},
- # Video. Measured on a B200: the training LOOP peaks at 11.2 GB (nf4, rank 32, 512px
- # stills, batch 1, gradient checkpointing), but the RUN peaks at 34.8 GB earlier, while
- # the Gemma3-12B conditioning stack is resident and captions are encoded -- before it is
- # freed and the transformer loads. The quoted figure covers the whole run, since that is
- # what a card has to hold. Lightricks quote 80 GB recommended / 32 GB with their int8
- # low-VRAM config for their own trainer.
+ # Measured on a B200: the training LOOP peaks at 11.2 GB, but the RUN peaks at 34.8 GB while the
+ # Gemma3-12B conditioning stack is resident, and the quoted figure covers the whole run.
"ltx-2": {
"params": "19B",
"qlora_vram_gb": 36,
"gated": False,
"note": "Video: trains a style LoRA on still images.",
},
- # Video + audio. Measured on a B200 at the released 768 short edge, 22-frame clips,
- # rank 16, batch 1, gradient checkpointing: the training LOOP peaks around 44 GB (nf4 with
- # the dtype-reading modules kept dense, see _H3_NF4_SKIP_MODULES), but the RUN peaks far
- # higher while the 63 GiB Qwen3-VL conditioner is resident and captions are encoded --
- # before it is freed and the transformer loads. The quoted figure covers the whole run,
- # since that is what a card has to hold: a real 20-step run on this branch peaked at
- # 77.76 GB, so 72 was under its own measurement and sized users onto a card that OOMs
- # only after the conditioner has loaded.
+ # Measured on a B200: the loop peaks near 44 GB, but a 20-step run peaked at 77.76 GB with the
+ # 63 GiB Qwen3-VL conditioner resident, so 72 sizes users onto a card that OOMs later.
"minimax-h3": {
"params": "31B",
"qlora_vram_gb": 80,
@@ -625,14 +595,13 @@ _FAMILY_TRAIN_SPECS: dict[str, dict[str, Any]] = {
"note": "Video with sound: trains on clips that have a soundtrack.",
},
}
-# Facts that differ between selectable checkpoints inside one family. Keys are canonical
-# upstream ids; family_train_infos also publishes the mirror aliases so a custom mirror pick
-# gets the same guidance. Values overlay the family facts in the client.
+# Keys are canonical upstream ids; family_train_infos also publishes the mirror aliases, and values
+# overlay the family facts in the client.
_BASE_TRAIN_SPECS: dict[str, dict[str, Any]] = {
"black-forest-labs/flux.2-klein-base-9b": {
"params": "9B",
- # The auto policy measures 16.4 GB for the bf16 text encoder alone. Leave room for
- # the VAE and runtime state rather than inheriting the 4B checkpoint's 10 GB floor.
+ # The bf16 text encoder alone measures 16.4 GB, so leave room for the VAE and runtime state rather
+ # than inheriting the 4B checkpoint's floor.
"qlora_vram_gb": 18,
},
}
@@ -650,13 +619,12 @@ def _family_vram_note(name: str) -> str:
return " ".join([head, *tail])
-# The flow-matching DiT families (run by diffusion_dit_trainer): they expose base_precision / compile and need bf16 on CUDA. SDXL is absent (own mixed_precision path).
+# The flow-matching DiT families (run by diffusion_dit_trainer): they expose base_precision /
+# compile and need bf16 on CUDA. SDXL is absent (own mixed_precision path).
_DIT_TRAIN_FAMILIES = frozenset(
{"flux.1", "qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2"}
)
-# Families with a flow-matching trainer that is NOT diffusion_dit_trainer, but which still
-# expose base_precision and need bf16 on CUDA. Kept separate so the DiT-specific levers
-# (compile, the shared sigma table) do not follow.
+# Kept separate so the DiT-specific levers (compile, the shared sigma table) do not follow.
_FLOW_TRAIN_FAMILIES = _DIT_TRAIN_FAMILIES | {"minimax-h3"}
@@ -673,14 +641,9 @@ def effective_mixed_precision(cfg: Any) -> str:
requested = str(getattr(cfg, "mixed_precision", "") or "")
if str(getattr(cfg, "resolved_family", "") or "").strip().lower() in _FLOW_TRAIN_FAMILIES:
- # No flow-matching trainer reads mixed_precision at all: weight_dtype is bf16 on CUDA and
- # fp32 otherwise, in the DiT loop and in the H3 loop alike. Recording the REQUEST for those
- # families put "fp16" or "no" in the identity of a run that executed in bf16, and a later
- # bf16 resume of it was then rejected as a precision mismatch between two runs that ran
- # identically. _FLOW_TRAIN_FAMILIES rather than _DIT_TRAIN_FAMILIES so the answer follows
- # the weight dtype rather than which loop happens to own the family: H3 refuses a non-bf16
- # request and writes no checkpoint today, so this is the same string either way for every
- # config it accepts, but the helper stops being the odd one out the day that changes.
+ # No flow-matching trainer reads mixed_precision (weight_dtype is bf16 on CUDA, fp32 otherwise),
+ # so recording the REQUEST failed a later bf16 resume as a precision mismatch between identical
+ # runs. Keyed on _FLOW_TRAIN_FAMILIES so the answer follows the weight dtype.
return "bf16" if torch.cuda.is_available() else "no"
if not torch.cuda.is_available():
return "no"
@@ -753,7 +716,8 @@ def dit_accelerator_missing_reason(resolved_family: str) -> Optional[str]:
try:
import torch
def probe(owner: Any) -> bool:
- # Each accelerator is probed on its own: torch.mps.is_available() only exists from torch 2.5 while the floor is 2.4, so a shared try/except would wave a CPU-only host through.
+ # Each accelerator is probed on its own: torch.mps.is_available() only exists from torch 2.5 while
+ # the floor is 2.4, so a shared try/except would wave a CPU-only host through.
try:
fn = getattr(owner, "is_available", None)
return bool(fn()) if callable(fn) else False
@@ -796,7 +760,8 @@ def training_precision_preflight_error(resolved_family: str, base_precision: str
fam = (resolved_family or "").strip().lower()
mode = (base_precision or "").strip().lower()
if fam in _FLOW_TRAIN_FAMILIES and mode in ("bf16", "int8", "fp8", "mxfp8"):
- # The DiT trainer's dense precisions all require CUDA, and bf16_unsupported_reason exempts a CPU-only host, so without this a dense request would evict residents then raise in the child.
+ # The DiT trainer's dense precisions all require CUDA, and bf16_unsupported_reason exempts a CPU-
+ # only host, so without this a dense request would evict residents then raise in the child.
try:
import torch
has_cuda = torch.cuda.is_available()
@@ -818,13 +783,15 @@ def training_precision_preflight_error(resolved_family: str, base_precision: str
"base_precision='int8' needs a functional torchao install; this host's torchao is "
"missing or the non-functional Windows-ROCm stub. Use 'nf4', 'bf16', or 'auto'."
)
- # Mirrors the child's fp8/mxfp8 stub guard so a doomed run is refused before eviction. Not has_functional_torchao(): that probes int8's symbols.
+ # Mirrors the child's fp8/mxfp8 stub guard so a doomed run is refused before eviction. Not
+ # has_functional_torchao(): that probes int8's symbols.
if mode in ("fp8", "mxfp8") and is_stubbed("torchao"):
return (
f"base_precision={mode!r} is not available on this host: torchao is the "
"non-functional Windows-ROCm stub. Use 'nf4', 'bf16', or 'auto'."
)
- # mxfp8 needs Blackwell (sm100+): its MX GEMM raises at the first training step, AFTER a full dense load. Re-check here so a stale client fails fast before eviction.
+ # mxfp8 needs Blackwell (sm100+): its MX GEMM raises at the first training step, AFTER a full dense
+ # load. Re-check here so a stale client fails fast before eviction.
if mode == "mxfp8":
try:
import torch
@@ -858,28 +825,21 @@ def family_train_infos() -> list[dict[str, Any]]:
fam = _trainable_family_spec(name)
if fam is None or not family_pipeline_available(fam):
continue
- # A video family carries no train_base_repos/deploy_base_repo: its own base repo is the
- # one training base, and a video LoRA has no image catalog to deploy into.
+ # A video family carries no train_base_repos/deploy_base_repo: its own base repo is the one training base.
repos = list(getattr(fam, "train_base_repos", ()) or ()) or [fam.base_repo]
- # base_precision applies to every flow-matching trainer, not only the shared DiT one:
- # MiniMax-H3 has its own trainer but the same base_precision lever and the same bf16-on-
- # CUDA requirement. Reading _DIT_TRAIN_FAMILIES here reported precision_modes = [] for it,
- # which the Train panel reads as "this GPU cannot train this family" and disables Start
- # with "Not supported on this GPU" -- the trainer was unreachable from Unsloth on every
- # host. SDXL keeps its mixed_precision lever and is in neither set.
+ # base_precision applies to every flow-matching trainer, not only the shared DiT one: reading
+ # _DIT_TRAIN_FAMILIES here reported precision_modes = [] for H3, which the Train panel shows as
+ # "Not supported on this GPU".
is_dit = name in _FLOW_TRAIN_FAMILIES
- # On a non-bf16 CUDA GPU the start preflight rejects EVERY DiT family, so advertise no precision, else /info offers an
- # nf4 DiT option that always 400s. Otherwise drop any scheme this family's DiT corrupts, plus any the TRAINING bar
- # holds back even though inference passes (fp8 on Qwen-Image: rendering is validated, a training run is not).
+ # On a non-bf16 CUDA GPU the start preflight rejects EVERY DiT family, so advertise no precision
+ # rather than an option that always 400s; also drop schemes the TRAINING bar holds back.
dit_block = (
bf16_unsupported_reason(name) or dit_accelerator_missing_reason(name)
if is_dit
else None
)
- # compile is the one DiT-only lever that does NOT follow the precision one. MiniMax-H3
- # refuses compile_transformer="on" -- its packed sequence changes length with every
- # caption, so each step would re-trace -- so advertising the control would offer a
- # selection that always 400s.
+ # H3 refuses compile_transformer="on" (its packed sequence changes length with every caption, so
+ # each step would re-trace), so advertising the control would offer a selection that always 400s.
supports_compile = bool(not dit_block) and name in _DIT_TRAIN_FAMILIES
if not is_dit or dit_block:
fam_modes: list[str] = []
@@ -898,8 +858,8 @@ def family_train_infos() -> list[dict[str, Any]]:
base_specs[repo_mirror] = dict(base_spec)
for training_repo, inference_repo in getattr(fam, "deploy_base_repos", ()):
deploy_bases[training_repo] = inference_repo
- # A custom base entered with the public mirror id must follow the same pairing as the
- # advertised vendor id. Return the inference mirror too, so Deploy stays ungated.
+ # A custom base entered with the public mirror id must follow the same pairing as the advertised
+ # vendor id; return the inference mirror too so Deploy stays ungated.
training_mirror = mirror_repo(training_repo)
if training_mirror:
deploy_bases[training_mirror] = mirror_repo(inference_repo) or inference_repo
@@ -911,37 +871,29 @@ def family_train_infos() -> list[dict[str, Any]]:
"base_repos": repos,
"defaults": train_defaults(name),
"vram_note": dit_block or _family_vram_note(name),
- # Chip fields. Dropped on a dit_block, since vram_note then carries the reason.
+ # Dropped on a dit_block, since vram_note then carries the reason.
"params": "" if dit_block else spec.get("params", ""),
"qlora_vram_gb": None if dit_block else spec.get("qlora_vram_gb"),
"gated": False if dit_block else bool(spec.get("gated", False)),
"note": "" if dit_block else spec.get("note", ""),
"precision_modes": fam_modes,
"recommended_precision": "nf4" if (not is_dit or dit_block) else dit_recommended,
- # compile is offered for SDXL's regional U-Net and the shared DiT trainer, except
- # a family the GPU cannot train in bf16, and except a trainer that cannot compile.
+ # compile is offered for SDXL's regional U-Net and the shared DiT trainer, except a family the
+ # GPU cannot train in bf16, and except a trainer that cannot compile.
"supports_compile": supports_compile or name == "sdxl",
- # Same reasoning one field up, for the other control a family can simply not
- # have. save_steps is REFUSED for a checkpointless family, not ignored, so a
- # panel that keeps offering "Checkpoint every" turns a nonzero value into a
- # rejected Start with no way to see why from the control itself.
+ # save_steps is REFUSED for a checkpointless family, not ignored, so a panel that keeps offering
+ # "Checkpoint every" turns a nonzero value into a rejected Start with no way to see why.
"supports_checkpoints": name not in CHECKPOINTLESS_FAMILIES,
- # And the third one, for the same reason. A batch > 1 is REFUSED for a family
- # whose forward covers one packed sequence, so a panel that keeps offering an
- # unrestricted Batch turns a perfectly reasonable 2 -- or a value carried over
- # from the family the user was on a moment ago -- into a rejected Start with
- # nothing on the control to say why.
+ # A batch > 1 is REFUSED for a family whose forward covers one packed sequence, so leaving the
+ # control unrestricted turns a reasonable 2 into a rejected Start with nothing to say why.
"max_train_batch_size": 1 if name in SINGLE_SEQUENCE_FAMILIES else None,
# Krea trains on Raw but previews adapters on Turbo; None elsewhere (and never for a video family).
"deploy_base": getattr(fam, "deploy_base_repo", None),
# Families with several train/deploy pairs cannot use the scalar above.
"deploy_bases": deploy_bases,
- # Checkpoint-specific facts overlay the family facts in the Train UI. Dropped on a
- # dit_block for the same reason the family chips above are: the overlay wins in
- # resolveDiffusionTrainingFacts, so a per-base entry would put the 9B / 18 GB chips
- # back, and FamilyFacts renders vram_note only when there are NO chips. Leaving
- # these populated therefore replaces the actionable hardware reason (no CUDA, no
- # native bf16) with a size the host cannot act on.
+ # Dropped on a dit_block for the same reason as the family chips: the overlay wins and
+ # FamilyFacts renders vram_note only when there are no chips, so keeping these would replace the
+ # actionable hardware reason with a size the host cannot act on.
"base_specs": {} if dit_block else base_specs,
}
)
@@ -956,67 +908,74 @@ class DiffusionLoraConfig:
base_model: str
data_dir: str
output_dir: str
- # Dreambooth-style caption applied to any image without its own. Required if the dataset has no captions.jsonl / sidecars.
+ # Dreambooth-style caption applied to any image without its own. Required if the dataset has no
+ # captions.jsonl / sidecars.
instance_prompt: Optional[str] = None
resolution: int = 1024
train_steps: int = 500
- # 0 = disabled (train for train_steps). Above 0 it overrides train_steps with num_epochs full passes (see resolve_train_steps).
+ # 0 = disabled (train for train_steps). Above 0 it overrides train_steps with num_epochs full passes
+ # LoRA EMA decay (DiT trainer only). 0.0 disables; a positive value keeps a warmup-ramped EMA of
+ # the trainable params and exports it under ema/.
num_epochs: int = 0
learning_rate: float = 1e-4
train_batch_size: int = 1
gradient_accumulation_steps: int = 1
lora_rank: int = 16
- lora_alpha: Optional[int] = None # defaults to lora_rank
+ lora_alpha: Optional[int] = None
lora_dropout: float = 0.0
lora_target_modules: tuple[str, ...] = DEFAULT_LORA_TARGETS
seed: int = 42
- mixed_precision: str = "bf16" # "bf16" | "fp16" | "no"
- snr_gamma: Optional[float] = 5.0 # min-SNR loss weighting; None disables
+ mixed_precision: str = "bf16"
+ snr_gamma: Optional[float] = 5.0
gradient_checkpointing: bool = True
max_grad_norm: float = 1.0
lr_scheduler: str = "constant"
lr_warmup_steps: int = 0
center_crop: bool = False
random_flip: bool = True
- caption_column: str = "text" # column in metadata.jsonl
+ caption_column: str = "text"
adapter_name: str = "default"
hf_token: Optional[str] = None
- # Derived by normalized(): the byte-identical mirror used by from_pretrained while
- # base_model remains the canonical id stored in metadata and resume identity.
+ # Derived by normalized(): the byte-identical mirror used by from_pretrained, while base_model stays
+ # the canonical id in metadata and resume identity.
fetch_base_model: Optional[str] = None
- # Precompute the VAE latents once (freeing the VAE) instead of re-encoding every step. ``cache_variants`` crop/flip draws are frozen per image; the per-step VAE sampling noise is preserved.
+ # cache_variants crop/flip draws are frozen per image; the per-step VAE sampling noise is preserved.
cache_latents: bool = True
cache_variants: int = 4
- # Persistent on-disk conditioning cache directory (DiT trainer only). None keeps the in-memory-only behavior; a
- # directory persists latent stats and caption embeddings keyed by content hash, so a warm cache skips the VAE and TEs.
+ # A directory persists latent stats and caption embeddings keyed by content hash, so a warm cache
+ # skips the VAE and text encoders.
cond_cache_dir: Optional[str] = None
- # LoRA EMA decay (DiT trainer only). 0.0 disables; a positive value keeps a warmup-ramped EMA of the trainable params and exports it under ema/.
+ # LoRA EMA decay (DiT trainer only). 0.0 disables; a positive value keeps a warmup-ramped EMA of
+ # the trainable params and exports it under ema/.
ema_decay: float = 0.0
- # Regional torch.compile of the transformer blocks: "off" | "on" | "auto" (auto turns it on only for a dense, non-bitsandbytes base where it is a clean win).
+ # Regional torch.compile of the transformer blocks: "off" | "on" | "auto" (auto turns it on only for
+ # a dense, non-bitsandbytes base where it is a clean win).
compile_transformer: str = "auto"
# TF32 matmuls + cudnn autotuning for the run. Near-lossless; disable for strict bit-reproducibility A/Bs.
enable_tf32: bool = True
# DiT base transformer precision: "nf4" (bitsandbytes QLoRA, the memory floor and default), "bf16" (dense, fastest
- # eager), "int8" (torchao weight-only), "fp8" (Ada/Hopper/Blackwell + compile) or "auto". Non-nf4 needs a dense base.
base_precision: str = "nf4"
- # Training-time timestep shift on the flow-matching sigma draw. None resolves per family in normalized(): "auto" for
- # qwen-image, 1.0 elsewhere. A number applies s*u/(1+(s-1)*u); "auto" without dynamic shifting falls back to identity.
- flow_shift: Optional[Any] = None # float | "auto" | None
- # Per-sample probability of replacing the caption with the empty prompt (classifier-free-guidance dropout). 0.0 disables.
+ # None resolves per family in normalized(); a number applies s*u/(1+(s-1)*u), and "auto" without
+ # dynamic shifting falls back to identity.
+ flow_shift: Optional[Any] = None
+ # Per-sample probability of replacing the caption with the empty prompt (classifier-free-guidance dropout).
+ # 0.0 disables.
cfg_dropout: float = 0.0
- # Per-sample loss weighting over the drawn timestep: "none" (unweighted MSE) or "bell" (Gaussian, normalized to mean 1).
+ # Per-sample loss weighting over the drawn timestep: "none" (unweighted MSE) or "bell" (Gaussian, normalized
+ # to mean 1).
weighting_scheme: str = "none"
# How often to emit a progress event (in optimizer steps).
log_every: int = 1
- # Periodic resume-checkpoint interval, in optimizer steps. 0 (the default) writes no periodic
- # checkpoints; a stop-and-save always writes one regardless, so the Resume action stays available.
+ # 0 writes no periodic checkpoints; a stop-and-save always writes one regardless, so the Resume
+ # action stays available.
save_steps: int = 0
# How many checkpoint-<N> bundles to keep in the output dir; 0 keeps every one.
save_total_limit: int = 2
- # Continue a previous run: the run's output_dir, or one of its checkpoint-<N> directories.
- # The start route resolves and validates it before the trainer spawns.
+ # The run's output_dir, or one of its checkpoint-<N> directories; the start route resolves and
+ # validates it before the trainer spawns.
resume_from_checkpoint: Optional[str] = None
- # Optional explicit family override; None = detect from base_model. ``resolved_family`` is filled by normalized() with the trainer family that will run.
+ # Optional explicit family override; None = detect from base_model. ``resolved_family`` is filled by
+ # normalized() with the trainer family that will run.
model_family: Optional[str] = None
resolved_family: str = "sdxl"
@@ -1043,8 +1002,8 @@ class DiffusionLoraConfig:
)
if self.resolution < 64 or self.resolution % 8 != 0:
raise ValueError("resolution must be a multiple of 8 and >= 64")
- # A video family's VAE compresses space by 32, so an off-grid resolution changes the
- # latent geometry silently. Refuse it here, before the GPU models are evicted.
+ # A video family's VAE compresses space by 32, so an off-grid resolution changes the latent geometry
+ # silently. Refuse it before the GPU models are evicted.
if (
resolved_family in TRAINABLE_VIDEO_FAMILIES
and self.resolution % _VIDEO_RESOLUTION_MULTIPLE != 0
@@ -1056,7 +1015,8 @@ class DiffusionLoraConfig:
)
if self.mixed_precision not in ("bf16", "fp16", "no"):
raise ValueError("mixed_precision must be one of bf16 / fp16 / no")
- # torch.manual_seed unpacks int64/uint64, so anything wider raises inside the trainer, after eviction. Catch it here.
+ # torch.manual_seed unpacks int64/uint64, so anything wider raises inside the trainer, after
+ # eviction. Catch it here.
if not -(2**63) <= int(self.seed) <= 2**64 - 1:
raise ValueError("seed must fit in torch's 64-bit range")
# Refuse fp16 for a bf16-only DiT family up front, before evicting resident models.
@@ -1070,8 +1030,8 @@ class DiffusionLoraConfig:
f"lr_scheduler must be one of {', '.join(sorted(_LR_SCHEDULERS))}; "
f"got {self.lr_scheduler!r}"
)
- # Do not rewrite the scheduler here: it is part of checkpoint identity, so doing so would
- # make legacy runs with ("constant", warmup > 0) impossible to resume.
+ # Do not rewrite the scheduler: it is part of checkpoint identity, so legacy runs with ("constant",
+ # warmup > 0) would become unresumable.
try:
lr_warmup_steps = int(self.lr_warmup_steps or 0)
except (TypeError, ValueError) as exc:
@@ -1082,7 +1042,6 @@ class DiffusionLoraConfig:
raise ValueError("lr_warmup_steps must be >= 0")
if not 1 <= int(self.cache_variants) <= 16:
raise ValueError("cache_variants must be between 1 and 16")
- # Checkpointing knobs. Rejected here, before the route evicts resident GPU models, rather than deep in the loop.
try:
save_steps = int(self.save_steps or 0)
save_total_limit = int(self.save_total_limit or 0)
@@ -1095,17 +1054,15 @@ class DiffusionLoraConfig:
raise ValueError("save_steps must be >= 0 (0 disables periodic checkpoints)")
if save_total_limit < 0:
raise ValueError("save_total_limit must be >= 0 (0 keeps every checkpoint)")
- # A blank resume path (the Unsloth default when the field is present but unset) means "fresh run", not the outputs root.
+ # A blank resume path (the Unsloth default when the field is present but unset) means "fresh run",
+ # not the outputs root.
resume_from_checkpoint = (
str(self.resume_from_checkpoint).strip()
if self.resume_from_checkpoint is not None
else ""
) or None
- # The H3 loop does not checkpoint: it neither writes a resume bundle nor restores one.
- # Accepting these two silently was the dangerous part -- a caller handing over a resume
- # bundle got a FRESH optimization that then overwrote the outputs it was meant to
- # continue, and one asking for periodic saves got none, both discovered only after an
- # expensive run. Refuse in validation, where it costs nothing, until the loop supports it.
+ # The H3 loop neither writes a resume bundle nor restores one, and accepting these silently gave a
+ # resume request a FRESH optimization that then overwrote the outputs it was meant to continue.
if resolved_family in CHECKPOINTLESS_FAMILIES:
if resume_from_checkpoint:
raise ValueError(
@@ -1122,7 +1079,8 @@ class DiffusionLoraConfig:
ema_decay = float(self.ema_decay or 0.0)
except (TypeError, ValueError) as exc:
raise ValueError(f"ema_decay must be a number, got {self.ema_decay!r}") from exc
- # decay = 1.0 would freeze the shadow at its init forever; the update is shadow * decay + param * (1 - decay), so valid decays live in [0, 1).
+ # decay = 1.0 would freeze the shadow at its init forever; the update is shadow * decay + param * (1
+ # - decay), so valid decays live in [0, 1).
if not 0.0 <= ema_decay < 1.0:
raise ValueError("ema_decay must be in [0, 1); 0 disables the EMA adapter")
# A blank cond_cache_dir (the Unsloth default when unset) means "off", not cwd.
@@ -1135,7 +1093,8 @@ class DiffusionLoraConfig:
base_precision = str(self.base_precision or "nf4").strip().lower()
if base_precision not in ("nf4", "bf16", "int8", "fp8", "mxfp8", "auto"):
raise ValueError("base_precision must be one of nf4 / bf16 / int8 / fp8 / mxfp8 / auto")
- # base_precision is a DiT-only lever, so the dense-mode gates apply only to the DiT families. The mode-name check above still runs for every family.
+ # base_precision is a DiT-only lever, so the dense-mode gates apply only to the DiT families. The
+ # mode-name check above still runs for every family.
if resolved_family != "sdxl" and base_precision in ("bf16", "int8", "fp8", "mxfp8"):
if repo_is_prequantized(self.base_model):
raise ValueError(
@@ -1148,12 +1107,10 @@ class DiffusionLoraConfig:
f"base_precision={base_precision!r} trains in bf16 compute; set "
f"mixed_precision to bf16."
)
- # Refuse a scheme this family's DiT is known to corrupt, and also one the training bar holds back while
- # inference allows it: qwen-image fp8 now renders inside the accuracy gate, but no one has measured whether a
- # LoRA converges against fp8-frozen linears, so it fails fast here rather than silently training on faith.
- # MiniMax-H3 runs all three modalities through one set of linears, so the
- # per-family activation range the fp8 module filter was measured against does not
- # describe it. Refuse the float8 modes rather than train against a clipped forward.
+ # qwen-image fp8 renders inside the accuracy gate, but no one has measured whether a LoRA converges
+ # against fp8-frozen linears, so training fails fast rather than training on faith.
+ # MiniMax-H3 runs all three modalities through one set of linears, so the per-family activation
+ # range the fp8 module filter was measured against does not describe it.
if resolved_family == "minimax-h3" and base_precision in ("fp8", "mxfp8"):
raise ValueError(
f"base_precision={base_precision!r} is not supported for minimax-h3: its "
@@ -1161,9 +1118,8 @@ class DiffusionLoraConfig:
f"so the activation range fp8 was measured against does not apply. Use "
f"'nf4', 'int8', 'bf16', or 'auto'."
)
- # _family_train_denied, not _family_denied: it is the strict superset (every inference
- # deny plus the training-only ones), so importing the narrower helper here would let a
- # scheme cleared only for rendering reach a trainer.
+ # _family_train_denied is the strict superset of _family_denied, so importing the narrower helper
+ # here would let a scheme cleared only for rendering reach a trainer.
from core.inference.diffusion_transformer_quant import _family_train_denied
if _family_train_denied(resolved_family, base_precision):
@@ -1171,7 +1127,8 @@ class DiffusionLoraConfig:
f"base_precision={base_precision!r} is not validated for training "
f"{resolved_family}. Use 'nf4', 'int8', 'bf16', or 'auto'."
)
- # flow_shift: None resolves to the family default ("auto" only for qwen-image, whose scheduler skips its static shift under use_dynamic_shifting); an explicit value is validated and kept.
+ # flow_shift: None resolves to the family default ("auto" only for qwen-image, whose scheduler skips its
+ # static shift under use_dynamic_shifting); an explicit value is validated and kept.
flow_shift = self.flow_shift
if flow_shift is None:
flow_shift = "auto" if resolved_family in AUTO_FLOW_SHIFT_FAMILIES else 1.0
@@ -1186,7 +1143,8 @@ class DiffusionLoraConfig:
) from exc
if not isinstance(flow_shift, str):
flow_shift = float(flow_shift)
- # isfinite as well as positive: JSON accepts 1e309, which floats to inf and would poison every sampled sigma while progress looks normal.
+ # isfinite as well as positive: JSON accepts 1e309, which floats to inf and would poison every
+ # sampled sigma while progress looks normal.
if not math.isfinite(flow_shift) or flow_shift <= 0:
raise ValueError(
"flow_shift must be a finite number > 0 (1.0 disables the shift), or 'auto'"
@@ -1200,7 +1158,8 @@ class DiffusionLoraConfig:
weighting_scheme = str(self.weighting_scheme or "none").strip().lower()
if weighting_scheme not in ("none", "bell"):
raise ValueError("weighting_scheme must be one of none / bell")
- # A zero/negative gamma would zero out (or invert) the min-SNR weight and silently train on a degenerate loss; None is the documented disable.
+ # A zero/negative gamma would zero out (or invert) the min-SNR weight and silently train on a
+ # degenerate loss; None is the documented disable.
if self.snr_gamma is not None and float(self.snr_gamma) <= 0:
raise ValueError("snr_gamma must be > 0, or null to disable min-SNR weighting")
# learning_rate can arrive as a string ("1e-4") from the Unsloth config path, so coerce it before AdamW sees it.
@@ -1212,7 +1171,8 @@ class DiffusionLoraConfig:
raise ValueError("learning_rate must be > 0")
alpha = self.lora_alpha if self.lora_alpha is not None else self.lora_rank
targets = tuple(self.lora_target_modules) or DEFAULT_LORA_TARGETS
- # A blank Hub token (the Unsloth default when none is configured) must load anonymously, not as an explicit empty credential.
+ # A blank Hub token (the Unsloth default when none is configured) must load anonymously, not as an
+ # explicit empty credential.
token = self.hf_token.strip() if isinstance(self.hf_token, str) else self.hf_token
from core.inference.diffusion_families import (
_is_local_path,
@@ -1225,27 +1185,13 @@ class DiffusionLoraConfig:
fetch_base_model = self.base_model
else:
fetch_base_model = prefer_ungated_mirror(self.base_model, token or None)
- # For a GATED upstream and no token, the cache preference has to be overridden: the
- # credentials this run lacks are the credentials the fetch needs, so a partial
- # snapshot cannot be completed and the start route's HEAD refuses even a complete
- # one. prefer_ungated_mirror's probe counts any cached weight as a hit, so a single
- # leftover shard from an interrupted or previously authorized download was enough to
- # keep the vendor id and hard-block the very case the mirrors exist for.
- #
- # Only gated, though. Most of the mirror table is ungated, mirrored to keep the
- # fetch inside unsloth/*, and there the upstream is reachable anonymously: an
- # override would discard a complete local cache and re-pull gigabytes, or fail
- # outright offline. Those keep prefer_ungated_mirror's cache-aware answer.
- #
- # UNSLOTH_DIFFUSION_NO_MIRROR still wins, exactly as it does inside
- # prefer_ungated_mirror: it is the documented way to pin the vendor repo, and an
- # override that ignored it would make that switch a lie on the training path only.
- #
- # A local clone wins over both. A base can be a directory named exactly like the
- # vendor id (the loaders resolve `black-forest-labs/FLUX.1-dev` on disk when it
- # exists), and prefer_ungated_mirror carves that out for the same reason: rewriting
- # it sends the fetch to the Hub past weights the user already has, so the run trains
- # on a different repo, or fails offline.
+ # For a GATED upstream with no token, override the cache preference: prefer_ungated_mirror's
+ # probe counts any cached weight as a hit, so one leftover shard kept the vendor id.
+ # Only gated: the rest of the mirror table is reachable anonymously, and an override there would
+ # discard a complete local cache and re-pull gigabytes, or fail offline.
+ # UNSLOTH_DIFFUSION_NO_MIRROR still wins, exactly as it does inside prefer_ungated_mirror.
+ # A local clone wins over both: a base can be a directory named exactly like the vendor id, and
+ # rewriting it sends the fetch to the Hub past weights the user already has.
if (
not token
and upstream_is_gated(self.base_model)
@@ -1253,10 +1199,9 @@ class DiffusionLoraConfig:
and not os.environ.get("UNSLOTH_DIFFUSION_NO_MIRROR", "").strip()
):
fetch_base_model = mirror_repo(self.base_model) or fetch_base_model
- # A blank caption_column means the default, as the start route's own preflight already
- # assumes ("or 'text'"). Without this the route and the trainer resolve different captions
- # from a metadata.jsonl, so their dataset fingerprints disagree and a resume that the
- # route accepted is then refused in the child -- after the GPU residents are freed.
+ # A blank caption_column means the default, as the start route's preflight assumes: otherwise
+ # route and trainer resolve different captions from a metadata.jsonl and their fingerprints
+ # disagree, so an accepted resume is refused in the child.
caption_column = str(self.caption_column or "").strip() or "text"
return replace(
self,
@@ -1324,7 +1269,8 @@ class PermutationBatchSampler:
self._pos = 0
def next_batch(self, k: int) -> list[int]:
- # k may exceed n (batch larger than the dataset): the permutation refills across cycles so the caller always gets exactly k indices.
+ # k may exceed n (batch larger than the dataset): the permutation refills across cycles so the
+ # caller always gets exactly k indices.
out: list[int] = []
while len(out) < k:
if self._pos >= len(self._order):
@@ -1359,12 +1305,8 @@ class PermutationBatchSampler:
restored = [int(i) for i in order]
except (TypeError, ValueError):
return False
- # A FULL permutation, or the initial empty state. A shortened or duplicate-carrying
- # order is in range and the same length as nothing in particular: the cycle then
- # reshuffles early or serves the same image twice, while the RNG has already been
- # restored to a point after the original draw. That is the silent reorder this
- # boolean exists to prevent, arriving through a truncated or hand-edited manifest
- # instead of a missing one.
+ # A shortened or duplicate-carrying order is in range and plausible, so the cycle reshuffles
+ # early or serves the same image twice with the RNG already past the original draw.
if restored and sorted(restored) != list(range(self._n)):
return False
self._order = restored
@@ -1372,10 +1314,8 @@ class PermutationBatchSampler:
pos = int(state.get("pos") or 0)
except (TypeError, ValueError):
return False
- # A position outside 0..len(order) is not a rounding error, it is a damaged manifest,
- # and clamping it re-serves the permutation from the top or ends the cycle early --
- # behind an RNG already restored past the draw. The same silent repeat-or-skip the
- # order check above refuses, so refuse it rather than quietly correcting it.
+ # A position outside 0..len(order) is a damaged manifest, and clamping it re-serves the permutation
+ # from the top or ends the cycle early behind an already-restored RNG.
if not 0 <= pos <= len(self._order):
return False
self._pos = pos
@@ -1415,8 +1355,7 @@ def discover_image_caption_pairs(
if not root.is_dir():
raise FileNotFoundError(f"data_dir is not a directory: {data_dir}")
- # The caption lookup resolves to the caption's own companion, which exists, so the pair
- # reads as a real one.
+ # The caption lookup resolves to the caption's own companion, which exists, so the pair reads as a real one.
images = drop_appledouble_metadata(
sorted(p for p in root.iterdir() if p.is_file() and p.suffix.lower() in _IMAGE_EXTS)
)
@@ -1427,7 +1366,8 @@ def discover_image_caption_pairs(
meta_path = root / meta_name
if not meta_path.is_file():
continue
- # Tolerate a bad upload (invalid UTF-8, or non-object JSON): skip the record so the instance_prompt fallback still applies.
+ # Tolerate a bad upload (invalid UTF-8, or non-object JSON): skip the record so the instance_prompt
+ # fallback still applies.
try:
meta_lines = meta_path.read_text(encoding = "utf-8").splitlines()
except (OSError, UnicodeError):
@@ -1452,7 +1392,8 @@ def discover_image_caption_pairs(
for img in images:
caption: Optional[str] = None
sidecar_present = False
- # 1. per-image sidecar caption (the user's explicit edit; wins over metadata). An EMPTY sidecar is a deliberate tombstone: it suppresses the metadata caption and leaves the image uncaptioned.
+ # An EMPTY sidecar is a deliberate tombstone: it suppresses the metadata caption and leaves the
+ # image uncaptioned.
for ext in _CAPTION_EXTS:
sidecar = img.with_suffix(ext)
if sidecar.is_file():
@@ -1460,10 +1401,12 @@ def discover_image_caption_pairs(
try:
caption = sidecar.read_text(encoding = "utf-8").strip()
except (OSError, UnicodeError):
- # An unreadable sidecar reads as the empty tombstone, so the instance_prompt fallback applies instead of a 500.
+ # An unreadable sidecar reads as the empty tombstone, so the instance_prompt fallback applies
+ # instead of a 500.
caption = ""
break
- # 2. metadata row keyed by file name (basename or relative path, as_posix so Windows paths match). A sidecar, even empty, wins.
+ # 2. metadata row keyed by file name (basename or relative path, as_posix so Windows paths
+ # match). A sidecar, even empty, wins.
if not sidecar_present:
caption = meta_caption.get(img.name) or meta_caption.get(
img.relative_to(root).as_posix()
@@ -1473,7 +1416,11 @@ def discover_image_caption_pairs(
caption = instance_prompt
if caption:
if verify_images:
- # Reject a corrupt/truncated image now via a cheap PIL header probe: otherwise it passes filename-only discovery, the start route frees the GPU models, and the trainer crashes in Image.open.
+ # Reject a corrupt or truncated image now: otherwise it passes filename-only discovery, the start
+ # route frees the GPU models, and the trainer crashes in Image.open.
+ # Epoch-mode payloads carry max_steps: 0 as the "use epochs" sentinel, which the alias copies as
+ # train_steps: 0. normalized() rejects train_steps < 1 before resolve_train_steps() applies
+ # num_epochs, so drop a falsy value here.
try:
from PIL import Image
with Image.open(img) as _probe:
@@ -1493,22 +1440,18 @@ def discover_image_caption_pairs(
return pairs
-# Families whose trainer has no checkpoint/resume support yet. The shared DiffusionLoraConfig
-# carries save_steps / resume_from_checkpoint for every family, so a loop that implements
-# neither has to say so rather than ignore them.
+# The shared DiffusionLoraConfig carries save_steps / resume_from_checkpoint for every family, so a
+# loop that implements neither has to say so rather than ignore them.
CHECKPOINTLESS_FAMILIES: frozenset[str] = frozenset({"minimax-h3"})
-# Families whose forward covers ONE packed sequence, so the batch axis is a pure replication
-# axis and a second clip cannot join it: the layout, the rotary grid and the row timesteps are
-# set by that clip's geometry and its caption's length. Kept beside the refusal it explains.
+# The batch axis is a pure replication axis for these: the layout, the rotary grid and the row
+# timesteps are set by one clip's geometry and its caption's length.
SINGLE_SEQUENCE_FAMILIES: frozenset[str] = frozenset({"minimax-h3"})
-# Families whose trainer loads its base through ``ModularPipeline.from_pretrained``. Their local
-# layout is ``modular_model_index.json`` and no ``model_index.json``, so the conventional shape
-# check in ``_assert_trusted_base_model`` refuses the only local form the family HAS. The trainer
-# knew this and passed allow_modular itself; the START ROUTE runs the same gate first, and
-# without the same answer a local H3 pipeline 400'd there before the trainer was ever reached.
-# One set, read by both, so the two cannot drift into disagreeing about the same directory.
+# Families whose trainer loads its base through ModularPipeline.from_pretrained. Their local
+# layout is modular_model_index.json and no model_index.json, so the conventional shape check
+# refuses the only local form the family HAS. One allow_modular set, read by the trainer and by
+# the START ROUTE, so the two cannot disagree about the same directory.
MODULAR_BASE_FAMILIES: frozenset[str] = frozenset({"minimax-h3"})
# MiniMax-H3 canvas multiple: a 16x VAE compression and a 2x patch.
@@ -1549,29 +1492,23 @@ def h3_train_unsupported_reason(cfg: Any) -> Optional[str]:
"audio at different sigmas in the same step, so a single weight over 'the' "
"timestep is ambiguous. Use weighting_scheme='none'."
)
- # The batch axis of an H3 forward is a pure replication axis: the layout, the rotary grid
- # and the row timesteps describe ONE packed sequence that every batch item shares. Two
- # clips with different captions have different text lengths and therefore different
- # layouts, so a batch > 1 cannot be formed without padding the model has no mask for.
+ # Two clips with different captions have different text lengths and therefore different layouts, so
+ # a batch > 1 cannot be formed without padding the model has no mask for.
if cfg.train_batch_size != 1:
return (
"MiniMax-H3 trains at batch size 1: one forward covers one packed sequence, whose "
"row layout is set by the clip's own geometry and its caption's length. Use "
"gradient_accumulation_steps to raise the effective batch."
)
- # torch.compile is never invoked here (the packed layout changes shape with every clip's
- # caption length, so each step would recompile), and the run already reports compiled=False.
- # "auto" means "the trainer decides", which it does; an explicit "on" is a policy that would
- # be accepted and then ignored.
+ # torch.compile is never invoked here (the packed layout changes shape with every caption length),
+ # so an explicit "on" would be accepted and then ignored.
if str(getattr(cfg, "compile_transformer", "auto") or "auto").strip().lower() == "on":
return (
"MiniMax-H3 does not compile: its packed sequence changes length with every clip's "
"caption, so torch.compile would re-trace each step. Use compile_transformer "
"'off' or 'auto'."
)
- # No conditioning cache exists on this path: the conditioner is loaded and every caption and
- # latent recomputed each run. Accepting the directory would promise a saving that never
- # happens, so an explicit one is refused rather than silently unused.
+ # No conditioning cache exists on this path, so accepting the directory would promise a saving that never happens.
if str(getattr(cfg, "cond_cache_dir", "") or "").strip():
return (
"MiniMax-H3 has no persistent conditioning cache yet: each run loads the "
@@ -1581,17 +1518,9 @@ def h3_train_unsupported_reason(cfg: Any) -> Optional[str]:
return None
-# The SCHEMA defaults the MiniMax-H3 loop cannot honour, and the value it actually runs with.
-# Unlike the fields ``h3_train_unsupported_reason`` refuses, these have a DEFAULT that the loop
-# disagrees with, so refusing them would 422 every untouched request; they are normalised
-# instead. center_crop/random_flip: every frame goes through the same centre cover-crop and
-# nothing is flipped (a per-frame flip tears a clip, and a per-clip one has nowhere to live --
-# the cached tensors carry no variant axis). snr_gamma: the step is a plain unweighted MSE.
-# cache_latents/cache_variants: the loop encodes every clip once, up front, into exactly one
-# cached (video_a, video_b, audio) tuple and then frees the VAEs, with no per-step encode path
-# to fall back to and nowhere to put a second draw. It never reads either field, so a request
-# asking for no cache, or for the schema's four variants, got one cache of one variant anyway
-# and a run record claiming otherwise.
+# These have a DEFAULT the H3 loop disagrees with, so they are normalised rather than refused
+# (which would 422 every untouched request): one centre cover-crop, nothing flipped, a plain
+# unweighted MSE, and exactly one cached tuple.
_H3_FIXED_RECIPE: dict[str, Any] = {
"center_crop": True,
"random_flip": False,
@@ -1616,8 +1545,7 @@ def train_recipe_overrides(cfg: Any) -> dict[str, Any]:
return dict(_H3_FIXED_RECIPE)
-# Families whose dataset is captioned video CLIPS rather than stills. LTX-2 is deliberately not
-# here: it trains a style LoRA FROM still images, so it keeps the image discovery.
+# LTX-2 is deliberately not here: it trains a style LoRA FROM still images, so it keeps the image discovery.
CLIP_TRAINED_FAMILIES: frozenset[str] = frozenset({"minimax-h3"})
@@ -1684,11 +1612,12 @@ def _plan_cache_variants(
return plan
-# Host-memory budget for the AUTOMATIC latent cache. It holds two fp32 posterior tensors per crop/flip variant per image,
-# so a few thousand images x cache_variants can exhaust pinned RAM; over budget it falls back to per-step VAE encoding.
+# Two fp32 posterior tensors per crop/flip variant per image, so a few thousand images can exhaust
+# pinned RAM; over budget it falls back to per-step VAE encoding.
_LATENT_CACHE_BUDGET_BYTES = 4 * 1024**3 # 4 GiB
-# Returned by the cache builders when the estimate exceeds budget: the caller keeps the VAE resident. Distinct from ``None`` (a stop requested mid-build).
+# Returned by the cache builders when the estimate exceeds budget: the caller keeps the VAE
+# resident. Distinct from ``None`` (a stop requested mid-build).
LATENT_CACHE_OVER_BUDGET: Any = object()
@@ -1741,13 +1670,15 @@ def _apply_perf_flags(
torch.backends.cudnn.allow_tf32 = True
torch.set_float32_matmul_precision("high")
else:
- # The opt-out is a strict-fp32 A/B mode, so actively clear the flags rather than inherit ambient state (cudnn TF32 defaults ON).
+ # The opt-out is a strict-fp32 A/B mode, so actively clear the flags rather than inherit ambient
+ # state (cudnn TF32 defaults ON).
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.set_float32_matmul_precision("highest")
if cudnn_benchmark:
torch.backends.cudnn.benchmark = True
- # The cuDNN SDPA TRAINING graph is broken for the FLUX attention shapes on torch 2.10 + cu130 (B200) and poisons the context, so pin flash / mem-efficient SDPA for the run (restored on exit).
+ # The cuDNN SDPA TRAINING graph is broken for the FLUX attention shapes on torch 2.10 + cu130 (B200)
+ # and poisons the context, so pin flash / mem-efficient SDPA for the run (restored on exit).
cuda_backends = getattr(torch.backends, "cuda", None)
if cuda_backends is not None and hasattr(cuda_backends, "enable_cudnn_sdp"):
try:
@@ -1774,7 +1705,8 @@ def _restore_perf_flags(snap: Optional[dict]) -> None:
if snap.get("matmul_precision"):
torch.set_float32_matmul_precision(snap["matmul_precision"])
- # Restore the exact pre-run cudnn SDPA state; None means the flag was unreadable at apply time and never touched.
+ # Restore the exact pre-run cudnn SDPA state; None means the flag was unreadable at apply time and
+ # never touched.
cuda_backends = getattr(torch.backends, "cuda", None)
if (
snap.get("cudnn_sdp") is not None
@@ -1786,28 +1718,25 @@ def _restore_perf_flags(snap: Optional[dict]) -> None:
pass
-# Official safetensors-only TRAINING bases trusted in addition to the inference allowlist. The FLUX.2 bases are in that
-# list too but kept here so the training gate stays independent. Exact-match lowercased; never add pickles or remote code.
+# Kept here so the training gate stays independent of the inference allowlist. Exact-match
+# lowercased; never add pickles or remote code.
_TRAIN_EXTRA_TRUSTED_REPOS = frozenset(
{
"black-forest-labs/flux.2-dev",
"black-forest-labs/flux.2-klein-4b",
"black-forest-labs/flux.2-klein-base-4b",
"black-forest-labs/flux.2-klein-base-9b",
- # LTX-2's official base. It is a video family, so the image-side inference allowlist
- # (_is_trusted_diffusion_repo) never covered it; safetensors-only, no remote code.
+ # A video family, so the image-side inference allowlist never covered it; safetensors-only, no remote code.
"lightricks/ltx-2",
# MiniMax-H3's official base, for the same reason: safetensors-only, no remote code.
"minimaxai/minimax-h3",
}
)
-# LTX-2.3 repos hold SINGLE-FILE checkpoints (ltx-2.3-22b-*.safetensors) and no diffusers layout:
-# no model_index.json, no transformer/ or scheduler/ subfolder. Inference assembles them with
-# from_single_file plus 2.3 config overrides and components borrowed from the 2.0 base (see
-# core/inference/video_ltx2.py); the trainer only knows LTX2Pipeline.from_pretrained, which cannot
-# read that layout. The name still resolves to the ``ltx-2`` family, so without an explicit refusal
-# the run passes preflight, evicts the user's resident models, and only then fails in the child.
+# LTX-2.3 repos hold SINGLE-FILE checkpoints and no diffusers layout; inference assembles them
+# with from_single_file plus 2.3 config overrides (see core/inference/video_ltx2.py), while the
+# trainer only knows LTX2Pipeline.from_pretrained. The name still resolves to the ltx-2 family, so
+# without an explicit refusal the run evicts residents and only then fails in the child.
_LTX23_TRAIN_UNSUPPORTED = ("lightricks/ltx-2.3", "lightricks/ltx-2.3-fp8")
@@ -1843,13 +1772,13 @@ def _assert_trusted_base_model(base_model: str, *, allow_modular: bool = False)
f"Refusing to train from untrusted base model '{base_model}'. Use a local path or "
f"a trusted repo (an unsloth/* repo or an official base)."
)
- # An existing LOCAL base is loaded as a full pipeline, which needs an index; reject a non-pipeline local dir before /diffusion/start frees the GPU models.
+ # An existing LOCAL base is loaded as a full pipeline, which needs an index; reject a non-pipeline
+ # local dir before /diffusion/start frees the GPU models.
_assert_local_base_is_pipeline(base_model, allow_modular = allow_modular)
+# One writer and one reader for BOTH trainers, so an SDXL and a DiT run resume from the same bundle shape.
# ── resume checkpoints ────────────────────────────────────────────────────────
-# One writer and one reader for BOTH trainers, so an SDXL and a DiT run resume from the same
-# bundle shape. The family-specific part (the deployable adapter export) stays in the trainers.
def trainable_state_dict(model: Any) -> dict[str, Any]:
"""The trainable (LoRA) parameters of ``model``, keyed by parameter name.
@@ -1892,11 +1821,8 @@ def load_trainable_state_dict(model: Any, state: Optional[dict[str, Any]]) -> in
)
p.copy_(saved.to(device = p.device, dtype = p.dtype))
restored += 1
- # BOTH directions. Counting only the checkpoint's own tensors proves every saved tensor
- # landed somewhere; it says nothing about a live trainable parameter the checkpoint never
- # had. A truncated or hand-edited adapter file holding a strict SUBSET therefore passed,
- # and the full optimizer state was then loaded on top: restored Adam moments driving
- # freshly initialised weights, while the run reported a clean resume.
+ # BOTH directions: counting only the checkpoint's own tensors let a truncated adapter holding a
+ # strict SUBSET pass, and the optimizer state then loaded on top of freshly initialised weights.
unsaved = sorted(trainable - set(state))
unknown = sorted(set(state) - trainable)
if unsaved or unknown:
@@ -1967,17 +1893,15 @@ def write_resume_checkpoint(
progress = _json_safe_progress(progress),
save_total_limit = int(cfg.save_total_limit or 0),
discard_existing = discard_existing,
- # What this run resumed from, so the "step already written" shortcut can tell
- # "we are re-saving the bundle we started from" apart from "another run left a
- # bundle at this number".
+ # Which bundle THIS is, so the "step already written" shortcut can tell a re-save of our own source
+ # apart from another run's bundle at the same number.
source_checkpoint = getattr(cfg, "resume_from_checkpoint", None),
- # Bundles that predated this run are never pruned to make room for its own: a
- # branched resume beside higher-numbered checkpoints used to delete them on the
- # first save, irreversibly.
+ # Bundles that predated this run are never pruned to make room for its own: a branched resume used
+ # to delete them irreversibly on the first save.
preexisting = preexisting,
)
- # Reported per save, not only at the end: a run that later CRASHES must still be known to
- # have resumable state (and a run whose write failed must still be known to be blocked).
+ # Reported per save, so a run that later crashes is still known to have resumable state and one
+ # whose write failed is still known to be blocked.
_emit(on_event, "checkpoint_saved", checkpoint_path = path, step = step)
return path, None
except Exception as exc: # noqa: BLE001 -- reported, never fatal to the run
@@ -2055,19 +1979,15 @@ def restore_resume_state(
load_trainable_state_dict(model, ckpt.tensors("adapter"))
optimizer_state = ckpt.torch_state("optimizer")
if optimizer_state is not None:
- # The trainers pick their optimizer from the HOST (bitsandbytes present, a fused kernel
- # available, UNSLOTH_DIFFUSION_FP32_OPTIM), not from the config, so a checkpoint can
- # legitimately arrive with foreign moments: AdamW8bit stores "state1"/"state2", torch
- # AdamW stores "exp_avg"/"exp_avg_sq". Shapes and counts match, so load_state_dict
- # accepts them and the first step dies on a bare KeyError. Refuse with a real reason.
+ # The trainers pick their optimizer from the HOST (bnb present, fused kernel available,
+ # UNSLOTH_DIFFUSION_FP32_OPTIM), so foreign moments arrive legitimately (state1/state2 versus
+ # exp_avg/exp_avg_sq): shapes match, load_state_dict accepts them, and the first step dies on a
+ # bare KeyError.
saved_optimizer = ckpt.optimizer_class
live_optimizer = optimizer_key(optimizer)
if not saved_optimizer:
- # This writer records the class whenever it writes moments, so an optimizer file
- # with no class beside it is a hand-edited or half-written bundle -- and letting it
- # through is the same failure the check below exists for: foreign moments load
- # cleanly (shapes and counts match) and die on the first step, in the child, after
- # the route preflight has already evicted the resident models.
+ # An optimizer file with no class beside it is a hand-edited or half-written bundle, and foreign
+ # moments load cleanly then die on the first step, after the preflight evicted the residents.
raise ResumeError(
"This checkpoint does not record which optimizer wrote its state, so its "
"moments cannot be safely restored. Resume from an earlier checkpoint, or "
@@ -2079,11 +1999,9 @@ def restore_resume_state(
f"machine builds {live_optimizer}. Install the same optimizer backend (or unset "
f"UNSLOTH_DIFFUSION_FP32_OPTIM) to continue this run."
)
- # Optimizer state is keyed by parameter POSITION, so load_state_dict rebinds the saved
- # moments onto whatever order this process built. The adapter tensors above were restored
- # by NAME, so a PEFT/diffusers upgrade that changes traversal order while keeping the same
- # names leaves the two disagreeing: many LoRA projections share a shape, so every moment
- # loads cleanly onto the wrong tensor and the continued trajectory is silently corrupt.
+ # Optimizer state is keyed by parameter POSITION while the adapter was restored by NAME, so a
+ # PEFT/diffusers upgrade that changes traversal order loads every moment cleanly onto a
+ # same-shaped wrong tensor.
saved_names = ckpt.optimizer_param_names
live_names = list(trainable_state_dict(model))
if saved_names is not None and saved_names != live_names:
@@ -2092,15 +2010,12 @@ def restore_resume_state(
"than this build produces, so its moments cannot be matched to this run's "
"tensors. Start a new run, or resume on the version that wrote it."
)
- # load_state_dict replaces the param groups too, so the checkpoint's learning rate wins
- # over a changed cfg.learning_rate -- the same semantics as HF Trainer's resume, and the
- # UI only ever replays the original run's config anyway.
+ # load_state_dict replaces the param groups too, so the checkpoint's learning rate wins over a
+ # changed cfg, the same semantics as HF Trainer's resume.
optimizer.load_state_dict(optimizer_state)
elif optimizer is not None:
- # The bundle lists no optimizer at all. Every bundle this writer produces for a real run
- # has one, so this is a hand-edited or half-written directory -- and continuing would
- # restart Adam's moments from zero at step N while reporting a clean resume, which looks
- # like a resume and trains like a fresh run with a warm LR.
+ # Every bundle this writer produces has an optimizer, so continuing without one restarts Adam's
+ # moments from zero at step N while reporting a clean resume.
raise ResumeError(
"This checkpoint carries no optimizer state, so the run cannot be continued from "
"it: the moments would restart from zero at the resumed step. Start a new run."
@@ -2110,18 +2025,14 @@ def restore_resume_state(
lr_scheduler.load_state_dict(scheduler_state)
_reapply_lr_schedule(optimizer, lr_scheduler)
elif lr_scheduler is not None:
- # Same for the schedule: a fresh LambdaLR at step 0 would re-warm the learning rate the
- # restored optimizer already moved past.
+ # A fresh LambdaLR at step 0 would re-warm the learning rate the restored optimizer already moved past.
raise ResumeError(
"This checkpoint carries no learning-rate scheduler state, so the schedule would "
"restart from step 0. Start a new run."
)
if sampler is not None and not sampler.load_state_dict(ckpt.sampler_state):
- # Every image checkpoint this format writes carries sampler state, so a bundle whose
- # state is missing or malformed is not one this code wrote. Continuing would leave a
- # fresh sampler behind an RNG restored to a point AFTER the saved permutation was
- # drawn, so the next batch is a different order: images silently skipped or repeated,
- # under a resume that reported success.
+ # Every image checkpoint carries sampler state, so a missing or malformed one leaves a fresh sampler
+ # behind an RNG restored past the saved permutation: images silently skipped or repeated.
raise ResumeError(
"This checkpoint's dataset sampler state is missing or unreadable, so the image "
"order cannot be continued. Start a new run."
@@ -2131,11 +2042,8 @@ def restore_resume_state(
if ema_state:
missing = ema.missing_from(ema_state)
if missing:
- # A readable but INCOMPLETE shadow set. load_state_dict keeps the freshly
- # initialised shadow for anything it cannot match, so continuing here would
- # blend restored EMA weights with initialisation noise for the rest -- and
- # report a clean resume while doing it. There is no honest way to continue
- # this run's EMA, so say so instead.
+ # load_state_dict keeps the freshly initialised shadow for anything it cannot match, so an
+ # incomplete set blends restored EMA weights with initialisation noise under a clean resume.
named = ", ".join(missing[:3]) + ("..." if len(missing) > 3 else "")
raise ResumeError(
f"This checkpoint's EMA state is missing or mis-shaped for {len(missing)} "
@@ -2144,11 +2052,8 @@ def restore_resume_state(
)
ema.load_state_dict(ema_state, updates = ckpt.ema_updates)
else:
- # EMA is not part of the validated identity, so a resume may turn it on for a run
- # that had it off. The EMA object is built BEFORE the adapter is restored, so its
- # shadow holds freshly initialised LoRA weights and there is nothing saved to
- # replace them with -- every later update, and the exported EMA adapter, would
- # blend the restored weights with that initialisation. Start from what was restored.
+ # EMA is not part of the validated identity, so a resume may turn it on: the object is built
+ # BEFORE the adapter is restored, so its shadow holds freshly initialised weights.
ema.reseed_from(model)
restore_rng_state(ckpt.rng_json, ckpt.torch_state("rng"), rng_streams)
_emit(
@@ -2157,9 +2062,8 @@ def restore_resume_state(
checkpoint_path = str(path),
step = step,
total_steps = cfg.train_steps,
- # Which bundle THIS is, not just where it sat. Another run can write its own
- # checkpoint-<N> over the same slot, and the pathname alone would then offer that
- # replacement back as this run's lineage.
+ # Which bundle THIS is, not just where it sat: another run can write its own checkpoint over the
+ # same slot, and the pathname alone would offer that replacement back as this run's lineage.
source_created_at = ckpt.manifest.get("created_at"),
)
return ckpt
@@ -2243,11 +2147,13 @@ def _write_lora_sidecar(
sidecar_path.write_text(json.dumps(meta, indent = 2), encoding = "utf-8")
-# Aliases from the generic Unsloth training payload onto DiffusionLoraConfig fields, so the shared request shape can also drive this trainer.
+# Aliases from the generic Unsloth training payload onto DiffusionLoraConfig fields, so the shared
+# request shape can also drive this trainer.
_CONFIG_ALIASES = {
"model_name": "base_model",
"max_steps": "train_steps",
- # num_epochs already matches the diffusion field name, but list it so the epochs override is threaded through explicitly.
+ # num_epochs already matches the diffusion field name, but list it so the epochs override is
+ # threaded through explicitly.
"num_epochs": "num_epochs",
"batch_size": "train_batch_size",
"lora_r": "lora_rank",
@@ -2287,8 +2193,8 @@ def _config_from_dict(config: dict) -> DiffusionLoraConfig:
for k, v in config.items():
if k in valid:
kwargs[k] = v
- # Epoch-mode payloads carry max_steps: 0 as the "use epochs" sentinel, which the alias copies as train_steps: 0.
- # normalized() rejects train_steps < 1 before resolve_train_steps() applies num_epochs, so drop a falsy value here.
+ # Epoch-mode payloads carry max_steps: 0 as the "use epochs" sentinel, and normalized() rejects
+ # train_steps < 1 before resolve_train_steps() applies num_epochs.
try:
_num_epochs = int(kwargs.get("num_epochs") or 0)
except (TypeError, ValueError):
diff --git a/studio/backend/core/training/diffusion_train_extras.py b/studio/backend/core/training/diffusion_train_extras.py
index be1ab7a24c8..89f6ab1aaff 100644
--- a/studio/backend/core/training/diffusion_train_extras.py
+++ b/studio/backend/core/training/diffusion_train_extras.py
@@ -40,10 +40,11 @@ import re
from pathlib import Path
from typing import Any, Iterable, Optional
+
+# Effective decay is min(decay, (1 + updates) / (WARMUP_OFFSET + updates)); at offset 10 step 1
+# averages aggressively (~0.18) and the ramp reaches 0.99 after ~1000 updates.
# ── LoRA EMA ──────────────────────────────────────────────────────────────────
-# Warmup horizon for the EMA decay ramp: effective decay is min(decay, (1 + updates) / (WARMUP_OFFSET + updates)).
-# With the offset at 10, step 1 averages aggressively (~0.18) and the ramp reaches 0.99 after ~1000 updates.
_EMA_WARMUP_OFFSET = 10.0
@@ -234,7 +235,7 @@ def _hub_cache_roots() -> list[str]:
longer changed this key and a warm run silently reused the old embeddings and latents. The
constant stays as a fallback, since the trainer subprocess may run without Unsloth's settings
module importable."""
- import os # noqa: PLC0415 — keep the module import list light for the subprocess
+ import os # noqa: PLC0415 - keep the module import list light for the subprocess
roots: list[str] = []
try:
@@ -256,12 +257,9 @@ def _hub_cache_roots() -> list[str]:
return roots
-# Pipeline subdirectories whose weights decide what the conditioning cache holds, so an in-place
-# edit of any of them must change the fingerprint. text_encoder*/tokenizer* produce the embeddings
-# and vae* the latents, for every family. "connectors" is LTX-2's: the Gemma3 hidden states are
-# per-layer stacked and only the connector output reaches the transformer, so that projection --
-# not the raw encoder state -- is what gets cached. No other supported family runs a module between
-# encode_prompt and the DiT; the rest go straight from the text encoders to the cached tensors.
+# An in-place edit of any of these must change the fingerprint: text_encoder*/tokenizer* produce
+# the embeddings and vae* the latents. "connectors" is LTX-2's, the only family running a module
+# between encode_prompt and the DiT: its connector output, not the raw Gemma3 state, is cached.
_CACHE_SOURCE_SUBDIRS = ("text_encoder", "tokenizer", "vae", "connectors")
@@ -275,7 +273,7 @@ def source_revision(ref: Any) -> str:
never touches the encoders, since the point of the cache is that a warm run does
not load them.
"""
- import os # noqa: PLC0415 — keep the module import list light for the subprocess
+ import os # noqa: PLC0415 - keep the module import list light for the subprocess
try:
name = str(ref or "").strip()
if not name:
@@ -313,7 +311,7 @@ def source_revision(ref: Any) -> str:
if len(names) == 1:
return f"rev-{names[0][:16]}"
return "unresolved"
- except Exception: # noqa: BLE001 — best-effort, never block a run
+ except Exception: # noqa: BLE001 - best-effort, never block a run
return "unresolved"
@@ -332,7 +330,6 @@ class PersistentConditioningCache:
self.resolution = int(resolution)
self.root.mkdir(parents = True, exist_ok = True)
- # -- keys --
def latent_key(
self,
image_path: str,
@@ -355,7 +352,6 @@ class PersistentConditioningCache:
def has(self, key: str) -> bool:
return self.path_for(key).is_file()
- # -- IO --
def put(self, key: str, tensors: Iterable[Any]) -> None:
"""Store an ordered tuple of tensors (None entries allowed: their slot
indices are recorded in the metadata so ``get`` restores them)."""
@@ -397,12 +393,14 @@ class PersistentConditioningCache:
return None
+# Buckets snap to 64 pixels: the DiT families divide by 8 in the VAE and 2 again in latent patching,
+# and regional torch.compile prefers few distinct shapes.
# ── aspect-ratio bucketing ────────────────────────────────────────────────────
-# Pixel-dimension divisor for bucket shapes: the DiT families divide by 8 in the VAE and 2 again in latent patching, and regional torch.compile prefers few distinct shapes, so buckets snap to 64 pixels.
BUCKET_DIVISOR = 64
-# Widest aspect ratio a bucket may take; anything more extreme clamps to it (matching the common practice of capping panoramas).
+# Widest aspect ratio a bucket may take; anything more extreme clamps to it (matching the common
+# practice of capping panoramas).
MAX_BUCKET_RATIO = 2.0
diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.py
index 2202a1d696a..afdef557d60 100644
--- a/studio/backend/core/training/diffusion_training_service.py
+++ b/studio/backend/core/training/diffusion_training_service.py
@@ -52,8 +52,8 @@ def _finite_or_none(value: Any) -> Optional[float]:
def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) -> None:
- # Fresh spawned interpreter: re-apply the OS-trust-store injection, inside
- # the secret scrub and before the trainer imports diffusers.
+ # Fresh spawned interpreter: re-apply the OS-trust-store injection, inside the secret scrub and
+ # before the trainer imports diffusers.
from utils.native_tls import activate_native_tls
activate_native_tls()
@@ -61,9 +61,8 @@ def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) ->
# Imported lazily so this module (and the route layer) stays torch-free at import.
from .diffusion_lora_trainer import run_diffusion_training_process
- # This child never runs LogConfig.setup_logging, so it installs the Hub default
- # here; the diffusers half is done inside the two trainer entrypoints, which are
- # the first points where diffusers is actually imported.
+ # This child never runs LogConfig.setup_logging, so it installs the Hub default here; the diffusers
+ # half is done inside the trainer entrypoints.
try:
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
@@ -74,14 +73,16 @@ def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) ->
def _default_target(*, event_queue: Any, stop_queue: Any, config: dict) -> None:
- # First thing in the child (before torch): self-bind to parent death and scrub the native path secret, like the other workers.
+ # First thing in the child (before torch): self-bind to parent death and scrub the native path
+ # secret, like the other workers.
from utils.native_path_leases import run_without_native_path_secret
run_without_native_path_secret(
_run_diffusion_child, event_queue = event_queue, stop_queue = stop_queue, config = config
)
-# Cap on retained metric points; over it, arrays are decimated so a long run stays bounded while the chart keeps its shape.
+# Cap on retained metric points; over it, arrays are decimated so a long run stays bounded while the
+# chart keeps its shape.
_METRIC_CAP = 4000
@@ -103,8 +104,8 @@ def _llm_training_active() -> bool:
return False
+# One JSON file per terminal run, not the LLM sqlite, so diffusion runs stay off the LLM Runs page.
# ── persisted run history ──────────────────────────────────────────────────────
-# Every terminal run is recorded as one JSON file (summary + scrubbed config + metric logs) for the Train tab's history. JSON, not the LLM sqlite, so diffusion runs stay off the LLM Runs page.
def _runs_dir() -> Path:
from utils.paths.storage_roots import studio_root
@@ -159,9 +160,8 @@ def _resume_fields(
return {
"can_resume": can_resume,
"checkpoint_step": state.get("checkpoint_step"),
- # The EXACT bundle this run would continue, so the client resumes the one it was shown
- # rather than "whatever is newest in that folder" (which, in a folder two runs share,
- # can be a different run's).
+ # The EXACT bundle this run would continue, so the client resumes the one it was shown rather than
+ # whatever is newest in a folder two runs share.
"checkpoint_path": state.get("checkpoint_path") if can_resume else None,
"resume_blocked_reason": state.get("resume_blocked_reason"),
}
@@ -185,21 +185,18 @@ def _refresh_resume_state(rec: dict) -> dict:
status = rec.get("status"),
started_at = rec.get("started_at"),
ended_at = rec.get("ended_at"),
- # Same rule as the write side, and via the same helper: a run that died during
- # model loading never got a "resumed" event to seed total_steps, and zero here
- # sends the calculation back to the checkpoint manifest's OLDER target. In EPOCH
- # mode train_steps is the request model's unused default, so recomputing from it
- # here undid the persisted answer on every read -- a 600-step checkpoint of a run
- # resolved to 1000 read as 600/500 and Resume was disabled.
+ # A run that died during model loading never got a "resumed" event to seed total_steps, and zero
+ # here sends the calculation back to the manifest's OLDER target, so a 600-step checkpoint read as
+ # 600/500 and Resume was disabled.
total_steps = _resolved_total_steps(rec, config if isinstance(config, dict) else {}),
write_error = rec.get("checkpoint_write_error"),
- # What this run itself resumed from, so a resumed run that died before its first
- # save can still offer the bundle it was validated against.
+ # What this run itself resumed from, so a resumed run that died before its first save can still
+ # offer the bundle it was validated against.
source_checkpoint = (
config.get("resume_from_checkpoint") if isinstance(config, dict) else None
),
- # And WHICH bundle that was, so a slot another run has since rewritten is not
- # offered back as this run's own lineage.
+ # And WHICH bundle that was, so a slot another run has since rewritten is not offered back as this
+ # run's own lineage.
source_created_at = rec.get("resumed_source_created_at"),
)
)
@@ -222,15 +219,14 @@ def list_diffusion_runs(limit: int = 20) -> list[dict]:
rec = json.loads(p.read_text(encoding = "utf-8"))
except Exception: # noqa: BLE001 -- a corrupt record never breaks the listing
continue
- # Skip a wrong-shape record so one bad file cannot blow up the route's DiffusionTrainingRunSummary(**r) or the whole panel.
+ # Skip a wrong-shape record so one bad file cannot blow up the route's
+ # DiffusionTrainingRunSummary(**r) or the whole panel.
if not isinstance(rec, dict):
continue
if not (isinstance(rec.get("job_id"), str) and isinstance(rec.get("status"), str)):
continue
- # Resume state comes from the checkpoints on disk NOW, before the config is dropped.
- # Per record, because the refresh reads counters straight out of the file: one
- # hand-edited or older record with a nonnumeric total_steps took the whole endpoint
- # down, where every other kind of corruption here is skipped.
+ # Resume state comes from the checkpoints on disk NOW, before the config is dropped. Per record,
+ # because one hand-edited nonnumeric total_steps otherwise took the whole endpoint down.
try:
_refresh_resume_state(rec)
_restate_live_job(rec)
@@ -282,10 +278,8 @@ def _restate_live_job(rec: dict) -> dict:
if rec.get("job_id") == live.get("job_id") and live.get("status") == "running":
rec["status"] = "running"
rec["message"] = live.get("message") or ""
- # And nothing resumable, which is what the interim record's error status made
- # _refresh_resume_state derive a moment ago. _UNRESUMABLE_STATUS rejects a running job
- # for a reason: its output directory is being written right now, so the only thing a
- # Resume action there can do is race the live run and 409.
+ # _UNRESUMABLE_STATUS rejects a running job because its output directory is being written right now,
+ # so a Resume there can only race the live run and 409.
rec["can_resume"] = False
rec["checkpoint_path"] = None
return rec
@@ -317,16 +311,14 @@ def _idle_state() -> dict[str, Any]:
"base_model": None,
"samples_per_second": None,
"peak_memory_gb": None,
- # Resume state for this job: where its newest checkpoint-<N> bundle is, the step it holds,
- # why one could not be written (surfaced as the Resume action's disabled tooltip), and the
- # step a resumed run picked up from.
+ # Where the newest checkpoint-<N> bundle is, the step it holds, why one could not be written (the
+ # Resume action's disabled tooltip), and the step a resumed run picked up from.
"checkpoint_path": None,
"checkpoint_step": None,
"resume_blocked_reason": None,
"resumed_from_step": None,
- # The manifest timestamp of the bundle this run resumed FROM. A pathname is not an
- # identity: another run can write over the same checkpoint-<N> slot, and the fallback
- # would then offer that replacement back under this run's lineage.
+ # A pathname is not an identity: another run can write over the same checkpoint-<N> slot, and the
+ # fallback would offer that replacement back under this run's lineage.
"resumed_source_created_at": None,
"started_at": None,
"updated_at": None,
@@ -335,16 +327,16 @@ def _idle_state() -> dict[str, Any]:
"metric_loss": [],
"metric_lr": [],
"metric_grad_norm": [],
- # The two halves of MiniMax-H3's joint objective. Null on every other family, and null
- # on H3 steps that reported only the combined loss, so the series stay index-aligned.
+ # Null on every other family, and on H3 steps that reported only the combined loss, so the series
+ # stay index-aligned.
"metric_video_loss": [],
"metric_audio_loss": [],
}
-# The value series, in append order, paired index-for-index with ``metric_steps``. One list so a
-# new series cannot be added to the appends and forgotten in the decimation below, which would
-# leave the curves silently misaligned rather than failing.
+# The value series, in append order, paired index-for-index with metric_steps. One list so a new
+# series cannot be added to the appends and forgotten in the decimation below, which would
+# silently misalign the curves.
_METRIC_SERIES: tuple[str, ...] = (
"metric_loss",
"metric_lr",
@@ -380,8 +372,7 @@ def _append_metric(
floss = _finite_or_none(loss)
if floss is None: # non-numeric or non-finite: skip, keep the curve JSON-safe
return
- # Every series but loss may be None or non-finite; non-finite is nulled (not dropped) to
- # stay index-aligned.
+ # Non-finite is nulled rather than dropped, to stay index-aligned.
values = {
"metric_loss": floss,
"metric_lr": _finite_or_none(lr),
@@ -397,8 +388,8 @@ def _append_metric(
steps = state["metric_steps"]
steps.append(istep)
for key in _METRIC_SERIES:
- # setdefault, not [key]: a run resumed from a record written before a series existed
- # carries a state dict without it, and losing the whole point is worse than a short tail.
+ # setdefault, not [key]: a run resumed from a record written before a series existed carries a state
+ # dict without it.
state.setdefault(key, []).append(values[key])
@@ -431,29 +422,28 @@ class DiffusionTrainingService:
self._ctx = ctx if ctx is not None else _CTX
self._target = target if target is not None else _default_target
self._lock = threading.Lock()
- # Set by reserve() while a start is in flight (before the route frees GPU models) so the load guards refuse a concurrent load. Cleared by unreserve().
+ # Set by reserve() while a start is in flight (before the route frees GPU models) so the load guards
+ # refuse a concurrent load. Cleared by unreserve().
self._reserved = False
- # Dataset mutations in flight. A start refuses while any is open and a mutation refuses once a start is reserved, both under _lock, so neither slips through the other's check-then-act window.
+ # Dataset mutations in flight. A start refuses while any is open and a mutation refuses once a
+ # start is reserved, both under _lock, so neither slips through the other's window.
self._dataset_mutations = 0
- # "Stop without saving" for the CURRENT job, remembered in the parent. The trainer
- # reports it on its completion event, but a child that is killed or OOMs after the
- # request never emits one -- and the unexpected-exit path then recorded a resumable
- # error run, re-offering Resume from the very checkpoint the user asked to discard.
+ # The trainer reports a no-save stop on its completion event, but a child killed or OOMed after
+ # the request never emits one, and the unexpected-exit path then re-offered Resume from the
+ # discarded checkpoint.
self._discard_requested = False
- # Bundle paths THIS job reported writing, so a discard the child could not carry out
- # itself removes exactly those and nothing that predated the run.
+ # Bundle paths THIS job reported writing, so a discard the child could not carry out removes exactly
+ # those and nothing that predated the run.
self._own_checkpoints: list[str] = []
- # Set when the child reported a discarded completion, which it emits only AFTER running
- # its own clear_own_checkpoints. That helper hands a displaced slot back, so the path
- # this run wrote to can now hold ANOTHER run's restored bundle -- and the parent-side
- # cleanup, which knows only pathnames, would delete it.
+ # The child emits a discarded completion only after its own clear_own_checkpoints, which hands a
+ # displaced slot back, so the parent's pathname-only cleanup would delete another run's bundle.
self._child_cleared_own = False
# Set by a checkpoint_saved event, cleared once the pump has written the record.
self._persist_interim = False
- # Whether this job has already had a stop signalled. Only the first one reaches the
- # child, so only the first one may set the parent's disposition.
+ # Only the first stop reaches the child, so only the first may set the parent's disposition.
self._stop_signalled = False
- # GPU load admissions in flight (a load between its training guard and its arbiter registration). Same two-sided rule as the dataset mutations.
+ # GPU load admissions in flight (a load between its training guard and its arbiter registration).
+ # Same two-sided rule as the dataset mutations.
self._gpu_admissions = 0
self._proc: Any = None
self._stop_queue: Any = None
@@ -490,14 +480,15 @@ class DiffusionTrainingService:
"then start the run."
)
if self._gpu_admissions:
- # A load already passed its training guard and is about to take the GPU. Reserving now would free residents it has not
- # registered yet. Refusing is safe: the admission is held only across the registration, not the load itself.
+ # A load past its training guard is about to take the GPU, and reserving now would free residents
+ # it has not registered yet; refusing is safe, since admission is held only across registration.
raise RuntimeError(
"A model is being loaded onto the GPU right now. Wait for that to finish, "
"then start the run."
)
- # The LLM trainer under the SAME lock, not just at the route's earlier check: several network-bound preflights separate the
- # two, so an LLM start could spawn in between. The LLM route holds gpu_load_admission() across its own spawn, so one of the two always raises.
+ # Under the SAME lock as the LLM trainer, not just at the route's earlier check: several
+ # network-bound preflights separate the two, and the LLM route holds gpu_load_admission() across
+ # its spawn, so one of the two always raises.
if _llm_training_active():
raise RuntimeError(
"An LLM training job is already running. "
@@ -569,15 +560,15 @@ class DiffusionTrainingService:
Raises ValueError for an unusable config (before any spawn) and RuntimeError if a
job is already running. Returns the new job id."""
- # Validate cheaply BEFORE spawning so a bad request fails fast with a clear error. The
- # normalised config is kept: it carries the resolved family the recipe overrides below
- # are keyed on, which the raw request dict does not have to name.
+ # Validate before spawning, and keep the normalised config: it carries the resolved family the
+ # recipe overrides are keyed on, which the raw request dict need not name.
from .diffusion_lora_trainer import _config_from_dict
from .diffusion_train_common import train_recipe_overrides
normalized_cfg = _config_from_dict(config).normalized()
- # Join a finished job's pump OUTSIDE the lock: its final state writes take this lock, so joining under it would stall the start and let the stale pump overwrite the new state.
+ # Join a finished job's pump OUTSIDE the lock: its final state writes take this lock, so joining
+ # under it would stall the start and let the stale pump overwrite the new state.
with self._lock:
if self._proc is not None and self._proc.is_alive():
raise RuntimeError("A diffusion training job is already running.")
@@ -613,7 +604,7 @@ class DiffusionTrainingService:
self._proc.start()
try:
from utils.process_lifetime import adopt_pid
- adopt_pid(self._proc.pid) # bind to parent lifetime (no zombie on exit)
+ adopt_pid(self._proc.pid)
except Exception: # noqa: BLE001 -- lifetime binding is best-effort
pass
@@ -628,18 +619,12 @@ class DiffusionTrainingService:
started_at = now,
updated_at = now,
)
- # AFTER the reset above, which replaces the whole state dict. The route has already
- # validated and PINNED a source bundle, so its identity is known before the child
- # runs: recording it here means a resume that dies during the model load -- before
- # the trainer can emit "resumed" -- still has the timestamp its fallback is checked
- # against, instead of trusting the pathname and offering back whatever later
- # occupies that slot.
+ # AFTER the reset above: the route has already pinned a source bundle, so recording its identity
+ # here means a resume that dies during the model load still has its timestamp to check against.
self._seed_source_identity(config)
- # Keep the config (minus secrets) for the persisted run record, with the fields this
- # family's loop REPLACES rather than honours set to what it will actually run. The
- # trainer applies the same table in the child, which the record is not written from,
- # so without this Previous runs described a recipe (cropping, flipping, min-SNR
- # weighting) that no step of the run ever used.
+ # Record the config with the fields this family's loop REPLACES set to what it will actually run:
+ # the trainer applies the same table in the child, so without this Previous runs described a
+ # recipe no step ever used.
self._config = {k: v for k, v in dict(config).items() if k != "hf_token"}
self._config.update(train_recipe_overrides(normalized_cfg))
self._pump = threading.Thread(
@@ -657,12 +642,9 @@ class DiffusionTrainingService:
if self._proc is None or not self._proc.is_alive() or self._stop_queue is None:
return False
if self._stop_signalled:
- # The child consumes the FIRST signal and acts on it. A second one cannot
- # un-save an adapter it has already exported, so honouring a later
- # stop-without-saving set a parent discard the child never carried out: the
- # run was marked discarded and its checkpoints deleted while the adapter and
- # the catalog entry it published stayed on disk. The disposition is whichever
- # one the child actually got.
+ # The child consumes the FIRST signal, so honouring a later stop-without-saving set a parent
+ # discard the child never carried out: the run was marked discarded and its checkpoints deleted
+ # while the adapter and catalog entry stayed on disk.
return True
try:
# Bare True = older wire format; the dict form carries the no-save cancel flag.
@@ -671,8 +653,8 @@ class DiffusionTrainingService:
return False
self._stop_signalled = True
if not save:
- # Remembered here so a child that dies before its discarded completion still
- # blocks the resume the user threw away.
+ # Remembered here so a child that dies before its discarded completion still blocks the resume the
+ # user threw away.
self._discard_requested = True
self._state["message"] = (
"Stop requested; finishing the current step and saving a partial adapter..."
@@ -726,9 +708,8 @@ class DiffusionTrainingService:
self._persist_interim = False
self._persist_run_record(interim = True)
if ev.get("type") in _TERMINAL:
- # An exception raised on the current step is a terminal `error`, and the pump
- # returns here rather than through the dead-process branch -- so the discard the
- # user asked for has to be applied on this path too.
+ # An exception raised on the current step is a terminal error and returns here rather than through
+ # the dead-process branch, so the discard has to be applied on this path too.
with self._lock:
discarding = self._discard_requested and self._proc is proc
child_cleared = self._child_cleared_own
@@ -826,10 +807,8 @@ class DiffusionTrainingService:
"ema_path": s.get("ema_path"),
"catalog_path": s.get("catalog_path"),
"saved": bool(s.get("lora_path")),
- # Resume lineage + state. output_dir is what a Resume replays, so it is recorded on
- # the run rather than only inside the config. can_resume / checkpoint_step are a
- # snapshot taken now and RE-DERIVED from disk on read, so deleting the checkpoints
- # later takes the action away instead of leaving a button that 400s.
+ # output_dir is what a Resume replays, so it is recorded on the run; can_resume / checkpoint_step
+ # are re-derived from disk, so deleting the checkpoints takes the action away.
"output_dir": str(adapter) if adapter else None,
"resumed_from_job_id": cfg.get("resumed_from_job_id") or None,
"resumed_from_step": s.get("resumed_from_step"),
@@ -839,18 +818,11 @@ class DiffusionTrainingService:
str(adapter) if adapter else None,
status = s.get("status"),
started_at = s.get("started_at"),
- # The REQUESTED target, falling back to the config the start preflight
- # accepted: total_steps is seeded by the "resumed" event, and a run that
- # dies during model loading never gets one. Zero there sent the resume
- # calculation back to the checkpoint manifest's older target, which then
- # reported "nothing left to train" for a run whose whole point was a
- # raised target.
- #
- # NOT in epoch mode, though: num_epochs overrides train_steps, which then
- # still carries the Pydantic default of 500. A run resolved to 1000 that
- # died before its "resumed" event would report a 600-step checkpoint as
- # 600/500 and refuse the resume. Zero there is honest, and the manifest's
- # own target (written with the resolved count) is the right answer.
+ # total_steps is seeded by the "resumed" event and a run that dies during model loading never gets
+ # one; zero there sent the resume calculation back to the manifest's older target, reporting
+ # "nothing left to train" for a run whose point was a raised target.
+ # NOT in epoch mode: num_epochs overrides train_steps, which keeps its Pydantic default of 500, so a
+ # run resolved to 1000 reported a 600-step checkpoint as 600/500 and refused the resume.
total_steps = _resolved_total_steps(s, cfg),
write_error = s.get("resume_blocked_reason"),
source_checkpoint = cfg.get("resume_from_checkpoint"),
@@ -893,7 +865,8 @@ class DiffusionTrainingService:
elif etype == "model_load_completed":
s.update(in_model_load = False, message = "Training...")
elif etype == "preparing":
- # A long precompute phase (e.g. the VAE latent cache) before the first step; surfaced so the UI shows progress instead of a silent stall.
+ # A long precompute phase (e.g. the VAE latent cache) before the first step; surfaced so the UI
+ # shows progress instead of a silent stall.
done, total = ev.get("done"), ev.get("total")
stage = str(ev.get("stage", "prepare")).replace("_", " ")
s.update(
@@ -909,27 +882,21 @@ class DiffusionTrainingService:
# Non-fatal trainer notes; keep training state, surface the text.
s["message"] = str(ev.get("message", "warning"))
elif etype == "resumed":
- # The trainer restored a checkpoint and is continuing from it. Recorded so the run
- # history shows lineage instead of an unexplained run that starts mid-way. This is
- # the step resumed FROM, not a checkpoint this run wrote, so it does not touch
- # checkpoint_step (which tracks this run's own newest bundle).
+ # This is the step resumed FROM, not a checkpoint this run wrote, so it does not touch checkpoint_step.
s.update(
resumed_from_step = ev.get("step"),
resumed_source_created_at = ev.get("source_created_at"),
message = f"Resuming from step {ev.get('step')}...",
)
- # Seed the LIVE counters from the same event. They are what the UI renders and
- # what an errored run persists, and until the first post-resume progress event
- # they still read 0/0 -- so a resume of step 400 of 500 shows "0/0", and an OOM
- # on the first step records a failed run at step 0 with a target of 0.
- # checkpoint_step stays untouched: this bundle is not one this run wrote.
+ # Seed the LIVE counters from the same event: until the first post-resume progress they read 0/0, so
+ # a resume of step 400 of 500 showed "0/0" and an OOM recorded a failed run at step 0 of 0.
if ev.get("step") is not None:
s["step"] = int(ev["step"])
if ev.get("total_steps") is not None:
s["total_steps"] = int(ev["total_steps"])
elif etype == "checkpoint_saved":
- # Folded as each bundle lands, not only at the end, so a run that later crashes is
- # still reported as resumable. A good write also clears an earlier write's failure.
+ # Folded as each bundle lands, not only at the end, so a run that later crashes is still reported as
+ # resumable; a good write also clears an earlier failure.
s.update(
checkpoint_path = ev.get("checkpoint_path"),
checkpoint_step = ev.get("step"),
@@ -938,23 +905,15 @@ class DiffusionTrainingService:
written = ev.get("checkpoint_path")
if written and written not in self._own_checkpoints:
self._own_checkpoints.append(str(written))
- # The bundle is on disk; the run JSON is not, and only a terminal event writes
- # one. Unsloth being killed or shut down after a periodic save therefore left a
- # resumable checkpoint that Previous runs -- which reads those JSONs and
- # nothing else -- had no entry for, so the Resume action did not exist. Written
- # as an interrupted run because that is what it IS until the run ends: the
- # terminal persist overwrites the same file, and the live job is reported as
- # running by the reader below.
+ # Only a terminal event writes the run JSON, so being killed after a periodic save left a resumable
+ # checkpoint Previous runs had no entry for.
self._persist_interim = True
elif etype == "checkpoint_failed":
- # Sticky: whatever older bundle is on disk predates the work this run did, so
- # resuming from it would silently lose steps. Mirrors the MLX resume_blocked flag.
+ # Sticky: an older bundle on disk predates the work this run did, so resuming from it would
+ # silently lose steps. Mirrors the MLX resume_blocked flag.
s["resume_blocked_reason"] = str(ev.get("message", "checkpoint write failed"))
- # Persisted for the same reason a successful write is. In memory only, an Unsloth
- # exit after this left the last record advertising the OLDER checkpoint as
- # resumable -- the one this service has just decided is stale -- and resuming it
- # rolls the run back past everything after it. A later checkpoint_saved clears
- # the reason and replaces the record.
+ # Persisted because, in memory only, an exit after this left the last record advertising the stale
+ # older checkpoint as resumable.
self._persist_interim = True
elif etype == "progress":
# Null any non-finite float so the JSON stays strict-parseable; a missing key keeps the last value.
@@ -968,9 +927,8 @@ class DiffusionTrainingService:
grad_norm = (
_finite_or_none(ev["grad_norm"]) if "grad_norm" in ev else s["grad_norm"]
)
- # The two halves of a joint objective, when the trainer reports them. Folded
- # like the rest rather than dropped: on MiniMax-H3 the combined loss can hold
- # steady while one modality degrades, and these are the only signal that says so.
+ # Folded like the rest rather than dropped: on MiniMax-H3 the combined loss can hold steady while
+ # one modality degrades, and these are the only signal that says so.
video_loss = (
_finite_or_none(ev["video_loss"]) if "video_loss" in ev else s["video_loss"]
)
@@ -1005,7 +963,8 @@ class DiffusionTrainingService:
ev.get("audio_loss"),
)
elif etype == "complete":
- # Reset in_model_load: a stop during model load emits complete with no preceding model_load_completed, leaving a stale indicator.
+ # Reset in_model_load: a stop during model load emits complete with no preceding
+ # model_load_completed, leaving a stale indicator.
s.update(
active = False,
in_model_load = False,
@@ -1027,17 +986,13 @@ class DiffusionTrainingService:
s["family"] = ev.get("family")
if ev.get("base_model") is not None:
s["base_model"] = ev.get("base_model")
- # Checkpoint state is folded from the per-save events above; only the lineage is
- # (re)confirmed here, for a pump that missed the earlier ``resumed`` event.
+ # Only the lineage is re-confirmed here, for a pump that missed the earlier "resumed" event.
if ev.get("resumed_from_step") is not None:
s["resumed_from_step"] = ev.get("resumed_from_step")
if ev.get("discarded"):
- # Cancelled with "stop without saving": the user threw this run away, so its
- # own periodic checkpoints must not keep offering to continue it.
- #
- # The child ran its own cleanup before emitting this, so the parent must not
- # repeat it by pathname: a slot this run overwrote has been handed back to
- # the bundle it displaced, and that one belongs to another run.
+ # A discarded run's own periodic checkpoints must not keep offering to continue it.
+ # The child ran its own cleanup before emitting this, so the parent must not repeat it by pathname:
+ # a slot this run overwrote has been handed back to another run's bundle.
self._child_cleared_own = True
s["resume_blocked_reason"] = (
"This run was stopped without saving, so it was discarded."
diff --git a/studio/backend/core/training/provenance.py b/studio/backend/core/training/provenance.py
index 1b6480ef74c..b29444c28e7 100644
--- a/studio/backend/core/training/provenance.py
+++ b/studio/backend/core/training/provenance.py
@@ -131,8 +131,8 @@ def exact_resume_requires_current_4bit(config: dict[str, Any]) -> bool:
return False
except Exception:
return False
- # The same disjunction effective_training_load_in_4bit tests: routes/training.py
- # fills require_exact_model_resource from exact_resume_resource_requirements and
+ # The same disjunction effective_training_load_in_4bit tests: routes/training.py fills
+ # require_exact_model_resource from exact_resume_resource_requirements and
# require_exact_resume_resources from resource_provenance_is_complete.
return bool(requires_exact_model or resource_provenance_is_complete(config))
diff --git a/studio/backend/core/training/resume.py b/studio/backend/core/training/resume.py
index c018084abbd..681b398d332 100644
--- a/studio/backend/core/training/resume.py
+++ b/studio/backend/core/training/resume.py
@@ -26,6 +26,7 @@ def _is_under_outputs(path: Path) -> bool:
resolved.relative_to(root)
return True
except (OSError, RuntimeError, ValueError):
+ # Unrecognized state-file formats are not usable resume state.
return False
@@ -88,7 +89,6 @@ def _valid_state_file(path: Path, require_tensor: bool = True) -> bool:
and info.file_size > 0
for info in infos
)
- # Unrecognized state-file formats are not usable resume state.
return False
except (OSError, ValueError, zipfile.BadZipFile):
return False
@@ -287,7 +287,8 @@ def can_resume_run(run: dict, *, resource_cache: Optional[dict[str, bool]] = Non
status = run.get("status")
if status == "error":
- # A save-time crash can report final_step == total_steps with no artifacts; checkpoint state alone decides resumability.
+ # A save-time crash can report final_step == total_steps with no artifacts; checkpoint state alone
+ # decides resumability.
resume_state_available = has_resume_state(run.get("output_dir"))
else:
final_step = run.get("final_step")
diff --git a/studio/backend/core/training/s3_dataset.py b/studio/backend/core/training/s3_dataset.py
index e25b1479419..d1c7e92e2d9 100644
--- a/studio/backend/core/training/s3_dataset.py
+++ b/studio/backend/core/training/s3_dataset.py
@@ -68,7 +68,7 @@ def _build_s3_client(s3_config: dict):
Uses explicit access keys when provided, otherwise falls back to the
default credential chain (IAM role / instance profile / env / shared creds).
"""
- import boto3 # lazy: optional dependency
+ import boto3
region = s3_config.get("region") or "us-east-1"
use_iam_role = bool(s3_config.get("use_iam_role"))
@@ -98,14 +98,14 @@ def _list_dataset_keys(client, bucket: str, prefix: Optional[str]) -> list[str]:
for obj in page.get("Contents", []):
key = obj["Key"]
if key.endswith("/"):
- continue # directory placeholder
+ continue
if os.path.basename(key).lower() in _IGNORED_METADATA_FILENAMES:
continue
if key.lower().endswith(SUPPORTED_EXTENSIONS):
keys.append(key)
- # A sync from a Mac uploads Finder metadata too, under the shard's own extension, and
- # _validate_single_extension_family cannot see it. Nothing to read here, so a key is
- # dropped only when the object it would describe is in the same listing.
+ # A Mac sync uploads Finder metadata under the shard's own extension and
+ # _validate_single_extension_family cannot see it, so a key is dropped only when the object it
+ # would describe is in the same listing.
return drop_shadowed_appledouble_names(keys)
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index 6fbc241a61a..be63f934464 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -11,13 +11,12 @@ import os
import sys
import types
-# Off on Linux so datasets' forked map() workers can't deadlock. On spawn platforms
-# map() runs in-process, so keep the fast tokenizer's Rust threads on.
+# Off on Linux so datasets' forked map() workers cannot deadlock; on spawn platforms map() runs in-
+# process, so the fast tokenizer's Rust threads stay on.
os.environ["TOKENIZERS_PARALLELISM"] = "true" if sys.platform in ("win32", "darwin") else "false"
-# Make compiled cache modules importable by any subprocess: spawned dataset.map()
-# workers re-import top-level modules whose cache files import torch + unsloth_zoo.
-# Do NOT import unsloth_zoo.compiler here -- it pulls in heavy torch/triton imports.
+# Spawned dataset.map() workers re-import top-level modules whose cache files import torch +
+# unsloth_zoo. Do NOT import unsloth_zoo.compiler here: it pulls in heavy torch/triton imports.
if sys.platform in ("win32", "darwin"):
_compile_cache = os.environ.get("UNSLOTH_COMPILE_LOCATION", "unsloth_compiled_cache")
if not os.path.isabs(_compile_cache):
@@ -94,8 +93,7 @@ from .dataset_bounds import bound_dataset_rows, world_size_env_report, world_siz
logger = get_logger(__name__)
-# Streaming eval has no __len__, so an unbounded eval would iterate the whole
-# source on every eval step. Cap it so each evaluation terminates.
+# Streaming eval has no __len__, so cap it or an unbounded eval iterates the whole source on every eval step.
STREAMING_EVAL_MAX_SAMPLES = 500
@@ -186,9 +184,8 @@ def _spark_tts_tokenizer_kwargs(audio_type: Optional[str], lookup_name: str) ->
name = str(lookup_name).replace("\\", "/").rstrip("/")
if name.endswith("/LLM"):
return {}
- # An LLM/ child is the canonical layout, and a cache-pinned or offline snapshot root has
- # one too. Treating that as "already at the tokenizer" sent AutoTokenizer at the root,
- # which holds no tokenizer, and failed the very case this helper exists for.
+ # A cache-pinned or offline snapshot root also has an LLM/ child, and treating that as "already at
+ # the tokenizer" sent AutoTokenizer at a root that holds none.
if os.path.isdir(os.path.join(lookup_name, "LLM")):
return {"subfolder": "LLM"}
# A local checkpoint that carries its own tokenizer is already the right directory.
@@ -197,12 +194,11 @@ def _spark_tts_tokenizer_kwargs(audio_type: Optional[str], lookup_name: str) ->
return {"subfolder": "LLM"}
-# Enough rows to see past a leading null or malformed value, few enough to stay cheap on a
-# streamed dataset, where each row is pulled over the network.
+# Enough rows to see past a leading null or malformed value, few enough to stay cheap on a streamed dataset.
_AUDIO_SNIFF_ROWS = 16
-# Kept in step with detect_multimodal_dataset, so the veto answers False only about the very
-# columns that set the flag it is vetoing.
+# Kept in step with detect_multimodal_dataset, so the veto answers False only about the very columns
+# that set the flag it is vetoing.
_AUDIO_COLUMN_KEYWORDS = ("audio", "speech", "wav", "waveform", "sound")
@@ -237,14 +233,9 @@ def _dataset_has_audio_column(dataset) -> Optional[bool]:
except Exception: # noqa: BLE001 - a mapping that is not a features dict
return None
- # No Audio feature, so nothing decodes here and reading rows cannot hit the
- # missing-FFmpeg path. Several rows, not one: a JSON/CSV audio dataset can carry a null
- # or malformed first value and real paths after it, and the preprocessors skip such a
- # row rather than fail, so row 0 alone would call it textual.
- #
- # Audio-like values count anywhere, but only an audio-NAMED column can answer False: a
- # transcript is populated in every row of a path-backed audio dataset, so counting it
- # would report "no audio here" on the strength of the text. No evidence stays None.
+ # Several rows, not one: a JSON/CSV audio dataset can carry a null or malformed first value and real paths after it.
+ # Only an audio-NAMED column can answer False: a transcript is populated in every row of a path-
+ # backed audio dataset. No evidence stays None.
try:
from utils.datasets.format_detection import (
_AUDIO_EXTENSIONS,
@@ -297,17 +288,16 @@ class UnslothTrainer:
self.save_on_stop = True
self.load_in_4bit = True
- self.is_cpt = False # Continued Pretraining
+ self.is_cpt = False
self.is_vlm = False
self.is_audio = False
- self.is_audio_vlm = False # e.g. Gemma 3N trained on audio
- self._audio_type = None # 'csm', 'whisper', 'snac', 'bicodec', 'dac'
- # True until a probe says otherwise, so a path that never probes cannot trip the
- # inconclusive-detection guard.
+ self.is_audio_vlm = False
+ self._audio_type = None
+ # True until a probe says otherwise, so a path that never probes cannot trip the inconclusive-detection guard.
self._audio_type_known = True
self._is_dataset_audio = False
- self._cuda_audio_used = False # Set once after audio CUDA preprocessing; never cleared
- self._spark_tts_repo_dir = None # Spark-TTS repo path, for BiCodecTokenizer
+ self._cuda_audio_used = False
+ self._spark_tts_repo_dir = None
self._spark_tts_code_dir = None
self._outetts_code_dir = None
self.model_name = None
@@ -350,10 +340,8 @@ class UnslothTrainer:
if hf_token:
os.environ["HF_TOKEN"] = hf_token
- # Detect audio type (config.json only, no VRAM). Checked, because an unreadable
- # tokenizer_config.json (gated repo, offline, or a cache holding weights but not the
- # tokenizer files) otherwise reads as "not an audio model" and sends a TTS run down
- # the text path.
+ # Checked, because an unreadable tokenizer_config.json (gated, offline, or a weights-only cache)
+ # otherwise reads as "not an audio model" and sends a TTS run down the text path.
self._audio_type, self._audio_type_known = detect_audio_type_checked(
lookup_name,
hf_token,
@@ -392,8 +380,8 @@ class UnslothTrainer:
self.is_vlm,
)
- # Load tokenizer/processor (CPU only, no VRAM)
- # Whisper needs AutoProcessor; others AutoTokenizer (CSM loads its own inline).
+ # Load tokenizer/processor (CPU only, no VRAM) Whisper needs AutoProcessor; others AutoTokenizer (CSM
+ # loads its own inline).
def _load_tokenizer_for_detected_type():
if self._audio_type == "whisper":
from transformers import AutoProcessor
@@ -412,17 +400,14 @@ class UnslothTrainer:
token = hf_token,
local_files_only = local_files_only,
revision = model_revision,
- # Spark-TTS keeps its tokenizer under LLM/, the subfolder _load_model
- # loads weights from. Reading the root finds no vocab and fails blaming
- # a missing sentencepiece that is installed and irrelevant.
+ # Spark-TTS keeps its tokenizer under LLM/; reading the root finds no vocab and fails blaming a
+ # missing sentencepiece that is installed and irrelevant.
**_spark_tts_tokenizer_kwargs(self._audio_type, lookup_name),
)
- # The probe and the tokenizer load are separate reads of the same repo, so a transient
- # timeout or 5xx can leave the probe inconclusive while the read after it succeeds.
- # Ask again rather than refuse the run later claiming tokenizer_config.json could not
- # be read when it just was. detect_audio_type_checked caches only definitive results,
- # so this costs a cache hit on the common path.
+ # The probe and the tokenizer load are separate reads of the same repo, so a transient 5xx can
+ # leave the probe inconclusive while the read after it succeeds; detect_audio_type_checked caches
+ # only definitive results.
def _retry_audio_detection() -> bool:
"""Re-probe an inconclusive type. True if the answer changed."""
if self._audio_type_known:
@@ -437,6 +422,7 @@ class UnslothTrainer:
if not retried_known:
return False
self._audio_type, self._audio_type_known = retried_type, True
+ # audio_vlm is detected as an audio_type now; handle separately
if self._audio_type == "audio_vlm":
self.is_audio = False
self.is_audio_vlm = is_dataset_audio
@@ -451,10 +437,8 @@ class UnslothTrainer:
)
return self._audio_type != previous_type
- # An inconclusive probe leaves _audio_type None and the kwargs are chosen from it, so
- # a Spark-TTS repo is read at its tokenizer-less root and raises here. Re-probe before
- # giving up, or the retry below is unreachable for exactly the models whose layout the
- # type decides.
+ # An inconclusive probe leaves _audio_type None, so a Spark-TTS repo is read at its tokenizer-less
+ # root; re-probe, or the retry below is unreachable for exactly those models.
try:
_load_tokenizer_for_detected_type()
except Exception:
@@ -469,9 +453,8 @@ class UnslothTrainer:
logger.info("Pre-loaded tokenizer for %s", model_name)
- # The loader and its kwargs are chosen from the type, so a retry that changes the
- # answer has left the wrong object in self.tokenizer: whisper needs an AutoProcessor
- # (_preprocess_whisper_dataset reads .feature_extractor and .tokenizer off it), and
+ # A retry that changes the answer leaves the wrong object in self.tokenizer: whisper needs an
+ # AutoProcessor (_preprocess_whisper_dataset reads .feature_extractor and .tokenizer off it) and
# Spark-TTS its LLM/ subfolder.
if _retry_audio_detection():
_load_tokenizer_for_detected_type()
@@ -520,10 +503,8 @@ class UnslothTrainer:
trainer_ref = self
class _ProgressCallback(TrainerCallback):
- # Per-batch evaluation progress used to exist only as HF's tqdm bar, which
- # this PR drops. A long eval would otherwise look stalled between the last
- # step log and the eval result, so it is republished, throttled, as status
- # and one structured line.
+ # Republished, throttled: without HF's tqdm bar a long eval looks stalled between the last step log
+ # and the eval result.
_eval_seen = 0
_eval_last_report = 0.0
@@ -538,8 +519,8 @@ class UnslothTrainer:
had_status = cls._eval_last_report > 0.0
cls._eval_seen = 0
cls._eval_last_report = 0.0
- # An empty status is ignored downstream on purpose, so the UI would
- # sit on "Evaluating..." for the rest of the run.
+ # An empty status is ignored downstream on purpose, so the UI would sit on "Evaluating..." for the
+ # rest of the run.
if had_status and not trainer_ref.should_stop:
trainer_ref._update_progress(status_message = "Training in progress...")
@@ -556,7 +537,7 @@ class UnslothTrainer:
total = None
try:
total = len(eval_dataloader) if eval_dataloader is not None else None
- except TypeError: # an iterable-style loader has no length
+ except TypeError:
total = None
now = time.time()
if cls._eval_last_report and (now - cls._eval_last_report) < 15.0:
@@ -576,11 +557,9 @@ class UnslothTrainer:
):
if not logs:
return
- # Trainer's end-of-run and end-of-eval summaries carry numbers that
- # appear nowhere else in Unsloth (train_samples_per_second,
- # train_steps_per_second, total_flos, memory, eval runtime). They used
- # to reach the log only through PrinterCallback's raw stdout dict, so
- # with that callback gone they are re-published here, structured, once.
+ # Trainer's end-of-run and end-of-eval summaries carry numbers that appear nowhere else
+ # (train_samples_per_second, train_steps_per_second, total_flos, memory, eval runtime) and used to
+ # reach the log only through PrinterCallback's raw stdout.
if any(k in logs for k in _TRAINER_SUMMARY_KEYS):
logger.info(
"trainer summary",
@@ -590,12 +569,8 @@ class UnslothTrainer:
if isinstance(k, str) and k not in _RESERVED_LOG_KEYS
},
)
- # HF logs the end-of-run summary as {"train_runtime": ..., "train_loss": ...}
- # with no "loss" key, and `train_loss` is the MEAN loss over the whole run,
- # not a step loss. Falling back to it reported the average as the final
- # step's value: it landed on the chart at the same global_step as the real
- # last step and became `final_loss` on /api/train/runs, disagreeing with the
- # checkpoint. Keep reporting it, as the summary it is.
+ # train_loss is the MEAN loss over the whole run, so falling back to it put the average on the chart
+ # at the last step and into final_loss, disagreeing with the checkpoint.
loss_value = logs.get("loss")
is_run_summary = loss_value is None and (
logs.get("train_loss") is not None or logs.get("train_runtime") is not None
@@ -692,8 +667,7 @@ class UnslothTrainer:
"seed": random_seed,
"output_dir": output_dir,
"report_to": _build_report_targets(training_args),
- # The subprocess stdout is teed into the server log, so HF's bar is
- # noise there; --verbose restores it. See _hf_stdout_progress_disabled.
+ # The subprocess stdout is teed into the server log, so HF's bar is noise there; --verbose restores it.
"disable_tqdm": _hf_stdout_progress_disabled(),
}
@@ -770,7 +744,6 @@ class UnslothTrainer:
sys.path.remove(path)
removed_paths.append(path)
- # Remove stale audio modules from sys.modules
prefixes = ("snac", "whisper", "sparktts", "outetts")
removed_modules = [key for key in sys.modules if key.startswith(prefixes)]
for key in removed_modules:
@@ -813,7 +786,6 @@ class UnslothTrainer:
"speaker_col": speaker_col,
}
- # Hardcoded fallback
audio_col = next((c for c in cols if c.lower() in ("audio", "speech")), None)
text_col = next(
(c for c in cols if c.lower() in ("text", "sentence", "transcript", "transcription")),
@@ -849,8 +821,8 @@ class UnslothTrainer:
model_revision: Optional[str] = None,
) -> bool:
"""Load model for training (supports both text and vision models)"""
- self.load_in_4bit = load_in_4bit # For training_meta.json
- self.trust_remote_code = trust_remote_code # For AutoProcessor etc. used during training
+ self.load_in_4bit = load_in_4bit
+ self.trust_remote_code = trust_remote_code
lookup_name = model_load_name or model_name
self.model_load_error = None
try:
@@ -868,8 +840,8 @@ class UnslothTrainer:
# Clear audio sys.path/sys.modules state; stale entries deadlock forked map() workers
self._cleanup_audio_artifacts()
- # Reload Unsloth-patched modeling modules before clearing the cache: __UNSLOTH_PATCHED__
- # blocks re-compilation, so clearing the disk cache alone would leave files missing.
+ # Reload the Unsloth-patched modeling modules first: __UNSLOTH_PATCHED__ blocks re-compilation, so
+ # clearing the disk cache alone would leave files missing.
import importlib
for _key, _mod in list(sys.modules.items()):
@@ -878,15 +850,13 @@ class UnslothTrainer:
try:
importlib.reload(_mod)
except Exception:
- pass # Non-critical — Unsloth handles stale modules
+ pass
- # Remove stale compiled cache so the new model gets a fresh one
from utils.cache_cleanup import clear_unsloth_compiled_cache
_preserve = ["Unsloth*Trainer.py"] if sys.platform in ("win32", "darwin") else None
clear_unsloth_compiled_cache(preserve_patterns = _preserve)
- # Detect audio type (config.json + tokenizer). Checked: this reassigns
- # _audio_type, so an unchecked answer would leave the flag describing the
+ # Checked: this reassigns _audio_type, so an unchecked answer would leave the flag describing the
# previous probe.
self._audio_type, self._audio_type_known = detect_audio_type_checked(
lookup_name,
@@ -895,10 +865,9 @@ class UnslothTrainer:
revision = model_revision,
)
self._is_dataset_audio = is_dataset_audio
- # audio_vlm is detected as an audio_type now; handle separately
if self._audio_type == "audio_vlm":
self.is_audio = False
- self.is_audio_vlm = is_dataset_audio # Only use audio VLM path if dataset has audio
+ self.is_audio_vlm = is_dataset_audio
self._audio_type = None
else:
self.is_audio = self._audio_type is not None
@@ -907,7 +876,6 @@ class UnslothTrainer:
if not self.is_audio and not self.is_audio_vlm:
self._cuda_audio_used = False
- # VLM: vision model + image dataset (mutually exclusive with audio)
vision = (
is_vision_model(
lookup_name,
@@ -987,16 +955,15 @@ class UnslothTrainer:
f"Using device_map='{device_map}' ({get_visible_gpu_count()} GPU(s) visible)"
)
- # ROCm without native bf16 (e.g. RDNA2/gfx103x) dies with an LLVM error on the first
- # bf16 kernel when dtype=None picks bf16, so force float16. NVIDIA keeps None so
- # unsloth's auto-detect is honored -- T4/V100 must NOT be coerced to float16.
+ # ROCm without native bf16 (e.g. RDNA2/gfx103x) dies with an LLVM error on the first bf16 kernel
+ # when dtype=None picks bf16, so force float16; NVIDIA keeps None so T4/V100 are not coerced.
_is_rocm = (
bool(getattr(torch.version, "hip", None)) or "rocm" in torch.__version__.lower()
)
_auto_dtype = torch.float16 if (_is_rocm and not is_bfloat16_supported()) else None
if self._audio_type == "csm":
- # CSM: FastModel, auto_model=CsmForConditionalGeneration, load_in_4bit=False
+ # Whisper: FastModel, auto_model=WhisperForConditionalGeneration, load_in_4bit=False
from unsloth import FastModel
from transformers import CsmForConditionalGeneration
@@ -1016,7 +983,6 @@ class UnslothTrainer:
logger.info("Loaded CSM audio model")
elif self._audio_type == "whisper":
- # Whisper: FastModel, auto_model=WhisperForConditionalGeneration, load_in_4bit=False
from unsloth import FastModel
from transformers import WhisperForConditionalGeneration
@@ -1034,7 +1000,6 @@ class UnslothTrainer:
revision = model_revision,
use_exact_model_name = model_revision is not None,
)
- # Generation settings (notebook lines 100-105)
self.model.generation_config.language = "<|en|>"
self.model.generation_config.task = "transcribe"
self.model.config.suppress_tokens = []
@@ -1058,13 +1023,17 @@ class UnslothTrainer:
logger.info(f"Loaded {self._audio_type} audio model (FastLanguageModel)")
elif self._audio_type == "bicodec":
- # Spark-TTS: download the full repo (sparktts + BiCodec weights), load only the LLM
- # subfolder. model_name is "Spark-TTS-0.5B/LLM" (YAML mapping) or "unsloth/Spark-TTS-0.5B".
+ # Download the full repo (sparktts + BiCodec weights) but load only the LLM subfolder; model_name is
+ # "Spark-TTS-0.5B/LLM" or "unsloth/Spark-TTS-0.5B".
+ # Use FastModel.get_peft_model (codec audio + audio VLM)
+ # Audio VLM (e.g. Gemma 3N): FastModel returns (model, processor).
+ # OuteTTS: uses FastModel (not FastLanguageModel) with load_in_4bit=False
+ # Spark-TTS: download the full repo (sparktts + BiCodec weights), load only the LLM subfolder.
+ # model_name is "Spark-TTS-0.5B/LLM" (YAML mapping) or "unsloth/Spark-TTS-0.5B".
from unsloth import FastModel
from huggingface_hub import snapshot_download
if model_name.endswith("/LLM"):
- # "Spark-TTS-0.5B/LLM" → repo "unsloth/Spark-TTS-0.5B"
hf_repo = f"unsloth/{model_name.rsplit('/', 1)[0]}"
else:
hf_repo = model_name
@@ -1076,7 +1045,7 @@ class UnslothTrainer:
hf_repo,
revision = model_revision,
)
- self._spark_tts_repo_dir = os.path.abspath(repo_path) # Absolute for sys.path
+ self._spark_tts_repo_dir = os.path.abspath(repo_path)
llm_path = os.path.join(self._spark_tts_repo_dir, "LLM")
self.model, self.tokenizer = FastModel.from_pretrained(
@@ -1093,7 +1062,6 @@ class UnslothTrainer:
logger.info("Loaded Spark-TTS (bicodec) model")
elif self._audio_type == "dac":
- # OuteTTS: uses FastModel (not FastLanguageModel) with load_in_4bit=False
from unsloth import FastModel
self.model, self.tokenizer = FastModel.from_pretrained(
lookup_name,
@@ -1109,7 +1077,6 @@ class UnslothTrainer:
logger.info("Loaded OuteTTS (dac) model (FastModel)")
elif self.is_audio_vlm:
- # Audio VLM (e.g. Gemma 3N): FastModel returns (model, processor).
from unsloth import FastModel
self.model, self.tokenizer = FastModel.from_pretrained(
model_name = lookup_name,
@@ -1140,7 +1107,6 @@ class UnslothTrainer:
)
logger.info("Loaded vision model")
- # Did FastVisionModel return a Processor or a raw tokenizer?
from transformers import ProcessorMixin
tok = self.tokenizer
@@ -1184,7 +1150,6 @@ class UnslothTrainer:
return False
if full_finetuning:
- # All params trainable (else frozen)
self.model.for_training()
self._update_progress(status_message = "Model loaded successfully")
@@ -1196,8 +1161,8 @@ class UnslothTrainer:
if "could not get source code" in str(e) and not getattr(
self, "_source_code_retried", False
):
- # Unsloth patching can leave stale state that breaks inspect.getsource() when
- # switching model families (e.g. gemma3 -> gemma3n); the first failure clears it.
+ # Unsloth patching can leave stale state that breaks inspect.getsource() when switching model
+ # families; the first failure clears it.
self._source_code_retried = True
logger.info(f"\n'could not get source code' — retrying once...\n")
return self.load_model(
@@ -1238,7 +1203,6 @@ class UnslothTrainer:
except Exception as e:
self.model_load_error = e
error_msg = str(e)
- # Surface a friendly message for gated/auth errors
error_lower = error_msg.lower()
if any(
k in error_lower
@@ -1264,7 +1228,6 @@ class UnslothTrainer:
def prepare_model_for_training(
self,
use_lora: bool = True,
- # Vision-specific LoRA parameters (only used if is_vlm=True)
finetune_vision_layers: bool = True,
finetune_language_layers: bool = True,
finetune_attention_modules: bool = True,
@@ -1286,7 +1249,6 @@ class UnslothTrainer:
if self.model is None:
raise ValueError("Model not loaded. Call load_model() first.")
- # Full finetuning: skip PEFT
if not use_lora:
self._update_progress(status_message = "Full finetuning mode - no LoRA adapters")
logger.info("Full finetuning mode - training all parameters\n")
@@ -1311,7 +1273,6 @@ class UnslothTrainer:
"down_proj",
]
- # Normalize gradient_checkpointing to True, False, or "unsloth"
if isinstance(use_gradient_checkpointing, str):
use_gradient_checkpointing = use_gradient_checkpointing.strip().lower()
if use_gradient_checkpointing == "" or use_gradient_checkpointing == "unsloth":
@@ -1351,12 +1312,12 @@ class UnslothTrainer:
)
if self._audio_type in ("csm", "bicodec", "dac") or self.is_audio_vlm:
- # Use FastModel.get_peft_model (codec audio + audio VLM)
from unsloth import FastModel
label = self._audio_type or "audio_vlm"
logger.info(f"{label} LoRA configuration:")
logger.info(f" - Target modules: {target_modules}")
+ # Audio VLM models support VLM-style layer selection
if self.is_audio_vlm:
logger.info(f" - Finetune vision layers: {finetune_vision_layers}")
logger.info(f" - Finetune language layers: {finetune_language_layers}")
@@ -1377,7 +1338,6 @@ class UnslothTrainer:
use_dora = use_dora,
loftq_config = {"loftq_bits": 4, "loftq_iter": 1} if use_loftq else None,
)
- # Audio VLM models support VLM-style layer selection
if self.is_audio_vlm:
peft_kwargs.update(
finetune_vision_layers = finetune_vision_layers,
@@ -1514,7 +1474,7 @@ class UnslothTrainer:
CsmOutputWithPast,
)
- base_csm = self.model.base_model.model # CsmForConditionalGeneration
+ base_csm = self.model.base_model.model
# Original forward (@can_return_tuple wrapped version)
_original_forward = CsmForConditionalGeneration.forward
@@ -1681,8 +1641,8 @@ class UnslothTrainer:
trust_remote_code = getattr(self, "trust_remote_code", False),
)
- # Some fine-tuned models save pad_to_multiple_of in tokenizer_config.json and
- # _merge_kwargs leaks it into audio_kwargs, where EncodecFeatureExtractor rejects it.
+ # Some fine-tuned models save pad_to_multiple_of in tokenizer_config.json and _merge_kwargs leaks it
+ # into audio_kwargs, where EncodecFeatureExtractor rejects it.
processor.tokenizer.init_kwargs.pop("pad_to_multiple_of", None)
resolved = self._resolve_audio_columns(dataset, custom_format_mapping)
@@ -1802,10 +1762,8 @@ class UnslothTrainer:
f"Audio VLM dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
- # Needed by the collator closure
self._audio_vlm_audio_col = audio_col
- # Cast audio to 16kHz (standard for speech models)
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 16000))
def format_messages(samples):
@@ -1867,7 +1825,6 @@ class UnslothTrainer:
max_length = self.max_seq_length or 2048
tokenizer = self.tokenizer
- # Orpheus special token IDs (hardcoded in tokenizer vocabulary)
START_OF_HUMAN = 128259
END_OF_HUMAN = 128260
START_OF_AI = 128261
@@ -1907,7 +1864,6 @@ class UnslothTrainer:
snac_model = SNAC.from_pretrained(SNAC_MODEL_NAME)
snac_model = snac_model.to(device).eval()
- # Resample transform (created once)
resample_transform = (
T.Resample(orig_freq = ds_sample_rate, new_freq = SNAC_SAMPLE_RATE)
if ds_sample_rate != SNAC_SAMPLE_RATE
@@ -1939,7 +1895,6 @@ class UnslothTrainer:
skipped += 1
continue
- # Encode audio with SNAC (notebook 122-142)
waveform = (
torch.from_numpy(audio_data["array"]).unsqueeze(0).to(dtype = torch.float32)
)
@@ -1950,7 +1905,6 @@ class UnslothTrainer:
with torch.inference_mode():
codes = snac_model.encode(waveform)
- # Interleave 7 codes per frame with layer offsets (notebook 134-142)
all_codes = []
for i in range(codes[0].shape[1]):
all_codes.append(codes[0][0][i].item() + AUDIO_OFFSET)
@@ -1965,14 +1919,12 @@ class UnslothTrainer:
skipped += 1
continue
- # Dedup consecutive frames with same first code (notebook 185-207)
deduped = all_codes[:7]
for i in range(7, len(all_codes), 7):
if all_codes[i] != deduped[-7]:
deduped.extend(all_codes[i : i + 7])
all_codes = deduped
- # Build text tokens (notebook 217-224)
text_prompt = (
f"{example[speaker_col]}: {text}"
if has_source and example.get(speaker_col)
@@ -1981,7 +1933,6 @@ class UnslothTrainer:
text_ids = tokenizer.encode(text_prompt, add_special_tokens = True)
text_ids.append(END_OF_TEXT)
- # Build full input_ids (notebook 225-234)
input_ids = (
[START_OF_HUMAN]
+ text_ids
@@ -2073,8 +2024,8 @@ class UnslothTrainer:
f"BiCodec dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
- # Cast so datasets 4.x AudioDecoder objects decode to dicts. No resample here --
- # BiCodec's target_sr may differ; the loop does it.
+ # Cast so datasets 4.x AudioDecoder objects decode to dicts; no resample here, since BiCodec's
+ # target_sr may differ.
from datasets import Audio
dataset = dataset.cast_column(audio_col, Audio())
@@ -2278,7 +2229,6 @@ class UnslothTrainer:
f"DAC dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
- # Cast audio to 24kHz (notebook: cast_column("audio", Audio(sampling_rate=24000)))
from datasets import Audio
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = 24000))
@@ -2330,13 +2280,11 @@ class UnslothTrainer:
audio_array = np.array(audio_data["array"], dtype = np.float32)
sampling_rate = audio_data.get("sampling_rate", 24000)
- # Convert to WAV bytes (Whisper needs a file path)
buf = io.BytesIO()
sf.write(buf, audio_array, sampling_rate, format = "WAV", subtype = "FLOAT")
buf.seek(0)
audio_bytes = buf.getvalue()
- # 1. Get word timings from Whisper
with tempfile.NamedTemporaryFile(
suffix = ".wav",
delete = False,
@@ -2369,7 +2317,6 @@ class UnslothTrainer:
skipped += 1
continue
- # 2. Create speaker representation with AudioProcessor
speaker_data_dict = {
"audio": {"bytes": audio_bytes},
"text": normalized_transcript,
@@ -2380,7 +2327,6 @@ class UnslothTrainer:
skipped += 1
continue
- # 3. Get training prompt from PromptProcessor
prompt = prompt_processor.get_training_prompt(speaker)
if prompt:
processed_examples.append({"text": prompt})
@@ -2395,7 +2341,6 @@ class UnslothTrainer:
status_message = f"Preprocessing audio with OuteTTS... {idx + 1}/{len(dataset)}"
)
- # Free Whisper from GPU (notebook: whisper_model.to('cpu'))
logger.info("Moving Whisper model to CPU...\n")
whisper_model.to("cpu")
del whisper_model
@@ -2442,10 +2387,8 @@ class UnslothTrainer:
f"Whisper dataset needs 'audio' and 'text' columns, got: {dataset.column_names}"
)
- # Cast audio to 16kHz (Whisper's expected sample rate)
dataset = dataset.cast_column(audio_col, Audio(sampling_rate = WHISPER_SAMPLE_RATE))
- # Train/eval split (notebook does dataset.train_test_split)
eval_dataset_raw = None
if eval_split:
splits = dataset.train_test_split(test_size = 0.06, seed = 42)
@@ -2474,11 +2417,9 @@ class UnslothTrainer:
skipped += 1
continue
- # Extract audio features (notebook 112-115)
features = self.tokenizer.feature_extractor(
audio_data["array"], sampling_rate = audio_data["sampling_rate"]
)
- # Tokenize text (notebook 116)
tokenized_text = self.tokenizer.tokenizer(text)
processed.append(
@@ -2580,22 +2521,16 @@ class UnslothTrainer:
"""
from core.training.s3_dataset import S3DownloadCancelled
- # `datasets` exposes no env var, so its bars can only be quieted through the
- # class, and only once the module is in. It is (module-level import above), and
- # this is the first point before any load: the local-file and Hub branches below
- # both call load_dataset(), whose "Generating train split" / "Downloading data"
- # bars would otherwise write into the worker's structured stream. It also covers
- # the map/filter bars of every branch further down.
+ # datasets exposes no env var, so its bars can only be quieted through the class, and this is the
+ # first point before any load_dataset() in either branch.
try:
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
except Exception: # noqa: BLE001 - never let log tidying stop a run
pass
- # An Audio column decodes inside load_dataset() below, before the audio branches
- # that already call this. This worker starts without the shim the API process
- # installs, so the load raised `datasets`' own "please install 'torchcodec'".
- # A False here is still reported by those branches, naming FFmpeg.
+ # An Audio column decodes inside load_dataset() below, and this worker starts without the shim the
+ # API process installs, so the load raised datasets' own "please install torchcodec".
try:
ensure_audio_decoding()
except Exception: # noqa: BLE001 - never let a broken librosa stop a text run
@@ -2608,7 +2543,7 @@ class UnslothTrainer:
dataset = None
eval_dataset = None
dataset_attestation_source = None
- has_separate_eval_source = False # True if eval comes from a separate HF split
+ has_separate_eval_source = False
eval_enabled = eval_steps is not None and eval_steps > 0
raw_text_mode = is_cpt or format_type == "raw"
dataset_loaded_from_cache = False
@@ -2672,8 +2607,8 @@ class UnslothTrainer:
logger.info(f"Downloaded {len(local_datasets)} file(s) from S3\n")
if local_datasets:
- # load_dataset() gives an Arrow-backed result; in-memory Dataset.from_list() has no
- # cache and forces num_proc=1 during tokenization/map (sharding needs Arrow files).
+ # load_dataset() is Arrow-backed; an in-memory Dataset.from_list() has no cache and forces
+ # num_proc=1 during tokenization.
all_files = self._resolve_local_files(local_datasets)
if all_files:
@@ -2744,11 +2679,11 @@ class UnslothTrainer:
)
self._update_progress(status_message = f"Streaming {dataset_source}")
else:
- # Non-streaming: with a slice end, stream only the needed rows and materialize them
- # (avoids downloading everything); the eager [start, end] trim happens below.
+ # With a slice end, stream only the needed rows and materialize them rather than downloading
+ # everything.
_slice_start = dataset_slice_start or 0
- # streaming=True rejects HF slice syntax (e.g. "train[:50%]") with "Bad split", so
- # fall back to the regular download path when train_split already carries a slice.
+ # streaming=True rejects HF slice syntax with "Bad split", so fall back to the regular download path
+ # when train_split already carries a slice.
_split_has_slice = (train_split or "").find("[") != -1
dataset = None
if dataset_local_files_only:
@@ -2843,7 +2778,6 @@ class UnslothTrainer:
logger.info("Stopped during dataset loading\n")
return None
- # Resolve eval split from a separate HF split (explicit or auto)
if eval_enabled:
effective_train = train_split or "train"
if eval_split and eval_split != effective_train:
@@ -2855,8 +2789,9 @@ class UnslothTrainer:
eval_load_kwargs["revision"] = dataset_revision
if dataset_streaming:
- # Probe splits first: load_dataset(streaming=True) returns an IterableDataset without
- # validating the name, so a typo would only surface on the first eval batch.
+ # load_dataset(streaming=True) returns an IterableDataset without validating the
+ # name, so a typo
+ # would only surface on the first eval batch.
from datasets import get_dataset_split_names
probe_kwargs = {"path": dataset_source}
@@ -2880,8 +2815,9 @@ class UnslothTrainer:
f"{available_splits}"
)
eval_dataset = load_dataset(**eval_load_kwargs, streaming = True)
- # Streaming eval has no __len__; bound it so each evaluation terminates. .take()
- # stays lazy and survives the later format/raw-text .map() passes.
+ # Streaming eval has no __len__; .take() stays lazy and survives the later format
+ # and raw-text map()
+ # passes.
if not hasattr(eval_dataset, "__len__"):
eval_dataset = eval_dataset.take(STREAMING_EVAL_MAX_SAMPLES)
logger.info(
@@ -2938,7 +2874,6 @@ class UnslothTrainer:
"Streaming mode does not support using the same split for both train and eval. "
"Please provide a separate eval split or set eval_steps to 0."
)
- # Same split as training — split 80/20 after formatting
logger.info(
f"Eval split '{eval_split}' is the same as train split — will split 80/20\n"
)
@@ -2947,7 +2882,6 @@ class UnslothTrainer:
raise ValueError(
"Streaming mode currently requires an explicit eval split when evaluation is enabled."
)
- # Auto-detect eval split from HF (separate dataset or None)
if dataset_loaded_from_cache:
split_info = getattr(getattr(dataset, "info", None), "splits", None)
available_splits = list(split_info or ())
@@ -2993,8 +2927,8 @@ class UnslothTrainer:
if dataset is None:
raise ValueError("No dataset provided")
- # Eager-only index slicing (inclusive both ends). Streaming already sliced lazily via
- # skip()/take(); the non-streaming path fetched end+1 rows and is trimmed here.
+ # Streaming already sliced lazily via skip()/take(); the non-streaming path fetched end+1 rows and
+ # is trimmed here.
if (not dataset_streaming) and (
dataset_slice_start is not None or dataset_slice_end is not None
):
@@ -3011,10 +2945,9 @@ class UnslothTrainer:
status_message = f"Sliced dataset to {len(dataset)} rows (indices {start}-{end})"
)
- # Bound before the formatting/template/tokenization passes, which map over
- # every row. Skipped when streaming (bounded lazily above) or when the user
- # named an explicit range, including a bracketed split like train[1000:2000]:
- # those are already the rows they asked for, not a corpus to sample from.
+ # Bound before the formatting and tokenization passes, and skipped when streaming or when the user
+ # named an explicit range, including a bracketed split like train[1000:2000]: those rows are
+ # already what they asked for.
if (
(not dataset_streaming)
and dataset_slice_start is None
@@ -3040,19 +2973,13 @@ class UnslothTrainer:
if self.should_stop:
logger.info("Stopped before applying chat template\n")
+ # No separate HF eval split - caller handles programmatic splitting
return None
# ========== AUDIO MODELS: custom preprocessing ==========
- # An inconclusive probe must not read as "not an audio model": falling through
- # with an audio dataset lands on the text path, which fails much later with
- # "Could not auto-detect format mapping", a column-mapping complaint that says
- # nothing about the repo read that actually failed.
- # getattr, because this method is also driven against lightweight stand-ins; both
- # defaults leave the guard closed.
- # _is_dataset_audio arrives from the client and is true on a column-NAME match
- # alone, so _dataset_has_audio_column is the tiebreaker: only a positive "no audio
- # here" reopens the text path, unknown still refuses.
- # raw/CPT is exempt: it reads the text column and ignores any audio one by design.
+ # An inconclusive probe must not read as "not an audio model": falling through lands on the text
+ # path, which fails later with a column-mapping complaint. _dataset_has_audio_column is the
+ # tiebreaker because _is_dataset_audio is true on a column-NAME match alone; raw/CPT is exempt.
if (
not self._audio_type
and not getattr(self, "_audio_type_known", True)
@@ -3101,6 +3028,10 @@ class UnslothTrainer:
# ========== RAW TEXT BYPASS ==========
if raw_text_mode:
+ # Say which variable said so: a stale size from an earlier mpirun or an HPC container image
+ # reads as a multi-rank launch on a one-process machine, and the only symptom is this run being
+ # told it makes several passes. The row bound already trusts the same variables; what was
+ # missing was a way to see them.
logger.info(
f"{_raw_mode_label().capitalize()} mode: bypassing chat template, "
"using raw text\n"
@@ -3127,8 +3058,7 @@ class UnslothTrainer:
f"({eval_rows}) kept as raw text\n"
)
elif eval_enabled and not has_separate_eval_source and not dataset_streaming:
- # _resolve_eval_split_from_dataset does a train_test_split (needs len/random access).
- # Streaming always has a separate eval split (route-enforced), so non-streaming only.
+ # _resolve_eval_split_from_dataset does a train_test_split, which needs len and random access.
split_result = self._resolve_eval_split_from_dataset(dataset)
if split_result is not None:
train_portion, eval_dataset = split_result
@@ -3142,8 +3072,8 @@ class UnslothTrainer:
)
logger.info(f"Raw-text dataset ready ({n_display} samples)\n")
- # Streaming datasets can report column_names as None, which would make the `in` check
- # raise; resolve_column_names falls back to features/first-row probing.
+ # Streaming datasets can report column_names as None, which would make the `in` check raise;
+ # resolve_column_names falls back to features/first-row probing.
train_columns = resolve_column_names(train_dataset)
if "text" not in train_columns:
raise ValueError(f"Raw-text dataset missing 'text' column: {train_columns}")
@@ -3188,7 +3118,6 @@ class UnslothTrainer:
)
logger.info(f"Dataset formatted successfully ({final_n} samples, {detected})\n")
- # ========== THEN SPLIT ==========
if has_separate_eval_source and eval_dataset is not None:
eval_n = len(eval_dataset) if hasattr(eval_dataset, "__len__") else "?"
logger.info(f"Formatting eval dataset ({eval_n} rows)...\n")
@@ -3284,7 +3213,6 @@ class UnslothTrainer:
f"from the training data when enough rows are available: {e}"
)
- # No separate HF eval split — caller handles programmatic splitting
return None
def _resolve_eval_split_from_dataset(self, dataset) -> Optional[tuple]:
@@ -3352,9 +3280,9 @@ class UnslothTrainer:
self._update_progress(error = "Model not loaded")
return False
- # Pre-import heavy transformers modules on the main thread: Unsloth's patched_import
- # isn't thread-safe with importlib's cache (KeyError: 'size') from a worker thread.
- import transformers # noqa: F401 – ensures submodules are cached
+ # Pre-import heavy transformers modules on the main thread: Unsloth's patched_import is not thread-
+ # safe with importlib's cache.
+ import transformers # noqa: F401 - ensures submodules are cached
from transformers import ( # noqa: F401
Trainer as _HFTrainer,
TrainingArguments as _TrainingArguments,
@@ -3458,35 +3386,22 @@ class UnslothTrainer:
self._online_eval_dataset = eval_dataset
train_dataset = dataset["dataset"] if isinstance(dataset, dict) else dataset
- # A step-capped run says nothing about passes, but Unsloth knows the rows
- # and microbatch size, so resolve it here. World size scales rows consumed
- # per step; omitting it would understate the passes.
+ # A step-capped run says nothing about passes, and world size scales the rows consumed per step.
resolved_epochs = None
max_steps = int(config_args.get("max_steps") or 0)
if max_steps > 0:
try:
rows = len(train_dataset)
- # Every launcher variable, not WORLD_SIZE alone: no MPI sets it (Open
- # MPI uses OMPI_COMM_WORLD_SIZE, Hydra/Intel MPI PMI_SIZE, mlx.launch's
- # CUDA-only NCCL backend MLX_WORLD_SIZE), LOCAL_WORLD_SIZE is defensive
- # since torchrun sets both and the max prefers the global count. Reading
- # one variable calls an mpirun launch single-process and engages a view
- # that re-tokenizes every extra pass; it also raises on junk like
- # int("auto"), leaving resolved_epochs None so the gate reads inf and
- # kills the feature. Env-only, deliberately NOT worker.py's
- # _data_parallel_world_size, which also counts visible CUDA devices:
- # that bounds a row subset where over-counting is free, this feeds a
- # veto where both directions are wrong. Unsloth's multi-GPU load is a
- # sharding device_map, model-parallel to transformers with _n_gpu = 1,
- # so a sharded 4-GPU run draws one GPU's rows per step and counting
- # devices would report 4x the passes, vetoing a qualifying run.
+ # Every launcher variable, not WORLD_SIZE alone: Open MPI sets OMPI_COMM_WORLD_SIZE, Hydra/Intel
+ # MPI PMI_SIZE, mlx.launch's CUDA-only NCCL backend MLX_WORLD_SIZE, and LOCAL_WORLD_SIZE is
+ # defensive since torchrun sets both. Reading one variable calls an mpirun launch single-process
+ # or raises on junk like int("auto"), leaving resolved_epochs None. Env-only, deliberately NOT
+ # worker.py's _data_parallel_world_size, which also counts visible CUDA devices: Unsloth's
+ # multi-GPU load is a sharding device_map, model-parallel to transformers.
world_size = world_size_from_env()
if world_size > 1:
- # Say which variable said so: a stale size from an earlier mpirun
- # or an HPC container image reads as a multi-rank launch on a
- # one-process machine, and the only symptom is this run being told
- # it makes several passes. The row bound already trusts the same
- # variables; what was missing was a way to see them.
+ # Say which variable said so: a stale size from an earlier mpirun reads as a multi-rank launch, and
+ # this veto is the only symptom.
logger.info(
f"Launcher environment reports {world_size} data-parallel "
f"processes ({world_size_env_report()}); a step-capped run "
@@ -3537,10 +3452,9 @@ class UnslothTrainer:
try:
text_field = config_args.get("dataset_text_field", "text") or "text"
max_length = int(config_args.get("max_seq_length") or 2048)
- # The generated `__init__` clamps `max_seq_length` to the model's cap
- # and derives `max_length` from it, but runs after this. Apply the same
- # cap here or the transform truncates wider than the eager map it
- # replaces, and the attestation claims a width nothing enforced.
+ # The generated __init__ clamps max_seq_length to the model's cap and derives max_length from it,
+ # but runs after this: apply the same cap here or the transform truncates wider than the eager map
+ # it replaces.
model_cap = getattr(self.model, "max_seq_length", None)
try:
if model_cap is not None and 0 < int(model_cap) < max_length:
@@ -3566,9 +3480,8 @@ class UnslothTrainer:
if isinstance(dataset, dict):
dataset["dataset"] = view
if eval_dataset is not None:
- # Probe the eval split's own first row: TRL calls _prepare_dataset
- # once per split, so reusing the train answer would tokenize eval
- # differently whenever the splits disagree about a leading BOS.
+ # Probe the eval split's own first row: TRL calls _prepare_dataset once per split, so reusing the
+ # train answer tokenizes eval differently whenever the splits disagree about a leading BOS.
eval_add_special_tokens = resolve_add_special_tokens(
self.tokenizer, first_sample_text(eval_dataset, text_field)
)
@@ -3615,8 +3528,8 @@ class UnslothTrainer:
except Exception as exc: # noqa: BLE001 - cleanup must never fail a finished run
logger.warning(f"Online tokenization worker shutdown failed: {exc}")
return
- # `_online_prewarm_batches` is left alone: it records how the run was
- # configured and the A/B harness reads it back. A second call is a no-op.
+ # _online_prewarm_batches records how the run was configured and the A/B harness reads it back; a
+ # second call is a no-op.
if released:
logger.info(f"Online tokenization: shut down {released} DataLoader workers\n")
@@ -3638,8 +3551,7 @@ class UnslothTrainer:
if prewarm:
from utils.datasets.online_tokenization import memoize_train_dataloader
- # Hold the loader this barrier fills, or train() forks fresh workers
- # and everything drained here is wasted.
+ # Hold the loader this barrier fills, or train() forks fresh workers and everything drained here is wasted.
memoize_train_dataloader(self.trainer)
try:
loader = self.trainer.get_train_dataloader()
@@ -3649,9 +3561,9 @@ class UnslothTrainer:
try:
next(iterator)
except StopIteration:
- break # a short split simply prewarms fewer
- # Drop the local names: on the eager path that tears the loader down;
- # on the online path the memo still holds it, so the workers survive.
+ break
+ # Drop the local names: on the eager path that tears the loader down, while on the online path the
+ # memo still holds it so the workers survive.
del iterator, loader
except StopIteration:
return (
@@ -3671,7 +3583,7 @@ class UnslothTrainer:
except Exception:
input_ids = getattr(batch, "input_ids", None)
if input_ids is None:
- return None # some collators omit input_ids
+ return None
seq_len = input_ids.shape[-1] if input_ids.ndim > 0 else 0
if not (input_ids.is_floating_point() or input_ids.numel() == 0 or seq_len == 0):
@@ -3708,8 +3620,8 @@ class UnslothTrainer:
never passed as a bare ``IterableDataset``.
"""
try:
- # On spawn platforms, put compiled-cache dirs on sys.path/PYTHONPATH before any
- # dataset.map() so spawned workers can import e.g. UnslothSFTTrainer.
+ # On spawn platforms, put the compiled-cache dirs on sys.path/PYTHONPATH before any dataset.map()
+ # so spawned workers can import e.g. UnslothSFTTrainer.
if sys.platform in ("win32", "darwin"):
from utils.cache_cleanup import register_compiled_cache_on_path
register_compiled_cache_on_path()
@@ -3730,7 +3642,6 @@ class UnslothTrainer:
output_dir = str(resolve_output_dir(training_args.get("output_dir")))
ensure_dir(Path(output_dir))
- # ========== AUDIO TRAINER BRANCH ==========
if self._audio_type == "csm":
# CSM uses plain HF Trainer with remove_unused_columns=False for the depth decoder.
from transformers import Trainer as HFTrainer, TrainingArguments
@@ -3750,8 +3661,10 @@ class UnslothTrainer:
args = TrainingArguments(**config),
)
self.trainer.add_callback(self._create_progress_callback())
- # Unsloth publishes progress itself, so HF's stdout callbacks are pure
- # duplication in a log that has no terminal. --verbose keeps them.
+ # Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that has
+ # no terminal; --verbose keeps them.
+ # Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that
+ # has no terminal. --verbose keeps them.
_drop_hf_stdout_callbacks(self.trainer)
batch_size = training_args.get("batch_size", 2)
@@ -3771,8 +3684,7 @@ class UnslothTrainer:
return
elif self._audio_type == "snac":
- # Orpheus: LM with SNAC codec tokens -- plain HF Trainer. DataCollatorForSeq2Seq
- # pads variable-length sequences per batch and pads labels with -100.
+ # DataCollatorForSeq2Seq pads variable-length sequences per batch and pads labels with -100.
from transformers import (
Trainer as HFTrainer,
TrainingArguments,
@@ -3791,8 +3703,6 @@ class UnslothTrainer:
),
)
self.trainer.add_callback(self._create_progress_callback())
- # Unsloth publishes progress itself, so HF's stdout callbacks are pure
- # duplication in a log that has no terminal. --verbose keeps them.
_drop_hf_stdout_callbacks(self.trainer)
batch_size = training_args.get("batch_size", 2)
@@ -3812,7 +3722,6 @@ class UnslothTrainer:
return
elif self._audio_type == "whisper":
- # Whisper: Seq2SeqTrainer with custom speech collator
from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments
from utils.datasets import DataCollatorSpeechSeq2SeqWithPadding
@@ -3838,8 +3747,6 @@ class UnslothTrainer:
self.trainer = Seq2SeqTrainer(**trainer_kwargs)
self.trainer.add_callback(self._create_progress_callback())
- # Unsloth publishes progress itself, so HF's stdout callbacks are pure
- # duplication in a log that has no terminal. --verbose keeps them.
_drop_hf_stdout_callbacks(self.trainer)
batch_size = training_args.get("batch_size", 2)
@@ -3882,7 +3789,6 @@ class UnslothTrainer:
data_collator = None
if is_deepseek_ocr:
- # DeepSeek OCR collator - auto-install if needed
logger.info("Detected DeepSeek OCR model\n")
if not _ensure_deepseek_ocr_installed():
error_msg = (
@@ -3900,8 +3806,8 @@ class UnslothTrainer:
logger.info("Configuring DeepSeek OCR data collator...\n")
FastVisionModel.for_training(self.model)
- # (image_size, base_size, crop_mode) is a coupled preset; changing image_size alone
- # desyncs the per-crop grid from num_queries. Use the Gundam preset.
+ # (image_size, base_size, crop_mode) is a coupled preset: changing image_size alone desyncs the per-
+ # crop grid from num_queries.
if training_args.get("vision_image_size") is not None:
logger.info(
"Vision image resize ignored for DeepSeek OCR "
@@ -3926,7 +3832,7 @@ class UnslothTrainer:
elif self.is_audio_vlm and not raw_text_mode:
# Audio VLM collator (e.g. Gemma 3N), mirrors the Gemma3N_(4B)-Audio notebook.
logger.info("Configuring audio VLM data collator...\n")
- processor = self.tokenizer # FastModel returns processor as tokenizer
+ processor = self.tokenizer
audio_col_name = getattr(self, "_audio_vlm_audio_col", "audio")
@@ -3944,7 +3850,6 @@ class UnslothTrainer:
batch = processor(text = texts, audio = audios, return_tensors = "pt", padding = True)
- # Labels = input_ids with special tokens masked
labels = batch["input_ids"].clone()
labels[labels == processor.tokenizer.pad_token_id] = -100
for attr in (
@@ -4003,11 +3908,9 @@ class UnslothTrainer:
"report_to": _build_report_targets(training_args),
"disable_tqdm": _hf_stdout_progress_disabled(),
"include_num_input_tokens_seen": True,
- # serial_as_none = False: this is a config boundary, not a map() call site. The audio
- # paths ask for 1 to keep dataset workers off a process holding audio/CUDA state, and
- # a config None reads as "auto-size me" downstream. Pass None (not a count) otherwise:
- # the shared policy sizes an auto request from CPU affinity + cgroup quota, while an
- # integer computed here comes from host os.cpu_count() and skips that.
+ # serial_as_none = False: this is a config boundary, not a map() call site. The audio paths ask
+ # for 1 to keep dataset workers off a process holding audio/CUDA state; pass None otherwise, so
+ # the shared policy sizes it from CPU affinity and cgroup quota rather than host os.cpu_count().
"dataset_num_proc": dataset_map_num_proc(
1 if (self.is_audio or self.is_audio_vlm or self._cuda_audio_used) else None,
serial_as_none = False,
@@ -4051,7 +3954,6 @@ class UnslothTrainer:
else:
logger.info(f"Training for {config_args['num_train_epochs']} epochs\n")
- # ========== EVAL CONFIGURATION ==========
eval_dataset = training_args.get("eval_dataset", None)
eval_steps_val = training_args.get("eval_steps", 0.00)
if eval_dataset is not None:
@@ -4076,13 +3978,12 @@ class UnslothTrainer:
else:
logger.info("No eval dataset — evaluation disabled\n")
- # Use training_args optim/lr_scheduler_type if given, else defaults
optim_value = training_args.get("optim", "adamw_8bit")
lr_scheduler_type_value = training_args.get("lr_scheduler_type", "linear")
if (self.is_vlm or self.is_audio_vlm) and not raw_text_mode:
- # Vision / audio VLM both need skip_prepare_dataset + remove_unused_columns;
- # raw-text VLM goes to the text path below.
+ # Vision and audio VLM both need skip_prepare_dataset + remove_unused_columns; raw-text VLM goes to
+ # the text path below.
label = "audio VLM" if self.is_audio_vlm else "vision"
logger.info(f"Configuring {label} model training parameters\n")
optim_value = training_args.get("optim", "adamw_torch_fused")
@@ -4131,7 +4032,7 @@ class UnslothTrainer:
f"Sequence packing: {'enabled' if packing_enabled else 'disabled'}\n"
)
- # Audio codec overrides — BiCodec/DAC use the text SFTTrainer path
+ # Audio codec overrides - BiCodec/DAC use the text SFTTrainer path
if self._audio_type == "bicodec":
config_args["packing"] = False
logger.info("Applied BiCodec overrides: packing=False\n")
@@ -4139,9 +4040,8 @@ class UnslothTrainer:
config_args["packing"] = False
logger.info("Applied DAC overrides: packing=False\n")
- # ========== ONLINE (OVERLAPPED) TOKENIZATION ==========
- # Plain-text single-pass runs tokenize in the DataLoader workers
- # instead of a blocking .map(); everything else stays eager.
+ # Plain-text single-pass runs tokenize in the DataLoader workers instead of a blocking .map();
+ # everything else stays eager.
self._online_prewarm_batches = 0
online_decision = self._configure_online_tokenization(
config_args = config_args,
@@ -4160,8 +4060,9 @@ class UnslothTrainer:
logger.info("Training configuration prepared\n")
# ========== TRAINER INITIALIZATION ==========
if self.is_audio_vlm and not raw_text_mode:
- # Audio VLM (e.g. Gemma 3N + audio): raw Dataset from _format_audio_vlm_dataset.
- # Notebook uses processing_class=processor.tokenizer; raw-text runs use the text path.
+ # Image VLM: dict wrapper from format_and_template_dataset (raw-text uses the text path).
+ # Audio VLM (e.g. Gemma 3N + audio): raw Dataset from _format_audio_vlm_dataset. Notebook uses
+ # processing_class=processor.tokenizer; raw-text runs use the text path.
train_dataset = dataset["dataset"] if isinstance(dataset, dict) else dataset
processing_class = (
self.tokenizer.tokenizer
@@ -4179,7 +4080,6 @@ class UnslothTrainer:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = SFTTrainer(**trainer_kwargs)
elif self.is_vlm and not raw_text_mode:
- # Image VLM: dict wrapper from format_and_template_dataset (raw-text uses the text path).
train_dataset = dataset["dataset"] if isinstance(dataset, dict) else dataset
trainer_kwargs = {
"model": self.model,
@@ -4192,8 +4092,10 @@ class UnslothTrainer:
trainer_kwargs["eval_dataset"] = eval_dataset
self.trainer = SFTTrainer(**trainer_kwargs)
else:
- # For text-only, unwrap a Processor (Gemma-3 returns ProcessorMixin even for text) to
- # the raw tokenizer; else SFTTrainer sets _is_vlm, skips _prepare_dataset, no input_ids.
+ # Unwrap a Processor for text-only (Gemma-3 returns ProcessorMixin even for text), else SFTTrainer
+ # sets _is_vlm, skips _prepare_dataset and produces no input_ids.
+ # For text-only, unwrap a Processor (Gemma-3 returns ProcessorMixin even for text) to the raw
+ # tokenizer; else SFTTrainer sets _is_vlm, skips _prepare_dataset, no input_ids.
from transformers import ProcessorMixin
sft_tokenizer = self.tokenizer
@@ -4254,8 +4156,7 @@ class UnslothTrainer:
self.trainer.processing_class = self.tokenizer
logger.info("Trainer initialized\n")
- # ========== TRAIN ON RESPONSES ONLY ==========
- # Raw-text datasets always train on all tokens.
+ # ========== TRAIN ON RESPONSES ONLY ========== Raw-text datasets always train on all tokens.
is_cpt = training_args.get("is_cpt", False)
train_on_responses_enabled = (
False
@@ -4270,10 +4171,8 @@ class UnslothTrainer:
"Raw-text mode: skipping train_on_responses_only — training on all tokens\n"
)
- # DeepSeek OCR handles this internally in its collator, so skip
- # Audio VLM handles label masking in its collator, so skip
- # Markers auto-detected from the chat template first, manual table as fallback;
- # gpt-oss stays on manual markers. See apply_completion_masking.
+ # Markers auto-detected from the chat template first, with the manual table as fallback; gpt-oss
+ # stays on manual markers. See apply_completion_masking.
if (
train_on_responses_enabled
and not self.is_audio_vlm
@@ -4290,8 +4189,8 @@ class UnslothTrainer:
else:
logger.info(f"{message}\n")
- # No try/except: the helper handles detection failures itself, so an exception here is
- # a real masking failure that must fail the run, not silently train full sequences.
+ # No try/except: the helper handles detection failures itself, so an exception here is a real
+ # masking failure that must fail the run, not silently train full sequences.
self.trainer, masking_applied = apply_completion_masking(
self.trainer,
self.model_name,
@@ -4306,10 +4205,9 @@ class UnslothTrainer:
if masking_applied:
try:
- # Safety net: train_on_responses_only masks non-response tokens with -100, and a row
- # becomes all -100 (Unsloth drops it) when the response template is missing from the
- # formatted text -- usually a dataset/template mismatch, sometimes max_seq_length
- # truncation. len()-based, so skip for streaming.
+ # Safety net: train_on_responses_only masks non-response tokens with -100, and a row becomes all
+ # -100 (Unsloth drops it) when the response template is missing from the formatted text;
+ # len()-based, so skipped for streaming.
if detect_streaming_dataset(self.trainer.train_dataset):
logger.info("Skipping post-filter length check for streaming dataset\n")
else:
@@ -4359,10 +4257,11 @@ class UnslothTrainer:
else:
logger.info("Training on full sequences (including prompts)\n")
- # ========== PROGRESS TRACKING ==========
self.trainer.add_callback(self._create_progress_callback())
- # Unsloth publishes progress itself, so HF's stdout callbacks are pure
- # duplication in a log that has no terminal. --verbose keeps them.
+ # Unsloth publishes progress itself, so HF's stdout callbacks duplicate a log that has no terminal;
+ # --verbose keeps them.
+ # Unsloth publishes progress itself, so HF's stdout callbacks are pure duplication in a log that has
+ # no terminal. --verbose keeps them.
_drop_hf_stdout_callbacks(self.trainer)
train_dataset_obj = dataset["dataset"] if isinstance(dataset, dict) else dataset
@@ -4398,8 +4297,8 @@ class UnslothTrainer:
)
self._update_progress(total_steps = total_steps)
- # ========== START TRAINING ==========
# Fail fast on an invalid first batch (empty/float input_ids) vs a step-1 crash.
+ # ========== START TRAINING ==========
preflight_error = self._preflight_first_batch()
if preflight_error:
logger.error(preflight_error)
@@ -4413,9 +4312,8 @@ class UnslothTrainer:
resume_from_checkpoint = training_args.get("resume_from_checkpoint")
)
finally:
- # Before _finalize_training, not after: merging and exporting are
- # where the memory goes, and the workers still hold a fork of this
- # process.
+ # Before _finalize_training: merging and exporting are where the memory goes, and the workers still
+ # hold a fork of this process.
self._release_online_dataloader()
# ========== SAVE MODEL ==========
@@ -4429,8 +4327,8 @@ class UnslothTrainer:
self._update_progress(is_training = False, error = str(e))
finally:
- # Backstop for returns that never reach train(), notably a preflight
- # error: that path has already forked the workers.
+ # Backstop for returns that never reach train(), notably a preflight error, which has already forked
+ # the workers.
self._release_online_dataloader()
self.is_training = False
@@ -4520,9 +4418,8 @@ def _ensure_deepseek_ocr_installed():
import os
script_dir = os.path.dirname(os.path.abspath(__file__))
- parent_dir = os.path.dirname(script_dir) # project root
+ parent_dir = os.path.dirname(script_dir)
- # Download to project root as 'deepseek_ocr' folder
local_dir = os.path.join(parent_dir, "deepseek_ocr")
snapshot_download("unsloth/DeepSeek-OCR", local_dir = local_dir, local_dir_use_symlinks = False)
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index 6ba57916778..fec2390b36c 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -49,13 +49,12 @@ def _env_int(name: str, default: int) -> int:
return default
-# Stop-watchdog escalation timeouts. Primary trigger: a short grace once "complete" (save
-# done). The absolute cap is a backstop: long for save=True, shorter for a cancel.
+# Primary trigger is a short grace once "complete" (save done); the absolute cap is a backstop, long
+# for save=True and shorter for a cancel.
_STOP_GRACE_S = _env_int("UNSLOTH_STUDIO_TRAINING_STOP_GRACE_S", 15)
_STOP_TIMEOUT_S = _env_int("UNSLOTH_STUDIO_TRAINING_STOP_TIMEOUT_S", 600)
_CANCEL_TIMEOUT_S = _env_int("UNSLOTH_STUDIO_TRAINING_CANCEL_TIMEOUT_S", 120)
-# Backstop to reclaim the GPU from a worker wedged in teardown. Generous: is_run_finished
-# already unwedges the UI, and a post-run wandb sync can legitimately take a while.
+# Generous: is_run_finished already unwedges the UI, and a post-run wandb sync can legitimately take a while.
_COMPLETE_EXIT_GRACE_S = _env_int("UNSLOTH_STUDIO_TRAINING_COMPLETE_EXIT_GRACE_S", 120)
# A few short retries so a transient SQLite lock doesn't lose the terminal state.
@@ -105,7 +104,7 @@ def _load_pyplot():
try:
import matplotlib
- matplotlib.use("Agg") # headless backend
+ matplotlib.use("Agg")
import matplotlib.pyplot as plt
_pyplot = plt
@@ -309,9 +308,9 @@ def _sanitize_db_config(config: dict[str, Any]) -> dict[str, Any]:
_MODEL_SNAPSHOT_METADATA = ("config.json", "adapter_config.json")
-# refs/main can point at a revision that only ever fetched metadata, so prefer a snapshot
-# that actually carries weights. Keep in step with _MODEL_WEIGHT_CANDIDATES in
-# routes/training.py: selecting a snapshot the start route rejects reproduces the 400.
+# refs/main can point at a revision that only ever fetched metadata, so prefer a snapshot that
+# carries weights. Keep in step with _MODEL_WEIGHT_CANDIDATES in routes/training.py: selecting a
+# snapshot the start route rejects reproduces the 400.
_MODEL_SNAPSHOT_WEIGHTS = (
"model.safetensors",
"model.safetensors.index.json",
@@ -344,8 +343,8 @@ def _resolve_model_snapshot(model_name: str, local_path: Optional[str]) -> Optio
repo_id = canonical_model_repo_id(model_name)
metadata_names = _with_load_subdirs(model_name, _MODEL_SNAPSHOT_METADATA)
weight_names = _with_load_subdirs(model_name, _MODEL_SNAPSHOT_WEIGHTS)
- # Pass 1 demands metadata AND weights, so neither a metadata-only refs/main nor a
- # weights-only fetch displaces a complete sibling. Pass 2 keeps the old metadata-only path.
+ # Pass 1 demands metadata AND weights, so neither a metadata-only refs/main nor a weights-only fetch
+ # displaces a complete sibling; pass 2 keeps the old metadata-only path.
passes: tuple[dict[str, Any], ...] = (
{"required_groups": (metadata_names, weight_names)},
{"metadata_filenames": metadata_names},
@@ -626,9 +625,8 @@ class TrainingProgress:
eval_loss: Optional[float] = None
peak_memory_gb: Optional[float] = None
output_dir: Optional[str] = None
- # Set on the end-of-run record HF emits after leaving the training loop. It has no
- # step loss, so the progress filter would drop it, and with it the only elapsed
- # time that includes the final evaluation, checkpoint save and best-model reload.
+ # The end-of-run record has no step loss, so the progress filter would drop it, and with it the only
+ # elapsed time that includes the final evaluation, checkpoint save and best-model reload.
is_run_summary: bool = False
@@ -800,10 +798,8 @@ class _MLXTrainerAdapter:
max_train_rows: Optional[int] = None,
max_train_rows_seed: int = 3407,
) -> Optional[tuple]:
- # Signature must match UnslothTrainer, which hands back this adapter on an
- # MLX host. The MLX worker loads its own data and derives the row bound from
- # its config, so the two bound arguments are accepted and deliberately not
- # forwarded: a copy here would be a second source of truth.
+ # Signature must match UnslothTrainer: the MLX worker loads its own data and derives the row bound
+ # from its config, so the two bound arguments are accepted and deliberately not forwarded.
self._dataset_config = {
"hf_dataset": dataset_source or "",
"local_datasets": local_datasets,
@@ -1097,8 +1093,7 @@ class TrainingBackend:
self._provenance_lock = threading.Lock()
self._run_intent_lock = threading.RLock()
- # Stop watchdog: escalates to force_terminate() if the worker doesn't exit in time. The
- # watched proc is tracked so a new run always gets its own watcher.
+ # The watched proc is tracked so a new run always gets its own watchdog.
self._stop_watchdog: Optional[threading.Thread] = None
self._stop_watchdog_proc: Optional[mp.Process] = None
self._complete_seen = threading.Event()
@@ -1112,8 +1107,8 @@ class TrainingBackend:
# Throttled training-status logging to the server log (not one line/step).
self._last_progress_log_ts: float = 0.0
self._last_progress_log_step: int = -1
- # (elapsed_seconds, num_tokens) at the previous logged line, so the next one
- # can report throughput over the interval between them.
+ # (elapsed_seconds, num_tokens) at the previous logged line, so the next one reports throughput over
+ # the interval between them.
self._last_progress_log_elapsed: Optional[float] = None
self._last_progress_log_tokens: Optional[int] = None
@@ -1155,8 +1150,6 @@ class TrainingBackend:
logger.info("TrainingBackend initialized (subprocess mode)")
- # --- Public API (called by routes/training.py) ---
-
def reserve_start_request(
self, start_request_id: str, job_id: str
) -> tuple[str, TrainingStartRequestRecord]:
@@ -1299,8 +1292,8 @@ class TrainingBackend:
return "cancelled", existing
if existing.state == "pending" and not owns_current:
- # Not the active run, so it does not get the owner's over-cap slot: start
- # plus cancel could otherwise be repeated to grow the table without bound.
+ # Not the active run, so it does not get the owner's over-cap slot: start plus cancel could
+ # otherwise be repeated to grow the table without bound.
self._reserve_start_cancel_tombstone_locked(start_request_id)
cancelled = replace(
existing,
@@ -1499,10 +1492,8 @@ class TrainingBackend:
raise TrainingStartCancellationCapacityError(
"Too many training start cancellations are pending"
)
- # Everything left is unexpired (pruned above), so evicting one would forget a
- # live cancellation and let its delayed /start spawn the job we just cancelled.
- # Overshoot instead: only the owner of the active start reaches this, and there
- # is at most one of those, so the table stays bounded.
+ # Everything left is unexpired, so evicting one would forget a live cancellation and let its delayed
+ # /start spawn the job just cancelled.
self._start_cancel_tombstone_reservations[start_request_id] = 1
return True
@@ -1543,8 +1534,8 @@ class TrainingBackend:
start_request_id: Optional[str] = None,
**kwargs,
) -> bool:
- # Reserve before lifecycle locking and synchronous validation: routes call start_training
- # from worker threads, so this compare-and-set stops two requests reaching the spawn.
+ # Reserve before lifecycle locking and validation: routes call start_training from worker threads,
+ # so this compare-and-set stops two requests reaching the spawn.
with self._new_job_spawn_reservation(job_id) as spawn_reserved:
if not spawn_reserved:
logger.warning("Training subprocess already running")
@@ -1606,7 +1597,7 @@ class TrainingBackend:
logger.warning("Training subprocess already running")
return False
- # Join prior pump thread — refuse to start if it won't die
+ # Wait for pump thread to finish DB finalization (8s covers SQLite's 5s lock timeout).
if self._pump_thread is not None and self._pump_thread.is_alive():
self._pump_thread.join(timeout = 5.0)
if self._pump_thread.is_alive():
@@ -1623,11 +1614,9 @@ class TrainingBackend:
initialize_resource_provenance(config)
- # Split GPU validation from placement around the VRAM hook:
- # * Explicit gpu_ids are validated here (raises -> the route 400s before any teardown)
- # and their placement is VRAM-independent, so it survives the hook freeing memory.
- # * Auto-selection ranks GPUs by *free* VRAM, so it is deferred until after the hook,
- # else it could pin training onto a GPU the hook is about to clear.
+ # Explicit gpu_ids are validated here, so the route 400s before any teardown and their placement
+ # survives the VRAM hook; auto-selection ranks GPUs by FREE VRAM, so it is deferred until after
+ # the hook, else it pins training onto a GPU the hook is about to clear.
from utils.hardware import hardware as _hw
gpu_ids = kwargs.get("gpu_ids")
@@ -1690,8 +1679,7 @@ class TrainingBackend:
start_request_id,
)
return False
- # Synchronous validation passed -> free VRAM (export + chat) before auto-selection and the
- # spawn. After the handshake, so a lost race can't tear down chat for a run that never spawns.
+ # Free VRAM after the handshake, so a lost race cannot tear down chat for a run that never spawns.
if before_spawn is not None:
try:
before_spawn()
@@ -1747,7 +1735,7 @@ class TrainingBackend:
self.current_job_id = job_id
self.current_start_request_id = start_request_id
try:
- adopt_pid(proc.pid) # bind to parent lifetime (Windows job / sweep)
+ adopt_pid(proc.pid)
except Exception:
logger.error(
"Failed to adopt training subprocess; terminating it",
@@ -1813,7 +1801,7 @@ class TrainingBackend:
self._metric_buffer.clear()
self._run_finalized = False
self._db_run_created = False
- self._db_create_in_progress = False # a stale watchdog create can't block this run
+ self._db_create_in_progress = False
self._db_total_steps_set = False
self._db_config = _sanitize_db_config(config)
self._db_started_at = datetime.now(timezone.utc).isoformat()
@@ -1824,8 +1812,8 @@ class TrainingBackend:
self._xet_fallback_used = False
self._needs_xet_respawn = False
- # Create the DB run row before the pump consumes events, so it appears in history during
- # model loading and a fast terminal worker can't race the pump. From here the pump finalizes.
+ # Create the DB run row before the pump consumes events, so it appears in history during model
+ # loading and a fast terminal worker cannot race the pump.
self._ensure_db_run_created()
if resume_source_run_id and not self._db_run_created:
if proc.is_alive():
@@ -1846,6 +1834,7 @@ class TrainingBackend:
self._stop_queue = stop_queue
self._proc = proc
self._pump_thread = new_pump
+ # Start under the lock so a concurrent _ensure_pump_alive can't spawn yet another pump.
new_pump.start()
if self._new_job_spawn_id == job_id:
self._spawn_in_progress = False
@@ -1915,9 +1904,8 @@ class TrainingBackend:
with self._lock:
if self.current_job_id != run_id:
return False
- # The pump can finish the run between the route's terminal check and this
- # lock, so re-test: latching _should_stop after the fact would report a
- # saved run as stopped for good.
+ # The pump can finish the run between the route's terminal check and this lock, so re-test: latching
+ # _should_stop after the fact would report a saved run as stopped for good.
if save and self._run_finished_locked():
return False
if save or not run_id:
@@ -1955,11 +1943,9 @@ class TrainingBackend:
expected_job_id is not None and self.current_job_id != expected_job_id
):
return "superseded"
- # An unscoped reset cannot prove it means THIS run, so it never force-
- # terminates: _cancel_requested is cleared after current_job_id is set, so a
- # bodyless reset landing in that window would kill the run that just started.
- # A stop already asked for means the pre-rework cancel-then-dismiss flow, so
- # let it clear its UI; otherwise this is a stale reset of a live run (409).
+ # An unscoped reset cannot prove it means THIS run, so it never force-terminates: _cancel_requested
+ # is cleared after current_job_id is set, so a bodyless reset landing in that window would kill the
+ # run that just started. Otherwise this is a stale reset of a live run (409).
if expected_job_id is None and is_active:
return "superseded" if self._cancel_requested else "active"
cancel_requested = self._cancel_requested
@@ -2115,10 +2101,10 @@ class TrainingBackend:
re-guarded on target_proc so a run that did replace the worker keeps its handle."""
with self._lock:
if target_proc is not None and self._proc is not target_proc:
- return # a new run replaced the worker; never touch its state
+ return
if watched_job_id is not None and self.current_job_id != watched_job_id:
- return # a new run is already starting up; leave its state alone
- run_id = self.current_job_id # == watched_job_id
+ return
+ run_id = self.current_job_id
self._progress.is_training = False
terminal_payload = self._terminal_finalize_kwargs()
status = terminal_payload["status"]
@@ -2134,8 +2120,8 @@ class TrainingBackend:
elif status != "completed":
self._progress.status_message = "Training stopped."
# A completed run keeps its message; reaping a wedged worker must not relabel it.
- # Create the row if a start-time create failed (no-op otherwise; skips when the pump
- # is mid-create, in which case its create-then-finalize records the run instead).
+ # Create the row if a start-time create failed; skipped while the pump is mid-create, whose create-
+ # then-finalize records the run instead.
self._ensure_db_run_created()
with self._provenance_lock:
with self._lock:
@@ -2151,7 +2137,7 @@ class TrainingBackend:
if clear_output_dir:
self._output_dir = self._progress.output_dir = None
if claim:
- self._run_finalized = True # claim this run's finalize
+ self._run_finalized = True
batch = list(self._metric_buffer)
del self._metric_buffer[: len(batch)]
final_step = self._progress.step
@@ -2253,7 +2239,7 @@ class TrainingBackend:
with self._lock:
proc = self._proc
if target_proc is not None and proc is not target_proc:
- return # superseded by a new run; do not touch the new worker
+ return
if proc is not None and proc.is_alive():
logger.info("Force-terminating training subprocess (pid=%s)", proc.pid)
proc.terminate()
@@ -2309,10 +2295,8 @@ class TrainingBackend:
if proc is not None and proc.is_alive():
proc.terminate()
if not recover:
- # terminate() is only a request: arm the same backstop as the other terminal
- # paths so a worker that ignores it cannot hold the GPU for good. Signal first,
- # since arming no-ops when a watchdog already watches this proc and only this
- # drops it to the grace.
+ # terminate() is only a request, so arm the same backstop as the other terminal paths. Signal first,
+ # since arming no-ops when a watchdog already watches this proc.
self._complete_seen.set()
self._start_stop_watchdog(
cancel = False,
@@ -2433,7 +2417,7 @@ class TrainingBackend:
new_proc.start()
from utils.process_lifetime import adopt_pid
- adopt_pid(new_proc.pid) # bind to parent lifetime (Windows job / sweep)
+ adopt_pid(new_proc.pid)
except Exception:
logger.error("Failed to respawn training subprocess", exc_info = True)
self._spawn_in_progress = False
@@ -2490,7 +2474,6 @@ class TrainingBackend:
)
new_pump = threading.Thread(target = self._pump_loop, daemon = True)
self._pump_thread = new_pump
- # Start under the lock so a concurrent _ensure_pump_alive can't spawn yet another pump.
new_pump.start()
return True
@@ -2509,7 +2492,7 @@ class TrainingBackend:
Status and progress read this so a finished run reports terminal at once; the GPU
admission guards keep using is_training_active(), since a lingering worker holds VRAM."""
if getattr(self, "_spawn_in_progress", False):
- return False # a new run is spawning; _progress is still the old run's
+ return False
with self._lock:
return self._run_finished_locked()
@@ -2599,8 +2582,6 @@ class TrainingBackend:
return self._create_loss_plot(progress, self.current_theme)
return None
- # --- Compatibility shims: routes/training.py accesses these ---
-
class _TrainerShim:
"""Minimal shim so routes that access backend.trainer.* still work."""
@@ -2630,8 +2611,6 @@ class TrainingBackend:
"""Compatibility shim for routes that access backend.trainer.*"""
return self._TrainerShim(self)
- # --- Event pump (background thread) ---
-
def _safe_handle_event(self, event: dict) -> None:
"""Apply one event, swallowing any handler error.
@@ -2673,8 +2652,8 @@ class TrainingBackend:
self._safe_handle_event(event)
continue
- # Snapshot: the watchdog drops _proc last, so a re-read can hit None and kill
- # this thread. A dropped handle means it already finalized.
+ # Snapshot: the watchdog drops _proc last, so a re-read can hit None and kill this thread; a dropped
+ # handle means it already finalized.
proc = self._proc
if proc is None:
self._pump_running = False
@@ -2687,8 +2666,8 @@ class TrainingBackend:
for e in self._drain_queue(self._event_queue):
self._safe_handle_event(e)
- # Model-load stall: respawn over HTTP instead of finalizing as failure. Starts a fresh pump
- # on this thread (no self-join); it takes over _pump_running, so this exit leaves it set.
+ # Model-load stall: respawn over HTTP instead of finalizing as failure. The fresh pump takes over
+ # _pump_running, so this exit leaves it set.
with self._lock:
needs_xet_respawn = self._needs_xet_respawn
self._needs_xet_respawn = False
@@ -2698,7 +2677,6 @@ class TrainingBackend:
):
return
- # Mark done if no explicit complete/error was received.
with self._lock:
if self._progress.is_training:
if self._should_stop:
@@ -2725,8 +2703,8 @@ class TrainingBackend:
return
def _has_current_resume_checkpoint(self, output_dir, step) -> bool:
- # A valid checkpoint at the current step means the stop-and-save landed on
- # disk even if the worker died before confirming it.
+ # A valid checkpoint at the current step means the stop-and-save landed on disk even if the worker
+ # died before confirming it.
if not output_dir or not isinstance(step, int) or step <= 0:
return False
from core.training.resume import get_resume_checkpoint_path
@@ -2846,8 +2824,8 @@ class TrainingBackend:
_safe_loss = None
_loss_is_nonfinite = _safe_loss is not None and not math.isfinite(_safe_loss)
if _loss_is_nonfinite:
- # Drop the value rather than laundering it back to the last finite loss; clients see
- # loss=None at this step so the NaN is not hidden. Training continues.
+ # Drop the value rather than laundering it back to the last finite loss: clients see loss=None at
+ # this step so the NaN is not hidden.
_safe_loss = None
if not getattr(self._progress, "_nonfinite_loss_warned", False):
self._progress._nonfinite_loss_warned = True
@@ -2890,9 +2868,8 @@ class TrainingBackend:
step = event.get("step", 0)
loss = _safe_loss
lr = _safe_lr
- # Only ever move forward. HF can log more than one record at the same
- # global_step around the end of a run, and each one used to add another
- # point, so a 30-step run charted 33 with the last few stacked on step 30.
+ # Only ever move forward: HF can log more than one record at the same global_step around the end of
+ # a run, so a 30-step run charted 33 points.
_last_step = self.step_history[-1] if self.step_history else None
if step > 0 and loss is not None and (_last_step is None or step > _last_step):
self.loss_history.append(loss)
@@ -3021,8 +2998,7 @@ class TrainingBackend:
elif etype == "error":
self._progress.is_training = False
self._progress.error = event.get("error", "Unknown error")
- # Nothing left to save: drop an in-flight watchdog to its grace, not the
- # save backstop.
+ # Nothing left to save: drop an in-flight watchdog to its grace, not the save backstop.
self._complete_seen.set()
if self._cancel_requested:
self._output_dir = self._progress.output_dir = None
@@ -3055,7 +3031,6 @@ class TrainingBackend:
}
self._terminal_finalize_payload = dict(db_action_kwargs)
- # --- DB I/O outside the lock ---
if db_action == "create_run":
self._ensure_db_run_created()
if self._db_run_created:
@@ -3079,8 +3054,8 @@ class TrainingBackend:
elif db_action == "finalize":
self._finalize_run_in_db(**db_action_kwargs)
- # Bound how long a worker that will not exit can hold the UI at 100%. No-ops on a
- # prompt exit. Outside the lock: _start_stop_watchdog takes it, not reentrant.
+ # Bound how long a worker that will not exit can hold the UI at 100%. Outside the lock:
+ # _start_stop_watchdog takes it and it is not reentrant.
if etype in ("complete", "error"):
self._start_stop_watchdog(
cancel = False,
@@ -3093,6 +3068,7 @@ class TrainingBackend:
self._log_training_progress()
def _persist_output_dir(self) -> None:
+ # Re-queue the claimed batch at the front so it retries on the next flush.
with self._lock:
if (
not self._output_dir
@@ -3124,11 +3100,9 @@ class TrainingBackend:
now = time.monotonic()
if prev >= 0 and step > prev and not is_final and (now - self._last_progress_log_ts) < 30.0:
return
- # Throughput over the interval since the previous logged line. Trainer speed is
- # the number people watch a training run for, and the only place it used to
- # appear was HF's own tqdm bar ("1.84s/it") and its per-step print
- # ("train_tokens_per_second"), both of which are raw stdout rather than
- # structured. Carry it here instead, so the structured line is not a downgrade.
+ # Throughput over the interval since the previous logged line: it used to appear only in HF's own
+ # tqdm bar ("1.84s/it") and its per-step train_tokens_per_second print, both raw stdout rather
+ # than structured.
elapsed = p.elapsed_seconds
tokens = p.num_tokens
s_per_step = tok_per_s = None
@@ -3141,11 +3115,8 @@ class TrainingBackend:
s_per_step = round(d_time / d_steps, 3)
if tokens is not None and prev_tokens is not None and tokens > prev_tokens:
tok_per_s = round((tokens - prev_tokens) / d_time, 1)
- # The first logged line reports no throughput on purpose: elapsed_seconds is
- # wall time since the worker started, which includes imports, the model
- # download and load and the dataset build, and on a resumed run the step and
- # token counters predate this process entirely. Dividing by it would report a
- # number nobody wants. The next line has a real in-training interval.
+ # The first logged line reports no throughput on purpose: elapsed_seconds is wall time since the
+ # worker started (imports, download, load, dataset build), and a resumed run's counters are older.
self._last_progress_log_ts = now
self._last_progress_log_step = step
@@ -3178,7 +3149,7 @@ class TrainingBackend:
):
self._run_intent_lock.release()
return
- self._db_create_in_progress = True # only one caller creates
+ self._db_create_in_progress = True
job_id = self.current_job_id
db_config = self._db_config
started_at = self._db_started_at or datetime.now(timezone.utc).isoformat()
@@ -3215,9 +3186,8 @@ class TrainingBackend:
logger.warning("Failed to create DB run record for early failure", exc_info = True)
finally:
with self._lock:
- # Publish the flags only if this is still the current run. A killed worker lets a new /start
- # proceed mid-create, and these flags are backend-wide, so a stale create must not satisfy
- # the new run's DB state (the row was created by id; the new run creates its own).
+ # Publish the flags only if this is still the current run: they are backend-wide, and a killed
+ # worker lets a new /start proceed mid-create.
if self.current_job_id == job_id:
if created:
self._db_run_created = True # publish only after the insert commits
@@ -3351,8 +3321,6 @@ class TrainingBackend:
)
return events
- # --- Plot generation ---
-
def _create_loss_plot(
self,
progress: TrainingProgress,
diff --git a/studio/backend/core/youtube_transcript.py b/studio/backend/core/youtube_transcript.py
index 9ec5cd27d44..0bec641ba86 100644
--- a/studio/backend/core/youtube_transcript.py
+++ b/studio/backend/core/youtube_transcript.py
@@ -41,8 +41,7 @@ _TIMEOUT = httpx.Timeout(20.0)
# Captions are text; a 4 MB track is already an outlier for a very long video.
_MAX_CAPTION_BYTES = 4 * 1024 * 1024
_MAX_PLAYER_BYTES = 4 * 1024 * 1024
-# Roughly 25k tokens. A three hour video's captions run past 200k characters, which
-# would swallow the model's context window on its own.
+# Roughly 25k tokens: a three hour video's captions would swallow the model's context window on their own.
_MAX_TRANSCRIPT_CHARS = 100_000
# Timedtext normally answers 200, but a hop is re-validated rather than refused.
_MAX_CAPTION_REDIRECTS = 3
@@ -201,8 +200,8 @@ def _select_track(
base = wanted.split("-")[0]
for want_generated in (False, True):
candidates = [t for t in tracks if (t.get("kind") == "asr") is want_generated]
- # exact locale before the base-language fallback: a pt-BR request must not
- # take a pt-PT track just because it is listed first
+ # exact locale before the base-language fallback: a pt-BR request must not take a pt-PT track just
+ # because it is listed first
for matches_wanted in (
lambda code: code == wanted,
lambda code: code.split("-")[0] == base,
@@ -257,8 +256,8 @@ async def _fetch_track_text(client: httpx.AsyncClient, base_url: str) -> str:
body = b""
for _ in range(_MAX_CAPTION_REDIRECTS + 1):
- # Redirects are followed by hand so the host allowlist covers every hop, not
- # just the URL the player response handed us.
+ # Redirects are followed by hand so the host allowlist covers every hop, not just the URL the player
+ # response handed us.
async with client.stream(
"GET", url, headers = {"User-Agent": _USER_AGENT}, follow_redirects = False
) as response:
diff --git a/studio/backend/hub/routes/inventory.py b/studio/backend/hub/routes/inventory.py
index 1a779cd918b..bd2fd37eaa3 100644
--- a/studio/backend/hub/routes/inventory.py
+++ b/studio/backend/hub/routes/inventory.py
@@ -59,8 +59,8 @@ async def list_local_models(
return await local_inventory.list_local_models_response(models_dir)
-# Plain `def` (not async): synchronous SQLite + filesystem work runs in
-# FastAPI's thread-pool instead of blocking the event loop.
+# Plain def, not async: synchronous SQLite and filesystem work runs in FastAPI's thread pool instead
+# of blocking the event loop.
@router.get("/scan-folders", response_model = ScanFoldersResponse)
def get_scan_folders(current_subject: str = Depends(get_current_subject)):
return local_inventory.get_scan_folders_response()
diff --git a/studio/backend/hub/schemas/downloads.py b/studio/backend/hub/schemas/downloads.py
index 9397bdc9196..93d4ae105d7 100644
--- a/studio/backend/hub/schemas/downloads.py
+++ b/studio/backend/hub/schemas/downloads.py
@@ -77,16 +77,15 @@ class DownloadStartResponse(BaseModel):
job_key: str
state: str
accepted: bool
- # True when this start attached to a job already running rather than
- # starting one. Accepted either way.
+ # True when this start attached to a job already running rather than starting one.
attached: bool = False
generation: int
- # The transport the job is really on: the one the backend resolved for a
- # fresh start, or the running job's own when this start adopted one.
- # Either can differ from what the client asked for.
+ # The transport the job is really on, which can differ from the one the client asked for.
transport: Optional[str] = None
- # Set only when an adopted job had fallen back from Xet to HTTP: stopping
- # it still writes the original marker, so it is a restart, not a resume.
+ # Set only when an adopted job had fallen back from Xet to HTTP: stopping it still writes the
+ # original marker, so it is a restart, not a resume.
+ # Set only on a job that fell back from Xet to HTTP mid-flight: cancelling it still writes the original
+ # transport's marker, so the partial is restart-only even though the worker is on resumable HTTP.
cancel_transport: Optional[str] = None
@@ -101,9 +100,6 @@ class ActiveDownload(BaseModel):
repo_id: Optional[str] = None
variant: Optional[str] = None
transport: Optional[str] = None
- # Set only on a job that fell back from Xet to HTTP mid-flight: cancelling
- # it still writes the original transport's marker, so the partial is
- # restart-only even though the worker is on resumable HTTP.
cancel_transport: Optional[str] = None
state: str
files: Optional[List[str]] = Field(
@@ -135,8 +131,8 @@ class TransportCapabilities(BaseModel):
xet: TransportCapability
auto_resolves_to: Literal["xet", "http"] = "xet"
auto_reason: Optional[str] = None
- # False when the installed huggingface_hub refetches an interrupted file from zero, so no
- # transport can offer a byte-resume and the UI must not label one.
+ # False when the installed huggingface_hub refetches an interrupted file from zero, so no transport
+ # can offer a byte-resume and the UI must not label one.
partials_resumable: bool = True
@@ -148,8 +144,8 @@ class TransportStatusResponse(BaseModel):
class DownloadProgressResponse(BaseModel):
downloaded_bytes: int
- # Finalized-blob bytes only (no ``.incomplete``). Registry-loss completion
- # fallbacks key off this so a partial isn't mistaken for a finished download.
+ # Finalized-blob bytes only: registry-loss completion fallbacks key off this, so a partial is not
+ # mistaken for a finished download.
completed_bytes: int = 0
complete_on_disk: bool = Field(
False,
diff --git a/studio/backend/hub/schemas/inventory.py b/studio/backend/hub/schemas/inventory.py
index de83251bedb..c7e31077f00 100644
--- a/studio/backend/hub/schemas/inventory.py
+++ b/studio/backend/hub/schemas/inventory.py
@@ -241,8 +241,8 @@ class CachedRepoBase(BaseModel):
runtime: ModelRuntime = "unknown"
format_variant: Optional[str] = None
capabilities: LocalModelCapabilities = Field(default_factory = LocalModelCapabilities)
- # Inferred pipeline task ("text-to-image" / "text-to-video" / a chat task / None). The task-scoped pickers filter On
- # Device rows on it and the chat picker routes a diffusion pick by it, so a row without one is dropped from those lists.
+ # The task-scoped pickers filter On Device rows on the inferred task and the chat picker routes a
+ # diffusion pick by it, so a row without one is dropped from those lists.
task: Optional[str] = None
audio_type: Optional[str] = None
@@ -270,17 +270,16 @@ class CachedModelRepo(CachedRepoBase):
pipeline_tag: Optional[str] = None
library_name: Optional[str] = None
tags: Optional[List[str]] = None
- # True for a diffusion-tagged repo with NO top-level model_index.json: a single-file checkpoint needing from_single_file
- # + a filename. Pickers must not offer it as a pipeline load unless the catalog carries a curated artifact for it.
+ # True for a diffusion-tagged repo with NO top-level model_index.json: a single-file checkpoint
+ # needing from_single_file plus a filename. Pickers must not offer it as a pipeline load unless
+ # the catalog carries a curated artifact.
single_file: bool = False
- # True for an sd.cpp companion mirror: a VAE / text-encoder repo with no denoiser, so it is
- # never a pick on ANY page. It still gets a row, because these run to tens of GB and the row
- # is how they are seen and deleted; the pickers filter on this instead.
+ # An sd.cpp companion mirror is never a pick on any page, but still gets a row, because these run to
+ # tens of GB and the row is how they are seen and deleted.
companion: bool = False
- # True when the SELECTED revision is a diffusers pipeline. An unrecognised one carries no
- # task and no root config for can_chat, so this flag is all that keeps it out of a chat
- # picker. Declared because response_model drops undeclared keys, which left the CLI
- # (reading the dict in-process) and the frontend disagreeing about the same row.
+ # An unrecognised pipeline carries no task and no root config for can_chat, so this flag is all
+ # that keeps it out of a chat picker. Declared because response_model drops undeclared keys, which
+ # left the CLI and the frontend disagreeing about the same row.
diffusers: bool = False
diff --git a/studio/backend/hub/services/datasets/cache_inventory.py b/studio/backend/hub/services/datasets/cache_inventory.py
index b70bad1c04b..f627563d2e6 100644
--- a/studio/backend/hub/services/datasets/cache_inventory.py
+++ b/studio/backend/hub/services/datasets/cache_inventory.py
@@ -201,21 +201,21 @@ _DATASET_NON_PAYLOAD_FILENAMES = (
)
-# suffixes no loader can turn into rows: none is in `datasets`' extension map, and a script also
-# needs `trust_remote_code`, which no load path here passes and `datasets>=4` dropped.
+# suffixes no loader can turn into rows; a script also needs trust_remote_code, which no load path
+# here passes and datasets>=4 dropped.
_DATASET_NON_PAYLOAD_SUFFIXES = frozenset({".cff", ".md", ".py", ".pyc"})
def _is_payload_dir(name: str) -> bool:
- # the only rule `datasets` applies to a directory, which also covers a Mac zip's `__MACOSX`.
- # the metadata FILE names must not be applied here: `license/train.parquet` loads fine, and
- # pruning that subtree hid the dataset from On Device.
+ # the only rule datasets applies to a directory, which also covers a Mac zip's __MACOSX. the
+ # metadata FILE names must not be applied here: license/train.parquet loads fine, and pruning that
+ # subtree hid the dataset from On Device.
return not name.startswith(".") and not name.startswith("__")
def _is_payload_name(name: str) -> bool:
# as above, plus the metadata names: AppleDouble sidecars, every dotfile the list does not
- # enumerate, and the cards and licences a cancelled download leaves behind.
+ # enumerate, and the cards a cancelled download leaves behind.
if not _is_payload_dir(name):
return False
return name.lower() not in _DATASET_NON_PAYLOAD_FILENAMES
@@ -224,21 +224,20 @@ def _is_payload_name(name: str) -> bool:
def _is_payload_file(name: str) -> bool:
if not _is_payload_name(name):
return False
- # the resolver's suffix rule, which walks the whole chain and drops a trailing compression
- # suffix: `train.parquet.backup` is still parquet, `data.py.gz` is still a script.
+ # the resolver's suffix rule drops a trailing compression suffix: train.parquet.backup is still
+ # parquet, data.py.gz is still a script.
suffix = local_options._data_suffix(name)
return suffix is None or suffix.lower() not in _DATASET_NON_PAYLOAD_SUFFIXES
def _is_present_payload_file(path: Path) -> bool:
- # existence is not enough. `_empty_payload` covers both shapes that look like payload and
- # are not: a zero-byte file, and a `blobs/` link whose blob was pruned.
+ # existence is not enough: a zero-byte file, and a blobs/ link whose blob was pruned, both look like
+ # payload and are not.
if not _is_payload_file(path.name):
return False
if local_options._empty_payload(path):
return False
- # bytes are not rows: `_rowless` probes a header-only csv or a `[]` json, reading a short
- # head for those two builders only. it drops the file, not the snapshot, as datasets does.
+ # bytes are not rows: a header-only csv or a [] json drops the file, not the snapshot, as datasets does.
module = local_options._file_module(path.name)
return module is None or not local_options._rowless(path, path.name, module)
@@ -255,9 +254,8 @@ def _snapshot_holds_payload(snapshot: Path) -> Optional[bool]:
nonlocal unreadable
unreadable = True
- # roots still to walk, and every directory already walked or queued by resolved path. a
- # junction pointing at its own ancestor resolves back inside the snapshot, so containment
- # alone leaves `data/loop/loop/...` descending until the path length gives out.
+ # a junction pointing at its own ancestor resolves back inside the snapshot, so containment alone
+ # leaves data/loop/loop/... descending until the path length gives out.
seen: set[Path] = set()
pending: list[Path] = []
@@ -286,30 +284,28 @@ def _snapshot_holds_payload(snapshot: Path) -> Optional[bool]:
base = Path(directory)
kept = []
for name in dirnames:
- # nothing under a hidden or `__`-prefixed dir can supply rows, so
- # `.hidden/notes.txt` must not clear `partial`.
+ # nothing under a hidden or __-prefixed dir can supply rows, so .hidden/notes.txt must not clear
+ # partial.
if not _is_payload_dir(name):
continue
entry = base / name
- # containment, not a link-type test: `is_symlink()` is false for a Windows
- # junction and `is_junction()` postdates 3.12, which this still supports, so
- # only comparing resolved paths catches every redirect on every runtime.
+ # containment, not a link-type test: is_symlink() is false for a Windows junction and is_junction()
+ # postdates 3.12, so only comparing resolved paths catches every redirect.
try:
redirected = not entry.resolve(strict = True).is_relative_to(root)
linked = entry.is_symlink()
except (OSError, RuntimeError, ValueError):
unreadable = True
continue
- # walked as a root of its own, not taken as proof: migrated caches keep
- # their data behind a redirect, and a stale one holds nothing.
+ # walked as a root of its own, not taken as proof: migrated caches keep their data behind a
+ # redirect, and a stale one holds nothing.
if redirected:
target = _book(entry)
if target is not None:
pending.append(target)
continue
- # a symlink back inside this root is skipped: the walk never descends one,
- # and booking its target would prune the real directory whenever `alias` is
- # listed before `data`. a junction reports False here, hence the visited set.
+ # a symlink back inside this root is skipped, since booking its target would prune the real
+ # directory whenever alias is listed before data; a junction reports False here.
if linked:
continue
if _book(entry) is None:
@@ -701,7 +697,7 @@ def _delete_cached_dataset_blocking(repo_id: str, cache_path: Optional[str] = No
target_root = resolve_delete_target_root("dataset", repo_id, cache_path, owners.keys())
# A processed-only dataset row sends its Arrow cache path, which is not a Hub datasets-- dir, so
- # resolve_delete_target_root returns None. Accept it and fall through to the processed delete.
+ # resolve_delete_target_root returns None.
if target_root is None and not (
cache_path
and (_is_processed_dataset_cache_path(repo_id, cache_path) or app_entry is not None)
@@ -733,9 +729,8 @@ def _delete_cached_dataset_blocking(repo_id: str, cache_path: Optional[str] = No
exc_info = True,
)
- # Restrict the processed Arrow-cache delete to the selected cache's datasets root. A processed
- # cache_path scopes to its own root; a Hub target to the datasets root sharing its cache home;
- # an unspecified cache_path stays global (legacy).
+ # A processed cache_path scopes to its own root, a Hub target to the datasets root sharing its cache
+ # home, and an unspecified one stays global.
processed_roots: Optional[set[Path]]
if not cache_path:
processed_roots = None
@@ -773,7 +768,7 @@ def _delete_cached_dataset_blocking(repo_id: str, cache_path: Optional[str] = No
),
)
- # ``scan_cache_dir()`` skips blob-only/corrupt repos the revision delete can't touch, yet the
+ # scan_cache_dir() skips blob-only or corrupt repos the revision delete cannot touch, yet the
# fallback scanner shows them, so purge the whole dir. Hub cache targets only.
cache_purged = partial_purged = state_purged = False
if target_root is not None:
diff --git a/studio/backend/hub/services/datasets/downloads.py b/studio/backend/hub/services/datasets/downloads.py
index f84ea4084c4..75299a25c28 100644
--- a/studio/backend/hub/services/datasets/downloads.py
+++ b/studio/backend/hub/services/datasets/downloads.py
@@ -70,8 +70,7 @@ def get_dataset_snapshot_metadata_cached(
size, hashes, restricted, cached_fp, ts = cached
if (time.monotonic() - ts) >= _DATASET_SIZE_POS_TTL:
del _dataset_size_cache[repo_id]
- # A gated/private repo's metadata is only served back to the token
- # that fetched it; another token may have no access at all.
+ # A gated or private repo's metadata is only served back to the token that fetched it.
elif not restricted or cached_fp == token_fp:
_dataset_size_cache.move_to_end(repo_id)
return size, hashes
@@ -165,7 +164,7 @@ async def download_dataset_response(
key = _download_job_key(repo_id)
# Off the event loop: resolving "auto" can run the Xet reachability probe, and a blackholed DNS
- # makes that outlast its 3s budget while every other Unsloth request waits behind it.
+ # makes that outlast its 3s budget while every other request waits behind it.
use_xet, transport_reason = await asyncio.to_thread(
download_lifecycle.resolve_requested_use_xet,
getattr(body, "transport_mode", None),
@@ -188,9 +187,8 @@ async def download_dataset_response(
)
generation = _registry.current_generation(key)
if not claimed:
- # Pollable when rejected by this repo's own in-flight job, and that is
- # also the only case that attached to anything; an in-progress delete
- # leaves no job, so both come from ``adoptable``.
+ # Both come from adoptable: an in-progress delete leaves no job, and only an in-flight job of this
+ # repo attached to anything.
adoptable = _registry.adoptable(key)
return {
"repo_id": repo_id,
@@ -198,11 +196,11 @@ async def download_dataset_response(
"accepted": adoptable,
"attached": adoptable,
"generation": generation,
- # An adopted job keeps the transport it started on, so report it
- # rather than let the caller assume the one it asked for.
+ # An adopted job keeps the transport it started on, so report it rather than let the caller assume
+ # the one it asked for.
"transport": _registry.job_transport(key),
- # And its cancel marker: a run that fell back from Xet to HTTP
- # still cancels into a restart-only partial.
+ # And its cancel marker: a run that fell back from Xet to HTTP still cancels into a restart-only
+ # partial.
"cancel_transport": _registry.job_cancel_transport(key),
}
download_manifest.clear_cancel_marker(
@@ -239,8 +237,7 @@ async def download_dataset_response(
"accepted": True,
"attached": False,
"generation": generation,
- # See models: the resolved transport, which a downgrade can make
- # different from the one requested.
+ # See models: the resolved transport, which a downgrade can make different from the one requested.
"transport": transport,
}
diff --git a/studio/backend/hub/services/datasets/formatting.py b/studio/backend/hub/services/datasets/formatting.py
index a4944d2f3de..7881947b6b4 100644
--- a/studio/backend/hub/services/datasets/formatting.py
+++ b/studio/backend/hub/services/datasets/formatting.py
@@ -160,11 +160,8 @@ def _serialize_preview_value(value):
value.keys() - {"bytes", "path"}
):
return _serialize_binary_value(raw)
- # A decoded Audio cell is {"array", "sampling_rate", "path"}. torchcodec hands back
- # an AudioDecoder, which falls through to str() below, but the soundfile fallback
- # returns the waveform and the dataset formatter turns it into a plain list -- one
- # float per sample, so ten preview rows of a few seconds each is tens of MB of JSON
- # and the client dies rendering it.
+ # A decoded Audio cell becomes one float per sample under the soundfile fallback, so ten preview
+ # rows of a few seconds each are tens of MB of JSON and the client dies rendering it.
if "sampling_rate" in value and isinstance(value.get("array"), (list, tuple)):
return _serialize_decoded_audio(value)
return {str(key): _serialize_preview_value(item) for key, item in value.items()}
@@ -441,7 +438,7 @@ def check_format_response(
)
if preview_slice is None:
- # Tier 2: full streaming (resolves all files — slow for large repos)
+ # Tier 2: full streaming (resolves all files - slow for large repos)
logger.info("Tier 2: falling back to full streaming load_dataset")
try:
load_kwargs = {
diff --git a/studio/backend/hub/services/datasets/local.py b/studio/backend/hub/services/datasets/local.py
index 4360128a122..28655b26fb4 100644
--- a/studio/backend/hub/services/datasets/local.py
+++ b/studio/backend/hub/services/datasets/local.py
@@ -26,8 +26,8 @@ from utils.paths.path_utils import (
is_appledouble_metadata,
)
-# Tabular formats are preferred over archives for Tier 1 preview: archives (e.g. images.zip)
-# load as ImageFolder with synthetic columns that don't match the real schema.
+# Archives load as ImageFolder with synthetic columns that do not match the real schema, so tabular
+# formats are preferred for Tier 1 preview.
_TABULAR_EXTS = (".parquet", ".json", ".jsonl", ".csv", ".tsv", ".arrow")
_ARCHIVE_EXTS = (".tar", ".tar.gz", ".tgz", ".gz", ".zst", ".zip", ".txt")
DATA_EXTS = _TABULAR_EXTS + _ARCHIVE_EXTS
diff --git a/studio/backend/hub/services/datasets/local_options.py b/studio/backend/hub/services/datasets/local_options.py
index fa9ff724fe2..ee49a7fbac7 100644
--- a/studio/backend/hub/services/datasets/local_options.py
+++ b/studio/backend/hub/services/datasets/local_options.py
@@ -220,8 +220,7 @@ def _snapshot_metadata_file(snapshot: Path, name: str) -> Optional[Path]:
return path if 0 < size <= _MAX_METADATA_BYTES else None
-# _read_card_metadata returns this when a card has front matter datasets cannot parse.
-# DatasetCard.load raises on it, so nothing in the snapshot is loadable.
+# DatasetCard.load raises on front matter datasets cannot parse, so nothing in the snapshot is loadable.
_UNPARSABLE_METADATA = object()
@@ -238,8 +237,7 @@ def _read_card_metadata(path: Path) -> Any:
except YAMLError:
return _UNPARSABLE_METADATA
if payload is None:
- # Empty, comment-only or null front matter, which RepoCard reads as an empty
- # card rather than refusing.
+ # Empty, comment-only or null front matter, which RepoCard reads as an empty card rather than refusing.
return {}
except StopIteration:
# Front matter that never closes is not front matter at all.
@@ -252,24 +250,17 @@ def _read_card_metadata(path: Path) -> Any:
return payload if isinstance(payload, dict) else _UNPARSABLE_METADATA
+# Mirrors datasets' get_data_patterns (sharded data/{split}-NNNNN files, then directory keywords,
+# then filename keywords, then one train split) without importing datasets, which
+# dataset_cache.py forbids on cache paths.
+# An offered split has to be trainable, and this is deliberately tighter in two places: a file
+# whose symlink leaves the repository is refused, and only trainable extensions are offered.
+
+# Past this the result would depend on traversal order, so nothing is offered.
# --- Split inference for cached datasets whose card declares nothing ------------------
-#
-# datasets resolves a metadata-free directory with get_data_patterns: it tries sharded
-# data/{split}-NNNNN-of-NNNNN files, then directory-name keywords, then filename keywords,
-# and falls back to one train split holding everything. The first stage that names any
-# split wins, and one builder is chosen for all of them. This mirrors that without
-# importing datasets, which dataset_cache.py forbids on cache paths.
-#
-# The contract is that an offered split has to be trainable: anything looser is offered and
-# then 422s, anything tighter hides a usable option. Deliberately tighter in two places,
-# both to keep training inside the cache: a file whose symlink leaves the repository is
-# refused, and only the extensions Unsloth can train on are offered.
-
-# Only names are read during the scan, so this can sit well above any real snapshot. Past
-# it the result would depend on traversal order, so nothing is offered.
_MAX_SNAPSHOT_DATA_FILES = 200_000
-# datasets drops these by basename before it infers anything, so a metadata-only cache is
-# empty rather than a bogus train split.
+# datasets drops these by basename before it infers anything, so a metadata-only cache is empty
+# rather than a bogus train split.
_IGNORED_DATA_FILENAMES = frozenset(
{
"README.md",
@@ -283,11 +274,11 @@ _IGNORED_DATA_FILENAMES = frozenset(
# What fsspec can decompress in an Unsloth install, as suffixes that sit after the real one.
_COMPRESSION_EXTENSIONS = frozenset({".gz", ".gzip", ".bz2", ".xz", ".zip"})
# Named by datasets but needing codecs an Unsloth install does not ship, so they raise
-# "Compression type not supported" and offering them would put a dead split in the picker.
-# .lzma is the legacy alone-format: datasets registers .xz for its filter, not this.
+# "Compression type not supported" and would put a dead split in the picker; .lzma is the legacy
+# alone-format, for whose filter datasets registers .xz.
_UNREADABLE_COMPRESSION = frozenset({".zst", ".zstd", ".lz4", ".lzma"})
-# datasets picks one builder for the whole dataset, from the extensions it finds. Splits
-# that disagree make it raise, so a snapshot mixing formats is not offerable at all.
+# datasets picks one builder for the whole dataset, and splits that disagree make it raise, so a
+# snapshot mixing formats is not offerable at all.
_MODULE_EXTENSIONS = {
"arrow": ".arrow",
"csv": ".csv",
@@ -300,19 +291,15 @@ _MODULE_EXTENSIONS = {
"xml": ".xml",
"hdf5": ".h5 .hdf5",
}
-# The folder builders. datasets registers only these in both letter cases, so only their
-# extensions match case-insensitively; everything else is matched by a case-sensitive glob.
+# datasets registers only the folder builders in both letter cases, so only their extensions match case-insensitively.
_FOLDER_EXTENSIONS = frozenset(
- # imagefolder
".apng .blp .bmp .bufr .bw .cur .dcx .dds .dib .emf .eps .fit .fits .flc .fli .ftc .ftu "
".gbr .gif .grib .icb .icns .ico .iim .im .j2c .j2k .jfif .jp2 .jpc .jpe .jpeg .jpf "
".jpg .jpx .msp .pbm .pcd .pcx .pgm .png .pnm .ppm .ps .psd .pxr .ras .rgb .rgba .sgi "
".tga .tif .tiff .vda .vst .webp .wmf .xbm .xpm "
- # audiofolder
".3g2 .3gp .aiff .asf .au .avr .caf .f4v .flac .flv .htk .ircam .m4v .mat4 .mat5 .mp3 "
".mpc2k .mpg .mxf .nist .nut .ogg .ogm .opus .paf .pvf .raw .rf64 .sd2 .sds .svx .voc "
".w64 .wav .wavex .webm .wma .wmv .wve .xi "
- # videofolder and pdffolder
".avi .mkv .mov .mp4 .mpeg .pdf".split()
)
_EXTENSION_MODULES = {
@@ -321,8 +308,8 @@ _EXTENSION_MODULES = {
# datasets infers a split's module from its first 200 files, in resolved (sorted) order.
_MAX_MODULE_INFERENCE_FILES = 200
# datasets' tie-break once the counts are level, then the extension string itself.
-# TRAINING_DATA_EXTS only ever resolves to these, so any other builder means the
-# snapshot holds nothing trainable and no file needs opening to find that out.
+# TRAINING_DATA_EXTS only ever resolves to these, so any other builder means the snapshot holds
+# nothing trainable.
_TRAINABLE_MODULES = frozenset({"csv", "json", "parquet"})
_ROW_PROBE_BYTES = 8192
_EXTENSION_PRIORITY = (".parquet", ".jsonl", ".json", ".csv")
@@ -348,8 +335,8 @@ _DIR_NAME_KEYWORD_PATTERNS = (
)
# data/{split}-NNNNN-of-NNNNN*.*, where a * matches nothing as happily as something.
_SHARDED_DATA_RE = re.compile(r"^data/(?P<split>[^/]*)-[0-9]{5}-of-[0-9]{5}[^/]*\.[^/]*$")
-# datasets' own split grammar, which is stricter than the one the picker accepts. A shard
-# named outside it makes the whole snapshot unloadable rather than falling through.
+# datasets' own split grammar is stricter than the one the picker accepts: a shard named outside it
+# makes the whole snapshot unloadable rather than falling through.
_SHARD_SPLIT_RE = re.compile(r"^\w+(\.\w+)*$")
@@ -423,15 +410,15 @@ def _snapshot_data_files(snapshot: Path) -> Optional[list[PurePosixPath]]:
for filename in filenames:
if filename in _IGNORED_DATA_FILENAMES or filename.startswith("."):
continue
- # resolve_pattern keeps a link only when its target is a file, so a dangling
- # one is not a file datasets sees, let alone one that condemns its split.
+ # resolve_pattern keeps a link only when its target is a file, so a dangling one is not a file
+ # datasets sees.
if not (base / filename).is_file():
continue
- # Files with no builder of their own are kept: they cannot win the vote, but a
- # split holding nothing else is one datasets refuses to build.
+ # Files with no builder of their own are kept: they cannot win the vote, but a split holding nothing
+ # else is one datasets refuses to build.
if len(found) >= _MAX_SNAPSHOT_DATA_FILES:
- # Past the cap this is a traversal-order prefix, which cannot be compared
- # with what the loader would resolve.
+ # Past the cap this is a traversal-order prefix, which cannot be compared with what the loader would
+ # resolve.
return None
found.append(PurePosixPath((relative / filename).as_posix()))
found.sort(key = lambda path: path.as_posix())
@@ -530,8 +517,8 @@ def _offerable(entries: list[PurePosixPath], snapshot: Path, module: str) -> Opt
resolved = resolved_dataset_snapshot_file(snapshot, path.as_posix())
if resolved is None or _blocked_by_compression(path.name, module):
return None
- # Once a builder is chosen, datasets reads only what that builder claims, so a
- # training file left behind by a folder builder is not data this split offers.
+ # Once a builder is chosen datasets reads only what that builder claims, so a training file left
+ # behind by a folder builder is not data this split offers.
if _file_module(path.name) != module or not _trainable_name(path.name):
continue
if _empty_payload(resolved):
@@ -540,8 +527,8 @@ def _offerable(entries: list[PurePosixPath], snapshot: Path, module: str) -> Opt
if _rowless(resolved, path.name, module):
continue
trainable = True
- # Every builder but json fails outright on a file with no rows, and datasets prepares
- # every split before handing one back, so such a file condemns its siblings too.
+ # Every builder but json fails outright on a file with no rows, and datasets prepares every split
+ # before handing one back, so such a file condemns its siblings too.
if empty and module != "json":
return None
return trainable
@@ -642,8 +629,8 @@ def _declares_configs(snapshot: Path, name: str) -> bool:
if not isinstance(payload, dict):
# DatasetCardData updates a dict from it, which raises on anything else.
return bool(payload)
- # Only configs count: 4.3.0 builds no config from dataset_info declared here, so a
- # feature schema in this file leaves the loader inferring the files by pattern.
+ # Only configs count: 4.3.0 builds no config from dataset_info declared here, so a feature schema
+ # leaves the loader inferring the files by pattern.
return bool(payload.get("configs"))
@@ -662,9 +649,8 @@ def _malformed_info(payload: Any) -> bool:
def _snapshot_options(snapshot: Path) -> set[tuple[str, str]]:
options: set[tuple[str, str]] = set()
- # Metadata we cannot read still names the loader's configs, so inference must not
- # step in beside it: a card too large or too unsafe to open, or the standalone yaml
- # file, which datasets reads as a card and we do not.
+ # Metadata we cannot read still names the loader's configs, so inference must not step in beside it:
+ # a card too large or unsafe to open, or the standalone yaml file.
declared = _unreadable_metadata(snapshot, "README.md") or _declares_configs(
snapshot, ".huggingface.yaml"
)
@@ -675,15 +661,15 @@ def _snapshot_options(snapshot: Path) -> set[tuple[str, str]]:
# datasets raises out of DatasetCard.load, so no option here would ever start.
return options
if isinstance(card_data, dict):
- # datasets merges the standalone YAML into the card, so a README that
- # declares nothing does not undo a declaration made there.
+ # datasets merges the standalone YAML into the card, so a README that declares nothing does not undo
+ # a declaration made there.
declared = declared or bool(card_data.get("configs"))
_add_config_options(options, card_data.get("configs"))
named = len(options)
info = card_data.get("dataset_info")
_add_dataset_info_options(options, info)
- # dataset_info carrying only a feature schema names no config, so datasets
- # still resolves the files by pattern and inference has to run.
+ # dataset_info carrying only a feature schema names no config, so datasets still resolves the files
+ # by pattern and inference has to run.
declared = (
declared or len(options) > named or _malformed_info(info) or _declares_splits(info)
)
@@ -691,7 +677,7 @@ def _snapshot_options(snapshot: Path) -> set[tuple[str, str]]:
for filename in ("dataset_infos.json", "dataset_info.json"):
metadata = _snapshot_metadata_file(snapshot, filename)
if metadata is None:
- # datasets json.loads dataset_infos.json unconditionally, so one it cannot
+ # datasets json.loads dataset_infos.json unconditionally while resolving configs, so one it cannot
# read raises before any split exists, inferred or not.
declared = declared or (
filename == "dataset_infos.json" and _metadata_present(snapshot, filename)
@@ -699,17 +685,16 @@ def _snapshot_options(snapshot: Path) -> set[tuple[str, str]]:
continue
payload = _safe_json_file(metadata, snapshot, allow_snapshot_symlink = True)
if filename == "dataset_infos.json":
- # datasets json.loads this one while resolving configs, so a file it cannot
- # parse raises before any split exists, inferred or not.
+ # datasets json.loads this one while resolving configs, so a file it cannot parse raises before any
+ # split exists, inferred or not.
declared = declared or payload is None
_add_dataset_info_options(options, payload)
else:
_add_info_options(options, payload)
if not options and not declared:
- # Nothing was declared, which is the case #8140 reports: the loader still resolves
- # the files by pattern, so the picker infers the same splits it would. A card that
- # did declare something names its own configs, which inference cannot reproduce.
+ # Nothing was declared, the case #8140 reports: the loader still resolves the files by pattern, so
+ # the picker infers the same splits.
options.update(_inferred_snapshot_options(snapshot))
return options
diff --git a/studio/backend/hub/services/download_lifecycle.py b/studio/backend/hub/services/download_lifecycle.py
index 787916ffec3..50997ca75da 100644
--- a/studio/backend/hub/services/download_lifecycle.py
+++ b/studio/backend/hub/services/download_lifecycle.py
@@ -82,16 +82,11 @@ def resolve_auto_use_xet() -> tuple[bool, str]:
if health is not None and not health.use_xet:
return (False, reason)
if _health_is_forced(health):
- # ``UNSLOTH_FORCE_XET=1``. The zoo's off switches (``UNSLOTH_DISABLE_XET``,
- # ``UNSLOTH_STABLE_DOWNLOADS``, ``HF_HUB_DISABLE_XET``) already win above, so the on switch
- # has to win here or the pair is asymmetric and the only escape from the RAM rule is
- # editing the request. Buffers are still clamped to free RAM; only the transport choice is
- # left to the operator, the same stand-down the zoo makes for a user-set
- # HF_XET_HIGH_PERFORMANCE.
+ # UNSLOTH_FORCE_XET=1 has to win here, since the zoo's off switches already win above and the pair
+ # would otherwise be asymmetric; buffers are still clamped to free RAM.
return (True, reason)
- # Health said Xet, or had no opinion at all. Free RAM is separate evidence, read from a
- # different zoo module, so it still gets a say: a missing health module says nothing about
- # whether this machine can afford Xet right now.
+ # Free RAM is separate evidence, read from a different zoo module: a missing health module says
+ # nothing about whether this machine can afford Xet right now.
pressure = _memory_pressure_reason()
if pressure is not None:
return (False, pressure)
@@ -188,31 +183,28 @@ def spawn_worker(
env["HF_HUB_DISABLE_TELEMETRY"] = "1"
env["HF_HUB_DISABLE_XET"] = "0" if use_xet else "1"
if use_xet:
- # hf_xet sizes its buffers from constants, not from the machine, and reads its config
- # natively at import, so the worker's env has to be sized here. unsloth_zoo decides, so
- # there is one rule and not two: it sizes from RAM/cores/disk and honours a user-set
- # high-performance flag (standing its own sizing down, since xet-core voids it anyway).
- # UNSLOTH_XET_FORCE_CAPS=1 bounds the machine regardless. Unsloth hand-rolled this, and the
- # copies drifted: on a 2TB host the worker got a 24GB laptop's buffer and ran 3.4x slower.
+ # hf_xet sizes its buffers from constants and reads its config natively at import, so the worker's
+ # env is sized here, and by unsloth_zoo so there is one rule and not two (UNSLOTH_XET_FORCE_CAPS=1
+ # bounds the machine regardless): the hand-rolled copies drifted, and on a 2TB host the worker got
+ # a laptop's 24GB buffer and ran 3.4x slower.
from utils import hf_xet_fallback
- # The worker's own cache, which the sizing measures: this backend's env may still name the
- # one it started with, since moving the cache in Settings does not rewrite the live process.
+ # The worker's own cache, which the sizing measures: this backend's env may still name the one it
+ # started with, since moving the cache in Settings does not rewrite the live process.
sized = hf_xet_fallback.apply_xet_env(env, env.get("HF_HUB_CACHE"))
if sized is None and not _allow_high_performance():
- # No tuning module: that unsloth_zoo is also the one setting HF_XET_HIGH_PERFORMANCE=1
- # at import, and `env` is seeded from the parent, so that inherited "1" would hand the
- # worker a 64GB ceiling (xet-core applies the preset AFTER reading the environment).
+ # No tuning module: that unsloth_zoo is also the one setting HF_XET_HIGH_PERFORMANCE=1 at import,
+ # and the inherited "1" would hand the worker a 64GB ceiling, since xet-core applies the preset
+ # AFTER reading the environment.
for key in ("HF_XET_HIGH_PERFORMANCE", "HF_XET_HP"):
env[key] = "0"
- # No token in Unsloth settings: fall back to the backend's own HF_TOKEN so
- # private repos stay downloadable (needed while inkling repos are private).
- # Not for a repo an API caller named: that would lend them the owner's identity.
+ # Fall back to the backend's own HF_TOKEN so private repos stay downloadable, but never for a repo
+ # an API caller named: that would lend them the owner's identity.
if not hf_token and allow_ambient_token:
hf_token = os.environ.get("HF_TOKEN") or None
env["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "0" if hf_token else "1"
- # hf_transfer's parallel Range chunks can leave sparse partials even in
- # "http" mode; disable so the worker's writer is always sequential.
+ # hf_transfer's parallel Range chunks can leave sparse partials even in "http" mode, so disable it
+ # and keep the worker's writer sequential.
env["HF_HUB_ENABLE_HF_TRANSFER"] = "0"
for token_key in (
"HF_TOKEN",
@@ -248,9 +240,8 @@ def spawn_worker(
return proc
finally:
if use_xet:
- # Tie the sizing's RAM reservation to the worker, so it frees when the worker exits and
- # a sibling starting in this window sizes against the remainder. A spawn that raised
- # passes None, which drops the reservation instead of leaking it.
+ # Tie the sizing's RAM reservation to the worker so it frees when the worker exits and a sibling
+ # sizes against the remainder; a spawn that raised passes None, dropping the reservation.
from utils import hf_xet_fallback
hf_xet_fallback.bind_worker_budget(proc.pid if proc is not None else None)
@@ -375,10 +366,8 @@ def finalize_worker_exit(
if state == "complete":
hf_cache_scan.invalidate_hf_cache_scans()
registry.set_job(key, "complete")
- # Where /v1 learns a new model exists: its resolver answers from a cached scan
- # with no watcher, so it would report the model absent and serve whatever is
- # resident. Models only: noting a dataset id as a local model would refuse a
- # bare request naming it instead of letting a foreign id fall through.
+ # Where /v1 learns a new model exists: its resolver answers from a cached scan with no watcher.
+ # Models only, since noting a dataset id as a local model would refuse a bare request naming it.
if repo_type == "model":
try:
from core.inference.local_model_resolver import (
@@ -389,8 +378,7 @@ def finalize_worker_exit(
note_downloaded(repo_id)
invalidate_index(additions_only = True)
- # Rebuild here, not on the first request, to keep the scan off the
- # request path.
+ # Rebuild here, not on the first request, to keep the scan off the request path.
warm_index_soon()
except Exception:
pass
@@ -404,8 +392,8 @@ def finalize_worker_exit(
else:
logger.info(f"{log_prefix} worker diagnostics for {label}: {stderr_text}")
logger.info(f"{log_prefix} complete: {label}")
- # Defensive cleanup: the canonical clear is at download-start; this
- # catches the rare case where that failed but the download succeeded.
+ # Defensive cleanup: the canonical clear is at download-start; this catches the rare case where that
+ # failed but the download succeeded.
if repo_type and repo_id:
try:
download_manifest.clear_cancel_marker(
@@ -417,8 +405,8 @@ def finalize_worker_exit(
except Exception as exc:
logger.debug(f"clear_cancel_marker failed for {repo_id} (rc=0): {exc}")
elif state == "cancelled":
- # Read metadata before the terminal set_job so a concurrent eviction
- # can't drop it; the job key is the fallback variant label.
+ # Read metadata before the terminal set_job so a concurrent eviction cannot drop it; the job key is
+ # the fallback variant label.
registry.set_job(key, "cancelled")
logger.info(f"{log_prefix} cancelled: {label} (rc={rc})")
download_registry.persist_cancel_marker(
@@ -616,9 +604,8 @@ def _try_transport_retry(
claimed, conflict_state = registry.claim(
key,
- # A XET retry re-claims the SAME transport, the permissive case in claim(): only a
- # cross-transport claim serializes against a sibling, since only that can mix an HTTP
- # resume with a XET rewrite over one shared blob.
+ # A XET retry re-claims the SAME transport, the permissive case in claim(): only a cross-transport
+ # claim can mix an HTTP resume with a XET rewrite over one shared blob.
retry_transport,
repo_type = repo_type,
repo_id = repo_id,
@@ -631,14 +618,14 @@ def _try_transport_retry(
cancel_marker_transport = original_metadata.transport,
hub_cache = original_metadata.hub_cache,
xet_cache = original_metadata.xet_cache,
- # Carry the scoped file list across the reclaim: the record it overwrites is what a later start for this scope slot is
- # compared against, so dropping it makes an identical start read as a different file set and 409 instead of adopting.
+ # Carry the scoped file list across the reclaim: the record it overwrites is what a later start is
+ # compared against, so dropping it makes an identical start 409 instead of adopting.
scoped_files = original_metadata.scoped_files or None,
)
if claimed:
break
- # An STT owner is not an active job, so mark_pending_cancel cannot reach
- # this loop; waiting it out here would be an uninterruptible spin.
+ # An STT owner is not an active job, so mark_pending_cancel cannot reach this loop and waiting it
+ # out would be an uninterruptible spin.
if conflict_state in ("deleting", "repository_owned"):
logger.debug(
"%s XET retry claim rejected for %s; blocked by %s",
@@ -671,7 +658,8 @@ def _try_transport_retry(
args.append("--dataset")
elif variant:
args.extend(["--variant", variant])
- # A scoped job must retry as the SAME scoped download; without its file list the recovery worker would fall through to a full snapshot.
+ # A scoped job must retry as the SAME scoped download; without its file list the recovery worker
+ # would fall through to a full snapshot.
if original_metadata.scoped_files:
args.extend(["--files-json", write_files_manifest(original_metadata.scoped_files)])
@@ -724,10 +712,8 @@ def _try_transport_retry(
)
registry.update_job_transport(key, original_metadata.transport)
if retry_over_xet:
- # The EXTRA worker could not be started (fd exhaustion, a broken interpreter path): a
- # local failure, not a verdict on the download, so drop to the rung this job would have
- # taken without the retry instead of stranding it in "error". Bounded: the HTTP
- # direction never re-enters this branch.
+ # A failed EXTRA worker spawn is a local failure, not a verdict on the download, so drop to the
+ # rung this job would have taken without the retry instead of stranding it in "error".
logger.warning(
"%s XET retry could not be spawned for %s; falling back to HTTP",
log_prefix,
@@ -936,11 +922,11 @@ def _start_stall_watchdog(
)
on_stall(message)
try:
- # Kill only: the _watch thread is already reaping this process, so a wait() here would
- # race it for the exit status.
+ # Kill only: the _watch thread is already reaping this process, so a wait() here would race it for
+ # the exit status.
proc.kill()
except ProcessLookupError:
- pass # already exited between the stall verdict and the kill
+ pass
except Exception:
logger.exception("%s failed to kill stalled worker for %s", log_prefix, label)
@@ -951,18 +937,14 @@ def _start_stall_watchdog(
cache_dir = cache_dir,
on_stall = _on_stall,
child_pid = proc.pid,
- # Scope the measurement to partials this worker holds open; otherwise the shared helper
- # stays repo-wide, child_pid does nothing, and two concurrent same-transport GGUF
- # variants of one repo reset each other's stall timer. This scopes the DATA clock only:
- # before its first byte a variant is still covered by the shared peer-progress check,
- # repo-wide by design so a lock wait behind a live sibling is not read as a hang.
+ # Scope the measurement to partials this worker holds open; otherwise the shared helper stays
+ # repo-wide, child_pid does nothing, and two concurrent same-transport GGUF variants of one repo
+ # reset each other's stall timer. The DATA clock only: before its first byte a variant is still
+ # covered by the repo-wide peer-progress check.
watch_new_partials_only = True,
- # The shared 90s zero-byte default assumes a single-file download whose pre-byte phase
- # is one HEAD. This worker calls snapshot_download(max_workers=1), so its pre-byte phase
- # is a model_info lookup with retries plus one sequential HEAD per file, which for an
- # already-cached repo is the ENTIRE job with no byte written. A few hundred files on a
- # slow link legitimately exceeds 90s, so a healthy worker would be killed before it
- # started.
+ # The shared 90s zero-byte default assumes a single-file download whose pre-byte phase is one
+ # HEAD; snapshot_download(max_workers=1) does a model_info lookup plus one sequential HEAD per
+ # file, which for an already-cached repo is the entire job with no byte written.
connect_timeout = 600.0,
xet_disabled = False,
)
@@ -1007,23 +989,22 @@ def register_worker(
_get_metadata = getattr(registry, "get_job_metadata", None)
_metadata = _get_metadata(key) if callable(_get_metadata) else None
_cache_dir = getattr(_metadata, "hub_cache", None) if _metadata is not None else None
- # The variant's own blobs, so a sibling variant writing into the same repo is not counted as
- # this job's progress. None for job shapes that cannot have a concurrent same-repo sibling.
+ # The variant's own blobs, so a sibling variant writing into the same repo is not counted as this job's progress.
_own_blob_hashes = (
getattr(_metadata, "blob_hashes", frozenset())
if getattr(_metadata, "variant", None)
else None
)
- # Companions (a shared mmproj) live only in the progress set, and this worker was writing
- # one when it died just as much as it was writing the main quant.
+ # Companions such as a shared mmproj live only in the progress set, and this worker was writing one
+ # as much as it was writing the main quant.
_owned_for_sweep = (
getattr(_metadata, "progress_blob_hashes", None) or _own_blob_hashes
if _own_blob_hashes is not None
else None
)
- # Sampled before the worker can write, so "did this job move bytes over Xet" is answerable on
- # exit. launch_worker samples BEFORE spawn(); sampling here would race a fast child that already
- # finalized its blobs, making a real transfer look like a no-op and leaving the streak uncleared.
+ # Sampled before the worker can write, as launch_worker samples BEFORE spawn(): sampling later
+ # would race a fast child that already finalized its blobs, making a real transfer look like a
+ # no-op and leaving the streak uncleared.
_bytes_before = (
_job_bytes_on_disk(repo_type, repo_id, _cache_dir, _own_blob_hashes)
if bytes_before is _UNSAMPLED
@@ -1041,10 +1022,8 @@ def register_worker(
)
is None
)
- # This path had NO stall detection: it relied on the worker's exit code, and a Xet
- # transfer that hangs with no progress and no error never produces one (the frozen
- # progress bar on the model-hub page). Watch the cache for byte-level progress and kill
- # a hung worker; the SIGKILL surfaces as "error", which triggers the HTTP retry below.
+ # A Xet transfer that hangs with no progress and no error never produces an exit code, so watch
+ # the cache for byte-level progress and kill it; the SIGKILL surfaces as "error".
if can_retry_http:
watchdog_stop = _start_stall_watchdog(
registry,
@@ -1072,13 +1051,11 @@ def register_worker(
defer_error = can_retry_http,
)
if watchdog_stop is not None:
- # Stop measuring once the worker is reaped: post-download work (symlinking,
- # verification) makes no byte-level progress and must not read as a stall.
+ # Stop measuring once the worker is reaped: post-download symlinking and verification make no byte-
+ # level progress and must not read as a stall.
watchdog_stop.set()
- # A hung Xet transfer usually clears on a fresh process, so spend one more XET worker
- # before changing transport. Only a DATA-phase verdict qualifies: retrying a pre-byte
- # trip would buy a second full 600s connect window before HTTP ever starts, and that
- # trip is as likely slow metadata as a broken Xet.
+ # Spend one more XET worker only on a DATA-phase verdict: retrying a pre-byte trip would buy a
+ # second full 600s connect window, and that trip is as likely slow metadata as a broken Xet.
retry_xet = (
can_retry_http
and state == "error"
@@ -1086,22 +1063,14 @@ def register_worker(
and _is_data_phase_stall(stalled[0])
and xet_attempt < _xet_attempt_budget()
)
- # Evidence for the WHOLE Xet phase: what earlier attempts held back, updated with this
- # worker's verdict. Reported once, when the phase ends.
+ # Evidence for the WHOLE Xet phase, reported once when the phase ends.
xet_failure = pending_xet_failure
if stalled:
- # Exclude only the PRE-BYTE trip: "did not start" means no byte ever arrived, as
- # likely slow metadata, a queue of HEADs or a cache lock as a broken Xet, and two
- # recorded failures pin this machine to HTTP for 24h. Everything else is evidence.
- # "did not resume" fires only after bytes HAVE flowed and is the shape this worker
- # hangs in most often (snapshot_download owns no partial between files); an earlier
- # allow-list keyed on "no progress" silently dropped it. Excluding one wording
- # rather than allowing one also fails cheaply if the shared wording changes.
- #
- # `state == "error"` is the other half: the watchdog appends its verdict before the
- # kill lands, so a worker that completed or was cancelled in that instant would be
- # charged a failure it did not earn -- and on the completed path that also skips the
- # success-clearing below, costing two streak steps the wrong way.
+ # Exclude only the PRE-BYTE trip: "did not start" is as likely slow metadata, a queue of HEADs or a
+ # cache lock as a broken Xet, and two recorded failures pin this machine to HTTP for 24h.
+ # state == "error" is the other half: the watchdog appends its verdict before the kill lands, so a
+ # worker that completed or was cancelled in that instant would be charged a failure it did not
+ # earn.
if state == "error" and _is_data_phase_stall(stalled[0]):
xet_failure = stalled[0]
else:
@@ -1114,23 +1083,17 @@ def register_worker(
if state == "cancelled" or (
state == "complete" and transport == download_registry.TRANSPORT_XET
):
- # A XET completion proved Xet works here, and a cancel was never evidence against
- # it: either way an earlier stall is dropped. An HTTP completion proves nothing
- # about Xet, so a verdict carried onto that rung is still charged below.
+ # A XET completion proved Xet works here and a cancel was never evidence against it; an HTTP
+ # completion proves nothing about Xet, so a verdict carried onto that rung is still charged.
xet_failure = None
if xet_failure is not None and not retry_xet:
# Xet phase over (HTTP next, or nothing left to try): report it, once.
_record_xet_failure(xet_failure, logger)
xet_failure = None
if not stalled and transport == download_registry.TRANSPORT_XET and state == "complete":
- # Clear the streak so "two failures in a row" means in a row: otherwise a stall
- # today and another next week count as consecutive despite every download between
- # them succeeding, pinning Auto to HTTP for no reason. Only a job that actually
- # moved bytes says anything about Xet's health though: a fully cached repo (the UI's
- # re-download on an up-to-date model) exits 0 without touching the network, and
- # clearing an earned demotion on that puts a bad machine back on Xet. Unmeasurable
- # means do not clear: a missed clear costs one streak entry, a wrong clear undoes
- # the demotion.
+ # Clear the streak so "two failures in a row" means in a row, but only for a job that moved bytes:
+ # a fully cached repo exits 0 without touching the network, and clearing an earned demotion on
+ # that puts a bad machine back on Xet.
bytes_after = _job_bytes_on_disk(repo_type, repo_id, _cache_dir, _own_blob_hashes)
if (
_bytes_before is not None
@@ -1138,9 +1101,8 @@ def register_worker(
and bytes_after > _bytes_before
):
_record_xet_success(logger)
- # Recovery: spend another XET worker if the stall looked recoverable and the budget
- # allows, else fall back to HTTP. One guard per direction: `transport == TRANSPORT_XET`
- # (inside can_retry_http) makes the HTTP rung terminal, `xet_attempt` bounds the XET one.
+ # One guard per direction: transport == TRANSPORT_XET makes the HTTP rung terminal, xet_attempt
+ # bounds the XET one.
if can_retry_http and state == "error":
_try_transport_retry(
registry,
@@ -1159,20 +1121,19 @@ def register_worker(
),
xet_attempt = xet_attempt + 1 if retry_xet else xet_attempt,
pending_xet_failure = xet_failure,
- # The ORIGINAL pre-Xet baseline: resampling would fold the killed worker's
- # partial writes in, so a recovered attempt would read as a cached no-op.
+ # The ORIGINAL pre-Xet baseline: resampling would fold the killed worker's partial writes in, so a
+ # recovered attempt would read as a cached no-op.
bytes_before = _bytes_before,
allow_ambient_token = allow_ambient_token,
)
except Exception:
if watchdog_stop is not None:
watchdog_stop.set()
- # finalize_worker_exit is the only thing that clears running/cancelling;
- # if it raises, force a terminal state so claim() isn't blocked until restart.
+ # finalize_worker_exit is the only thing that clears running/cancelling, so if it raises, force a
+ # terminal state or claim() is blocked until restart.
logger.exception("download watcher crashed for %s", key)
- # finalize may have raised before reaping the worker; terminate the
- # still-registered Popen first, else the terminal set_job clears the
- # repo guard and a live worker would race a retry on the same repo.
+ # finalize may have raised before reaping the worker, so terminate the still-registered Popen
+ # first: the terminal set_job clears the repo guard and a live worker would race a retry.
try:
kill_and_reap_process(proc, label = label, logger = logger)
except Exception:
@@ -1196,15 +1157,10 @@ def register_worker(
except Exception:
logger.exception("post-finalize marker cleanup failed for %s", key)
try:
- # The second look for anything prepare_cache_for_transport spared as too
- # recently written. A download runs for long enough that a partial orphaned
- # before it started is well past the grace by now, and the peer set still
- # shields a same-repo variant writing a shared companion right now.
- # This job's own blobs skip the abandonment wait, but ONLY once the job is
- # genuinely finished. A cancelled worker's partial was written seconds ago, so
- # the wait would strand it for the rest of the session; and its writer has just
- # been reaped, which is the very thing the wait is there to guess at. A retry
- # relaunched above leaves the job active, and then it is not ours to assume.
+ # The second look for anything prepare_cache_for_transport spared as too recently written. This
+ # job's own blobs skip the abandonment wait, but ONLY once the job is genuinely finished: a
+ # cancelled worker's partial was written seconds ago, and a retry relaunched above leaves the job
+ # active, so it is not ours to assume.
terminal = registry.get_job(key).state not in ("running", "cancelling")
_owned, _owns_all = (
_sweep_ownership(
@@ -1217,13 +1173,12 @@ def register_worker(
repo_type,
repo_id,
protected_blob_hashes = registry.peer_blob_hashes(key),
- # A companion a sibling is writing right now is still held back by
- # peer_blob_hashes above, whatever this job believes it owns.
+ # A companion a sibling is writing right now is still held back by peer_blob_hashes above, whatever
+ # this job believes it owns.
owned_blob_hashes = _owned,
owns_all_blobs = _owns_all,
- # The cache this worker actually wrote to. Resolving the live one instead
- # would miss the orphan whenever the download location changed mid-run,
- # and sweep a cache this job never touched.
+ # The cache this worker actually wrote to: resolving the live one would miss the orphan whenever the
+ # download location changed mid-run, and sweep a cache this job never touched.
root = _cache_dir,
)
if swept:
@@ -1254,13 +1209,12 @@ def launch_worker(
watch_name: str,
allow_ambient_token: bool = True,
) -> str:
- # Only the Xet success-recording consumes this, and sampling lazy-loads unsloth_zoo (so torch
- # and transformers) on the request path. An HTTP start skips it entirely.
+ # Only the Xet success-recording consumes this, and sampling lazy-loads unsloth_zoo, so torch and
+ # transformers, on the request path.
_baseline: Optional[int] = None
if transport == download_registry.TRANSPORT_XET:
# Before spawn(), deliberately: a small download can finalize its blobs while we are still
- # registering the process, and a later baseline would show no growth for a transfer that
- # really happened, leaving the streak uncleared.
+ # registering the process, and a later baseline would show no growth for a real transfer.
_get_metadata = getattr(registry, "get_job_metadata", None)
_metadata = _get_metadata(key) if callable(_get_metadata) else None
_baseline = _job_bytes_on_disk(
@@ -1313,8 +1267,8 @@ def cancel_worker(
logger,
) -> str:
proc = registry.get_process(key)
- # No worker process yet: arm a pending cancel so register_process kills it on
- # arrival during the claim-to-register window.
+ # No worker process yet: arm a pending cancel so register_process kills it on arrival during the
+ # claim-to-register window.
if proc is None:
if registry.mark_pending_cancel(key, generation):
return "cancelling"
@@ -1337,8 +1291,8 @@ def cancel_worker(
if not registry.request_cancel(key, proc, generation):
return registry.get_job(key).state
- # No eager marker: finalize_worker_exit writes it on a "cancelled" exit.
- # Persisting before the kill races a clean completion and strands a stale marker.
+ # No eager marker: finalize_worker_exit writes it on a "cancelled" exit, and persisting before the
+ # kill races a clean completion and strands a stale marker.
try:
proc.kill()
except ProcessLookupError:
@@ -1388,8 +1342,8 @@ def active_download_refs(
else:
ref_repo_id = metadata.repo_id if metadata is not None else ref.key
variant = None
- # Scoped jobs share one slot per repo, so publish the file list an adopting client needs to recognise its own transfer.
- # Absent metadata (a job hydrated before the registry knew it) reports null, which the client reads as unprovable.
+ # Scoped jobs share one slot per repo, so publish the file list an adopting client needs; absent
+ # metadata reports null, which the client reads as unprovable.
scoped_files = list(metadata.scoped_files) if metadata is not None else []
downloads.append(
ActiveDownload(
diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.py
index b1633572d63..67607cbe618 100644
--- a/studio/backend/hub/services/models/cache_inventory.py
+++ b/studio/backend/hub/services/models/cache_inventory.py
@@ -41,11 +41,9 @@ from hub.services.models.common import (
_runtime_for_format,
)
-# Imported at module scope (not inside the per-repo scan loop) so a broken
-# import surfaces at startup instead of silently emptying the inventory: the
-# scan loop swallows per-repo exceptions and would drop every repo. Lives under
-# ``utils`` (not ``utils.models``) to avoid the eager model-config/checkpoint
-# imports in ``utils/models/__init__.py``.
+# Imported at module scope so a broken import surfaces at startup instead of silently emptying the
+# inventory: the scan loop swallows per-repo exceptions. Lives under utils, not utils.models, to
+# avoid the eager model-config imports in utils/models/__init__.py.
from utils.paths.path_utils import is_appledouble_metadata
from utils.audio_tokens import detect_local_tts_audio_type
from utils.hidden_models import (
@@ -76,14 +74,14 @@ _cached_inventory_flights: dict[
tuple[asyncio.AbstractEventLoop, tuple[str, int]], asyncio.Task[list[dict]]
] = {}
-# Retrying a superseded scan is only worth it while invalidations are occasional;
-# past this the endpoint has to answer instead of restarting the walk forever.
+# Retrying a superseded scan is only worth it while invalidations are occasional; past this the
+# endpoint has to answer instead of restarting the walk forever.
_INVENTORY_SCAN_MAX_ATTEMPTS = 8
# Last scan per inventory name that confirmed its epoch, served when the cap is hit.
_last_confirmed_inventory: dict[str, list[dict]] = {}
-# Identity for a cached file with no HF blob (Windows without Developer Mode: hf
-# moves the blob into snapshots/ and leaves blobs/ empty).
+# Identity for a cached file with no HF blob: on Windows without Developer Mode hf moves the blob
+# into snapshots/ and leaves blobs/ empty.
_LOCAL_SIZE_IDENTITY_PREFIX = "size:"
@@ -187,9 +185,8 @@ def _repo_gguf_last_modified(repo_info) -> float:
def _repo_has_mmproj(repo_info) -> bool:
- # An mmproj file only makes a repo vision-capable when it is an actual GGUF
- # projector; a non-GGUF sidecar (e.g. mmproj_config.json) does not, and the
- # runtime's projector detection is GGUF-only.
+ # Only an actual GGUF projector makes a repo vision-capable; a non-GGUF sidecar
+ # (e.g. mmproj_config.json) does not, and the runtime's projector detection is GGUF-only.
return any(
_is_gguf_filename(f.file_name) and _is_mmproj_filename(f.file_name)
for revision in repo_info.revisions
@@ -217,8 +214,8 @@ def cached_repo_files(revision) -> list:
Every classification below reads these by name, and a "._" companion carries the described
file's own name, so it answers each one the way the real file does.
"""
- # The "._" name is on the snapshot entry, the bytes are in the content-addressed blob whose
- # own name carries no prefix -- but the entry is a symlink to it, so one open reads both.
+ # The "._" name is on the snapshot entry while the bytes are in the content-addressed blob, but the
+ # entry is a symlink to it, so one open reads both.
return [
f
for f in getattr(revision, "files", ()) or ()
@@ -409,8 +406,8 @@ def _cache_inventory_fields(
active_hub_cache = active_hub_cache,
payload_snapshots = payload_snapshots,
)
- # The directory this row loads from: the non-GGUF scan passes only
- # *identity*, so snapshot_path alone classified nothing.
+ # The directory this row loads from: the non-GGUF scan passes only identity, so snapshot_path alone
+ # classified nothing.
classify_snapshot = identity.load_snapshot or snapshot_path
can_chat_override = None
if classify_snapshot is not None:
@@ -433,27 +430,19 @@ def _cache_inventory_fields(
else repo_info is not None and _repo_has_mmproj(repo_info)
):
capabilities["supports_vision"] = True
- # Qwen3-ASR's required mmproj is an audio projector, not a vision
- # projector. Curated STT rows are visible only to task-scoped consumers and
- # must never become eligible for chat auto-load.
- # ``stt_only`` covers any repo whose config sniffs as Whisper, curated or not: a
- # third-party checkpoint or a user's own fine-tune is just as unchattable, and can_chat
- # is what auto-load and the chat picker filter on, neither of which looks at the task.
+ # Qwen3-ASR's required mmproj is an audio projector, not a vision one, and stt_only covers any
+ # repo whose config sniffs as Whisper: can_chat is what auto-load and the chat picker filter on.
if stt_only or is_curated_stt_repo_id(repo_id):
capabilities["supports_vision"] = False
capabilities["can_chat"] = False
- # Same rule for the speech-emitting half of the Audio page. The codec probe in
- # _local_transformers_can_chat covers uncurated safetensors copies; native audio
- # architectures are also passed explicitly. A GGUF repo ships no tokenizer_config
- # to probe, so the curated ids answer for those.
+ # The codec probe covers uncurated safetensors copies and native audio architectures are passed
+ # explicitly; a GGUF repo ships no tokenizer_config to probe, so the curated ids answer for those.
if tts_only or is_curated_tts_repo_id(repo_id):
capabilities["can_chat"] = False
if hidden_infra:
capabilities["can_chat"] = False
- # A VAE / text-encoder mirror holds no language model, so it cannot chat whatever its weight
- # format says. Set HERE rather than left to the row's companion flag alone: startup auto-load
- # filters on capabilities.can_chat (isChattableCachedRepo), not on that flag, so a row that
- # only carried the flag was still auto-loadable as a chat model.
+ # A VAE / text-encoder mirror holds no language model. Set HERE rather than left to the row's
+ # companion flag alone: startup auto-load filters on capabilities.can_chat, not on that flag.
if companion:
capabilities["can_chat"] = False
return {
@@ -502,9 +491,8 @@ def _cached_row_companion(repo_id: str, snapshot: Optional[Path] = None) -> bool
if normalized in native_companions:
return True
- # MOSS Local may name a different compatible tokenizer in its processor
- # config. Classify the codec architecture too, so that dynamically
- # resolved companion cannot surface as a chat checkpoint after download.
+ # MOSS Local may name a different compatible tokenizer, so classify the codec architecture too or
+ # that dynamically resolved companion surfaces as a chat checkpoint.
config = _read_json_object(snapshot / "config.json") if snapshot is not None else {}
model_type = str(config.get("model_type") or "").strip().lower()
if model_type in {
@@ -536,8 +524,8 @@ def _cached_row_task(
pick by it, so a row that arrives without one is dropped from those lists entirely.
"""
try:
- # Module-qualified: rebinding these names here re-points a load that resolved to
- # routes.models before the move, which verify_import_hoist.py blocks.
+ # Module-qualified: rebinding these names re-points a load that resolved to routes.models before the
+ # move, which verify_import_hoist.py blocks.
from hub.services.models import catalog_classification
return (
catalog_classification._repo_gguf_task(repo_info, selected)
@@ -581,14 +569,8 @@ def _scan_cached_gguf(
active_hub_cache = active_hub_cache,
)
except Exception as e:
- # Symmetric with _scan_cached_models below. The index is built once for the
- # whole scan and outside the per-repository try, so one malformed repo
- # identity anywhere in the cache took the endpoint with it: a cache
- # directory name holding a byte the filesystem encoding cannot decode
- # arrives as a lone surrogate, hashing it for the repo key raises, and
- # /api/hub/cached-gguf answers 500 with every valid row hidden. Falling
- # back to the per-repo reads this replaced costs speed on a broken cache
- # and nothing else.
+ # The index is built once for the whole scan and outside the per-repository try, so one undecodable
+ # cache directory name, hashed for the repo key, answered 500 with every valid row hidden.
logger.warning("Could not build shared cached-GGUF state index: %s", e)
variant_states = None
@@ -622,20 +604,16 @@ def _scan_cached_gguf(
str(snapshot_path) if snapshot_path is not None else None,
)
is_curated_stt = is_curated_stt_repo_id(repo_id)
- # Hide infra repos unless the user downloaded a variant via
- # the Hub; variant state only exists for user downloads. Exact
- # curated STT repos are still emitted as management rows and
- # the frontend's task allowlist keeps them out of Chat.
+ # Hide infra repos unless the user downloaded a variant: variant state only exists for user
+ # downloads, and curated STT repos are still emitted as management rows.
if is_hidden_infra and not is_curated_stt and not has_variant_state:
continue
if total_size == 0 and not has_variant_state:
continue
# Must run after the skips above and before the partial walk it scopes.
gguf_snapshot, gguf_payload_snapshots = _repo_gguf_payload_snapshots(repo_info)
- # Resolved before the row is classified, not inside _cache_inventory_fields
- # below: a load_id left as the repo id resolves through refs/main, which can
- # name an older revision than the newest payload snapshot. Classifying the
- # newest then described a revision the load never reads.
+ # Resolved before the row is classified: a load_id left as the repo id resolves through refs/main,
+ # which can name an older revision than the newest payload snapshot.
gguf_identity = _resolve_load_identity(
repo_id,
repo_path = repo_path,
@@ -677,8 +655,8 @@ def _scan_cached_gguf(
"partial": partial,
# A marker-only sibling moves neither size nor mtime.
"has_variant_state": has_variant_state,
- # GGUF row-level transport is ambiguous (variants may differ);
- # per-variant detail lives on GgufVariantDetail.
+ # GGUF row-level transport is ambiguous, since variants may differ; per-variant detail lives on
+ # GgufVariantDetail.
"partial_transport": None,
"partial_resumable": False,
}
@@ -729,9 +707,8 @@ def _scan_cached_inventory_snapshot(scanner, expected_epoch: int) -> list[dict]:
if hf_cache_scan.hf_cache_scans_epoch() != expected_epoch:
raise _CacheSourceChanged
rows = scanner(cache_scans = cache_scans, active_hub_cache = active_hub_cache)
- # The walk itself takes seconds, so a delete or a finished download landing
- # during it supersedes these rows just as surely as one landing before it:
- # without this a repo deleted mid-scan is still listed in the response.
+ # The walk itself takes seconds, so a delete or finished download landing during it supersedes these
+ # rows as surely as one landing before it.
if hf_cache_scan.hf_cache_scans_epoch() != expected_epoch:
raise _CacheSourceChanged
return rows
@@ -752,9 +729,8 @@ async def _shared_cached_inventory_scan(name: str, scanner) -> _CachedInventoryS
continue
_last_confirmed_inventory[name] = rows
return _CachedInventoryScan(rows, True)
- # Invalidations are arriving faster than the walk completes, so no scan will
- # confirm as current. Answer with the last one that did rather than spin a
- # full cache walk per epoch forever.
+ # Invalidations are arriving faster than the walk completes, so answer with the last scan that
+ # confirmed rather than spin a full cache walk per epoch forever.
logger.warning(
"Cached %s inventory kept racing cache invalidations; serving the last confirmed scan",
name,
@@ -907,15 +883,12 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload:
if snapshot is not None:
for config_name, key in (
("config.json", "has_config"),
- # A diffusers pipeline's root manifest, which plays exactly the role config.json
- # plays for transformers: from_pretrained reads it at the snapshot root to learn
- # the component layout. Without it here no pure pipeline snapshot could classify
- # (real repos ship model_index.json and per-component configs, never a root
- # config.json), payload_snapshots came back empty, and every cached diffusion base
- # pipeline was force-flagged partial and dropped from the On Device lists.
+ # model_index.json plays for a diffusers pipeline the role config.json plays for transformers:
+ # without it no pure pipeline snapshot could classify, and every cached diffusion base was
+ # force-flagged partial and dropped from On Device.
("model_index.json", "has_config"),
- # Modular Diffusers pipelines use this root manifest instead. Saved or custom
- # modular snapshots may omit both conventional root manifests.
+ # Modular Diffusers pipelines use this root manifest instead, and saved or custom modular snapshots
+ # may omit both.
("modular_model_index.json", "has_config"),
("adapter_config.json", "has_adapter_config"),
):
@@ -934,9 +907,8 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload:
)
def _revisions_of(fmt: str) -> tuple[list, list]:
- # Weights pool across revisions, so the pinned snapshot must classify alone; trusted=False
- # because from_pretrained needs config.json in that one directory. Filename classification
- # also matches torn revisions, so the whole ones are tracked separately.
+ # Weights pool across revisions, so the pinned snapshot must classify alone; trusted=False because
+ # from_pretrained needs config.json in that one directory.
snapshots = [
snapshot
for snapshot, flags in revision_flags
@@ -948,9 +920,9 @@ def _repo_non_gguf_model_payload(repo_info) -> _CachedNonGgufPayload:
payload_snapshots, complete = _revisions_of(model_format)
if not complete:
- # The repo-wide flags are OR-ed across revisions, so they can name a format whose every
- # revision is torn while another format has a whole one sitting right there. Load what
- # loads: an interrupted safetensors attempt must not hide a complete checkpoint.
+ # The repo-wide flags are OR-ed across revisions, so they can name a format whose every revision
+ # is torn while another format has a whole one: an interrupted safetensors attempt must not hide
+ # a complete checkpoint.
for candidate in ("safetensors", "checkpoint", "adapter"):
if candidate == model_format:
continue
@@ -1154,14 +1126,12 @@ def _scan_cached_models(
else None
),
)
- # A companion-only prefetch (pipeline manifest + VAE / text-encoder but no transformer shards, left by a GGUF load) passes
- # the download check yet cannot from_pretrained, so mark it partial and the pickers stop advertising it as on-device.
- # Scoped to the row's own snapshot, like the download check above it: the repo-wide twin picks the newest revision for
- # itself, so a newer companion-only snapshot beside the complete one refs/main resolves to marked this row partial.
+ # A companion-only prefetch passes the download check yet cannot from_pretrained, so mark it
+ # partial.
companion_only = hf_cache_scan.snapshot_pipeline_missing_denoiser(load_snapshot)
snapshot_partial = download_partial or companion_only
- # Flags are OR-ed over revisions, so no payload snapshot means no directory
- # serves the row and it would reach for the Hub.
+ # Flags are OR-ed over revisions, so no payload snapshot means no directory serves the row and it
+ # would reach for the Hub.
if not payload.payload_snapshots:
snapshot_partial = True
try:
@@ -1176,8 +1146,9 @@ def _scan_cached_models(
if is_whisper_stt
else (
"text-to-speech"
- # The probe answers for a repo whose card says nothing, so the
- # Audio page, which selects by task, still lists it.
+ # The probe answers for a repo whose card says nothing, so the Audio page, which
+ # selects by task,
+ # still lists it.
if is_output_audio or local_metadata.get("pipeline_tag") == "text-to-speech"
else _cached_row_task(repo_info, gguf = False, selected = load_snapshot)
)
@@ -1202,7 +1173,9 @@ def _scan_cached_models(
repo_id,
repo_cache_dir = repo_path,
)
- # Only a genuine download partial has a transport; a companion-only snapshot arrived intact and has no Resume story.
+ # Only a genuine download partial has a transport; a companion-only snapshot arrived
+ # intact and has
+ # no Resume story.
if download_partial
else None
),
@@ -1215,15 +1188,14 @@ def _scan_cached_models(
if download_partial
else False
),
- # Diffusion repos with no pipeline index load only via from_single_file, so the task pickers must not offer them as
- # pipeline loads. Without this the picker's single_file gate sees undefined on every hub-sourced row. Read from the
- # row's snapshot: the manifest a sibling revision carries is not the one this row's load_id opens.
+ # Diffusion repos with no pipeline index load only via from_single_file, so the task pickers must
+ # not offer them as pipeline loads.
"single_file": bool(
row_task is not None
and not hf_cache_scan.snapshot_has_pipeline_index(load_snapshot)
),
- # Listed so tens of GB of companion weights stay visible and deletable, but
- # flagged so no picker offers a denoiser-less repo as a load.
+ # Listed so tens of GB of companion weights stay visible and deletable, but flagged so no picker
+ # offers a denoiser-less repo as a load.
"companion": _cached_row_companion(repo_id, load_snapshot),
"diffusers": _cached_row_is_diffusers(repo_info, load_snapshot),
**local_metadata,
@@ -1246,9 +1218,8 @@ def _scan_cached_models(
tts_only = is_output_audio,
)
)
- # Native backend selection reads the load identity itself. A custom native
- # fork addressed only by repo id is indistinguishable from an ordinary LLM,
- # even though this inventory row detected its architecture from the snapshot.
+ # Native backend selection reads the load identity itself, so a custom native fork addressed only by
+ # repo id is indistinguishable from an ordinary LLM.
if native_audio_type and load_snapshot is not None:
row["load_id"] = str(load_snapshot)
if _prefer_cache_row(row, existing):
diff --git a/studio/backend/hub/services/models/catalog_classification.py b/studio/backend/hub/services/models/catalog_classification.py
index deef60252bd..22c6c5a3445 100644
--- a/studio/backend/hub/services/models/catalog_classification.py
+++ b/studio/backend/hub/services/models/catalog_classification.py
@@ -84,11 +84,9 @@ def _is_h3_bundle_gguf_hint(hint: Optional[str]) -> bool:
def _gguf_architecture(path: str) -> Optional[str]:
- # Every read inventory classification makes goes through here, so this is where "never open
- # a cloud placeholder" holds: opening one recalls its whole payload. The task probes below
- # ask the same question again because an unhydrated file needs a different ROUTE, not just a
- # skipped read; the audio-type probe has no route to pick and relies on this alone. Load-time
- # inspection calls read_gguf_architecture directly and still hydrates, on purpose.
+ # Every read inventory classification makes goes through here, so this is where "never open a
+ # cloud placeholder" holds: opening one recalls its whole payload. Load-time inspection calls
+ # read_gguf_architecture directly and still hydrates, on purpose.
if not file_contents_available_locally(path):
return None
from utils.models.gguf_metadata import read_gguf_architecture
@@ -165,11 +163,8 @@ def _unhydrated_gguf_task(name_hints: tuple[Optional[str], ...]) -> Optional[str
"""
if any(_is_h3_bundle_gguf_hint(hint) for hint in name_hints):
return _VIDEO_GEN_TASK
- # Leaves only. A filesystem row hands over its whole path, and family detection matches a
- # keyword in ANY segment of it, so a chat GGUF under .../FLUX.1-dev-GGUF/extra/ read as
- # text-to-image. With an architecture that mismatch only picks the wrong family; here the
- # name is the entire case, so an ancestor directory -- the user's shelf, not this model --
- # must not decide it.
+ # Leaves only: family detection matches a keyword in ANY path segment, so a chat GGUF under
+ # .../FLUX.1-dev-GGUF/extra/ read as text-to-image.
return _name_hint_media_task(tuple(_hint_leaf(hint) for hint in name_hints if hint), None)
@@ -274,8 +269,8 @@ def _gguf_folder_task(
fallback: Optional[str] = None
try:
scored: list[tuple[tuple[str, str], Path]] = []
- # Recorded as the tail is dropped, never inferred from `scored` afterwards: trimming cuts
- # back to the cap, so an overflowing folder is indistinguishable from one that fit.
+ # Recorded as the tail is dropped, never inferred from scored afterwards: trimming cuts back to the
+ # cap, so an overflowing folder would be indistinguishable from one that fit.
overflowed = False
for path in _iter_gguf_paths(root, deadline):
name = path.name
@@ -288,8 +283,7 @@ def _gguf_folder_task(
overflowed = True
scored.sort(key = lambda item: item[0])
paths = [path for _, path in scored[:_MAX_TASK_CLASSIFY_GGUFS]]
- # Whether every candidate made it into `paths`: the walk gives up at its own deadline and
- # the cap drops the tail, so either can leave a sibling unseen.
+ # The walk gives up at its own deadline and the cap drops the tail, so either can leave a sibling unseen.
complete = (
not overflowed
and len(scored) <= _MAX_TASK_CLASSIFY_GGUFS
@@ -309,8 +303,8 @@ def _gguf_folder_task(
if file_contents_available_locally(path):
task = _arch_to_task(_gguf_architecture(str(path)), name_hints = hints)
else:
- # Its header stays unread, so a name that says nothing leaves the candidate
- # unclassified rather than voting text-generation for the whole folder.
+ # Its header stays unread, so a name that says nothing leaves the candidate unclassified rather than
+ # voting text-generation for the whole folder.
task = _unhydrated_gguf_task(hints)
except Exception:
# Unread, so unranked: this file might have been the runnable sibling.
@@ -322,8 +316,8 @@ def _gguf_folder_task(
continue
if task in _LOADABLE_MEDIA_GGUF_TASKS:
return task
- # Speech is last resort: nothing here runs a llama-csm GGUF, so answering speech while a
- # sibling is loadable hides that sibling. A speech-only folder still tags speech.
+ # Speech is last resort: nothing here runs a llama-csm GGUF, so answering speech while a sibling is
+ # loadable hides that sibling.
if task == _SPEECH_TASK:
if speech is None:
speech = task
@@ -541,10 +535,9 @@ def _repo_is_diffusers(repo_info, selected: Optional[Path] = None) -> bool:
return True
except Exception:
pass
- # Video too, as _local_is_diffusers already asks. _cached_repo_task returns None for an
- # unbuildable video repo, so a single-file video checkpoint with no pipeline index carried
- # no task and no diffusers flag, and an inconclusive config leaves can_chat set: every gate
- # the chat picker has passes and the video weights reach the text loader.
+ # _cached_repo_task returns None for an unbuildable video repo, so a single-file video checkpoint
+ # with no pipeline index carried no task and no diffusers flag, and an inconclusive config leaves
+ # can_chat set: the video weights reach the text loader.
try:
from core.inference.video_families import detect_video_family
return detect_video_family(repo_id) is not None
diff --git a/studio/backend/hub/services/models/common.py b/studio/backend/hub/services/models/common.py
index d50a5644515..3b7f9bc9d87 100644
--- a/studio/backend/hub/services/models/common.py
+++ b/studio/backend/hub/services/models/common.py
@@ -34,8 +34,8 @@ LocalModelSource = Literal["models_dir", "hf_cache", "lmstudio", "ollama", "cust
def _safe_is_dir(path) -> bool:
- # Py >= 3.12 propagates PermissionError (EACCES) from is_dir(); folder scans
- # probe root-owned system dirs, so treat un-stat-able paths as not-a-dir.
+ # Py >= 3.12 propagates PermissionError (EACCES) from is_dir(), and folder scans probe root-owned
+ # system dirs, so treat un-stat-able paths as not-a-dir.
try:
return Path(path).is_dir()
except OSError:
@@ -165,8 +165,7 @@ _NON_GENERATIVE_ARCHITECTURE_SUFFIXES = (
"ForVideoClassification",
"ForZeroShotImageClassification",
)
-# Generative, but not of a chat reply: they match a generative suffix below yet
-# cannot answer a text turn, and they are small enough to be tried first.
+# Generative, but not of a chat reply: they match a generative suffix below yet cannot answer a text turn.
_NON_CHAT_GENERATIVE_MODEL_TYPES = frozenset(
{
"blip",
@@ -250,8 +249,8 @@ _ENCODER_ONLY_MODEL_TYPES = frozenset(
"squeezebert",
"vision-text-dual-encoder",
"xlm-roberta",
- # Vision and audio backbones: their bare ``*Model`` names carry no task
- # suffix, so only the model type identifies them.
+ # Vision and audio backbones: their bare *Model names carry no task suffix, so only the model type
+ # identifies them.
"beit",
"convnext",
"convnextv2",
@@ -292,8 +291,8 @@ def _read_local_json_object(path: Path) -> dict:
return {}
data = json.loads(path.read_text(encoding = "utf-8"))
return data if isinstance(data, dict) else {}
- # ValueError covers JSONDecodeError and UnicodeDecodeError; deeply nested
- # JSON raises RecursionError, which is neither.
+ # ValueError covers JSONDecodeError and UnicodeDecodeError; deeply nested JSON raises
+ # RecursionError, which is neither.
except (ValueError, OSError, RecursionError):
return {}
@@ -309,14 +308,12 @@ def _local_transformers_can_chat(path: Path) -> Optional[bool]:
if not _safe_is_dir(path):
return None
- # Before every architecture test below: a TTS model is an ordinary causal LM wearing
- # a codec vocabulary (Orpheus is LlamaForCausalLM), so the suffix rules answer True
- # and auto-load, which prefers the smallest, then picks it.
+ # Before every architecture test below: a TTS model is an ordinary causal LM wearing a codec
+ # vocabulary (Orpheus is LlamaForCausalLM), so the suffix rules answer True and auto-load picks it.
if detect_local_tts_audio_type(path) is not None:
return False
- # SentenceTransformers exports carry this even when the config names a
- # broadly reusable encoder class.
+ # SentenceTransformers exports carry this even when the config names a broadly reusable encoder class.
try:
if (path / "modules.json").is_file():
return False
@@ -341,8 +338,7 @@ def _local_transformers_can_chat(path: Path) -> Optional[bool]:
)
model_type_raw = config.get("model_type")
normalized_type = model_type_raw.strip().lower() if isinstance(model_type_raw, str) else ""
- # Before the generative suffix: Whisper and friends end in
- # ForConditionalGeneration but cannot answer a text turn.
+ # Before the generative suffix: Whisper and friends end in ForConditionalGeneration but cannot answer a text turn.
if normalized_type in _NON_CHAT_GENERATIVE_MODEL_TYPES or any(
name in _NON_CHAT_GENERATIVE_ARCHITECTURES for name in names
):
@@ -351,16 +347,13 @@ def _local_transformers_can_chat(path: Path) -> Optional[bool]:
return True
if names and all(name.endswith(_NON_GENERATIVE_ARCHITECTURE_SUFFIXES) for name in names):
return False
- # AutoModel.save_pretrained on a chat family writes the backbone name, and a
- # backbone has no LM head. Listed explicitly, not shape-matched, so an
- # unfamiliar FooModel still fails open.
+ # AutoModel.save_pretrained on a chat family writes the backbone name, which has no LM head. Listed
+ # explicitly, not shape-matched, so an unfamiliar FooModel still fails open.
if names and all(name in _BARE_TEXT_BACKBONE_ARCHITECTURES for name in names):
return False
- # The type alone decides: requiring the name shape too kept rows chat-capable
- # when it did not fit, e.g. google/siglip2-* omits architectures entirely and
- # CLIPTextModelWithProjection ends in neither Model nor a known suffix.
- # Anything generative returned True above, so no chat row reaches here.
+ # The type alone decides: requiring the name shape too kept rows chat-capable when it did not fit,
+ # e.g.
if normalized_type in _ENCODER_ONLY_MODEL_TYPES:
return False
return None
@@ -393,18 +386,16 @@ def _base_transformers_can_chat(
except (OSError, RuntimeError, ValueError):
return None
- # The adapter's own root first, then the active root, then the configured ones. The scan
- # covers legacy and previously configured roots, so an adapter can be listed from an
- # inactive root with its base cached beside it; the active root alone answered None there,
- # and None is inconclusive, which left a Whisper or encoder LoRA in the chat picker.
+ # The scan covers legacy and previously configured roots, so an adapter can be listed from an
+ # inactive root with its base cached beside it; the active root alone answered None, which is
+ # inconclusive and left encoder LoRAs in the chat picker.
try:
from huggingface_hub import try_to_load_from_cache
except Exception:
return None
- # Each source collected independently: under one try, a failure enumerating the OPTIONAL
- # extra roots discarded the adapter's own root too and answered None, the same fail-open
- # this function exists to close.
+ # Each source collected independently: under one try, a failure enumerating the OPTIONAL extra roots
+ # discarded the adapter's own root too and answered None.
roots: list[Path] = []
def _add(root: Optional[Path]) -> None:
@@ -438,7 +429,6 @@ def _base_transformers_can_chat(
except Exception:
continue
# A non-str is _CACHED_NO_EXIST ("we know it is absent here") or None ("unknown"),
- # and neither rules the base out of a different root.
if isinstance(found, str):
config_path = found
break
@@ -553,8 +543,8 @@ def _apply_format_aware_partial(
if not target:
rewritten.append(row)
continue
- # GGUF row-level transport is ambiguous (variants may differ); per-variant
- # detail lives on GgufVariantDetail.partial_transport via the variants endpoint.
+ # GGUF row-level transport is ambiguous, since variants may differ; per-variant detail lives on
+ # GgufVariantDetail.partial_transport.
partial_transport = None if row.model_format == "gguf" else snapshot_partial_transport
rewritten.append(
row.model_copy(
diff --git a/studio/backend/hub/services/models/companion_cleanup.py b/studio/backend/hub/services/models/companion_cleanup.py
index 3c251c236e6..856d5bac61f 100644
--- a/studio/backend/hub/services/models/companion_cleanup.py
+++ b/studio/backend/hub/services/models/companion_cleanup.py
@@ -115,8 +115,8 @@ def _remaining_main_gguf_variants(repo_info, *, excluding: Optional[str] = None)
pass
if not _is_main_gguf_filename(name):
continue
- # The delete this previews ignores proven metadata, so counting it here reports a
- # checkpoint as surviving that the deletion itself does not see.
+ # The delete this previews ignores proven metadata, so counting it here would report a checkpoint as
+ # surviving that the deletion itself does not see.
if path and is_appledouble_metadata(Path(path)):
continue
key = gguf_variant_key(name).lower()
@@ -157,7 +157,6 @@ def _delete_impact_blocking(repo_id: str, variant: Optional[str]) -> dict:
reclaimed += _variant_bytes(repo_info, variant) if variant else _repo_blob_bytes(repo_info)
# Would this delete leave the repo with no runnable checkpoint? Only then can its companions
- # become reclaimable; while a sibling quant survives they stay in use.
removes_last_checkpoint = True
if variant:
for repo_info in repos:
@@ -185,18 +184,13 @@ def _delete_impact_blocking(repo_id: str, variant: Optional[str]) -> dict:
entry = {"repo_id": display, "size_bytes": base_bytes, "needed_by": holders}
if holders:
retained.append(entry)
- # The SAME offerability test orphan_companions_response applies, because this row is a
- # pointer at that list. A borrowed chat GGUF repo (the native Qwen-Image text encoder) is a
- # curated companion id but holds a denoiser, so the orphan endpoint skips it; advertising
- # it here sent the user to Free up space to remove a row that is never there. Per copy,
- # like the listing: one pipeline copy in another cache root must not hide a companion-only
- # copy that Free up space really will offer.
+ # The SAME offerability test orphan_companions_response applies, since this row points at that
+ # list: a borrowed chat GGUF repo is a curated companion id but holds a denoiser, so advertising
+ # it sent the user to Free up space to remove a row that is never there.
elif base_key in offerable and any(not _repo_holds_denoiser(r) for r in base_repos):
freeable.append(entry)
- # Else: a base only a recorded link names. It is unheld, but the orphan endpoint is
- # table-only by design (a mis-recorded link must never turn an unrelated repo into a
- # delete candidate), so advertising it here pointed the user at a Free up space list it
- # will never appear in.
+ # A base only a recorded link names: the orphan endpoint is table-only by design, so advertising it
+ # here pointed the user at a Free up space list it will never appear in.
return {
"repo_id": repo_id,
@@ -204,10 +198,9 @@ def _delete_impact_blocking(repo_id: str, variant: Optional[str]) -> dict:
"reclaimed_bytes": reclaimed,
"retained_companions": retained,
"freeable_companions": freeable,
- # Same predicate the destructive path uses, so the dialog says what the delete will do.
- # A variant delete usually cannot strand anything, but the native Qwen-Image encoder is a
- # named quant inside a chat GGUF repo, and previewing only whole-repo deletes left Delete
- # enabled on it and the refusal to arrive after the user confirmed.
+ # Same predicate the destructive path uses: the native Qwen-Image encoder is a named quant inside
+ # a chat GGUF repo, so previewing only whole-repo deletes left Delete enabled and the refusal
+ # arriving after the user confirmed.
"blocked_by": (
companion_dependents(repo_id, scans, ignore_repo_ids = [repo_id])
if companion_assets.is_companion_base(repo_id)
@@ -245,27 +238,20 @@ def _orphan_companions_blocking() -> dict:
if required.get(base_key):
continue
repos = by_id[base_key]
- # A repo that holds a runnable denoiser is a model the user installed, not a leftover.
- # Several curated bases are perfectly good pipelines in their own right, so this is the
- # difference between the two ways the same repo id reaches the cache: a companion fetch
- # takes everything BUT the denoiser folder (``_base_file_downloaded`` skips
- # ``transformer/``), while a pipeline pick takes it. Its presence is therefore the
- # derived answer to "did the user ask for this repo, or did a GGUF drag it in".
- # Per COPY, not pooled: the loop below emits one row per cache root because a delete is
- # scoped to one, so a full pipeline copy in one root must not suppress an orphaned
- # companion-only copy in another that nothing can otherwise reclaim.
+ # A repo holding a runnable denoiser is a model the user installed: a companion fetch takes
+ # everything BUT transformer/, while a pipeline pick takes it, so its presence answers whether the
+ # user asked for this repo. Per COPY, since a delete is scoped to one cache root.
repos = [r for r in repos if not _repo_holds_denoiser(r)]
if not repos:
continue
- # One row per cache root. A delete is scoped to a single cache, so pooling copies from
- # several would promise bytes one removal cannot deliver.
+ # One row per cache root: a delete is scoped to a single cache, so pooling copies from several would
+ # promise bytes one removal cannot deliver.
for repo in repos:
size = _repo_blob_bytes(repo)
if size <= 0:
continue
- # The repo dir itself, not its parent: ``scoped_delete_root`` resolves the owning
- # cache by walking up to the ``models--`` component, so a bare root resolves to
- # nothing and the delete comes back "Invalid cache_path".
+ # The repo dir itself, not its parent: scoped_delete_root walks up to the models-- component, so a
+ # bare root resolves to nothing and the delete comes back "Invalid cache_path".
try:
cache_path = str(Path(getattr(repo, "repo_path")))
except (TypeError, OSError):
diff --git a/studio/backend/hub/services/models/deletion.py b/studio/backend/hub/services/models/deletion.py
index 726c728cd30..eead89e4d5c 100644
--- a/studio/backend/hub/services/models/deletion.py
+++ b/studio/backend/hub/services/models/deletion.py
@@ -126,8 +126,7 @@ def _repo_file_matches(target_repo, predicate) -> list[tuple[Path, Optional[Path
continue
if not file_path:
continue
- # Every predicate here keys on the name, which a sidecar answers exactly as its
- # neighbour does, so it would be counted as a deleted model in its own right.
+ # Every predicate here keys on the name, which a sidecar answers exactly as its neighbour does.
# Proven metadata only: anything else carrying this key is a file to delete.
if is_appledouble_metadata(Path(file_path)):
continue
@@ -156,10 +155,9 @@ def _remove_empty_variant_dirs(target_repos: list, variant: str) -> tuple[int, l
"""Remove now-empty ``snapshots/<rev>/<quant>/`` folders for *variant* (the
quant label names the folder); only empty dirs go, so siblings are safe.
Returns (count removed, removal failures other than a concurrent refill)."""
- # A qualified key names its own folder; its quant token belongs to sibling checkpoints too,
- # so it must not reach for a <quant>/ dir it does not own. Qualified means a path
- # (``distilled/...-Q6_K``), an H3 root stem, or a bpw modifier (``IQ4_XS-3.53bpw``, whose
- # token-only ``IQ4_XS/`` folder is a different build's).
+ # A qualified key names its own folder and must not reach for a <quant>/ dir it does not own;
+ # qualified means a path (distilled/...-Q6_K), an H3 root stem, or a bpw modifier
+ # (IQ4_XS-3.53bpw, whose token-only IQ4_XS/ folder is a different build's).
qualified = (
is_qualified_gguf_variant_key(variant)
or (quant_token_with_bpw(variant) or "").lower() == variant.lower()
@@ -201,8 +199,8 @@ def _remove_empty_variant_dirs(target_repos: list, variant: str) -> tuple[int, l
sub.rmdir()
removed += 1
except OSError as e:
- # A concurrent download refilling the dir (ENOTEMPTY) is not a
- # failure; a read-only cache or locked dir is, so surface it.
+ # A concurrent download refilling the dir (ENOTEMPTY) is not a failure; a read-only cache or locked
+ # dir is, so surface it.
if e.errno != errno.ENOTEMPTY:
failures.append(f"{sub.name}: {e}")
return removed, failures
@@ -255,8 +253,8 @@ def _variant_keys_to_delete(target_repo, variant: str) -> set[str]:
}
if wanted in keys:
return {wanted}
- # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant
- # names both partitions, so it must not delete either.
+ # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant names
+ # both partitions, so it must not delete either.
aliased = {key for key in keys if "/" in key and bare_quant_alias(key).lower() == wanted}
return aliased if len(aliased) == 1 else {wanted}
@@ -295,11 +293,7 @@ def _delete_gguf_variant_from_repos(
if matched and not sibling_active and not _has_remaining_main_gguf(target_repo):
companion_matches = _repo_file_matches(
target_repo,
- # Companions: mmproj and the drafters Unsloth downloads (MTP with
- # every variant, DSpark on opt-in). No main GGUF is left, so they
- # cannot be launched; reclaim them with the last variant. An imatrix
- # joins them: no longer offered as a variant, so a copy an older build
- # fetched as one would be unreachable from the UI.
+ # No main GGUF is left, so the companions cannot be launched; reclaim them with the last variant.
lambda name: _is_gguf_filename(name)
and (
_is_mmproj_filename(name)
@@ -510,8 +504,8 @@ def reclaim_replaced_gguf_variant(
and gguf_variant_key(name).lower() == variant_key,
)
for snap, blob, name in matches:
- # Prune only a file we can identify as a real, stale cache blob. A
- # no-symlink snapshot file has no identifiable blob hash, so keep it.
+ # Prune only a file we can identify as a real, stale cache blob: a no-symlink snapshot file has no
+ # identifiable blob hash, so keep it.
blob_hash = (
_blob_hash_from_path(blob)
if cache_inventory._is_real_cache_blob(blob, repo_dir)
@@ -691,7 +685,8 @@ def _diffusion_blocks_delete(repo_id: str) -> Optional[str]:
if status.get("loaded") and status.get("repo_id"):
if _loaded_id_matches_repo(str(status["repo_id"]), repo_id):
return "Unload the model before deleting"
- # sd.cpp re-reads companion VAE / text-encoder files every generation and status().repo_id covers only the main GGUF, so refuse the companions too.
+ # sd.cpp re-reads companion VAE / text-encoder files every generation and status().repo_id covers
+ # only the main GGUF, so refuse the companions too.
for lid in getattr(engine, "loaded_repo_ids", tuple)():
if _loaded_id_matches_repo(str(lid), repo_id):
return "Unload the model before deleting"
@@ -716,13 +711,14 @@ def _video_blocks_delete(repo_id: str) -> Optional[str]:
return None
status = backend.status()
if status.get("loaded"):
- # repo_id names the checkpoint; for a GGUF / single-file load the companion base supplies the VAE and text encoders, so refuse it too.
+ # repo_id names the checkpoint; for a GGUF / single-file load the companion base supplies the VAE
+ # and text encoders, so refuse it too.
for key in ("repo_id", "base_repo"):
held = status.get(key)
if held and _loaded_id_matches_repo(str(held), repo_id):
return "Unload the model before deleting"
# The native H3 runtime re-reads its Qwen encoder and both VAEs from companion repos that are
- # neither of the two ids above, so refuse those as well, exactly as the Images guard does.
+ # neither of the two ids above, so refuse those as well.
for lid in getattr(backend, "loaded_repo_ids", tuple)():
if _loaded_id_matches_repo(str(lid), repo_id):
return "Unload the model before deleting"
@@ -807,10 +803,7 @@ async def delete_cached_model_response(
detail = f"Invalid gguf_variant: {variant!r}",
)
- # Guard fails closed: if a live backend's load state can't be read, abort
- # with 503 rather than risk unlinking weights under a running process.
- # Every guard is sync and the chat ones reach get_inference_backend(), whose cold build waits
- # on hardware detection. One worker keeps the `or` short-circuit and keeps the event loop free.
+ # Fail closed with 503 rather than unlink weights under a running process.
def _load_state_blocks_delete() -> Optional[str]:
if _llama_cpp_blocks_delete(repo_id, variant) or (
_inference_backend_blocks_delete(repo_id)
@@ -842,10 +835,9 @@ async def delete_cached_model_response(
)
raise HTTPException(status_code = 400, detail = detail)
try:
- # Re-derived now the scope is reserved, as only_if_orphan re-derives its own answer
- # below. The first read ran before the reservation existed, so a load starting in
- # between published its claim too late to be seen, and begin_delete misses it too:
- # image and video loads download directly rather than through a registry claim.
+ # Re-derived now the scope is reserved, as only_if_orphan re-derives its own answer below: the
+ # first read ran before the reservation existed, so a load starting in between published its claim
+ # too late, and begin_delete misses it too.
try:
blocks_detail = await asyncio.to_thread(_load_state_blocks_delete)
except Exception as e:
@@ -880,12 +872,8 @@ def _delete_cached_model_blocking(
*,
only_if_orphan: bool = False,
) -> dict:
- # Free up space asks for this: the row it is removing was an orphan when the list was built,
- # and the list can be minutes old. A download of that same repo finishing in the background
- # turns it into an installed checkpoint, and neither guard below catches that -- begin_delete
- # only refuses a download still in flight, and the companion guard deliberately ignores the
- # target as its own dependent. Re-derived here, after begin_delete has closed the repo to new
- # downloads, so the answer cannot go stale between the check and the unlink.
+ # Free up space's list can be minutes old, and a background download finishing turns that orphan
+ # into an installed checkpoint neither guard below catches.
if only_if_orphan:
from hub.services.models import companion_cleanup
from hub.utils import companion_assets
@@ -894,9 +882,8 @@ def _delete_cached_model_blocking(
copies = companion_cleanup._repos_by_id(cache_inventory.all_hf_cache_scans()).get(
repo_id.strip().lower(), []
)
- # Only the copy being removed. The orphan listing emits one row per cache root
- # precisely because a delete is scoped to one, so a full-pipeline copy in another
- # remembered cache must not veto removing the companion-only copy that was listed.
+ # Only the copy being removed: the orphan listing emits one row per cache root, so a full-pipeline
+ # copy in another remembered cache must not veto removing the companion-only copy listed.
if cache_path:
wanted = Path(cache_path)
copies = [
@@ -905,11 +892,8 @@ def _delete_cached_model_blocking(
if getattr(r, "repo_path", None) and Path(getattr(r, "repo_path")) == wanted
]
if not copies:
- # No fallback to the other copies. An empty match means the target root is
- # not in this scan (its scan failed, or the copy is gone), and the delete
- # below can still purge that directory by path -- so concluding "orphan" from
- # copies we did not look at is exactly the fail-open this precondition exists
- # to prevent. Raising here lands in the fail-closed 503 below.
+ # An empty match means the target root is not in this scan, and concluding "orphan" from copies we
+ # did not look at is the fail-open this precondition prevents; raising lands in the 503.
raise RuntimeError(f"cache root not present in the scan: {cache_path}")
still_orphan = not any(companion_assets.repo_holds_denoiser(repo) for repo in copies)
except Exception as e:
@@ -928,26 +912,18 @@ def _delete_cached_model_blocking(
)
# A companion base repo carries the text encoders, VAE and tokenizer for every quant of its
- # family, so removing it while one is installed leaves that quant unloadable with nothing on
- # screen to say why. Derived from what is installed right now, never from a stored count, and
- # only for a WHOLE-repo delete: a variant delete cannot touch another repo. Deleting the
- # dependants first makes the base an orphan, which Free up space then offers.
- #
- # Here rather than in the async caller so it shares this function's cache walk and its stubs:
- # the check IS part of the destructive stage, and a caller that replaces that stage should not
- # end up with half of it still running.
- # A variant delete normally cannot touch another repo, so the guard is a whole-repo check.
- # The exception is a companion whose asset IS a named GGUF variant: native Qwen-Image opens
- # exactly Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf inside a chat GGUF repo, so removing that one
- # quant strands the image checkpoint however many siblings remain, and none of them is a
- # substitute for a fixed filename. A FLAG, never a rewrite of `variant`: that name is the
- # destructive scope, and widening it here would delete every revision and manifest the user
- # did not ask for, and purge a sibling quant out from under an in-flight download.
+ # family, so removing it while one is installed leaves that quant unloadable. Derived from what is
+ # installed right now, and only for a WHOLE-repo delete.
+ # Here rather than in the async caller so it shares this function's cache walk and stubs: the check
+ # IS part of the destructive stage.
+ # The exception is a companion whose asset IS a named GGUF variant: native Qwen-Image opens one
+ # fixed filename inside a chat GGUF repo, so removing that quant strands the image checkpoint. A
+ # FLAG, never a rewrite of `variant`: widening the scope would delete revisions the user did not
+ # ask for.
guard_this_delete = variant is None or _variant_is_a_required_companion_asset(repo_id, variant)
if guard_this_delete and _is_companion_base_repo(repo_id):
- # Fails CLOSED, and only here: the lookup above already established this repo IS a
- # companion base, so an unreadable cache means the dependants cannot be enumerated, not
- # that there are none. Every other repo skips the check entirely and is unaffected.
+ # Fails CLOSED, and only here: the lookup above already established this repo IS a companion base,
+ # so an unreadable cache means the dependants cannot be enumerated, not that there are none.
try:
shared_detail = _companion_share_blocks_delete(repo_id)
except Exception as e:
@@ -963,17 +939,16 @@ def _delete_cached_model_blocking(
raise HTTPException(status_code = 400, detail = shared_detail)
try:
- # If a sibling quant is downloading concurrently, restrict this delete to
- # the variant's own files and leave the shared mmproj companion for it.
+ # If a sibling quant is downloading concurrently, restrict this delete to the variant's own files
+ # and leave the shared mmproj companion for it.
sibling_active = bool(
variant and downloads.registry.has_active_peer_variant(repo_id, variant)
)
cache_scans = cache_inventory.all_hf_cache_scans()
- # A repo can live in several remembered caches. Group its copies by the
- # cache root that owns each, then target exactly one cache so a delete
- # never removes copies in other, previously selected caches.
+ # A repo can live in several remembered caches, so target exactly one or a delete removes copies in
+ # other, previously selected caches.
owners: dict = {}
for hf_cache in cache_scans:
for repo_info in hf_cache.repos:
diff --git a/studio/backend/hub/services/models/downloads.py b/studio/backend/hub/services/models/downloads.py
index 7bca8476f77..4704466408b 100644
--- a/studio/backend/hub/services/models/downloads.py
+++ b/studio/backend/hub/services/models/downloads.py
@@ -47,7 +47,8 @@ def _download_job_key(repo_id: str, variant: Optional[str]) -> str:
)
-# A scope rides the variant slot as "@name". No GGUF quant label starts with "@", so a scoped job never collides with a real variant or the full snapshot.
+# A scope rides the variant slot as "@name". No GGUF quant label starts with "@", so a scoped job
+# never collides with a real variant or the full snapshot.
_SCOPE_PREFIX = "@"
@@ -217,7 +218,7 @@ async def download_model_response(
variant = scope_variant
key = _download_job_key(repo_id, variant)
# Off the event loop: resolving "auto" can run the Xet reachability probe, and a blackholed DNS
- # makes that outlast its 3s budget while every other Unsloth request waits behind it.
+ # makes that outlast its 3s budget while every other request waits behind it.
use_xet, transport_reason = await asyncio.to_thread(
download_lifecycle.resolve_requested_use_xet,
getattr(body, "transport_mode", None),
@@ -303,9 +304,9 @@ async def download_model_response(
"this one."
),
)
- # claim_state is the blocking job's state. Attaching and accepting are
- # one verdict: only this key's own in-flight job can be joined, and a
- # cross-variant conflict or in-progress delete joined nothing.
+ # claim_state is the blocking job's state. Attaching and accepting are one verdict: only this
+ # key's own in-flight job can be joined, and a cross-variant conflict or in-progress delete joined
+ # nothing.
adoptable = _registry.adoptable(key)
return {
"job_key": key,
@@ -313,11 +314,11 @@ async def download_model_response(
"accepted": adoptable,
"attached": adoptable,
"generation": generation,
- # An adopted job keeps the transport it started on, so report it
- # rather than let the caller assume the one it asked for.
+ # An adopted job keeps the transport it started on, so report it rather than let the caller assume
+ # the one it asked for.
"transport": _registry.job_transport(key),
- # And its cancel marker: a run that fell back from Xet to HTTP
- # still cancels into a restart-only partial.
+ # And its cancel marker: a run that fell back from Xet to HTTP still cancels into a restart-only
+ # partial.
"cancel_transport": _registry.job_cancel_transport(key),
}
download_manifest.clear_cancel_marker(
@@ -326,8 +327,8 @@ async def download_model_response(
variant,
hub_cache = cache_paths.hub_cache,
)
- # Blobs a concurrent same-repo variant is already writing (e.g. a shared
- # mmproj). The worker must not purge these during cache preparation.
+ # Blobs a concurrent same-repo variant is already writing, such as a shared mmproj: the worker must
+ # not purge these during cache preparation.
protected_blob_hashes = _registry.peer_blob_hashes(key) if variant else frozenset()
label = f"{repo_id}{f' [{variant}]' if variant else ''}"
@@ -361,9 +362,8 @@ async def download_model_response(
"accepted": True,
"attached": False,
"generation": generation,
- # The transport that was actually resolved: an explicit "xet" is
- # downgraded to HTTP where hf_xet is unavailable, and a client that
- # assumed its request stood would offer the wrong stop control.
+ # The transport that was actually resolved: an explicit "xet" is downgraded to HTTP where hf_xet is
+ # unavailable, and a client that assumed its request stood would offer the wrong stop control.
"transport": transport,
}
@@ -456,9 +456,8 @@ def _variant_transport_status(repo_id: str, variant: str, hf_token: Optional[str
repo_id,
variant,
)
- # A partial only counts toward "resumable" while a writer that reopens it is installed,
- # since the UI turns the flag into "Resume with HTTP to keep the progress you already
- # have" and the next download start sweeps whatever cannot be reopened.
+ # A partial counts toward "resumable" only while a writer that reopens it is installed, since the
+ # next download start sweeps whatever cannot be reopened.
resumable_hashes = download_registry.incomplete_blob_hashes(
"model",
repo_id,
@@ -540,36 +539,29 @@ def _variant_manifest_decision(
caller's catalog-hinted total. Active cache first, so the common case is one
lookup; every candidate found has to agree before one is returned.
"""
- # The active cache's manifest is a candidate like any other, NOT an early return. Its repo
- # dir can be gone while its scoped state still holds an old manifest, and idle progress goes
- # on scanning the remembered caches -- so returning it unexamined applies a stale revision's
- # hashes to a remembered cache that has the complete variant, and filters every blob of it
- # out. That is the same wrong answer as two remembered caches disagreeing.
+ # The active cache's manifest is a candidate like any other, NOT an early return: its repo dir can
+ # be gone while its scoped state holds an old manifest, and applying those hashes to a remembered
+ # cache that has the complete variant filters out every blob of it.
found: list[download_manifest.Manifest] = []
- # ``active_root`` is the root the job records, which is the one snapshot_progress scans; it
- # is not necessarily the configured default (a cache moved mid-download), and reading the
- # default there would be another cache's answer again.
+ # active_root is the root the job records, which is the one snapshot_progress scans and not
+ # necessarily the configured default.
active_manifest = download_manifest.read_manifest(
"model", repo_id, variant, hub_cache = active_root
)
if active_manifest is not None:
found.append(active_manifest)
- # The active cache was just probed by the call above and a state-dir miss is not free, so
- # skip the entry that repeats it. In the common case preferred_repo_cache_dirs returns only
- # that entry and this loop does no work at all.
+ # The active cache was just probed by the call above and a state-dir miss is not free, so skip the
+ # entry that repeats it; in the common case preferred_repo_cache_dirs returns only that entry.
active = download_manifest._canonical_hub_cache(active_root)
- # The SAME cache dirs snapshot_progress will scan. A running or cancelling job writes into
- # the active root and is read only from there, so a remembered cache's manifest for the same
- # variant is not merely a second opinion -- its hashes would be applied to the active root's
- # blobs and filter out every byte the live download has written, leaving the card at 0 B
- # until Hub metadata comes back.
+ # The SAME cache dirs snapshot_progress will scan: a remembered cache's manifest for the same
+ # variant would have its hashes applied to the active root's blobs, leaving the card at 0 B.
for entry in preferred_repo_cache_dirs(
"model", repo_id, force_active = force_active, active_root = active_root
):
if active is not None and download_manifest._canonical_hub_cache(entry.parent) == active:
if active_manifest is None:
- # The cache snapshot_progress will scan, with no manifest of its own. Anything
- # returned here would be another cache's answer applied to its blobs.
+ # Anything returned for a cache with no manifest of its own would be another cache's answer applied
+ # to its blobs.
return ("refused", None)
continue
manifest = download_manifest.read_manifest(
@@ -579,23 +571,18 @@ def _variant_manifest_decision(
hub_cache = entry.parent,
)
if manifest is None:
- # A scanned cache that contributed NOTHING. Its snapshot may be the complete one --
- # a manifest can be deleted, or never written by an older build -- and returning
- # some other cache's hashes filters every blob of it out AND disables the per-entry
- # name-based fallback that would still have counted them. Refuse instead.
+ # A scanned cache that contributed NOTHING may hold the complete snapshot, since a manifest can be
+ # deleted or never written, and another cache's hashes would filter out every blob AND disable the
+ # name-based fallback.
return ("refused", None)
found.append(manifest)
if not found:
return ("absent", None)
# One answer, or several that agree: safe to apply to every scanned entry, which is what
- # snapshot_progress does with the hash set this produces.
- #
- # Several that DISAGREE is the case worth refusing. snapshot_progress picks its reading by
- # bytes, across all the preferred cache dirs, but the hashes come from this single lookup --
- # so handing it the first cache's older revision filters out every blob of a LATER cache that
- # holds the complete variant, and reports 0 or partial for a finished download. That is the
- # exact failure this fallback exists to prevent, just sourced from the wrong cache. Returning
- # None instead degrades to the name-based fallback, which stays attributable per entry.
+ # snapshot_progress does with this hash set.
+ # Several that DISAGREE must be refused: snapshot_progress picks its reading by bytes across all
+ # preferred cache dirs while the hashes come from one lookup, so the first cache's older revision
+ # filters out every blob of a later complete one. None degrades to the name-based fallback.
first = _manifest_hashes(found[0])
if any(_manifest_hashes(m) != first for m in found[1:]):
return ("refused", None)
@@ -640,8 +627,7 @@ async def get_gguf_download_progress_response(
return requirement.download_size_bytes, requirement.required_hashes
job_key = _download_job_key(resolved_repo_id, progress_variant)
job = _registry.get_job(job_key)
- # getattr, the same way snapshot_progress reads it: a registry without the accessor
- # simply has no recorded root, which is the "use the configured one" case.
+ # getattr, the same way snapshot_progress reads it: a registry without the accessor simply has no recorded root.
get_job_metadata = getattr(_registry, "get_job_metadata", None)
job_metadata = get_job_metadata(job_key) if callable(get_job_metadata) else None
hub_cache = getattr(job_metadata, "hub_cache", None)
@@ -658,12 +644,9 @@ async def get_gguf_download_progress_response(
frozenset(file.sha256 for file in manifest.expected_files if file.sha256),
)
if verdict == "refused":
- # A refusal is not a miss. The lookup above found manifests and ruled that none of
- # them may be applied across the caches snapshot_progress scans; the blob-hash
- # helper reads the DEFAULT cache's manifest with none of that scoping, so falling
- # through here reinstates the very hashes just rejected and filters out every blob
- # of whichever cache actually holds the variant. An empty set degrades to the
- # per-entry name-based fallback, which stays attributable to the entry it counted.
+ # A refusal is not a miss: the blob-hash helper reads the DEFAULT cache's manifest with none of
+ # this scoping, so falling through reinstates the hashes just rejected. An empty set degrades to
+ # the per-entry name-based fallback.
return (expected_total, frozenset())
return (
expected_total,
@@ -697,15 +680,10 @@ async def get_gguf_download_progress_response(
return requirement.expected_files if requirement is not None else ()
def _variant_file_matcher(path: str, *, companions: bool = True) -> bool:
- # Which snapshot files a quant owns, for the reading snapshot_progress
- # falls back to when the blob hashes cannot be resolved. Main shards are
- # matched by quant label; mmproj and the MTP drafter are downloaded with
- # every variant, so they belong to whichever one is being polled.
- #
- # ``companions`` False asks the narrower question -- does this path prove the quant
- # ITSELF is here -- which the caller uses first: shared companions belong to every
- # quant in the repo, so counting them for a variant whose main shard was deleted
- # reported bytes for a file that is gone.
+ # Main shards are matched by quant label; mmproj and the MTP drafter are downloaded with every
+ # variant, so they belong to whichever one is being polled.
+ # companions=False asks the narrower question, whether this path proves the quant ITSELF is here:
+ # shared companions belong to every quant, so counting them reported bytes for a deleted file.
if progress_variant is None:
return False
if gguf_plan.is_main_gguf_variant_path(path, progress_variant):
diff --git a/studio/backend/hub/services/models/folder_browser.py b/studio/backend/hub/services/models/folder_browser.py
index d6d0d50586e..45fe758a73a 100644
--- a/studio/backend/hub/services/models/folder_browser.py
+++ b/studio/backend/hub/services/models/folder_browser.py
@@ -130,13 +130,12 @@ def _is_path_inside_allowlist(target: Path, allowed_roots: list[Path]) -> bool:
return True
drive, tail = os.path.splitdrive(root_real)
if os.path.dirname(root_real) == root_real and not drive:
- # Bare POSIX filesystem root ("/"): equality above is the only
- # match; do not let it authorize arbitrary descendants.
+ # Bare POSIX filesystem root: the equality above is the only match, so it must not authorize
+ # arbitrary descendants.
continue
if drive.startswith(("\\\\", "//")) and not tail:
- # Bare UNC share root (\\server\share): os.path.commonpath raises
- # "can't mix absolute and relative" on it, so authorize its
- # descendants with a boundary-safe prefix test (normcase applied).
+ # os.path.commonpath raises "can't mix absolute and relative" on a bare UNC share root, so authorize
+ # its descendants with a boundary-safe prefix test.
if target_real.startswith(root_real.rstrip("\\/") + os.sep):
return True
continue
diff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.py
index 36905580584..d5731ac38e6 100644
--- a/studio/backend/hub/services/models/gguf_variants.py
+++ b/studio/backend/hub/services/models/gguf_variants.py
@@ -72,14 +72,12 @@ _VARIANT_HASH_CACHE: "OrderedDict[tuple[str, str, str, bool], tuple[frozenset[st
_VARIANT_REQUIREMENT_CACHE: "OrderedDict[tuple[str, str, str], tuple[_GgufVariantRequirement, float]]" = OrderedDict()
_VARIANT_REQUIREMENT_NEG_CACHE: "OrderedDict[tuple[str, str], float]" = OrderedDict()
_VARIANT_HASH_MAX = 512
-# Blob hashes are derived from the same mutable remote revision metadata as
-# variant requirements, so they must not outlive that freshness window.
+# Blob hashes are derived from the same mutable remote revision metadata as variant requirements, so
+# they must not outlive that freshness window.
_VARIANT_HASH_POS_TTL = 60.0
# Refresh resolved variant requirements so a moved repo revision is picked up
-# within the session instead of being pinned for the backend's lifetime.
_VARIANT_REQUIREMENT_POS_TTL = 60.0
# Suppress retries on a metadata-fetch failure so a slow/flaky link doesn't
-# re-hammer the API on every page refresh.
_VARIANT_REQUIREMENT_NEG_TTL = 60.0
# Fail fast on a slow link so the variant render isn't blocked for seconds.
_GGUF_METADATA_TIMEOUT_SECONDS = 5.0
@@ -119,6 +117,7 @@ def _variant_requirement_neg_cache_active(key: tuple[str, str]) -> bool:
return False
if (time.monotonic() - cached_at) < _VARIANT_REQUIREMENT_NEG_TTL:
_VARIANT_REQUIREMENT_NEG_CACHE.move_to_end(key)
+ # Split-named link on an ordinary target: loads as-is, nothing missing.
return True
_VARIANT_REQUIREMENT_NEG_CACHE.pop(key, None)
return False
@@ -381,8 +380,8 @@ def _variant_dependency_key(repo_id: str, filename: str) -> Optional[str]:
unknown = hashlib.sha256(filename.lower().encode("utf-8")).hexdigest()[:16]
return f"{fam.name}:unknown:{unknown}"
encoders = sd_cpp_text_encoders_for(fam, repo_id, filename, inner_dim = inner_dim)
- # Hashed, not joined raw: the encoder table is long, and the key is opaque
- # to the client, which only ever compares it for equality.
+ # Hashed, not joined raw: the encoder table is long and the key is opaque to the client, which only
+ # ever compares it for equality.
digest = hashlib.sha256(
"\n".join("/".join(str(part) for part in entry) for entry in encoders).encode("utf-8")
).hexdigest()[:16]
@@ -519,8 +518,7 @@ def _local_main_gguf_blobs_by_quant(
if not hashes:
continue
if _is_imatrix_filename(normalized):
- # Not a companion either: nothing fetches an imatrix, so a stale copy
- # on disk must not vouch for a variant's blobs.
+ # Nothing fetches an imatrix, so a stale copy on disk must not vouch for a variant's blobs.
continue
if _is_mmproj_filename(normalized) or _is_mtp_drafter_path(normalized):
companion_blobs.setdefault(normalized, set()).update(
@@ -528,9 +526,8 @@ def _local_main_gguf_blobs_by_quant(
)
continue
quant = gguf_variant_key(normalized).lower()
- # The endian predicate reads a quant TOKEN so it can tell a parent-only quant
- # from a big-endian build; the qualified key makes it misread the path and drop
- # the blob, which leaves update detection with no local main files to compare.
+ # The endian predicate reads a quant TOKEN, so a qualified key makes it misread the path and drop
+ # the blob, leaving update detection with no local main files to compare.
if is_big_endian_gguf_path(normalized, extract_quant_label(normalized)):
continue
bucket = result.setdefault(quant, {}).setdefault(normalized, set())
@@ -595,8 +592,8 @@ def delete_variant_incomplete_blobs_result(
companions: bool = True,
root: Optional[Path] = None,
) -> VariantIncompleteDeleteResult:
- # With a sibling still downloading, ``companions=False`` keeps a shared mmproj
- # from being unlinked out from under it; the repo's last delete reclaims it.
+ # With a sibling still downloading, companions=False keeps a shared mmproj from being unlinked out
+ # from under it; the repo's last delete reclaims it.
target_hashes = (
gguf_variant_blob_hashes(repo_id, variant, hf_token, include_companions = companions)
| extra_hashes
@@ -614,9 +611,8 @@ def delete_variant_incomplete_blobs_result(
unresolved = has_variant_partial_state and has_repo_partials,
)
deleted = 0
- # Destructive iterator: only the exact-case match (or abort if ambiguous),
- # so a case-variant sibling repo's partials are never unlinked. ``root`` scopes
- # the purge to one cache so a delete never touches another cache's partials.
+ # Destructive iterator: only the exact-case match, or abort if ambiguous, so a case-variant sibling
+ # repo's partials are never unlinked; root scopes the purge to one cache.
for entry in iter_destructive_repo_cache_dirs("model", repo_id, root = root):
blobs_dir = entry / "blobs"
if not blobs_dir.is_dir():
@@ -736,8 +732,8 @@ def _direct_gguf_loads(path: Path) -> bool:
)
-# llama.cpp's split grammar (model_config._GGUF_SPLIT_FILE_RE): five digits exactly, since a
-# shorter name loads as an ordinary file, not the cache scan's looser -\d{3,}- resume form.
+# llama.cpp's split grammar (model_config._GGUF_SPLIT_FILE_RE) wants five digits exactly: a
+# shorter name loads as an ordinary file, unlike the cache scan's looser resume form.
_DIRECT_SPLIT_RE = re.compile(r"^(?P<stem>.+)-(?P<index>\d{5})-of-(?P<total>\d{5})$", re.IGNORECASE)
@@ -780,7 +776,6 @@ def _direct_gguf_split_is_whole(path: Path) -> bool:
"""
m = _DIRECT_SPLIT_RE.match(target.name.rsplit(".", 1)[0])
if m is None:
- # Split-named link on an ordinary target: loads as-is, nothing missing.
return True
target_total = int(m.group("total"))
pattern = re.compile(
@@ -798,12 +793,11 @@ def _direct_gguf_split_is_whole(path: Path) -> bool:
found = _indexes_beside(path, sibling)
if not found >= set(range(1, total + 1)) and path.is_symlink():
# _local_gguf_load_path resolves siblings from the TARGET, so a renamed alias
- # still loads the target's real set -- and a torn target stays torn.
return _target_set_is_whole(path.resolve())
except OSError:
return True
- # Declared indexes, not a count: an over-indexed stray must not stand in for a missing
- # shard, and a zero-byte sibling is an interrupted copy.
+ # Declared indexes, not a count: an over-indexed stray must not stand in for a missing shard, and a
+ # zero-byte sibling is an interrupted copy.
return found >= set(range(1, total + 1))
@@ -836,10 +830,8 @@ def _will_serve(resolved: Optional[str]) -> bool:
return False
try:
path = Path(resolved)
- # The resolver answers for nonexistent paths, so absence is caught here: "no such
- # file" is definite, a read error (Windows lock window) stays unknown and serves.
- # stat(), not exists(), which swallows every OSError on 3.14 and would read a
- # sharing violation as absence.
+ # The resolver answers for nonexistent paths, so absence is caught here. stat(), not exists(), which
+ # swallows every OSError on 3.14 and would read a sharing violation as absence.
try:
size = path.stat().st_size
except (FileNotFoundError, NotADirectoryError):
@@ -859,10 +851,9 @@ def _loadable_variants(identifier: str, variants):
"""
from utils.models.model_config import _find_local_gguf_by_variant
- # from_identifier only consults the variant for a DIRECTORY; a direct file loads itself
- # regardless, so leave it unanswered rather than be stricter than the load. stat(), not
- # is_file(): a locked file must stay unanswered (an empty list is authoritative at the
- # gate), which is_file() cannot express -- it raises here, and answers False from 3.14.
+ # from_identifier only consults the variant for a DIRECTORY, so leave a direct file unanswered
+ # rather than be stricter than the load. stat(), not is_file(): a locked file must stay unanswered,
+ # which is_file() cannot express, since it raises here and answers False from 3.14.
import stat as _stat
try:
@@ -873,9 +864,9 @@ def _loadable_variants(identifier: str, variants):
except OSError:
return None
- # The resolver walks the tree per call, so spellings are deduped against `seen` first:
- # ~2 calls per row (36 for 18 quants, 179ms over 144 files), not one per spelling. Each
- # alias is still confirmed, so a token binding a different file is never advertised.
+ # The resolver walks the tree per call, so spellings are deduped against `seen` first (~2 calls
+ # per row); each alias is still confirmed, so a token binding a different file is never
+ # advertised.
accepted: list = []
seen = set()
for variant in variants:
@@ -892,10 +883,9 @@ def _loadable_variants(identifier: str, variants):
if key not in seen:
seen.add(key)
accepted.append(quant)
- # The resolver also takes the snapshot-relative stem (BF16/model) and the basename's
- # own tokens; derive those and let it confirm each. It returns an absolute path, so a
- # relative identifier must be resolved the same way or its alias is lost. Unresolved
- # first: a symlink out of the tree still answers BF16/model there, but not resolved.
+ # The resolver also takes the snapshot-relative stem and the basename's own tokens, and returns an
+ # absolute path, so a relative identifier must be resolved the same way or its alias is lost.
+ # Unresolved first: a symlink out of the tree still answers there, but not resolved.
relative = Path(bound).name
for base_raw, bound_path in (
(Path(identifier).expanduser(), Path(bound)),
@@ -978,11 +968,8 @@ def _complete_with_servable(snapshot: str, complete, variants):
return repaired
-# One scan per identical request in flight. Aborting the HTTP request cannot stop the scan
-# already running in its thread, so the picker's Retry, and reopening a row, would each start
-# another against a filesystem that is not answering: measured 23 retries filling all 20
-# default-executor workers, starving unrelated offloaded work and holding up exit. Joining the
-# running scan costs at most its own duration in staleness, well inside the client's cache TTL.
+# One scan per identical request in flight: aborting the HTTP request cannot stop the scan already
+# running, and 23 retries filled all 20 default-executor workers.
_VARIANTS_INFLIGHT: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary()
@@ -1001,7 +988,7 @@ async def _shared_variants_scan(key: tuple, compute):
def _release(finished: asyncio.Future) -> None:
inflight.pop(key, None)
if not finished.cancelled():
- finished.exception() # retrieved, so a caller-less failure stays quiet
+ finished.exception()
inflight[key] = task
task.add_done_callback(_release)
@@ -1046,12 +1033,11 @@ async def get_gguf_variants_answer(
with file sizes, whether the model supports vision, and the recommended
default variant.
"""
- # Set by whichever branch answers, and returned with the listing: the HF cache answers
- # before local_path, so a caller cannot infer the copy from the request alone.
+ # Returned with the listing because the HF cache answers before local_path, so a caller cannot infer
+ # the copy from the request alone.
answered_from: list[Optional[str]] = [None]
- # Set by the existence-first local branch below: a repo-shaped id resolving to a directory
- # is answered by that directory alone, not the HF cache of the same-named repo -- else a
- # GGUF-less directory could evict the resident model via a cleanable row from the cache.
+ # A repo-shaped id resolving to a directory is answered by that directory alone, not the HF cache
+ # of the same-named repo, else a GGUF-less directory could evict the resident model.
answered_locally = [False]
def _compute() -> GgufVariantsResponse:
@@ -1212,13 +1198,11 @@ async def get_gguf_variants_answer(
return response
return response.model_copy(update = {"variants": [*response.variants, *extra]})
- # Local directory path (e.g. LM Studio models) — scan filesystem. Load-path parity:
- # from_identifier resolves existence-first, so a marker-less relative name that exists
- # here is a local model, not a Hub id, and a direct .gguf file loads without the
- # metadata siblings the directory scan requires. It also normalizes first, so every
- # question below must ask about the same path: under WSL "C:\\models\\qwen" maps to
- # /mnt/c/models/qwen, and probing the raw spelling would report a working model
- # unloadable.
+ # Load-path parity: from_identifier resolves existence-first, so a marker-less relative name that
+ # exists here is a local model, not a Hub id, and a direct .gguf file loads without the metadata
+ # siblings. It normalizes first, so every question below asks about the same path: under WSL
+ # C:\models\qwen maps to /mnt/c/models/qwen, and probing the raw spelling would report a working
+ # model unloadable.
local_id = _loader_normalize_path(repo_id) if is_local_path(repo_id) else repo_id
local_target = None
try:
@@ -1229,8 +1213,7 @@ async def get_gguf_variants_answer(
local_target = None
if local_target is not None:
variants, has_vision = list_local_gguf_variants(local_id)
- # The load id is this path, so a scan-torn quant is still ready when the file the
- # resolver binds opens fine.
+ # The load id is this path, so a scan-torn quant is still ready when the file the resolver binds opens fine.
complete = _complete_with_servable(local_id, _complete_quants_under(local_id), variants)
if (
not variants
@@ -1238,14 +1221,14 @@ async def get_gguf_variants_answer(
and local_target.suffix.lower() == ".gguf"
and _direct_gguf_loads(local_target)
):
- # An unmarked-parent .gguf is skipped by the directory scan but detect_gguf_model
- # still loads it; falling back only here keeps a marked parent's siblings.
+ # An unmarked-parent .gguf is skipped by the directory scan but detect_gguf_model still loads it;
+ # falling back only here keeps a marked parent's siblings.
try:
size = local_target.stat().st_size
except OSError:
size = 0
- # The load resolver's own extractor over the context it reads, so the quant is
- # what the echoed load resolves (the hub one differs on F16-checkpoint-Q4_K_M).
+ # The load resolver's own extractor over the context it reads, so the quant is what the echoed load
+ # resolves; the hub one differs on F16-checkpoint-Q4_K_M.
from utils.models.model_config import _extract_quant_label
from utils.models.model_config import colocated_split_shards
@@ -1266,15 +1249,13 @@ async def get_gguf_variants_answer(
shard_count = len(shards) if split_complete and len(shards) > 1 else 0,
)
]
- # The shard scan resolves a file to its marked parent, so an unmarked one walks
- # a bare file and misreports the row. Ask the file itself: whole unless it is a
- # split missing a sibling or a zero-byte interrupted copy.
+ # The shard scan resolves a file to its marked parent, so an unmarked one walks a bare file and
+ # misreports the row; ask the file itself.
complete = None if size > 0 and _direct_gguf_split_is_whole(local_target) else set()
answered_from[0] = repo_id
answered_locally[0] = True
- # Surface the resolution so the CLI gate matches on the local resolver's exact
- # labels rather than the id's shape, and answers its real question -- would this
- # load resolve -- with the loader itself, so no client mirrors its grammar.
+ # Surface the resolution so the CLI gate matches the local resolver's exact labels rather than the
+ # id's shape, and no client mirrors its grammar.
return _local_response(repo_id, variants, has_vision, complete).model_copy(
update = {
"resolved_locally": True,
@@ -1283,8 +1264,8 @@ async def get_gguf_variants_answer(
}
)
- # Reject invalid remote repo_ids up front (like download/delete) so a
- # malformed id returns 400 instead of a 500 from the HF client.
+ # Reject invalid remote repo_ids up front (like download/delete) so a malformed id returns 400 instead
+ # of a 500 from the HF client.
if not _is_valid_repo_id(repo_id):
raise HTTPException(status_code = 400, detail = f"Invalid repo_id: {repo_id!r}")
@@ -1321,8 +1302,8 @@ async def get_gguf_variants_answer(
variants, has_vision, complete, snapshot = _merge_when_the_repo_id_loads(
repo_id, cached, hub_cache
)
- # Name the answering snapshot: a repo-wide walk could read a different cache's
- # context length, and a repo-dir walk a sibling revision's.
+ # Name the answering snapshot: a repo-wide walk could read a different cache's context length, and a
+ # repo-dir walk a sibling revision's.
answered_from[0] = str(snapshot)
# The lister leaves torn quants in: they stay listed for management, but not ready.
return _with_state_partials(
@@ -1372,8 +1353,8 @@ async def get_gguf_variants_answer(
repo_id, cached, hub_cache
)
answered_from[0] = str(snapshot)
- # Same reason as the local_only branch above, state partials included:
- # an unreachable Hub is exactly when a resume has nowhere else to surface.
+ # Same reason as the local_only branch above: an unreachable Hub is exactly when a resume has
+ # nowhere else to surface, so state partials are included.
return _with_state_partials(
_local_response(repo_id, variants, has_vision, complete)
)
@@ -1392,8 +1373,8 @@ async def get_gguf_variants_answer(
return fallback
raise
- # siblings is None only when the lister answered from its own repo-wide cache. Falling
- # through is deliberate: readiness below still counts against this request's own cache.
+ # siblings is None only when the lister answered from its own repo-wide cache; falling through is
+ # deliberate, since readiness still counts against this request's own cache.
if siblings is None:
if not cache_reads_authorized:
# `variants` is already the cache's answer, so declining a second one is
@@ -1409,10 +1390,8 @@ async def get_gguf_variants_answer(
best = pick_best_gguf(_default_variant_candidates(variants))
default_variant = gguf_variant_key(best) if best else None
- # Per-snapshot accounting: a variant counts as present only when one
- # snapshot holds all its files (split GGUFs need every shard together),
- # sizes are max across snapshots so shared blobs aren't double-counted,
- # and keys are lowercased since cache dir casing can differ from repo_id.
+ # Per-snapshot accounting: split GGUFs need every shard together, sizes are max across snapshots
+ # so shared blobs are not double-counted, and keys are lowercased since cache casing can differ.
cached_filenames_by_snapshot: list[dict[str, int]] = []
cached_quant_bytes_by_snapshot: list[dict[str, int]] = []
if _is_valid_repo_id(repo_id):
@@ -1524,8 +1503,8 @@ async def get_gguf_variants_answer(
)
):
return True
- # Byte fallback so a present quant isn't demoted by a filename mismatch;
- # vision repos still need an mmproj cached (any precision).
+ # Byte fallback so a present quant is not demoted by a filename mismatch; vision repos still need an
+ # mmproj cached, at any precision.
if not _quant_bytes_present(quant, variant.size_bytes):
return False
if (
@@ -1557,8 +1536,8 @@ async def get_gguf_variants_answer(
repo_signal_applies = hf_cache_scan.repo_signal_applies_to_snapshot(
repo_cache_dir, scan_snapshot_dir
)
- # The excuse is that this snapshot holds the quant whole, so a quant it lacks stays the
- # cancelled download and keeps its resume and delete affordances.
+ # The excuse is that this snapshot holds the quant whole, so a quant it lacks stays the cancelled
+ # download and keeps its resume and delete affordances.
excused_quants = (
frozenset()
if repo_signal_applies or scan_snapshot_dir is None
@@ -1570,8 +1549,8 @@ async def get_gguf_variants_answer(
def _repo_signals_apply_to(quant: str) -> bool:
return repo_signal_applies or quant.lower() not in excused_quants
- # Manifest + marker + main incomplete-blob check: catches variants whose
- # download was cancelled or whose expected shards are missing/undersized.
+ # Manifest + marker + main incomplete-blob check: catches variants whose download was cancelled or whose
+ # expected shards are missing/undersized.
for variant in variants:
try:
requirement = requirements_by_quant.get(variant.quant.lower())
@@ -1610,7 +1589,6 @@ async def get_gguf_variants_answer(
if requirement is None or not _repo_signals_apply_to(variant.quant):
continue
# companion_hashes adds the MTP drafter (mmproj_hashes covers
- # every mmproj precision in the repo, not just the planned one).
if (
(requirement.mmproj_hashes | requirement.companion_hashes) & incomplete_hashes
) and _filenames_cached(
@@ -1671,8 +1649,8 @@ async def get_gguf_variants_answer(
)
def _compute_with_cleanables() -> VariantsAnswer:
- # Returned with the answer, not read from the closure afterwards: coalesced callers
- # share one computation and must all see the copy it actually answered from.
+ # Returned with the answer, not read from the closure afterwards: coalesced callers share one
+ # computation and must all see the copy it answered from.
return VariantsAnswer(_compute_response(), answered_from[0])
def _compute_response() -> GgufVariantsResponse:
@@ -1680,9 +1658,7 @@ async def get_gguf_variants_answer(
try:
response = _compute()
except Exception:
- # Offline / metadata fetch failed with only an empty leftover
- # <quant>/ folder cached: still surface it so the UI can delete it,
- # otherwise re-raise the original error.
+ # Surface an empty leftover <quant>/ folder so the UI can delete it; otherwise re-raise the original error.
if skip:
raise
enriched = _mark_empty_dir_cleanables(
@@ -1709,8 +1685,7 @@ async def get_gguf_variants_answer(
bool(offline),
local_path or "",
hf_cache_scan.token_fingerprint(hf_token),
- # Switching cache storage must start a fresh scan rather than join one that is
- # stuck on the old volume.
+ # Switching cache storage must start a fresh scan rather than join one that is stuck on the old volume.
configured_cache_key(),
)
try:
diff --git a/studio/backend/hub/services/models/local_inventory.py b/studio/backend/hub/services/models/local_inventory.py
index 5f2e5a00021..7e77d61b4a0 100644
--- a/studio/backend/hub/services/models/local_inventory.py
+++ b/studio/backend/hub/services/models/local_inventory.py
@@ -65,16 +65,15 @@ _local_inventory_flights: dict[
] = {}
-# Retrying a superseded scan is only worth it while invalidations are occasional;
-# past this the endpoint must answer instead of restarting the walk forever.
+# Retrying a superseded scan is only worth it while invalidations are occasional; past this the endpoint must answer.
_LOCAL_INVENTORY_MAX_ATTEMPTS = 8
class _LocalCacheChanged(RuntimeError):
def __init__(self, response: LocalModelListResponse) -> None:
super().__init__("local inventory sources changed during the scan")
- # Carried so the attempt cap can serve the freshest scan it has instead
- # of looping forever or answering with nothing.
+ # Carried so the attempt cap can serve the freshest scan it has instead of looping forever or
+ # answering with nothing.
self.response = response
@@ -121,6 +120,8 @@ def _has_immediate_model_weight(
if entry.is_file() and _is_immediate_model_weight_file(entry):
return True
except OSError:
+ # Skip individual children that are unreadable (permissions, broken symlinks, etc.) rather than
+ # failing the entire scan.
continue
except OSError:
return False
@@ -218,8 +219,6 @@ def _scan_models_dir(
continue
has_model_files = is_gguf_file or _has_immediate_model_signal(child)
except OSError:
- # Skip individual children that are unreadable (permissions, broken
- # symlinks, etc.) rather than failing the entire scan.
continue
if not has_model_files:
continue
@@ -311,9 +310,8 @@ def _scan_hf_cache(
if not discovered:
return []
if variant_states is None:
- # Reached precisely when the caller's own guarded build already failed, so
- # leaving this one bare handed the same exception straight back and undid
- # that guard. Degrade to the per-repo reads instead, as the callers do.
+ # Reached precisely when the caller's own guarded build already failed, so leaving this bare handed
+ # the same exception back and undid that guard; degrade to the per-repo reads instead.
try:
variant_states = download_manifest.build_variant_state_index(
[("model", model_id, cache_dir) for _repo, model_id, _updated in discovered],
@@ -364,8 +362,8 @@ def _scan_hf_cache(
resolved = hf_cache_scan.resolve_hf_cache_realpath(repo_dir)
scan_path = Path(resolved) if resolved else repo_dir
load_path = repo_dir if active_cache else scan_path
- # partial=False here; _apply_format_aware_partial below rewrites per-row
- # so a hybrid repo's gguf row doesn't taint its safetensors row.
+ # _apply_format_aware_partial below rewrites per row, so a hybrid repo's gguf row does not taint its
+ # safetensors row.
rows = _classify_local_path(
scan_path,
"hf_cache",
@@ -394,8 +392,7 @@ def _scan_hf_cache(
)
]
else:
- # Fallback row's model_format is "unknown"; either signal
- # applies because we can't dispatch to a specific predicate.
+ # The fallback row's model_format is "unknown", so either signal applies.
rows = [
_local_model_info(
scan_path = repo_dir,
@@ -445,8 +442,8 @@ def _scan_lmstudio_dir(lm_dir: Path, *, entry_limit: int | None = None) -> List[
if not lm_dir.exists() or not lm_dir.is_dir():
return []
- # If the dir is itself a model dir (config + weights, or a diffusers pipeline root), it's not
- # an LM Studio publisher structure -- return it as a single entry rather than descend.
+ # A directory that is itself a model dir is not an LM Studio publisher structure, so return it as a
+ # single entry rather than descend.
if _is_model_directory(lm_dir) or _is_diffusers_pipeline_dir(lm_dir):
try:
updated_at = lm_dir.stat().st_mtime
@@ -494,8 +491,8 @@ def _scan_lmstudio_dir(lm_dir: Path, *, entry_limit: int | None = None) -> List[
)
continue
- # Child is itself a model dir: surface it directly, not as a publisher. A diffusers
- # pipeline counts, or its component subdirs are walked as if they were models.
+ # A child that is itself a model dir is surfaced directly, not as a publisher; a diffusers pipeline
+ # counts, or its component subdirs are walked as models.
if _is_model_directory(child) or _is_diffusers_pipeline_dir(child):
try:
updated_at = child.stat().st_mtime
@@ -579,8 +576,7 @@ def _inventory_path_identity(raw_path: str) -> str:
normalized = normalize_path(raw)
return os.path.normcase(os.path.realpath(os.path.expanduser(normalized)))
except (OSError, UnicodeError, ValueError):
- # Keep malformed sources distinct until the worker's existing request or
- # per-folder error boundary turns them into a 403/skip.
+ # Keep malformed sources distinct until the worker's error boundary turns them into a 403 or a skip.
return os.path.normcase(raw)
@@ -696,8 +692,8 @@ async def _collect_models_from_default_sources(
lambda path: _discover_hf_cache(path, entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES),
folder_path,
)
- # Carry the registered path: the status registry is keyed on the row, not
- # on the normalized Path this scan walks.
+ # Carry the registered path: the status registry is keyed on the row, not on the normalized Path
+ # this scan walks.
custom_sources.append((folder_path, discovered, str(folder["path"])))
state_repositories.extend(
("model", model_id, folder_path) for _repo, model_id, _updated in discovered
@@ -745,8 +741,8 @@ async def _collect_models_from_default_sources(
if isinstance(e, OSError):
record_scan_failure(row_path, e)
continue
- # Off the loop, like the scan above it: the probe opens directories, and on a
- # stalled network mount scandir sits in the kernel with nothing to yield to.
+ # Off the loop, like the scan above it: the probe opens directories, and on a stalled network mount
+ # scandir sits in the kernel with nothing to yield to.
await asyncio.to_thread(note_scan_folder_scanned, row_path, found = bool(custom_models))
local_models.extend(_promote_to_custom_source(model) for model in custom_models)
@@ -765,9 +761,8 @@ def _scan_custom_folder(
supported_formats: set[ModelFormat] = {"gguf", "safetensors", "adapter"}
def _is_supported(m: LocalModelInfo) -> bool:
- # A diffusers pipeline keeps its weights in component subdirs, so the root has no loose
- # weight file to classify and lands as "unknown". It is exactly what the Images and Video
- # loaders take, so judge it on its shape rather than on a format the layout cannot report.
+ # A diffusers pipeline keeps its weights in component subdirs, so its root lands as "unknown"; judge
+ # it on its shape rather than on a format the layout cannot report.
if m.model_format in supported_formats:
return True
return _is_diffusers_pipeline_dir(Path(m.path))
@@ -834,9 +829,8 @@ def _promote_to_custom_source(model: LocalModelInfo) -> LocalModelInfo:
"custom",
partial = model.partial,
requires_variant = model.capabilities.requires_variant,
- # Rebuilding from the format alone restored can_chat on rows the
- # classifier had ruled out. The format is unchanged here, so
- # carrying the old verdict through is idempotent.
+ # Rebuilding from the format alone restored can_chat on rows the classifier had ruled out; the
+ # format is unchanged here, so carrying the old verdict through is idempotent.
can_chat_override = model.capabilities.can_chat,
),
}
@@ -967,10 +961,8 @@ async def list_local_models_response(models_dir: str = "./models") -> LocalModel
"""Coalesce overlapping local inventory requests for the same models root."""
def classify(response: LocalModelListResponse) -> LocalModelListResponse:
- # These rows feed the same pickers as /api/models/local. Classified inside the
- # shared worker so retrying waiters do not repeat GGUF metadata reads, and only
- # for a response that is actually about to be served.
- # Classification reads GGUF headers, so keep it off the event loop too.
+ # Classified inside the shared worker so retrying waiters do not repeat GGUF metadata reads, and off
+ # the event loop, since classification reads GGUF headers.
try:
from hub.services.models import catalog_classification
@@ -1005,8 +997,8 @@ async def list_local_models_response(models_dir: str = "./models") -> LocalModel
# Discard obsolete results and retry their waiters against the current cache epoch.
superseded: Optional[LocalModelListResponse] = None
for _attempt in range(_LOCAL_INVENTORY_MAX_ATTEMPTS):
- # Epoch first: the sources and folders below are read after it, so any
- # change to them lands in a later epoch and the post-scan check sees it.
+ # Epoch first: the sources and folders below are read after it, so any change to them lands in a
+ # later epoch and the post-scan check sees it.
epoch = hf_cache_scan.hf_cache_scans_epoch()
custom_folders = await _load_custom_folders()
sources = _local_inventory_sources()
@@ -1029,9 +1021,7 @@ async def list_local_models_response(models_dir: str = "./models") -> LocalModel
except _LocalCacheChanged as changed:
superseded = changed.response
continue
- # Invalidations are outpacing the walk, so no scan will ever confirm as
- # current. Answer with the freshest one (the loop only reaches here through
- # the retry path, so there is always one) instead of rescanning forever.
+ # Invalidations are outpacing the walk, so answer with the freshest scan instead of rescanning forever.
logger.warning("Local inventory kept racing cache invalidations; serving the last scan")
return await asyncio.to_thread(classify, superseded)
@@ -1043,9 +1033,8 @@ def get_models_folder_response() -> dict:
the desktop app reveals it in the OS file manager.
"""
path = _resolve_hf_cache_dir()
- # Create it if missing so "Open folder" works before the first download:
- # HF builds the cache lazily, and studio only pre-creates the *default*
- # dir, not a user's explicit HF_HOME / HF_HUB_CACHE.
+ # Create it if missing so "Open folder" works before the first download: HF builds the cache
+ # lazily, and studio pre-creates only the default dir, not a user's explicit HF_HOME/HF_HUB_CACHE.
try:
path.mkdir(parents = True, exist_ok = True)
except OSError as e:
diff --git a/studio/backend/hub/services/models/ollama.py b/studio/backend/hub/services/models/ollama.py
index 62bf3c7b86e..7edccb20fdc 100644
--- a/studio/backend/hub/services/models/ollama.py
+++ b/studio/backend/hub/services/models/ollama.py
@@ -48,8 +48,7 @@ _OLLAMA_LOADABLE_LAYER_MEDIA_TYPES = frozenset(
{
"application/vnd.ollama.image.model",
"application/vnd.ollama.image.projector",
- # License text does not affect model behavior and does not need to be
- # carried into llama.cpp.
+ # License text does not affect model behavior and does not need to be carried into llama.cpp.
"application/vnd.ollama.image.license",
}
)
@@ -149,8 +148,8 @@ def _ollama_links_dir(ollama_dir: Path) -> Optional[Path]:
if _ensure_writable_dir(primary) is not None:
return primary
- # Namespace by a hash of the ollama_dir so two different Ollama roots
- # don't collide. This is a cache path, not a security boundary.
+ # Namespace by a hash of the ollama_dir so two different Ollama roots do not collide. A cache path,
+ # not a security boundary.
try:
digest = hashlib.sha256(str(ollama_dir.resolve()).encode()).hexdigest()[:12]
except (OSError, RuntimeError):
@@ -192,8 +191,7 @@ def _make_ollama_blob_link(link_dir: Path, link_name: str, target: Path) -> Opti
logger.debug("Could not resolve Ollama blob %s: %s", target, e)
return None
- # Skip if the link already points at the same blob. Use samefile, not size:
- # `ollama pull` can swap a tag to a same-sized blob, leaving a stale link.
+ # samefile, not size: `ollama pull` can swap a tag to a same-sized blob, leaving a stale link.
try:
if link_path.exists() and os.path.samefile(str(link_path), str(resolved)):
return str(link_path)
diff --git a/studio/backend/hub/services/snapshot_progress.py b/studio/backend/hub/services/snapshot_progress.py
index 449ed185c29..3f42de8bc92 100644
--- a/studio/backend/hub/services/snapshot_progress.py
+++ b/studio/backend/hub/services/snapshot_progress.py
@@ -37,17 +37,15 @@ logger = get_logger(__name__)
# (repo_id, hf_token) -> (expected_total_bytes, expected_blob_hashes)
SnapshotMetadataResolver = Callable[[str, Optional[str]], "tuple[int, frozenset[str]]"]
-# (repo_id, hf_token) -> the files HF says this target should contain. Optional,
-# and the only thing that lets a materialized snapshot with no manifest settle.
+# The files HF says this target should contain: optional, and the only thing that lets a
+# materialized snapshot with no manifest settle.
SnapshotExpectedFilesResolver = Callable[
[str, Optional[str]], Sequence["download_manifest.ExpectedFile"]
]
-# repo-relative snapshot path -> whether it belongs to the variant being polled.
# Supplied per repo kind, so this module keeps knowing nothing about quant labels.
VariantFileMatcher = Callable[[str], bool]
# One progress log per 10% step per job, so an active download reports progress
-# without emitting a line on every poll.
_progress_step_lock = threading.Lock()
_last_progress_step: dict[str, int] = {}
@@ -146,9 +144,8 @@ def _variant_bytes_on_disk(
if not download_manifest.expected_path_is_safe(expected.path):
continue
if expected.sha256 and expected.sha256 in active_partial_hashes:
- # A force/retry can leave the previous materialized file in the snapshot while
- # a replacement for the same logical blob is being written. Count the current
- # partial, not both generations of the file.
+ # A force or retry can leave the previous materialized file beside a replacement for the same
+ # logical blob; count the current partial, not both generations.
continue
try:
total += (snapshot_dir / expected.path).stat().st_size
@@ -206,9 +203,8 @@ def _variant_main_shard_present(
"""
if snapshot_dir is None or variant_file_matcher is None:
return None
- # An entry we could not read may BE the main shard -- a transient network-filesystem hiccup
- # or a Windows ACL denial on one directory is not evidence the variant is gone. A positive
- # match elsewhere still settles it; otherwise the reading stays unknown.
+ # An entry we could not read may BE the main shard, so a transient failure is not evidence the
+ # variant is gone: a positive match elsewhere still settles it, otherwise the reading is unknown.
entries, complete = _walk_files(snapshot_dir)
for path in entries:
relative = path.relative_to(snapshot_dir).as_posix()
@@ -267,24 +263,21 @@ def _materialized_bytes(snapshot_dir: Path, variant_file_matcher: "VariantFileMa
never linked contributes nothing, and the Windows copy layout is read as is.
"""
try:
- entries = list(snapshot_dir.rglob("*")) # unordered: the result is a sum
+ entries = list(snapshot_dir.rglob("*"))
except OSError:
return 0
# Same false "still active" as the companion clause below, reached by a stranded sidecar.
entries = [path for path in entries if not is_appledouble_metadata(path)]
def _accepts(relative: str, *, companions: bool) -> bool:
- # A matcher that understands the distinction gets asked for it; an older one is
- # answered as before. Only the GGUF matcher takes the keyword today.
+ # A matcher that understands the distinction gets asked for it; only the GGUF matcher takes the keyword today.
try:
return bool(variant_file_matcher(relative, companions = companions))
except TypeError:
return bool(variant_file_matcher(relative))
- # Companions (mmproj, the MTP drafter) are shared by every quant in the repo, so on their
- # own they are not evidence that THIS one is here. Deleting a quant while a sibling keeps
- # its companion left a positive reading for the deleted variant, which hydration reads as
- # active: it re-adopts the stale job and blocks a fresh download of the same quant.
+ # Companions are shared by every quant in the repo, so alone they are not evidence THIS one is
+ # here: a stranded companion left a positive reading that hydration re-adopts.
owns_a_main = False
for path in entries:
try:
@@ -343,21 +336,11 @@ def _snapshot_complete_on_disk(
return False
manifest = entry_manifest.get()
if manifest is None:
- # No manifest, but HF metadata describes the same thing a manifest does:
- # the exact files this download was supposed to fetch, at their declared
- # sizes. Verify against that rather than refusing outright, because a
- # manifest that was never written (metadata was unreachable when the run
- # ended), was deleted, or sits under a cache scope this reader cannot
- # name is not evidence of an unfinished download -- and refusing left a
- # materialized snapshot partial forever.
- #
- # Nothing weaker will do here. The caller's expected_bytes is a catalog
- # hint, and a byte total assembled from the shared blobs/ dir cannot
- # tell this variant's bytes from a sibling's or a stray companion's, so
- # neither is grounds for calling a download complete. Checking the named
- # files against the snapshot dir also catches the case a blob-level
- # tally structurally cannot: HF writes a blob and then links it, so a
- # run killed in between leaves bytes that nothing points at.
+ # HF metadata names the same files a manifest does, so verify against it: a manifest never
+ # written, deleted, or under an unnameable cache scope is not evidence of an unfinished download,
+ # and refusing left a materialized snapshot partial forever.
+ # Nothing weaker will do: expected_bytes is a catalog hint, and a blobs/ tally cannot tell this
+ # variant's bytes from a sibling's.
metadata_expected = metadata_files.get()
if not metadata_expected:
return False
@@ -368,15 +351,11 @@ def _snapshot_complete_on_disk(
started_at = "",
expected_files = metadata_expected,
)
- # ANY retained snapshot: the variant can be complete in an older revision while the newest
- # holds only a sibling, and checking the newest alone left that download at 99% forever.
- #
- # But an older revision can carry the same FILENAMES at the same sizes and different
- # content, and verify_against_disk does not read sha256 -- so with the resolved hashes in
- # hand, require the snapshot's entries to actually resolve to them. Without that, a blob
- # finalized but not yet linked (a crash between the two) let a stale revision satisfy the
- # check and the job settled on files the app would not load. No resolved hashes means no
- # such claim is possible, and the filename check stands alone as before.
+ # ANY retained snapshot: the variant can be complete in an older revision while the newest holds
+ # only a sibling, and checking the newest alone left that download at 99% forever.
+ # An older revision can carry the same FILENAMES at the same sizes with different content and
+ # verify_against_disk does not read sha256, so require the entries to resolve to known hashes;
+ # with none resolved the filename check stands alone.
for snap in snapshots:
if not download_manifest.verify_against_disk(manifest, snap).ok:
continue
@@ -489,26 +468,20 @@ def compute_snapshot_progress(
active_root = Path(metadata_hub_cache) if metadata_hub_cache else None
expected_total = max(expected_bytes, 0)
- # Always resolve the revision's blob hashes so stale blobs from a superseded
- # revision can't inflate the count; hashes degrade to empty (count-all) only
- # when metadata is unavailable (e.g. offline). Take the larger total so a low
- # caller hint can't cap the bar below the revision's real size.
+ # Always resolve the revision's blob hashes so stale blobs from a superseded revision cannot
+ # inflate the count; they degrade to empty (count-all) only when metadata is unavailable (e.g.
+ # offline). Take the larger total so a low caller hint cannot cap the bar.
meta_total, expected_hashes = metadata_resolver(repo_id, hf_token)
meta_total = max(0, meta_total)
expected_total = max(expected_total, meta_total)
- # Without resolved hashes, a variant must not count unscoped blobs: sibling
- # quants share one blobs/ dir, so a sibling's bytes (or .incomplete) would be
- # misattributed and make the bar jump backward. A no-variant snapshot owns
- # the whole dir, so it counts unscoped.
+ # Without resolved hashes a variant must not count unscoped blobs, since sibling quants share one
+ # blobs/ dir; a no-variant snapshot owns the whole dir and counts unscoped.
count_unscoped = variant is None
- # An empty hash set is "the expected file set could not be determined", not
- # "this variant has no bytes" -- model_info failing (offline, or a 401 on a
- # gated repo) is negatively cached, so one failed lookup keeps the set empty
- # for the whole TTL. Filtering every blob out then reports a finished 33 GB
- # variant as "0 B of 33 GB" and, because completion is never observed, leaves
- # Retry/Resume on the card. Fall back to the variant's own files in the
- # snapshot dir, which stay attributable when the blob hashes do not.
+ # An empty hash set means the expected file set could not be determined, not that the variant has
+ # no bytes: model_info failing (offline, or a 401 on a gated repo) is negatively cached, so one
+ # failed lookup would report a finished 33 GB variant as "0 B of 33 GB" for the whole TTL. Fall
+ # back to the snapshot dir's own files.
variant_file_set_unknown = variant is not None and not expected_hashes
# Resolved at most once, and only if a reading gets far enough to need it.
metadata_files: "_Lazy[tuple[download_manifest.ExpectedFile, ...]]" = _Lazy(
@@ -520,9 +493,8 @@ def compute_snapshot_progress(
)
readings: list[tuple[int, int, Optional[str], bool, Optional[bool]]] = []
- # The enumeration suppresses OSError per root, so an unreadable cache root came back as
- # "no dirs" -- indistinguishable from a wiped cache, and hydration retires the job on
- # that. Collected so the empty answer below can say unknown instead of absent.
+ # The enumeration suppresses OSError per root, so an unreadable cache root came back as "no dirs",
+ # indistinguishable from a wiped cache, which hydration retires the job on.
scan_errors: list = []
cache_dirs = (
preferred_repo_cache_dirs(
@@ -539,21 +511,19 @@ def compute_snapshot_progress(
)
for entry in cache_dirs:
completed_bytes = 0
- # Keyed by logical blob: a broken advisory lock leaves several process-unique writers
- # racing on one etag, and each downloads the WHOLE file, so summing them overshoots.
+ # Keyed by logical blob: a broken advisory lock leaves several writers racing on one etag, each
+ # downloading the WHOLE file, so summing them overshoots.
partial_bytes: dict[str, int] = {}
completed_hashes: set[str] = set()
- # A partial this reading could not attribute to any target. It is not evidence FOR this
- # variant -- it may be a sibling quant's -- but with the hashes unresolved it is not
- # evidence against it either, and the by-name scan cannot see it because a partial is
- # not linked into a snapshot yet.
+ # A partial attributable to no target is not evidence for this variant, nor against it while the
+ # hashes are unresolved, and the by-name scan cannot see it.
unattributable_partial = False
cache_path = hf_cache_scan.resolve_hf_cache_realpath(entry)
blobs_dir = entry / "blobs"
+ # Skip a blob that vanished mid-poll rather than zeroing the reading.
try:
- # os.stat, not Path.is_dir(): is_dir() swallows the OSError and answers False, so a
- # Windows ACL or network-filesystem failure on the directory ITSELF read as "no
- # blobs here" -- a MEASURED absence, which hydration retires the job on.
+ # os.stat, not Path.is_dir(): is_dir() swallows the OSError and answers False, turning a Windows ACL
+ # or network-filesystem failure into a MEASURED absence hydration retires on.
blobs_present = stat_module.S_ISDIR(os.stat(blobs_dir).st_mode)
except FileNotFoundError:
blobs_present = False
@@ -564,13 +534,11 @@ def compute_snapshot_progress(
try:
blob_entries = list(blobs_dir.iterdir())
except OSError as exc:
- # An unreadable blobs dir is not an empty one. Swallowing it produced a
- # measured zero -- target_present false, cache_measured true -- and hydration
- # retires a job whose cache was never actually read.
+ # An unreadable blobs dir is not an empty one: swallowing it produced a measured zero
+ # (target_present false, cache_measured true) and retired a job whose cache was never read.
scan_errors.append(exc)
blob_entries = []
for f in blob_entries:
- # Skip a blob that vanished mid-poll rather than zeroing the reading.
try:
if not f.is_file():
continue
@@ -594,29 +562,19 @@ def compute_snapshot_progress(
completed_hashes.add(f.name)
completed_bytes += f.stat().st_size
except OSError as exc:
- # A blob we could not inspect is not a blob that is not there. Swallowing it
- # produced a MEASURED absence, which hydration reads as "gone" and retires a
- # persisted download whose target may be intact behind the error.
+ # A blob we could not inspect is not a blob that is not there: swallowing it produced a MEASURED
+ # absence, which hydration reads as gone.
scan_errors.append(exc)
continue
- # A finalized blobs/<hash> supersedes every partial for the same logical blob: the racer
- # that installed the file settled it, so a leftover writer's bytes are a duplicate, not
- # progress. Counting both overshot the expected total and, because completion requires
- # no bytes in flight, pinned a fully downloaded variant at 0.99 until the orphan was
- # swept -- and an orphan outlives the process that would have unlinked it on the way out.
+ # A finalized blobs/<hash> supersedes every partial for the same logical blob, so counting both
+ # overshot the expected total and pinned a downloaded variant at 0.99 until the orphan was swept.
for blob_hash in completed_hashes:
partial_bytes.pop(blob_hash, None)
- # Largest wins, deliberately, even though a killed attempt's partial can sit beside its
- # replacement's under the same etag. Preferring the freshest mtime reads better against
- # a corpse but oscillates between two GENUINELY live writers, which is what a broken
- # advisory lock produces: each write makes a different one newest, so the bar would
- # jump between the leader and the straggler.
- #
- # A corpse is not this reading's problem to solve, because it should not outlive the
- # job that made it: a terminal job sweeps its own blobs without waiting out the
- # abandonment grace, and a backend that died before it could is caught at boot. If one
- # survives both (an unrelated client holding the blob lock over it) the bar over-reads
- # until the next sweep, which is a smaller wrong than a reading that will not sit still.
+ # Largest wins deliberately: preferring the freshest mtime reads better against a corpse but
+ # oscillates between two genuinely live writers, which is what a broken advisory lock produces.
+ # A corpse should not outlive the job that made it (a terminal job sweeps its own blobs, and a
+ # backend that died first is caught at boot); if one survives both, over-reading until the next
+ # sweep is a smaller wrong than a reading that will not sit still.
in_progress_bytes = sum(partial_bytes.values())
snapshot_dirs: "_Lazy[list[Path]]" = _Lazy(
lambda entry = entry: _retained_snapshot_dirs(entry)
@@ -630,12 +588,9 @@ def compute_snapshot_progress(
)
)
if variant is not None:
- # The best reading across every retained snapshot, for the same reason presence is
- # established across all of them: the variant can live in an older revision while
- # the newest holds a sibling, and reading only the newest reported 0 bytes for a
- # complete cached quant. Do this even when hashes resolved: huggingface_hub 1.18's
- # Windows copy layout can move a completed file directly into the snapshot and
- # leave no finalized entry in blobs/, so a blob-only tally stays at zero.
+ # The best reading across every retained snapshot, since the variant can live in an older
+ # revision, and because huggingface_hub 1.18's Windows copy layout can move a completed file
+ # straight into the snapshot and leave a blob-only tally at zero.
manifest = entry_manifest.get()
on_disk = max(
(
@@ -650,50 +605,34 @@ def compute_snapshot_progress(
),
default = 0,
)
- # Clamped, because the matcher behind the no-manifest half of that reading accepts
- # every companion in the repo and so can overshoot.
+ # Clamped, because the matcher behind the no-manifest half accepts every companion in the repo and
+ # so can overshoot.
if expected_total > 0:
on_disk = min(on_disk, expected_total)
completed_bytes = max(completed_bytes, on_disk)
- # Does THIS target have anything here, as opposed to the repo dir existing at all?
- # Sibling quants share one repo cache dir, so deleting a variant's files leaves the
- # dir standing and the reading came back "zero bytes, cache_path names a directory"
- # -- which hydration reads as resumable and adopts as a phantom, blocking a fresh
- # download of that same variant until the idle grace expires. False only on positive
- # evidence of absence: a named variant, a file set we could resolve, no manifest of
- # its own and nothing counted. Anything less certain stays None.
+ # Sibling quants share one repo cache dir, so deleting a variant's files leaves the dir standing
+ # and the reading came back "zero bytes, cache_path names a directory", which hydration adopts as
+ # a phantom. False only on positive evidence of absence; anything less certain stays None.
target_present: Optional[bool] = None
if variant is not None and not variant_file_set_unknown:
- # The MATERIALIZED file, not the blob tally. These counters come from the shared
- # blobs/ dir: deleting a variant's snapshot symlink leaves its finalized blob
- # behind, and a companion blob shared with a sibling keeps the count positive on its
- # own -- so a quant that is gone read as present and hydration re-adopted the
- # phantom. The by-name scan the unknown-hash branch already uses answers this
- # properly; bytes only stand in when it cannot (nothing readable to look at), where
- # in-progress bytes are still real evidence that this download is live.
+ # The MATERIALIZED file, not the blob tally: deleting a snapshot symlink leaves the finalized blob
+ # behind and a shared companion keeps the count positive, so a quant that is gone read as present.
+ # Bytes only stand in when there is nothing readable to scan.
scanned = _variant_present_in_any_snapshot(entry, variant_file_matcher)
if scanned is not None:
target_present = scanned or bool(in_progress_bytes)
else:
target_present = bool(completed_bytes or in_progress_bytes)
elif variant is not None:
- # An unresolvable file set does not have to mean unknown. The byte reading above
- # already walked the snapshot dir, whose entries are named per file, so it can
- # answer the narrower question the hash filter could not: is a main shard of THIS
- # quant here? Without this, a repo dir kept alive by a sibling read as "zero bytes,
- # cache_path names a directory", which hydration adopts as a resumable phantom and
- # which then blocks a fresh download of the deleted quant until the idle grace runs
- # out -- the same trap as above, on the metadata-unavailable path.
- # ...across EVERY snapshot the entry retains, not only the newest by mtime. A cache
- # can hold several revisions, and the requested quant living in an older one while
- # the newest holds a sibling read as absent -- and hydration retires a job whose
- # target is still usable.
+ # The byte reading already walked the snapshot dir, whose entries are named per file, so it can
+ # answer whether a main shard of THIS quant is here; without it a repo dir kept alive by a sibling
+ # read as "zero bytes, cache_path names a directory" and was adopted as a resumable phantom.
+ # Across EVERY snapshot the entry retains, not only the newest: a quant living in an older
+ # revision read as absent and hydration retired a job whose target is still usable.
scanned = _variant_present_in_any_snapshot(entry, variant_file_matcher)
if scanned is not None:
- # ...unless the shared blobs/ dir holds a partial nothing here can attribute.
- # An idle or restarted download whose hashes were refused has its bytes in an
- # .incomplete blob that is not linked into any snapshot yet, so the by-name scan
- # answers a confident "absent" -- and hydration retires a job that is resumable.
+ # Unless the shared blobs/ dir holds an unattributable partial: an idle or restarted download
+ # whose hashes were refused has its bytes in an .incomplete blob no snapshot links.
target_present = None if (not scanned and unattributable_partial) else scanned
readings.append(
(
@@ -719,52 +658,39 @@ def compute_snapshot_progress(
selected = max(
readings,
- # complete_on_disk last-but-one: two remembered caches can clamp to the SAME byte total
- # while only one has a manifest that verifies against disk, and root order then carried
- # the unverified reading -- the response stayed capped at 99% and kept offering Retry.
+ # complete_on_disk last-but-one: two remembered caches can clamp to the SAME byte total while only
+ # one has a manifest that verifies against disk, and root order then capped the response at 99%.
key = lambda item: (item[0] + item[1], bool(item[3]), item[0]),
default = None,
)
if selected is None:
- # Nothing measured AND a root that could not be listed: the cache may be entirely
- # intact behind that error, so this is unknown, not gone.
+ # Nothing measured AND a root that could not be listed: the cache may be entirely intact behind that
+ # error, so this is unknown, not gone.
if scan_errors:
return _empty_progress(expected_bytes, measured = False)
return empty
completed_bytes, in_progress_bytes, cache_path, complete_on_disk, target_present = selected
- # Presence is a property of the SET of caches, not of the one that happened to hold the most
- # bytes. A sibling-only repo dir and a cache that still holds this variant's manifest both
- # read as zero bytes, so root order alone could pick the False and retire a job whose target
- # another scanned cache proves is there. A positive reading anywhere wins.
+ # Presence is a property of the SET of caches: a sibling-only repo dir and a cache still holding
+ # this variant's manifest both read as zero bytes, so a positive reading anywhere wins.
presence = [reading[4] for reading in readings]
if any(verdict is True for verdict in presence):
target_present = True
elif any(verdict is None for verdict in presence):
- # And absence needs EVERY scanned cache to say so. One unknown reading -- a cache with
- # no readable snapshot to identify the variant from, whose shared blobs dir may still
- # hold an unattributable partial -- is not evidence the target is gone.
+ # Absence needs EVERY scanned cache to say so: one unknown reading is not evidence the target is gone.
target_present = None
downloaded_bytes = completed_bytes + in_progress_bytes
- # A reading taken while some root could not be listed is only ever a LOWER bound. The
- # active root raising EACCES/EIO while a remembered cache still holds the repo dir (with a
- # sibling quant in it) gives a non-null selection carrying target_present False and zero
- # bytes -- and hydration reads that as "the variant was deleted" and retires the job,
- # though the inaccessible root may hold every byte of it. Absence claims need a complete
- # scan behind them, so downgrade them to unknown; a positive reading is unaffected,
- # because bytes we did see are bytes that are there.
+ # A reading taken while some root could not be listed is only ever a LOWER bound: the active root
+ # raising EACCES/EIO while a remembered cache holds the repo dir gives target_present False and
+ # zero bytes, which hydration reads as "deleted". Downgrade absence claims to unknown; a positive
+ # reading is unaffected.
scan_incomplete = bool(scan_errors)
if scan_incomplete:
if not target_present:
target_present = None
- # Subtract the companion baseline only while still counted in completed_bytes
- # and the variant is not yet verified complete, else genuine progress reads as
- # 0-byte. A baseline that covers the whole expected total is never subtracted
- # either: a variant that was already on disk when the job was claimed carries
- # exactly that, and netting it out leaves "0 B of 0 B" -- nothing for the bar
- # to draw and, to the frontend, a job to evict. complete_on_disk covered that
- # only while completion could be verified, which is the one thing an
- # unresolvable file set takes away.
+ # Subtract the companion baseline only while it is still counted in completed_bytes and the
+ # variant is unverified, and never when it covers the whole expected total: that leaves
+ # "0 B of 0 B", which the frontend evicts as a dead job.
effective_baseline_bytes = (
completed_baseline_bytes
if (
@@ -801,8 +727,8 @@ def compute_snapshot_progress(
"cache_measured": not scan_incomplete,
}
- # Cap at 0.99 until the manifest-backed disk check verifies completion: on
- # resume, completed bytes can sit above the threshold while files still download.
+ # Cap at 0.99 until the manifest-backed disk check verifies completion: on resume, completed bytes
+ # can sit above the threshold while files still download.
progress = (
1.0
if complete_on_disk
diff --git a/studio/backend/hub/storage/scan_folders.py b/studio/backend/hub/storage/scan_folders.py
index c19f43fac9a..2fa3a57b994 100644
--- a/studio/backend/hub/storage/scan_folders.py
+++ b/studio/backend/hub/storage/scan_folders.py
@@ -32,8 +32,8 @@ def _denied_path_prefixes() -> list[str]:
if system == "Linux":
return ["/proc", "/sys", "/dev", "/etc", "/boot", "/run"]
if system == "Darwin":
- # realpath() resolves /etc -> /private/etc, /tmp -> /private/tmp on macOS,
- # so include the /private variants to avoid bypasses.
+ # realpath() resolves /etc -> /private/etc and /tmp -> /private/tmp on macOS, so include the
+ # /private variants to avoid bypasses.
return [
"/System",
"/Library",
@@ -127,8 +127,8 @@ def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:
if not os.path.isdir(normalized):
raise ValueError("Path must be a directory, not a file")
if is_local_filesystem_root(normalized):
- # A local fs root ("/", "C:\\") would expose denied system dirs via browse;
- # a UNC share root (\\server\share) has none under it and stays registerable.
+ # A local fs root would expose denied system dirs via browse; a UNC share root has none under it and
+ # stays registerable.
raise ValueError("The filesystem root cannot be registered")
if _contains_sensitive_path_component(normalized):
raise ValueError("Credential or configuration directories are not allowed")
diff --git a/studio/backend/hub/tests/test_companion_assets.py b/studio/backend/hub/tests/test_companion_assets.py
index 5827faef4bb..15dc2152f57 100644
--- a/studio/backend/hub/tests/test_companion_assets.py
+++ b/studio/backend/hub/tests/test_companion_assets.py
@@ -92,8 +92,6 @@ def _install(monkeypatch, *repos):
# --------------------------------------------------------------------------------------------
-# 1. Compatible quants reuse ONE cached copy of the companion assets.
-# --------------------------------------------------------------------------------------------
def test_two_quants_of_one_family_resolve_to_a_single_companion_base():
@@ -105,8 +103,6 @@ def test_two_quants_of_one_family_resolve_to_a_single_companion_base():
# --------------------------------------------------------------------------------------------
-# 2. Deletion shows what is reclaimed and what remains.
-# --------------------------------------------------------------------------------------------
def test_deleting_one_of_two_quants_retains_the_companions(monkeypatch):
@@ -140,8 +136,6 @@ def test_whole_repo_delete_reclaims_every_quant(monkeypatch):
# --------------------------------------------------------------------------------------------
-# 3. Shared assets are not removed while another installed model needs them. ADVERSARIAL.
-# --------------------------------------------------------------------------------------------
def test_shared_base_cannot_be_deleted_while_a_quant_is_installed(monkeypatch):
@@ -184,8 +178,8 @@ def test_a_base_reached_only_through_a_card_tag_is_still_protected(monkeypatch):
gguf = _repo("unsloth/FLUX.2-klein-9B-GGUF", [("flux-2-klein-9b-Q4_K_M.gguf", Q4_K_M_BYTES)])
other_base = "black-forest-labs/FLUX.2-klein-9B"
_install(monkeypatch, gguf, _base_repo(other_base))
- # Before any load has been recorded: the id says klein-9B, so the curated klein-9B base is
- # derived even though the family default is klein-4B.
+ # Before any load has been recorded: the id says klein-9B, so the curated klein-9B base is derived
+ # even though the family default is klein-4B.
assert companion_cleanup.companion_dependents(other_base) == ["unsloth/FLUX.2-klein-9B-GGUF"]
companion_assets.record_companion_link("unsloth/FLUX.2-klein-9B-GGUF", other_base)
assert companion_cleanup.companion_dependents(other_base) == ["unsloth/FLUX.2-klein-9B-GGUF"]
@@ -221,8 +215,6 @@ def test_the_guard_leaves_ordinary_repos_alone(monkeypatch):
# --------------------------------------------------------------------------------------------
-# 4. Orphaned companion assets can be found and removed without hand-editing the HF cache.
-# --------------------------------------------------------------------------------------------
def test_orphan_listing_is_empty_while_a_quant_is_installed(monkeypatch):
@@ -421,8 +413,8 @@ def test_reusing_an_existing_link_still_refreshes_its_recency(monkeypatch):
monkeypatch.setattr(companion_assets, "_MAX_LINKS", 2)
assert companion_assets.record_companion_link("unsloth/old-GGUF", BASE_REPO) is True
assert companion_assets.record_companion_link("unsloth/new-GGUF", BASE_REPO) is True
- # The old one resolves again to the SAME base: nothing new to record, but it is now the
- # freshest link, so the next checkpoint displaces the other one.
+ # The old one resolves again to the SAME base: nothing new to record, but it is now the freshest
+ # link, so the next checkpoint displaces the other one.
assert companion_assets.record_companion_link("unsloth/old-GGUF", BASE_REPO) is False
assert companion_assets.read_companion_links()["unsloth/old-gguf"] == [BASE_REPO]
assert companion_assets.record_companion_link("unsloth/third-GGUF", BASE_REPO) is True
diff --git a/studio/backend/hub/tests/test_dataset_local_options.py b/studio/backend/hub/tests/test_dataset_local_options.py
index a7549c3da41..c650eac8b8b 100644
--- a/studio/backend/hub/tests/test_dataset_local_options.py
+++ b/studio/backend/hub/tests/test_dataset_local_options.py
@@ -233,6 +233,7 @@ def test_snapshot_options_infer_one_train_split_from_unlabelled_files(tmp_path):
snapshot = _snapshot(tmp_path)
_rows(snapshot, "records.jsonl")
+ # A card with no bytes declares nothing, so it blocks nothing.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -255,6 +256,11 @@ def test_snapshot_options_reject_a_shard_name_the_loader_refuses(tmp_path):
_rows(snapshot, "data/train-clean-00000-of-00001.parquet")
# datasets raises on the name rather than falling through to the keyword stages.
+ # datasets calls .get on each entry, so a list of scalars raises.
+ # DatasetCard.load reads the card as utf-8 and raises before any data file.
+ # The inner .jsonl still votes, so datasets picks json and dies on the .zst.
+ # DatasetCardData updates a dict from it and raises on a scalar.
+ # imagefolder wins the vote and drops the jsonl, so there is nothing to train on.
assert local_options._snapshot_options(snapshot) == set()
@@ -290,6 +296,9 @@ def test_snapshot_options_ignore_a_metadata_file_when_it_steers_a_split(tmp_path
(snapshot / "test").mkdir()
(snapshot / "test" / "dataset_info.json").write_text("{}", encoding = "utf-8")
+ # The header alone yields no row, and datasets still builds train around it.
+ # resolve_pattern keeps a link only when its target is a file, so the loader never sees this one and the
+ # surviving split still loads.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -298,6 +307,7 @@ def test_snapshot_options_match_extensions_case_sensitively(tmp_path):
_rows(snapshot, "TRAIN.JSONL")
# A POSIX glob never matches .JSONL, so datasets finds no data files here.
+ # datasets treats the empty declaration as authoritative and exposes no config.
assert local_options._snapshot_options(snapshot) == set()
@@ -379,7 +389,6 @@ def test_snapshot_options_read_a_blob_symlink_the_cache_owns(tmp_path):
snapshot.mkdir(parents = True)
(snapshot / "records.jsonl").symlink_to(blobs / "abc123")
- # The normal cache layout: every file is a link into blobs.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -388,8 +397,8 @@ def test_snapshot_options_leave_a_declared_card_to_its_own_configs(tmp_path):
_card(snapshot, "configs:\n- config_name: foo\n data_dir: foo\n")
_rows(snapshot, "foo/records.jsonl", "test.jsonl")
- # The card names the loader's configs, and inference cannot reproduce them, so the
- # picker does not invent a default one beside them.
+ # The card names the loader's configs, and inference cannot reproduce them, so the picker does not
+ # invent a default one beside them.
assert local_options._snapshot_options(snapshot) == set()
@@ -422,7 +431,6 @@ def test_snapshot_options_stay_empty_for_a_readme_only_cache(tmp_path):
snapshot = _snapshot(tmp_path)
(snapshot / "README.md").write_text("# Alpaca\n", encoding = "utf-8")
- # The first #8140 negative: nothing to train on, so nothing is offered.
assert local_options._snapshot_options(snapshot) == set()
@@ -431,7 +439,6 @@ def test_snapshot_options_stay_empty_for_a_licence_only_subdirectory(tmp_path):
(snapshot / "legal").mkdir()
(snapshot / "legal" / "LICENSE").write_text("MIT\n", encoding = "utf-8")
- # The second #8140 negative.
assert local_options._snapshot_options(snapshot) == set()
@@ -669,7 +676,6 @@ def test_snapshot_options_reject_a_split_whose_module_needs_a_missing_codec(tmp_
(snapshot / "train.csv").write_text("text\nrow\n", encoding = "utf-8")
(snapshot / "train2.jsonl.zst").write_bytes(b"\x28\xb5\x2f\xfd\x00\x00")
- # The inner .jsonl still votes, so datasets picks json and dies on the .zst.
assert local_options._snapshot_options(snapshot) == set()
@@ -686,7 +692,6 @@ def test_snapshot_options_stand_down_beside_a_card_that_cannot_be_decoded(tmp_pa
(snapshot / "README.md").write_bytes(b"\xff\xfe---\nconfigs: x\n---\n")
_rows(snapshot, "train.jsonl")
- # DatasetCard.load reads the card as utf-8 and raises before any data file.
assert local_options._snapshot_options(snapshot) == set()
@@ -695,7 +700,6 @@ def test_snapshot_options_stand_down_beside_a_malformed_dataset_info_list(tmp_pa
_card(snapshot, "dataset_info: [foo]\n")
_rows(snapshot, "train.jsonl")
- # datasets calls .get on each entry, so a list of scalars raises.
assert local_options._snapshot_options(snapshot) == set()
@@ -704,8 +708,6 @@ def test_snapshot_options_ignore_a_dangling_link_beside_a_good_file(tmp_path):
_rows(snapshot, "train.jsonl")
(snapshot / "train-missing.jsonl").symlink_to("../../blobs/gone")
- # resolve_pattern keeps a link only when its target is a file, so the loader
- # never sees this one and the surviving split still loads.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -714,7 +716,6 @@ def test_snapshot_options_drop_a_header_only_csv_split(tmp_path):
(snapshot / "train.csv").write_text("text\nrow\n", encoding = "utf-8")
(snapshot / "test.csv").write_text("text\n", encoding = "utf-8")
- # The header alone yields no row, and datasets still builds train around it.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -730,7 +731,6 @@ def test_snapshot_options_stand_down_beside_an_empty_split_declaration(tmp_path)
_card(snapshot, "dataset_info:\n splits: {}\n")
_rows(snapshot, "train.jsonl")
- # datasets treats the empty declaration as authoritative and exposes no config.
assert local_options._snapshot_options(snapshot) == set()
@@ -754,6 +754,7 @@ def test_snapshot_options_keep_a_compressed_csv_split(tmp_path):
snapshot = _snapshot(tmp_path)
(snapshot / "train.csv.gz").write_bytes(gzip.compress(b"text\nrow\n"))
+ # `[]` parses fine and yields no row, so datasets builds train around it.
# Compressed bytes say nothing about the rows inside, so the split still stands.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -763,7 +764,6 @@ def test_snapshot_options_drop_a_split_holding_an_empty_json_container(tmp_path)
_rows(snapshot, "train.jsonl")
(snapshot / "test.json").write_text("[]", encoding = "utf-8")
- # `[]` parses fine and yields no row, so datasets builds train around it.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
diff --git a/studio/backend/hub/tests/test_dataset_services.py b/studio/backend/hub/tests/test_dataset_services.py
index 816748a0cdb..d78b502e602 100644
--- a/studio/backend/hub/tests/test_dataset_services.py
+++ b/studio/backend/hub/tests/test_dataset_services.py
@@ -254,8 +254,8 @@ def test_raw_dataset_cache_has_data_does_not_loop_on_a_link_to_an_ancestor(monke
repo_root = _dataset_snapshot(monkeypatch, tmp_path, ("README.md", "data/notes.md"))
data = next((repo_root / "snapshots").iterdir()) / "data"
(data / "loop").symlink_to(data, target_is_directory = True)
- # a junction resolves like a link while reporting `is_symlink()` False, and Linux cannot
- # create one, so this is a symlink that answers the way a junction does.
+ # a junction resolves like a link while reporting is_symlink() False, and Linux cannot create one,
+ # so this is a symlink that answers the way a junction does
real_is_symlink = Path.is_symlink
monkeypatch.setattr(
Path,
@@ -442,8 +442,8 @@ def test_raw_dataset_cache_has_data_never_walks_outside_the_snapshot(monkeypatch
(outside / "deep").mkdir(parents = True)
(outside / "deep" / "huge.parquet").write_bytes(b"PAR1")
repo_root = _dataset_snapshot(monkeypatch, tmp_path, ("README.md",))
- # Named like clutter, so it is not payload evidence either, and the tree behind it is
- # never entered -- which is what stops a scan from stalling on someone else's disk.
+ # Named like clutter, so it is not payload evidence either, and the tree behind it is never entered,
+ # which is what stops a scan from stalling on someone else's disk.
(repo_root / "snapshots" / "abc" / ".hidden_link").symlink_to(outside, target_is_directory = True)
assert cache_inventory._raw_dataset_cache_has_data("Org/Data", repo_root) is False
@@ -704,8 +704,8 @@ def test_delete_processed_dataset_scopes_to_selected_root(monkeypatch, tmp_path)
)
assert result == {"status": "deleted", "repo_id": "Org/Data"}
- assert not (selected_root / "Org___Data").exists() # the selected copy is deleted
- assert (other_root / "Org___Data").exists() # the other cache home is untouched
+ assert not (selected_root / "Org___Data").exists()
+ assert (other_root / "Org___Data").exists()
def _app_cache_entry(monkeypatch, hub_cache: Path, repo_id: str, commit_hash: str):
diff --git a/studio/backend/hub/tests/test_download_lifecycle.py b/studio/backend/hub/tests/test_download_lifecycle.py
index 241b47354fd..6e8a1b53940 100644
--- a/studio/backend/hub/tests/test_download_lifecycle.py
+++ b/studio/backend/hub/tests/test_download_lifecycle.py
@@ -97,9 +97,9 @@ def test_completion_invalidates_inventory_before_publishing_state(monkeypatch):
def test_xet_failure_retries_over_http_for_model_and_dataset(monkeypatch, tmp_path):
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
monkeypatch.setattr(download_lifecycle.threading, "Thread", _ImmediateThread)
- # _ImmediateThread mutates the stdlib threading module, which the shared Zoo watchdog also
- # imports, so it would run INLINE and block in Event.wait() before finalize_worker_exit could
- # set its stop flag. Stub the seam instead.
+ # _ImmediateThread mutates the stdlib threading module the shared Zoo watchdog also imports, so it
+ # would run INLINE and block in Event.wait() before finalize_worker_exit could stop it. Stub the
+ # seam instead.
monkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", lambda *a, **k: None)
register_worker = download_lifecycle.register_worker
@@ -303,7 +303,7 @@ def test_a_verdict_carried_onto_the_http_rung_is_still_charged(monkeypatch, tmp_
spawned.append(use_xet)
if use_xet:
raise OSError("cannot fork")
- return _Proc(0) # the HTTP worker starts and completes
+ return _Proc(0)
monkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", _start)
monkeypatch.setattr(download_lifecycle, "spawn_worker", flaky_spawn)
@@ -334,9 +334,8 @@ def test_a_verdict_carried_onto_the_http_rung_is_still_charged(monkeypatch, tmp_
transport = download_registry.TRANSPORT_XET,
watch_name = "model-watch",
)
- # Before the verdict: a double whose signature has fallen behind spawn_worker raises
- # TypeError at the call, which reads as a spawn failure and still records the verdict via
- # _give_up() -- green while this path never ran. Assert the HTTP worker actually started.
+ # Before the verdict: a double whose signature has fallen behind spawn_worker raises TypeError,
+ # which reads as a spawn failure and still records the verdict, staying green.
assert spawned == [True, False], "the HTTP rung never ran, so the verdict proves nothing"
assert recorded == [verdict], "a real Xet stall was dropped when HTTP finished the download"
@@ -344,9 +343,8 @@ def test_a_verdict_carried_onto_the_http_rung_is_still_charged(monkeypatch, tmp_
def test_http_failure_remains_terminal(monkeypatch, tmp_path):
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
monkeypatch.setattr(download_lifecycle.threading, "Thread", _ImmediateThread)
- # _ImmediateThread mutates the stdlib threading module, which the shared Zoo watchdog also
- # imports, so it would run INLINE and block in Event.wait() before finalize_worker_exit could
- # set its stop flag. Stub the seam instead.
+ # _ImmediateThread mutates the stdlib threading module the shared Zoo watchdog also imports, so stub
+ # the seam instead.
monkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", lambda *a, **k: None)
register_worker = download_lifecycle.register_worker
registry = download_registry.DownloadRegistry()
@@ -515,7 +513,7 @@ def _trip_xet_worker(
monkeypatch.setattr(download_lifecycle.threading, "Thread", _ImmediateThread)
def _start(registry, key, proc, *, on_stall, **kwargs):
- on_stall(message) # the watchdog tripped
+ on_stall(message)
return None
monkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", _start)
@@ -688,7 +686,7 @@ def test_the_xet_baseline_is_sampled_before_the_worker_spawns(monkeypatch, tmp_p
monkeypatch.setattr(download_lifecycle, "_start_stall_watchdog", lambda *a, **k: None)
order = []
- sizes = iter([0, 5_000]) # nothing before the spawn, bytes present after it
+ sizes = iter([0, 5_000])
monkeypatch.setattr(
download_lifecycle,
@@ -743,7 +741,7 @@ def test_a_stall_verdict_racing_a_completed_worker_is_not_recorded(monkeypatch,
monkeypatch,
tmp_path,
"Download appears stalled (xet transport) -- no progress for 30s",
- rc = 0, # the worker actually finished
+ rc = 0,
)
== []
), "a completed worker was charged a stall it raced"
diff --git a/studio/backend/hub/tests/test_download_manifest_scoping.py b/studio/backend/hub/tests/test_download_manifest_scoping.py
index 7cbd137fdda..691702824f1 100644
--- a/studio/backend/hub/tests/test_download_manifest_scoping.py
+++ b/studio/backend/hub/tests/test_download_manifest_scoping.py
@@ -77,8 +77,8 @@ def test_purge_state_preserves_active_legacy_when_deleting_inactive_cache(monkey
removed = download_manifest.purge_state("model", "Org/Model", hub_cache = str(previous))
assert removed is True
- assert not scoped.is_file() # the inactive cache's copy is gone
- assert legacy.is_file() # the active cache's legacy state survives
+ assert not scoped.is_file()
+ assert legacy.is_file()
def test_purge_state_removes_legacy_owned_by_the_deleted_cache(monkeypatch, tmp_path):
@@ -100,7 +100,7 @@ def test_purge_state_removes_legacy_owned_by_the_deleted_cache(monkeypatch, tmp_
removed = download_manifest.purge_state("model", "Org/Model", hub_cache = str(previous))
assert removed is True
- assert not legacy.is_file() # owned by the deleted cache -> purged
+ assert not legacy.is_file()
def test_scope_digest_is_shared_with_the_ownership_canonicalization(monkeypatch, tmp_path):
@@ -302,7 +302,7 @@ def test_windows_shaped_copy_cache_scope_survives_a_restart(monkeypatch, tmp_pat
hub_cache = tmp_path / "Hub"
snapshot = hub_cache / "models--Org--Model" / "snapshots" / "rev0"
snapshot.mkdir(parents = True)
- (snapshot / "model.gguf").write_bytes(b"x" * 16) # a copy, not a symlink
+ (snapshot / "model.gguf").write_bytes(b"x" * 16)
assert not (snapshot / "model.gguf").is_symlink()
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
monkeypatch.setattr(
@@ -741,14 +741,14 @@ def test_an_unreadable_cache_root_is_unknown_rather_than_absent(monkeypatch, tmp
metadata_resolver = lambda *a, **k: (33_000_000_000, frozenset()),
)
assert "cache_path" not in reading
- # And it survives DownloadProgressResponse, which defaults cache_path to None and would
- # otherwise reinstate the omission as an explicit "absent" before the frontend saw it.
+ # And it survives DownloadProgressResponse, which defaults cache_path to None and would otherwise reinstate
+ # the omission as an explicit "absent" before the frontend saw it.
assert reading["cache_measured"] is False
from hub.schemas.downloads import DownloadProgressResponse
- # DECLARED on the response model, or FastAPI drops it before the frontend sees it: these
- # readings are serialized through DownloadProgressResponse, whose cache_path defaults to
- # None, so the omission alone was reinstated as an explicit "absent" on the wire.
+ # DECLARED on the response model, or FastAPI drops it before the frontend sees it: these readings
+ # serialize through DownloadProgressResponse, whose cache_path defaults to None, so the omission
+ # alone was reinstated as an explicit "absent".
assert "cache_measured" in DownloadProgressResponse.__annotations__
assert reading["downloaded_bytes"] == 0
@@ -772,7 +772,7 @@ def test_a_scope_whose_payload_is_lost_reads_back_as_a_digest(monkeypatch, tmp_p
((variant, path),) = download_manifest.iter_variant_markers(
"model", "Org/Model", hub_cache = str(hub_cache)
)
- assert variant == "@diffusion" # intact while the payload is readable
+ assert variant == "@diffusion"
assert "--variant--@sha256-" in path.name
payload = json.loads(path.read_text(encoding = "utf-8"))
@@ -804,7 +804,7 @@ def test_a_variant_with_nothing_of_its_own_says_so(monkeypatch, tmp_path):
entry = tmp_path / "hub" / "models--unsloth--Model-GGUF"
(entry / "blobs").mkdir(parents = True)
- (entry / "blobs" / "sibling").write_bytes(b"x" * 32) # Q8_0's, not ours
+ (entry / "blobs" / "sibling").write_bytes(b"x" * 32)
monkeypatch.setattr(snapshot_progress, "preferred_repo_cache_dirs", lambda *a, **k: [entry])
diff --git a/studio/backend/hub/tests/test_empty_variant_folder.py b/studio/backend/hub/tests/test_empty_variant_folder.py
index dc4872b0709..cb9d312d837 100644
--- a/studio/backend/hub/tests/test_empty_variant_folder.py
+++ b/studio/backend/hub/tests/test_empty_variant_folder.py
@@ -17,7 +17,7 @@ def _make_snapshot(root: Path) -> Path:
(snap / "UD-IQ1_M").mkdir(parents = True)
(snap / "UD-IQ1_M" / "GLM-UD-IQ1_M-00001-of-00002.gguf").write_bytes(b"x")
(snap / "UD-IQ1_M" / "GLM-UD-IQ1_M-00002-of-00002.gguf").write_bytes(b"y")
- (snap / "UD-IQ1_S").mkdir(parents = True) # empty leftover
+ (snap / "UD-IQ1_S").mkdir(parents = True)
return snap
@@ -29,17 +29,17 @@ def test_list_empty_gguf_variant_dirs_finds_empty_leftover(tmp_path, monkeypatch
def test_list_empty_excludes_quant_with_files_in_another_snapshot(tmp_path, monkeypatch):
snap1 = tmp_path / "s1" / "snapshots" / "rev"
- (snap1 / "UD-IQ1_S").mkdir(parents = True) # empty here
+ (snap1 / "UD-IQ1_S").mkdir(parents = True)
snap2 = tmp_path / "s2" / "snapshots" / "rev"
(snap2 / "UD-IQ1_S").mkdir(parents = True)
- (snap2 / "UD-IQ1_S" / "m-UD-IQ1_S-00001-of-00001.gguf").write_bytes(b"z") # has shards
+ (snap2 / "UD-IQ1_S" / "m-UD-IQ1_S-00001-of-00001.gguf").write_bytes(b"z")
monkeypatch.setattr(gguf, "iter_hf_cache_snapshots", lambda repo_id: iter([snap1, snap2]))
assert gguf.list_empty_gguf_variant_dirs("org/Repo-GGUF") == set()
def test_list_empty_ignores_non_quant_dirs(tmp_path, monkeypatch):
snap = tmp_path / "snapshots" / "rev"
- (snap / "not-a-quant").mkdir(parents = True) # empty but not a quant label
+ (snap / "not-a-quant").mkdir(parents = True)
monkeypatch.setattr(gguf, "iter_hf_cache_snapshots", lambda repo_id: iter([snap]))
assert gguf.list_empty_gguf_variant_dirs("org/Repo-GGUF") == set()
@@ -143,8 +143,8 @@ def test_mark_empty_dir_cleanables_flips_listed_variant(monkeypatch):
def _force_compute_to_raise(monkeypatch):
- # Drive _compute() down its remote path, fail metadata, and have both cache
- # fallbacks miss so the original error re-raises.
+ # Drive _compute() down its remote path, fail metadata, and have both cache fallbacks miss so the
+ # original error re-raises.
def _boom(*a, **k):
raise RuntimeError("offline")
@@ -164,8 +164,8 @@ def _force_compute_to_raise(monkeypatch):
def test_get_variants_surfaces_cleanable_when_metadata_fails(monkeypatch):
- # Offline / model_info fails and only an empty leftover folder is cached:
- # the cleanable must still be returned instead of the error propagating.
+ # Offline / model_info fails and only an empty leftover folder is cached: the cleanable must still
+ # be returned instead of the error propagating.
import asyncio
_force_compute_to_raise(monkeypatch)
diff --git a/studio/backend/hub/tests/test_model_services.py b/studio/backend/hub/tests/test_model_services.py
index ded44508c2a..32146b25899 100644
--- a/studio/backend/hub/tests/test_model_services.py
+++ b/studio/backend/hub/tests/test_model_services.py
@@ -232,7 +232,8 @@ def test_custom_inventory_filters_mtp_companions_at_registered_root(tmp_path, mo
def test_custom_inventory_filters_dspark_companions_at_registered_root(tmp_path):
- # Registering dspark/ as the scan root strips it from every relative path, so the basename prefix carries the exclusion.
+ # Registering dspark/ as the scan root strips it from every relative path, so the basename prefix
+ # carries the exclusion.
root = tmp_path / "dspark"
root.mkdir()
for name in (
@@ -887,7 +888,7 @@ def test_cached_inventory_discards_a_scan_that_raced_an_invalidation(
def scan(**_kwargs):
scans.append(1)
if len(scans) == 1:
- epoch[0] += 1 # a deletion completes while this walk runs
+ epoch[0] += 1
return [{"repo_id": "Org/Deleted"}]
return [{"repo_id": "Org/Kept"}]
@@ -913,7 +914,7 @@ def test_cached_inventory_scan_stops_retrying_under_constant_invalidation(monkey
def scan(**_kwargs):
scans.append(1)
- epoch[0] += 1 # every walk is superseded before it returns
+ epoch[0] += 1
return [{"repo_id": "Org/Model"}]
monkeypatch.setattr(cache_inventory, "all_hf_cache_scans", lambda: [])
@@ -962,7 +963,7 @@ def test_local_inventory_scan_stops_retrying_under_constant_invalidation(monkeyp
async def fake_scan(models_dir, custom_folders, sources):
scans.append(1)
- epoch[0] += 1 # every walk is superseded before it returns
+ epoch[0] += 1
return SimpleNamespace(models = [], model_copy = lambda update = None: SimpleNamespace(models = []))
async def no_folders():
@@ -1010,11 +1011,8 @@ def test_local_inventory_filters_and_dedupes_off_event_loop(monkeypatch):
@pytest.mark.parametrize("change_kind", ["folders", "epoch"])
def test_local_inventory_requests_share_scan(monkeypatch, change_kind):
- # What the flight key needs from the helper is that it is total and stable on
- # hostile input, not that it echoes it back. POSIX realpath() rejects an
- # embedded NUL with ValueError so the raw string is normcased through; Windows
- # non-strict realpath falls back to abspath and joins the cwd instead. Assert
- # the property, not one platform's spelling of it.
+ # Assert the property, not one platform's spelling: POSIX realpath() rejects an embedded NUL with
+ # ValueError while Windows non-strict realpath falls back to abspath and joins the cwd.
for hostile in ("\0", "\ud800"):
identity = local_inventory._inventory_path_identity(hostile)
assert identity == local_inventory._inventory_path_identity(hostile)
@@ -1390,9 +1388,9 @@ def test_legacy_unscoped_download_state_falls_back_only_for_selected_cache(monke
@pytest.mark.parametrize(
"variant",
[
- "unknown/variant with spaces", # unsafe characters force the hashed filename
- "double--hyphen", # aliases the --variant-- delimiter, also hashed
- "Q4_K_M", # spelled literally, the control
+ "unknown/variant with spaces",
+ "double--hyphen",
+ "Q4_K_M",
],
)
def test_index_finds_an_unreadable_marker_stored_under_a_hashed_filename(
@@ -1447,11 +1445,9 @@ def test_cached_gguf_scan_degrades_when_the_shared_index_cannot_be_built(monkeyp
"utils.hf_cache_settings.get_hf_cache_paths",
lambda: SimpleNamespace(hub_cache = hub_cache),
)
- # An undecodable byte in a cache directory name reaches us as a lone surrogate,
- # and the repo key cannot be hashed from it. Spelled directly rather than via
- # os.fsdecode(b"...\xff...") because Windows decodes filenames as UTF-8 with
- # surrogatepass, where a bare 0xff raises instead of producing a surrogate.
- # Nothing here touches the filesystem, so the string alone is the whole case.
+ # An undecodable byte in a cache directory name reaches us as a lone surrogate, and the repo key
+ # cannot be hashed from it. Spelled directly rather than via os.fsdecode(b"...\xff...") because
+ # Windows decodes filenames as UTF-8 with surrogatepass, where a bare 0xff raises.
bad_repo = "Org/Re\udcffpo"
repos = [
SimpleNamespace(
@@ -1509,9 +1505,8 @@ def test_state_scan_survives_a_state_filename_with_an_undecodable_byte(monkeypat
with open(corrupt, "wb") as handle:
handle.write(json.dumps({"version": 2, "repo_type": "model"}).encode("utf-8"))
except (OSError, UnicodeError) as e:
- # Only POSIX filesystems that accept arbitrary bytes can hold this name, so
- # only there is the bug reachable. macOS rejects it with EILSEQ and Windows
- # has no byte-level filename API at all.
+ # Only POSIX filesystems that accept arbitrary bytes can hold this name: macOS rejects it with
+ # EILSEQ and Windows has no byte-level filename API at all.
pytest.skip(f"filesystem will not hold an undecodable filename: {e}")
index = download_manifest.build_variant_state_index(
@@ -1938,8 +1933,8 @@ def _diffusion_scan(
monkeypatch.setattr(
cache_inventory, "all_hf_cache_scans", lambda: [SimpleNamespace(repos = [repo])]
)
- # The real signature takes snapshot_dir; a double that omits it raises TypeError, which the
- # per-repo except swallows into an empty row list, silently skipping every assertion below.
+ # The real signature takes snapshot_dir; a double that omits it raises TypeError, which the per-repo
+ # except swallows into an empty row list, silently skipping every assertion below.
monkeypatch.setattr(
cache_inventory.hf_cache_scan,
"is_snapshot_partial",
@@ -1994,10 +1989,9 @@ def test_cached_models_scan_keeps_a_complete_pipeline_loadable(monkeypatch, tmp_
_file("model_index.json", 900),
_file("vae/diffusion_pytorch_model.safetensors", 300_000_000),
_file("text_encoder/model.safetensors", 900_000_000),
- # Unsharded, because this fixture stands for a COMPLETE pipeline. It used to name a
- # lone "-00001-of-00002" shard with no index, which is not loadable at all: with no
- # index diffusers reads the component as unsharded and asks for the plain name, never
- # the numbered file. The scan now calls that partial, as it always should have.
+ # Unsharded, because this fixture stands for a COMPLETE pipeline: it used to name a lone
+ # "-00001-of-00002" shard with no index, and with no index diffusers reads the component as
+ # unsharded and asks for the plain name, so a lone shard is not loadable.
_file("transformer/diffusion_pytorch_model.safetensors", 4_000_000_000),
],
task = "text-to-image",
@@ -2065,8 +2059,8 @@ def test_a_companion_mirror_carries_the_flag_on_the_hub_row(monkeypatch, tmp_pat
)
assert row["companion"] is True
- # Startup auto-load filters on capabilities.can_chat (isChattableCachedRepo), never on the
- # flag, so a row carrying only the flag was still auto-loadable as a chat model.
+ # Startup auto-load filters on capabilities.can_chat, never on the flag, so a row carrying only the
+ # flag was still auto-loadable as a chat model.
assert row["capabilities"]["can_chat"] is False
@@ -2268,8 +2262,7 @@ def test_cached_scans_hide_embedders_configured_by_snapshot_path(monkeypatch, tm
def test_cached_models_scan_keeps_unrelated_repo_with_custom_generic_embedder(
monkeypatch, tmp_path
):
- # A custom embedder with a generic basename ("org/model") must be hidden by EXACT repo-id
- # match only: substring basename matching used to drop real chat models from the inventory.
+ # EXACT repo-id match only: substring basename matching used to drop real chat models from the inventory.
from core.rag import config as rag_config
monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: "org/model")
@@ -3006,11 +2999,8 @@ def test_gguf_download_progress_fallback_logs_warning(monkeypatch):
)
)
- # cache_path is ABSENT, not null. Null means "no cache dir for this repo exists", which
- # hydration acts on by retiring the persisted job; a scan that threw is not evidence of
- # that, and dropping a job whose partial cache is still on disk costs the user the resume.
- # cache_measured carries the same distinction through DownloadProgressResponse, which
- # defaults cache_path to None and would otherwise turn the omission back into an "absent".
+ # cache_path is ABSENT, not null: null means no cache dir exists, which hydration acts on by
+ # retiring the persisted job, and cache_measured carries the same distinction.
assert result == {
"downloaded_bytes": 0,
"completed_bytes": 0,
@@ -3289,8 +3279,8 @@ def test_gguf_progress_shows_main_when_companion_left_the_count(monkeypatch, tmp
def test_gguf_progress_complete_on_disk_ignores_full_baseline(monkeypatch, tmp_path):
- # A variant already complete on disk carries a baseline equal to its full size; subtracting it
- # would report 0/0 for a finished variant, which the frontend evicts as gone.
+ # A variant already complete on disk carries a baseline equal to its full size; subtracting it would
+ # report 0/0, which the frontend evicts as gone.
entry = tmp_path / "models--Org--Model-GGUF"
snap = entry / "snapshots" / "rev0"
blobs = entry / "blobs"
@@ -3386,13 +3376,13 @@ def test_gguf_progress_complete_on_disk_ignores_full_baseline(monkeypatch, tmp_p
def test_gguf_progress_scoped_hashes_exclude_sibling_quant(monkeypatch, tmp_path):
- # The "instant ~900 MB" bug: with this variant's hashes resolved, progress counts ONLY its
- # in-progress blob, never a sibling quant's finalized bytes in the shared blobs/ dir.
+ # The "instant ~900 MB" bug: with this variant's hashes resolved, progress counts ONLY its in-
+ # progress blob, never a sibling quant's finalized bytes in the shared blobs/ dir.
entry = tmp_path / "models--Org--Model-GGUF"
blobs = entry / "blobs"
blobs.mkdir(parents = True)
- (blobs / "siblinghash").write_bytes(b"z" * 900) # other quant, complete
- (blobs / "mainhash.incomplete").write_bytes(b"x" * 5) # this variant, started
+ (blobs / "siblinghash").write_bytes(b"z" * 900)
+ (blobs / "mainhash.incomplete").write_bytes(b"x" * 5)
requirement = gguf_variants._GgufVariantRequirement(
main_filenames = frozenset({"model-Q4_K_M.gguf"}),
@@ -3451,10 +3441,8 @@ def test_gguf_progress_scoped_hashes_exclude_sibling_quant(monkeypatch, tmp_path
def test_gguf_progress_unknown_hashes_does_not_count_foreign_blobs(monkeypatch, tmp_path):
- # With a variant's hashes unresolved (metadata flaked, no manifest), the
- # shared blobs/ dir's FINALIZED blobs must NOT be counted wholesale: a cached
- # sibling quant (``siblinghash``) alongside is the "instant ~900 MB" bug.
- # With no .incomplete present, downloaded must be 0.
+ # With hashes unresolved, the shared blobs/ dir's FINALIZED blobs must not be counted wholesale: a
+ # cached sibling quant alongside is the "instant ~900 MB" bug.
entry = tmp_path / "models--Org--Model-GGUF"
snap = entry / "snapshots" / "rev0"
blobs = entry / "blobs"
@@ -3503,16 +3491,13 @@ def test_gguf_progress_unknown_hashes_does_not_count_foreign_blobs(monkeypatch,
def test_gguf_progress_unknown_hashes_drops_unscoped_incomplete_blob(monkeypatch, tmp_path):
- # With hashes unresolved, an .incomplete in the shared blobs/ dir can't be
- # attributed to this variant (it may be a concurrent sibling's active write),
- # so it is dropped, mirroring the finalized-blob guard. In production the
- # worker writes the manifest before any .incomplete exists, so hashes resolve
- # via the manifest backstop and this window never suppresses real progress.
+ # With hashes unresolved an .incomplete cannot be attributed to this variant, so it is dropped; in
+ # production the worker writes the manifest before any .incomplete exists.
entry = tmp_path / "models--Org--Model-GGUF"
blobs = entry / "blobs"
blobs.mkdir(parents = True)
- (blobs / "activehash.incomplete").write_bytes(b"x" * 50) # unattributable
- (blobs / "siblinghash").write_bytes(b"z" * 900) # finalized sibling
+ (blobs / "activehash.incomplete").write_bytes(b"x" * 50)
+ (blobs / "siblinghash").write_bytes(b"z" * 900)
async def _run_inline(fn, *args, **kwargs):
return fn(*args, **kwargs)
@@ -3547,22 +3532,20 @@ def test_gguf_progress_unknown_hashes_drops_unscoped_incomplete_blob(monkeypatch
)
)
- assert result["downloaded_bytes"] == 0 # unscoped .incomplete not leaked
- assert result["completed_bytes"] == 0 # finalized sibling still ignored
+ assert result["downloaded_bytes"] == 0
+ assert result["completed_bytes"] == 0
def test_gguf_progress_unknown_hashes_no_backward_dip_when_variant_finalizes(monkeypatch, tmp_path):
- # Regression for the two-variant dip: with hashes unresolved, the first quant
- # finalizes while the sibling still writes its .incomplete. The sibling's
- # bytes used to leak into this numerator, dipping the bar ~99% -> ~78% for
- # one poll. The unscoped .incomplete must be dropped so the reading stays 0.
+ # Regression for the two-variant dip: the sibling's .incomplete bytes used to leak into this
+ # numerator, dipping the bar to ~78% for one poll.
entry = tmp_path / "models--unsloth--SmolLM2-360M-Instruct-GGUF"
blobs = entry / "blobs"
snap = entry / "snapshots" / "rev0"
blobs.mkdir(parents = True)
snap.mkdir(parents = True)
- own_total = 218_673_760 # Q2_K finished blob size (denominator)
- sibling_total = 234_686_560 # Q3_K_M total
+ own_total = 218_673_760
+ sibling_total = 234_686_560
def _sparse_file(path: Path, size: int) -> None:
with path.open("wb") as handle:
@@ -3607,8 +3590,8 @@ def test_gguf_progress_unknown_hashes_no_backward_dip_when_variant_finalizes(mon
)
)
- assert result["downloaded_bytes"] == 0 # sibling .incomplete did not leak
- assert result["progress"] == 0 # no ~0.78 backward dip
+ assert result["downloaded_bytes"] == 0
+ assert result["progress"] == 0
def _unresolvable_variant_metadata(
@@ -3672,7 +3655,7 @@ def test_gguf_progress_unknown_hashes_reports_the_variant_files_on_disk(monkeypa
blobs.mkdir(parents = True)
(snap / "model-Q4_K_M.gguf").write_bytes(b"x" * 100)
(snap / "mmproj-F16.gguf").write_bytes(b"y" * 30)
- (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900) # sibling quant
+ (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900)
(blobs / "mainhash").write_bytes(b"x" * 100)
(blobs / "siblinghash").write_bytes(b"z" * 900)
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
@@ -3686,9 +3669,9 @@ def test_gguf_progress_unknown_hashes_reports_the_variant_files_on_disk(monkeypa
)
)
- assert result["completed_bytes"] == 130 # main quant + shared companion
+ assert result["completed_bytes"] == 130
assert result["downloaded_bytes"] == 130
- assert result["progress"] > 0 # not the "0 B of 130" card
+ assert result["progress"] > 0
def test_gguf_progress_unknown_hashes_keeps_a_total_under_a_full_baseline(monkeypatch, tmp_path):
@@ -3761,7 +3744,7 @@ def test_gguf_progress_unknown_hashes_prefers_the_manifest_file_set(monkeypatch,
)
assert result["completed_bytes"] == 100
- assert result["complete_on_disk"] is True # manifest verifies against disk
+ assert result["complete_on_disk"] is True
assert result["progress"] == 1.0
@@ -3942,7 +3925,7 @@ def test_gguf_progress_without_a_manifest_needs_every_expected_blob(monkeypatch,
snap.mkdir(parents = True)
blobs.mkdir(parents = True)
(snap / "model-Q4_K_M-00001-of-00002.gguf").write_bytes(b"x" * 400)
- (blobs / "shard1").write_bytes(b"x" * 400) # shard2 never arrived
+ (blobs / "shard1").write_bytes(b"x" * 400)
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
async def _run_inline(fn, *args, **kwargs):
@@ -3989,7 +3972,7 @@ def test_gguf_progress_without_a_manifest_needs_every_expected_blob(monkeypatch,
)
assert result["complete_on_disk"] is False
- assert result["progress"] == 0.99 # capped, not settled
+ assert result["progress"] == 0.99
def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypatch, tmp_path):
@@ -4017,8 +4000,8 @@ def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypat
blobs = entry / "blobs"
snap.mkdir(parents = True)
blobs.mkdir(parents = True)
- # Copy layout: the snapshot holds real files, and blobs/ was never populated
- # with anything this reading could have matched by hash.
+ # Copy layout: the snapshot holds real files, and blobs/ was never populated with anything this reading
+ # could have matched by hash.
(snap / "model-Q4_K_M.gguf").write_bytes(b"x" * 33_000)
assert not (snap / "model-Q4_K_M.gguf").is_symlink()
@@ -4027,9 +4010,8 @@ def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypat
"utils.hf_cache_settings.get_hf_cache_paths",
lambda: SimpleNamespace(hub_cache = str(hub_cache)),
)
- # A manifest as an older build filed it: hashed from the unresolved spelling,
- # and with no sha256 because HF metadata was already unreachable when the
- # worker recorded it from the finished snapshot.
+ # A manifest as an older build filed it: hashed from the unresolved spelling, and with no sha256
+ # because HF metadata was already unreachable when the worker recorded it.
legacy = state_dir.manifest_path(
"model",
"Org/Model-GGUF",
@@ -4064,10 +4046,10 @@ def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypat
)
)
- assert result["downloaded_bytes"] == 33_000 # not "0 B"
- assert result["completed_bytes"] == 33_000 # what settles the job terminal
+ assert result["downloaded_bytes"] == 33_000
+ assert result["completed_bytes"] == 33_000
assert result["expected_bytes"] == 33_000
- assert result["complete_on_disk"] is True # so Retry/Resume goes away
+ assert result["complete_on_disk"] is True
assert result["progress"] == 1.0
@@ -4090,8 +4072,8 @@ def test_gguf_progress_without_a_manifest_needs_the_snapshot_materialized(monkey
blobs = entry / "blobs"
snap.mkdir(parents = True)
blobs.mkdir(parents = True)
- (blobs / "mainhash").write_bytes(b"x" * 100) # never linked into rev0
- (snap / "mmproj-F32.gguf").write_bytes(b"y" * 5_000) # not in this plan
+ (blobs / "mainhash").write_bytes(b"x" * 100)
+ (snap / "mmproj-F32.gguf").write_bytes(b"y" * 5_000)
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
async def _run_inline(fn, *args, **kwargs):
@@ -4147,7 +4129,7 @@ def test_running_job_never_reads_progress_from_a_remembered_cache(monkeypatch, t
complete = previous / "models--Org--Model" / "blobs"
complete.mkdir(parents = True)
(complete / "mainhash").write_bytes(b"x" * 100)
- active = tmp_path / "active" # not created yet
+ active = tmp_path / "active"
monkeypatch.setattr(
hf_cache_state,
"hf_cache_roots",
@@ -4476,9 +4458,8 @@ def test_completed_gguf_split_variant_requires_all_shards(tmp_path):
def test_completed_gguf_variants_ignores_big_endian_by_the_loader_label(tmp_path):
- # The scan reads Q4_K_M here, the loader reads F16 and refuses the file as big-endian. By
- # the scan's label it would vouch for Q4_K_M, and _complete_with_servable skips complete
- # quants, so the torn split below would be reported downloaded.
+ # The scan reads Q4_K_M while the loader reads F16 and refuses the file as big-endian, so by the
+ # scan's label it would vouch for Q4_K_M and _complete_with_servable would skip the torn split.
snapshot = tmp_path / "snapshot"
snapshot.mkdir()
(snapshot / "0-F16-be-checkpoint-Q4_K_M.gguf").write_bytes(b"GGUF")
@@ -4658,9 +4639,8 @@ def test_download_registry_repo_keys_are_case_insensitive():
repo_id = "Org/Repo",
variant = "Q8_0",
)
- # The same variant under a different-cased repo id resolves to the same
- # job, so the second claim attaches to the running one instead of starting
- # a duplicate.
+ # The same variant under a different-cased repo id resolves to the same job, so the second claim
+ # attaches to the running one instead of starting a duplicate.
duplicate_claimed, duplicate_state = registry.claim(
"org/repo::Q8_0",
download_registry.TRANSPORT_HTTP,
@@ -4709,9 +4689,9 @@ def test_download_registry_allows_disjoint_gguf_variant_downloads():
def test_download_registry_allows_overlapping_same_transport_variant_downloads():
- # Two variants sharing one mmproj blob still download together on one
- # transport: huggingface_hub's per-blob lock serializes the shared write and
- # prepare_cache_for_transport never purges a blob a peer is writing.
+ # Two variants sharing one mmproj blob still download together on one transport:
+ # huggingface_hub's per-blob lock serializes the shared write and prepare_cache_for_transport
+ # never purges a blob a peer is writing.
registry = download_registry.DownloadRegistry()
claimed, state = registry.claim(
@@ -4740,9 +4720,8 @@ def test_download_registry_allows_overlapping_same_transport_variant_downloads()
def test_download_registry_variant_delete_does_not_block_sibling_download():
- # Deleting one quant's partial must be allowed while a different quant of the
- # same repo is downloading, and must protect every blob the live sibling is
- # writing (including a shared mmproj companion).
+ # Deleting one quant's partial must be allowed while a different quant of the same repo is
+ # downloading, and must protect every blob the live sibling is writing.
registry = download_registry.DownloadRegistry()
registry.claim(
"Org/Repo::Q8_0",
@@ -4764,8 +4743,8 @@ def test_download_registry_variant_delete_does_not_block_sibling_download():
assert registry.has_active_peer_variant("Org/Repo", "Q4_K_M") is True
assert registry.has_active_peer_variant("Org/Repo", "Q8_0") is False
- # While Q4_K_M is being deleted, re-downloading it is blocked but an
- # untouched third variant may still start.
+ # While Q4_K_M is being deleted, re-downloading it is blocked but an untouched third variant may still
+ # start.
blocked, blocked_state = registry.claim(
"Org/Repo::Q4_K_M",
download_registry.TRANSPORT_HTTP,
@@ -4790,8 +4769,9 @@ def test_download_registry_variant_delete_does_not_block_sibling_download():
def test_partial_gguf_reconstruction_dedupes_variant_casing(monkeypatch):
- # The manifest keeps original casing while the marker is lowercased; offline
- # reconstruction must collapse them to ONE entry (manifest's casing), not two.
+ # The manifest keeps original casing while the marker is lowercased; offline reconstruction must
+ # collapse them to ONE entry, in the manifest's casing.
+ # Per-variant blob hashes (distinct main shard, shared mmproj companion).
monkeypatch.setattr(
download_manifest,
"iter_variant_manifests",
@@ -4812,16 +4792,13 @@ def test_partial_gguf_reconstruction_dedupes_variant_casing(monkeypatch):
def test_partial_gguf_reconstruction_drops_a_variant_read_off_the_filename(monkeypatch):
- # An unreadable payload leaves the reader the filename, whose digest fragment
- # names nothing and would re-key to a further hash on resume. A variant genuinely
- # called sha256-<32 hex> reads the same but is stored under the hash of itself, so
- # the file it came from tells them apart where the spelling cannot.
+ # An unreadable payload leaves only the filename, whose digest fragment names nothing; a variant
+ # genuinely called sha256-<32 hex> reads the same but is stored under the hash of itself.
digest = "sha256-" + "0" * 32
entries = [
- # variant, the file it was recovered from
- (f"@{digest}", Path(f"repo--variant--@{digest}.json")), # scope, payload lost
- (digest, Path(f"repo--variant--{digest}.json")), # ditto, older tag
- (digest, Path("repo--variant--@sha256-" + "c" * 32 + ".json")), # real, payload intact
+ (f"@{digest}", Path(f"repo--variant--@{digest}.json")),
+ (digest, Path(f"repo--variant--{digest}.json")),
+ (digest, Path("repo--variant--@sha256-" + "c" * 32 + ".json")),
("Q4_K_M", Path("repo--variant--q4_k_m.json")),
]
monkeypatch.setattr(
@@ -4838,8 +4815,8 @@ def test_partial_gguf_reconstruction_drops_a_variant_read_off_the_filename(monke
def test_download_registry_serializes_cross_transport_variant_downloads():
- # An HTTP append-resume and an XET rewrite of the same shared blob would
- # corrupt each other, so different-transport variants are serialized.
+ # An HTTP append-resume and an XET rewrite of the same shared blob would corrupt each other, so
+ # different-transport variants are serialized.
registry = download_registry.DownloadRegistry()
claimed, state = registry.claim(
@@ -4868,10 +4845,9 @@ def test_download_registry_serializes_cross_transport_variant_downloads():
def test_download_registry_allows_unknown_hash_gguf_variant_downloads():
- # Resolved blob hashes are NOT required to run two same-transport variants
- # concurrently: on-disk safety comes from each worker purging only its own
- # main-quant blobs plus huggingface_hub's per-etag lock. Requiring them here
- # used to reject the second variant whenever a metadata fetch flaked.
+ # Resolved blob hashes are NOT required to run two same-transport variants concurrently: safety
+ # comes from each worker purging only its own blobs plus huggingface_hub's per-etag lock, and
+ # requiring them rejected the second variant whenever a metadata fetch flaked.
registry = download_registry.DownloadRegistry()
claimed, state = registry.claim(
@@ -4902,9 +4878,8 @@ def test_download_registry_allows_unknown_hash_gguf_variant_downloads():
def test_finalize_worker_exit_never_kills_a_healthy_worker(monkeypatch, tmp_path):
- # finalize_worker_exit relies solely on the worker's exit code and never kills
- # a live process: huggingface_hub already bounds reads with timeouts, so a
- # liveness kill could only false-cancel a healthy download.
+ # finalize_worker_exit relies solely on the worker's exit code and never kills a live process:
+ # huggingface_hub already bounds reads with timeouts, so a liveness kill could only false-cancel.
import inspect
import io
import logging
@@ -5022,9 +4997,8 @@ def test_prepare_cache_for_transport_purges_cross_transport_companion(monkeypatc
blobs = _vision_cache_root(monkeypatch, tmp_path)
companion = frozenset({"shared-mmproj"})
- # An interrupted XET download stamps the companion marker "xet" and leaves a
- # sparse partial. A later HTTP download of a different variant must purge it,
- # else the HTTP resumer appends to the sparse bytes and corrupts the blob.
+ # An interrupted XET download leaves a sparse partial, so a later HTTP download of a different
+ # variant must purge it, else the HTTP resumer appends to the sparse bytes and corrupts the blob.
download_registry.prepare_cache_for_transport(
"model",
"Org/Vision",
@@ -5050,8 +5024,7 @@ def test_prepare_cache_for_transport_purges_cross_transport_companion(monkeypatc
def test_prepare_cache_for_transport_preserves_same_transport_companion(monkeypatch, tmp_path):
"""Only a hub that can still append to the partial earns the same-transport reprieve."""
- # The purge asks partial_is_resumable, so patching the hub-version helper it wraps would
- # be a no-op here.
+ # The purge asks partial_is_resumable, so patching the hub-version helper it wraps would be a no-op here.
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: True)
blobs = _vision_cache_root(monkeypatch, tmp_path)
companion = frozenset({"shared-mmproj"})
@@ -5569,10 +5542,8 @@ def test_model_download_watcher_invalidates_hf_cache_scan(monkeypatch):
def test_two_concurrent_same_repo_variants_both_complete(monkeypatch, tmp_path):
- # End-to-end proof that two GGUF variants of ONE repo download concurrently
- # without cancelling each other, with real registry/finalize/subprocess/watch
- # threads exercising the claim gate, register, finalize funnel, and
- # classify_exit under true concurrency.
+ # End-to-end proof that two GGUF variants of one repo download concurrently without cancelling each
+ # other, with real registry, finalize, subprocess and watch threads under true concurrency.
import subprocess
import time
@@ -5592,7 +5563,6 @@ def test_two_concurrent_same_repo_variants_both_complete(monkeypatch, tmp_path):
"download_transport_unavailable_reason",
lambda _transport: None,
)
- # Per-variant blob hashes (distinct main shard, shared mmproj companion).
monkeypatch.setattr(
downloads.gguf_variants,
"gguf_variant_blob_hashes",
@@ -5718,8 +5688,8 @@ def _patch_variant_delete_side_effects(monkeypatch, hub_cache = None):
"purge_state",
lambda *_args, **_kwargs: False,
)
- # The repo under test lives in this cache; make it the active one so the
- # delete scopes to it (default target root is the active hub cache).
+ # The repo under test lives in this cache; make it the active one so the delete scopes to it (default target
+ # root is the active hub cache).
if hub_cache is not None:
monkeypatch.setattr(
"utils.hf_cache_settings.get_hf_cache_paths",
@@ -6000,8 +5970,8 @@ def test_a_partial_scan_cannot_report_the_target_as_gone(monkeypatch, tmp_path):
def test_a_complete_scan_still_reports_the_target_as_gone(monkeypatch, tmp_path):
- # The same reading with no scan error keeps the positive-evidence verdict, or the fix
- # above would simply disable the phantom-adoption guard it is protecting.
+ # The same reading with no scan error keeps the positive-evidence verdict, or the fix above would
+ # simply disable the phantom-adoption guard it is protecting.
entry = tmp_path / "models--Org--Model-GGUF"
(entry / "blobs").mkdir(parents = True)
(entry / "snapshots" / "rev0").mkdir(parents = True)
@@ -6058,7 +6028,7 @@ def test_delete_variant_keeps_blob_shared_with_other_snapshot(monkeypatch, tmp_p
assert not (repo_dir / "snapshots" / "rev1" / "model-Q4_K_M.gguf").exists()
assert (repo_dir / "blobs" / "sharedblob").exists()
extra = repo_dir / "snapshots" / "rev1" / "extra-copy.gguf"
- assert extra.is_symlink() and extra.exists() # not dangling
+ assert extra.is_symlink() and extra.exists()
def test_delete_variant_unlinks_unshared_blob(monkeypatch, tmp_path):
@@ -6785,8 +6755,8 @@ def test_gguf_progress_unknown_hashes_calls_a_sibling_only_dir_absent(monkeypatc
snap = entry / "snapshots" / "rev0"
snap.mkdir(parents = True)
(entry / "blobs").mkdir(parents = True)
- (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900) # the sibling that keeps the dir alive
- (snap / "mmproj-F16.gguf").write_bytes(b"y" * 30) # shared by every quant in the repo
+ (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900)
+ (snap / "mmproj-F16.gguf").write_bytes(b"y" * 30)
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
_unresolvable_variant_metadata(monkeypatch, entry)
@@ -6816,8 +6786,7 @@ def test_gguf_progress_target_presence_is_aggregated_across_caches(monkeypatch,
holder = tmp_path / "b" / "models--Org--Model-GGUF"
(holder / "snapshots" / "rev0").mkdir(parents = True)
(holder / "blobs").mkdir(parents = True)
- # The variant's own file, named but empty: zero bytes keeps the tie with the sibling-only
- # reading, and the name is what proves the target is in this cache.
+ # Zero bytes keeps the tie with the sibling-only reading, and the name is what proves the target is in this cache.
(holder / "snapshots" / "rev0" / "model-Q4_K_M.gguf").write_bytes(b"")
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
assert download_manifest.write_manifest(
@@ -6992,7 +6961,7 @@ def test_one_unknown_cache_keeps_absence_unknown(monkeypatch, tmp_path):
(sibling / "blobs").mkdir(parents = True)
(sibling / "snapshots" / "rev0" / "model-Q2_K.gguf").write_bytes(b"z" * 900)
unknown = tmp_path / "b" / "models--Org--Model-GGUF"
- (unknown / "blobs").mkdir(parents = True) # no snapshot dir at all: nothing to identify
+ (unknown / "blobs").mkdir(parents = True)
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
_unresolvable_variant_metadata(monkeypatch, sibling, state = "idle")
monkeypatch.setattr(
@@ -7050,7 +7019,7 @@ def test_a_subtree_that_cannot_be_scanned_keeps_presence_unknown(monkeypatch, tm
snap = entry / "snapshots" / "rev0"
snap.mkdir(parents = True)
(entry / "blobs").mkdir(parents = True)
- (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900) # a sibling keeps the dir alive
+ (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900)
denied = snap / "split"
denied.mkdir()
(denied / "model-Q4_K_M.gguf").write_bytes(b"x" * 100)
@@ -7281,7 +7250,7 @@ def test_a_deleted_snapshot_link_is_absent_even_with_its_blob_left_behind(monkey
blobs.mkdir(parents = True)
# The finalized blob survives; the snapshot entry that named it does not.
(blobs / "mainhash").write_bytes(b"x" * 100)
- (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900) # a sibling keeps the repo dir alive
+ (snap / "model-Q2_K.gguf").write_bytes(b"z" * 900)
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
async def _run_inline(fn, *args, **kwargs):
@@ -7326,7 +7295,7 @@ def test_a_stale_revisions_filenames_do_not_settle_the_resolved_one(monkeypatch,
stale.mkdir(parents = True)
blobs.mkdir(parents = True)
(blobs / "oldhash").write_bytes(b"y" * 100)
- (blobs / "newhash").write_bytes(b"x" * 100) # finalized, never linked
+ (blobs / "newhash").write_bytes(b"x" * 100)
os.symlink(blobs / "oldhash", stale / "model-Q4_K_M.gguf")
monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
assert download_manifest.write_manifest(
@@ -7451,7 +7420,7 @@ def test_local_inventory_classifies_a_superseded_result_off_the_event_loop(monke
response.model_copy = lambda update: SimpleNamespace(models = update["models"])
async def always_superseded(*_args):
- epoch[0] += 1 # every walk is invalidated before it returns
+ epoch[0] += 1
return response
async def no_folders():
@@ -7668,8 +7637,8 @@ def test_every_row_key_the_scanner_emits_survives_the_response_schema():
}
return set()
- # Each scanner against ITS OWN schema. A union would let a key emitted on a model row pass
- # because the GGUF schema happens to declare it, which is not what response_model does.
+ # Each scanner against ITS OWN schema: a union would let a key emitted on a model row pass because
+ # the GGUF schema happens to declare it, which is not what response_model does.
emitted = literal_keys("_cache_inventory_fields") | literal_keys("_scan_cached_models")
watched = ("diffusers", "companion", "single_file", "partial", "load_id", "task")
for flag in watched:
@@ -7679,8 +7648,8 @@ def test_every_row_key_the_scanner_emits_survives_the_response_schema():
f"FastAPI's response_model strips it before the frontend sees the row."
)
- # And prove it end to end rather than by field name alone: a declared-but-mistyped field is
- # dropped or 500s at serialization time, which a model_fields check cannot see.
+ # Prove it end to end rather than by field name alone: a declared-but-mistyped field is dropped or
+ # 500s at serialization time, which a model_fields check cannot see.
row = {
"repo_id": "Org/Pipeline",
"size_on_disk": 4096,
diff --git a/studio/backend/hub/tests/test_process_unique_incomplete_repro.py b/studio/backend/hub/tests/test_process_unique_incomplete_repro.py
index 6a7aa36f560..f2138a4d258 100644
--- a/studio/backend/hub/tests/test_process_unique_incomplete_repro.py
+++ b/studio/backend/hub/tests/test_process_unique_incomplete_repro.py
@@ -48,8 +48,8 @@ def test_registry_groups_duplicate_process_unique_writers_by_blob(monkeypatch, t
)
assert download_registry.incomplete_blob_hashes("model", "Org/Model") == {_BLOB_HASH}
- # Nonce partials are refetched rather than resumed, so none of those bytes are bytes the
- # next attempt gets to skip. Their grouping is still asserted, one blob not two.
+ # Nonce partials are refetched rather than resumed, so none of those bytes are bytes the next
+ # attempt skips; their grouping is still asserted, one blob not two.
assert (
download_registry.existing_blob_bytes(
"model",
diff --git a/studio/backend/hub/tests/test_resumable_partials.py b/studio/backend/hub/tests/test_resumable_partials.py
index 3fb0a3ee59c..e6ae51b5f92 100644
--- a/studio/backend/hub/tests/test_resumable_partials.py
+++ b/studio/backend/hub/tests/test_resumable_partials.py
@@ -71,8 +71,6 @@ def _patched_writer(module):
# ---------------------------------------------------------------------------------------------
-# When it engages
-# ---------------------------------------------------------------------------------------------
@pytest.mark.parametrize(
@@ -207,6 +205,7 @@ def test_each_cache_root_is_judged_on_its_own_filesystem(tmp_path, monkeypatch):
local_root.mkdir()
network_root.mkdir()
# Past the version gate, so the filesystem is what is left to decide.
+ # And out through the public entry point, past the version gate.
monkeypatch.setattr("huggingface_hub.__version__", "1.28.0", raising = False)
monkeypatch.setattr(rp, "_hub_is_patchable", lambda: True)
monkeypatch.setattr(
@@ -354,7 +353,6 @@ def test_an_unrunnable_probe_travels_up_rather_than_answering(tmp_path, monkeypa
with pytest.raises(rp._ProbeUnavailable):
rp._exclusion_is_provable()
- # And out through the public entry point, past the version gate.
monkeypatch.setattr("huggingface_hub.__version__", "1.28.0", raising = False)
monkeypatch.setattr(rp, "_hub_is_patchable", lambda: True)
with pytest.raises(rp._ProbeUnavailable):
@@ -390,15 +388,13 @@ def test_changing_the_cache_home_invalidates_the_verdict(monkeypatch):
assert hf_cache_state.hf_partials_are_resumable() is True
monkeypatch.setattr(rp, "can_restore_partials", lambda _c = None: False)
- # No cache at this layer any more: the verdict follows the filesystem, and a result kept
- # against the path alone outlives a remount at the same name.
+ # No cache at this layer any more: the verdict follows the filesystem, and a result kept against the
+ # path alone outlives a remount at the same name.
assert hf_cache_state.hf_partials_are_resumable() is False
hf_cache_state.invalidate_partial_resumability()
# ---------------------------------------------------------------------------------------------
-# What it does once it has
-# ---------------------------------------------------------------------------------------------
def test_it_appends_to_the_stable_name_and_says_how_far_it_got(monkeypatch, tmp_path):
@@ -466,10 +462,8 @@ def test_a_partial_left_by_another_user_is_not_built_on(monkeypatch, tmp_path):
partial = tmp_path / "abc.incomplete"
partial.write_bytes(b"poison" * 100)
- # Only the planted file reads as somebody else's: owning one for real needs root, and the
- # fresh partial that replaces it has to still be ours or the test proves nothing about the
- # retry. Keyed on which open it is rather than on the inode, since a filesystem is free to
- # hand the replacement the inode the unlinked file just released.
+ # Only the planted file reads as somebody else's, and the fresh partial replacing it has to still
+ # be ours; keyed on which open it is, since a filesystem may reuse the released inode.
real_fstat = os.fstat
opens: list[int] = []
@@ -819,8 +813,6 @@ def test_patching_twice_keeps_one_layer(monkeypatch):
# ---------------------------------------------------------------------------------------------
-# The capability the UI reads
-# ---------------------------------------------------------------------------------------------
def test_the_ui_is_told_partials_are_resumable_again(monkeypatch):
diff --git a/studio/backend/hub/tests/test_unresumable_partial_purge.py b/studio/backend/hub/tests/test_unresumable_partial_purge.py
index b5c1e228347..6346fab0e00 100644
--- a/studio/backend/hub/tests/test_unresumable_partial_purge.py
+++ b/studio/backend/hub/tests/test_unresumable_partial_purge.py
@@ -120,7 +120,7 @@ def test_a_nonce_partial_is_unresumable_even_under_a_legacy_writer(monkeypatch):
def test_unresumable_partial_is_purged_despite_a_matching_marker(monkeypatch, blobs):
"""The marker vouches for provenance, which is worth nothing with no resumer left."""
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)
- _prepare() # writes the http marker
+ _prepare()
partial = blobs / _NONCE_PARTIAL
partial.write_bytes(b"x" * 25)
_abandon(partial)
@@ -137,6 +137,7 @@ def test_resumable_partial_survives_a_matching_marker(monkeypatch, blobs):
partial.write_bytes(b"x" * 25)
_abandon(partial)
+ # Too fresh at download start, so prepare leaves it alone.
assert _prepare() == 0
assert partial.exists()
@@ -146,7 +147,7 @@ def test_a_partial_still_being_written_is_left_alone(monkeypatch, blobs):
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)
_prepare()
partial = blobs / _NONCE_PARTIAL
- partial.write_bytes(b"x" * 25) # mtime is now, as a live writer's would be
+ partial.write_bytes(b"x" * 25)
assert _prepare() == 0
assert partial.exists()
@@ -217,7 +218,6 @@ def test_a_skipped_partial_is_swept_once_it_ages_out(monkeypatch, blobs):
partial = blobs / _NONCE_PARTIAL
partial.write_bytes(b"x" * 25)
- # Too fresh at download start, so prepare leaves it alone.
assert _prepare() == 0
assert partial.exists()
@@ -313,7 +313,7 @@ def test_a_finalized_blob_still_counts_against_the_disk_check(monkeypatch, blobs
def test_startup_sweep_does_not_depend_on_a_breadcrumb(monkeypatch, tmp_path, blobs):
"""finalize_worker_exit drops the breadcrumb, so the boot sweep cannot be driven off one."""
workers = tmp_path / "workers"
- workers.mkdir() # deliberately empty, as it is once drop_process has run
+ workers.mkdir()
partial = blobs / _NONCE_PARTIAL
partial.write_bytes(b"x" * 25)
_abandon(partial)
@@ -354,7 +354,7 @@ def test_a_reaped_job_does_not_wait_out_the_grace_on_its_own_blobs(monkeypatch,
"""Cancelling writes the partial seconds before the sweep, so waiting strands it."""
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)
partial = blobs / _NONCE_PARTIAL
- partial.write_bytes(b"x" * 25) # freshly written, as a just-cancelled download's would be
+ partial.write_bytes(b"x" * 25)
monkeypatch.setattr(
download_registry,
"iter_destructive_repo_cache_dirs",
@@ -434,8 +434,8 @@ def test_the_sweep_accepts_the_string_root_the_metadata_holds(monkeypatch, tmp_p
partial.write_bytes(b"x" * 25)
_abandon(partial)
- # A Path-only signature raised AttributeError here, and the caller's broad except
- # swallowed it, so the terminal sweep silently did nothing for every real download.
+ # A Path-only signature raised AttributeError here and the caller's broad except swallowed it, so
+ # the terminal sweep silently did nothing for every real download.
swept = download_registry.sweep_abandoned_partials(
"model",
"Org/Model",
@@ -450,7 +450,7 @@ def test_a_job_owning_its_whole_repo_needs_no_hash_list(monkeypatch, blobs):
"""A download with no variant resolves no blob hashes, and claim() gives it the repo."""
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)
partial = blobs / _NONCE_PARTIAL
- partial.write_bytes(b"x" * 25) # fresh, as a just-cancelled snapshot download's would be
+ partial.write_bytes(b"x" * 25)
monkeypatch.setattr(
download_registry,
"iter_destructive_repo_cache_dirs",
@@ -504,7 +504,7 @@ def test_the_boot_sweep_runs_after_the_orphan_is_killed(monkeypatch, tmp_path, b
def _kill(_pid):
order.append("kill")
- locked["held"] = False # the lock dies with the process
+ locked["held"] = False
return True
monkeypatch.setattr(download_registry, "_kill_orphan", _kill)
@@ -525,7 +525,7 @@ def test_a_companion_the_dead_worker_was_writing_is_owned_too(monkeypatch, blobs
"""A shared mmproj lives in progress_blob_hashes, never in the main blob_hashes set."""
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)
companion = blobs / f"{_PEER}.feedface{hf_cache_state.INCOMPLETE_SUFFIX}"
- companion.write_bytes(b"x" * 25) # fresh, as a just-cancelled worker's companion would be
+ companion.write_bytes(b"x" * 25)
monkeypatch.setattr(
download_registry,
"iter_destructive_repo_cache_dirs",
@@ -567,7 +567,7 @@ def test_the_reaper_waits_for_the_worker_to_actually_die(monkeypatch):
download_registry._kill_orphan(4242)
- assert alive["n"] == 0 # returned only once the process was gone, not straight after kill
+ assert alive["n"] == 0
def test_a_worker_that_will_not_die_keeps_its_breadcrumb_and_its_partial(
@@ -597,14 +597,14 @@ def test_a_worker_that_will_not_die_keeps_its_breadcrumb_and_its_partial(
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)
monkeypatch.setattr(download_registry, "_process_alive", lambda _pid: True)
monkeypatch.setattr(download_registry, "_is_our_worker", lambda *_a: True)
- monkeypatch.setattr(download_registry, "_kill_orphan", lambda _pid: False) # would not die
+ monkeypatch.setattr(download_registry, "_kill_orphan", lambda _pid: False)
monkeypatch.setattr(download_registry, "hf_cache_roots", lambda *_a, **_k: [tmp_path / "none"])
download_registry.reap_orphan_workers()
_join_background_sweep()
assert partial.exists()
- assert crumb.exists() # still tracked, so the next boot tries again
+ assert crumb.exists()
def test_a_locked_peer_partial_still_counts_against_the_disk_check(monkeypatch, blobs):
@@ -673,7 +673,7 @@ def test_ownership_is_recovered_from_the_manifest_when_hashes_never_resolved(mon
metadata, frozenset(), frozenset(), "model", "Org/Model"
)
- assert owns_all is False # a variant job never owns its siblings' blobs
+ assert owns_all is False
assert owned == frozenset({_MAIN})
@@ -702,8 +702,8 @@ def test_a_filesystem_without_flock_does_not_escape_the_probe(monkeypatch, tmp_p
# No lock file: nobody has locked this blob, whatever the filesystem supports.
assert hf_cache_state.blob_download_lock_held(entry, _MAIN) is False
- # With one, the answer is "held" rather than an exception -- which is also what a
- # SoftFileLock would say, since its file IS the lock and that file is present.
+ # With one, the answer is "held" rather than an exception, which is also what a SoftFileLock would
+ # say, since its file IS the lock.
lock_path.touch()
assert hf_cache_state.blob_download_lock_held(entry, _MAIN) is True
@@ -759,7 +759,7 @@ def test_unreadable_breadcrumbs_do_not_cancel_the_cache_sweep(monkeypatch, tmp_p
def test_an_owned_partial_that_is_still_growing_is_spared(monkeypatch, blobs):
"""Ownership proves OUR writer died, never that no other process shares the cache."""
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)
- monkeypatch.setattr(download_registry, "blob_download_lock_held", lambda *_a: False) # lies
+ monkeypatch.setattr(download_registry, "blob_download_lock_held", lambda *_a: False)
monkeypatch.setattr(download_registry, "_STILLNESS_PROBE_SECONDS", 0.05)
monkeypatch.setattr(
download_registry,
@@ -774,7 +774,7 @@ def test_an_owned_partial_that_is_still_growing_is_spared(monkeypatch, blobs):
def _write_while_we_watch(_seconds):
real_sleep(_seconds)
with partial.open("ab") as handle:
- handle.write(b"y" * 10) # an external writer, mid-transfer
+ handle.write(b"y" * 10)
monkeypatch.setattr(download_registry.time, "sleep", _write_while_we_watch)
@@ -798,7 +798,7 @@ def test_an_owned_partial_that_never_moves_is_swept_without_the_full_grace(monke
lambda *_a, **_k: [blobs.parent],
)
partial = blobs / _NONCE_PARTIAL
- partial.write_bytes(b"x" * 25) # written seconds ago, far inside the abandonment grace
+ partial.write_bytes(b"x" * 25)
swept = download_registry.sweep_abandoned_partials(
"model",
@@ -828,11 +828,11 @@ def test_a_breadcrumb_whose_worker_already_exited_is_claimed(monkeypatch, tmp_pa
encoding = "utf-8",
)
partial = blobs / _NONCE_PARTIAL
- partial.write_bytes(b"x" * 25) # fresh, so only an ownership claim reaches it
+ partial.write_bytes(b"x" * 25)
monkeypatch.setattr(download_registry.state_dir, "workers_dir", lambda: workers)
monkeypatch.setattr(download_registry, "partial_is_resumable", lambda _name, _root = None: False)
- monkeypatch.setattr(download_registry, "_process_alive", lambda _pid: False) # already dead
+ monkeypatch.setattr(download_registry, "_process_alive", lambda _pid: False)
monkeypatch.setattr(download_registry, "_STILLNESS_PROBE_SECONDS", 0.05)
monkeypatch.setattr(download_registry, "_settle_orphaned_download", lambda *_a, **_k: None)
monkeypatch.setattr(download_registry, "hf_cache_roots", lambda *_a, **_k: [tmp_path / "none"])
diff --git a/studio/backend/hub/utils/companion_assets.py b/studio/backend/hub/utils/companion_assets.py
index a674c7531d0..c96968bb6f3 100644
--- a/studio/backend/hub/utils/companion_assets.py
+++ b/studio/backend/hub/utils/companion_assets.py
@@ -51,8 +51,8 @@ _LINKS_VERSION = 1
_MAX_LINKS = 512
# ... and per checkpoint, for the same reason: one key must not grow the file on its own.
_MAX_BASES_PER_CHECKPOINT = 16
-# Serialises the read-modify-write below. Losing a link to a lost update is the one direction
-# that matters: an unrecorded base can look orphaned and be offered for removal.
+# Serialises the read-modify-write below: losing a link to a lost update is the one direction that
+# matters, since an unrecorded base can look orphaned and be offered for removal.
_WRITE_LOCK = threading.Lock()
@@ -105,10 +105,7 @@ def _write_companion_links(links: dict[str, list[str]]) -> bool:
payload = {"version": _LINKS_VERSION, "links": links}
tmp = path.with_name(f".{path.name}.tmp-{uuid.uuid4().hex[:8]}")
try:
- # NOT sort_keys: json.loads keeps document order, so the file IS the recency record the
- # trim above reads. Sorting it made the next read alphabetical, and the trim then evicted
- # the lexicographically smallest link rather than the oldest -- so a link recorded minutes
- # ago could go, and its base become deletable while the checkpoint was still installed.
+ # NOT sort_keys: json.loads keeps document order, so the file IS the recency record the trim reads.
tmp.write_text(json.dumps(payload, indent = 2), encoding = "utf-8")
os.replace(tmp, path)
return True
@@ -147,18 +144,11 @@ def record_companion_link(checkpoint_repo_id: str, base_repo_id: str) -> bool:
key = _normalise(checkpoint)
existing = links.get(key, [])
known = any(_normalise(b) == _normalise(base) for b in existing)
- # Re-inserted at the end, not updated in place: recording is the only recency signal the
- # trim has, so a checkpoint that just resolved must not keep an old position and be
- # evicted ahead of one nothing has touched since. That applies to a REPEAT resolution too
- # -- a checkpoint reloaded every day but first recorded long ago is the most-used link
- # there is, and returning early on it let the cap throw it away.
+ # Re-inserted at the end, not updated in place: recording is the only recency signal the trim has,
+ # and returning early on a REPEAT resolution let the cap throw away the most-used link.
links.pop(key, None)
- # Recency inside the list too, and capped: an explicit base_repo (or a card tag that
- # changes) makes one checkpoint resolve a new base each load, and appending forever grew
- # the file that _MAX_LINKS is supposed to bound -- it counts checkpoints, not their bases,
- # and every dependency check parses and mirror-expands the whole list. A checkpoint has
- # one or two bases in practice, so the cap is only ever reached by the runaway writer it
- # is here for, and dropping the least recently resolved degrades to derivation.
+ # Recency inside the list too, and capped: an explicit base_repo or a changing card tag makes one
+ # checkpoint resolve a new base each load, and appending forever grew the file _MAX_LINKS bounds.
fresh = [b for b in existing if _normalise(b) != _normalise(base)]
links[key] = [*fresh, base][-_MAX_BASES_PER_CHECKPOINT:]
wrote = _write_companion_links(links)
@@ -337,16 +327,14 @@ def _family_bases(repo_id: str, gguf_filename: Optional[str] = None) -> set[str]
except Exception: # noqa: BLE001
return set()
if fam is None:
- # The loader resolves a base with this same call. No family means no load, so there is
- # no dependent here to protect -- not a gap, the two fail together.
+ # The loader resolves a base with this same call, so no family means no load and there is no
+ # dependent here to protect: the two fail together.
return set()
bases = {resolve_base_repo(fam, None)}
bases |= _curated_variant_bases(fam, repo_id, gguf_filename)
- # The NATIVE engine's companions as well. It never reads the diffusers base: it fetches a
- # single-file VAE and text encoder from their own repos, and those repos are offerable for
- # deletion, so a pre-existing native GGUF with no recorded link would have had its encoder
- # listed as an unused asset and removed underneath it. The recorded links cover a load that
- # has happened since; this covers the install that predates them.
+ # The NATIVE engine never reads the diffusers base: it fetches a single-file VAE and text encoder
+ # from their own repos, so a pre-existing native GGUF with no recorded link would have had its
+ # encoder listed as unused and removed underneath it.
bases |= {repo for repo, _file in _sd_cpp_component_specs(fam, repo_id, gguf_filename)}
return _with_mirrors(bases)
@@ -413,10 +401,8 @@ def _sd_cpp_component_specs(
vae = getattr(fam, "sd_cpp_vae", None)
if vae and vae[0]:
specs.add((vae[0], vae[1]))
- # EVERY set the family could pick, not the one the string fallback guesses. There is no
- # header to read here, and a renamed FLUX.2-klein 9B file carries no size token either,
- # so guessing answers 4B and leaves the 9B encoder the load actually fetched unprotected.
- # Naming both costs a delete that is refused; guessing costs an installed model.
+ # EVERY set the family could pick: there is no header to read and a renamed FLUX.2-klein 9B file
+ # carries no size token, so guessing answers 4B and leaves the 9B encoder unprotected.
for encoder in sd_cpp_text_encoder_candidates(fam) or ():
if encoder and encoder[0]:
specs.add((encoder[0], encoder[1]))
@@ -435,8 +421,7 @@ def required_companion_asset_files(cache_scans) -> dict[str, set[str]]:
pick_names = _cached_checkpoint_pick_names(cache_scans)
files: dict[str, set[str]] = {}
for repo_id in _cached_model_repo_ids(cache_scans):
- # Every cached GGUF, because one repo can hold checkpoints of two families and each opens
- # its own components.
+ # One repo can hold checkpoints of two families, and each opens its own components.
for name in dict.fromkeys([None, *pick_names.get(_normalise(repo_id), [])]):
fam = _detect_family(repo_id, name)
if fam is None:
@@ -497,17 +482,12 @@ def known_companion_base_ids() -> set[str]:
Deliberately NOT "anything a link ever named": orphan cleanup offers only repos this set
recognises, so a mis-recorded link can never turn an unrelated repo into a delete candidate.
"""
- # Family bases and their gated/mirror variants. The pairs are companion bases by construction
- # -- they exist only because a GGUF pick of that family needs an ungated copy of them -- and
- # they cover the variants one family entry serves through a card tag (klein-9B under the
- # klein-4B family, and so on).
+ # The pairs are companion bases by construction, existing only because a GGUF pick of that family
+ # needs an ungated copy, and they cover the variants one family entry serves through a card tag.
bases = _curated_base_ids()
- # The native engine's component-only repos (a single-file VAE, a text encoder) are curated
- # table entries too, and for an sd.cpp pick they ARE the companions -- the largest half of the
- # footprint. Leaving them out made them link-only strangers: unlisted by Free up space and
- # dropped from the delete preview, so the assets this cleanup exists for stayed invisible.
- # Safe against the one hazard that table warns about, a chat model borrowed as a text encoder:
- # the orphan listing skips any repo holding a GGUF, so it is never offered as a leftover.
+ # The native engine's component-only repos ARE the companions for an sd.cpp pick, and the largest
+ # half of the footprint; leaving them out made them link-only strangers. Safe against a chat model
+ # borrowed as a text encoder, since the orphan listing skips any repo holding a GGUF.
try:
from core.inference.diffusion_families import sd_cpp_companion_only_repo_ids
bases |= set(sd_cpp_companion_only_repo_ids())
@@ -517,12 +497,9 @@ def known_companion_base_ids() -> set[str]:
def _mirror_pair_ids() -> set[str]:
- # The WHOLE table, gated and ungated. Gating decides whether a fetch may override a user's
- # cache; it says nothing about whether a base can strand companions, and most of the table
- # is ungated (Klein 4B and base-4B, HiDream Dev / Fast, SDXL Turbo, ...). Reading only the
- # gated half would drop those from the curated ids, so before a companion link is recorded
- # the cleanup would not recognise them and could offer an installed checkpoint's companions
- # for deletion.
+ # The WHOLE table, gated and ungated: gating decides whether a fetch may override a user's cache,
+ # not whether a base can strand companions, so reading only the gated half would offer an
+ # installed checkpoint's companions for deletion.
try:
from core.inference.diffusion_families import _MIRROR_PAIRS
except Exception: # noqa: BLE001
@@ -542,10 +519,8 @@ def is_companion_base(repo_id: str) -> bool:
key = _normalise(repo_id)
if key in known_companion_base_ids():
return True
- # Through the same identity expansion required_companion_bases uses, so the two agree on
- # WHICH copy is protected. A link recorded against the unsloth mirror is satisfied on an
- # upgraded install by the community repack the fetch fell back to, and comparing the literal
- # id alone left that copy deletable while the checkpoint holding it was still installed.
+ # Through the same identity expansion required_companion_bases uses, so the two agree WHICH copy
+ # is protected: comparing the literal id left an upgraded install's repack copy deletable.
return any(
key in {_normalise(rid) for rid in _with_mirrors([base])}
for bases in read_companion_links().values()
@@ -595,10 +570,8 @@ def required_companion_bases(
recorded = links.get(key)
if recorded:
bases |= _with_mirrors(recorded)
- # Canonical, not literal: a cached MIRROR of a base resolves its own family back to the
- # UPSTREAM id, which would make each identity a dependent of the other and leave a cache
- # holding both unable to delete either. They are copies of one repo, not a pair that
- # needs each other.
+ # Canonical, not literal: a cached MIRROR resolves its own family back to the UPSTREAM id, which
+ # would make each identity a dependent of the other and leave a cache holding both stuck.
self_keys = {key, _normalise(_canonical(repo_id))}
for base in bases:
base_key = _normalise(base)
diff --git a/studio/backend/hub/utils/dataset_format.py b/studio/backend/hub/utils/dataset_format.py
index f6d92ea581e..310367f4ed8 100644
--- a/studio/backend/hub/utils/dataset_format.py
+++ b/studio/backend/hub/utils/dataset_format.py
@@ -403,13 +403,11 @@ def detect_multimodal_dataset(dataset):
detected_text_col = None
if audio_columns:
- # Two passes, not one list: a set carrying both an instruction-like "prompt" and a
- # real "transcript" would otherwise be mapped by schema order, and training an ASR
- # set against its instructions instead of its ground truth fails silently.
+ # Two passes, not one list: a set carrying both an instruction-like "prompt" and a real
+ # "transcript" would be mapped by schema order, training an ASR set against its instructions.
transcript_names = ("text", "sentence", "transcript", "transcription", "label")
- # TTS corpora name the line to speak rather than a transcript of it: every
- # svjack/SparkTTS_* set uses "prompt", LJSpeech derivatives use "normalized_text".
- # Without these the set needs a manual mapping it cannot satisfy.
+ # TTS corpora name the line to speak rather than a transcript of it: SparkTTS sets use "prompt",
+ # LJSpeech derivatives "normalized_text".
fallback_names = ("prompt", "normalized_text")
for candidates in (transcript_names, fallback_names):
for col_name in column_names:
diff --git a/studio/backend/hub/utils/download_manifest.py b/studio/backend/hub/utils/download_manifest.py
index e70c076c5fb..5ac8fe3cdd0 100644
--- a/studio/backend/hub/utils/download_manifest.py
+++ b/studio/backend/hub/utils/download_manifest.py
@@ -119,14 +119,9 @@ class VariantState:
def has_marker(self, variant: str) -> bool:
if variant.lower() in self._markers:
return True
- # An unreadable marker keeps its entry fail-closed but loses the payload,
- # so its only remaining identity is the filename. For a variant that has to
- # be stored hashed (slashes, spaces, `--`, an existing hash prefix) that
- # identity is the digest, not the variant, and a plain lookup misses it --
- # leaving a cancelled variant advertised as complete even though
- # has_cancel_marker still finds the file by rebuilding the same digest.
- # Match the spellings the variant could have been written under, as that
- # per-variant lookup does.
+ # An unreadable marker loses its payload, so its only identity is the filename, which for a
+ # variant stored hashed is the digest rather than the variant: a plain lookup missed it and left a
+ # cancelled variant advertised as complete.
try:
fragments = variant_key_fragments(variant)
except (UnicodeError, ValueError):
@@ -176,9 +171,8 @@ def _hub_cache_spellings(
hub_cache = get_hf_cache_paths().hub_cache
except Exception:
return None, None
- # Shared with state_dir.cache_scope_name so the ownership string recorded in
- # a payload and the cache-<digest> directory it is filed under can never be
- # derived from two different normalizations of the same directory.
+ # Shared with state_dir.cache_scope_name so the ownership string recorded in a payload and the
+ # cache-<digest> directory it is filed under can never come from two different normalizations.
return normalize_hub_cache(hub_cache), hub_cache
@@ -225,9 +219,8 @@ def _scope_spellings(hub_cache: str | Path) -> tuple[str, ...]:
def _read_state_payload(path: Path) -> Optional[dict]:
try:
data = json.loads(path.read_text(encoding = "utf-8"))
- # The decoder raises RecursionError for adversarially deep JSON. One corrupt
- # state file must not abort the shared one-pass index: returning None lets
- # manifests fail open and cancellation markers retain their fail-closed path.
+ # The decoder raises RecursionError for adversarially deep JSON, and one corrupt state file must not
+ # abort the shared one-pass index.
except (OSError, ValueError, RecursionError) as exc:
logger.debug("Could not read Hub state %s: %s", path, exc)
return None
@@ -339,8 +332,8 @@ def _state_read_path(
def applies(path: Path) -> bool:
payload = _read_state_payload(path)
plausible = _state_payload_identity_matches_entry(path, payload, repo_type)
- # Cancellation remains fail-closed when a marker cannot identify its
- # owner. Parseable state from another repository is never borrowed.
+ # Cancellation stays fail-closed when a marker cannot identify its owner; parseable state from
+ # another repository is never borrowed.
if fail_closed and not plausible:
return True
if plausible and variant is not None:
@@ -402,14 +395,10 @@ def _state_paths(
collapses to one entry whenever the two spellings agree, which is every path
that resolves to itself.
"""
- # _scope_spellings, not cache_scope_names, for the reason the repo-wide purge already
- # uses it: the single-variant delete route hands us a root that resolve_delete_target_root
- # has ALREADY resolved, so the raw spelling reproduces the canonical digest and the
- # pre-resolve scope is never probed. The read paths do probe it (they are fed the raw
- # configured setting), so an exact-variant delete left a legacy-scoped manifest or cancel
- # marker behind for the next read to resurrect the variant as partial or cancelled.
- # _scope_spellings adds the configured cache's own spelling only when it names the same
- # directory, so deleting out of a non-active cache still cannot touch the active one.
+ # _scope_spellings, not cache_scope_names: the single-variant delete route hands over a root
+ # resolve_delete_target_root has ALREADY resolved, so the raw spelling reproduces the canonical
+ # digest and the pre-resolve scope is never probed, leaving state for the next read to resurrect.
+ # It adds the configured cache's own spelling only when it names the same directory.
scopes = (
(None,)
if hub_cache is None
@@ -827,18 +816,11 @@ def verify_against_disk(manifest: Manifest, snapshot_dir: Path) -> VerifyResult:
missing: list[str] = []
mismatched: list[str] = []
for expected in manifest.expected_files:
- # Counted missing rather than skipped, so an unverifiable entry can
- # never read as verified. A Windows-separator path lands here, and
- # cannot be folded onto its posix spelling in expected_path_is_safe:
- # that guard also fronts resolved_dataset_snapshot_file, which splits on
- # PurePosixPath, where "a\b" is one component and accepting it would
- # hand a traversal straight through on the platform that disagrees. No
- # writer in this package produces one -- HF rfilenames are posix and
- # expected_files_from_snapshot_dir calls as_posix -- and one that
- # reached a payload would already have failed _manifest_from_payload's
- # identical check, so this stays defence in depth for a Manifest built
- # some other way. Should such a writer ever appear, normalize it in
- # _payload_file_path, not here.
+ # Counted missing rather than skipped, so an unverifiable entry can never read as verified. A
+ # Windows-separator path cannot be folded onto its posix spelling in expected_path_is_safe: that
+ # guard also fronts resolved_dataset_snapshot_file, which splits on PurePosixPath, where "a\b" is
+ # one component and accepting it would hand a traversal through. No writer here produces one (HF
+ # rfilenames are posix and expected_files_from_snapshot_dir calls as_posix).
if not expected_path_is_safe(expected.path):
missing.append(expected.path)
continue
@@ -871,8 +853,8 @@ def expected_files_from_snapshot_dir(snapshot_dir: Path) -> list[ExpectedFile]:
"""
out: list[ExpectedFile] = []
try:
- # Baking a companion into the completion contract makes the download read as partial
- # forever once macOS cleans it up.
+ # Baking a companion into the completion contract makes the download read as partial forever once
+ # macOS cleans it up.
entries = drop_appledouble_metadata(sorted(snapshot_dir.rglob("*")))
except OSError:
return out
@@ -1130,10 +1112,9 @@ def purge_state(
else:
requested, raw = _hub_cache_spellings(hub_cache)
candidates: list[Path] = []
- # Legacy unscoped state is shared: an unowned file belongs to the active
- # cache (per _legacy_state_applies), so only purge it when it belongs to
- # the cache being deleted -- else deleting an inactive cache would erase
- # the active cache's resume/cancel state.
+ # Legacy unscoped state is shared: an unowned file belongs to the active cache, so purge it only
+ # when it belongs to the cache being deleted, else deleting an inactive cache erases the active
+ # cache's state.
for path_factory, fail_closed in (
(manifest_path, False),
(marker_path, True),
@@ -1213,13 +1194,11 @@ def purge_all_state_for_repo(
]
else:
# This cache's scoped dirs plus the legacy unscoped base; glob (not rglob) so other caches are untouched.
- # Both scope spellings, else a repo delete leaves the pre-resolve copy
- # behind for a later read to resurrect as state for a cache that is gone.
+ # Both scope spellings, else a repo delete leaves the pre-resolve copy behind for a later read to
+ # resurrect as state for a cache that is gone.
search = []
- # _scope_spellings, not cache_scope_names: every production caller of this
- # function resolves its target root first, so the raw-spelling probe would
- # be inert here while the read path still has it -- a purged variant that
- # the next read brings back.
+ # _scope_spellings, not cache_scope_names: every production caller resolves its target root first,
+ # so the raw-spelling probe would be inert here while the read path still has it.
scopes = _scope_spellings(hub_cache)
for path_factory, base, cancel_markers in (
(manifest_path, manifests_dir(create = False), False),
@@ -1457,14 +1436,10 @@ def build_variant_state_index(
repo_keys.setdefault(prefix[: -len("--variant--")], set()).add(
(repo_type, normalized_repo)
)
- # Both spellings of the scope dir, so state filed under the pre-resolve
- # digest is indexed rather than silently read as "no manifest". Derived
- # from the caller's own spelling, not the canonical one, which resolves
- # to itself and would yield the canonical digest twice. Normally one key.
- # _scope_spellings for the same reason as the delete: the inventory callers
- # reach here with a directory derived from huggingface_hub.scan_cache_dir,
- # which has already resolved it, so the raw probe alone finds nothing and
- # the cached-model views would disagree with the progress endpoint.
+ # Both spellings of the scope dir, derived from the caller's own rather than the canonical one, so
+ # state filed under the pre-resolve digest is indexed instead of read as "no manifest": the
+ # inventory callers arrive with a directory huggingface_hub.scan_cache_dir already resolved, so
+ # the raw probe alone finds nothing.
for scope in _scope_spellings(hub_cache):
caches_by_scope.setdefault(scope, set()).add(canonical_cache)
@@ -1505,15 +1480,10 @@ def build_variant_state_index(
try:
canonical = marker_path(repo_type, repo_id, variant, create = False)
except (UnicodeError, ValueError, OSError):
- # A state filename can carry a byte the filesystem encoding cannot
- # decode, which iterdir() surfaces as a lone surrogate; hashing that
- # for the canonical spelling raises UnicodeEncodeError. Per-repo
- # reads used to contain the damage to the one repo that owned the
- # file. This index is built once for the whole scan and outside the
- # per-repo try, so letting it escape turns one corrupt filename into
- # a 500 that hides every cached model. `canonical` only decides the
- # "is this the canonical filename" half of `priority`, so treat the
- # entry as non-canonical and keep indexing.
+ # A state filename can carry a byte the filesystem encoding cannot decode, which iterdir()
+ # surfaces as a lone surrogate, and hashing it raises UnicodeEncodeError. This index is built once
+ # for the whole scan and outside the per-repo try, so letting it escape turns one corrupt filename
+ # into a 500 that hides every cached model; treat the entry as non-canonical and keep indexing.
canonical = None
add_entry(
cache,
@@ -1597,14 +1567,10 @@ def _iter_variant_state_files(
return
path_factory = marker_path if cancel_markers else manifest_path
requested, raw = _hub_cache_spellings(hub_cache)
- # Every scope this cache's state can sit in, so a variant filed under the
- # pre-resolve digest is enumerated instead of reading as "no variant state".
- #
- # _scope_spellings, not cache_scope_names, for the same reason the delete and index paths
- # use it: cache_scope_names recovers the legacy digest only from an UNRESOLVED path, and a
- # variant request carrying local_path arrives here already resolved by
- # _repo_cache_dir_for_request. On a cache reached through a symlink or a junction that left
- # the offline listing unable to see its own partial download, or its resume control.
+ # Every scope this cache's state can sit in, so a variant filed under the pre-resolve digest is
+ # enumerated instead of reading as "no variant state".
+ # _scope_spellings, since cache_scope_names recovers the legacy digest only from an UNRESOLVED path,
+ # while a variant request carrying local_path arrives here already resolved.
scopes = _scope_spellings(raw) if requested is not None and raw is not None else (None,)
scoped_dirs: list[Path] = []
for scope in scopes:
@@ -1679,10 +1645,8 @@ def _iter_variant_state_files(
variant_payload = None
variant = _variant_from_state_payload(variant_payload, fallback)
try:
- # Unscoped: only the basename is compared below, and that is
- # the same under every scope. Asking for a scoped path would
- # re-derive the digest -- and pay its ``resolve`` -- once per
- # candidate file, for a directory component never read.
+ # Unscoped: only the basename is compared below, and a scoped path would re-derive the digest, and
+ # pay its resolve, once per candidate file.
canonical = path_factory(
repo_type,
repo_id,
@@ -1691,10 +1655,7 @@ def _iter_variant_state_files(
create = False,
)
except (UnicodeError, ValueError, OSError):
- # Same undecodable-filename case guarded in add_path: a lone
- # surrogate from iterdir() cannot be hashed for the canonical
- # spelling. Before, this loop yielded such an entry as-is;
- # letting the hash escape would instead abort the whole
+ # Same undecodable-filename case as add_path: letting the hash escape would abort the whole
# iteration and lose the repo's valid state alongside it.
canonical = None
priority = (not legacy, canonical is not None and canonical.name == entry.name)
diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.py
index 296df2d4d91..1c9f3843954 100644
--- a/studio/backend/hub/utils/download_registry.py
+++ b/studio/backend/hub/utils/download_registry.py
@@ -55,9 +55,8 @@ from typing import Callable, Iterator, Literal, NamedTuple, Optional, Sequence
from loggers import get_logger
-# One floor, one name, shared. Written out by hand in each module it was needed
-# in at first, and the site that got missed was missed because "who enforces it"
-# was a question you answered by reading rather than by grepping.
+# One floor, one name, shared: hand-written copies meant the site that got missed was missed because
+# "who enforces it" had to be read rather than grepped.
from utils.process_lifetime import is_signalable_pid
from hub.utils import state_dir
@@ -99,12 +98,11 @@ class DownloadTransportCapability:
class DownloadTransportCapabilities:
http: DownloadTransportCapability
xet: DownloadTransportCapability
- # What "auto" would pick right now, and why, so the picker can say
- # "Auto (HTTP -- Xet stalled twice on this machine)" instead of just "Auto".
+ # What "auto" would pick right now, and why, so the picker can say "Auto (HTTP -- Xet stalled twice
+ # on this machine)" instead of just "Auto".
auto_resolves_to: str = TRANSPORT_XET
auto_reason: Optional[str] = None
- # Whether an interrupted HTTP transfer leaves bytes the next attempt can append to. False on
- # huggingface_hub >= 1.18, so the UI stops offering a byte-resume no writer can honour.
+ # False on huggingface_hub >= 1.18, so the UI stops offering a byte-resume no writer can honour.
partials_resumable: bool = True
@@ -126,22 +124,16 @@ def get_download_transport_capabilities(
try:
from utils.hf_xet_fallback import cached_xet_health, xet_health
- # Ordinary UI polls are read-only and must not load Zoo. `probe=True` is different:
- # the frontend sends it only while resolving Auto for a download, then submits the
- # concrete answer as xet/http, so this is the actual first-download decision.
- # `ram_gate` loads it too, without probing: an empty cache reads as the optimistic
- # Xet, so the row promised Xet while the next download loaded an unhealthy verdict
- # and chose HTTP. Still `probe=probe`, so only a download start probes live.
+ # Ordinary UI polls are read-only and must not load Zoo; probe=True is the actual first-download
+ # decision, and ram_gate loads it too without probing: an empty cache reads as the optimistic Xet,
+ # so the row promised Xet while the next download chose HTTP.
health_fn = xet_health if (probe or ram_gate) else cached_xet_health
health = health_fn(probe = probe)
if health is not None:
auto_transport = TRANSPORT_XET if health.use_xet else TRANSPORT_HTTP
auto_reason = str(health.reason)
- # UNSLOTH_FORCE_XET=1: an operator override, not a measurement, so the free-RAM
- # gate below stands down exactly as `resolve_auto_use_xet` does. Same helper, so
- # the probe and the API "auto" path cannot disagree about what "forced" means.
- # Imported separately and guarded: an older or stubbed shim that lacks it must not
- # cost us the health verdict just recorded above.
+ # UNSLOTH_FORCE_XET=1 is an operator override, not a measurement, so the free-RAM gate below stands
+ # down exactly as resolve_auto_use_xet does.
try:
from utils.hf_xet_fallback import xet_health_is_forced
auto_forced = bool(xet_health_is_forced(health))
@@ -156,10 +148,9 @@ def get_download_transport_capabilities(
and auto_transport == TRANSPORT_XET
and not auto_forced
):
- # Free RAM belongs in the same verdict: this IS the Auto decision, since the UI submits the
- # answer as an explicit xet/http. Read outside the health try, because a health module that
- # is missing or raising says nothing about RAM. Never on an ordinary poll, which stays
- # read-only and still does not load Zoo -- only for a probe or an explicit ram_gate.
+ # Free RAM belongs in the same verdict, since the UI submits the answer as an explicit xet/http.
+ # Read outside the health try, because a missing health module says nothing about RAM, and never
+ # on an ordinary poll: only for a probe or an explicit ram_gate.
try:
from utils.hf_xet_fallback import free_ram_pressure_reason
pressure = free_ram_pressure_reason()
@@ -305,8 +296,8 @@ def _is_our_worker(pid: int, repo_id: Optional[str]) -> bool:
return False
if "hub.workers.hf_download" not in cmdline:
return False
- # Exact --repo-id match: a substring match would let a stale breadcrumb for
- # Org/Model reap a live worker for Org/Model-v2.
+ # Exact --repo-id match: a substring match would let a stale breadcrumb for Org/Model reap a live
+ # worker for Org/Model-v2.
if isinstance(repo_id, str) and repo_id:
return _cmdline_repo_id(cmdline) == repo_id
return True
@@ -321,9 +312,8 @@ def _kill_orphan(pid: int) -> bool:
reason to hold up startup -- and answering False there matters: a survivor must keep its
breadcrumb and must not have its live partial claimed as ours to delete.
"""
- # Its caller floors the pid too. Repeated here because this one sends the
- # signal, and a helper that kills should not depend on every future caller
- # having checked first.
+ # Repeated here because this one sends the signal, and a helper that kills should not depend on
+ # every future caller having checked first.
if not is_signalable_pid(pid):
return False
try:
@@ -421,8 +411,8 @@ def reap_orphan_workers() -> None:
try:
entries = list(parent.iterdir())
except OSError:
- # Unreadable breadcrumbs means no worker can be claimed as reaped, not that the caches
- # go unswept: they are a separate tree and may well be readable.
+ # Unreadable breadcrumbs means no worker can be claimed as reaped, not that the caches go unswept:
+ # they are a separate tree.
_boot_sweep(reaped)
return
for entry in entries:
@@ -435,26 +425,22 @@ def reap_orphan_workers() -> None:
continue
pid = data.get("pid") if isinstance(data, dict) else None
repo_id = data.get("repo_id") if isinstance(data, dict) else None
- # pid 1 as well as 0 and negatives. The cmdline check below is what
- # actually keeps this honest and init cannot pass it, but the same was
- # true of the reaper's start-time check until a record naming pid 1
- # slipped through it, so the cheap bound goes in front of the clever one.
+ # pid 1 as well as 0 and negatives: the cmdline check below is what keeps this honest, but a
+ # record naming pid 1 once slipped through the reaper's start-time check.
signalable = is_signalable_pid(pid)
try:
if not signalable:
- # Nothing is signalled on this record's word, but its repo fields
- # are still readable, and the settle below is what preserves the
- # partial's resume marker. Unlinking straight from here would cost
- # the user a restarted download to pay for a bug that is ours.
+ # Its repo fields are still readable, and the settle below preserves the partial's resume marker:
+ # unlinking from here would cost the user a restarted download to pay for a bug that is ours.
pass
elif not _process_alive(pid):
- # Already gone, which is better proof than killing it ourselves. Its partial
- # is ours to sweep even though this invocation reaped nothing.
+ # Already gone, which is better proof than killing it ourselves; its partial is ours to sweep even
+ # though this invocation reaped nothing.
reaped.append((data.get("repo_type") or "model", repo_id, data.get("hub_cache")))
elif _is_our_worker(pid, repo_id):
if not _kill_orphan(pid):
- # Still running. Keeping the breadcrumb keeps it tracked for the next boot,
- # and claiming no ownership keeps its live partial out of the sweep.
+ # Still running: keeping the breadcrumb keeps it tracked for the next boot, and claiming no
+ # ownership keeps its live partial out of the sweep.
logger.warning(
"Could not reap download worker pid=%s repo=%s; leaving its "
"breadcrumb and partial in place.",
@@ -462,9 +448,8 @@ def reap_orphan_workers() -> None:
repo_id,
)
continue
- # Its partial is unreadable now and its writer is gone, but only as of this
- # line. The sweep has to come after the kill, not before, or it reads the
- # still-held blob lock and spares a file nothing will ever finish.
+ # The sweep has to come after the kill, not before, or it reads the still-held blob lock and spares
+ # a file nothing will ever finish.
reaped.append((data.get("repo_type") or "model", repo_id, data.get("hub_cache")))
logger.warning(
"Reaped orphaned download worker pid=%s repo=%s from a "
@@ -533,8 +518,8 @@ class _PurgeOutcome(NamedTuple):
failed: int
-# Only the unresumable sweep waits out ABANDONED_PARTIAL_SECONDS: it reclaims disk, while a
-# marker-mismatch purge exists to stop a corrupt append and cannot defer.
+# Only the unresumable sweep waits out ABANDONED_PARTIAL_SECONDS: it reclaims disk, while a marker-
+# mismatch purge exists to stop a corrupt append and cannot defer.
def _purge_incomplete_blobs(
@@ -590,10 +575,8 @@ def _purge_incomplete_blobs(
if unresumable_only:
if partial_is_resumable(blob.name, entry.parent):
continue
- # Two independent ways to spot a live writer, because neither is sufficient
- # alone: the lock is the precise signal but upstream calls it best-effort and
- # some filesystems grant it to everyone, while mtime cannot tell a dead writer
- # from one stalled on a slow network for longer than the grace.
+ # Neither signal is sufficient alone: the lock is precise but upstream calls it best-effort and
+ # some filesystems grant it to everyone, while mtime cannot tell a dead writer from a stalled one.
if blob_download_lock_held(entry, blob_hash):
continue
owned = owns_all_blobs or bool(owned_hashes and blob_hash in owned_hashes)
@@ -607,8 +590,8 @@ def _purge_incomplete_blobs(
blob.unlink()
removed += 1
except FileNotFoundError:
- # A peer finalized or removed it after enumeration. The requested
- # end state was reached, so this is not a failed purge.
+ # A peer finalized or removed it after enumeration, so the requested end state was reached and this
+ # is not a failed purge.
continue
except OSError:
failed += 1
@@ -617,8 +600,7 @@ def _purge_incomplete_blobs(
return _PurgeOutcome(removed + watched_outcome.removed, failed + watched_outcome.failed)
-# One shared pause is enough to tell a frozen corpse from a writer mid-transfer: hf writes a
-# partial continuously, so anything untouched across it is not being written by anyone.
+# One shared pause is enough to tell a frozen corpse from a writer mid-transfer: hf writes a partial continuously.
_STILLNESS_PROBE_SECONDS = 2.0
@@ -638,7 +620,7 @@ def _purge_still_partials(watched: "Sequence[tuple[Path, str, int, float]]") ->
try:
stat = blob.stat()
if stat.st_size != size or stat.st_mtime != mtime:
- continue # it moved, so somebody owns it after all
+ continue
blob.unlink()
removed += 1
except FileNotFoundError:
@@ -713,9 +695,8 @@ def _marker_path(entry: Path, variant: Optional[str] = None) -> Path:
def _is_transport_marker_file(path: Path) -> bool:
- # Matches ".transport", its tmps, and variant-scoped ".transport.gguf-*".
- # Real HF cache entries (blobs/refs/snapshots/.no_exist) never start with
- # ".transport.".
+ # Matches ".transport", its tmps and variant-scoped ".transport.gguf-*"; real HF cache entries
+ # (blobs/refs/snapshots/.no_exist) never start with ".transport.".
return path.name == TRANSPORT_MARKER_NAME or path.name.startswith(f"{TRANSPORT_MARKER_NAME}.")
@@ -729,22 +710,21 @@ def _read_marker_value(marker: Path) -> Optional[str]:
return None
value = marker.read_text(encoding = "utf-8").strip()
except (OSError, UnicodeDecodeError):
- # UnicodeDecodeError is a ValueError, so it would escape and abort
- # prepare_cache_for_transport. An unknown value just purges and restarts.
+ # UnicodeDecodeError is a ValueError, so it would escape and abort prepare_cache_for_transport; an
+ # unknown value just purges and restarts.
return None
return value if value in VALID_TRANSPORTS else None
def _write_marker_value(marker: Path, mode: str) -> None:
try:
- # tmp + rename so a SIGKILL mid-write can't leave a half-written marker.
- # The tmp name is per-process so concurrent writers don't clobber tmps.
+ # tmp + rename so a SIGKILL mid-write cannot leave a half-written marker, with a per-process tmp
+ # name so concurrent writers do not clobber tmps.
tmp = marker.with_name(f"{marker.name}.tmp-{os.getpid()}")
tmp.write_text(mode, encoding = "utf-8")
os.replace(tmp, marker)
except OSError:
- # Best-effort: a missing marker next run purges the partial defensively,
- # the safe failure mode.
+ # Best-effort: a missing marker next run purges the partial defensively, which is the safe failure mode.
pass
@@ -814,8 +794,8 @@ def prepare_cache_for_transport(
"""
if mode not in VALID_TRANSPORTS:
if mode == TRANSPORT_AUTO:
- # "auto" is a request preference, not a cache writer: it must be resolved to xet/http
- # before reaching this layer. Naming it turns "invalid transport" into the actual bug.
+ # "auto" is a request preference, not a cache writer: naming it turns "invalid transport" into the
+ # actual bug.
raise ValueError(
f"{TRANSPORT_AUTO!r} must be resolved to a concrete transport before preparing the "
f"cache; expected one of {sorted(VALID_TRANSPORTS)}"
@@ -833,9 +813,8 @@ def prepare_cache_for_transport(
except OSError:
return 0
if not entries:
- # First download: pre-create the repo dir so the marker lands before the
- # worker writes any bytes. Otherwise a SIGKILL mid-download leaves a
- # partial with no marker that the resume then purges.
+ # Pre-create the repo dir so the marker lands before the worker writes any bytes; otherwise a
+ # SIGKILL mid-download leaves a partial with no marker that the resume then purges.
canonical = repo_cache_dir_name(repo_type, repo_id)
new_entry = root / canonical
try:
@@ -852,11 +831,7 @@ def prepare_cache_for_transport(
if mode == TRANSPORT_XET:
main_purge = _purge_incomplete_blobs(entry, only_blob_hashes, protected)
else:
- # A matching marker vouches for provenance, which is only worth something if
- # something can still append to the partial it vouches for. When nothing can, what
- # survives is dead weight that holds the disk the refetch needs and, carrying the
- # etag of the blob being refetched, pins the bar to its own stale high-water mark
- # until the new attempt overtakes it. Sweep it, but only once abandoned.
+ # A matching marker only matters while something can still append to the partial it vouches for.
if _read_marker(entry, variant) != mode:
main_purge = _purge_incomplete_blobs(entry, only_blob_hashes, protected)
else:
@@ -981,14 +956,13 @@ def sweep_abandoned_partials(
this when a download reaches a terminal state and every file skipped then gets a second
look, by which point the grace has long since elapsed.
"""
- # DownloadMetadata.hub_cache is a str, and every caller hands its captured root straight
- # through, so normalize here rather than trusting each one to remember.
+ # DownloadMetadata.hub_cache is a str and every caller hands its captured root straight through, so
+ # normalize here rather than trusting each one.
if isinstance(root, str):
root = Path(root) if root else None
removed = 0
- # The destructive iterator, not the active one: this deletes, and on a case-insensitive
- # collision the active iterator yields every spelling while this one resolves to the exact
- # directory or refuses. A job owning "its" repo does not own a differently-cased neighbour.
+ # The destructive iterator, not the active one: on a case-insensitive collision the active iterator
+ # yields every spelling while this one resolves to the exact directory or refuses.
for entry in iter_destructive_repo_cache_dirs(repo_type, repo_id, root = root):
outcome = _purge_incomplete_blobs(
entry,
@@ -1195,11 +1169,9 @@ def existing_blob_bytes(
download still needs to write before the run starts."""
if not blob_hashes:
return 0
- # One tally for ALL the repo dirs the root holds, not one per dir. The Hub resolves repo ids
- # case-insensitively while huggingface_hub keeps the caller's casing in the folder name, so
- # a case-sensitive filesystem really does end up with models--Org--Model beside
- # models--org--model, each holding its own copy of the same blob. Summing the dirs counted
- # that shard twice, and the caller's max(0, total - have) then clamped to zero.
+ # One tally for ALL the repo dirs the root holds: the Hub resolves repo ids case-insensitively
+ # while huggingface_hub keeps the caller's casing, so a case-sensitive filesystem holds two copies
+ # of one blob and summing the dirs counted that shard twice.
present = {blob_hash: 0 for blob_hash in blob_hashes}
for entry in iter_active_repo_cache_dirs(repo_type, repo_id, root = root):
blobs_dir = entry / "blobs"
@@ -1222,28 +1194,22 @@ def existing_blob_bytes(
and not partial_is_resumable(blob.name, entry.parent)
and not blob_download_lock_held(entry, blob_hash)
):
- # Callers spend this on "bytes we will not have to fetch again", and
- # _preflight_disk_space subtracts it from the space a download needs. An
- # unresumable partial is refetched in full into a new path, so counting it
- # would clear a download for a disk that cannot hold it. A LOCKED one is
- # different: a live peer is finishing it and snapshot_download will block
- # on that lock and reuse the result, so those bytes are not ours to find
- # room for. Two GGUF variants sharing an mmproj hit this every time.
+ # An unresumable partial is refetched in full into a new path, so counting it would clear a
+ # download for a disk that cannot hold it. A LOCKED one is different: a live peer is finishing it
+ # and snapshot_download blocks on that lock and reuses the result, so those bytes are not ours to
+ # find room for.
continue
- # A partial is measured by the bytes actually ON DISK, not by its logical
- # length: hf_transfer's parallel Range writer leaves a sparse file whose
- # st_size runs ahead of what has been written -- observed at 1.2 GB reported
- # against 112 MB present, and once st_size reached the declared size the
- # remainder read as zero for a file barely started. A finalized blob is whole
- # by construction, so it keeps st_size: st_blocks is smaller than the file on
- # a compressing filesystem.
+ # Measured by the bytes actually ON DISK: hf_transfer's parallel Range writer leaves a sparse file
+ # whose st_size runs ahead of what was written, observed at 1.2 GB reported against 112 MB. A
+ # finalized blob is whole by construction, so it keeps st_size: st_blocks is smaller than the file
+ # on a compressing filesystem.
bytes_here = (
blob_bytes_present(blob)
if partial_hash is not None
else max(0, int(blob.stat().st_size))
)
- # Broken advisory locks can leave several process-unique writers for one etag.
- # They are duplicate attempts, not additive completion, so keep the largest.
+ # Broken advisory locks can leave several process-unique writers for one etag: duplicate attempts,
+ # not additive completion, so keep the largest.
present[blob_hash] = max(present[blob_hash], max(0, int(bytes_here)))
except OSError:
continue
@@ -1269,14 +1235,11 @@ class DownloadMetadata:
variant: Optional[str]
transport: Optional[str]
cancel_marker_transport: Optional[str] = None
- # GGUF variant main/writable hashes, identifying the variant-specific shards
- # for concurrency decisions.
+ # GGUF variant main/writable hashes, identifying the variant-specific shards for concurrency decisions.
blob_hashes: frozenset[str] = field(default_factory = frozenset)
- # Full required hash set for progress/completion (includes the shared mmproj
- # companion for vision GGUF repos).
+ # Includes the shared mmproj companion for vision GGUF repos.
progress_blob_hashes: frozenset[str] = field(default_factory = frozenset)
# Bytes already complete before this job started; not counted as this run's
- # progress.
completed_baseline_bytes: int = 0
hub_cache: Optional[str] = None
xet_cache: Optional[str] = None
@@ -1380,8 +1343,8 @@ class DownloadRegistry:
self._cancel_marker_transports: dict[str, str] = {}
self._pending_cancel: dict[str, Optional[int]] = {}
self._generations: dict[str, int] = {}
- # Monotonic across keys so an evicted then re-claimed key never reuses a
- # prior generation (which would let a stale cancel match a new run).
+ # Monotonic across keys so an evicted then re-claimed key never reuses a prior generation, which
+ # would let a stale cancel match a new run.
self._generation_seq = 0
self._deleting: dict[str, set[Optional[str]]] = {}
# Publish external cache owners under the same lock as Model Hub jobs.
@@ -1633,11 +1596,8 @@ class DownloadRegistry:
with self._lock:
if repo in self._repository_owners:
return False, "repository_owned"
- # Run the final external admission check while the registry lock is
- # held, immediately before inspecting and publishing active state.
- # The GGUF load path establishes its marker before calling
- # its active-job probe, so either this claim observes that marker
- # or the load's later probe observes this claim.
+ # Run the final admission check under the registry lock: the GGUF load path establishes its marker
+ # before its active-job probe, so either this claim sees that marker or the load sees this claim.
if admission_check is not None and not admission_check():
return False, "admission_blocked"
deleting_scopes = self._deleting.get(repo)
@@ -1656,11 +1616,9 @@ class DownloadRegistry:
stale_keys.append(other_key)
continue
other_metadata = self._metadata.get(other_key)
- # Same-transport variants of one model run concurrently: each
- # worker purges only its own re-resolved main blobs and the
- # shared companion is guarded by its marker. Cross-transport
- # stays serialized so an HTTP resume and an XET rewrite never
- # write one shared blob at once.
+ # Same-transport variants of one model run concurrently, since each worker purges only its own
+ # re-resolved main blobs and the shared companion is guarded by its marker; cross-transport stays
+ # serialized so an HTTP resume and an XET rewrite never write one blob at once.
concurrent_gguf_variants = (
repo_type == "model"
and bool(variant)
@@ -1679,8 +1637,8 @@ class DownloadRegistry:
return False, conflict_state
current = self._jobs.get(key, DownloadState("idle")).state
if current in _ACTIVE_STATES and not replace_active:
- # A scope slot is shared by every file set that rides it (two quants of one repo both key as "@diffusion"), so adopting
- # the live job would let the caller wait on files it never asked for. Reject instead, under the lock.
+ # A scope slot is shared by every file set that rides it, so adopting the live job would let the
+ # caller wait on files it never asked for.
live = self._metadata.get(key)
if (
scoped_files is not None
@@ -1840,9 +1798,8 @@ class DownloadRegistry:
candidate_keys = list(self._repo_active.get(repo_key, set()))
else:
candidate_keys = [key for active in self._repo_active.values() for key in active]
- # An XET->HTTP retry handoff briefly drops its key from _repo_active
- # while its job stays active; include those released-but-active jobs
- # so the waiting retry still lists and can be adopted or cancelled.
+ # An XET->HTTP retry handoff briefly drops its key from _repo_active while its job stays active, so
+ # include those released-but-active jobs.
seen = set(candidate_keys)
for key, job in self._jobs.items():
if key in seen or job.state not in _ACTIVE_STATES:
@@ -1937,11 +1894,8 @@ class DownloadRegistry:
continue
if self._active_job_variant_locked(key) != target:
return True
- # An XET->HTTP retry peer between release_active_slot() and its reclaim
- # is briefly absent from _repo_active while its job stays active and
- # still owns the shared companion; mirror the released-but-active scan
- # used by _delete_blocked_by_active_locked so it still blocks companion
- # deletion of a different variant.
+ # A retry peer between release_active_slot() and its reclaim is briefly absent from _repo_active
+ # while it still owns the shared companion, so mirror the released-but-active scan.
for key, job in self._jobs.items():
if key in active_keys or _repo_of_key(key) != repo_id:
continue
@@ -1981,18 +1935,12 @@ class DownloadRegistry:
]
live_keys = {key for key, _proc, _metadata in live}
# Flag as an intentional stop so the watcher's exit classification
- # reports them cancelled rather than an OOM/crash once SIGKILL lands.
for key, _proc, _metadata in live:
if self._jobs.get(key, DownloadState("idle")).state == "running":
self._jobs[key] = DownloadState("cancelling")
- # Settle active jobs without a live worker too. Two cases: an
- # XET->HTTP retry parked in the reclaim wait loop has dropped its
- # worker and slot guard, so it is absent from `live`; and a
- # registered worker that already exited with an error but whose
- # watcher has not yet run would otherwise stay `running` and spawn an
- # HTTP retry after this shutdown snapshot. Skip a registered worker
- # that exited cleanly (rc == 0): it completed and the watcher will
- # mark it done, so marking it cancelling would strand a stale marker.
+ # Settle active jobs without a live worker: a retry parked in the reclaim wait has dropped its
+ # worker, and a registered worker that errored before its watcher ran would stay running and spawn
+ # an HTTP retry. Skip one that exited cleanly, which would strand a stale marker.
for key, job in list(self._jobs.items()):
if job.state not in _ACTIVE_STATES or key in live_keys:
continue
@@ -2000,22 +1948,16 @@ class DownloadRegistry:
if proc is not None:
if proc.poll() == 0:
continue
- # A registered worker that exited nonzero on its own over HTTP
- # is a genuine terminal download failure, not a shutdown cancel
- # and not retry-capable: leave its error status intact rather
- # than persisting a cancel marker that would read as
- # cancelled/resumable after restart. Only an exited XET worker
- # could still spawn a post-shutdown HTTP retry, so only that
- # needs settling here.
+ # A registered worker that exited nonzero over HTTP is a genuine terminal failure, not a shutdown
+ # cancel: only an exited XET worker could still spawn a post-shutdown HTTP retry.
metadata = self._metadata.get(key)
if metadata is not None and metadata.transport == TRANSPORT_HTTP:
continue
self._pending_cancel[key] = self._generations.get(key)
self._jobs[key] = DownloadState("cancelling")
settled_no_proc.append(self._metadata.get(key))
- # Persist a cancel marker for each settled no-live-worker job outside the
- # lock (mirroring the reaped path) so shutdown records resumable/cancelled
- # state even if it returns before the daemon watcher wakes to do so.
+ # Persist the cancel marker outside the lock so shutdown records resumable state even if it returns
+ # before the daemon watcher wakes.
for metadata in settled_no_proc:
if metadata is not None:
persist_cancel_marker(
@@ -2051,8 +1993,7 @@ class DownloadRegistry:
logger.warning(f"shutdown: {kind} worker for {key} did not exit after kill")
except Exception:
pass
- # Mark only genuinely interrupted workers (rc != 0, or None on wait
- # timeout); persisting before the exit is known would strand a stale
+ # Mark only genuinely interrupted workers: persisting before the exit is known would strand a stale
# marker on a worker that completed cleanly during shutdown.
if metadata is not None and proc.poll() != 0:
persist_cancel_marker(
diff --git a/studio/backend/hub/utils/gguf.py b/studio/backend/hub/utils/gguf.py
index 181504eaa78..0986e154420 100644
--- a/studio/backend/hub/utils/gguf.py
+++ b/studio/backend/hub/utils/gguf.py
@@ -95,10 +95,9 @@ def is_mmproj_filename(filename: str) -> bool:
return "mmproj" in filename.lower()
-# Anchored at an END of the stem, never a substring, for the reason
-# is_mtp_drafter_path documents: a name that merely contains the word
-# (``Qwen3-Imatrix-Tuned-Q4_K_M.gguf``) is a real model, while every published
-# imatrix leads or closes with it.
+# Anchored at an END of the stem, never a substring, for the reason is_mtp_drafter_path documents:
+# a name that merely contains the word (Qwen3-Imatrix-Tuned-Q4_K_M.gguf) is a real model, while
+# every published imatrix leads or closes with it.
_IMATRIX_TOKEN_RE = re.compile(r"^imatrix(?:[._\-]|$)|[._\-]imatrix$", re.IGNORECASE)
@@ -122,12 +121,11 @@ def is_imatrix_filename(path: str) -> bool:
return bool(_IMATRIX_TOKEN_RE.search(stem)) or name.lower().endswith(".imatrix")
-# Separate-file drafter kinds. dspark and dflash are the same DeepSeek V4 Flash
-# drafter: the folder it ships in and the architecture it reports.
+# dspark and dflash are the same DeepSeek V4 Flash drafter: the folder it ships in and the architecture it reports.
_DRAFTER_KINDS = ("mtp", "dspark", "dflash")
-# Directories only: mtp/ and dspark/ are always a publisher's companion folder,
-# while dflash/ is a family name a user picks for real weights.
+# Directories only: mtp/ and dspark/ are always a publisher's companion folder, while dflash/ is a
+# family name a user picks for real weights.
_DRAFTER_DIR_KINDS = ("mtp", "dspark")
@@ -226,10 +224,9 @@ def is_gguf_filename(filename: str) -> bool:
return filename.lower().endswith(".gguf")
-# Every repo that bundles H3's denoisers together with its companion models. The Unsloth mirror
-# carries the Qwen3-VL text-encoder quants beside the denoisers, so it needs the same filtering as
-# the community repack it replaced; listing only one of them would let the encoder GGUFs be
-# aggregated as if they were selectable transformer quants.
+# Every repo that bundles H3's denoisers with its companion models: the Unsloth mirror carries the
+# Qwen3-VL encoder quants beside the denoisers, so listing one repo would aggregate encoder GGUFs
+# as selectable transformer quants.
_H3_BUNDLE_REPOS = frozenset({"leejet/minimax-h3-gguf", "unsloth/minimax-h3-gguf"})
@@ -237,10 +234,9 @@ def is_h3_bundle_repo(repo_id: str) -> bool:
return repo_id.strip().lower() in _H3_BUNDLE_REPOS
-# Both released denoiser partitions are valid picks -- which one is picked IS the task. Kept in
+# Both released denoiser partitions are valid picks and which one is picked IS the task. Kept in
# step with validate_h3_transformer_filename in core/inference/video_minimax_h3.py, which the load
-# enforces; listing only FL2VA hid every published Ref2VA quant from the picker even though the
-# loader routes it and the reference UI depends on it.
+# enforces: listing only FL2VA hid every published Ref2VA quant from the picker.
_H3_DENOISER_PARTITIONS = ("minimax_h3_fl2va", "minimax_h3_ref2va")
@@ -394,20 +390,17 @@ def extract_quant_token(filename: str) -> Optional[str]:
return None
-# A bits-per-weight modifier trailing the quant token. Two builds of one base quant at different
-# bpw are two checkpoints (byteshape ships IQ4_XS at 3.53, 3.97 and 4.19), and the loader's own
-# label keeps the modifier for exactly that reason. The variant KEY has to keep it too: without it
-# the lister advertises IQ4_XS-3.53bpw while the plan, the download map and the delete predicate
-# all say IQ4_XS, so the advertised name 404s and the collapsed one unlinks every build.
-#
-# Applied with ``match`` against the text that follows the token, never ``search``: only a
-# modifier IMMEDIATELY after the quant qualifies it, so ``flux1-dev-Q8_0-fp32-08.577bpw`` keeps
-# the bare ``Q8_0`` rather than borrowing a number that describes something else in the name.
+# Two builds of one base quant at different bpw are two checkpoints (byteshape ships IQ4_XS at
+# 3.53, 3.97 and 4.19), so the variant KEY has to keep the modifier: without it the advertised
+# name 404s.
+# match, never search: only a modifier IMMEDIATELY after the quant qualifies it, so
+# flux1-dev-Q8_0-fp32-08.577bpw keeps the bare Q8_0 rather than borrowing a number describing
+# something else.
_GGUF_BPW_SUFFIX_RE = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw", re.IGNORECASE)
-# The same modifier ending a name of its own, with or without the extension still on it. Used
-# only for the basename under a quant DIRECTORY (``Q6_K/model-3.5bpw.gguf``), where the token is
-# in the parent and the number is all the file has to say which build it is.
+# The same modifier ending a name of its own, used only for the basename under a quant DIRECTORY
+# (Q6_K/model-3.5bpw.gguf), where the token is in the parent and the number is all the file has to
+# identify the build.
_GGUF_BPW_TRAILING_RE = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw(?=\.[A-Za-z0-9]+$|$)", re.IGNORECASE)
@@ -834,9 +827,8 @@ def bare_quant_alias(key: str) -> str:
token = quant_token_with_bpw(basename)
if token is not None:
return token
- # Nothing in the name is a quant, so the alias is the unknown-variant spelling. That one
- # still re-stems, and a key arrives already extension-stripped, so hand it an extension to
- # strip rather than let it cut at the dot in "ltx-2.3".
+ # Nothing in the name is a quant, so the alias is the unknown-variant spelling; hand it an extension
+ # to strip rather than let it cut at the dot in "ltx-2.3".
return extract_quant_label(f"{basename}.gguf")
@@ -892,9 +884,8 @@ def gguf_variant_key(filename: str) -> str:
quant = quant_token_with_bpw(path)
if quant is None:
return _unknown_gguf_variant_key(path)
- # MiniMax H3 bundles two denoiser partitions, and may publish both full and pruned
- # builds at one quant. Each file is a different loadable checkpoint, so the bare
- # quant cannot identify the row, download state, or file the loader should open.
+ # MiniMax H3 bundles two denoiser partitions and may publish full and pruned builds at one quant,
+ # so the bare quant cannot identify the row, its download state, or the file to open.
if path.rsplit("/", 1)[-1].lower().startswith(_H3_DENOISER_PARTITIONS):
return _unknown_gguf_variant_key(path)
parents = path.rpartition("/")[0]
@@ -924,9 +915,8 @@ def _apply_gguf_display_labels(variants: list[GgufVariantInfo]) -> None:
]
ambiguous = len(unknown_variants) > 1
- # The bpw modifier is part of the key but reads perfectly well on its own
- # ("IQ4_XS-3.53bpw"), so a key that is only the token plus its bpw suffix is NOT a
- # path-qualified one and needs no scope label.
+ # The bpw modifier reads perfectly well on its own, so a key that is only the token plus its bpw
+ # suffix is NOT path-qualified and needs no scope label.
def _plain_key(variant) -> Optional[str]:
return quant_token_with_bpw(variant.filename)
@@ -1009,8 +999,8 @@ def iter_hf_cache_snapshots(repo_id: str, root: Optional[Path] = None):
)
for repo_dir in repo_dirs:
snapshots_dir = repo_dir / "snapshots"
- # is_dir() ignores only ENOENT/ENOTDIR/EBADF/ELOOP, so an unreadable root raised
- # EACCES up to 3.13 (3.14 returns False, gh-101357). Skip it instead of 500ing.
+ # is_dir() ignores only ENOENT/ENOTDIR/EBADF/ELOOP, so an unreadable root raised EACCES up to 3.13
+ # (3.14 returns False, gh-101357); skip it instead of 500ing.
try:
if not snapshots_dir.is_dir():
continue
@@ -1143,8 +1133,7 @@ def merge_sibling_snapshot_variants(
merged_complete.add(variant.quant)
whole.add(key)
if changed:
- # Labels disambiguate within a revision, so a merged set can hold names only the
- # merge brings together.
+ # Labels disambiguate within a revision, so a merged set can hold names only the merge brings together.
_apply_gguf_display_labels(merged)
return merged, has_vision or any(merged_vision.values()), merged_complete, snapshot
@@ -1189,9 +1178,8 @@ def list_partial_gguf_variants_from_state(
"""
from hub.utils import download_manifest
- # Variant identity on disk is case-insensitive (_entry_key lowercases it), so
- # dedupe on the lowercased key. Manifests are read first to keep their
- # original-casing label over a lowercased cancel marker for the same variant.
+ # Variant identity on disk is case-insensitive, so dedupe on the lowercased key; manifests are read
+ # first to keep their original-casing label over a lowercased cancel marker.
seen: set[str] = set()
ordered: list[str] = []
sources = (
@@ -1247,14 +1235,13 @@ def list_partial_gguf_variants_from_state(
if not is_gguf_filename(expected.path):
continue
if is_imatrix_filename(expected.path):
- # A manifest predating this filtering can still name one; it is
- # neither weights nor a companion, so it counts towards neither size.
+ # A manifest predating this filtering can still name one: it is neither weights nor a companion, so
+ # it counts towards neither size.
imatrix_only = True
continue
if is_mtp_drafter_path(expected.path):
- # Downloaded with every variant (like mmproj) but not a
- # selectable quant; count it so the shown download size
- # matches what is fetched.
+ # Downloaded with every variant like mmproj but not a selectable quant; counted so the shown
+ # download size matches what is fetched.
companion_bytes += max(0, int(expected.size or 0))
continue
if is_mmproj_filename(expected.path):
@@ -1266,11 +1253,9 @@ def list_partial_gguf_variants_from_state(
main_filenames.append(expected.path)
size_bytes += max(0, int(expected.size or 0))
if main_filename is None:
- # An older build could download the imatrix as a variant of its own, so its
- # interrupted state is still on disk. Naming the synthetic file after the
- # variant would put that row back in the menu, at zero bytes, on exactly the
- # offline path this listing serves. Only when NOTHING eligible was found:
- # a marker-only quant keeps its synthetic row so it can be resumed or deleted.
+ # An older build could download the imatrix as a variant of its own, so naming the synthetic file
+ # after the variant would put that interrupted row back in the menu at zero bytes. Only when
+ # NOTHING eligible was found.
if imatrix_only or is_imatrix_filename(variant):
continue
main_filename = f"{variant}.gguf"
@@ -1384,8 +1369,10 @@ def list_gguf_variants(
if is_mmproj_filename(filename):
has_vision = True
continue
- # The two extractors disagree on F16-be-checkpoint-Q4_K_M shapes; judge with the
- # loader's label so no row is advertised for a file the remote detector refuses.
+ # The two extractors disagree on F16-be-checkpoint-Q4_K_M shapes; judge with the loader's label so
+ # no row is advertised for a file the detector refuses.
+ # The two extractors disagree on F16-be-checkpoint-Q4_K_M shapes; judge with the loader's label so no
+ # row is listed for a file the local detector refuses.
from utils.models.model_config import _extract_quant_label as _loader_quant
if is_big_endian_gguf_path(filename, _loader_quant(filename)):
@@ -1440,8 +1427,8 @@ def list_local_gguf_variants(
main_files: list[tuple[str, int]] = []
has_vision = False
- # Match the cache dir of ANY H3 bundle repo, not just one of them: the same aggregation runs
- # over whichever mirror the user actually downloaded.
+ # Match the cache dir of ANY H3 bundle repo: the aggregation runs over whichever mirror the user
+ # actually downloaded.
root_key = root.as_posix().lower()
h3_bundle_repo = next(
(r for r in _H3_BUNDLE_REPOS if f"models--{r.replace('/', '--')}" in root_key), None
@@ -1453,9 +1440,8 @@ def list_local_gguf_variants(
if is_imatrix_filename(file.name):
continue
if is_mmproj_filename(file.name):
- # Header metadata distinguishes vision projectors from audio-only ones. Read it
- # only when Windows reports the file fully present; opening a cloud placeholder
- # would recall the projector during discovery.
+ # Header metadata distinguishes vision projectors from audio-only ones, read only when Windows
+ # reports the file fully present: opening a cloud placeholder would recall it during discovery.
try:
info = file.stat()
has_vision = has_vision or (
@@ -1473,8 +1459,6 @@ def list_local_gguf_variants(
rel = file.relative_to(root).as_posix()
if _is_local_mtp_drafter(file, custom_root, rel):
continue
- # The two extractors disagree on F16-be-checkpoint-Q4_K_M shapes; judge with the
- # loader's label so no row is listed for a file the local detector refuses.
from utils.models.model_config import _extract_quant_label as _loader_quant
if is_big_endian_gguf_path(rel, _loader_quant(rel)):
diff --git a/studio/backend/hub/utils/gguf_plan.py b/studio/backend/hub/utils/gguf_plan.py
index 4e1f22f558b..154f2b29268 100644
--- a/studio/backend/hub/utils/gguf_plan.py
+++ b/studio/backend/hub/utils/gguf_plan.py
@@ -84,9 +84,8 @@ def is_main_gguf_variant_path(path: str, variant: str) -> bool:
and not is_mmproj_filename(path)
and not is_mtp_drafter_path(path)
and not is_imatrix_filename(path)
- # The endian predicate reads a quant TOKEN, so it gets the label: handed the qualified key
- # it cannot see a parent-only quant and drops the file, leaving the plan with no main
- # files at all and an interrupted download with no hashes to resume against.
+ # The endian predicate reads a quant TOKEN, so hand it the label: given the qualified key it
+ # cannot see a parent-only quant and drops the file, leaving the plan with no main files.
and not is_big_endian_gguf_path(path, extract_quant_label(path))
and gguf_variant_key(path).lower() == variant.lower()
)
@@ -218,9 +217,9 @@ def dflash_plan_files(
def build_gguf_variant_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]:
- # Family grouping keeps the family holding the lexicographically first name, which is the
- # "._" one, so the plan fetched the sidecar and marked the variant complete -- leaving local
- # discovery, which judges by header, no main GGUF to load.
+ # Family grouping keeps the family holding the lexicographically first name, which is the "._"
+ # one, so the plan fetched the sidecar and marked the variant complete, leaving header-based local
+ # discovery no main GGUF to load.
siblings = drop_shadowed_appledouble_siblings(list(siblings))
main: dict[str, list] = {}
all_mmproj = mmproj_siblings(siblings)
@@ -242,24 +241,22 @@ def build_gguf_variant_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]:
name = _gguf_rfilename(sibling)
if name is None:
continue
- # Companions are folded into every plan below; keep them out of the
- # quant grouping so a drafter never lands in a variant's main files
- # (the root mtp-*.gguf carries a quant label, e.g. Q8_0).
- # An imatrix leaves entirely rather than joining companions_expected: no
- # variant needs llama-quantize's calibration data downloaded.
+ # Keep companions out of the quant grouping so a drafter never lands in a variant's main files: the
+ # root mtp-*.gguf carries a quant label.
+ # An imatrix leaves entirely rather than joining companions_expected: no variant needs llama-
+ # quantize's calibration data downloaded.
if is_mmproj_filename(name) or is_mtp_drafter_path(name) or is_imatrix_filename(name):
continue
quant = gguf_variant_key(name).lower()
- # The endian predicate reads a quant TOKEN -- it decides whether the quant came from the
- # parent directory only -- so a qualified key would make it misread the path and drop the
- # file from every plan.
+ # The endian predicate reads a quant TOKEN, so a qualified key would make it misread the path and
+ # drop the file from every plan.
if is_big_endian_gguf_path(name, extract_quant_label(name)):
continue
main.setdefault(quant, []).append(sibling)
plans: dict[str, GgufVariantPlan] = {}
- # Every weight in the listing, so the ranking can tell a sidecar naming a
- # neighbouring family from one naming this variant's.
+ # Every weight in the listing, so the ranking can tell a sidecar naming a neighbouring family from
+ # one naming this variant's.
all_weight_names = [
name.rsplit("/", 1)[-1]
for quant_siblings in main.values()
@@ -272,10 +269,9 @@ def build_gguf_variant_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]:
for sibling in target_main_siblings
if (file := expected_file_from_sibling(sibling)) is not None
)
- # Per variant, unlike mmproj and the MTP drafter: ranked against the weight
- # being fetched, so a multi-family repo does not hand B the drafter naming A.
- # Against the family plan_from_expected_files KEEPS, not the listing's first,
- # or a two-family variant key pairs the discarded one's sidecar.
+ # Per variant, unlike mmproj and the MTP drafter: ranked against the weight being fetched, and
+ # against the family plan_from_expected_files KEEPS, or a two-family variant key pairs the wrong
+ # sidecar.
kept_main = _one_shard_family(main_expected)
target_weight_name = (
min(file.path for file in kept_main).rsplit("/", 1)[-1] if kept_main else None
@@ -316,8 +312,8 @@ def plan_for_variant(plans: dict[str, GgufVariantPlan], variant: str) -> Optiona
exact = plans.get(wanted)
if exact is not None:
return exact
- # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant
- # names both partitions, and picking either would load a different task.
+ # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant names
+ # both partitions, and picking either would load a different task.
matches = [key for key in plans if "/" in key and bare_quant_alias(key).lower() == wanted]
return plans[matches[0]] if len(matches) == 1 else None
@@ -354,15 +350,13 @@ def plan_from_expected_files(
expected = tuple(expected_files)
all_main = tuple(file for file in expected if is_main_gguf_variant_path(file.path, variant))
main_files = _one_shard_family(all_main)
- # A discarded family has to leave the plan ENTIRELY, not just its main files. It is
- # target_filenames, required_hashes and download_size_bytes that the worker fetches and the
- # manifest checks against; leaving the copy there downloaded it, then reclaim deleted it as
- # not-ours (it is absent from main_hashes) and the job reported partial and fetched it again.
+ # A discarded family has to leave the plan ENTIRELY: target_filenames, required_hashes and
+ # download_size_bytes are what the worker fetches, so leaving the copy there downloaded it, then
+ # reclaim deleted it as not-ours (absent from main_hashes) and the job fetched it again.
kept = {file.path for file in main_files}
expected = tuple(file for file in expected if file not in all_main or file.path in kept)
companion_files = tuple(file for file in expected if is_companion_gguf_path(file.path))
- # Manifest-resume fallback for the mmproj fields below: companion_files
- # also holds the MTP drafter, so keep an mmproj-only view.
+ # companion_files also holds the MTP drafter, so keep an mmproj-only view for the manifest-resume fallback.
mmproj_files = tuple(file for file in companion_files if is_mmproj_filename(file.path))
main_hashes = frozenset(file.sha256 for file in main_files if file.sha256)
companion_hashes = frozenset(file.sha256 for file in companion_files if file.sha256)
diff --git a/studio/backend/hub/utils/hf_cache_state.py b/studio/backend/hub/utils/hf_cache_state.py
index 2f3cf101810..0042b9cb961 100644
--- a/studio/backend/hub/utils/hf_cache_state.py
+++ b/studio/backend/hub/utils/hf_cache_state.py
@@ -18,9 +18,8 @@ EXIT_CANCELLED = 130
TRANSPORT_HTTP = "http"
TRANSPORT_XET = "xet"
VALID_TRANSPORTS = frozenset({TRANSPORT_HTTP, TRANSPORT_XET})
-# A *request* preference, deliberately NOT in VALID_TRANSPORTS: "auto" is resolved to a real
-# transport before anything is spawned, and the on-disk .transport marker must keep naming the
-# writer that produced a partial, or a resume picks the wrong strategy.
+# A request preference, deliberately NOT in VALID_TRANSPORTS: the on-disk .transport marker must
+# keep naming the writer that produced a partial, or a resume picks the wrong strategy.
TRANSPORT_AUTO = "auto"
VALID_TRANSPORT_MODES = frozenset({TRANSPORT_HTTP, TRANSPORT_XET, TRANSPORT_AUTO})
TRANSPORT_MARKER_NAME = ".transport"
@@ -51,10 +50,8 @@ def incomplete_blob_hash(name: str) -> Optional[str]:
# The last huggingface_hub line whose partials a later attempt can append to.
_LAST_RESUMABLE_PARTIAL_VERSION = (1, 17)
-# How long a partial must sit untouched before it reads as abandoned rather than in flight.
-# huggingface_hub writes to one continuously, so anything still advancing has a live writer --
-# possibly a client in another process that no registry here can see. Shared by the sweep that
-# deletes abandoned partials and the progress scan that must not report one as current.
+# huggingface_hub writes to a partial continuously, so anything still advancing has a live writer,
+# possibly a client in another process no registry here can see.
ABANDONED_PARTIAL_SECONDS = 120
@@ -146,8 +143,8 @@ def blob_download_lock_held(entry: Path, blob_hash: str) -> bool:
"""
lock_path = entry.parent / ".locks" / entry.name / f"{blob_hash}.lock"
if not lock_path.exists():
- # hf creates the lock file before taking the lock, so no file means no writer. It is
- # also the answer for a SoftFileLock, whose file IS the lock (see below).
+ # hf creates the lock file before taking the lock, so no file means no writer; it is also the answer
+ # for a SoftFileLock, whose file IS the lock.
return False
try:
from filelock import FileLock, Timeout
@@ -159,14 +156,10 @@ def blob_download_lock_held(entry: Path, blob_hash: str) -> bool:
except Timeout:
return True
except Exception: # noqa: BLE001 - deliberately broad, see below
- # A filesystem without flock raises NotImplementedError here, and upstream's
- # WeakFileLock answers it by retrying as a SoftFileLock (huggingface_hub
- # utils/_fixes.py). Retrying is pointless for a PROBE: a soft lock is its file, and we
- # only reach this line because that file exists, so the soft answer is "held" too.
- # What matters is that the exception does not escape -- it used to travel out through
- # the purge and fail the download on every retry. Any other unprobeable error answers
- # the same way, because the caller's remaining guard is a staleness check that an
- # ownership claim may skip, and a wrong "free" there deletes a live writer's file.
+ # A filesystem without flock raises NotImplementedError, which upstream answers by retrying as a
+ # SoftFileLock; retrying is pointless for a PROBE, since we only reach here because the file
+ # exists. What matters is that the exception does not escape: a wrong "free" deletes a live
+ # writer's file.
return True
@@ -248,10 +241,9 @@ def hf_cache_roots(scan_errors: Optional[list] = None) -> list[Path]:
try:
key = str(path.resolve())
except OSError as exc:
- # A root that stats but will not resolve -- an intermittent network mount, a
- # Windows reparse point -- is a root we could not read, not a root that is gone.
- # Dropping it silently let the progress scan answer "measured, no cache", which
- # hydration acts on by retiring a download whose files may be entirely intact.
+ # A root that stats but will not resolve is a root we could not read, not one that is gone:
+ # dropping it silently let the progress scan answer "measured, no cache", which retires a download
+ # whose files may be intact.
if scan_errors is not None:
scan_errors.append(exc)
return
@@ -305,11 +297,9 @@ def blob_bytes_present(path: Path) -> int:
blocks = getattr(st, "st_blocks", None)
if blocks is not None and blocks > 0:
return min(blocks * 512, st.st_size)
- # A present zero is not a missing field. A parallel writer that sets the partial to its
- # final length before its first chunk lands sits exactly here, and reading st_size then
- # says "0 B left" on a download that has transferred nothing. The zero alone is not enough
- # to act on -- a mount that never populates st_blocks looks the same -- so confirm the
- # emptiness directly before believing it.
+ # A present zero is not a missing field: a parallel writer that sets the partial to its final
+ # length before its first chunk lands sits exactly here, and a mount that never populates
+ # st_blocks looks the same, so confirm the emptiness directly before believing st_size.
if blocks == 0 and _holds_no_data(path):
return 0
if sys.platform == "win32":
@@ -481,8 +471,7 @@ def latest_snapshot_from_cache_path(
try:
def has_metadata(path: Path) -> bool:
- # required_groups is an AND of ORs: the snapshot must carry at least one file from every
- # group. That is what "loadable" means: metadata alone or weights alone is not enough.
+ # required_groups is an AND of ORs: "loadable" means metadata alone or weights alone is not enough.
for group in required_groups:
if not any((path / name).is_file() for name in group):
return False
@@ -619,7 +608,7 @@ def iter_active_repo_cache_dirs(
for entry in root.iterdir():
if entry.name.lower() == target:
yield entry
- except OSError as exc: # see iter_repo_cache_dirs on why the caller may want this
+ except OSError as exc:
if scan_errors is not None:
scan_errors.append(exc)
return
@@ -633,22 +622,17 @@ def preferred_repo_cache_dirs(
active_root: Optional[Path] = None,
scan_errors: Optional[list] = None,
) -> list[Path]:
- # iter_active_repo_cache_dirs already matches case-insensitively, so a repo
- # dir the active root does hold is found whatever its casing; the canonical
- # name below is only ever a placeholder for one that is not there yet.
+ # iter_active_repo_cache_dirs already matches case-insensitively, so the canonical name below is
+ # only ever a placeholder for a repo dir that is not there yet.
active_entries = list(
iter_active_repo_cache_dirs(repo_type, repo_id, root = active_root, scan_errors = scan_errors)
)
if active_entries:
return active_entries
if force_active:
- # A running or cancelling job writes into the active root and nowhere
- # else, so its progress may only ever be read from there. hf_cache_root
- # returns None for a root that is not a directory yet -- the first
- # download into a freshly configured cache creates it -- and falling
- # through from there would read a previous cache's completed copy as
- # this run's progress and finalize a job that has not written a byte.
- # Name the directory this run will create instead.
+ # A running or cancelling job writes into the active root and nowhere else, so its progress may
+ # only be read from there: hf_cache_root returns None for a root not yet created, and falling
+ # through would read a previous cache's completed copy as this run's progress.
root = hf_cache_root(root = active_root) or active_root or _configured_hub_cache()
if root is not None:
canonical = repo_cache_dir_name(repo_type, repo_id)
@@ -657,9 +641,8 @@ def preferred_repo_cache_dirs(
def _configured_hub_cache() -> Optional[Path]:
- # Path()-wrapped: the setting is typed Path, but a caller (or a test) that
- # hands back a str would otherwise turn `root / name` above into a TypeError
- # that surfaces as an empty progress reading -- the very card being fixed.
+ # Path()-wrapped: the setting is typed Path, but a caller handing back a str would turn `root /
+ # name` into a TypeError that surfaces as an empty progress reading.
try:
from utils.hf_cache_settings import get_hf_cache_paths
configured = get_hf_cache_paths().hub_cache
@@ -833,9 +816,8 @@ def with_load_subdirs(model_name: str, names: tuple[str, ...]) -> tuple[str, ...
from utils.security import security_load_subdirs
subdirs = security_load_subdirs(model_name, local_files_only = True)
except Exception:
- # Degrading to root-only is fail-closed at every caller, so nothing is wrongly accepted. But a
- # real cache permission or corruption fault then reaches the user as "your cached model isn't
- # cached" with no clue why, and four sites now share this handler.
+ # Degrading to root-only is fail-closed at every caller, but a real cache permission or corruption
+ # fault then reaches the user as "your cached model isn't cached" with no clue why.
from loggers import get_logger
get_logger(__name__).debug(
"Load-subdir detection failed for %s; using root only.",
diff --git a/studio/backend/hub/utils/inventory_scan.py b/studio/backend/hub/utils/inventory_scan.py
index e0fca5a9a1b..4eb6ab87239 100644
--- a/studio/backend/hub/utils/inventory_scan.py
+++ b/studio/backend/hub/utils/inventory_scan.py
@@ -49,9 +49,8 @@ from hub.utils.hf_cache_state import (
)
from utils.paths.path_utils import drop_appledouble_metadata, is_appledouble_metadata
-# Inventory is invalidated explicitly on every app-driven cache mutation, so
-# this TTL only bounds staleness from out-of-band edits while skipping re-walks
-# on rapid UI navigation.
+# Inventory is invalidated explicitly on every app-driven cache mutation, so this TTL only bounds
+# staleness from out-of-band edits.
_HF_CACHE_SCANS_TTL_SECONDS = 15.0
_GGUF_SPLIT_RE = re.compile(r"-(\d{3,})-of-(\d{3,})(?=\.gguf$)", re.IGNORECASE)
# transformers shard naming: each shard names the set's total.
@@ -70,9 +69,8 @@ class _HfCacheScanFlight:
_hf_cache_scans_flight: Optional[_HfCacheScanFlight] = None
_hf_cache_scans_result: Optional[list] = None
_hf_cache_scans_cached_at: float = 0.0
-# Bumped on every invalidation. A scan tags itself with the epoch it began
-# under; an invalidation mid-scan changes the epoch so the in-flight result is
-# neither cached nor served to callers that arrived after the mutation.
+# A scan tags itself with the epoch it began under, so an invalidation mid-scan makes the in-flight
+# result neither cached nor served to callers that arrived after the mutation.
_hf_cache_scans_epoch: int = 0
_T = TypeVar("_T")
@@ -123,9 +121,8 @@ def all_hf_cache_scans() -> list:
return list(_hf_cache_scans_result)
start_epoch = _hf_cache_scans_epoch
flight = _hf_cache_scans_flight
- # Only coalesce onto an in-flight scan from the current epoch; one that
- # began before an intervening invalidation is superseded so
- # post-mutation callers never receive pre-mutation data.
+ # Only coalesce onto an in-flight scan from the current epoch, so post-mutation callers never
+ # receive pre-mutation data.
if flight is None or flight.epoch != start_epoch:
flight = _HfCacheScanFlight(event = threading.Event(), epoch = start_epoch)
_hf_cache_scans_flight = flight
@@ -183,7 +180,7 @@ _HF_REPO_TYPES = frozenset({"model", "dataset", "space"})
# Mirrors huggingface_hub's Cached{File,Revision,Repo}Info field-for-field; frozen because
-# HFCacheInfo.delete_revisions() set-diffs ``revisions``.
+# HFCacheInfo.delete_revisions() set-diffs revisions.
@dataclass(frozen = True)
class _RecoveredFileInfo:
file_name: str
@@ -723,8 +720,8 @@ def _repo_cache_dir_has_snapshot_legacy_partial(
) -> bool:
if _repo_cache_dir_has_non_gguf_broken_snapshot_symlinks(repo_cache_dir, snapshot_dir):
return True
- # ``.incomplete`` blobs carry no revision, so they need attributing. Judged on this row's
- # weights alone: a torn quant beside them is another row's payload, not this one's.
+ # ".incomplete" blobs carry no revision, so they need attributing; judged on this row's weights
+ # alone, since a torn quant beside them is another row's payload.
if snapshot_dir is not None and not _repo_signal_applies_to_snapshot(
repo_cache_dir, snapshot_dir, quants = False
):
@@ -797,15 +794,13 @@ def _completed_gguf_variants(snapshot_dir: Optional[Path]) -> set[str]:
or is_imatrix_filename(rel)
):
continue
- # Metadata vouching for a quant marks a torn snapshot ready: a set whose sidecars are
- # all present but whose weights are not answers the shard count exactly as the real
- # files would, in its own family and under their quant.
+ # Metadata vouching for a quant marks a torn snapshot ready: a set whose sidecars are all present
+ # answers the shard count exactly as the real files would.
if is_appledouble_metadata(path):
continue
quant = gguf_variant_key(rel)
- # Mirror the lister: a big-endian build is never offered, so it cannot vouch for the
- # quant. Judged with the loader's label, since the two extractors disagree on
- # F16-be-checkpoint-Q4_K_M and this file must not mark Q4_K_M complete.
+ # A big-endian build is never offered, so it cannot vouch for the quant; judged with the loader's
+ # label, since the two extractors disagree on F16-be-checkpoint-Q4_K_M.
from utils.models.model_config import _extract_quant_label as _loader_quant
if is_big_endian_gguf_path(rel, _loader_quant(rel)):
@@ -1090,12 +1085,12 @@ def _snapshot_payload(snapshot_dir: Path) -> Optional[_SnapshotPayload]:
if _required_config_is_unreadable(snapshot_dir / config_name, config_empty):
unreadable.update(formats)
model_format = _classify_non_gguf_model_format(**flags, trusted_hf_cache_repo = False)
- # from_pretrained never globs, so shards with no index are invisible and neither serve nor veto.
- # An unusable index is picked and failed on instead.
+ # from_pretrained never globs, so shards with no index are invisible and neither serve nor veto; an
+ # unusable index is picked and failed on instead.
unloadable: set = set()
invisible: set = set()
- # Selected by its own name, so it counts even when the walk grouped no shard of it, and it
- # carries its own paths: every weight_map entry resolves against the index.
+ # Selected by its own name, so it counts even when the walk grouped no shard of it, and every
+ # weight_map entry resolves against the index.
root_indexes: set[str] = set()
unusable_root_indexes: set[str] = set()
for suffix, canonical in _LOADER_WEIGHT_NAMES["base"].items():
@@ -1153,9 +1148,8 @@ def _index_cannot_serve_its_shards(index_path: Path, family_files: set[str]) ->
with index_path.open(encoding = "utf-8") as handle:
index = json.load(handle)
except (OSError, UnicodeDecodeError, ValueError, RecursionError):
- # RecursionError (deeply nested json) escapes every caller's fail-open guard. The loader
- # parses this index with the same json module, so one too deep to parse there cannot serve
- # its shards here either.
+ # RecursionError escapes every caller's fail-open guard, and the loader parses this index with the
+ # same json module, so one too deep to parse there cannot serve its shards here either.
return True
weight_map = index.get("weight_map") if isinstance(index, dict) else None
if not isinstance(weight_map, dict) or not weight_map:
@@ -1166,16 +1160,15 @@ def _index_cannot_serve_its_shards(index_path: Path, family_files: set[str]) ->
if not isinstance(shard, str) or not shard:
return True
parts = PurePosixPath(shard.replace("\\", "/"))
- # is_absolute() is per flavour: PurePosixPath reads "C:/weights/x.safetensors" as a relative
- # subdirectory called "C:", but the join below is a platform Path, so on Windows that name
- # replaces the index directory outright. A drive is never part of a shard name.
+ # is_absolute() is per flavour: PurePosixPath reads "C:/weights/x.safetensors" as a relative "C:"
+ # subdirectory, but the join below is a platform Path, so on Windows that name replaces the
+ # index directory outright.
windows = PureWindowsPath(shard)
if parts.is_absolute() or ".." in parts.parts or windows.is_absolute() or windows.drive:
return True
shards.add(parts)
named = {shard.name for shard in shards}
- # Coverage matters only for the family this index describes: one it names nothing of is stale
- # content beside it, which the loader never reads because it opens weight_map and nothing else.
+ # Coverage matters only for the family this index describes: the loader opens weight_map and nothing else.
if not family_files <= named and not named.isdisjoint(family_files):
return True
for shard in shards:
@@ -1187,7 +1180,7 @@ def _index_cannot_serve_its_shards(index_path: Path, family_files: set[str]) ->
except (OSError, ValueError):
return True
# A shard names its own total, so an index listing one of a set has to list the whole set: the
- # loader opens exactly what is mapped and silently drops whatever the map leaves out.
+ # loader silently drops whatever the map leaves out.
declared: dict[tuple[str, str, int], set[int]] = {}
for shard in shards:
match = _WEIGHT_SHARD_RE.search(shard.name)
@@ -1225,15 +1218,14 @@ def _snapshot_lacks_a_complete_weight_family(snapshot_dir: Path) -> bool:
# The loader opens this name first and finds it empty. Same exemption as below.
return kind == wanted or wanted not in payload.ungrouped
if canonical_empty is False:
- # Only the row's own kind proves it loads, and it vetoes nothing once that kind's
- # payload is here but names no family.
+ # Only the row's own kind proves it loads, and it vetoes nothing once that kind's payload is here
+ # but names no family.
return kind != wanted and wanted not in payload.ungrouped
- # from_pretrained reads the snapshot root, so only families named there are judged; a
- # subdirectory layout is carried by ungrouped instead.
+ # from_pretrained reads the snapshot root, so only families named there are judged; a subdirectory
+ # layout is carried by ungrouped instead.
if kind == "base" and suffix in payload.root_indexes:
- # Selected and loaded for exactly what it names, wherever those paths point, so
- # judge its contents rather than this walk's families: a numbered file it names
- # nothing of is never read. The next name is never tried.
+ # Selected and loaded for exactly what it names, wherever those paths point, so judge its contents
+ # rather than this walk's families; the next name is never tried.
if suffix in payload.unusable_root_indexes:
return kind == wanted or wanted not in payload.ungrouped
return kind != wanted and wanted not in payload.ungrouped
@@ -1586,16 +1578,16 @@ def _current_revisions(repo_info):
return revisions
-# One definition of "the denoiser is on disk", shared by the repo-wide and the snapshot-scoped
-# check so the two cannot drift into disagreeing about the same directory.
+# One definition of "the denoiser is on disk", shared by the repo-wide and snapshot-scoped checks so
+# the two cannot drift into disagreeing about the same directory.
_DENOISER_DIRS = ("transformer", "unet")
_DENOISER_WEIGHT_SUFFIXES = (".safetensors", ".bin")
-# The two names a default load can end up opening at the component root: the safetensors one
-# _get_model_file is asked for first, and the .bin the pickle fallback under it drops to.
+# The two names a default load can open at the component root: the safetensors one _get_model_file
+# asks for first, and the .bin its pickle fallback drops to.
_DEFAULT_DENOISER_WEIGHTS = frozenset(
f"diffusion_pytorch_model{suffix}" for suffix in _DENOISER_WEIGHT_SUFFIXES
)
-# The one sharded index a default load resolves: use_safetensors unset coerces to True, and
+# The one sharded index a default load resolves: use_safetensors unset coerces to True and
# _fetch_index_file then builds only _add_variant(SAFE_WEIGHTS_INDEX_NAME, variant), so with
# variant unset this exact name. Our load path passes neither (core/inference/{diffusion,video}.py).
_SELECTED_DENOISER_INDEX = "diffusion_pytorch_model.safetensors.index.json"
@@ -1649,9 +1641,9 @@ def _manifest_denoiser_components(snapshot: Path) -> Optional[tuple[str, ...]]:
name = key.lower()
if name != "unet" and "transformer" not in name:
continue
- # A component is a [library, class] pair keyed by its directory; [null, null] means
- # deliberately absent (Wan 2.2's 5B transformer_2). Anything else names no directory to
- # infer (ACE-STEP maps "transformer" to a config dict), so leave it to the loader.
+ # A component is a [library, class] pair keyed by its directory; [null, null] means deliberately
+ # absent (Wan 2.2's 5B transformer_2), and anything else names no directory to infer (ACE-STEP
+ # maps "transformer" to a config dict).
if not isinstance(value, (list, tuple)) or not any(v for v in value):
continue
found.append(key)
@@ -1668,7 +1660,6 @@ def _denoiser_index_shards(index: Path) -> Optional[set[str]]:
with index.open("r", encoding = "utf-8") as fh:
weight_map = json.load(fh).get("weight_map")
except (OSError, ValueError, AttributeError, RecursionError):
- # RecursionError (deeply nested json) would escape the caller's fail-open guard.
return None
if not isinstance(weight_map, dict):
return None
@@ -1704,28 +1695,25 @@ def _component_weights_complete(component: Path) -> bool:
So with no selected index there are exactly two names left, the pair ``_get_model_file`` is
handed, and every other weight in the directory is one ``from_pretrained`` never resolves.
"""
- # iterdir() raises on an unreadable dir, reaching the caller's fail-open guard; glob() would
- # swallow that OSError and read as "no weights".
+ # iterdir() raises on an unreadable dir, reaching the caller's fail-open guard; glob() would swallow
+ # that OSError and read as "no weights".
next(component.iterdir(), None)
- # Existence alone makes the component sharded (``is_sharded`` comes from ``is_file()``), so an
- # index we cannot read is not no-evidence here, it IS the failure. ``is_file()`` is also the
- # loader's own test, so a dangling snapshot symlink is absent to both.
+ # Existence alone makes the component sharded (is_sharded comes from is_file()), so an index we
+ # cannot read IS the failure; is_file() is the loader's own test too.
selected = component / _SELECTED_DENOISER_INDEX
if selected.is_file():
if _index_cannot_serve_its_shards(selected, set()):
return False
- # The loader opens exactly what ``weight_map`` lists and reads each one as a checkpoint,
- # so a map naming something that is not a weight file -- a ``config.json`` a corrupt fetch
- # left behind -- fails at load however present that file is.
+ # The loader opens exactly what weight_map lists and reads each one as a checkpoint, so a map naming
+ # a config.json a corrupt fetch left behind fails at load however present that file is.
shards = _denoiser_index_shards(selected)
return bool(shards) and all(
name.lower().endswith(_DENOISER_WEIGHT_SUFFIXES) for name in shards
)
- # No index, so the component is not sharded to the loader either, and ``_get_model_file`` is
- # asked for the safetensors default and then the ``.bin`` under it and opens nothing else. A
- # weight under any other name is one ``from_pretrained`` never resolves: a numbered shard is
- # reachable only THROUGH an index, a dtype twin only under a matching ``variant``, and a
- # ``model.safetensors`` or adapter sidecar never at all.
+ # No index means the component is not sharded to the loader either, and _get_model_file opens only
+ # the safetensors default and the .bin under it: a numbered shard is reachable only THROUGH an
+ # index, a dtype twin only under a matching variant, and a model.safetensors or adapter sidecar
+ # never at all.
return any((component / name).is_file() for name in _DEFAULT_DENOISER_WEIGHTS)
@@ -1749,14 +1737,14 @@ def snapshot_pipeline_missing_denoiser(snapshot: Optional[Path]) -> bool:
root = Path(snapshot)
declared = _manifest_denoiser_components(root)
if declared is not None:
- # all(()) is True: a manifest declaring no denoiser has none to prove absent, so it
- # reads complete rather than being hunted for one it never had.
+ # all(()) is True: a manifest declaring no denoiser has none to prove absent, so it reads complete
+ # rather than being hunted for one it never had.
return not all(
(root / name).is_dir() and _component_weights_complete(root / name)
for name in declared
)
- # Unreadable manifest: either fixed name will do, since a UNet pipeline has no
- # transformer/ and a DiT one no unet/.
+ # Unreadable manifest: either fixed name will do, since a UNet pipeline has no transformer/ and a
+ # DiT one no unet/.
return not any(
(root / name).is_dir() and _component_weights_complete(root / name)
for name in _DENOISER_DIRS
@@ -2041,9 +2029,8 @@ def partial_resume_available(
if partial_transport_for(repo_type, repo_id, variant, repo_cache_dir) != "http":
return False
- # Same root the transport was read from. A row can be displayed from a remembered, legacy
- # or custom cache, and both halves have to answer about THAT directory: the active root
- # neither holds its partials nor shares its manifest scope.
+ # Same root the transport was read from: a row can be displayed from a remembered, legacy or custom
+ # cache, and the active root neither holds its partials nor shares its manifest scope.
return download_registry.is_resumable_partial(
repo_type,
repo_id,
diff --git a/studio/backend/hub/utils/paths.py b/studio/backend/hub/utils/paths.py
index d36a0e8468a..dd887575092 100644
--- a/studio/backend/hub/utils/paths.py
+++ b/studio/backend/hub/utils/paths.py
@@ -18,8 +18,8 @@ from loggers import get_logger
from utils.paths import path_utils as _path_utils
from utils.paths.path_utils import wsl_automount_root
-# One policy, defined in utils.paths.storage_roots. The copy that used to live here
-# drifted: a BOM'd settings.json was honoured by one side and dropped by the other (#9748).
+# One policy, defined in utils.paths.storage_roots: the copy that used to live here drifted, and a
+# BOM'd settings.json was honoured by one side and dropped by the other (#9748).
from utils.paths.storage_roots import (
lmstudio_model_dirs,
ollama_model_dirs,
@@ -111,8 +111,7 @@ def hf_default_cache_dir() -> Path:
return Path.home() / ".cache" / "huggingface" / "hub"
-# normalize_path reads these at call time and tests set them
-# (test_gguf_variants_local_resolution), so they stay attributes of this module.
+# normalize_path reads these at call time and tests set them, so they stay attributes of this module.
_IS_WSL = _path_utils._IS_WSL
_WSL_AUTOMOUNT_ROOT = wsl_automount_root()
@@ -165,8 +164,8 @@ def is_valid_repo_id(repo_id: str) -> bool:
segments = repo_id.split("/")
if len(segments) not in (1, 2):
return False
- # Match huggingface_hub.validate_repo_id: the 96-char limit applies per segment (repo name /
- # namespace), not to the whole "namespace/repo_name" string.
+ # Match huggingface_hub.validate_repo_id: the 96-char limit applies per segment, not to the whole
+ # "namespace/repo_name" string.
return all(
segment not in ("", ".", "..")
and len(segment) <= _MAX_REPO_ID_LENGTH
@@ -196,7 +195,7 @@ def is_valid_gguf_variant(variant: str) -> bool:
return all(segment not in ("", ".", "..") for segment in normalized.split("/"))
-# Per-process memo for resolve_cached_repo_id_case. Bounded LRU so a long-lived process can't
+# Per-process memo for resolve_cached_repo_id_case. Bounded LRU so a long-lived process cannot
# grow it without limit; evicted cold entries simply recompute.
_CACHE_CASE_RESOLUTION_MEMO_MAX = 512
_CACHE_CASE_RESOLUTION_MEMO: "OrderedDict[tuple[str, str], str]" = OrderedDict()
@@ -288,8 +287,8 @@ def resolve_dataset_path(path_value: str) -> Path:
raw = str(path_value or "").strip()
if "\x00" in raw:
raise ValueError("dataset path may not contain null bytes")
- # Normalize first so Windows/UNC and backslash paths resolve like the rest of the Hub path
- # layer, and a backslashed '..' is caught by the traversal guard below.
+ # Normalize first so Windows/UNC and backslash paths resolve like the rest of the Hub path layer,
+ # and a backslashed ".." is caught by the traversal guard below.
normalized = normalize_path(raw)
path = Path(normalized).expanduser()
if ".." in path.parts:
@@ -375,8 +374,8 @@ def resolve_cached_repo_id_case(
continue
if entry.name.lower() != expected_lower:
continue
- # The lowercased full-name match already proves the prefix matches; a case-sensitive
- # startswith would reject a mixed-case imported dir such as Models--Org--Repo.
+ # The lowercased full-name match already proves the prefix matches; a case-sensitive startswith
+ # would reject a mixed-case imported dir such as Models--Org--Repo.
repo_part = entry.name[len(prefix) :]
if not repo_part:
continue
diff --git a/studio/backend/hub/utils/resumable_partials.py b/studio/backend/hub/utils/resumable_partials.py
index f7a62c6de40..52a790bd3bf 100644
--- a/studio/backend/hub/utils/resumable_partials.py
+++ b/studio/backend/hub/utils/resumable_partials.py
@@ -58,16 +58,14 @@ LAST_STOCK_RESUMABLE_VERSION = (1, 17)
# The newest major whose internals this has been read against. A 2.x is not assumed to look alike.
MAX_SUPPORTED_MAJOR = 1
-# What flock reports when another holder has the lock, and nothing else. EACCES belongs to fcntl,
-# not flock; ENOLCK and EOPNOTSUPP mean this filesystem cannot lock at all.
+# What flock reports when another holder has the lock, and nothing else: EACCES belongs to fcntl,
+# and ENOLCK / EOPNOTSUPP mean this filesystem cannot lock at all.
_CONTENDED = frozenset({errno.EWOULDBLOCK, errno.EAGAIN})
-# Filesystems backed by storage attached to this host, so a lock taken here is the only lock.
# An allowlist rather than a list of network types: a FUSE mount reports whatever name its daemon
-# chose, and unless it negotiates FUSE_FLOCK_LOCKS the kernel answers flock locally (libfuse
-# fuse_lowlevel.h), so fuse.rclone or fuse.s3fs over shared object storage passes the probe while
-# excluding nobody. Naming the ones we know instead means an unrecognised or blank type keeps the
-# stock writer rather than silently re-enabling a shared one.
+# chose and, unless it negotiates FUSE_FLOCK_LOCKS, the kernel answers flock locally, so
+# fuse.rclone or fuse.s3fs over shared object storage would pass a network-name test. An
+# unrecognised or blank type keeps the stock writer.
_LOCAL_FSTYPES = frozenset(
{
# Linux
@@ -156,8 +154,8 @@ def _probe_dir(hub_cache: Optional[Path | str] = None) -> Optional[Path]:
logger.debug("resumable partials: no hub cache to probe (%s)", exc)
return None
if hub_cache is not None:
- # Asked about a named root, so only report on one that is there. Creating it would
- # resurrect a cache the user detached, and an absent root holds no partials to judge.
+ # Asked about a named root, so only report on one that is there: creating it would resurrect a cache
+ # the user detached, and an absent root holds no partials to judge.
return root if root.is_dir() else None
try:
root.mkdir(parents = True, exist_ok = True)
@@ -205,7 +203,7 @@ def _filesystem_is_local_on(directory: str, device: int) -> bool:
"""
path = Path(directory).resolve()
if str(path).startswith("\\\\") or str(path).startswith("//"):
- return False # UNC share
+ return False
try:
table = _mounts()
except Exception as exc: # noqa: BLE001 - unreadable now does not mean unreadable next time
@@ -234,8 +232,8 @@ def _lock_is_honoured_at(directory: str) -> bool:
return _lock_is_honoured_on(directory, _device_at(directory))
-# Keyed on the directory and the device, so moving the cache, or swapping the mount under it,
-# re-probes instead of reusing a verdict about a filesystem that is no longer there.
+# Keyed on the directory and the device, so moving the cache or swapping the mount under it re-
+# probes instead of reusing a verdict about a filesystem that is gone.
@lru_cache(maxsize = 8)
def _lock_is_honoured_on(directory: str, device: int) -> bool:
"""Take the lock twice and require the second to be refused.
@@ -247,8 +245,8 @@ def _lock_is_honoured_on(directory: str, device: int) -> bool:
"""
import fcntl
- # A random, exclusively created file: the cache can be shared, and a predictable name there
- # lets another user pre-place a symlink that an unguarded open would follow and truncate.
+ # A random, exclusively created file: the cache can be shared, and a predictable name lets another
+ # user pre-place a symlink an unguarded open would follow and truncate.
try:
handle, name = tempfile.mkstemp(dir = directory, prefix = ".unsloth-flock-probe.")
except Exception as exc: # noqa: BLE001 - nowhere to probe now is not nowhere to probe later
@@ -300,16 +298,13 @@ def _exclusion_is_provable(hub_cache: Optional[Path | str] = None) -> bool:
try:
import fcntl # noqa: F401
except ImportError:
- # No fcntl on Windows, where huggingface_hub locks via msvcrt: mandatory rather than
- # advisory. A network share still cannot be spoken for, which the locality check catches.
- # Windows has no way to establish who owns a partial: os.stat reports st_uid 0 for every
- # file, and reading an ACL needs pywin32, which is not a dependency here. Without an owner
- # check, another account on a shared NTFS cache could leave a partial with a chosen prefix
- # and have the remaining range appended to it, which the size-only check downstream would
- # pass. So the shared name stays off here and Windows keeps the stock writer.
+ # No fcntl on Windows, where huggingface_hub locks via msvcrt, and Windows has no way to establish
+ # who owns a partial (os.stat reports st_uid 0 for every file and reading an ACL needs pywin32),
+ # so another account on a shared NTFS cache could leave a partial with a chosen prefix and have
+ # the remaining range appended to it, which the size-only check would pass.
return False
- # _ProbeUnavailable is deliberately not caught: a probe that could not run is not an answer,
- # and letting it out keeps any caller from caching one. Callers turn it into "not proven".
+ # _ProbeUnavailable is deliberately not caught: a probe that could not run is not an answer, and
+ # letting it out keeps any caller from caching one.
return _filesystem_is_local(str(directory)) and _lock_is_honoured_at(str(directory))
@@ -356,8 +351,7 @@ def _objection_to(descriptor: int) -> Optional[str]:
return "hard linked from elsewhere"
euid = getattr(os, "geteuid", None)
if euid is None:
- # No owner to compare against, so nothing here can be vouched for. Reachable only if the
- # writer is ever enabled without geteuid; _exclusion_is_provable refuses that today.
+ # No owner to compare against, so nothing here can be vouched for.
return "on a platform where ownership cannot be established"
if info.st_uid != euid():
return "owned by another user"
@@ -419,10 +413,8 @@ def _open_stable_partial(path: Path) -> Optional[Any]:
if exc.errno in (errno.ELOOP, errno.EMLINK):
objection = "a symlink"
else:
- # Anything else is a partial this account cannot use and must not judge: a
- # 0600 one left by another user answers EACCES, a directory in its place
- # EISDIR. Raising would fail every attempt at the blob for good, while the
- # stock writer creates a file of its own and does not need this one at all.
+ # Anything else is a partial this account cannot use and must not judge (another user's 0600 file
+ # answers EACCES, a directory EISDIR); raising would fail every attempt at the blob for good.
logger.warning(
"Cannot open the download partial at %s (%s); leaving it alone and "
"letting the stock writer fetch the file.",
@@ -451,8 +443,8 @@ def restore_resumable_partials() -> bool:
try:
permitted = can_restore_partials()
except _ProbeUnavailable as exc:
- # The worker calls this at import, so an escaping exception would take the whole download
- # process down instead of leaving it with the stock writer it can always fall back on.
+ # The worker calls this at import, so an escaping exception would take the whole download process
+ # down instead of leaving it the stock writer.
logger.debug("resumable partials: %s", exc)
return False
if not permitted:
@@ -478,8 +470,8 @@ def restore_resumable_partials() -> bool:
if destination_path.exists() and not force_download:
return
- # A XET-backed repo still comes down over HTTP when hf_xet is absent or disabled, so what
- # matters is whether XET will run, not whether its metadata exists. XET writes its own way.
+ # A XET-backed repo still comes down over HTTP when hf_xet is absent or disabled, so what matters is
+ # whether XET will run, not whether its metadata exists.
uses_xet = xet_file_data is not None and file_download.is_xet_available()
if force_download or uses_xet:
return stock(
@@ -512,10 +504,8 @@ def restore_resumable_partials() -> bool:
with opened as handle:
resume_size = handle.tell()
if expected_size is not None and resume_size > expected_size:
- # Longer than the file is supposed to be, so there is nothing to resume from: a
- # Range starting past the end answers 416 and would do so on every retry, where
- # the stock writer's fresh name recovers. Truncating through the handle we already
- # validated, rather than unlinking, keeps the name from being swapped underneath.
+ # Longer than the file is supposed to be, so there is nothing to resume from: a Range starting past
+ # the end answers 416 on every retry.
logger.warning(
"Restarting '%s': the partial holds %s bytes but the file is %s.",
filename,
@@ -543,9 +533,8 @@ def restore_resumable_partials() -> bool:
expected_size = expected_size,
tqdm_class = kwargs.get("tqdm_class"),
)
- # _chmod_and_move resolves the name again, so publish only if the name still holds the
- # file that was actually written. Otherwise another account could swap something in
- # after the last write and have it installed as the blob under a verified descriptor.
+ # _chmod_and_move resolves the name again, so publish only if the name still holds the file that was
+ # actually written; otherwise another account could swap something in after the last write.
if not _still_the_written_file(incomplete_path, written):
logger.warning(
"Not publishing '%s': the partial at %s was replaced while it was being written.",
diff --git a/studio/backend/hub/utils/snapshot_filters.py b/studio/backend/hub/utils/snapshot_filters.py
index 67f5b47bdcb..65ea85552f6 100644
--- a/studio/backend/hub/utils/snapshot_filters.py
+++ b/studio/backend/hub/utils/snapshot_filters.py
@@ -44,8 +44,8 @@ def _size(sibling) -> int:
def repo_ships_transformers_weights(filenames: Iterable[str]) -> bool:
- # A "._consolidated.safetensors" does not start with "consolidated", so it would answer yes
- # here and then have every consolidated* file stripped from the download.
+ # A "._consolidated.safetensors" does not start with "consolidated", so it would answer yes here and
+ # then have every consolidated* file stripped from the download.
for name in drop_shadowed_appledouble_names(list(filenames)):
base = name.rsplit("/", 1)[-1].lower()
if base.startswith("consolidated"):
@@ -94,9 +94,8 @@ def total_size_for_siblings(siblings: Iterable) -> int:
def blob_hashes_for_siblings(siblings: Iterable) -> frozenset[str]:
- # Blob filename == file etag (LFS sha256, else git blob id). Collecting both
- # lets progress count exactly this revision's files without summing stale
- # blobs from other revisions.
+ # Blob filename == file etag (LFS sha256, else git blob id), so collecting both lets progress count
+ # exactly this revision's files without summing stale blobs from other revisions.
hashes: set[str] = set()
for sibling in siblings:
sha = getattr(getattr(sibling, "lfs", None), "sha256", None)
diff --git a/studio/backend/hub/utils/state_dir.py b/studio/backend/hub/utils/state_dir.py
index 45cf9950048..35ad21e918f 100644
--- a/studio/backend/hub/utils/state_dir.py
+++ b/studio/backend/hub/utils/state_dir.py
@@ -57,8 +57,7 @@ _MAX_VARIANT_FRAGMENT_LENGTH = 64
_CACHE_SCOPE_DIGEST_LENGTH = 32
_HASH_PREFIXES = ("sha256-", "@sha256-")
_LEGACY_HASH_FRAGMENT = re.compile(r"sha256-[0-9a-f]{32}")
-# Either tag _variant_fragment stamps, so a reader can spot a digest handed to it
-# in place of a variant name.
+# Either tag _variant_fragment stamps, so a reader can spot a digest handed to it in place of a variant name.
_HASHED_FRAGMENT = re.compile(r"@?sha256-[0-9a-f]{32}")
@@ -91,9 +90,8 @@ def _subdir(name: str, *, create: bool = False) -> Optional[Path]:
def repo_cache_basename(repo_type: RepoType, repo_id: str) -> str:
- # Reject a bad repo_type at runtime: a wrong value would silently produce a
- # wrong filename and a misclassified scanner row (the Literal only guards
- # statically; dynamic/JSON-sourced values slip past it).
+ # Reject a bad repo_type at runtime: the Literal only guards statically and dynamic/JSON-sourced
+ # values slip past it, silently producing a wrong filename and a misclassified scanner row.
if repo_type not in _VALID_REPO_TYPES:
raise ValueError(f"repo_type must be one of {_VALID_REPO_TYPES}, got {repo_type!r}")
return f"{repo_type}s--{repo_id.replace('/', '--')}".lower()
@@ -187,8 +185,8 @@ def _variant_fragment(
legacy_hash_key: bool = False,
) -> str:
normalized_variant = variant.strip().lower()
- # Double-hyphen variants can alias a repository component plus the
- # ``--variant--`` delimiter, so give them an injective hashed fragment.
+ # Double-hyphen variants can alias a repository component plus the --variant-- delimiter, so give
+ # them an injective hashed fragment.
if _SAFE_VARIANT_FRAGMENT.fullmatch(normalized_variant) and (
legacy_variant_key
or ("--" not in normalized_variant and not normalized_variant.startswith(_HASH_PREFIXES))
@@ -271,12 +269,9 @@ def normalize_hub_cache(hub_cache: str | Path) -> str:
try:
resolved = str(Path(hub_cache).expanduser().resolve(strict = False))
except (OSError, RuntimeError, ValueError):
- # Windows can refuse to resolve a path it can still open (OneDrive
- # placeholders, a locked junction). Degrade to the expanded spelling
- # rather than dropping the scope. It has to be exactly what
- # legacy_cache_scope_name below builds, or that read-side counterpart
- # would not recover the state this branch writes -- so expanduser here
- # too, and fall back again if even that is unavailable.
+ # Windows can refuse to resolve a path it can still open (OneDrive placeholders, a locked
+ # junction), so degrade to the expanded spelling; it has to be exactly what
+ # legacy_cache_scope_name builds, or the read side cannot recover this state.
try:
resolved = str(Path(hub_cache).expanduser())
except (OSError, RuntimeError, ValueError):
@@ -307,9 +302,8 @@ def legacy_cache_scope_name(hub_cache: str | Path) -> str:
try:
normalized = os.path.normcase(str(Path(hub_cache).expanduser()))
except (OSError, RuntimeError, ValueError):
- # Guarded like normalize_hub_cache, and for the same reason: this is now
- # fed the caller's raw spelling, so a homeless "~" that expanduser
- # refuses would otherwise escape a plain read as a RuntimeError.
+ # Guarded like normalize_hub_cache: this is fed the caller's raw spelling, so a homeless "~" that
+ # expanduser refuses would otherwise escape a plain read as a RuntimeError.
normalized = os.path.normcase(str(hub_cache))
return _cache_scope_digest(normalized)
@@ -337,9 +331,7 @@ def _cache_scope(
) -> Optional[Path]:
if hub_cache is None:
return parent
- # A precomputed scope skips re-deriving the digest -- and with it the
- # ``resolve`` inside it -- once per probe when a caller fans one cache path
- # out across the filename-migration spellings.
+ # A precomputed scope skips re-deriving the digest, and the resolve inside it, once per probe.
scoped = parent / (cache_scope or cache_scope_name(hub_cache))
if not create:
return scoped
diff --git a/studio/backend/hub/workers/hf_download.py b/studio/backend/hub/workers/hf_download.py
index b66c1e2249f..be44372e096 100644
--- a/studio/backend/hub/workers/hf_download.py
+++ b/studio/backend/hub/workers/hf_download.py
@@ -60,16 +60,16 @@ from hub.utils.gguf_plan import (
from hub.utils.state_dir import RepoType
from hub.utils.resumable_partials import restore_resumable_partials
-# Put huggingface_hub's 1.17 HTTP writer back where it is safe to. The SIGKILL then restart loop
-# documented above reads ``.incomplete`` for its resume offset, and 1.18+ leaves nothing to read.
+# Put huggingface_hub's 1.17 HTTP writer back: the SIGKILL then restart loop reads .incomplete for
+# its resume offset, and 1.18+ leaves nothing to read.
_PARTIALS_RESUMABLE = restore_resumable_partials()
# typing.Union, not `str | bool | None`: an alias is evaluated on import and PEP 604 raises below 3.10.
HfTokenArg = Union[str, bool, None]
-# Bound the metadata fetch so a stalled connection fails the worker (exit 1) instead of hanging
-# at 0%. The file download itself is governed by huggingface_hub's own timeout.
+# Bound the metadata fetch so a stalled connection fails the worker instead of hanging at 0%; the
+# file download itself is governed by huggingface_hub's own timeout.
_METADATA_REQUEST_TIMEOUT = 10.0
_METADATA_RETRY_TIMEOUT = 30.0
_METADATA_RETRY_DELAY = 1.0
@@ -154,10 +154,8 @@ def _parent_is_alive(parent_pid: int) -> bool:
def _terminate_orphaned_self() -> None:
- # Hard exit from the watchdog thread: a self-SIGTERM would be deferred while the main thread is
- # GIL-blocked in a C socket read. The partial .incomplete resumes byte-exact and marker/manifest
- # writes are atomic, so code 130 is safe. The diagnostic is best-effort: a dead parent's closed
- # stderr pipe can raise BrokenPipeError, which must never preempt the exit.
+ # Hard exit from the watchdog thread: a self-SIGTERM would be deferred while the main thread is GIL-
+ # blocked in a C socket read, and the partial resumes byte-exact with atomic marker writes.
try:
print(
"Parent process exited; stopping orphaned download worker.",
@@ -239,8 +237,8 @@ def _dataset_info_with_retry(repo_id: str, hf_token: str | None):
)
-# Tied to drain_stderr_excerpt's 500-byte head/tail window in the parent: listing every expected
-# file would blow past it and lose the diagnostic, so cap the preview.
+# Tied to drain_stderr_excerpt's 500-byte head/tail window: listing every expected file would blow
+# past it and lose the diagnostic.
_VERIFY_PATH_LIST_CAP = 10
@@ -554,9 +552,8 @@ def _download_snapshot(repo_id: str, hf_token: str | None, mode: str) -> None:
from hub.utils.download_registry import prepare_cache_for_transport
from hub.utils import download_manifest
- # One metadata fetch powers both the ignore-pattern decision (drop consolidated.* when
- # transformers weights exist) and the manifest's expected_files. A failure is non-fatal: fall
- # back to the legacy ignore set and skip the manifest, losing verification but not the download.
+ # One metadata fetch powers both the ignore-pattern decision and the manifest's expected_files; a
+ # failure is non-fatal and falls back to the legacy ignore set, losing verification only.
try:
info = _model_info_with_retry(repo_id, hf_token)
except Exception as e:
@@ -570,9 +567,8 @@ def _download_snapshot(repo_id: str, hf_token: str | None, mode: str) -> None:
download_manifest.clear_cancel_marker("model", repo_id, None)
if info is not None:
ignore_patterns, expected_files = _snapshot_download_plan(info)
- # Written for every transport. The manifest verifies the finalized files under snapshots/, which
- # both transports produce identically (XET also renames a full, correctly-sized blob into place).
- # XET's block-level dedup lives only in the chunk-cache, so per-file size verification is valid.
+ # The manifest verifies the finalized files under snapshots/, which both transports produce
+ # identically; XET's block-level dedup lives only in the chunk cache.
download_manifest.write_manifest("model", repo_id, None, expected_files, mode)
else:
ignore_patterns = list(SNAPSHOT_IGNORE_PATTERNS)
@@ -624,9 +620,9 @@ def _gguf_variant_target_plan(
raise RuntimeError(
f"Metadata unavailable while resolving GGUF variant '{variant}' " f"for {repo_id}"
) from e
- # plan_for_variant, not .get: a repo that files every variant under one shared container
- # qualifies every key, and a stored pin or an explicit repo:Q4_K_M then missed the map and
- # the worker exited with "No GGUF shards matching variant".
+ # plan_for_variant, not .get: a repo filing every variant under one shared container qualifies
+ # every key, so a stored pin or an explicit repo:Q4_K_M missed the map and the worker exited with
+ # "No GGUF shards matching variant".
return plan_for_variant(build_gguf_variant_plans(list(info.siblings)), variant)
@@ -662,8 +658,8 @@ def _download_gguf_variant(repo_id: str, variant: str, hf_token: str | None, mod
mode,
)
else:
- # Metadata unreachable (offline / gated / private). Resume the exact shards the original attempt
- # recorded so snapshot_download can range over the surviving .incomplete blobs.
+ # Metadata unreachable: resume the exact shards the original attempt recorded so snapshot_download
+ # can range over the surviving .incomplete blobs.
manifest = download_manifest.read_manifest("model", repo_id, variant)
if manifest is None or not manifest.expected_files:
print(
@@ -707,8 +703,8 @@ def _download_gguf_variant(repo_id: str, variant: str, hf_token: str | None, mod
"hashes; starting without partial cache reuse.",
file = sys.stderr,
)
- # Main quant blobs are owned by this variant (variant-scoped marker). The shared vision companion
- # (mmproj) has its own marker and is never purged while a concurrent peer is writing it.
+ # Main quant blobs are owned by this variant; the shared mmproj companion has its own marker and is
+ # never purged while a concurrent peer is writing it.
purged = prepare_cache_for_transport(
"model",
repo_id,
@@ -830,8 +826,8 @@ def _download_scoped_snapshot(
max_workers = 1,
)
if info is None:
- # With no metadata there is no manifest, so verification is a no-op, and snapshot_download RETURNS
- # AN EXISTING SNAPSHOT FOLDER when repo_info also fails -- flipping the job to complete with no weights.
+ # With no metadata there is no manifest, and snapshot_download RETURNS AN EXISTING SNAPSHOT FOLDER
+ # when repo_info also fails, flipping the job to complete with no weights.
root = Path(snapshot_path)
absent = tuple(f for f in files if not (root / f).exists())
if absent:
@@ -1002,9 +998,9 @@ def main() -> None:
except SystemExit:
raise
except Exception as e:
- # Surface a precise message so the UI doesn't show a generic "worker exited with code 1".
- # huggingface_hub recommends force_download=True to recover, which our "Restart" UI maps to a
- # fresh start by purging the partial via prepare_cache_for_transport.
+ # Surface a precise message rather than a generic "worker exited with code 1": huggingface_hub
+ # recommends force_download=True to recover, which our Restart maps to purging the partial via
+ # prepare_cache_for_transport.
print(f"{type(e).__name__}: {e}", file = sys.stderr)
sys.exit(1)
diff --git a/studio/backend/lan_access.py b/studio/backend/lan_access.py
index 11bb319f65e..acc3e41719f 100644
--- a/studio/backend/lan_access.py
+++ b/studio/backend/lan_access.py
@@ -44,8 +44,8 @@ _START_TIMEOUT = 10.0
# kept under the ~5s Windows console-close budget run.py's shutdown path works to
_STOP_TIMEOUT = 3.0
-# a LAN request already accepted can run for minutes, and it stays a remote caller
-# for all of them; on expiry the trust flag is left active rather than downgraded
+# a LAN request already accepted can run for minutes and stays a remote caller throughout; on expiry
+# the trust flag is left active rather than downgraded
_DRAIN_TIMEOUT = 300.0
# networking mode rarely changes, but a failed wslinfo probe must eventually recover
@@ -62,8 +62,8 @@ _serve_loop: Any = None
_sockets: tuple[socket.socket, ...] = ()
_port: Optional[int] = None
_error: Optional[str] = None
-# stopped listeners whose accepted requests are still running; they remain remote
-# callers, so the trust flag stays up until every one of them has drained
+# stopped listeners whose accepted requests are still running remain remote callers, so the trust
+# flag stays up until every one has drained
_pending_drains = 0
# rebound whole, never mutated: request_on_lan_listener reads it without the lock
_bound_addresses: tuple[str, ...] = ()
@@ -77,10 +77,8 @@ def detect_lan_addresses(ip_version: int = 4) -> list[str]:
-- a cloud VM binding its own public IP is the same operation as a laptop
binding its Wi-Fi address, and the caller decides whether that is wanted.
"""
- # WSL's NAT-side address belongs to a private Hyper-V network, not the
- # physical LAN. A second device cannot open it directly. Mirrored mode is
- # different: WSL participates in the host's network and its addresses can be
- # reached subject to the host firewall.
+ # WSL's NAT-side address belongs to a private Hyper-V network a second device cannot open; mirrored
+ # mode is different, since WSL joins the host's network
if _wsl_networking_mode() not in (None, "mirrored"):
return []
@@ -123,8 +121,8 @@ def detect_lan_addresses(ip_version: int = 4) -> list[str]:
if probe is not None:
probe.close()
- # every other adapter that is up: a route probe picks one source address, and
- # an isolated LAN has no route at all, so neither it nor the hostname enumerates them
+ # a route probe picks one source address and an isolated LAN has no route at all, so neither it nor
+ # the hostname enumerates the other adapters
for address in _interface_addresses(ip_version):
_add(address)
return addresses
@@ -322,8 +320,8 @@ def start_lan_listener(
logger.info("LAN access skipped unbindable addresses: %s", "; ".join(failures))
server = uvicorn.Server(_listener_config(app, bound[0], port))
- # published before the socket can accept: a request served in between would
- # still read the loopback-only trust defaults
+ # published before the socket can accept: a request served in between would still read the loopback-
+ # only trust defaults
set_lan_connector_active(True)
serving = server.serve(sockets = sockets)
try:
@@ -434,8 +432,8 @@ def stop_lan_listener() -> bool:
"""
global _server, _serve_loop, _sockets, _bound_addresses, _port, _error
- # a start holds _lock while waiting for this loop to run serve(), so a stop that
- # arrives on the loop itself must not block on it or the two wait each other out
+ # a start holds _lock while waiting for this loop to run serve(), so a stop arriving on the loop
+ # itself must not block on it
if not _lock.acquire(blocking = not _running_on_event_loop()):
logger.info("LAN access stop deferred: a listener change is in flight")
return False
@@ -451,9 +449,8 @@ def stop_lan_listener() -> bool:
return True
server.should_exit = True
if _running_on_event_loop():
- # /api/shutdown tears down from a task on this very loop; waiting would deadlock.
- # ownership is kept because uvicorn cannot close the sockets until the loop is
- # free again, and _graceful_shutdown blocks it for seconds stopping subprocesses
+ # /api/shutdown tears down from a task on this very loop, so waiting would deadlock; ownership is
+ # kept because uvicorn cannot close the sockets until the loop is free again
logger.info("LAN access stopping")
return True
if loop is None or loop.is_closed() or not loop.is_running():
@@ -468,8 +465,8 @@ def stop_lan_listener() -> bool:
_release_listener_state()
logger.info("LAN access stopped")
return True
- # ownership is kept so a retry waits on these same sockets, and so a second
- # stop cannot report success while the port may still be accepting
+ # ownership is kept so a retry waits on these same sockets, and so a second stop cannot report
+ # success while the port may still be accepting
_error = "stop_timed_out"
logger.warning("LAN access did not release port %s within %ss", port, _STOP_TIMEOUT)
return False
diff --git a/studio/backend/loggers/config.py b/studio/backend/loggers/config.py
index 3e322af4d3d..22376597d06 100644
--- a/studio/backend/loggers/config.py
+++ b/studio/backend/loggers/config.py
@@ -34,16 +34,11 @@ def _env_int(name: str, default: int) -> int:
return default
-# Cap on the rendered traceback in a single log record. A traceback is normally a few
-# KB, but an exception whose message embeds a request body is not: a binary upload
-# rejected by request validation produced one 2.2 MB line. Keep the head (where the
-# raising frame is) and the tail (where the actual exception type and message are),
-# and say how much was dropped. 0 disables the cap.
+# An exception whose message embeds a request body is not a few KB: a rejected binary upload
+# produced one 2.2 MB line.
_MAX_EXC_CHARS = _env_int("UNSLOTH_STUDIO_MAX_EXCEPTION_CHARS", 16384)
_EXC_TAIL_CHARS = 2048
-# The middleware logs the same exception twice: once rendered as a traceback under
-# "exception" and once as str(exc) under "error". Capping only the first still lets an
-# exception whose message embeds the request body through, so bound both.
+# The middleware logs the same exception twice, rendered as a traceback and as str(exc), so bound both.
_MAX_ERROR_CHARS = 2048
@@ -77,14 +72,13 @@ def truncate_exception(event_dict: dict) -> dict:
if isinstance(error, str):
event_dict["error"] = _truncate_middle(error, message_cap, _EXC_TAIL_CHARS)
# f-string call sites interpolate the exception straight into the message
- # (routes/inference.py: logger.error(f"...: {e}", exc_info = True)), so the event
- # itself is a third copy that can carry the whole payload.
+ # (routes/inference.py: logger.error(f"...: {e}", exc_info = True)), so the event itself is a
+ # third copy that can carry the whole payload.
event = event_dict.get("event")
if isinstance(event, str):
event_dict["event"] = _truncate_middle(event, message_cap, _EXC_TAIL_CHARS)
- # logger.error("stream error: %s", exc) keeps the exception under positional_args,
- # and the chain has no PositionalArgumentsFormatter, so the renderer stringifies it
- # untouched. Render and cap it here instead.
+ # logger.error("...: %s", exc) keeps the exception under positional_args and the chain has no
+ # PositionalArgumentsFormatter, so render and cap it here instead.
args = event_dict.get("positional_args")
if isinstance(args, (list, tuple)) and args:
event_dict["positional_args"] = [
@@ -110,25 +104,17 @@ def _plain_tracebacks_enabled() -> bool:
)
-# Prefix on every echoed line. NOT whitespace: RFC 8259 lets a parser skip leading
-# space/tab, so json.loads(' {"event": ...}') SUCCEEDS and a request-derived exception
-# message could forge a record (CWE-117). "| " cannot begin a JSON value, so no echoed
-# line parses as one, and a records-only reader can drop them on the prefix alone.
+# NOT whitespace: RFC 8259 lets a parser skip leading space/tab, so json.loads(' {"event": ...}')
+# SUCCEEDS and a request-derived exception message could forge a record (CWE-117); "| " cannot
+# begin a JSON value.
_TRACEBACK_ECHO_PREFIX = "| "
-# Unicode's Bidi_Control set (PropList.txt), exactly what the UAX #9 algorithm acts on and
-# what UTR #36 / Trojan Source (CVE-2021-42574) name. json.dumps already escapes these
-# (ensure_ascii), so only the echo would emit them raw.
-#
-# Deliberately NOT all of category Cf: U+200B-200D, U+00AD and U+FEFF occur in ordinary
-# text (ZWNJ in Persian/Arabic, ZWJ in emoji) and reorder nothing.
-_BIDI_CONTROLS = frozenset(
- "؜" # ARABIC LETTER MARK
- "‎‏" # LEFT-TO-RIGHT / RIGHT-TO-LEFT MARK
- "‪‫‬‭‮" # LRE, RLE, PDF, LRO, RLO
- "⁦⁧⁨⁩" # LRI, RLI, FSI, PDI
-)
+# Unicode's Bidi_Control set (PropList.txt), exactly what UAX #9 acts on and what UTR #36 /
+# Trojan Source (CVE-2021-42574) name; json.dumps already escapes these (ensure_ascii), so only
+# the echo would emit them raw. Deliberately NOT all of category Cf: U+200B-200D, U+00AD and
+# U+FEFF occur in ordinary text (ZWNJ in Persian/Arabic, ZWJ in emoji) and reorder nothing.
+_BIDI_CONTROLS = frozenset("؜‎‏‪‫‬‭‮⁦⁧⁨⁩")
def _escape_unprintable(text: str) -> str:
@@ -207,8 +193,8 @@ def _cap_echoed_lines(lines: list[str], limit: int) -> str:
used = 0
for line in reversed(lines[len(head) :]):
if used + len(line) + 1 > tail_budget:
- # Cut the boundary line rather than drop it: losing a traceback's last line
- # leaves the reader every frame and no reason.
+ # Cut the boundary line rather than drop it: losing a traceback's last line leaves the reader every
+ # frame and no reason.
room = tail_budget - used - 1
if room > 0:
tail.append(line[:room])
@@ -265,17 +251,16 @@ def with_readable_traceback(renderer):
return _render
-# Set alongside HF_HUB_DISABLE_PROGRESS_BARS when the value is Unsloth's default rather
-# than the operator's, so allow_progress_bars() can tell them apart.
+# Set alongside HF_HUB_DISABLE_PROGRESS_BARS when the value is Unsloth's default rather than the
+# operator's, so allow_progress_bars() can tell them apart.
_PROGRESS_BARS_DEFAULTED = "UNSLOTH_STUDIO_PROGRESS_BARS_DEFAULTED"
-# huggingface_hub's own spelling of truth (utils/_runtime.py ENV_VARS_TRUE_VALUES),
-# so "off" and "no" mean "keep the bars" here exactly as they do there.
+# huggingface_hub's own spelling of truth (utils/_runtime.py ENV_VARS_TRUE_VALUES), so "off" and
+# "no" mean "keep the bars" here exactly as they do there.
_ENV_TRUE = frozenset({"1", "on", "yes", "true"})
-# Set once this process has deliberately taken its bars back (the export worker draws
-# them, the training worker reads them). quiet_third_party_progress_bars() then stops
-# being a switch a later call can flip the other way.
+# Set once this process has deliberately taken its bars back, so quiet_third_party_progress_bars()
+# stops being a switch a later call can flip the other way.
_BARS_RESTORED = False
@@ -323,6 +308,7 @@ def _silence_datasets_bar_output() -> None:
stream keeps the counter (and the status) alive while the log stays clean.
"""
if "datasets" not in sys.modules:
+ # Operator asked to keep them; leave every library alone.
return
try:
from datasets.utils.tqdm import tqdm as bar_cls
@@ -386,7 +372,6 @@ def keep_progress_bars_countable() -> None:
"""
value = os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS")
if value is None or not _env_is_true(value):
- # Nothing quieted them here: --verbose, or an operator who asked to keep them.
return
if not os.environ.get(_PROGRESS_BARS_DEFAULTED):
# The operator turned them off; that is not ours to undo.
@@ -452,42 +437,35 @@ def quiet_third_party_progress_bars() -> None:
about to replace with its transformers sidecar. `--verbose` skips it entirely.
"""
if _BARS_RESTORED:
- # This process took its bars back on purpose: the export worker shows them, the
- # training worker reads them out of tqdm._instances (where a disabled bar is
- # never registered) and has already redirected their output.
+ # This process took its bars back on purpose: the training worker reads them out of tqdm._instances,
+ # where a disabled bar is never registered, and has already redirected their output.
return
if _verbose_logging_requested() and os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS") is None:
- # --verbose promises everything back, so it must not install this default
- # either; the flag is inherited by the workers, which would stay quiet.
+ # --verbose promises everything back, so it must not install this default either; the flag is
+ # inherited by the workers.
return
if os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS") is None:
os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
- # Marks the value as ours rather than the operator's, so a process that needs
- # bars back (the export worker streams Hub upload progress into the export
- # dialog) can tell the difference. Inherited by every child process.
+ # Marks the value as ours rather than the operator's, so a process that needs bars back can tell the
+ # difference. Inherited by every child process.
os.environ[_PROGRESS_BARS_DEFAULTED] = "1"
elif not _env_is_true(os.environ["HF_HUB_DISABLE_PROGRESS_BARS"]):
# Operator asked to keep them; leave every library alone.
return
- # Only touch Hub if something already imported it. Importing it here would cache
- # the base environment's copy before a subprocess prepends its transformers
- # sidecar to sys.path, leaving that process on an incompatible Hub.
+ # Only touch Hub if something already imported it: importing it here would cache the base
+ # environment's copy before a subprocess prepends its transformers sidecar to sys.path.
if "huggingface_hub" in sys.modules:
try:
from huggingface_hub.utils import disable_progress_bars
disable_progress_bars()
- except Exception: # noqa: BLE001 — quieting logs must never break startup
+ except Exception: # noqa: BLE001 - quieting logs must never break startup
pass
- # transformers derives its own _tqdm_active from the hub flag at import time,
- # so a module imported BEFORE this ran still needs the explicit call.
- #
- # datasets is handled separately (see _silence_datasets_bar_output): its `Map:` and
- # `Standardizing chat format (num_proc=8):` bars from dataset preparation were the
- # ones actually landing inside JSON records, but the UI reads their counter, so
- # only the output goes. datasets is imported long after logging setup, which is why
- # this function is safe to call again once a library is in.
+ # transformers derives its own _tqdm_active from the hub flag at import time, so a module imported
+ # BEFORE this ran still needs the explicit call.
+ # datasets is handled separately: the UI reads its bar counters, so only the output goes, and it is
+ # imported long after logging setup, which is why this function is safe to call again.
for _mod in ("transformers", "diffusers"):
module = sys.modules.get(_mod)
if module is None:
@@ -519,8 +497,8 @@ class LogConfig:
log_level_name = os.getenv("LOG_LEVEL", "INFO").upper()
log_level = getattr(logging, log_level_name, logging.INFO)
- # Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows,
- # LANG=C), so key off the stream, not the platform.
+ # Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows, LANG=C), so key off the
+ # stream, not the platform.
for stream in (sys.stdout, sys.stderr):
if getattr(stream, "encoding", "") and not str(stream.encoding).lower().replace(
"-", ""
@@ -534,21 +512,20 @@ class LogConfig:
structlog.configure(
processors = [
# Ordered to control output field order.
- structlog.processors.TimeStamper(fmt = "iso"), # timestamp first
- structlog.processors.add_log_level, # level second
+ structlog.processors.TimeStamper(fmt = "iso"),
+ structlog.processors.add_log_level,
structlog.contextvars.merge_contextvars,
structlog.processors.format_exc_info,
filter_sensitive_data,
- # After redaction, not before: redact_native_paths replaces exact
- # strings, so cutting the middle out of a traceback first could leave
- # half a path behind for it to miss.
+ # After redaction, not before: redact_native_paths replaces exact strings, so cutting the middle out
+ # of a traceback first could leave half a path behind for it to miss.
_truncate_exception_processor,
# Flatten the extra field into the main dict.
lambda logger, method_name, event_dict: {
"timestamp": event_dict.get("timestamp"),
"level": event_dict.get("level"),
"event": event_dict.get("event"),
- **(event_dict.get("extra", {})), # Flatten extra into main dict
+ **(event_dict.get("extra", {})),
**{
k: v
for k, v in event_dict.items()
diff --git a/studio/backend/loggers/handlers.py b/studio/backend/loggers/handlers.py
index 7b7a8226c85..f57f012f39c 100644
--- a/studio/backend/loggers/handlers.py
+++ b/studio/backend/loggers/handlers.py
@@ -18,8 +18,7 @@ from typing import TYPE_CHECKING
import structlog
-# Annotations only: a runtime import makes the ASGI stack a hard dependency of
-# every CLI command.
+# Annotations only: a runtime import makes the ASGI stack a hard dependency of every CLI command.
if TYPE_CHECKING:
from starlette.types import ASGIApp, Message, Receive, Scope, Send
@@ -36,34 +35,29 @@ def _env_int(name: str, default: int) -> int:
return default
-# Collapse identical GET/2xx logs within the window (the SPA fans one invalidation
-# into many list fetches). Mutations and errors always log. 0 = off.
+# Collapse identical GET/2xx logs within the window, since the SPA fans one invalidation into many
+# list fetches; mutations and errors always log.
_ACCESS_LOG_DEDUP_MS = _env_int("UNSLOTH_STUDIO_ACCESS_LOG_DEDUP_MS", 300)
# Liveness/UI polls whose line means only "still polling"; collapse to a longer
-# heartbeat. First hit and errors still log. 0 = off.
_QUIET_POLL_DEDUP_MS = _env_int("UNSLOTH_STUDIO_ACCESS_LOG_POLL_DEDUP_MS", 10000)
-# The desktop watchdog probe is slower than every other poll here: 15s between rounds
-# (HEALTH_WATCHDOG_INTERVAL, src-tauri/src/commands.rs) plus up to a 10s probe budget,
-# ~19s in the sample below. A 10s window that stamps only on emit can never close over two
-# of those, so it would collapse nothing. Its own window, wide enough to span a round.
-# 0 = off.
+# The desktop watchdog probe rounds are ~19s apart, so a 10s window that stamps only on emit would
+# collapse nothing; it gets its own, wider window.
_WATCHDOG_POLL_DEDUP_MS = _env_int("UNSLOTH_STUDIO_ACCESS_LOG_WATCHDOG_DEDUP_MS", 60000)
-# Both windows off is what --verbose sets; the drop-the-2xx suppressor below has no
-# window of its own, so it must read the same signal to honour --verbose.
+# Both windows off is what --verbose sets, and the drop-the-2xx suppressor below has no window of
+# its own, so it reads the same signal.
_VERBOSE_ACCESS_LOG = _ACCESS_LOG_DEDUP_MS <= 0 and _QUIET_POLL_DEDUP_MS <= 0
_QUIET_POLL_PATHS = {
"/api/health",
"/api/auth/status",
"/api/inference/status",
"/api/inference/monitor",
- # The loaded-models indicator polls all four runtimes every 5s for as long
- # as the app is open, and on the desktop every line is mirrored into
- # tauri.log. /api/inference/status is already above; these are its siblings.
+ # The loaded-models indicator polls all four runtimes every 5s for as long as the app is open, and
+ # on the desktop every line is mirrored into tauri.log.
"/api/inference/images/status",
"/api/inference/video/status",
"/api/inference/audio/stt/status",
- # Re-read whenever the settings dialog or the remote-access section is open, and it is
- # a plain read of a toggle: 116 lines in the 4h sample.
+ # Re-read whenever the settings dialog or the remote-access section is open, and it is a plain read
+ # of a toggle: 116 lines in the 4h sample.
"/api/settings/remote-access",
# List polls the tabs refetch on a timer and on every tab switch.
"/api/train/runs",
@@ -79,30 +73,20 @@ _QUIET_POLL_PATHS = {
"/api/inference/video/generate-progress",
# Polled every 1.5s while the train UI is open.
"/api/train/diffusion/status",
- # Unlike /api/inference/load-progress, these handlers log nothing and
- # diffusion.loaded / video.loaded are terminal, so a minutes-long load would
- # otherwise emit nothing at all.
+ # These handlers log nothing and diffusion.loaded / video.loaded are terminal, so a minutes-long
+ # load would otherwise emit nothing at all.
"/api/inference/images/load-progress",
"/api/inference/video/load-progress",
# Templated; matched through normalize_poll_path. See _TEMPLATED_POLL_PATHS.
"/api/chat/threads/{id}",
"/api/chat/threads/{id}/forks",
}
-# The pure-liveness subset of _QUIET_POLL_PATHS. Every one of these answers the same
-# question ("the server is up and answering"), and the SPA fires them together in one
-# burst, so heartbeating them independently emits one line per path per window instead
-# of one line per window. They share a single bucket: the first of the burst logs with
-# its real path, the rest of that window is dropped. Measured over four Unsloth sessions
-# these were 39-69% of the access log and the shared bucket removed 25-47% of it.
-#
-# Only this group is shared. The other _QUIET_POLL_PATHS entries (/api/train/runs,
-# /api/models/checkpoints, /api/models/local, /api/rag/knowledge-bases, the download
-# polls) each report on a different subsystem, so they keep their own heartbeat.
-# Two paths are deliberately NOT here because their latency is worth seeing on its
-# own rather than being stamped out by a cheap sibling: /api/health (main.py waits up
-# to a second for hardware detection, and the desktop preflight has a two-second
-# deadline) and /api/inference/status (its handler reads llama.cpp capabilities and
-# runs a release-freshness check in an executor).
+# The pure-liveness subset: the SPA fires them together in one burst, so they share a single bucket,
+# the first logging with its real path and the rest of the window dropping.
+# Only this group is shared: the other quiet paths each report on a different subsystem, so they
+# keep their own heartbeat.
+# Two paths are deliberately NOT here because their latency is worth seeing: /api/health waits on
+# hardware detection, and /api/inference/status reads llama.cpp capabilities in an executor.
_LIVENESS_POLL_PATHS = frozenset(
{
"/api/auth/status",
@@ -112,14 +96,10 @@ _LIVENESS_POLL_PATHS = frozenset(
"/api/inference/audio/stt/status",
}
)
-# The desktop shell's own watchdog probe. /api/health was already quiet but its sibling was
-# in no suppressor at all: 760 lines on an idle 4h session, 14% of tauri.log, to say the
-# process is still up. Out of _QUIET_POLL_PATHS because that window is narrower than the
-# poll interval; see _WATCHDOG_POLL_DEDUP_MS. Start/stop transitions reach the phase log
-# either way.
+# The desktop shell's watchdog probe was in no suppressor at all: 760 lines on an idle 4h session.
+# Out of _QUIET_POLL_PATHS because that window is narrower than the poll interval.
_WATCHDOG_POLL_PATHS = {"/api/liveness"}
-# Bucket key for the group above. Not a real (method, path, query, status), so it can
-# never collide with one.
+# Bucket key for the group above; not a real (method, path, query, status), so it can never collide with one.
_LIVENESS_DEDUP_KEY = ("GET", "\x00liveness", b"", 200)
_DEDUP_MAP_MAX = 4096
_NATIVE_PATH_LEASE_RE = re.compile(
@@ -141,9 +121,8 @@ _EXCLUDED_SUFFIXES = (
".woff2",
".ttf",
)
-# GET polls whose 2xx line carries no signal (their progress/phase events and the UI
-# do), so drop it entirely; non-2xx still logs. Only /api/hub download polls emit
-# events; the legacy /api/models and /api/datasets ones heartbeat via _QUIET_POLL_PATHS.
+# GET polls whose 2xx line carries no signal, so drop it entirely; non-2xx still logs. Only /api/hub
+# download polls emit events.
_QUIET_SUCCESS_PATHS = {
"/api/inference/load-progress",
"/api/llama/update-status",
@@ -159,39 +138,33 @@ _QUIET_SUCCESS_PATHS = {
"/api/hub/datasets/download-progress",
"/api/hub/datasets/active-downloads",
"/api/hub/datasets/transport-status",
- # Boot-burst catalog reads. The SPA refetches these on every auth change, store
- # rehydration and settings/provider dialog open, and each one is a plain read whose
- # outcome is already visible as the populated (or empty) list in the UI.
- # Provider registry + configs: syncExternalProvidersFromBackend fetches the pair in
- # one Promise.all, so they always arrive as two lines saying the same thing.
+ # Boot-burst catalog reads: the SPA refetches these on every auth change and dialog open, and each
+ # outcome is already visible in the UI.
+ # syncExternalProvidersFromBackend fetches the pair in one Promise.all, so they always arrive as two
+ # lines saying the same thing.
"/api/providers/registry",
"/api/providers/",
- # Fetched in the same Promise.all as /api/models/list and /api/inference/status, both
- # of which keep their access line, so the resync is still traceable without it.
+ # Fetched in the same Promise.all as two routes that keep their access line, so the resync is still
+ # traceable without it.
"/api/models/loras",
- # Re-read once per auth generation at boot. Suppression is GET-only, so the PUT that
- # actually writes a profile change still logs.
+ # Suppression is GET-only, so the PUT that actually writes a profile change still logs.
"/api/settings/personalization",
}
-# The token-refresh route. Its first 2xx means the client has obtained a valid
-# session, so from then on chat 401s are real failures and must stay visible.
+# The token-refresh route: its first 2xx means the client has a valid session, so from then on chat
+# 401s are real failures and must stay visible.
_AUTH_REFRESH_PATH = "/api/auth/refresh"
-# High-frequency chat list polls; their 2xx is covered by generation/tool-call/stats
-# events. Exact paths only, so detail/message reads (/threads/{id}, .../messages,
-# /projects/{id}) keep their logs. The pre-auth 401 race also fires on these polls.
+# Exact paths only, so detail and message reads keep their logs; the pre-auth 401 race also fires on these polls.
_CHAT_LIST_PATHS = {
"/api/chat/threads",
"/api/chat/projects",
}
-# Templated paths. The sets above match EXACTLY, which #7087 chose so detail reads kept
-# their access line; what changed is that streaming now drives these two on a loop (25 and
-# 21 lines, 34% of the access log, over a 20s four-tab session). So they get a heartbeat
-# rather than silence: the first of each window logs with its real path, the rest drop.
+# The sets above match EXACTLY so detail reads keep their access line, but streaming drives these
+# two on a loop (34% of the access log over a 20s session), so they get a heartbeat instead.
_CHAT_THREAD_DETAIL = "/api/chat/threads/{id}"
_CHAT_THREAD_FORKS = "/api/chat/threads/{id}/forks"
_TEMPLATED_POLL_PATHS = frozenset({_CHAT_THREAD_DETAIL, _CHAT_THREAD_FORKS})
-# One id segment, no slashes: a deeper path such as /threads/{id}/messages/{mid} must NOT
-# collapse into the detail bucket, since a message read is a different question.
+# One id segment, no slashes: a deeper path such as /threads/{id}/messages/{mid} must NOT collapse
+# into the detail bucket.
_CHAT_THREAD_PATH_RE = re.compile(r"^/api/chat/threads/(?!$)[^/]+(/forks)?$")
@@ -208,13 +181,10 @@ def normalize_poll_path(path: str) -> str:
return _CHAT_THREAD_FORKS if m.group(1) else _CHAT_THREAD_DETAIL
-# The log viewer polls these while reading the very file this middleware writes,
-# so unsuppressed each poll appends a record the next poll reads back and the log
-# grows forever. _is_redundant_repeat does not cover it: it keys on the query
-# string, and every poll carries a fresh cursor.
-# Separate from _QUIET_SUCCESS_PATHS because --verbose must NOT lift this one:
-# --verbose is what someone debugging turns on, and here the extra noise (~390
-# bytes/s at the 1 Hz Live poll rate) buries the failure they opened the viewer for.
+# The log viewer polls these while reading the very file this middleware writes, so unsuppressed
+# each poll appends a record the next reads back; _is_redundant_repeat cannot cover it.
+# Separate from _QUIET_SUCCESS_PATHS because --verbose must NOT lift this one: the extra noise
+# buries the failure the viewer was opened for.
_SELF_READ_PATHS = {
"/api/settings/debug/logs",
"/api/settings/debug/logs/sources",
@@ -238,17 +208,14 @@ def _is_quiet_success(method: str, path: str, status_code: int, pre_auth: bool)
return pre_auth and status_code == 401 and path in _CHAT_LIST_PATHS
-# An unhandled request exception is logged twice: once here as a structured
-# request_failed event whose "exception" field already carries the whole traceback
-# (format_exc_info renders it, see loggers/config.py), and then again by uvicorn on
-# stderr as "Exception in ASGI application" after the re-raise. The desktop shell
-# mirrors every stderr line separately into tauri.log, so the second copy alone costs
-# ~90 lines per failure. Keep the structured copy and drop uvicorn's.
+# An unhandled request exception is logged twice: here as a structured request_failed event whose
+# "exception" field carries the whole traceback (format_exc_info renders it, see
+# loggers/config.py), and again by uvicorn on stderr, which the desktop shell mirrors into
+# tauri.log at ~90 lines per failure. Keep the structured copy.
_UVICORN_ASGI_EXC_MSG = "Exception in ASGI application"
-# Set on the exception instance itself rather than tracked in a side table: the object
-# is what uvicorn hands us, so the match cannot go stale or collide with a recycled id,
-# and an exception raised above this middleware (CORS, remote-access, the protocol
-# layer) carries no marker and keeps uvicorn's traceback.
+# Set on the exception instance itself rather than a side table: the object is what uvicorn hands
+# us, so the match cannot go stale or collide with a recycled id, and an exception raised above
+# this middleware (CORS, remote-access, the protocol layer) keeps uvicorn's traceback.
_LOGGED_EXC_ATTR = "_unsloth_request_failed_logged"
@@ -294,8 +261,8 @@ class LoggingMiddleware:
self.app = app
# (method, path, query, status_code) -> monotonic ts of the last EMITTED log.
self._last_log: dict[tuple[str, str, bytes, int], float] = {}
- # Flips True after the first successful /api/auth/refresh; before that, chat
- # list-poll 401s are the transient bootstrap race and are suppressed.
+ # Before the first successful /api/auth/refresh, chat list-poll 401s are the transient bootstrap
+ # race and are suppressed.
self._auth_refreshed = False
def _is_redundant_repeat(
@@ -306,9 +273,8 @@ class LoggingMiddleware:
heartbeat. Stamps only on emit, so steady polls still log."""
if method != "GET" or not (200 <= status_code < 300):
return False
- # A query makes the request something other than the background poll:
- # /api/inference/audio/stt/status?model=... extends the downloaded check to a
- # custom repo, so it keeps its own identity rather than joining the bucket.
+ # A query makes the request something other than the background poll, so it keeps its own identity
+ # rather than joining the bucket.
is_liveness = path in _LIVENESS_POLL_PATHS and not query
# Bucketed by template, so four tabs polling four threads share one heartbeat.
norm = normalize_poll_path(path) if not query else path
@@ -321,9 +287,8 @@ class LoggingMiddleware:
window_ms = _ACCESS_LOG_DEDUP_MS
if window_ms <= 0:
return False
- # The liveness group shares one bucket, so a burst of them logs once, not once
- # per path. Only the query-less form joins it, so a parameterized call still
- # gets its own status and latency line.
+ # The liveness group shares one bucket, so a burst logs once rather than once per path; only the
+ # query-less form joins it.
key = _LIVENESS_DEDUP_KEY if is_liveness else (method, norm, query, status_code)
last = self._last_log.get(key)
if last is not None and (now - last) * 1000.0 < window_ms:
diff --git a/studio/backend/mcp_server.py b/studio/backend/mcp_server.py
index e950842fcf5..d2820180ed7 100644
--- a/studio/backend/mcp_server.py
+++ b/studio/backend/mcp_server.py
@@ -27,8 +27,8 @@ class BearerTokenMiddleware:
# A non-ASCII token cannot be sent in an HTTP header; reject it here.
raise ValueError("Unsloth MCP bearer token must contain ASCII characters only")
self.app = app
- # Compare on raw header bytes: str hmac.compare_digest raises on non-ASCII
- # input, which would surface as a 500 instead of a clean 401.
+ # Compare on raw header bytes: str hmac.compare_digest raises on non-ASCII input, which would
+ # surface as a 500 instead of a clean 401.
self.expected = token.encode("utf-8")
async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
@@ -168,9 +168,8 @@ def create_studio_mcp() -> FastMCP:
from models.data_recipe import RecipePayload
from routes.data_recipe.validate import validate
- # Direct call, so the ViaApiKey dependency never runs and its `= False`
- # default would read as a UI session. This surface is a remote static
- # bearer, so say so explicitly.
+ # Direct call, so the ViaApiKey dependency never runs and its `= False` default would read as a UI
+ # session; this surface is a remote static bearer.
return _dump(validate(RecipePayload(recipe = recipe), via_api_key = True))
@mcp.tool
diff --git a/studio/backend/models/auth.py b/studio/backend/models/auth.py
index 9eabc6d4ee8..e0251a6c0d8 100644
--- a/studio/backend/models/auth.py
+++ b/studio/backend/models/auth.py
@@ -76,8 +76,6 @@ class ChangePasswordRequest(BaseModel):
# ---------------------------------------------------------------------------
-# API key schemas
-# ---------------------------------------------------------------------------
class CreateApiKeyRequest(BaseModel):
diff --git a/studio/backend/models/data_recipe.py b/studio/backend/models/data_recipe.py
index e6f27e64dfa..b9b89ca82bb 100644
--- a/studio/backend/models/data_recipe.py
+++ b/studio/backend/models/data_recipe.py
@@ -81,7 +81,6 @@ class SeedInspectUploadRequest(BaseModel):
block_id: str | None = None
file_ids: list[str] | None = None
file_names: list[str] | None = None
- # Shared fields
preview_size: int = Field(default = 10, ge = 1, le = 50)
seed_source_type: str | None = None
unstructured_chunk_size: int | None = Field(default = None, ge = 1, le = 20000)
@@ -122,7 +121,7 @@ class UnstructuredFileUploadResponse(BaseModel):
file_id: str
filename: str
size_bytes: int
- status: str # "ok" or "error"
+ status: str
error: str | None = None
diff --git a/studio/backend/models/export.py b/studio/backend/models/export.py
index 3be2d1e3af6..49eb7b049fb 100644
--- a/studio/backend/models/export.py
+++ b/studio/backend/models/export.py
@@ -107,9 +107,8 @@ class ExportStatusResponse(BaseModel):
False,
description = "True while a load / export / cleanup operation is running",
)
- # Recovery fields: when a blocking export POST is cut off by a Cloudflare tunnel
- # timeout (524 at ~100s), the client polls this endpoint to learn the real
- # outcome of the operation that kept running on the backend.
+ # Recovery fields: when a blocking export POST is cut off by a Cloudflare tunnel timeout (524 at
+ # ~100s), the client polls this endpoint to learn the real outcome.
active_op_kind: Optional[str] = Field(
None,
description = "Kind of the currently running op (load_checkpoint / export_* / cleanup)",
@@ -209,8 +208,6 @@ class ExportMergedModelRequest(ExportCommonOptions):
class ExportBaseModelRequest(ExportCommonOptions):
"""Request for exporting a non-PEFT (base) model."""
- # Uses fields from ExportCommonOptions only
-
class ExportGGUFRequest(BaseModel):
"""Request for exporting the current model to GGUF format."""
diff --git a/studio/backend/models/mcp_servers.py b/studio/backend/models/mcp_servers.py
index 354356164ae..6beb116823c 100644
--- a/studio/backend/models/mcp_servers.py
+++ b/studio/backend/models/mcp_servers.py
@@ -66,5 +66,5 @@ class McpServerImportRequest(BaseModel):
class McpServerImportResult(BaseModel):
created: list[McpServerResponse] = Field(default_factory = list)
- skipped: list[str] = Field(default_factory = list) # display names skipped as duplicates
+ skipped: list[str] = Field(default_factory = list)
errors: list[str] = Field(default_factory = list)
diff --git a/studio/backend/models/models.py b/studio/backend/models/models.py
index 33468d70550..cd9bb7f0c24 100644
--- a/studio/backend/models/models.py
+++ b/studio/backend/models/models.py
@@ -153,9 +153,9 @@ class GgufVariantDetail(BaseModel):
filename: str = Field(..., description = "GGUF filename (e.g., 'gemma-3-4b-it-Q4_K_M.gguf')")
quant: str = Field(..., description = "Quantization label or internal GGUF variant key")
- # Mirrors hub.schemas.inventory.GgufVariantDetail. The route builds THIS model, so a field
- # that exists only on the hub twin is dropped by pydantic without a word, and a qualified
- # row falls back to rendering its whole relative path.
+ # Mirrors hub.schemas.inventory.GgufVariantDetail. The route builds THIS model, so a field that
+ # exists only on the hub twin is dropped by pydantic without a word and a qualified row falls back
+ # to rendering its whole relative path.
display_label: Optional[str] = Field(
None, description = "Optional user-facing label when quant is an internal key"
)
diff --git a/studio/backend/models/responses.py b/studio/backend/models/responses.py
index 59130dc6732..862b70b402f 100644
--- a/studio/backend/models/responses.py
+++ b/studio/backend/models/responses.py
@@ -8,9 +8,6 @@ from pydantic import BaseModel, Field
from typing import Optional, List
-# --- Training route response models ---
-
-
class TrainingStopResponse(BaseModel):
"""Response for stopping a training job"""
@@ -34,9 +31,6 @@ class TrainingMetricsResponse(BaseModel):
current_step: Optional[int] = Field(None, description = "Most recent step number")
-# --- Model management route response models ---
-
-
class LoRABaseModelResponse(BaseModel):
"""Response for getting a LoRA's base model"""
diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py
index d7af4ca4b69..6ec7d81f818 100644
--- a/studio/backend/models/training.py
+++ b/studio/backend/models/training.py
@@ -38,8 +38,7 @@ _MAX_LORA_ALPHA = 32_768
_MIN_VISION_IMAGE_SIZE = 256
# 2048 is the highest most llms stay stable at
_MAX_VISION_IMAGE_SIZE = 2048
-# Upper bound for dataset slice indices, capping `.skip(n)` on streaming datasets so an absurd
-# index can't iterate effectively forever. 1e9 is far beyond any realistic dataset row count.
+# Caps .skip(n) on streaming datasets so an absurd index cannot iterate effectively forever.
_MAX_DATASET_SLICE_INDEX = 1_000_000_000
@@ -81,6 +80,7 @@ class S3Config(BaseModel):
def _parse_lr(v: Any) -> float:
"""Parse learning_rate as a positive float strictly below _MAX_LR_VALUE."""
+ # mode="before" sees True/False as bool (not 1/0) for a precise error.
if v is None:
raise ValueError("learning_rate is required")
if isinstance(v, bool):
@@ -206,8 +206,8 @@ class TrainingStartRequest(BaseModel):
def _normalize_project_name(cls, value: Optional[str]) -> Optional[str]:
return normalize_project_name(value)
- # NOTE: pydantic runs all `mode="after"` validators in definition order, and
- # `_check_steps_or_epochs` is lower in this class; keep these checks order-independent.
+ # pydantic runs all mode="after" validators in definition order and _check_steps_or_epochs is lower
+ # in this class, so keep these checks order-independent.
@model_validator(mode = "after")
def _validate_dataset_slice(self) -> "TrainingStartRequest":
# start == end is intentionally allowed (a deliberate single-row slice); the trainer warns.
@@ -316,7 +316,7 @@ class TrainingStartRequest(BaseModel):
@field_validator("num_epochs")
@classmethod
def _check_num_epochs(cls, v: int) -> int:
- # 0 is a sentinel for "use max_steps instead" (frontend toggle).
+ # 0 is the frontend's sentinel for "use max_steps instead".
if v is None:
return 1
if v < 0 or v > _MAX_EPOCHS:
@@ -466,9 +466,8 @@ class TrainingStartRequest(BaseModel):
max_steps: Optional[int] = Field(None, description = "Maximum training steps")
save_steps: int = Field(100, description = "Steps between checkpoints")
weight_decay: float = Field(0.001, description = "Weight decay")
- # All three clip knobs are finite as well as non-negative: JSON 1e309 (and
- # FastAPI's Infinity literal) floats to inf, which clears ge=0 but never
- # binds, so the run would train unclipped while reporting a threshold.
+ # Finite as well as non-negative: JSON 1e309 floats to inf, which clears ge=0 but never binds, so
+ # the run would train unclipped while reporting a threshold.
max_grad_norm: Optional[float] = Field(
None,
ge = 0,
@@ -536,7 +535,6 @@ class TrainingStartRequest(BaseModel):
use_dora: bool = Field(False, description = "Use DoRA")
train_on_completions: bool = Field(False, description = "Train on completions only")
- # Vision-specific LoRA parameters
finetune_vision_layers: bool = Field(False, description = "Finetune vision layers")
finetune_language_layers: bool = Field(True, description = "Finetune language layers")
finetune_attention_modules: bool = Field(True, description = "Finetune attention modules")
@@ -582,8 +580,8 @@ class TrainingStartRequest(BaseModel):
@model_validator(mode = "after")
def _validate_streaming_splits(self) -> "TrainingStartRequest":
- # Streaming load_dataset does not accept HF slice syntax (probe-confirmed: ValueError: Bad
- # split). Reject early with a clear message so the user knows to use a plain split name.
+ # Streaming load_dataset does not accept HF slice syntax (probe-confirmed "Bad split"), so reject
+ # early with a clear message.
if self.dataset_streaming:
for field_name, split_val in (
("train_split", self.train_split),
@@ -605,9 +603,8 @@ class TrainingStartRequest(BaseModel):
@model_validator(mode = "after")
def _validate_lora_variant_flags(self) -> "TrainingStartRequest":
- # The frontend only ever sends one of these and never under Full Finetuning, but a direct
- # API/YAML/CLI caller can bypass that. Nothing downstream breaks, but reject early for a
- # clear error instead of a silently-ignored flag.
+ # A direct API, YAML or CLI caller can bypass the frontend; nothing downstream breaks, but reject
+ # early instead of silently ignoring the flag.
active = [
name
for name, enabled in (
@@ -622,8 +619,8 @@ class TrainingStartRequest(BaseModel):
f"Only one LoRA variant may be enabled at a time; got {active}. "
"use_rslora, use_loftq, and use_dora are mutually exclusive."
)
- # getattr, not self.training_type: model_construct() (used by single-field tests) leaves
- # required fields unset, and this mode="after" validator still runs on that partial instance.
+ # getattr, not self.training_type: model_construct() leaves required fields unset and this
+ # mode="after" validator still runs on that partial instance.
if getattr(self, "training_type", None) == "Full Finetuning" and active:
raise ValueError(
f"{active[0]} requires an adapter method (LoRA/QLoRA or "
@@ -735,8 +732,7 @@ class TrainingRunSummary(BaseModel):
error_message: Optional[str] = None
loss_sparkline: Optional[List[float]] = None
can_resume: bool = False
- # Why resume is unavailable when the reason is the recorded resource provenance rather than
- # the checkpoint. None when resumable or when the checkpoint itself is the problem.
+ # Why resume is unavailable when the reason is the recorded resource provenance rather than the checkpoint itself.
resume_blocked_reason: Optional[str] = None
resumed_later: bool = False
artifacts_available: bool = False
@@ -827,35 +823,36 @@ class DiffusionTrainingStartRequest(BaseModel):
"ceil(N / (batch x grad_accum)) optimizer steps over the N-image dataset"
),
)
- # Upper bound as well as positive, matching the LLM schema: JSON accepts 1e309, which floats to inf and satisfies a
- # gt-only constraint, so the route would evict residents and start AdamW at an infinite rate. Values >= 1.0 diverge.
+ # Upper bound as well as positive: JSON 1e309 floats to inf and satisfies a gt-only constraint, so
+ # the route would evict residents and start AdamW at an infinite rate; values >= 1.0 diverge.
learning_rate: float = Field(1e-4, gt = 0, lt = 1.0)
train_batch_size: int = Field(1, ge = 1, le = 64)
gradient_accumulation_steps: int = Field(1, ge = 1, le = 256)
lora_rank: int = Field(16, ge = 1, le = 320)
lora_alpha: Optional[int] = Field(None, ge = 1, le = 640, description = "Defaults to lora_rank")
- # Strictly below 1: PEFT turns lora_dropout into nn.Dropout(p=...), so 1.0 zeroes the LoRA branch and the run saves an
- # untrained adapter while reporting normal progress. Matches TrainingStartRequest's validator.
+ # Strictly below 1: PEFT turns lora_dropout into nn.Dropout(p=...), so 1.0 zeroes the LoRA branch
+ # and the run saves an untrained adapter while reporting normal progress.
lora_dropout: float = Field(0.0, ge = 0.0, lt = 1.0)
- # Mirror the remaining training-affecting knobs of DiffusionLoraConfig so a client that sets them is not silently trained with defaults. Defaults to the SDXL projections.
+ # Mirror the remaining training-affecting knobs of DiffusionLoraConfig so a client that sets them is
+ # not silently trained with defaults. Defaults to the SDXL projections.
lora_target_modules: List[str] = Field(
default_factory = lambda: ["to_k", "to_q", "to_v", "to_out.0"],
description = "U-Net modules to attach LoRA to",
)
- # Finite as well as non-negative: JSON accepts 1e309 (and FastAPI's parser the Infinity literal), which floats to inf and satisfies a ge-only
- # constraint. clip_grad_norm_ then clamps its coefficient to 1.0, so the run trains completely unclipped while the config reports clipping.
+ # Finite as well as non-negative: an inf max_grad_norm makes clip_grad_norm_ clamp its coefficient
+ # to 1.0, so the run trains completely unclipped while the config reports clipping.
max_grad_norm: float = Field(
1.0,
ge = 0,
allow_inf_nan = False,
description = "Gradient clipping max-norm; 0 disables clipping",
)
- # Bounded to what torch.manual_seed unpacks (int64 low / uint64 high): an out-of-range value otherwise passes every
- # preflight, evicts the resident models, spawns the trainer, and only then dies unpacking long long.
+ # Bounded to what torch.manual_seed unpacks: an out-of-range value otherwise passes every preflight,
+ # evicts the resident models, spawns the trainer, and only then dies unpacking long long.
seed: int = Field(42, ge = -(2**63), le = 2**64 - 1)
mixed_precision: Literal["bf16", "fp16", "no"] = Field("bf16")
- # Finite for the same reason as max_grad_norm: an inf gamma passes gt here and in normalized(), then collapses every
- # min-SNR weight to 1.0, silently training on plain unweighted MSE. null is the documented disable.
+ # Finite for the same reason as max_grad_norm: an inf gamma collapses every min-SNR weight to 1.0,
+ # silently training on plain unweighted MSE.
snr_gamma: Optional[float] = Field(
5.0, gt = 0, allow_inf_nan = False, description = "Min-SNR loss weighting; null disables"
)
@@ -904,7 +901,8 @@ class DiffusionTrainingStartRequest(BaseModel):
"or auto (pick by free VRAM + GPU class). Dense modes need a non-prequant base."
),
)
- # DiT-only levers the trainer implements. Undeclared, they were silently dropped by model_dump(); the defaults match DiffusionLoraConfig.
+ # DiT-only levers the trainer implements. Undeclared, they were silently dropped by model_dump();
+ # the defaults match DiffusionLoraConfig.
ema_decay: float = Field(
0.0, ge = 0.0, lt = 1.0, description = "EMA of the LoRA weights; 0 disables it"
)
@@ -979,8 +977,8 @@ class DiffusionMetricHistory(BaseModel):
loss: List[float] = Field(default_factory = list)
lr: List[Optional[float]] = Field(default_factory = list)
grad_norm: List[Optional[float]] = Field(default_factory = list)
- # The two halves of a joint video+audio objective (MiniMax-H3). All-null on every other
- # family, which is what lets the chart decide whether to draw the split curves at all.
+ # All-null on every family but MiniMax-H3, which is what lets the chart decide whether to draw the
+ # split curves at all.
video_loss: List[Optional[float]] = Field(default_factory = list)
audio_loss: List[Optional[float]] = Field(default_factory = list)
@@ -999,8 +997,7 @@ class DiffusionTrainingStatusResponse(BaseModel):
learning_rate: Optional[float] = None
# Total pre-clip gradient norm from the last optimizer step (health signal the UI charts).
grad_norm: Optional[float] = None
- # The last per-modality losses of a joint video+audio run (MiniMax-H3), None elsewhere. The
- # combined loss can hold steady while one modality degrades, so these are reported apart.
+ # The combined loss can hold steady while one modality degrades, so these are reported apart.
video_loss: Optional[float] = None
audio_loss: Optional[float] = None
num_images: Optional[int] = None
@@ -1009,15 +1006,16 @@ class DiffusionTrainingStatusResponse(BaseModel):
lora_path: Optional[str] = None
# The second, EMA-averaged adapter written in the run's ema subdir when ema_decay was enabled.
ema_path: Optional[str] = None
- # Where the adapter was mirrored into the Unsloth LoRA catalog, and what family / base it trained from, so the UI can deploy it.
+ # Where the adapter was mirrored into the Unsloth LoRA catalog, and what family / base it trained
+ # from, so the UI can deploy it.
catalog_path: Optional[str] = None
family: Optional[str] = None
base_model: Optional[str] = None
# Live throughput + peak VRAM (from the trainer's progress events).
samples_per_second: Optional[float] = None
peak_memory_gb: Optional[float] = None
- # Resume state for this job: the newest checkpoint bundle written, the step it holds, why one
- # could not be written (the Resume action's disabled tooltip), and where a resumed run started.
+ # The newest checkpoint bundle written, the step it holds, why one could not be written (the Resume
+ # action's disabled tooltip), and where a resumed run started.
checkpoint_path: Optional[str] = None
checkpoint_step: Optional[int] = None
resume_blocked_reason: Optional[str] = None
@@ -1049,13 +1047,13 @@ class DiffusionTrainingRunSummary(BaseModel):
ended_at: Optional[float] = None
# The run's adapter directory, which is also what a Resume replays as resume_from_checkpoint.
output_dir: Optional[str] = None
- # Whether this run can be continued, re-derived from the checkpoints on disk at read time (not
- # the value frozen when the run ended), with the step the newest bundle holds. When it cannot,
- # resume_blocked_reason says why, and the UI shows that as the disabled action's tooltip.
+ # Re-derived from the checkpoints on disk at read time, not the value frozen when the run ended,
+ # so deleting them takes the action away. When it cannot resume, resume_blocked_reason says why
+ # and the UI shows it as the disabled action's tooltip.
can_resume: bool = False
checkpoint_step: Optional[int] = None
- # The exact bundle a resume would continue. Clients send it back as resume_from_checkpoint so
- # they get the one they were shown, not whatever is newest in a folder two runs may share.
+ # The exact bundle a resume would continue, sent back so clients get the one they were shown rather
+ # than whatever is newest in a folder two runs may share.
checkpoint_path: Optional[str] = None
resume_blocked_reason: Optional[str] = None
# Lineage: the run this one continued, and the step it picked up from.
@@ -1086,8 +1084,11 @@ class DiffusionDatasetSummary(BaseModel):
name: str
path: str
image_count: int
- # Clips, for the families that train from video. Defaults to 0 so an older backend's
- # payload (and every image-only caller) stays valid.
+ # Defaults to 0 so an older backend's payload, and every image-only caller, stays valid.
+ # Clips, for the families that train from video. Defaults to 0 so an older backend's payload (and every
+ # image-only caller) stays valid.
+ # Clips, for the families that train from video. Defaults to 0 so an older backend's payload (and every
+ # image-only caller) stays valid.
clip_count: int = 0
caption_count: int
@@ -1106,22 +1107,20 @@ class DiffusionTrainableFamily(BaseModel):
qlora_vram_gb: Optional[int] = None
gated: bool = False
note: str = ""
- # base_precision modes this machine supports for the family (empty = no selector, e.g. SDXL), the recommended pick, and
- # whether regional torch.compile applies. Defaults keep older backends' payloads valid.
+ # base_precision modes this machine supports for the family (empty = no selector, e.g. SDXL), the
+ # recommended pick, and whether regional torch.compile applies. Defaults keep older backends'
+ # payloads valid.
precision_modes: List[str] = Field(default_factory = list)
recommended_precision: str = "nf4"
supports_compile: bool = False
- # Whether this family's loop writes checkpoint bundles. False makes the panel drop the
- # "Checkpoint every" control: save_steps is refused, not ignored, for such a family, so
- # offering the control means offering a value that rejects Start. Defaults True so an older
- # backend's payload, which has no such families in it, keeps the control.
+ # save_steps is refused, not ignored, for a checkpointless family, so offering the control means
+ # offering a value that rejects Start; defaults True so an older backend's payload keeps it.
supports_checkpoints: bool = True
- # The largest batch this family's loop can form, or None for unrestricted. 1 for a family
- # whose forward covers one packed sequence: the panel then drops the Batch control, since a
- # value above the cap is refused rather than clamped. Declared here or Pydantic silently
- # drops it from the response and the panel's clamp never receives its input.
+ # 1 for a family whose forward covers one packed sequence: a value above the cap is refused rather
+ # than clamped, and declaring it here is what stops Pydantic dropping it from the response.
max_train_batch_size: Optional[int] = None
- # When set, a LoRA trained on this family previews on this repo instead of the training base (Krea trains on Raw, runs on Turbo).
+ # When set, a LoRA trained on this family previews on this repo instead of the training base (Krea
+ # trains on Raw, runs on Turbo).
deploy_base: Optional[str] = None
# Variant-specific training-base to inference-base pairs, including public mirror ids.
deploy_bases: Dict[str, str] = Field(default_factory = dict)
@@ -1146,8 +1145,6 @@ class DiffusionDatasetUploadResponse(BaseModel):
name: str
path: str
image_count: int
- # Clips, for the families that train from video. Defaults to 0 so an older backend's
- # payload (and every image-only caller) stays valid.
clip_count: int = 0
caption_count: int
uploaded: int
@@ -1162,9 +1159,8 @@ class DiffusionDatasetImageRecord(BaseModel):
filename: str
caption: Optional[str] = None
caption_source: Literal["sidecar", "metadata", "none"] = "none"
- # Which kind of item this is. Clips are listed so a caller can see everything the folder
- # holds under a stem, but they carry no pixel dimensions and the labeling grid skips them.
- # Defaults to "image" so an older backend's payload still validates.
+ # Clips are listed so a caller can see everything the folder holds under a stem, but they carry no
+ # pixel dimensions and the labeling grid skips them; defaults to "image" for older payloads.
kind: Literal["image", "clip"] = "image"
width: int
height: int
@@ -1220,8 +1216,6 @@ class DiffusionDatasetImportResponse(BaseModel):
name: str
path: str
image_count: int
- # Clips, for the families that train from video. Defaults to 0 so an older backend's
- # payload (and every image-only caller) stays valid.
clip_count: int = 0
caption_count: int
imported: int
diff --git a/studio/backend/picker/routes/templates.py b/studio/backend/picker/routes/templates.py
index 8ca20e23798..f410fc104e2 100644
--- a/studio/backend/picker/routes/templates.py
+++ b/studio/backend/picker/routes/templates.py
@@ -38,8 +38,7 @@ async def get_default_chat_template_route(
hf_token: HfTokenArg = Depends(get_request_hf_token),
current_subject: str = Depends(get_current_subject),
) -> ModelTemplateResponse:
- # Cached repos resolve from disk, but a cache miss falls through to the hub and
- # offline that costs one retry backoff per candidate template file.
+ # A cache miss falls through to the hub, and offline that costs one retry backoff per candidate template file.
from core.inference.llama_cpp import _hf_offline_if_unreachable_for
def _read():
diff --git a/studio/backend/picker/schemas.py b/studio/backend/picker/schemas.py
index c43d769817d..c32cce3065e 100644
--- a/studio/backend/picker/schemas.py
+++ b/studio/backend/picker/schemas.py
@@ -5,9 +5,8 @@ from typing import Optional
from pydantic import BaseModel, Field, field_validator
-# Mirror the frontend's 64 KiB chat-template contract (per-model-config.ts) at
-# the API boundary so a direct caller cannot make Jinja parse an oversized
-# template. MaxBodyMiddleware only caps the whole request body, not this field.
+# Mirror the frontend's 64 KiB chat-template contract (per-model-config.ts) at the API boundary:
+# MaxBodyMiddleware only caps the whole request body, not this field.
MAX_CHAT_TEMPLATE_BYTES = 65_536
diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py
index d4d46da370f..8f0739b9b68 100644
--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py
+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/__init__.py
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-# Intentionally empty. Data-designer loads submodules lazily via qualified names
-# in plugin.py, so importing this package must not touch data_designer.engine.*
-# during Unsloth bootstrap (circular import).
+# Intentionally empty: data-designer loads submodules lazily by qualified name in plugin.py, so
+# importing this package must not touch data_designer.engine.* during bootstrap (circular import).
diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py
index 330e917f3f0..97411c57f73 100644
--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py
+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/impl.py
@@ -16,10 +16,8 @@ from .config import GitHubRepoSeedSource
from .scraper import ScrapeConfig, materialize_to_jsonl
-# In-process cache: config signature -> JSONL path. A recipe job reads the seed
-# multiple times (validation, preview, sampling); memoize to avoid re-scraping
-# GitHub on each call. Key uses a short SHA-256 of the token (never the raw value)
-# so token never hits memory twice and rotation invalidates cleanly.
+# Memoize seed materialization so a recipe job does not re-scrape GitHub on every call; the key
+# uses a short SHA-256 of the token, so the raw value never hits memory twice.
_SCRAPE_CACHE: dict[tuple, str] = {}
_SCRAPE_CACHE_LOCK = threading.Lock()
diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py
index 0ba3394ca3f..2d90b58e61a 100644
--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py
+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/gh_client.py
@@ -133,6 +133,7 @@ class GitHubClient:
)
def _check_rate_and_wait(self, kind: str) -> None:
+ # Reset remaining so we don't spin
if kind == "graphql":
remaining = self.graphql_remaining
reset = self.graphql_reset
@@ -144,7 +145,6 @@ class GitHubClient:
if remaining is not None and remaining < min_remaining:
if reset:
self._sleep_until(reset)
- # Reset remaining so we don't spin
if kind == "graphql":
self.graphql_remaining = None
else:
diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py
index a7ddaef5fef..7664510eb25 100644
--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py
+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/scraper.py
@@ -49,15 +49,12 @@ class RepoScraper:
self.base_dir = base_dir
self.client = client
self.trial_limits = trial_limits or {}
- # light=True uses trimmed GraphQL queries (no reviewThreads/reviews/
- # commits/timelineItems/files) so PR pages can be larger without
- # hitting GitHub's node-count ceiling.
+ # light=True uses trimmed GraphQL queries so PR pages can be larger without hitting GitHub's node-count ceiling.
self.light = light
self.repo_dir = base_dir / f"{owner}__{name}"
self.repo_dir.mkdir(parents = True, exist_ok = True)
self.state = StateStore(base_dir / "state" / f"{owner}__{name}.json")
- # Writers
self.writers: Dict[str, JsonlWriter] = {}
for key in (
"issues",
@@ -115,8 +112,8 @@ class RepoScraper:
return 0
total_new = 0
page = 0
- # Light query skips heavy nested fields; safe at 50/page. Clamp by
- # trial_limit so limit=1 asks for first:1, not a full 50-item page.
+ # The light query skips heavy nested fields, so 50/page is safe; clamp by trial_limit so limit=1
+ # does not fetch a full page.
page_cap = 50 if self.light else 15
trial_cap = self.trial_limits.get(key)
per_page = min(page_cap, trial_cap) if trial_cap and trial_cap > 0 else page_cap
@@ -222,9 +219,8 @@ class RepoScraper:
return 0
total_new = 0
page = 0
- # Heavy nested PR query caps at 3/page (GitHub node-count ceiling);
- # light query skips nested fields and goes to 25/page. Clamp by
- # trial_limit so limit=1 does not fetch a whole 25-item page.
+ # The heavy nested PR query caps at 3/page against GitHub's node-count ceiling and the light one at
+ # 25; clamp by trial_limit so limit=1 does not fetch a whole page.
page_cap = 25 if self.light else 3
trial_cap = self.trial_limits.get(key)
per_page = min(page_cap, trial_cap) if trial_cap and trial_cap > 0 else page_cap
diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py
index 1a580c45b35..330afdd9ca1 100644
--- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py
+++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py
@@ -73,9 +73,8 @@ def _read_line(raw: bytes, codepage: str) -> _Reading:
a JSON backslash, so the record fails to parse and its id is forgotten.
"""
as_utf8 = _parse(raw, "utf-8")
- # A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a
- # 76 MB shard, and these reach gigabytes. Only a dict, since key lookup falls
- # through to the codepage when UTF-8 yields none.
+ # A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a 76 MB shard, and
+ # these reach gigabytes.
if isinstance(as_utf8, dict):
return _Reading(as_utf8, None)
for encoding in (codepage, "latin-1", *_DOUBLE_BYTE_ENCODINGS):
@@ -90,11 +89,11 @@ def _read_line(raw: bytes, codepage: str) -> _Reading:
class _Scan(NamedTuple):
"""What a pass over an existing shard established about it."""
- legacy: bool # enough evidence to trust the codepage reading's keys
+ legacy: bool
readable: bool
- saw_non_ascii: bool # some line's meaning depends on the encoding
- utf8_keys: set # keys from lines UTF-8 could read
- legacy_keys: set # keys only the codepage reading yields
+ saw_non_ascii: bool
+ utf8_keys: set
+ legacy_keys: set
class StateStore:
@@ -103,12 +102,10 @@ class StateStore:
self.path.parent.mkdir(parents = True, exist_ok = True)
self._lock = threading.Lock()
self._data: Dict[str, Any] = {}
- # Read whole, and UTF-8 only unlike the shards below: a checkpoint holds
- # nothing but base64 cursors and booleans, so a codepage retry could only ever
- # add non-ASCII. That would resume on a mojibaked cursor, which GitHub rejects
- # with INVALID_CURSOR_ARGUMENTS, and the empty page it returns marks the stream
- # done and skips the rest for good. Dropping a damaged checkpoint re-scrapes
- # from the first page, which the writers dedup.
+ # Read whole, and UTF-8 only unlike the shards below: a checkpoint holds nothing but base64
+ # cursors and booleans, so a codepage retry could only resume on a mojibaked cursor GitHub rejects
+ # with INVALID_CURSOR_ARGUMENTS, whose empty page marks the stream done. Dropping a damaged
+ # checkpoint re-scrapes from page one, which the writers dedup.
if self.path.exists():
try:
raw = self.path.read_bytes()
@@ -165,9 +162,8 @@ class JsonlWriter:
if scan.legacy:
self._count_seen_keys |= scan.legacy_keys
if scan.saw_non_ascii or not scan.readable:
- # Never convert: the writing encoding is unrecoverable and guessing
- # mojibakes the records. Pure ASCII appends store identically under
- # every codepage, and json.loads turns the \uXXXX escapes back.
+ # Never convert: the writing encoding is unrecoverable and guessing mojibakes the records. Pure
+ # ASCII appends store identically under every codepage, and json.loads turns the escapes back.
encoding = "ascii"
self._ensure_ascii = True
self._fh = self.path.open("a", buffering = 1, encoding = encoding, errors = "strict")
diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py
index 47dc5db140c..2ba60b3ac92 100644
--- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py
+++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py
@@ -146,10 +146,7 @@ def materialize_unstructured_seed_dataset(
def materialize_multi_file_unstructured_seed(
- *,
- file_entries: list[tuple[Path, str]], # (extracted_txt_path, original_filename)
- chunk_size: int,
- chunk_overlap: int,
+ *, file_entries: list[tuple[Path, str]], chunk_size: int, chunk_overlap: int
) -> tuple[Path, list[dict[str, str]]]:
"""Chunk multiple files into one parquet dataset with a source_file column."""
chunk_size, chunk_overlap = resolve_chunking(chunk_size, chunk_overlap)
diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py
index 825b050e070..f5393292299 100644
--- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py
+++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py
@@ -30,8 +30,8 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]):
meta = json_mod.loads(meta_path.read_text(encoding = "utf-8"))
orig_name = meta.get("original_filename", path_obj.name)
except (json_mod.JSONDecodeError, OSError, UnicodeDecodeError):
- # Undecodable metadata is as malformed as invalid JSON, so
- # fall back to the file's own name rather than abort the seed.
+ # Undecodable metadata is as malformed as invalid JSON, so fall back to the file's own name rather
+ # than abort the seed.
pass
file_entries.append((path_obj, orig_name))
diff --git a/studio/backend/startup_banner.py b/studio/backend/startup_banner.py
index 145c7f6dcf5..b8bc8360524 100644
--- a/studio/backend/startup_banner.py
+++ b/studio/backend/startup_banner.py
@@ -114,12 +114,11 @@ def print_studio_access_banner(
else:
external_url = f"http://{display_host}:{port}"
- # The exact aliases the canned loopback_url below is valid for; any other bind
- # (e.g. a specific LAN IP) must show its real address, not http://127.0.0.1.
+ # The exact aliases the canned loopback_url is valid for; any other bind (e.g. a specific LAN IP)
+ # must show its real address, not http://127.0.0.1.
loopback_bind = bind_host in ("127.0.0.1", "localhost", "::1")
- # Use the loopback URL only when reachable on loopback; otherwise show
- # the actual bound address.
+ # Use the loopback URL only when reachable on loopback; otherwise show the actual bound address.
primary_url = loopback_url if listen_all or loopback_bind else external_url
api_base = primary_url
diff --git a/studio/backend/state/active_generations.py b/studio/backend/state/active_generations.py
index 0c0bb584cc3..9b17627d25b 100644
--- a/studio/backend/state/active_generations.py
+++ b/studio/backend/state/active_generations.py
@@ -23,8 +23,8 @@ import time
import uuid
from typing import Any, Optional
-# handle id -> entry. Keyed by handle, not thread_id: a tool continuation can register
-# before the previous leg unregisters, and one key would drop the other.
+# Keyed by handle, not thread_id: a tool continuation can register before the previous leg
+# unregisters, and one key would drop the other.
_ACTIVE: dict[str, dict[str, Any]] = {}
_LOCK = threading.Lock()
@@ -64,9 +64,8 @@ class ActiveGeneration:
def __enter__(self) -> "ActiveGeneration":
with _LOCK:
- # A durable supervisor registers before model loading starts. The
- # route later enters its normal tracker with that exact event/run;
- # borrow the outer registration so lifecycle counts stay truthful.
+ # A durable supervisor registers before model loading starts and the route later enters its normal
+ # tracker with that same event and run, so borrow the outer registration.
if self.run_id:
for entry in _ACTIVE.values():
if entry["run_id"] != self.run_id or entry["event"] is not self.cancel_event:
diff --git a/studio/backend/state/tool_approvals.py b/studio/backend/state/tool_approvals.py
index f66226b61d5..059f960022b 100644
--- a/studio/backend/state/tool_approvals.py
+++ b/studio/backend/state/tool_approvals.py
@@ -24,12 +24,11 @@ import secrets
import threading
from typing import Optional
-# Generous ceiling so a user can deliberate; cancellation (stop button /
-# disconnect) still breaks the wait early via ``cancel_event``.
+# Generous ceiling so a user can deliberate; the stop button or a disconnect still breaks the wait
+# early via cancel_event.
_DECISION_TIMEOUT = 3600.0
-# Fed to the model as the tool result when the user denies a call, so it
-# can adapt and keep responding instead of the turn ending abruptly.
+# Fed to the model as the tool result when the user denies a call, so it can adapt instead of the turn ending abruptly.
TOOL_REJECTED_MESSAGE = "The user declined to run this tool call."
_lock = threading.Lock()
diff --git a/studio/backend/storage/api_usage_db.py b/studio/backend/storage/api_usage_db.py
index d022929248c..ba36fd759e9 100644
--- a/studio/backend/storage/api_usage_db.py
+++ b/studio/backend/storage/api_usage_db.py
@@ -18,9 +18,8 @@ from typing import Callable, Optional
from storage.studio_db import get_connection, is_sqlite_busy_error
-# Kept aligned with the API monitor's defensive upper bound. The storage layer
-# validates independently because callers can invoke it directly in tests or
-# future integrations.
+# Kept aligned with the API monitor's defensive upper bound; the storage layer validates
+# independently because callers can invoke it directly.
MAX_TOKEN_COUNT = 1 << 40
MAX_RECEIPT_ID_CHARS = 128
MAX_SUBJECT_CHARS = 256
@@ -164,9 +163,8 @@ def record_api_usage(receipt: ApiUsageReceipt) -> bool:
except sqlite3.OperationalError as exc:
if not _is_busy_error(exc) or attempt + 1 == _WRITE_RETRIES:
raise
- # The worker is the only production writer of these receipts. A
- # short bounded backoff lets unrelated Unsloth transactions finish
- # without ever holding up the inference/streaming caller.
+ # The worker is the only production writer of these receipts, so a short bounded backoff lets
+ # unrelated transactions finish without holding up the streaming caller.
_sleep_after_busy(min(0.01 * (2**attempt), _WORKER_BUSY_RETRY_SECONDS))
if inserted:
@@ -214,7 +212,6 @@ class ApiUsageWriter:
self._stopped = True
self._queue.put_nowait(_STOP)
# Production calls this through asyncio.to_thread so even the bounded
- # wait cannot pause inference or the event loop.
self._thread.join(timeout = max(0.0, timeout))
drained = not self._thread.is_alive()
if not drained:
@@ -241,11 +238,9 @@ class ApiUsageWriter:
if not _is_busy_error(exc):
logger.warning("api usage receipt persistence failed", exc_info = True)
break
- # record_api_usage already made its bounded fast retries.
- # Retain this accepted item at the head of the single
- # writer until a normal long Unsloth transaction releases
- # SQLite. The stop sentinel remains behind it, so final
- # shutdown drains rather than silently dropping usage.
+ # record_api_usage already made its bounded fast retries: retain this accepted item at
+ # the head of
+ # the single writer until a long transaction releases SQLite, with the stop sentinel behind it.
busy_failures += 1
if busy_failures == 1 or busy_failures % 20 == 0:
logger.warning(
diff --git a/studio/backend/storage/chat_generation_runs_db.py b/studio/backend/storage/chat_generation_runs_db.py
index 8720dc92cc6..fcb097d30bd 100644
--- a/studio/backend/storage/chat_generation_runs_db.py
+++ b/studio/backend/storage/chat_generation_runs_db.py
@@ -22,9 +22,8 @@ _EVENTS_CHANGED = threading.Condition()
_RUN_TOMBSTONE_PREFIX = "chat-generation-run-tombstone:"
ChatGenerationEventInput = Union[tuple[str, dict[str, Any]], tuple[str, dict[str, Any], int]]
-# Progress lease columns, added here rather than in _ensure_schema so the base table
-# definition stays owned by studio_db. Named _schema_ready to match the flag the test
-# harness resets on every storage module when it swaps UNSLOTH_STUDIO_HOME.
+# Progress lease columns live here rather than in _ensure_schema so the base table stays owned by
+# studio_db; named _schema_ready to match the flag the test harness resets.
_schema_ready = False
_schema_lock = threading.Lock()
@@ -65,9 +64,8 @@ def _connect() -> sqlite3.Connection:
conn.commit()
_schema_ready = True
except sqlite3.OperationalError:
- # A writer holds the database. The columns are additive, so let this call through
- # and migrate later rather than turning contention into a failed history read;
- # _without_lease_columns below covers the paths that then cannot assume them.
+ # A writer holds the database, and the columns are additive, so let this call through and migrate
+ # later rather than turning contention into a failed history read.
conn.rollback()
except Exception:
conn.close()
@@ -208,8 +206,8 @@ def _touch_progress_locked(conn: sqlite3.Connection, run_id: str, tokens: int) -
(now, now, max(0, int(tokens)), run_id),
)
except sqlite3.OperationalError as exc:
- # The migration has not landed yet. The run keeps streaming and simply ages out
- # on started_at/created_at, which the sweep already falls back to.
+ # The migration has not landed yet, so the run ages out on started_at/created_at, which the sweep
+ # already falls back to.
if not _missing_lease_columns(exc):
raise
@@ -548,8 +546,8 @@ def append_events(
conn.commit()
return []
sequences = _append_events_locked(conn, run_id, batch)
- # The producer's only regular write, so it is also the lease renewal: output
- # reaching the database is the definition of progress this sweep reaps on.
+ # The producer's only regular write, so it is also the lease renewal: output reaching the database
+ # is the definition of progress this sweep reaps on.
_touch_progress_locked(conn, run_id, sum(1 for event in batch if event[0] == "chunk"))
_commit(conn, notify = True)
return sequences
@@ -769,8 +767,8 @@ def reconcile_runs(
WHERE status IN ('queued','running','cancelling')"""
args: tuple[Any, ...] = ()
if stale_after_ms is not None:
- # started_at/created_at carry a run that has not streamed anything yet,
- # so a producer wedged before its first token still ages out.
+ # started_at/created_at carry a run that has not streamed anything yet, so a producer wedged before
+ # its first token still ages out.
sql += " AND COALESCE(progress_at, started_at, created_at) <= ?"
args = (completed - int(stale_after_ms),)
try:
@@ -778,11 +776,9 @@ def reconcile_runs(
except sqlite3.OperationalError as exc:
if not _missing_lease_columns(exc):
raise
- # Contention blocked the migration, so no run can persist progress. Falling
- # back to started_at/created_at is not conservative but the opposite: those
- # stamps are older by the whole life of the run, so one that streamed a chunk
- # moments ago is reaped once its total AGE passes the timeout. Boot reconcile
- # passes no stale_after_ms and is unaffected.
+ # Contention blocked the migration, so falling back to started_at/created_at is the opposite of
+ # conservative: those stamps are older by the whole life of the run, so one that streamed moments
+ # ago is reaped once its total AGE passes the timeout. Boot reconcile passes no stale_after_ms.
if stale_after_ms is not None:
conn.rollback()
return []
@@ -793,9 +789,8 @@ def reconcile_runs(
).fetchall()
for row in rows:
run_id = row["id"]
- # A Stop that was already recorded outlives the restart. Reporting it as a
- # backend failure would tell the user Studio broke when they stopped it, and
- # finish_run settles this same case as cancelled.
+ # A Stop that was already recorded outlives the restart, and reporting it as a backend failure
+ # would tell the user Studio broke when they stopped it; finish_run settles this case as cancelled.
if str(row["status"]) == "cancelling" or bool(row["cancel_requested"]):
status, finish_reason, message = "cancelled", "cancelled", None
terminal = ("run.cancelled", {"status": status, "finishReason": finish_reason})
@@ -809,8 +804,8 @@ def reconcile_runs(
updated_at=?, completed_at=? WHERE id=?""",
(status, finish_reason, message, completed, completed, run_id),
)
- # Stamps incomplete: {reason: "interrupted"} on the assistant message, which
- # is what releases the frontend's "generating" state and restores Send.
+ # Stamps incomplete on the assistant message, which is what releases the frontend's "generating"
+ # state and restores Send.
_sync_assistant_status_locked(conn, run_id, status)
settled.append(str(run_id))
_commit(conn, notify = bool(settled))
diff --git a/studio/backend/storage/profile_stats_db.py b/studio/backend/storage/profile_stats_db.py
index 477b4ee9edd..ba109cac443 100644
--- a/studio/backend/storage/profile_stats_db.py
+++ b/studio/backend/storage/profile_stats_db.py
@@ -33,8 +33,8 @@ from storage.studio_db import count_chat_message_attachments, get_connection
logger = get_logger(__name__)
-# Gaps longer than this end a "sitting at the keyboard" stretch: without the cap
-# a thread reopened a week later would report a week-long chat.
+# Gaps longer than this end a sitting-at-the-keyboard stretch: without the cap a thread reopened a
+# week later would report a week-long chat.
SESSION_GAP_SECONDS = 30 * 60
# Cap on the daily activity series handed to the UI (the heatmap draws a year).
MAX_DAILY_DAYS = 366
@@ -43,8 +43,8 @@ MAX_TZ_OFFSET_MINUTES = 14 * 60
# Top-N lists returned to the client.
TOP_MODELS = 8
RECENT_RUNS = 5
-# Serve a memoised payload for this long even if the fingerprint is unchanged,
-# so a chat that is mid-stream still refreshes reasonably promptly.
+# Serve a memoised payload for this long even when the fingerprint is unchanged, so a chat that is
+# mid-stream still refreshes promptly.
CACHE_TTL_SECONDS = 20.0
_cache_lock = threading.Lock()
@@ -227,7 +227,6 @@ def _fold_api_usage(conn, zone, subject: str) -> _ApiUsageFold:
fold.prompt_tokens += prompt_tokens
fold.completion_tokens += completion_tokens
# Preserve the provider's authoritative total even when it differs
- # from the input/output sum.
fold.total_tokens += total_tokens
fold.requests += 1
@@ -242,8 +241,7 @@ def _fold_api_usage(conn, zone, subject: str) -> _ApiUsageFold:
model_id,
{"id": model_id, "label": _model_label(model_id), "messages": 0, "tokens": 0},
)
- # One terminal API request represents one model response in the
- # combined leaderboard.
+ # One terminal API request represents one model response in the combined leaderboard.
model["messages"] += 1
model["tokens"] += total_tokens
return fold
@@ -362,20 +360,15 @@ def _fold_messages(conn, zone) -> _MessageFold:
thread_messages = 0
previous_created = None
- # Compare mode stores one thread per pane under a shared pair_id, and
- # the sidebar shows them as a single conversation. Count them that way
- # too, or one comparison inflates the chat total and drags the average
- # tokens per chat down.
+ # Compare mode stores one thread per pane under a shared pair_id and the sidebar shows them as a
+ # single conversation, so counting them twice inflates the chat total.
conversation_id = row["pair_id"] or thread_id
# A fork is its own visible conversation, so it counts towards the chat
- # total from the moment it exists, before any new turn is added.
fold.threads.add(conversation_id)
- # Forking clones the whole ancestry into the new thread, keeping each
- # copy's original timestamp. Skip a clone while the row it was taken
- # from is still there to be counted; once that original is gone, only
- # the elected fork stands in for it, so nothing is lost or doubled.
+ # Forking clones the whole ancestry keeping each copy's timestamp, so skip a clone while the row
+ # it came from is still countable; once that original is gone the elected fork stands in.
source_id = row["forked_from_thread_id"]
if source_id and created_at < _as_int(row["thread_created_at"]):
original = (source_id, created_at, row["role"])
@@ -418,9 +411,8 @@ def _fold_messages(conn, zone) -> _MessageFold:
usage = usage if isinstance(usage, dict) else {}
timing = timing if isinstance(timing, dict) else {}
- # llama.cpp reports its own counters under serverTimings when the
- # provider sends no usage chunk. The response-details sheet already
- # falls back to these, so the totals here have to as well.
+ # llama.cpp reports its own counters under serverTimings when the provider sends no usage chunk, and
+ # the response-details sheet already falls back to these.
server = metadata.get("serverTimings")
server = server if isinstance(server, dict) else {}
@@ -431,7 +423,6 @@ def _fold_messages(conn, zone) -> _MessageFold:
total_tokens = _as_int(usage.get("totalTokens"))
if completion_tokens == 0:
# Local engines occasionally omit the usage chunk; the adapter's
- # own token count is the next best estimate.
completion_tokens = _as_int(timing.get("tokenCount"))
if total_tokens == 0:
total_tokens = prompt_tokens + completion_tokens
@@ -445,11 +436,8 @@ def _fold_messages(conn, zone) -> _MessageFold:
fold.tool_calls += _as_int(timing.get("toolCallCount"))
message_tokens = total_tokens
- # responseDetails carries the model that actually answered, which
- # differs from the requested checkpoint whenever a provider routes
- # or resolves an alias. contextUsage.modelId is the request, so it
- # is only the fallback. The thread's model_id is never used: it
- # tracks the current selection, not the one that ran.
+ # responseDetails carries the model that actually answered, which differs from the requested
+ # checkpoint whenever a provider routes or resolves an alias.
details = metadata.get("responseDetails")
details = details if isinstance(details, dict) else {}
model_id = _clean_str(details.get("responseModelId")) or _clean_str(
@@ -555,18 +543,16 @@ def _training_stats(conn) -> dict[str, Any]:
"""
).fetchone()
- # A resumed run continues its source's step and token counters from the
- # checkpoint, so both absolute totals already include the source's work.
- # Only a run superseded by a resume is dropped: create_run's claim sets
- # resume_blocked while leaving output_dir intact, whereas cancelling clears
- # output_dir, so a cancelled run keeps contributing the work it did do.
+ # A resumed run continues its source's counters, so only a run superseded by a resume is dropped:
+ # create_run's claim sets resume_blocked while leaving output_dir intact, whereas cancelling
+ # clears it, so a cancelled run keeps the work it did do.
steps = conn.execute(
f"SELECT COALESCE(SUM(r.final_step), 0) FROM training_runs r WHERE NOT ({_superseded()})"
).fetchone()[0]
- # num_tokens is state.num_input_tokens_seen, a running total logged at each
- # step, so summing the samples multiplies the real figure. Take each run's
- # final counter, the same value get_run_metrics reports.
+ # num_tokens is state.num_input_tokens_seen, a running total logged at each step, so summing the
+ # samples multiplies the real figure; take each run's final counter, the value get_run_metrics
+ # reports.
tokens = conn.execute(
f"""
SELECT COALESCE(SUM(run_tokens), 0) FROM (
@@ -603,7 +589,6 @@ def _training_stats(conn) -> dict[str, Any]:
{
"id": item["id"],
# A renamed run keeps the name the user gave it; otherwise fall
- # back to the short model label rather than the full repo id.
"name": _clean_str(item["display_name"]) or _model_label(item["model_name"] or ""),
"modelLabel": _model_label(item["model_name"] or ""),
"datasetLabel": _model_label(item["dataset_name"] or ""),
@@ -681,14 +666,13 @@ def compute_profile_stats(
_merge_api_activity(fold, api_fold)
training = _training_stats(conn)
- # "Today" has to match the buckets above, or the newest column and the
- # current streak drift by a day whenever the caller is elsewhere.
+ # "Today" has to match the buckets above, or the newest column and the current streak drift by a day
+ # whenever the caller is elsewhere.
today = (_local_stamp(int(time.time() * 1000), zone) or datetime.now()).date()
streak = _streaks(set(fold.by_day.keys()), today)
daily = _daily_series(fold, today, days)
- # The grid stops at today and the streaks ignore anything later, so the
- # day-based headlines have to as well. A skewed client clock would
+ # The grid stops at today and the streaks ignore anything later, so a skewed client clock would
# otherwise name a peak day that is nowhere in the chart.
past_days = {day: bucket for day, bucket in fold.by_day.items() if day <= today}
peak_day = max(past_days.items(), key = lambda item: item[1]["tokens"], default = None)
diff --git a/studio/backend/storage/providers_db.py b/studio/backend/storage/providers_db.py
index 5d49bef2fb7..55cb9a25461 100644
--- a/studio/backend/storage/providers_db.py
+++ b/studio/backend/storage/providers_db.py
@@ -113,8 +113,8 @@ def provider_bundle_transaction() -> Iterator[sqlite3.Connection]:
a new endpoint with the previous key (or the inverse) while a provider edit
is in progress.
"""
- # Ensure both tables exist before opening the transaction. The credential
- # module commits schema initialization on its own connection.
+ # Ensure both tables exist before opening the transaction. The credential module commits schema
+ # initialization on its own connection.
from storage import credential_secrets
credential_secrets.ensure_schema()
diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.py
index 9641eda868b..1e53fda315f 100644
--- a/studio/backend/storage/rag_db.py
+++ b/studio/backend/storage/rag_db.py
@@ -47,15 +47,12 @@ class RagExtensionUnavailable(RuntimeError):
_schema_lock = threading.Lock()
_schema_ready = False
-# The dylib is either there or it is not, and the UI polls the KB list on a timer, so
-# one warning per process says everything the repeat lines would. Same shape as the
-# per-job throttle in hub/services/snapshot_progress.py.
+# The dylib is either there or it is not, and the UI polls the KB list on a timer, so one warning
+# per process says everything the repeats would.
_unavailable_lock = threading.Lock()
_unavailable_warned = False
-# Set once a connection has actually loaded the extension, so the request gate can
-# answer without reopening the database. Only the positive verdict is kept: a failure
-# stays retried per connection exactly as it was before, so a one-off cannot latch RAG
-# off for the rest of the session.
+# Only the positive verdict is kept: a failure stays retried per connection, so a one-off cannot
+# latch RAG off for the rest of the session.
_extension_loaded = False
@@ -246,8 +243,8 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
cols = {r[1] for r in conn.execute("PRAGMA table_info(documents)").fetchall()}
if "project_id" not in cols:
conn.execute("ALTER TABLE documents ADD COLUMN project_id TEXT")
- # Lazy upgrade: which embedder produced a document's vectors (NULL = legacy,
- # assumed current). Dedupe re-ingests when it no longer matches.
+ # Which embedder produced a document's vectors (NULL = legacy, assumed current); dedupe re-ingests
+ # when it no longer matches.
if "embedding_model" not in cols:
conn.execute("ALTER TABLE documents ADD COLUMN embedding_model TEXT")
# Folder ownership makes crash cleanup unambiguous without changing retrieval.
@@ -255,24 +252,22 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
conn.execute("ALTER TABLE documents ADD COLUMN linked_folder_id TEXT")
if "linked_relative_path" not in cols:
conn.execute("ALTER TABLE documents ADD COLUMN linked_relative_path TEXT")
- # How many messages an archived turn was rendered from. NULL for everything else and
- # for older archives, which fall back to counting role labels in the rendered text.
+ # NULL for everything else and for older archives, which fall back to counting role labels in the rendered text.
if "archive_messages" not in cols:
conn.execute("ALTER TABLE documents ADD COLUMN archive_messages INTEGER")
- # An archived turn group's position in its conversation. NULL elsewhere and for older
- # archives, which fall back to created_at ordering. Not backfilled: created_at cannot
+ # NULL for older archives, which fall back to created_at ordering. Not backfilled: created_at cannot
# recover the order within a compaction epoch.
if "archive_ordinal" not in cols:
conn.execute("ALTER TABLE documents ADD COLUMN archive_ordinal INTEGER")
- # Partial, so it is empty until a chat is compacted and the MAX() that allocates the
- # next ordinal is an index probe rather than a scan.
+ # Partial, so it is empty until a chat is compacted and the MAX() that allocates the next ordinal is
+ # an index probe rather than a scan.
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_documents_archive_ordinal "
"ON documents(scope, archive_ordinal) WHERE archive_ordinal IS NOT NULL"
)
# After the ALTER that adds the column on an older database. Partial, so it holds only
- # folder-owned rows and is empty with nothing linked, which keeps the lexical fast-path
- # gate an index probe rather than a scan of documents.
+ # folder-owned rows and is empty with nothing linked, which keeps the lexical fast-path gate an
+ # index probe rather than a scan.
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_documents_linked_folder "
"ON documents(linked_folder_id) WHERE linked_folder_id IS NOT NULL"
@@ -312,9 +307,8 @@ def get_connection() -> sqlite3.Connection:
ensure_dir(db_path.parent)
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
- # Wait for a lock instead of erroring immediately: a figure/scan-heavy ingest can
- # hold its connection across many seconds of vision calls, and a concurrent ingest
- # or autoinject read would otherwise hit "database is locked".
+ # Wait for a lock instead of erroring immediately: a figure-heavy ingest can hold its connection
+ # across many seconds of vision calls, and a concurrent read would hit "database is locked".
conn.execute("PRAGMA busy_timeout = 5000")
try:
conn.enable_load_extension(True)
@@ -324,8 +318,8 @@ def get_connection() -> sqlite3.Connection:
conn.close()
_warn_unavailable_once(exc)
raise RagExtensionUnavailable(_RAG_UNAVAILABLE_MSG) from exc
- # Set before the schema step: the library loaded, so RAG runs on this machine
- # whatever a broken database does next. A monotonic flip, so no lock.
+ # Set before the schema step: the library loaded, so RAG runs on this machine whatever a broken
+ # database does next. A monotonic flip, so no lock.
_extension_loaded = True
if not _schema_ready:
@@ -423,17 +417,16 @@ def reconcile_orphaned_ingestion_jobs() -> int:
live backend is left alone until its lease expires. No-op without RAG. Returns
the number of jobs reset.
"""
- # rag_available(), not RAG_AVAILABLE: a venv with the package but no vec0 binary
- # would otherwise raise out of startup and be logged as a reconcile failure, when
- # there is simply nothing here to reconcile.
+ # rag_available(), not RAG_AVAILABLE: a venv with the package but no vec0 binary would raise out of
+ # startup and be logged as a reconcile failure when there is nothing to reconcile.
if not rag_available():
return 0
conn = get_connection()
try:
conn.execute("BEGIN IMMEDIATE")
now = datetime.now(timezone.utc).isoformat()
- # 'cancelled' is terminal too: the job stopped because its document was deleted, so
- # rewriting it to failed would report a deliberate cancellation as an indexing failure.
+ # "cancelled" is terminal too: the job stopped because its document was deleted, so rewriting it to
+ # failed would report a deliberate cancellation as an indexing failure.
rows = conn.execute(
"SELECT j.id, j.document_id FROM ingestion_jobs j "
"WHERE j.status NOT IN ('completed', 'failed', 'cancelled') AND NOT EXISTS ("
@@ -446,10 +439,8 @@ def reconcile_orphaned_ingestion_jobs() -> int:
"SELECT status FROM documents WHERE id=?", (row["document_id"],)
).fetchone()
if doc is not None and doc["status"] == "completed":
- # Worker finished indexing before the crash but didn't retire the
- # job row. Mark the job completed (not failed) and keep its chunks,
- # so the UI's getJob fallback after restart doesn't flag a
- # searchable document as a failed ingestion.
+ # The worker finished indexing before the crash but did not retire the job row: mark it completed
+ # and keep its chunks, so the UI does not flag a searchable document as a failed ingestion.
conn.execute(
"UPDATE ingestion_jobs SET status='completed', stage='done', "
"progress=1.0, error=NULL WHERE id=?",
@@ -466,9 +457,8 @@ def reconcile_orphaned_ingestion_jobs() -> int:
"WHERE id=? AND status NOT IN ('completed', 'failed')",
(row["document_id"],),
)
- # A failed or still-in-flight doc must not leave citable chunks
- # (retrieval filters by scope, not status); also drops any chunks of a
- # doc already 'failed' before the crash.
+ # A failed or still-in-flight doc must not leave citable chunks, since retrieval filters by scope
+ # and not status.
_delete_document_chunks(conn, row["document_id"])
conn.execute(
"DELETE FROM rag_job_leases WHERE kind='ingestion' AND job_id=?",
diff --git a/studio/backend/storage/research_runs_db.py b/studio/backend/storage/research_runs_db.py
index 44f7779d475..6c4f1740191 100644
--- a/studio/backend/storage/research_runs_db.py
+++ b/studio/backend/storage/research_runs_db.py
@@ -172,9 +172,8 @@ def _bind_assistant_locked(
existing_run_id = (
existing_metadata.get("researchRunId") if isinstance(existing_metadata, dict) else None
)
- # Only bind to an empty placeholder or this run's own message: an untagged
- # reply carries text/source parts that _update_assistant drops on completion,
- # so binding one silently overwrites an existing answer.
+ # Only bind to an empty placeholder or this run's own message: an untagged reply carries parts
+ # _update_assistant drops on completion, so binding one silently overwrites an existing answer.
existing_answer = any(
isinstance(part, dict)
and (
@@ -363,11 +362,10 @@ def rebind_cancelled(
plan_revision = revision,
created = now,
)
- # retry_count is the attempt epoch every event is stamped with, and the new question
- # is a new attempt: without the bump its report would carry the stopped question's
- # reasoning (get_reasoning_text joins every event at the run's current attempt) and its
- # activity panel would fold the two questions together. The retry BUDGET is counted per
- # question in retry(), so spending an epoch here does not spend a retry.
+ # retry_count is the attempt epoch every event is stamped with, and a new question is a new
+ # attempt: without the bump its report would carry the stopped question's reasoning, since
+ # get_reasoning_text joins every event at the run's current attempt. The retry BUDGET is counted
+ # per question, so spending an epoch here spends no retry.
conn.execute(
"UPDATE research_runs SET user_message_id=?, assistant_message_id=?, "
"status='planning', cancel_requested=0, plan_json=NULL, plan_hash=NULL, "
@@ -389,8 +387,8 @@ def rebind_cancelled(
conn.execute("DELETE FROM research_plan_steps WHERE run_id=?", (run_id,))
conn.execute("DELETE FROM research_sources WHERE run_id=?", (run_id,))
conn.execute("DELETE FROM research_document_sources WHERE run_id=?", (run_id,))
- # Events replay into the activity panel, so a kept approval would show "Plan approved"
- # for a question this run no longer researches.
+ # Events replay into the activity panel, so a kept approval would show "Plan approved" for a
+ # question this run no longer researches.
conn.execute(
"DELETE FROM research_events WHERE run_id=? AND event_type='run.approved'", (run_id,)
)
@@ -787,9 +785,8 @@ def retry(run_id: str, max_retries: int = 3) -> str:
raise KeyError(run_id)
if row["status"] not in {"failed", "cancelled"}:
raise ResearchConflictError("Only failed or cancelled runs can be retried")
- # Counted per question, not per run row: a thread holds one run for its lifetime and
- # re-points it at each new question, so a raw retry_count would hand a fresh question
- # whatever the stopped one left over -- and nothing at all once three were spent.
+ # Counted per question, not per run row: a thread re-points one run at each new question, so a raw
+ # retry_count would hand a fresh question whatever the stopped one left over.
spent = conn.execute(
"SELECT COUNT(*) FROM research_events WHERE run_id=? AND event_type='run.retried' "
"AND seq > COALESCE((SELECT MAX(seq) FROM research_events "
@@ -871,8 +868,8 @@ def _has_claimable(now: int) -> bool:
def claim_next(worker_id: str, lease_ms: int = 120_000) -> dict | None:
- # Advisory only: the row can disappear between this probe and the transaction below,
- # which the "row is None" branch inside already handles.
+ # Advisory only: the row can disappear between this probe and the transaction below, which the "row
+ # is None" branch already handles.
if not _has_claimable(now_ms()):
return None
conn = get_connection()
@@ -1342,7 +1339,6 @@ def wait_for_events(
return events
with _EVENTS_CHANGED:
# Recheck under the condition lock so a commit cannot be missed between
- # the initial query and waiting for its notification.
events = list_events(run_id, after)
if events:
return events
diff --git a/studio/backend/tests/test_mcp_upgrade_compat.py b/studio/backend/tests/test_mcp_upgrade_compat.py
index 434bc698be0..c1adc3f98a1 100644
--- a/studio/backend/tests/test_mcp_upgrade_compat.py
+++ b/studio/backend/tests/test_mcp_upgrade_compat.py
@@ -29,6 +29,7 @@ ENV = "UNSLOTH_STUDIO_MAX_MCP_SESSIONS"
# --------------------------------------------------------------------------
+
# Entry points an old install may still call
# --------------------------------------------------------------------------
@@ -45,6 +46,7 @@ def test_the_old_close_name_takes_the_same_arguments():
# --------------------------------------------------------------------------
+
# Session cap: the name changed, the setting must not
# --------------------------------------------------------------------------
@@ -90,6 +92,7 @@ def test_the_cap_never_drops_below_one(monkeypatch, raw, expected):
# --------------------------------------------------------------------------
+
# The fastmcp surface this module relies on
# --------------------------------------------------------------------------
@@ -127,6 +130,7 @@ def test_tool_errors_are_still_distinguishable_from_transport_errors():
# --------------------------------------------------------------------------
+
# Rows written by an older Studio
# --------------------------------------------------------------------------
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index 12d1afeed3b..e1956b82c16 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -16,14 +16,13 @@ import os, importlib.util, platform, sys
os.environ["UNSLOTH_IS_PRESENT"] = "1"
-# Transformers 4.x imports TensorFlow / Flax merely because they are installed
-# (`processing_utils` -> `image_transforms`), breaking Unsloth, which uses neither.
-# It reads these variables once at its own import, so this has to land first. An
-# explicit opt-in still wins; 5.x dropped both backends and ignores all of this.
+# Transformers 4.x imports TensorFlow / Flax merely because they are installed (processing_utils
+# -> image_transforms); it reads these variables once at its own import, so they have to land
+# first. An explicit opt-in still wins, and 5.x ignores all of this.
if "transformers" not in sys.modules:
- _TRUE = {"1", "ON", "YES", "TRUE"} # Transformers' ENV_VARS_TRUE_VALUES
- # Overwrite, not `setdefault`: unset means `AUTO`, i.e. "enable if installed".
- # An imported backend is in use though, so opting it out breaks a `from_tf` load.
+ _TRUE = {"1", "ON", "YES", "TRUE"}
+ # Overwrite, not `setdefault`: unset means AUTO ("enable if installed"), but an already-imported
+ # backend is in use, so opting it out breaks a `from_tf` load.
for _var, _modules, _opt_ins in (
("USE_TF", ("tensorflow",), ("USE_TF", "FORCE_TF_AVAILABLE")),
("USE_FLAX", ("flax", "jax"), ("USE_FLAX",)),
@@ -35,12 +34,10 @@ if "transformers" not in sys.modules:
os.environ[_var] = "0"
del _TRUE, _var, _modules, _opt_ins
else:
- # Transformers derives `_tf_available` / `_flax_available` from `find_spec` alone,
- # so clearing the cached flags keeps `image_transforms` off a broken backend.
- # `"transformers" in sys.modules` does not mean it finished importing (Python
- # publishes the module before its body runs), so write the variables too: inert
- # once read, decisive in that window, unconditional since waiting deadlocks.
- _TRUE = {"1", "ON", "YES", "TRUE"} # Transformers' ENV_VARS_TRUE_VALUES
+ # Transformers derives _tf_available / _flax_available from find_spec alone, and being in
+ # sys.modules does not mean its body ran, so clear the cached flags AND write the variables:
+ # inert once read, decisive in that window, unconditional since waiting deadlocks.
+ _TRUE = {"1", "ON", "YES", "TRUE"}
_import_utils = sys.modules.get("transformers.utils.import_utils")
for _var, _flag, _const, _modules, _opt_ins, _cached in (
(
@@ -64,14 +61,14 @@ else:
continue
if any(os.environ.get(_v, "").upper() in _TRUE for _v in _opt_ins):
continue
- # An opt-in can be consumed and then restored, so read the snapshot
- # Transformers used. Env `USE_FLAX` lands in the constant `USE_JAX`.
+ # An opt-in can be consumed and then restored, so read the snapshot Transformers used. Env USE_FLAX
+ # lands in the constant USE_JAX.
if any(str(getattr(_import_utils, _v, "")).upper() in _TRUE for _v in _cached):
continue
os.environ[_var] = "0"
try:
- # `import_utils` copies the env into `USE_TF` / `USE_JAX` (lines 102-104)
- # and derives the flags at 264 / 355, so mid-body only the constant works.
+ # import_utils copies the env into USE_TF / USE_JAX (lines 102-104) and derives the flags at
+ # 264 / 355, so mid-body only the constant works.
if hasattr(_import_utils, _const):
setattr(_import_utils, _const, "0")
# Absent on 5.x, and a module proxy can refuse the write.
@@ -81,16 +78,14 @@ else:
pass
del _TRUE, _import_utils, _var, _flag, _const, _modules, _opt_ins, _cached
-# Relax Metal's context-store timeout before MLX modules can initialize Metal.
-# Keep an explicit user value authoritative.
+# Relax Metal's context-store timeout before MLX modules can initialize Metal; an explicit user
+# value stays authoritative.
if platform.system() == "Darwin" and platform.machine() == "arm64":
os.environ.setdefault("AGX_RELAX_CDM_CTXSTORE_TIMEOUT", "1")
+# Legacy Windows consoles (cp1252) cannot encode Unsloth's emoji/box-drawing glyphs and crash with
+# UnicodeEncodeError; errors="replace" guarantees no crash on an unencodable glyph.
# ── Windows console UTF-8 safety ─────────────────────────────────────────────
-# Legacy Windows consoles (cp1252) can't encode Unsloth's emoji/box-drawing
-# glyphs and crash with UnicodeEncodeError. Force stdout/stderr to UTF-8 only on
-# Windows and only when not already UTF-8; no-op elsewhere. errors="replace"
-# guarantees we never crash on an unencodable glyph.
if platform.system() == "Windows":
import sys as _sys
for _name in ("stdout", "stderr"):
@@ -125,9 +120,8 @@ def _bytes_to_gb(value):
def _is_mlx_available():
- # Transitional import barrier: keep non-Apple-Silicon imports from touching
- # unsloth_zoo until unsloth_zoo.mlx is import-safe on GPU hosts. Then this
- # can collapse back to the centralized zoo runtime call below.
+ # Transitional import barrier: keep non-Apple-Silicon imports from touching unsloth_zoo until
+ # unsloth_zoo.mlx is import-safe on GPU hosts.
if (
os.environ.get("UNSLOTH_FORCE_GPU_PATH", "0") == "1"
or platform.system() != "Darwin"
@@ -146,8 +140,8 @@ def _is_mlx_available():
_IS_MLX = _is_mlx_available()
if _IS_MLX:
- # _gpu_init does this on the GPU path; the MLX path never reaches it, so
- # torchao 0.18 + torch < 2.10 dies on `ScalingType`. No-op otherwise.
+ # _gpu_init does this on the GPU path and the MLX path never reaches it, so torchao 0.18 + torch <
+ # 2.10 dies on `ScalingType`.
try:
from .import_fixes import fix_torchao_torch_symbol_skew as _fix_torchao
_fix_torchao()
@@ -155,16 +149,15 @@ if _IS_MLX:
except Exception:
pass
try:
- # Same reason: MLX audio reaches xcodec2 -> torchtune -> the old
- # torchao.dtypes.nf4tensor path.
+ # Same reason: MLX audio reaches xcodec2 -> torchtune -> the old torchao.dtypes.nf4tensor path.
from .import_fixes import fix_torchao_nf4tensor_move as _fix_nf4
_fix_nf4()
del _fix_nf4
except Exception:
pass
try:
- # Same reason: this branch imports transformers itself further down, so a
- # --no-deps floor miss surfaces here with the same wrong remedy.
+ # Same reason: this branch imports transformers itself further down, so a --no-deps floor miss would
+ # surface here with the same wrong remedy.
from .import_fixes import check_transformers_dependency_versions as _check_tf_deps
_check_tf_deps()
del _check_tf_deps
@@ -177,9 +170,8 @@ if _IS_MLX:
"Unsloth: MLX support requires `unsloth-zoo` with MLX modules. "
"Reinstall with `pip install unsloth-zoo` or rerun install.sh."
) from _e
- # An older unsloth-zoo satisfies `import unsloth_zoo` but lacks the
- # mlx.trainer / mlx.loader submodules. Surface a friendly install hint
- # instead of a raw ImportError on the submodule path.
+ # An older unsloth-zoo satisfies `import unsloth_zoo` but lacks the mlx.trainer / mlx.loader
+ # submodules; surface an install hint instead of a raw ImportError.
try:
from unsloth_zoo.mlx.trainer import (
MLXTrainer,
@@ -202,9 +194,8 @@ if _IS_MLX:
import types as _types
import warnings as _warnings
- # unsloth_zoo is a different distribution, pinned `>=`, so borrowing its number
- # reported neither the installed core nor the latest zoo. `_version` imports nothing,
- # so this stays torch-free while agreeing with the GPU path and `pip show unsloth`.
+ # unsloth_zoo is a different distribution, pinned >=, so borrowing its number reported neither the
+ # installed core nor the latest zoo. `_version` imports nothing, so this stays torch-free.
from ._version import __version__
DEVICE_TYPE = "mlx"
@@ -259,8 +250,8 @@ if _IS_MLX:
target = kwargs.get("device", device)
if _is_mlx_cuda_device_target(target):
return self
- # device given by keyword: don't also pass the positional None, or the
- # original raises "multiple values for 'device'" (e.g. .to(device="cpu")).
+ # device given by keyword: do not also pass the positional None, or the original raises "multiple
+ # values for 'device'".
if "device" in kwargs:
return original_to(self, *args, **kwargs)
return original_to(self, device, *args, **kwargs)
@@ -271,8 +262,8 @@ if _IS_MLX:
_patch_mlx_batch_encoding_to_cuda()
- # Load raw_text helpers without executing dataprep/__init__.py, which
- # imports synthetic.py -> torch and would defeat the torch-free MLX path.
+ # Load raw_text helpers without executing dataprep/__init__.py, which imports synthetic.py -> torch
+ # and would defeat the torch-free MLX path.
from pathlib import Path as _Path
_raw_text_path = _Path(__file__).resolve().parent / "dataprep" / "raw_text.py"
@@ -635,8 +626,8 @@ if _IS_MLX:
try:
return _normalize_mlx_optimizer_name(value)
except ValueError:
- # Older unsloth-zoo lacks CUDA/TRL optimizer aliases; map common
- # adamw_* names so notebook defaults (optim="adamw_8bit") still work.
+ # Older unsloth-zoo lacks the CUDA/TRL optimizer aliases, so map the common adamw_* names and keep
+ # notebook defaults (optim="adamw_8bit") working.
opt = str(getattr(value, "value", value) or "adamw").strip().lower()
opt = opt.rsplit(".", 1)[-1].replace("-", "_")
if opt in _MLX_ADAMW_OPTIMIZER_ALIASES:
@@ -708,8 +699,7 @@ if _IS_MLX:
return False
if _is_mlx_training_args_like(args[1]):
return True
- # TRL callers often pass explicit defaults:
- # SFTTrainer(model, None, None, train_dataset, ...)
+ # TRL callers often pass explicit defaults: SFTTrainer(model, None, None, train_dataset, ...)
return len(args) >= 3 and args[1] is None and (args[2] is None or callable(args[2]))
def _assign_mlx_positional_kwarg(kwargs, name, value):
@@ -834,9 +824,8 @@ if _IS_MLX:
)
max_length_value = kwargs.get("max_length", None)
- # Only the canonical max_seq_length marks context length explicit; TRL
- # max_length stays a compatibility alias and defers to the model's
- # context length when one is available.
+ # Only the canonical max_seq_length marks context length explicit; TRL max_length stays a
+ # compatibility alias that defers to the model's context length.
max_seq_length_explicit = (
_positive_mlx_context_length(kwargs.get("max_seq_length", None)) is not None
)
@@ -1326,7 +1315,7 @@ if _IS_MLX:
kwargs[collator_key] = collator_kwargs[collator_key]
_raise_unsupported_mlx_vision_collator_kwargs(collator_kwargs)
elif _is_mlx_native_text_collator(data_collator):
- pass # redundant on MLX; MLXTrainer batches/masks/pads natively
+ pass
else:
raise NotImplementedError(
"Unsloth MLX: custom data_collator is not supported by "
@@ -1434,9 +1423,8 @@ if _IS_MLX:
def train_on_responses_only(*args, **kwargs):
"""Mask non-response tokens through the shared zoo dataset helper."""
- # Prefer the chat_templates export, which bounds the zoo's dataset.map()
- # worker count (issue #2693). It is None on a torch-free host; fall back
- # so the zoo raises its own ImportError, not "NoneType is not callable".
+ # Prefer the chat_templates export, which bounds the zoo's dataset.map() worker count (#2693); it is
+ # None on a torch-free host, so fall back and let the zoo raise its own ImportError.
from .chat_templates import train_on_responses_only as _train_on_responses_only
if _train_on_responses_only is None:
@@ -1460,9 +1448,8 @@ if _IS_MLX:
safe_exports.append(name)
return safe_exports
- # trl trainers with no MLX implementation yet. Swap them for stubs that fail
- # with a clear message instead of importing the real torch/CUDA trainer and
- # crashing deep inside it, so an unmigrated GRPO/DPO/ORPO notebook is legible.
+ # Stub the trl trainers with no MLX implementation yet, so an unmigrated GRPO/DPO/ORPO notebook
+ # fails with a clear message instead of crashing deep inside the real torch/CUDA trainer.
_MLX_UNSUPPORTED_TRL_TRAINERS = (
"GRPOTrainer",
"DPOTrainer",
@@ -1516,17 +1503,13 @@ if _IS_MLX:
_trl.SFTTrainer = UnslothTrainer
_trl.SFTConfig = _MLXSFTConfig
- # Only retarget trainers the installed trl actually exposes (don't invent
- # attributes); idempotent so re-importing unsloth is a no-op.
- # Decide what to stub from trl's declared exports (__all__) and already
- # materialized attrs only. A getattr probe here would trigger trl's lazy
- # trainer import, pulling torch and breaking `import unsloth` on torch-free
- # MLX just to check existence.
+ # Only retarget trainers the installed trl actually exposes; idempotent, so re-importing unsloth is a no-op.
+ # Names come from trl's __all__ and already materialized attrs, never a getattr probe: resolving
+ # one triggers trl's lazy trainer import and pulls torch, breaking `import unsloth` on MLX.
_trl_exports = set(getattr(_trl, "__all__", ()) or ())
- # Stub every non-SFT trainer trl exposes, not just a fixed list, so newer
- # trainers (RLOOTrainer, ...) also fail with a clear MLX message instead
- # of importing the real torch trainer. Names come from __all__ so we never
- # resolve them (that would trigger trl's lazy import and pull torch).
+ # Stub every non-SFT trainer trl exposes, not just a fixed list, so newer trainers (RLOOTrainer, ...)
+ # also fail with a clear MLX message instead of importing the real torch trainer. Names come from
+ # __all__ so we never resolve them (that would trigger trl's lazy import and pull torch).
_unsupported = set(_MLX_UNSUPPORTED_TRL_TRAINERS) | {
_n for _n in _trl_exports if _n.endswith("Trainer") and _n != "SFTTrainer"
}
@@ -1595,10 +1578,9 @@ else:
pass
-# Both paths, GPU and MLX: a `pip install --target` / PYTHONPATH layout makes
-# dill pickle whole modules by value, and every training path here builds a
-# `datasets.Dataset`, which fingerprints through dill. No-op on an ordinary
-# install. See `import_fixes.fix_dill_module_by_value_pickling`.
+# A `pip install --target` / PYTHONPATH layout makes dill pickle whole modules by value, and every
+# training path here builds a datasets.Dataset, which fingerprints through dill. See
+# import_fixes.fix_dill_module_by_value_pickling.
try:
from .import_fixes import fix_dill_module_by_value_pickling as _fix_dill
_fix_dill()
diff --git a/unsloth/_compressed_quantize.py b/unsloth/_compressed_quantize.py
index ec7564ef2f3..217cbfba253 100644
--- a/unsloth/_compressed_quantize.py
+++ b/unsloth/_compressed_quantize.py
@@ -65,9 +65,8 @@ def compressed_ignore_patterns(config):
class name) match.
"""
ignore = ["lm_head"]
- # Skip the same modules RedHatAI/NVIDIA skip for the Qwen3.5 / Qwen3-Next family (these also
- # have shapes not divisible by the grouped-scheme group_size, which would otherwise error).
- # No-ops elsewhere. Hybrid linear attention, VLM vision tower, and the MTP/speculative head.
+ # Skip the same modules RedHatAI/NVIDIA skip for the Qwen3.5 / Qwen3-Next family: their shapes are
+ # not divisible by the grouped-scheme group_size and would error. No-ops elsewhere.
ignore += ["re:.*\\.linear_attn\\..*", "re:.*\\.visual\\..*", "re:.*mtp.*"]
if _is_moe(config):
# Keep MoE routing layers unquantized: the router gate and (Qwen) shared-expert gate.
@@ -96,8 +95,8 @@ def _build_calibration_dataset(tokenizer, kind, value, num_samples, max_seq_leng
except (ValueError, KeyError):
from datasets import get_dataset_split_names
try:
- # Resolve the first split name so only num_samples rows are fetched, instead of
- # downloading/materializing the whole dataset just to take a small slice.
+ # Resolve the first split name so only num_samples rows are fetched, instead of materializing the
+ # whole dataset just to take a small slice.
split = get_dataset_split_names(value)[0]
ds = load_dataset(value, split = f"{split}[:{num_samples}]")
except Exception:
@@ -136,16 +135,16 @@ def _build_calibration_dataset(tokenizer, kind, value, num_samples, max_seq_leng
cols = set(ds.column_names)
if "input_ids" in cols:
- # Drop non-model-input columns (e.g. a leftover 'messages' list) so llm-compressor's
- # collator does not try to batch them.
+ # Drop non-model-input columns (e.g. a leftover 'messages' list) so llm-compressor's collator does
+ # not try to batch them.
keep = {"input_ids", "attention_mask", "labels", "position_ids"}
extra = [c for c in ds.column_names if c not in keep]
if extra:
ds = ds.remove_columns(extra)
return ds
if "messages" in cols:
- # Base / non-chat tokenizers have no chat template; concatenate message contents instead
- # of calling apply_chat_template (which would raise).
+ # Base / non-chat tokenizers have no chat template, and apply_chat_template would raise, so
+ # concatenate message contents instead.
has_chat_template = bool(getattr(_tok, "chat_template", None))
def _content_to_text(content):
@@ -233,8 +232,8 @@ def main():
from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
- # Import the VLM auto-class only when needed - some transformers versions lack it, and the
- # text path must not fail just because that newer class is unavailable.
+ # Import the VLM auto-class only when needed: some transformers versions lack it, and the text path
+ # must not fail over a newer class.
if args.is_vlm:
from transformers import AutoProcessor
try:
@@ -255,8 +254,8 @@ def main():
model.eval()
# A tokenizer may be absent if the caller saved it separately; only calibration needs one.
try:
- # The tokenizer/processor has its own trust flag: consent for one component must not
- # let the other's custom code run.
+ # The tokenizer/processor has its own trust flag: consent for one component must not let the other's
+ # custom code run.
tokenizer = auto_proc.from_pretrained(
args.model, trust_remote_code = args.trust_remote_code_tokenizer
)
@@ -268,8 +267,8 @@ def main():
)
tokenizer = None
- # MoE models: keep the router/gate unquantized (it decides expert routing) and calibrate every
- # expert even if the sample set does not route tokens to all of them.
+ # MoE models: keep the router/gate unquantized and calibrate every expert even if the sample set
+ # does not route tokens to all of them.
config = getattr(model, "config", None)
is_moe = _is_moe(config)
ignore = compressed_ignore_patterns(config)
@@ -286,10 +285,9 @@ def main():
args.num_calibration_samples,
args.max_seq_length,
)
- # Use the sequential pipeline: it onloads layer-by-layer, so models that do not fit in
- # memory at once can still calibrate. Running here in a clean process (Unsloth's attention
- # patches are absent) means tracing works; fall back to the memory-hungry "basic" pipeline
- # only if tracing fails.
+ # The sequential pipeline onloads layer-by-layer, so a model that does not fit at once still
+ # calibrates; tracing works only because this runs in a clean process without Unsloth's attention
+ # patches. Fall back to the memory-hungry "basic" pipeline only if tracing fails.
try:
oneshot(
model = model,
@@ -306,10 +304,8 @@ def main():
"retrying with the 'basic' pipeline (needs the full model to fit in memory).",
flush = True,
)
- # Free the partially-processed model before loading a fresh copy, so the fallback does
- # not transiently hold two copies on GPU. llm-compressor keeps the model in a global
- # session after a failed run, so reset it first; also drop the traceback frames (e) and
- # the local reference that pin the model.
+ # Free the partially-processed model before loading a fresh copy so the fallback never holds two
+ # on GPU: llm-compressor keeps the model in a global session after a failed run.
import gc as _gc
import torch as _torch
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index d843e179dbd..bf771f0c677 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -23,7 +23,6 @@ os.environ["UNSLOTH_IS_PRESENT"] = "1"
critical_modules = ["trl", "transformers", "peft"]
already_imported = [mod for mod in critical_modules if mod in sys.modules]
-# Fix some issues before importing other packages
from .import_fixes import (
fix_message_factory_issue,
fix_torch_check_is_size,
@@ -41,15 +40,11 @@ from .import_fixes import (
fix_huggingface_hub,
)
-# Redirect a read-only Hugging Face cache before anything below imports
-# huggingface_hub / transformers / vllm (disable_broken_vllm probes `import vllm`
-# and its compiled extensions, check_fbgemm_gpu_version imports transformers,
-# fix_huggingface_hub imports huggingface_hub) -- any of which would freeze Hub's
-# cache constants with the un-redirected paths. unsloth_zoo runs the same redirect
-# at import, but only after these probes. hf_cache.py is stdlib-only, so load it
-# straight from its file without triggering the full unsloth_zoo init this early;
-# the zoo's later call is an idempotent no-op. Older unsloth_zoo without it is
-# skipped silently.
+# Redirect a read-only Hugging Face cache before anything below imports huggingface_hub /
+# transformers / vllm (disable_broken_vllm probes `import vllm`, check_fbgemm_gpu_version imports
+# transformers, fix_huggingface_hub imports huggingface_hub), any of which would freeze Hub's
+# cache constants with the un-redirected paths. hf_cache.py is stdlib-only, so it loads straight
+# from its file without triggering the full unsloth_zoo init this early.
try:
import importlib.util as _importlib_util
from pathlib import Path as _Path
@@ -79,22 +74,16 @@ disable_broken_vllm()
fix_message_factory_issue()
fix_torch_check_is_size()
fix_torchao_torch_symbol_skew()
-# The above fixes THIS process only; vLLM's model-architecture inspector
-# is a subprocess that imports torchao itself and hits the same ImportError.
+# The above fixes THIS process only; vLLM's model-architecture inspector is a subprocess that
+# imports torchao itself and hits the same ImportError.
propagate_torchao_fix_to_subprocesses()
-# Warn, do not raise: this only adds the correct remedy just before transformers
-# prints its misleading one, without turning a metadata-derived floor into a hard
-# import failure of our own.
+# Warn, do not raise: this only adds the correct remedy just before transformers prints its misleading one.
check_transformers_dependency_versions()
check_fbgemm_gpu_version()
torchvision_compatibility_check()
-# Ahead of `import unsloth_zoo` below, not with the other import fixes further
-# down. unsloth_zoo's temporary_patches reach transformers.processing_utils,
-# which imports transformers.audio_utils, which imports torchaudio -- so a
-# torchaudio that raises at extension init takes the whole unsloth import down
-# roughly 95 lines before the late block would have neutralised it. Measured:
-# Kaggle-Muse_Glimmer_(30B)-GRPO died at cell 4 with the guard present but not
-# yet run.
+# Must precede `import unsloth_zoo`: its temporary_patches reach transformers.processing_utils ->
+# audio_utils -> torchaudio, so a torchaudio raising at extension init takes the whole unsloth
+# import down before the late guard would have neutralised it.
disable_torchaudio_if_cuda_mismatched()
fix_diffusers_warnings()
fix_huggingface_hub()
@@ -112,8 +101,8 @@ del torchvision_compatibility_check
del fix_diffusers_warnings
del fix_huggingface_hub
-# Unsloth patches these libraries at import time; if imported first, the
-# unoptimized versions run, risking OOM or slower training.
+# Unsloth patches these libraries at import time; if they are imported first the unoptimized
+# versions run, risking OOM or slower training.
if already_imported:
# stacklevel=2 points the warning at the user's import line
warnings.warn(
@@ -125,8 +114,7 @@ if already_imported:
)
del already_imported, critical_modules
-# Pin BNB_ROCM_VERSION before bitsandbytes is first imported (`import
-# unsloth_zoo` below pulls it in on ROCm hosts).
+# Pin BNB_ROCM_VERSION before bitsandbytes is first imported (`import unsloth_zoo` below pulls it in on ROCm hosts).
from .import_fixes import maybe_set_windows_rocm_bnb_version
maybe_set_windows_rocm_bnb_version()
@@ -137,15 +125,11 @@ del maybe_set_windows_rocm_bnb_version
# Fixes https://github.com/unslothai/unsloth/issues/1266
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
-# [TODO] Check why some GPUs don't work
-# "pinned_use_cuda_host_register:True,"\
-# "pinned_num_register_threads:8"
-
from importlib.metadata import version as importlib_version
from importlib.metadata import PackageNotFoundError
-# Check for unsloth_zoo
+# Try importing PyTorch and check version
try:
unsloth_zoo_version = importlib_version("unsloth_zoo")
if Version(unsloth_zoo_version) < Version("2026.8.15"):
@@ -153,14 +137,6 @@ try:
"Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n"
"Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`"
)
- # if os.environ.get("UNSLOTH_DISABLE_AUTO_UPDATES", "0") == "0":
- # try:
- # os.system("pip install --upgrade --no-cache-dir --no-deps unsloth_zoo")
- # except:
- # try:
- # os.system("pip install --upgrade --no-cache-dir --no-deps --user unsloth_zoo")
- # except:
- # raise ImportError("Unsloth: Please update unsloth_zoo via `pip install --upgrade --no-cache-dir --no-deps unsloth_zoo`")
import unsloth_zoo
except PackageNotFoundError:
raise ImportError(
@@ -170,7 +146,6 @@ except:
raise
del PackageNotFoundError, importlib_version
-# Try importing PyTorch and check version
try:
import torch
except ModuleNotFoundError:
@@ -191,7 +166,6 @@ from unsloth_zoo.device_type import (
)
from .device_type import arch_lacks_bf16, hip_visible_archs
-# Fix other issues
from .import_fixes import (
fix_transformers5_bare_annotation_configs,
fix_transformers_fully_masked_rows,
@@ -230,19 +204,18 @@ from .import_fixes import (
# Must run first: guards PretrainedConfig before vLLM defines its config classes.
fix_transformers5_bare_annotation_configs()
-# Probe-gated: no-ops unless this transformers really hands SDPA a query row
-# that attends to nothing. Ordered here, before anything imports a model, so a
-# plain `transformers.generate` in the same process is covered too -- which is
-# the case unsloth #9708 was measured on.
+# Probe-gated: no-ops unless this transformers really hands SDPA a query row that attends to
+# nothing. Ordered here, before anything imports a model, so a plain transformers.generate in the
+# same process is covered too (#9708).
fix_transformers_fully_masked_rows()
fix_xformers_performance_issue()
-# Must run AFTER fix_xformers_performance_issue (it rewrites xformers' cutlass.py on disk, so it
-# must precede any xformers import) and BEFORE models/_utils.py imports xformers.ops.
+# Must run AFTER fix_xformers_performance_issue (it rewrites xformers' cutlass.py on disk) and
+# BEFORE models/_utils.py imports xformers.ops.
fix_flash_attn_4_namespace_shadow()
fix_vllm_aimv2_issue()
fix_vllm_lora_tokenizer_module()
-# torchao 0.18.0 moved nf4tensor; torchtune (via xcodec2) still imports the
-# old path. Lazy alias, so it costs nothing unless asked for.
+# torchao 0.18.0 moved nf4tensor; torchtune (via xcodec2) still imports the old path. Lazy alias, so
+# it costs nothing unless asked for.
fix_torchao_nf4tensor_move()
# Check vLLM + torch < 2.9.0 + SM100 compatibility BEFORE importing vLLM
check_vllm_torch_sm100_compatibility()
@@ -250,9 +223,8 @@ fix_vllm_guided_decoding_params()
fix_trl_vllm_ascend()
fix_vllm_pdl_blackwell()
fix_triton_compiled_kernel_missing_attrs()
-# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss temporary
-# patches raise the dynamo recompile limits, so those settings reach the
-# autograd worker threads on torch >= 2.12.
+# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss patches raise the dynamo
+# recompile limits, so those settings reach the autograd worker threads on torch >= 2.12.
fix_dynamo_config_thread_visibility()
patch_trunc_normal_precision_issue()
ignore_logger_messages()
@@ -270,14 +242,14 @@ patch_vllm_for_notebooks()
patch_torchcodec_audio_decoder()
disable_torchcodec_if_broken()
disable_broken_wandb()
-# Must run before patch_peft_weight_converter_compatibility: stubs the
-# transformers v5 submodules peft 0.19.x imports, so the next patch can wrap
-# build_peft_weight_mapping instead of being swallowed by its ImportError.
+# Must run before patch_peft_weight_converter_compatibility: it stubs the transformers v5
+# submodules peft 0.19.x imports, so the next patch can wrap build_peft_weight_mapping instead of
+# being swallowed by its ImportError.
fix_peft_transformers_tensor_parallel_import_compat()
fix_peft_transformers_weight_conversion_import()
patch_peft_weight_converter_compatibility()
-# After peft is importable, so the already-bound `is_torchao_available` in
-# peft.tuners.lora.torchao is replaced too, not just import_utils'.
+# After peft is importable, so the already-bound is_torchao_available in peft.tuners.lora.torchao is
+# replaced too, not just import_utils'.
fix_peft_stale_torchao_import_error()
patch_accelerate_recursively_apply()
@@ -316,16 +288,9 @@ del patch_accelerate_recursively_apply
# Torch 2.4 has including_emulation
if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():
- # UNSLOTH_ALLOW_CPU=1 is the documented way to import on a host that has a
- # CUDA-built torch and no usable device (driverless container, CI runner, a
- # laptop with the runtime and no card). get_device_type() deliberately keeps
- # DEVICE_TYPE at "cuda" there, so this branch is entered with nothing to
- # query and torch.cuda.get_device_capability() raises out of _lazy_init().
- # Ask whether a device is present before asking what it can do.
- #
- # No device means no capability to report, so claim the conservative answer.
- # SUPPORTS_BFLOAT16 = False only costs float32; True would fail at the first
- # cast. is_bf16_supported() is stubbed to match rather than left to raise.
+ # UNSLOTH_ALLOW_CPU=1 keeps DEVICE_TYPE at "cuda" on a host with a CUDA-built torch and no usable
+ # device, so ask whether a device is present before asking what it can do: get_device_capability()
+ # would raise out of _lazy_init(). is_bf16_supported() is stubbed to match.
SUPPORTS_BFLOAT16 = False
torch.cuda.is_bf16_supported = lambda *args, **kwargs: False
elif DEVICE_TYPE == "cuda":
@@ -360,8 +325,8 @@ elif DEVICE_TYPE == "hip":
torch.cuda.is_bf16_supported = is_bf16_supported
del old_is_bf16_supported
elif DEVICE_TYPE == "xpu":
- # torch.xpu.is_bf16_supported() does not have including_emulation
- # set SUPPORTS_BFLOAT16 as torch.xpu.is_bf16_supported()
+ # torch.xpu.is_bf16_supported() does not have including_emulation set SUPPORTS_BFLOAT16 as
+ # torch.xpu.is_bf16_supported()
SUPPORTS_BFLOAT16 = torch.xpu.is_bf16_supported()
# For Gradio HF Spaces?
@@ -371,6 +336,7 @@ import triton
if DEVICE_TYPE == "cuda":
libcuda_dirs = lambda: None
if Version(triton.__version__) >= Version("3.0.0"):
+ # Try loading bitsandbytes and triton
try:
from triton.backends.nvidia.driver import libcuda_dirs
except:
@@ -378,13 +344,11 @@ if DEVICE_TYPE == "cuda":
else:
from triton.common.build import libcuda_dirs
- # Try loading bitsandbytes and triton
try:
import bitsandbytes as bnb
- # Bind the submodule by name: a half-imported bitsandbytes leaves the parent
- # without a `functional` attribute, which would otherwise be misreported below
- # as a CUDA linking failure. See unsloth/kernels/utils.py.
+ # Bind the submodule by name: a half-imported bitsandbytes leaves the parent without a `functional`
+ # attribute, which would otherwise be misreported below as a CUDA linking failure.
import bitsandbytes.functional as bnb_functional
except:
print(
@@ -397,12 +361,9 @@ if DEVICE_TYPE == "cuda":
libcuda_dirs()
except:
if not torch.cuda.is_available():
- # UNSLOTH_ALLOW_CPU=1 on a driverless host. DEVICE_TYPE is "cuda"
- # only because the caller asked the import to survive without a
- # device, so a missing libcuda is the expected state, not broken
- # linkage. Repairing it would ldconfig the host's linker cache (this
- # branch runs as root, the default in a container) through an
- # unguarded `ls` subprocess, to link a device that is not there.
+ # UNSLOTH_ALLOW_CPU=1 on a driverless host: a missing libcuda is the expected state, not broken
+ # linkage, and repairing it would ldconfig the host's linker cache as root for a device that is
+ # not there.
pass
elif hasattr(os, "geteuid") and os.geteuid() == 0:
warnings.warn("Unsloth: Running `ldconfig /usr/lib64-nvidia` to link CUDA.")
@@ -434,12 +395,14 @@ if DEVICE_TYPE == "cuda":
if bnb is not None:
importlib.reload(bnb)
importlib.reload(triton)
+ # Same degradation as the cuda branch above: no bnb means no 4bit, not a failed `import unsloth`.
try:
libcuda_dirs = lambda: None
if Version(triton.__version__) >= Version("3.0.0"):
try:
from triton.backends.nvidia.driver import libcuda_dirs
except:
+ # TODO: check triton for intel installed properly.
pass
else:
from triton.common.build import libcuda_dirs
@@ -466,8 +429,7 @@ elif DEVICE_TYPE == "hip":
# NO-OP for rocm device
pass
elif DEVICE_TYPE == "xpu":
- # Same degradation as the cuda branch above: no bnb means no 4bit, not a
- # failed `import unsloth`.
+ # Same degradation as the cuda branch above: no bnb means no 4bit, not a failed `import unsloth`.
try:
import bitsandbytes as bnb
except Exception:
@@ -476,7 +438,6 @@ elif DEVICE_TYPE == "xpu":
)
bnb = None
- # TODO: check triton for intel installed properly.
pass
from .models import *
@@ -497,8 +458,7 @@ from unsloth_zoo.rl_environments import (
launch_openenv,
)
-# Patch TRL trainers for backwards compatibility. Skipped under
-# UNSLOTH_ALLOW_CPU=1 (CPU-only CI): rebinding trl.SFTTrainer.__init__
-# changes inspect.getsource() and corrupts downstream drift detectors.
+# Skipped under UNSLOTH_ALLOW_CPU=1 (CPU-only CI): rebinding trl.SFTTrainer.__init__ changes
+# inspect.getsource() and corrupts downstream drift detectors.
if os.environ.get("UNSLOTH_ALLOW_CPU", "0") != "1":
_patch_trl_trainer()
diff --git a/unsloth/bnb_availability.py b/unsloth/bnb_availability.py
index 9d14bbb0f30..7176aedf2ec 100644
--- a/unsloth/bnb_availability.py
+++ b/unsloth/bnb_availability.py
@@ -40,8 +40,8 @@ _C_SYMBOLS = (
"cdequantize_blockwise_fp16_nf4",
"cdequantize_blockwise_bf16_nf4",
)
-# 4bit inference is a gemv on xpu and a naive gemm elsewhere; probing the wrong pair
-# would write off a perfectly good wheel.
+# 4bit inference is a gemv on xpu and a naive gemm elsewhere; probing the wrong pair would write off
+# a perfectly good wheel.
_C_SYMBOLS_XPU = (
"cgemv_4bit_inference_fp16",
"cgemv_4bit_inference_bf16",
@@ -70,8 +70,8 @@ def check_native_kernels(bnb, device_type):
raise ImportError("Unsloth: `bitsandbytes` is not installed.")
functional = getattr(bnb, "functional", None)
if functional is None:
- # A part-initialised bitsandbytes leaves the parent without the attribute while
- # the submodule stays in sys.modules, which `import x.y as z` reads directly.
+ # A part-initialised bitsandbytes leaves the parent without the attribute while the submodule stays
+ # in sys.modules, which `import x.y as z` reads directly.
import bitsandbytes.functional as functional
lib = functional.lib
diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index 317b4106ac5..15d081e5c08 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -43,8 +43,8 @@ try:
standardize_data_formats,
)
except ImportError:
- # dataset_utils pulls torch; keep chat_templates importable on torch-free
- # (MLX) hosts, which expose these via the backend-specific wrappers instead.
+ # dataset_utils pulls torch; keep chat_templates importable on torch-free (MLX) hosts, which expose
+ # these via the backend-specific wrappers.
_zoo_train_on_responses_only = standardize_data_formats = None
if _zoo_train_on_responses_only is None:
@@ -57,12 +57,10 @@ else:
@_functools.wraps(_zoo_train_on_responses_only)
def train_on_responses_only(*args, **kwargs):
- # The zoo still auto-sizes its dataset.map() workers with the uncapped
- # min(max(cpu_count + 4, 2), 64) heuristic (issue #2693) and is a
- # separate package, so bound the count on the way in. See
- # resolve_responses_only_num_proc; the local copy is only the fallback
- # for a zoo predating it. Top level, not under unsloth.utils, whose
- # __init__ imports torch: this path has to stay MLX-safe.
+ # The zoo auto-sizes its dataset.map() workers with the uncapped min(max(cpu_count + 4, 2), 64)
+ # heuristic (#2693) and is a separate package, so bound the count on the way in; see
+ # resolve_responses_only_num_proc. Top level, not under unsloth.utils, whose __init__ imports
+ # torch: this path has to stay MLX-safe.
try:
from unsloth_zoo.dataset_num_proc import resolve_responses_only_num_proc
except ImportError:
@@ -71,7 +69,7 @@ else:
try:
bound = _ZOO_RESPONSES_ONLY_SIGNATURE.bind_partial(*args, **kwargs)
except TypeError:
- # Signature drift, or a bad call. Let the zoo raise its own error.
+ # Signature drift, or a bad call: let the zoo raise its own error.
return _zoo_train_on_responses_only(*args, **kwargs)
# The zoo returns the masking closure before reading num_proc.
@@ -93,7 +91,6 @@ def _ollama_template(name: str):
return OLLAMA_TEMPLATES[name]
# =========================================== Unsloth
-# Unsloth efficient template leverages from Zephyr
unsloth_template = \
"{{ bos_token }}"\
"{% if messages[0]['role'] == 'system' %}"\
@@ -122,8 +119,8 @@ unsloth_eos_token = "eos_token"
CHAT_TEMPLATES["unsloth"] = (unsloth_template, unsloth_eos_token, False, unsloth_ollama,)
DEFAULT_SYSTEM_MESSAGE["unsloth"] = "You are a helpful assistant to the user"
-# =========================================== Zephyr
# Zephyr has no BOS!
+# =========================================== Zephyr
zephyr_template = \
"{% for message in messages %}"\
"{% if message['role'] == 'user' %}"\
@@ -142,10 +139,10 @@ zephyr_ollama = _ollama_template("zephyr")
zephyr_eos_token = "eos_token"
CHAT_TEMPLATES["zephyr"] = (zephyr_template, zephyr_eos_token, False, zephyr_ollama,)
-DEFAULT_SYSTEM_MESSAGE["zephyr"] = None # No system message in Zephyr
+DEFAULT_SYSTEM_MESSAGE["zephyr"] = None
+# ChatML has no BOS and no EOS: <|im_start|> / <|im_end|> act as BOS / EOS.
# =========================================== ChatML
-# ChatML has no BOS and not EOS! Rather <|im_start|> and <|im_end|> acts as BOS / EOS.
chatml_template = \
"{% for message in messages %}"\
"{% if message['role'] == 'user' %}"\
@@ -164,10 +161,10 @@ chatml_ollama = _ollama_template("chatml")
chatml_eos_token = "<|im_end|>"
CHAT_TEMPLATES["chatml"] = (chatml_template, chatml_eos_token, True, chatml_ollama,)
-DEFAULT_SYSTEM_MESSAGE["chatml"] = None # No system message in ChatML
+DEFAULT_SYSTEM_MESSAGE["chatml"] = None
-# =========================================== Mistral-1
# Mistral Instruct doesn't allow system prompts, so we append it to the user message.
+# =========================================== Mistral-1
mistral_template = \
"{{ bos_token }}"\
"{% if messages[0]['role'] == 'system' %}"\
@@ -196,10 +193,10 @@ mistral_ollama = _ollama_template("mistral")
mistral_eos_token = "eos_token"
CHAT_TEMPLATES["mistral"] = (mistral_template, mistral_eos_token, False, mistral_ollama,)
-DEFAULT_SYSTEM_MESSAGE["mistral"] = None # No system message in Mistral
+DEFAULT_SYSTEM_MESSAGE["mistral"] = None
-# =========================================== Llama-2
# Adds BOS to every convo! And weird <<SYS>> system messages.
+# =========================================== Llama-2
llama_template = \
"{% if messages[0]['role'] == 'system' %}"\
"{% if messages[1]['role'] == 'user' %}"\
@@ -227,10 +224,10 @@ llama_ollama = _ollama_template("llama")
llama_eos_token = "eos_token"
CHAT_TEMPLATES["llama"] = (llama_template, llama_eos_token, False, llama_ollama,)
-DEFAULT_SYSTEM_MESSAGE["llama"] = None # No system message in Llama
+DEFAULT_SYSTEM_MESSAGE["llama"] = None
-# =========================================== Vicuna
# https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template
+# =========================================== Vicuna
vicuna_template = \
"{{ bos_token }}"\
"{% if messages[0]['role'] == 'system' %}"\
@@ -260,8 +257,8 @@ vicuna_eos_token = "eos_token"
CHAT_TEMPLATES["vicuna"] = (vicuna_template, vicuna_eos_token, False, vicuna_ollama,)
DEFAULT_SYSTEM_MESSAGE["vicuna"] = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
-# =========================================== Vicuna Old
# https://github.com/lm-sys/FastChat/blob/main/docs/vicuna_weights_version.md#prompt-template
+# =========================================== Vicuna Old
vicuna_old_template = \
"{{ bos_token }}"\
"{% if messages[0]['role'] == 'system' %}"\
@@ -293,8 +290,8 @@ DEFAULT_SYSTEM_MESSAGE["vicuna_old"] = "A chat between a curious human and an ar
CHAT_TEMPLATES["vicuna old"] = CHAT_TEMPLATES["vicuna_old"]
DEFAULT_SYSTEM_MESSAGE["vicuna old"] = DEFAULT_SYSTEM_MESSAGE["vicuna_old"]
-# =========================================== Alpaca multi turn
# https://github.com/tatsu-lab/stanford_alpaca Changed for multi-turn convos
+# =========================================== Alpaca multi turn
alpaca_template = \
"{{ bos_token }}"\
"{% if messages[0]['role'] == 'system' %}"\
@@ -323,10 +320,9 @@ alpaca_eos_token = "eos_token"
CHAT_TEMPLATES["alpaca"] = (alpaca_template, alpaca_eos_token, False, alpaca_ollama,)
DEFAULT_SYSTEM_MESSAGE["alpaca"] = "Below are some instructions that describe some tasks. Write responses that appropriately complete each request."
+# |trim is required for lstrip/rstrip. <start_of_turn> maps to 106 and <end_of_turn> to 107; user
+# and model are normal 1-word tokens.
# =========================================== Gemma
-# https://huggingface.co/google/gemma-7b-it
-# Notice we must use |trim for lstrip and rstrip. <start_of_turn> maps to 106.
-# <end_of_turn> maps to 107. user and model are normal 1 word tokens.
gemma_template = \
"{{ bos_token }}"\
"{% if messages[0]['role'] == 'system' %}"\
@@ -351,10 +347,10 @@ gemma_ollama = _ollama_template("gemma")
gemma_eos_token = "<end_of_turn>"
CHAT_TEMPLATES["gemma"] = (gemma_template, gemma_eos_token, True, gemma_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gemma"] = None # No system message in Gemma
+DEFAULT_SYSTEM_MESSAGE["gemma"] = None
+# <eos> is still more appropriate here than a ChatML end token.
# =========================================== Gemma with ChatML instead
-# We find using <eos> is still more appropriate!
gemma_chatml_template = "{{ bos_token }}" + chatml_template
gemma_chatml_ollama = _ollama_template("gemma_chatml")
@@ -364,26 +360,26 @@ gemma_chatml_eos_token = (
"<|im_end|>",
)
CHAT_TEMPLATES["gemma_chatml"] = (gemma_chatml_template, gemma_chatml_eos_token, True, gemma_chatml_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gemma_chatml"] = None # No system message in Gemma
+DEFAULT_SYSTEM_MESSAGE["gemma_chatml"] = None
-# =========================================== Gemma 2
# Same as Gemma 1, but with sliding window attention!
# https://ollama.com/library/gemma2/blobs/6522ca797f47
+# =========================================== Gemma 2
gemma2_template = gemma_template
gemma2_ollama = _ollama_template("gemma2")
gemma2_eos_token = "<end_of_turn>"
CHAT_TEMPLATES["gemma2"] = (gemma2_template, gemma2_eos_token, True, gemma2_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gemma2"] = None # No system message in Gemma 2
+DEFAULT_SYSTEM_MESSAGE["gemma2"] = None
# =========================================== Gemma 2 with ChatML instead
gemma2_chatml_template = gemma_chatml_template
gemma2_chatml_ollama = _ollama_template("gemma2_chatml")
gemma2_chatml_eos_token = gemma_chatml_eos_token
CHAT_TEMPLATES["gemma2_chatml"] = (gemma2_chatml_template, gemma2_chatml_eos_token, True, gemma2_chatml_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gemma2_chatml"] = None # No system message in Gemma 2
+DEFAULT_SYSTEM_MESSAGE["gemma2_chatml"] = None
+# The \n\n really is needed.
# =========================================== Llama-3
-# Weirdly \n\n is needed?
llama3_template = \
"{{ bos_token }}"\
"{% for message in messages %}"\
@@ -405,14 +401,14 @@ llama3_ollama = _ollama_template("llama-3")
llama3_template_eos_token = "eos_token"
CHAT_TEMPLATES["llama-3"] = (llama3_template, llama3_template_eos_token, False, llama3_ollama,)
-DEFAULT_SYSTEM_MESSAGE["llama-3"] = None # No system message in Llama-3
+DEFAULT_SYSTEM_MESSAGE["llama-3"] = None
CHAT_TEMPLATES["llama3"] = (llama3_template, llama3_template_eos_token, False, llama3_ollama,)
-DEFAULT_SYSTEM_MESSAGE["llama3"] = None # No system message in Llama-3
+DEFAULT_SYSTEM_MESSAGE["llama3"] = None
-# =========================================== Phi-3
# "{{ bos_token }}"\ # Phi-3.5 removes BOS?
+# =========================================== Phi-3
phi3_template = \
"{% for message in messages %}"\
"{% if message['role'] == 'user' %}"\
@@ -432,15 +428,14 @@ phi3_ollama = _ollama_template("phi-3")
phi3_template_eos_token = "<|end|>"
CHAT_TEMPLATES["phi-3"] = (phi3_template, phi3_template_eos_token, False, phi3_ollama,)
-DEFAULT_SYSTEM_MESSAGE["phi-3"] = None # No system message in Phi-3
+DEFAULT_SYSTEM_MESSAGE["phi-3"] = None
CHAT_TEMPLATES["phi-35"] = CHAT_TEMPLATES["phi-3"]
-DEFAULT_SYSTEM_MESSAGE["phi-35"] = None # No system message in Phi-3.5
+DEFAULT_SYSTEM_MESSAGE["phi-35"] = None
CHAT_TEMPLATES["phi-3.5"] = CHAT_TEMPLATES["phi-3"]
-DEFAULT_SYSTEM_MESSAGE["phi-3.5"] = None # No system message in Phi-3.5
+DEFAULT_SYSTEM_MESSAGE["phi-3.5"] = None
-# =========================================== Llama-3.1
"""
No trimming in Llama 3.1 Instruct!
Also an extra newline for Cutting Knowledge Date
@@ -574,10 +569,10 @@ llama31_ollama = _ollama_template("llama-3.1")
llama31_template_eos_token = "eos_token"
CHAT_TEMPLATES["llama-3.1"] = (llama31_template, llama31_template_eos_token, False, llama31_ollama,)
-DEFAULT_SYSTEM_MESSAGE["llama-3.1"] = "" # Llama3.1 default system message is empty + the dates
+DEFAULT_SYSTEM_MESSAGE["llama-3.1"] = ""
CHAT_TEMPLATES["llama-31"] = (llama31_template, llama31_template_eos_token, False, llama31_ollama,)
-DEFAULT_SYSTEM_MESSAGE["llama-31"] = "" # Llama3.1 default system message is empty + the dates
+DEFAULT_SYSTEM_MESSAGE["llama-31"] = ""
for version in ("llama-3.2", "llama-3.3", "llama-32", "llama-33"):
CHAT_TEMPLATES[version] = CHAT_TEMPLATES["llama-3.1"]
@@ -643,19 +638,19 @@ qwen25_ollama = _ollama_template("qwen-2.5")
qwen25_template_eos_token = "eos_token"
qwen25_default_system_message = "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."
CHAT_TEMPLATES["qwen-2.5"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,)
-DEFAULT_SYSTEM_MESSAGE["qwen-2.5"] = qwen25_default_system_message # No system message in Qwen 2.5
+DEFAULT_SYSTEM_MESSAGE["qwen-2.5"] = qwen25_default_system_message
CHAT_TEMPLATES["qwen-25"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,)
-DEFAULT_SYSTEM_MESSAGE["qwen-25"] = qwen25_default_system_message # No system message in Qwen 2.5
+DEFAULT_SYSTEM_MESSAGE["qwen-25"] = qwen25_default_system_message
CHAT_TEMPLATES["qwen25"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,)
-DEFAULT_SYSTEM_MESSAGE["qwen25"] = qwen25_default_system_message # No system message in Qwen 2.5
+DEFAULT_SYSTEM_MESSAGE["qwen25"] = qwen25_default_system_message
CHAT_TEMPLATES["qwen2.5"] = (qwen25_template, qwen25_template_eos_token, False, qwen25_ollama,)
-DEFAULT_SYSTEM_MESSAGE["qwen2.5"] = qwen25_default_system_message # No system message in Qwen 2.5
+DEFAULT_SYSTEM_MESSAGE["qwen2.5"] = qwen25_default_system_message
-# =========================================== Phi-4
# "{{ bos_token }}"\ # Phi-4 removes BOS?
+# =========================================== Phi-4
phi4_template = \
"{% for message in messages %}"\
"{% if (message['role'] == 'system') %}"\
@@ -680,12 +675,11 @@ phi4_ollama = _ollama_template("phi-4")
phi4_template_eos_token = "<|im_end|>"
CHAT_TEMPLATES["phi-4"] = (phi4_template, phi4_template_eos_token, False, phi4_ollama,)
-DEFAULT_SYSTEM_MESSAGE["phi-4"] = None # No system message in Phi-4
+DEFAULT_SYSTEM_MESSAGE["phi-4"] = None
+# Obtained via print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))
# =========================================== Gemma-3
-# Obtained via
-# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))
gemma3_template = \
"""{{ bos_token }}
{%- if messages[0]['role'] == 'system' -%}
@@ -734,13 +728,13 @@ gemma3_ollama = _ollama_template("gemma-3")
gemma3_template_eos_token = "<end_of_turn>"
CHAT_TEMPLATES["gemma-3"] = (gemma3_template, gemma3_template_eos_token, False, gemma3_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gemma-3"] = None # No system message in Gemma-3
+DEFAULT_SYSTEM_MESSAGE["gemma-3"] = None
CHAT_TEMPLATES["gemma3"] = (gemma3_template, gemma3_template_eos_token, False, gemma3_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gemma3"] = None # No system message in Gemma-3
+DEFAULT_SYSTEM_MESSAGE["gemma3"] = None
-# =========================================== Qwen-3
# Official Qwen-3 chat template (see https://ollama.com/library/qwen3/blobs/eb4402837c78)
+# =========================================== Qwen-3
qwen3_template = \
"""
{%- if tools %}
@@ -846,14 +840,13 @@ qwen3_template = \
qwen3_ollama = _ollama_template("qwen-3")
qwen3_template_eos_token = "<|im_end|>"
CHAT_TEMPLATES["qwen-3"] = (qwen3_template, qwen3_template_eos_token, False, qwen3_ollama,)
-DEFAULT_SYSTEM_MESSAGE["qwen-3"] = None # No default system message for Qwen-3
+DEFAULT_SYSTEM_MESSAGE["qwen-3"] = None
CHAT_TEMPLATES["qwen3"] = (qwen3_template, qwen3_template_eos_token, False, qwen3_ollama,)
-DEFAULT_SYSTEM_MESSAGE["qwen3"] = None # No default system message for Qwen-3
+DEFAULT_SYSTEM_MESSAGE["qwen3"] = None
+# Obtained via print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))
# =========================================== Gemma-3n
-# Obtained via
-# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))
gemma3n_template = \
"""{{ bos_token }}
{%- if messages[0]['role'] == 'system' -%}
@@ -903,13 +896,13 @@ gemma3n_template = \
gemma3n_ollama = _ollama_template("gemma-3n")
gemma3n_template_eos_token = "<end_of_turn>"
CHAT_TEMPLATES["gemma-3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = None # No system message in Gemma-3n
+DEFAULT_SYSTEM_MESSAGE["gemma-3n"] = None
CHAT_TEMPLATES["gemma3n"] = (gemma3n_template, gemma3n_template_eos_token, False, gemma3n_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None # No system message in Gemma-3n
+DEFAULT_SYSTEM_MESSAGE["gemma3n"] = None
-# =========================================== Gemma-4
# Gemma-4 uses <|turn>role\n...<turn|>\n format
+# =========================================== Gemma-4
gemma4_template = \
"""{%- macro strip_thinking(text) -%}
{%- set ns = namespace(result='') -%}
@@ -1065,9 +1058,8 @@ DEFAULT_SYSTEM_MESSAGE["gemma-4-thinking"] = None
CHAT_TEMPLATES["gemma4-thinking"] = (gemma4_thinking_template, gemma4_template_eos_token, False, gemma4_ollama,)
DEFAULT_SYSTEM_MESSAGE["gemma4-thinking"] = None
+# Obtained via print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))
# =========================================== GPT-OSS
-# Obtained via
-# print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))
gptoss_template = \
"""{#-
In addition to the normal inputs of `messages` and `tools`, this template also accepts the
@@ -1602,10 +1594,10 @@ PARAMETER top_p 1.0
gptoss_template_template_eos_token = "<|return|>"
CHAT_TEMPLATES["gpt-oss"] = (gptoss_template, gptoss_template_template_eos_token, False, gptoss_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gpt-oss"] = None # No system message in GPT-oss
+DEFAULT_SYSTEM_MESSAGE["gpt-oss"] = None
CHAT_TEMPLATES["gptoss"] = (gptoss_template, gptoss_template_template_eos_token, False, gptoss_ollama,)
-DEFAULT_SYSTEM_MESSAGE["gptoss"] = None # No system message in GPT-oss
+DEFAULT_SYSTEM_MESSAGE["gptoss"] = None
# =========================================== Qwen3-Instruct
qwen3_instruct_template = \
@@ -1698,7 +1690,7 @@ qwen3_instruct_template = \
qwen3_template_eos_token = "<|im_end|>"
CHAT_TEMPLATES["qwen3-instruct"] = (qwen3_instruct_template, qwen3_template_eos_token, False, _ollama_template("qwen3-instruct"),)
-DEFAULT_SYSTEM_MESSAGE["qwen3-instruct"] = None # No system message in Qwen3
+DEFAULT_SYSTEM_MESSAGE["qwen3-instruct"] = None
# =========================================== Qwen3-Thinking
@@ -1796,7 +1788,7 @@ CHAT_TEMPLATES["qwen3-thinking"] = (
False,
_ollama_template("qwen3-thinking"),
)
-DEFAULT_SYSTEM_MESSAGE["qwen3-thinking"] = None # No system message in Qwen3
+DEFAULT_SYSTEM_MESSAGE["qwen3-thinking"] = None
# =========================================== Liquid-LFM2
@@ -1809,11 +1801,12 @@ liquid_lfm2_template = \
liquid_lfm2_template_eos_token = "<|im_end|>"
CHAT_TEMPLATES["lfm-2"] = (liquid_lfm2_template, liquid_lfm2_template_eos_token, False, None)
-DEFAULT_SYSTEM_MESSAGE["lfm-2"] = None # No system message in Phi-3
+DEFAULT_SYSTEM_MESSAGE["lfm-2"] = None
CHAT_TEMPLATES["lfm-2.5"] = (liquid_lfm2_template, liquid_lfm2_template_eos_token, False, None)
DEFAULT_SYSTEM_MESSAGE["lfm-2.5"] = None
+
# =========================================== Starling-LM
starling_template = \
@@ -1833,6 +1826,7 @@ CHAT_TEMPLATES["starling"] = (starling_template, starling_template_eos_token, Fa
DEFAULT_SYSTEM_MESSAGE["starling"] = None
+
# =========================================== Yi-chat
yi_chat_template = \
@@ -1850,9 +1844,8 @@ yi_chat_template_eos_token = "<|endoftext|>"
CHAT_TEMPLATES["yi-chat"] = (yi_chat_template, yi_chat_template_eos_token, False, yi_chat_ollama)
DEFAULT_SYSTEM_MESSAGE["yi-chat"] = None
-# Caller text is spliced into Jinja '...' / "..." literals: a bare quote closes the
-# literal, a backslash reads as an escape, and \r becomes \n before unescaping. Backslash
-# first, else it re-escapes the rest; escaping both quotes is safe in either style.
+# Caller text is spliced into Jinja '...' / "..." literals: a bare quote closes the literal, a
+# backslash reads as an escape, and \r becomes \n before unescaping.
def _escape_jinja_literal(text):
return text.replace("\\", "\\\\").replace("'", "\\'").replace('"', '\\"').replace("\r", "\\r")
@@ -1861,8 +1854,8 @@ def _change_system_message(template: str, type_chat_template: str, system_messag
# For predefined templates, check if default system message exists
default_system_message = DEFAULT_SYSTEM_MESSAGE.get(f"{type_chat_template}", None)
- # Custom templates have no default but may carry a {system_message} placeholder;
- # fill it before the no-default return below. A missing message here is an error.
+ # Custom templates have no default but may carry a {system_message} placeholder; fill it before the
+ # no-default return below. A missing message here is an error.
if default_system_message is None and "{system_message}" in template:
if system_message is None:
raise ValueError("Unsloth: You need to provide a system message for custom templates.")
@@ -1880,8 +1873,8 @@ def _change_system_message(template: str, type_chat_template: str, system_messag
# For predefined templates with default system message
message_to_use = system_message if system_message is not None else default_system_message
- # Predefined templates hold {system_message} inside a Jinja literal, so escape it.
- # Custom ones (above) are filled verbatim: their placeholder may sit in raw text.
+ # Predefined templates hold {system_message} inside a Jinja literal, so escape it; custom ones are
+ # filled verbatim, since their placeholder may sit in raw text.
new_template = template.replace("{system_message}", _escape_jinja_literal(message_to_use))
return new_template, message_to_use
@@ -1903,8 +1896,8 @@ def get_chat_template(
use_zoo_tokenizer_patch = is_mlx_backend
old_tokenizer = tokenizer
- # mlx-lm's TokenizerWrapper._tokenizer is the HF tokenizer, not the Rust
- # backend the vocab-edit paths below need; unwrap here, re-wrap before return.
+ # mlx-lm's TokenizerWrapper._tokenizer is the HF tokenizer, not the Rust backend the vocab-edit
+ # paths below need; unwrap here, re-wrap before return.
_mlx_tokenizer_wrapper = None
if is_mlx_backend and tokenizer.__class__.__name__ == "TokenizerWrapper":
_inner_tokenizer = getattr(tokenizer, "_tokenizer", None)
@@ -1917,16 +1910,6 @@ def get_chat_template(
if chat_template == "chatml": chat_template = "gemma_chatml"
IS_GEMMA = True
- # We add a check for Llama-3
- # if chat_template == "llama-3":
- # tokenizer._using_llama3_template = True
- # else:
- # llama3_tokens = set(["<|end_header_id|>", "<|eot_id|>", "<|start_header_id|>"])
- # check_llama3_tokens = llama3_tokens & set(str(x) for x in tokenizer.added_tokens_decoder.values())
- # if len(check_llama3_tokens) == len(llama3_tokens):
- # tokenizer._using_llama3_template = True
- # pass
- # pass
# We first check if the tokenizer is a fast one. If not, we cannot convert this!
is_fast_tokenizer = getattr(tokenizer, "is_fast", False)
@@ -1936,8 +1919,7 @@ def get_chat_template(
type_chat_template = None
if type(chat_template) in (list, tuple,):
- # For changing system message later
- # Since it's not supported yet, we will raise an error first!
+ # For changing system message later Since it's not supported yet, we will raise an error first!
type_chat_template = chat_template[0].lower()
chat_template, stop_word = chat_template
assert(type(chat_template) is str)
@@ -1950,8 +1932,8 @@ def get_chat_template(
chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template]
- # Check mapping to eos_token. The template can veto the mapping, but it must not
- # force it back on: `map_eos_token = False` is an explicit choice by the caller.
+ # The template can veto the eos mapping, but it must not force it back on: map_eos_token = False is
+ # an explicit choice by the caller.
if not yes_map_eos_token and map_eos_token: map_eos_token = False
if type(stop_word) in (list, tuple,):
@@ -1962,12 +1944,10 @@ def get_chat_template(
assert(type(stop_word) is str)
- # gemma_chatml and gemma2_chatml build <|im_end|> by renaming <eos>, and that rename
- # runs whether or not the caller opts out, while the rebuilt tokenizer only carries
- # eos_token = stop_word when the mapping is on: honouring the opt-out here lets the
- # tokenizer class default re-add <eos> as a fresh id past the end of the embeddings.
- # Key on the mapping, not on tokenizer.eos_token, or a Gemma checkpoint whose
- # eos_token is <end_of_turn> (gemma-3-270m-it, gemma-3-1b-it) slips through.
+ # gemma_chatml / gemma2_chatml build <|im_end|> by renaming <eos> whether or not the caller opts
+ # out, while the rebuilt tokenizer carries eos_token = stop_word only when the mapping is on, so
+ # honouring the opt-out lets the class default re-add <eos> past the end of the embeddings. Key on
+ # the mapping, or a Gemma checkpoint whose eos_token is <end_of_turn> slips through.
if not map_eos_token and yes_map_eos_token and token_mapping is not None:
logger.warning_once(
f"Unsloth: {type_chat_template} builds {stop_word} by renaming existing "\
@@ -1975,24 +1955,16 @@ def get_chat_template(
)
map_eos_token = True
- # Check fast tokenizer
if not is_fast_tokenizer:
pass
- # print(
- # "Unsloth: Not a fast tokenizer, so can't process it as of yet :(\n"\
- # "Please log a Github issue if you want this as a new feature!\n"\
- # "Your chat template will still work, but it won't add or edit tokens."
- # )
elif token_mapping is not None:
- # token_mapping = {"<start_of_turn>" : "<|im_start|>", "<end_of_turn>" : "<|im_end|>"}
- # For Gemma :)
string_vocab = tokenizer._tokenizer.to_str()
skipped = 0
- # Only mirror applied mappings into the spm model; a skipped one would
- # rename a piece the JSON never changed and desync the two.
+ # Only mirror applied mappings into the spm model; a skipped one would rename a piece the JSON never
+ # changed and desync the two.
applied_mapping = {}
for old_token, new_token in token_mapping.items():
old_count = string_vocab.count(f'"{old_token}"')
@@ -2046,9 +2018,9 @@ def get_chat_template(
elif map_eos_token and (stop_word != "eos_token"):
logger.warning_once(f"Unsloth: Will map {stop_word} to EOS = {tokenizer.eos_token}.")
- # HACK: replace old EOS with a new one (e.g. ChatML <|im_end|>) to
- # avoid the slow lm_head/embedding retraining of new tokens.
- # Idea from https://huggingface.co/cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laser
+ # Replace the old EOS with a new one (e.g. ChatML <|im_end|>) to avoid slow lm_head/embedding
+ # retraining of new tokens. Idea from
+ # huggingface.co/cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laser
old_bos_token = getattr(tokenizer, "bos_token", None)
old_eos_token = getattr(tokenizer, "eos_token", None)
@@ -2058,7 +2030,6 @@ def get_chat_template(
string_vocab = tokenizer._tokenizer.to_str()
# First check if new stop_word is in the tokenizer
if stop_word in string_vocab:
- # We shall swap them around
temporary_stop_token = "<|:__TEMP//STOP//TOKEN__:|>"
string_vocab = string_vocab.replace(old_eos_token, temporary_stop_token)
string_vocab = string_vocab.replace(stop_word, old_eos_token)
@@ -2096,13 +2067,11 @@ def get_chat_template(
f"{CHAT_TEMPLATES.keys()}"
)
- # Careful on Gemma
- # bos_token is a must or else losses become too high
+ # bos_token is a must on Gemma or losses become too high.
if IS_GEMMA and not chat_template.startswith(("{{ bos_token }}", "{{- bos_token }}")):
chat_template = "{{ bos_token }}" + chat_template
- # For ShareGPT role -> from and content -> value
- # The spliced values land inside Jinja literals, so escape them.
+ # The spliced ShareGPT values land inside Jinja literals, so escape them.
new_chat_template = chat_template\
.replace("'role'", "'" + _escape_jinja_literal(mapping["role"]) + "'")\
.replace("'content'", "'" + _escape_jinja_literal(mapping["content"]) + "'")\
@@ -2110,8 +2079,8 @@ def get_chat_template(
.replace("'assistant'", "'" + _escape_jinja_literal(mapping["assistant"]) + "'")
if use_zoo_tokenizer_patch:
- # Unsloth MLX avoids the model-utils tokenizer wrapper because that
- # import path pulls in Torch/GPU-specific modules before MLX training.
+ # Unsloth MLX avoids the model-utils tokenizer wrapper: that import path pulls Torch/GPU-specific
+ # modules in before MLX training.
from unsloth_zoo.tokenizer_utils import patch_tokenizer
else:
from .models._utils import patch_tokenizer
@@ -2145,14 +2114,11 @@ def get_chat_template(
if not same_padding_token:
if old_pad_token != new_pad_token: tokenizer.pad_token = old_pad_token
- # stopping_criteria = create_stopping_criteria(tokenizer, stop_word)
- # Patch saving functions
if patch_saving and not is_mlx_backend:
from .save import patch_saving_functions
tokenizer = patch_saving_functions(tokenizer)
- # Add Ollama
tokenizer._ollama_modelfile = ollama_modelfile
tokenizer._system_message = system_message
@@ -2167,7 +2133,7 @@ def get_chat_template(
getattr(tokenizer, "chat_template", None) is not None
)
tokenizer = _mlx_tokenizer_wrapper
- return tokenizer#, stopping_criteria
+ return tokenizer
def remove_special_tokens(tokenizer, prompt):
@@ -2179,7 +2145,6 @@ def remove_special_tokens(tokenizer, prompt):
def _parse_combined_prompt(combined_prompt, dataset):
- # Find {...}
possible_columns = re.findall(r"\{(.+?)\}", combined_prompt)
dataset_columns = set(dataset.column_names)
for column in possible_columns:
@@ -2189,30 +2154,25 @@ def _parse_combined_prompt(combined_prompt, dataset):
f"Only allowed columns are {list(dataset_columns)}"
)
- # Find [[...]]
optional_prompts = list(re.finditer(r"\[\[.+?\]\]", combined_prompt, flags = re.DOTALL | re.MULTILINE))
optional_prompts = [(x.span(), x.group(0)) for x in optional_prompts]
final_optional_prompts = []
if len(optional_prompts) != 0:
- # Add left
left = optional_prompts[0]
l = left[0][0]
if l != 0: final_optional_prompts.append(combined_prompt[:l])
- # Add in between
for left, right in zip(optional_prompts[:-1], optional_prompts[1:]):
l, r = left[0][-1], right[0][0]
final_optional_prompts.append(left)
if l != r: final_optional_prompts.append(combined_prompt[l : r])
final_optional_prompts.append(optional_prompts[-1])
- # Add right
right = optional_prompts[-1]
r = right[0][1]
if r != len(combined_prompt): final_optional_prompts.append(combined_prompt[r:])
else:
- # Just add in the entire string
final_optional_prompts.append(combined_prompt)
check_combined = "".join(x if type(x) is str else x[1] for x in final_optional_prompts)
@@ -2255,12 +2215,7 @@ def _create_formatter(possible_columns, final_optional_prompts, user_column_name
texts = []
for row_idx in range(n_rows):
- # Coerce missing (None) columns to "" so they do not render as the
- # literal string "None" in the emitted text. In a [[...]] block only
- # the first column gates the block, so a later column can still be
- # None here; required columns can be None too. Coercing at the source
- # covers both; since None is now "", the gate below only needs to
- # test for "" (an empty first column still drops the block).
+ # Coerce missing (None) columns to "" so they do not render as the literal string "None".
row_values = {
column: ("" if (value := examples[column][row_idx]) is None else value)
for column in columns
@@ -2317,9 +2272,8 @@ def to_sharegpt(
formatter = _create_formatter(possible_columns, final_optional_prompts, merged_column_name)
dataset = dataset.map(formatter, batched = True, desc = "Merging columns")
elif merged_column_name not in dataset.column_names:
- # Without a merged_prompt there is nothing to build the input from, so the
- # column has to be there already. Running the formatter anyway would fill
- # merged_column_name with empty strings and silently blank every human turn.
+ # Without a merged_prompt there is nothing to build the input from. Running the formatter anyway
+ # would fill merged_column_name with empty strings and silently blank every human turn.
raise KeyError(
f"Unsloth: `to_sharegpt` needs an input column named '{merged_column_name}', "
f"but the dataset has {dataset.column_names}. Pass `merged_column_name` to "
@@ -2334,10 +2288,8 @@ def to_sharegpt(
"Unsloth: Input and output columns must have matching batch lengths. "
f"Got {len(users)} {merged_column_name} rows and {len(assistants)} {output_column_name} rows."
)
- # A null cell is an absent value, not the word "None". _create_formatter
- # already coalesces it on the merged path, and skipping the merge must
- # not reintroduce it; the output column never went through that path at
- # all, so it leaked here either way.
+ # A null cell is an absent value, not the word "None": _create_formatter coalesces it on the merged
+ # path, and the output column never went through that path at all.
texts = [
[
{"from" : "human", "value" : "" if user is None else str(user) },
@@ -2351,11 +2303,9 @@ def to_sharegpt(
__convert_to_sharegpt__,
batched = True,
desc = "Converting to ShareGPT",
- # Remove unused columns!
remove_columns = dataset.column_names if remove_unused_columns else None,
)
- # Randomnly concat conversations to create a long stream!
from datasets import concatenate_datasets
n_extensions = max(conversation_extension-1, 0)
if n_extensions == 0: return dataset
@@ -2367,7 +2317,6 @@ def to_sharegpt(
all_shuffled.append(shuffled)
dataset = concatenate_datasets(all_shuffled, axis = 1)
- # Combine them into 1
n_extensions += 1
conversation_columns = [f"conversations{j}" for j in range(n_extensions)]
def __combine_conversations__(examples):
@@ -2384,7 +2333,6 @@ def to_sharegpt(
__combine_conversations__,
batched = True,
desc = "Extending conversations",
- # Remove unused columns!
remove_columns = dataset.column_names if remove_unused_columns else None,
)
return dataset
@@ -2402,13 +2350,11 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []):
added_tokens_decoder = [x for x in added_tokens_decoder if x != tokenizer.bos_token]
repeated_tokens = []
- # Join all vocab
joined_text = "\x01\x00".join(added_tokens_decoder)
for token in added_tokens_decoder:
n = len(token)
repeated_counts = joined_text.count(token[:n//2])
- # Try finding longer than 1/2 of the token in the rest
- # For eg <|reserved_special_token_0|>, <|reserved_special_token_1|>
+ # Longer than half the token, for e.g. <|reserved_special_token_0|> vs <|reserved_special_token_1|>.
if repeated_counts > 2:
for j in range(n//2+1, n):
if joined_text.count(token[:j]) < repeated_counts:
@@ -2418,13 +2364,11 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []):
repeated_tokens.append(token[:j])
break
- # Remove duplicates
split = joined_text.split("\x01\x00")
final_eos_tokens = [old for old, new in zip(added_tokens_decoder, split) if old == new]
final_eos_tokens += extra_eos_tokens
final_eos_tokens += repeated_tokens
- # Remove new lines, spaces and HTML tags
filtered_eos_tokens = []
for token in final_eos_tokens:
if token.count("\n") == len(token): continue
@@ -2463,8 +2407,8 @@ extra_eos_tokens = None,
You must use {INPUT}, {OUTPUT} twice, and {SYSTEM} is optional.
"""
- # Strip only the left: trailing whitespace can be part of the repeated example
- # (e.g. "{OUTPUT}\n"). Accidental trailing whitespace (#992) is retried on failure.
+ # Strip only the left: trailing whitespace can be part of the repeated example (e.g. "{OUTPUT}\n").
+ # Accidental trailing whitespace (#992) is retried on failure.
chat_template = chat_template.lstrip()
assert(tokenizer is not None)
@@ -2496,31 +2440,28 @@ extra_eos_tokens = None,
"Unsloth: Your tokenizer does not have an EOS token? Please provide one via extra_eos_tokens!"
)
- # Check tokenizer types
tokenizer_name = tokenizer.name_or_path.lower()
if tokenizer_name.startswith(("unsloth/llama-3-8b-instruct", "unsloth/llama-3-70b-instruct")):
- # Add <|eot_id|>
extra_eos_tokens.append("<|eot_id|>")
elif ("<|eot_id|>" in extra_eos_tokens or "<|eot_id|>" in chat_template) and \
tokenizer_name.startswith(("unsloth/llama-3-8b", "unsloth/llama-3-70b")):
- # Warn
logger.warning(
"Unsloth: Base llama-3 models did not train <|eot_id|>.\n"\
"Please use the instruct version or use <|end_of_text|>"
)
- # dict.fromkeys, not set: `extra_eos_tokens.insert(0, tokenizer.eos_token)` above
- # sets a priority that `extra_eos_tokens[0]` relies on when appending the EOS.
+ # dict.fromkeys, not set: extra_eos_tokens.insert(0, tokenizer.eos_token) above sets a priority that
+ # extra_eos_tokens[0] relies on when appending the EOS.
extra_eos_tokens = list(dict.fromkeys(extra_eos_tokens))
count_eos = 0
for eos in extra_eos_tokens:
count_eos += len(re.findall(r"{OUTPUT}" + re.escape(eos), chat_template))
- # This forces you to provide 2 input and outputs
+ # This forces the caller to provide 2 input/output pairs.
final_combined_check = False
try:
- # O(N^2) search finding 2 repeatted pieces of text
+ # O(N^2) search for 2 repeated pieces of text.
j = len(chat_template)-1
at_least_one = False
while j > 0:
@@ -2536,19 +2477,15 @@ extra_eos_tokens = None,
# Must be equivalent to left
final_combined_check = True
- # Repeatted text
instruction_response = chat_template[j:]
if instruction_response.count("{INPUT}") != 1 or instruction_response.count("{OUTPUT}") != 1:
raise RuntimeError(error_msg)
- # 1st System, Instruction, Output pair
left = chat_template[:j]
- # 2nd Instruction, Output pair
right = chat_template[j:]
final_combined_check = left if final_combined_check else chat_template
- # Isolate input
# Regex alternations prefer the first match, so match prefix tokens last.
eos_tokens_for_matching = sorted(extra_eos_tokens, key = len, reverse = True)
extra_eos_tokens_regex = "|".join(f"(?:{re.escape(x)})" for x in eos_tokens_for_matching)
@@ -2563,14 +2500,11 @@ extra_eos_tokens = None,
input_end = input_end.span(0)[1]
input_part = right[:input_end]
- # Isolate output
output_part = right[input_end:]
- # Isolate system
where_system = left.find(input_part)
system_part = left[:where_system if where_system != -1 else len(left)]
- # Check if the user provided a correct prompt
combined = system_part + input_part + output_part
if combined != final_combined_check:
combined_changed = combined .replace('\n', '\\n')
@@ -2582,8 +2516,8 @@ extra_eos_tokens = None,
f"{left_changed}"
)
except:
- # Accidental trailing whitespace (#992) desyncs the two-example detection,
- # so retry once without it. Templates that parse as-is are never altered.
+ # Accidental trailing whitespace (#992) desyncs the two-example detection, so retry once without it.
+ # Templates that parse as-is are never altered.
rstripped_chat_template = chat_template.rstrip()
if rstripped_chat_template != chat_template:
try:
@@ -2648,9 +2582,8 @@ extra_eos_tokens = None,
eos = extra_eos_tokens[0]
output_part = output_part + eos
- # Ollama modelfile parts
- # Check bos_token is in system prompt
+ # bos_token must be in the system prompt.
ollama_system = system_part
has_bos_token = False
always_bos_token = False
@@ -2659,7 +2592,6 @@ extra_eos_tokens = None,
if ollama_system.startswith(tokenizer.bos_token):
has_bos_token = True
ollama_system = ollama_system[len(tokenizer.bos_token):]
- # Check system
if "{SYSTEM}" in ollama_system:
system_modelfile = "{{ if .System }}" + ollama_system.replace("{SYSTEM}", "{{ .System }}") + "{{ end }}"
else:
@@ -2667,20 +2599,17 @@ extra_eos_tokens = None,
input_modelfile = "{{ if .Prompt }}" + input_part .replace("{INPUT}", "{{ .Prompt }}") + "{{ end }}"
output_modelfile = output_part.replace("{OUTPUT}", "{{ .Response }}")
- # Ollama EOS
ollama_eos = get_ollama_eos_tokens(tokenizer, extra_eos_tokens)
ollama_eos = '\n'.join(f'PARAMETER stop "{eos}"' for eos in ollama_eos)
- # Add temperature and min_p to counteract gibberish
+ # Temperature and min_p counteract gibberish.
ollama_eos += "\nPARAMETER temperature 1.5\nPARAMETER min_p 0.1"
- # Ollama modelfile
part = '"""'
modelfile = 'FROM {__FILE_LOCATION__}\n\n'\
'TEMPLATE ' + part + system_modelfile + input_modelfile + output_modelfile + \
part + '\n\n' + ollama_eos
- # HF Jinja Chat template
def process(part, which, content = "message['content']"):
# Escape the literal pieces only; the placeholder becomes Jinja syntax below.
part = which.join(_escape_jinja_literal(piece) for piece in part.split(which))
@@ -2709,7 +2638,6 @@ extra_eos_tokens = None,
"{{ '" + _escape_jinja_literal(output_part[:output_part.find("{OUTPUT}")]) + "' }}"\
"{% endif %}"
- # Now add system prompt to jinja
if len(system_part) != 0:
# Strip the BOS while raw: an escaped bos_token no longer matches.
if has_bos_token:
@@ -2735,14 +2663,13 @@ extra_eos_tokens = None,
"{% set loop_messages = messages %}"\
"{% endif %}"
elif "{SYSTEM}" in system_part:
- # Only bind loop_messages when the template can render a caller system
- # message. A static prefix with no {SYSTEM} must still raise, not drop it.
+ # Only bind loop_messages when the template can render a caller system message; a static prefix with
+ # no {SYSTEM} must still raise, not drop it.
partial_system += "{% else %}"\
"{% set loop_messages = messages %}"\
"{% endif %}"
else:
- # Emit a static prefix on both branches so the collapse below
- # makes it unconditional.
+ # Emit a static prefix on both branches so the collapse below makes it unconditional.
partial_system += "{% else %}"\
"{{ " + system_expr + " }}"\
"{% set loop_messages = messages %}"\
@@ -2937,7 +2864,6 @@ def test_chat_templates():
{"role": "user", "content": " No it's 100% 5! "},
]
- # Zephyr
from transformers import AutoTokenizer
template = zephyr_template
correct_tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-beta")
@@ -2946,7 +2872,6 @@ def test_chat_templates():
our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
assert(correct_prompt == our_prompt)
- # Chatml
template = chatml_template
correct_tokenizer = AutoTokenizer.from_pretrained("teknium/OpenHermes-2.5-Mistral-7B")
correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
@@ -2954,7 +2879,6 @@ def test_chat_templates():
our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
assert(correct_prompt == our_prompt)
- # Mistral
template = mistral_template
correct_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)
@@ -2962,7 +2886,6 @@ def test_chat_templates():
our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)
assert(correct_prompt == our_prompt)
- # Llama
template = llama_template
correct_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-2-7b-chat")
correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
@@ -2970,7 +2893,6 @@ def test_chat_templates():
our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
assert(correct_prompt == our_prompt)
- # Vicuna
try:
from fastchat.conversation import get_conv_template
except:
@@ -3003,17 +2925,15 @@ def test_chat_templates():
correct_prompt = correct_tokenizer.bos_token + correct_prompt.get_prompt()
correct_tokenizer.chat_template = template
our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)
- # We add </s> ourselves
+ # </s> is added here rather than by the template.
assert(correct_prompt == our_prompt.replace("</s>", ""))
- # Gemma
correct_tokenizer = AutoTokenizer.from_pretrained("unsloth/gemma-7b-it")
correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)
correct_tokenizer.chat_template = gemma_template
our_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)
assert(our_prompt == correct_prompt)
- # Llama-3
template = llama3_template
correct_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-3-8b-Instruct")
correct_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
@@ -3021,7 +2941,6 @@ def test_chat_templates():
our_prompt = correct_tokenizer.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
assert(correct_prompt == our_prompt)
- # Phi-3
template = phi3_template
correct_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
correct_prompt = correct_tokenizer.apply_chat_template(messages[1:], tokenize = False, add_generation_prompt = True)
@@ -3055,8 +2974,8 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf")
### Response:
{}""".format(
- "Describe the city given eloquently.", # instruction
- "The lost city of Atlantis.", # input
+ "Describe the city given eloquently.",
+ "The lost city of Atlantis.",
"", # output - leave this blank for generation!
)
prompts = [ prompt, ]
@@ -3067,7 +2986,7 @@ def test_hf_gguf_equivalence(tokenizer, gguf_model = "./model-unsloth.F16.gguf")
prompts.append(prompt)
for prompt in prompts:
- # Use a list of args with shell=False so prompt content is passed literally.
+ # A list of args with shell=False so prompt content is passed literally.
command = [
"./llama.cpp/llama-cli",
"-m", gguf_model,
diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py
index 73d17bba977..37f204e9e47 100644
--- a/unsloth/dataprep/raw_text.py
+++ b/unsloth/dataprep/raw_text.py
@@ -90,8 +90,8 @@ class RawTextDataLoader:
)
all_chunks.extend(chunks)
if not all_chunks:
- # All files empty/whitespace: raise like load_from_file instead of
- # create_causal_dataset([]) returning a 0-row text-column dataset.
+ # All files empty/whitespace: raise like load_from_file instead of create_causal_dataset([])
+ # returning a 0-row text-column dataset.
raise ValueError("All files are empty or contain only whitespace")
return self.create_causal_dataset(all_chunks)
@@ -108,7 +108,6 @@ class RawTextDataLoader:
def create_causal_dataset(self, chunks):
"""Create dataset for causal language modeling"""
if chunks and isinstance(chunks[0], dict):
- # Already-tokenized chunks: reshape for Dataset.from_dict
input_ids = [chunk["input_ids"] for chunk in chunks]
attention_mask = [chunk["attention_mask"] for chunk in chunks]
# Labels == input_ids for causal LM
@@ -121,7 +120,6 @@ class RawTextDataLoader:
}
)
else:
- # Text strings (backward compatibility)
return Dataset.from_dict({"text": chunks})
def smart_chunk_text(
@@ -150,9 +148,8 @@ class RawTextDataLoader:
f"stride ({stride}) must be smaller than chunk_size ({chunk_size}) to progress the chunking loop"
)
- # Skip empty/whitespace text before tokenizing: BPE/SentencePiece emit
- # real tokens for spaces/newlines, so a len(tokens)==0 check misses it
- # and would yield a degenerate lone-EOS sample. Mirrors load_from_file.
+ # Skip empty/whitespace text before tokenizing: BPE/SentencePiece emit real tokens for
+ # spaces/newlines, so a len(tokens)==0 check misses it and would yield a degenerate lone-EOS sample.
if not text or not text.strip():
return []
@@ -194,7 +191,7 @@ class RawTextDataLoader:
chunk_tokens.tolist() if hasattr(chunk_tokens, "tolist") else list(chunk_tokens)
)
- # Append EOS on the last or a full chunk
+ # Append EOS on the last or a full chunk.
if end_idx == len(tokens) or len(chunk_tokens_list) == chunk_size:
eos_token_id = getattr(self.tokenizer, "eos_token_id", None)
if eos_token_id is not None:
@@ -204,10 +201,9 @@ class RawTextDataLoader:
chunks.append({"input_ids": chunk_tokens_list, "attention_mask": attention_mask})
else:
- # Decode back to text (backward compatibility)
chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True)
- # Append EOS on the last or a full chunk
+ # Append EOS on the last or a full chunk.
if end_idx == len(tokens) or len(chunk_tokens) == chunk_size:
eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else ""
chunk_text += eos_token
@@ -223,16 +219,15 @@ class RawTextDataLoader:
def _read_file_by_format(self, file_path, file_format):
"""Read file content based on detected format."""
- # utf-8-sig: Windows tooling (PowerShell's Out-File, Excel's "CSV UTF-8") prepends
- # a BOM that plain utf-8 keeps as a leading character. Without a BOM it decodes
- # exactly like utf-8.
+ # utf-8-sig: Windows tooling (PowerShell Out-File, Excel "CSV UTF-8") prepends a BOM that plain
+ # utf-8 keeps as a leading character. Without a BOM it decodes exactly like utf-8.
with open(file_path, "r", encoding = "utf-8-sig") as f:
if file_format == "plain_text" or file_format == "markdown":
return f.read()
elif file_format == "json_lines":
if Path(file_path).suffix.lower() == ".json":
- # A .json file is a single JSON document (commonly a list
- # of records), so parsing it per line drops the whole file.
+ # A .json file is a single JSON document (commonly a list of records), so parsing it per line drops
+ # the whole file.
try:
parsed = json.load(f)
records = parsed if isinstance(parsed, list) else [parsed]
@@ -241,8 +236,8 @@ class RawTextDataLoader:
f.seek(0)
records = self._iter_json_lines(f)
else:
- # A .jsonl file is one JSON value per line: stay streaming so
- # a large file is never held in memory all at once.
+ # A .jsonl file is one JSON value per line: stay streaming so a large file is never held in memory
+ # at once.
records = self._iter_json_lines(f)
lines = []
for data in records:
@@ -260,7 +255,6 @@ class RawTextDataLoader:
return "\n\n".join(texts)
return ""
- # Cache text fields/columns for better performance
_TEXT_FIELDS = ("text", "content", "message", "body", "description", "prompt")
_TEXT_COLUMNS = _TEXT_FIELDS
@@ -277,8 +271,8 @@ class RawTextDataLoader:
def _extract_text_from_json(self, data):
"""Extract text from JSON object using common field names."""
- # Skip non-object lines (str/list/number): `field in data` would be a
- # substring/membership test, not a key lookup, and `data[field]` raises.
+ # Skip non-object lines (str/list/number): `field in data` would be a substring/membership test, not
+ # a key lookup, and `data[field]` raises.
if not isinstance(data, dict):
return ""
for field in self._TEXT_FIELDS:
@@ -311,7 +305,6 @@ def _iter_column(dataset, column):
class TextPreprocessor:
- # Compile regex patterns once for better performance
_WHITESPACE_PATTERN = re.compile(r"[^\S\n]+")
_INVALID_CHARS_PATTERN = re.compile(r"[^\x20-\x7E\n]")
_MULTIPLE_SPACES_PATTERN = re.compile(r"[ ]{2,}")
@@ -336,8 +329,8 @@ class TextPreprocessor:
"""Extract specific sections (e.g., code blocks, quotes)"""
sections = []
for pattern in patterns:
- # Compile pattern on first use and cache? Well, patterns are user-provided,
- # so just use re.findall with compiled flags
+ # Compile pattern on first use and cache? Well, patterns are user-provided, so just use re.findall
+ # with compiled flags
matches = re.findall(pattern, text, re.MULTILINE | re.DOTALL)
sections.extend(matches)
return sections
@@ -373,8 +366,8 @@ class TextPreprocessor:
"warnings": [],
}
- # `column_names` is HF Dataset-only; None means a mapping-like (DataFrame,
- # dict, custom __getitem__), which this method has always accepted.
+ # `column_names` is HF Dataset-only; None means a mapping-like (DataFrame, dict, custom
+ # __getitem__), which this method has always accepted.
column_names = getattr(dataset, "column_names", None)
if column_names is None or "text" in column_names:
texts = _iter_column(dataset, "text")
@@ -402,35 +395,30 @@ class TextPreprocessor:
stats["empty_samples"] += 1
continue
- # Check for encoding issues
try:
text.encode("utf-8")
except UnicodeEncodeError:
stats["encoding_issues"] += 1
- # Calculate lengths
length = len(text)
text_lengths.append(length)
stats["min_length"] = min(stats["min_length"], length)
stats["max_length"] = max(stats["max_length"], length)
- # Check for repeated content
text_hash = hash(text.strip())
if text_hash in seen_texts:
stats["repeated_content"] += 1
else:
seen_texts.add(text_hash)
- # Calculate average length
if text_lengths:
stats["avg_length"] = sum(text_lengths) / len(text_lengths)
- # No sample had content, so min_length is still its float("inf") seed. Report 0,
- # like max_length and avg_length beside it, rather than handing back an infinity.
+ # No sample had content, so min_length is still its float("inf") seed; report 0 like max_length and
+ # avg_length beside it.
if stats["min_length"] == float("inf"):
stats["min_length"] = 0
- # Generate warnings
if stats["empty_samples"] > 0:
stats["warnings"].append(f"Found {stats['empty_samples']} empty samples")
@@ -440,8 +428,8 @@ class TextPreprocessor:
if stats["encoding_issues"] > 0:
stats["warnings"].append(f"Found {stats['encoding_issues']} encoding issues")
- # Guard on text_lengths, not on min_length: with nothing measured it is now 0,
- # and zero measured samples is not "some samples are very short".
+ # Guard on text_lengths, not on min_length: with nothing measured it is now 0, and zero measured
+ # samples is not "some samples are very short".
if text_lengths and stats["min_length"] < 10:
stats["warnings"].append("Some samples are very short (< 10 characters)")
diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py
index befb5f013a4..514953aa9b6 100644
--- a/unsloth/dataprep/synthetic.py
+++ b/unsloth/dataprep/synthetic.py
@@ -183,7 +183,7 @@ class SyntheticDataKit:
float8_kv_cache = False,
conservativeness = 1.0,
token = None,
- timeout = 1200, # maybe this is not enough for large models if we need to download
+ timeout = 1200,
**kwargs,
):
assert type(model_name) is str
@@ -232,7 +232,6 @@ class SyntheticDataKit:
elif dtype_val == torch.float32:
dtype_val = "float32"
engine_args["dtype"] = dtype_val
- # Convert torch dtype to valid CLI string
if hasattr(dtype_val, "name"):
engine_args["dtype"] = dtype_val.name
elif isinstance(dtype_val, str) and dtype_val.startswith("torch."):
@@ -254,9 +253,7 @@ class SyntheticDataKit:
for key, value in engine_args.items():
flag = key.replace("_", "-")
if key == "compilation_config":
- # [TODO] Unsure why subprocess doesn't process json properly
- # Also -O3 breaks on T4!
- # subprocess_commands += ["-O3",]
+ # -O3 breaks on T4.
continue
which = str(value).replace("torch.", "")
if which == "True":
@@ -265,10 +262,8 @@ class SyntheticDataKit:
"--" + flag,
]
elif which == "False":
- # Ignore flag
pass
elif which == "None":
- # Ignore flag
pass
else:
subprocess_commands += [
@@ -300,7 +295,7 @@ class SyntheticDataKit:
ready_regex = None,
text = False,
)
- # we don't print stderr to console but self.stderr_capture.tail(200) will print the last 200 lines
+ # stderr is not printed to console, but self.stderr_capture.tail(200) prints the last 200 lines.
self._await_vllm_server(timeout = timeout)
print("vLLM Server Ready Detected")
@@ -341,11 +336,9 @@ class SyntheticDataKit:
"""
deadline = _deadline(timeout)
while True:
- # Cap each lap at the time left: a flat `poll_interval` overshoots
- # any shorter timeout, and readiness inside that overshoot would
- # return success from an expired deadline. No deadline means full
- # laps, which still notice a dead child where the bare
- # `Event.wait(None)` this replaced would have hung forever.
+ # Cap each lap at the time left: a flat poll_interval overshoots any shorter timeout, and
+ # readiness inside that overshoot would return success from an expired deadline. No deadline means
+ # full laps, which still notice a dead child where the bare Event.wait(None) this replaced hung.
remaining = _remaining(deadline)
if remaining is not None and remaining <= 0:
self._fail_vllm_server(f"was not ready within {timeout} seconds")
@@ -357,8 +350,8 @@ class SyntheticDataKit:
self._fail_vllm_server(f"exited with code {returncode} before it was ready")
if self.stdout_capture.has_closed():
self._fail_vllm_server("closed its stdout before it was ready")
- # Expiry is checked at the top, so a dead or closed child keeps
- # its own message rather than being reported as a timeout.
+ # Expiry is checked at the top, so a dead or closed child keeps its own message rather than being
+ # reported as a timeout.
def _fail_vllm_server(self, what_happened):
"""Terminate the server and raise, quoting what it managed to say.
@@ -412,8 +405,7 @@ class SyntheticDataKit:
response = requests.get("http://localhost:8000/metrics", timeout = 5)
return response.status_code == 200
except requests.exceptions.RequestException:
- # ConnectionError alone let a read timeout escape as a stray
- # traceback out of the readiness loop.
+ # ConnectionError alone let a read timeout escape as a stray traceback out of the readiness loop.
return False
def cleanup(self):
@@ -445,7 +437,6 @@ class SyntheticDataKit:
torch.cuda.empty_cache()
gc.collect()
- # Delete vLLM module as well
if hasattr(self, "_delete_vllm"):
self._delete_vllm(llm = None)
@@ -477,8 +468,8 @@ class SyntheticDataKit:
if max_tokens <= 5:
raise RuntimeError("Generation length is way too long!")
if max_tokens <= self.overlap:
- # A non-positive stride (max_tokens - overlap) makes the n_chunks
- # computation below divide by zero or go negative, so reject it.
+ # A non-positive stride (max_tokens - overlap) makes the n_chunks computation below divide by zero
+ # or go negative.
raise RuntimeError(
f"The chunk size (max_seq_length - 2 * max_generation_tokens - 128 = "
f"{max_tokens}) must be larger than the overlap ({self.overlap}). "
@@ -489,21 +480,16 @@ class SyntheticDataKit:
# Get left and right boundaries
length = len(input_ids)
if length <= max_tokens:
- # The whole document fits in one chunk window, so emit it as a single
- # chunk. Routing it through the multi-chunk path below would drop it
- # (the linspace/stack pairing emits one fewer range than boundary
- # points) or, for a document shorter than the overlap, slice the wrong
- # tokens via negative start indices. Empty doc -> no chunk.
+ # The whole document fits one chunk window: the multi-chunk path below would drop it (the
+ # linspace/stack pairing emits one fewer range than boundary points) or, for a document shorter
+ # than the overlap, slice the wrong tokens via negative start indices.
boundaries = [[0, length]] if length > 0 else []
else:
- # length > max_tokens > overlap here, so length - overlap > 0 and the
- # linspace boundaries below are always non-negative.
- # Minimal count: overlapping chunks cover `length` in
- # ceil((length - overlap) / stride) chunks, not ceil(length / stride)
- # which over-splits just past a stride multiple.
+ # Minimal count: overlapping chunks cover `length` in ceil((length - overlap) / stride) chunks, not
+ # ceil(length / stride), which over-splits just past a stride multiple.
n_chunks = int(np.ceil((length - self.overlap) / (max_tokens - self.overlap)))
- # n_chunks + 1 points: [:-1]/[1:] pairing yields n_chunks ranges; using
- # n_chunks points gave one fewer, oversized chunk (over max_tokens).
+ # n_chunks + 1 points: the [:-1]/[1:] pairing yields n_chunks ranges; n_chunks points gave one
+ # fewer, oversized chunk (over max_tokens).
boundaries = np.ceil(np.linspace(0, length - self.overlap, n_chunks + 1)).astype(int)
boundaries = np.stack((boundaries[:-1], (boundaries + self.overlap)[1:])).T
boundaries = np.minimum(boundaries, length).tolist()
diff --git a/unsloth/dataset_num_proc.py b/unsloth/dataset_num_proc.py
index f891074d3b3..38b4df82a79 100644
--- a/unsloth/dataset_num_proc.py
+++ b/unsloth/dataset_num_proc.py
@@ -43,31 +43,28 @@ __all__ = [
"resolve_responses_only_num_proc",
]
-# Escape hatch: a positive integer forces that count verbatim (no cap, no
-# start-method veto), "0"/"none" forces in-process tokenization.
+# Escape hatch: a positive integer forces that count verbatim (no cap, no start-method veto),
+# "0"/"none" forces in-process tokenization.
NUM_PROC_ENV_VAR = "UNSLOTH_DATASET_NUM_PROC"
-# Upper bound for the *auto* count only; raise it via NUM_PROC_ENV_VAR. The old
-# min(max(cpu_count + 4, 2), 64) forked up to 64 workers, each handed its own
-# dill-pickled tokenizer closure and an Arrow shard over a pipe. Measured on
-# 8000 rows, Qwen2.5 fast tokenizer, 3 reps: None 6.3s, 1 9.5s, 8 8.1s,
-# 32 14.2s, 64 21.7s -- more workers were slower than none at every size.
+# Upper bound for the AUTO count only; raise it via NUM_PROC_ENV_VAR. The old
+# min(max(cpu_count + 4, 2), 64) forked up to 64 workers, each handed its own dill-pickled
+# tokenizer closure and an Arrow shard over a pipe: measured on 8000 rows, more workers were
+# slower than none at every size (None 6.3s against 64 workers 21.7s).
AUTO_NUM_PROC_CAP = 8
-# ~680 MB peak RSS per worker on that run, flat across counts; 1 GB for headroom.
+# ~680 MB peak RSS per worker, flat across counts; 1 GB for headroom.
WORKER_MEMORY_BUDGET_GB = 1.0
-# Share of *available* RAM tokenization may spend; the rest belongs to the run
-# that follows. This is what bounds issue #2693, where OOM-killed workers
-# surface only as "One of the subprocesses has abruptly died during map
-# operation", the real cause discarded.
+# Share of AVAILABLE RAM tokenization may spend; the rest belongs to the run that follows. This is
+# what bounds #2693, where OOM-killed workers surface only as "One of the subprocesses has
+# abruptly died during map operation".
MEMORY_BUDGET_FRACTION = 0.5
-# Mirrors _MIN_ROWS_FOR_MULTIPROC, a local inside
-# dataset_utils.train_on_responses_only (hence not importable): below it that
-# helper maps a split in-process, unless handed an explicit count.
-# resolve_responses_only_num_proc needs the threshold to keep that guard; a
-# canary in tests/test_dataset_num_proc.py catches drift.
+# Mirrors _MIN_ROWS_FOR_MULTIPROC, a local inside dataset_utils.train_on_responses_only (hence not
+# importable): below it that helper maps a split in-process unless handed an explicit count.
+# resolve_responses_only_num_proc needs the threshold to keep that guard; a canary in
+# tests/test_dataset_num_proc.py catches drift.
ZOO_MIN_ROWS_FOR_MULTIPROC = 5_000
# Warn at most once per process per distinct reason.
@@ -105,9 +102,8 @@ def _unpinned_default_start_method(module) -> Optional[str]:
try:
name = module.context._default_context._default_context._name
- # Only if the platform actually offers it: a Windows runner answered
- # 'fork' while get_all_start_methods() was ['spawn'], which would read
- # Windows as forkable -- the spawn re-import loop of #3211 / #3397.
+ # Only if the platform actually offers it: a Windows runner answered 'fork' while
+ # get_all_start_methods() was ['spawn'], which would read Windows as forkable (#3211 / #3397).
if isinstance(name, str) and name and (not methods or name in methods):
return name
except Exception:
@@ -140,6 +136,7 @@ def multiprocessing_start_method() -> Optional[str]:
try:
return _module_start_method(module_name)
except Exception:
+ # Skip rather than let it mask a sized sibling.
continue
return None
@@ -187,9 +184,8 @@ CGROUP_ROOT = "/sys/fs/cgroup"
def _cgroup_first_line(path: str) -> Optional[str]:
try:
- # encoding named explicitly: these are ASCII kernel files, but a
- # locale-dependent read crashes or produces mojibake on a Windows
- # console codepage, and CI polices every read/write for it.
+ # encoding named explicitly: a locale-dependent read of these ASCII kernel files crashes or produces
+ # mojibake on a Windows console codepage, and CI polices every read/write for it.
with open(path, "r", encoding = "utf-8") as f:
return f.readline().strip()
except OSError:
@@ -250,8 +246,8 @@ def _cgroup_free_bytes_unaided() -> Optional[int]:
if os.path.isdir(CGROUP_ROOT):
rel = None
- # The v2 line is "0::<path>"; under systemd hybrid mode v1 lines share
- # the file, so scan rather than taking the first.
+ # The v2 line is "0::<path>"; under systemd hybrid mode v1 lines share the file, so scan rather than
+ # taking the first.
for line in lines:
if line.startswith("0::"):
rel = line[3:].strip()
@@ -309,10 +305,9 @@ def _cgroup_free_bytes() -> Optional[int]:
_read_first_line,
)
except Exception:
- # An older unsloth_zoo has no such private helpers. Do the same pairing
- # here rather than reading the public limit alone: an 8GB cgroup with 6GB
- # already resident would otherwise report 8GB free, and memory pressure
- # is the one condition this ceiling exists for.
+ # An older unsloth_zoo has no such private helpers. Do the same pairing here rather than reading
+ # the public limit alone: an 8GB cgroup with 6GB already resident would otherwise report 8GB free,
+ # and memory pressure is the one condition this ceiling exists for.
return _cgroup_free_bytes_unaided()
def _used(path):
@@ -404,8 +399,7 @@ def _clamp_by_memory(num_proc: int) -> Optional[int]:
"""
affordable = _affordable_workers()
if affordable is None:
- # No memory reading, so honour the request rather than serialising a
- # machine that may be perfectly capable.
+ # No memory reading, so honour the request rather than serialising a machine that may be perfectly capable.
return num_proc
if affordable < 2:
@@ -531,17 +525,15 @@ def get_dataset_num_proc(
A worker count >= 2, or this layer's in-process sentinel (``None`` at a
call site, ``1`` at the config layer).
"""
- # 1. The environment override wins over everything, uncapped and unvetoed,
- # so a user who knows their workload is fork-safe is never downgraded.
+ # 1. The environment override wins over everything, uncapped and unvetoed, so a user who knows
+ # their workload is fork-safe is never downgraded.
env_set, env_value = _from_environment()
if env_set:
- # In-process still has to be encoded for this layer, or
- # UNSLOTH_DATASET_NUM_PROC=0 -- the hatch the dead-worker message
- # recommends -- would inflate a config instead of removing workers.
+ # In-process still has to be encoded for this layer, or UNSLOTH_DATASET_NUM_PROC=0 (the hatch the
+ # dead-worker message recommends) would inflate a config instead of removing workers.
return _serial(serial_as_none) if env_value is None else env_value
- # 2. Workers are unusable whatever was requested; see
- # `_workers_unusable_reason` for the two refusals.
+ # 2. Workers are unusable whatever was requested; see _workers_unusable_reason.
unusable = _workers_unusable_reason()
if unusable is not None:
if isinstance(desired, int) and not isinstance(desired, bool) and desired > 1:
@@ -553,13 +545,13 @@ def get_dataset_num_proc(
)
return _serial(serial_as_none)
- # 3. Normalise "no multiprocessing" requests. `1` is the trap: callers pass
- # it meaning "serial" and datasets >= 4.1 hands them a Pool(1).
+ # 3. Normalise "no multiprocessing" requests. `1` is the trap: callers pass it meaning "serial"
+ # and datasets >= 4.1 hands them a Pool(1).
if isinstance(desired, int) and not isinstance(desired, bool) and desired <= 1:
return _serial(serial_as_none)
- # 4. Auto-size when no usable request was made, then bound by memory. A
- # non-int, or a bool (an int subclass), is not a request.
+ # 4. Auto-size when no usable request was made, then bound by memory. A non-int, or a bool (an int
+ # subclass), is not a request.
if desired is None or not isinstance(desired, int) or isinstance(desired, bool):
num_proc = _auto_num_proc()
else:
@@ -570,9 +562,9 @@ def get_dataset_num_proc(
return num_proc
-# The message datasets raises when a pool worker dies. It never reads the child's
-# exit status, so an OOM kill, a segfault and a genuine exception all arrive as
-# this one string -- which is why #2693 looks untraceable.
+# The message datasets raises when a pool worker dies. It never reads the child's exit status, so
+# an OOM kill, a segfault and a genuine exception all arrive as this one string, which is why
+# #2693 looks untraceable.
_WORKER_DIED = "subprocesses has abruptly died"
@@ -634,7 +626,6 @@ def _largest_split_rows(trainer) -> Optional[int]:
try:
rows = len(split)
except Exception:
- # Skip rather than let it mask a sized sibling.
continue
measured = True
largest = max(largest, rows)
@@ -699,8 +690,8 @@ def resolve_responses_only_num_proc(trainer, num_proc):
have had: worth it only when a split is big enough to have been
parallelized at all, which is what the row check below establishes.
"""
- # Mirror that helper's own test, so "explicit" means the same on both sides
- # (it treats bools as auto, since type(True) is not int).
+ # Mirror that helper's own test, so "explicit" means the same on both sides (it treats bools as
+ # auto, since type(True) is not int).
was_auto = num_proc is None or type(num_proc) is not int
rows = _largest_split_rows(trainer)
small = rows is None or rows < ZOO_MIN_ROWS_FOR_MULTIPROC
@@ -708,11 +699,7 @@ def resolve_responses_only_num_proc(trainer, num_proc):
if not was_auto:
resolved = get_dataset_num_proc(num_proc, serial_as_none = False)
if resolved == 1 and small:
- # Serial, and every split is under the threshold, so the helper's own
- # guard runs each of them in-process, which an older unsloth_zoo
- # would not do for the 1 expressible here: reaching this with
- # UNSLOTH_DATASET_NUM_PROC=0 and a config sentinel of 1 was the
- # escape hatch still building a Pool(1) there.
+ # Serial with every split under the threshold, so the helper's own guard runs each in-process.
return None
return _serial_for_the_zoo(resolved)
@@ -721,10 +708,8 @@ def resolve_responses_only_num_proc(trainer, num_proc):
env_set, env_value = _from_environment()
if env_set and env_value is not None:
return env_value
- # Otherwise it would have gone in-process anyway, and its guard yields
- # None, which is more in-process than the 1 expressible here. Safe whatever
- # the start method: every split here is under the threshold, so its own
- # per-split guard serialises them all.
+ # Otherwise it would have gone in-process anyway, and its guard yields None, which is more in-
+ # process than the 1 expressible here. Safe whatever the start method.
return num_proc
return _serial_for_the_zoo(get_dataset_num_proc(None, serial_as_none = False))
diff --git a/unsloth/device_type.py b/unsloth/device_type.py
index ebb3001e840..d6793ed421b 100644
--- a/unsloth/device_type.py
+++ b/unsloth/device_type.py
@@ -60,13 +60,12 @@ def is_hip():
@functools.cache
def get_device_type():
- # MLX first: torch is never imported on the MLX runtime, so claiming "cuda" here
- # would NameError in get_device_count. Matches unsloth/__init__.py and
- # unsloth_zoo.device_type, which both pick MLX ahead of the CPU fallback.
+ # MLX first: torch is never imported on the MLX runtime, so claiming "cuda" here would NameError in
+ # get_device_count. Matches unsloth/__init__.py and unsloth_zoo.device_type.
if _IS_MLX:
return "mlx"
- # Test-only CPU fallback: report "cuda" so every DEVICE_TYPE == "cuda"
- # branch behaves identically. Read once per process (function is cached).
+ # Test-only CPU fallback: report "cuda" so every DEVICE_TYPE == "cuda" branch behaves identically.
+ # Read once per process (function is cached).
if os.environ.get("UNSLOTH_ALLOW_CPU", "0") == "1":
return "cuda"
if hasattr(torch, "cuda") and torch.cuda.is_available():
@@ -75,7 +74,6 @@ def get_device_type():
return "cuda"
elif hasattr(torch, "xpu") and torch.xpu.is_available():
return "xpu"
- # Check torch.accelerator
if hasattr(torch, "accelerator"):
if not torch.accelerator.is_available():
raise NotImplementedError("Unsloth cannot find any torch accelerator? You need a GPU.")
@@ -110,28 +108,19 @@ def get_device_count():
DEVICE_COUNT: int = get_device_count()
-# 4-bit quantization requires a block size of 64
-# | Device Type | Warp Size | Block Size |
-# |-----------------|-----------|------------|
-# | CUDA | 32 | 32 |
-# | Radeon (Navi) | 32 | 32 |
-# | Instinct (MI) | 64 | 32 |
-#
-# Since bitsandbytes 0.49.0, pre-quantized models with 64 blockwise now works
-# on Radeon GPUs, but not Instinct MI300x for eg
-# See https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1748
-#
-# Since bitsandbytes 0.49.2, blocksize=64 4-bit quantization is supported on
-# CDNA (MI Instinct / gfx9xx) GPUs as well
-# See https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1856
+# 4-bit quantization requires a block size of 64: Instinct (MI) has a warp size of 64 against 32
+# elsewhere. Since bitsandbytes 0.49.0 pre-quantized 64-blockwise models work on Radeon (Navi)
+# but not Instinct (bitsandbytes-foundation/bitsandbytes#1748); since 0.49.2 blocksize=64 4-bit
+# is supported on CDNA (MI Instinct / gfx9xx) too (#1856).
+# |-----------------|-----------|------------|
ALLOW_PREQUANTIZED_MODELS: bool = True
# HSA_STATUS_ERROR_EXCEPTION checks - sometimes AMD fails for BnB
ALLOW_BITSANDBYTES: bool = True
-# Unusable bitsandbytes on any backend, not just hip: clear the flags the loader reads
-# before it picks a 4bit checkpoint. A guarded import, not find_spec, since importable
-# is not usable - from 0.46 a dead native library still resolves every ctypes handle to
-# a closure that raises only when called, so 4bit would die mid-run, not fall back here.
+# Unusable bitsandbytes on any backend, not just hip: clear the flags the loader reads before it
+# picks a 4bit checkpoint. A guarded import, not find_spec, since importable is not usable: from
+# 0.46 a dead native library still resolves every ctypes handle to a closure that raises only when
+# called, so 4bit would die mid-run rather than fall back here.
try:
import bitsandbytes as _bnb_probe
except Exception:
@@ -142,10 +131,8 @@ else:
ALLOW_PREQUANTIZED_MODELS = False
ALLOW_BITSANDBYTES = False
del _bnb_probe
-# gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this
-# legacy GCN arch (ROCm dropped it after 6.3) - compiled graphs crash or miscompile
-# while the eager path trains fine. Default compile off; setdefault so a user
-# override wins.
+# gfx906 (MI50 / Radeon VII / Vega 20): Dynamo/Inductor codegen is broken on this legacy GCN arch
+# (ROCm dropped it after 6.3), so compiled graphs crash or miscompile while eager trains fine.
if DEVICE_TYPE == "hip":
try:
_gcn_arch = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0].strip().lower()
@@ -174,7 +161,7 @@ if DEVICE_TYPE == "hip":
pass
elif Version(bitsandbytes.__version__) >= Version("0.49.0"):
try:
- # Pre-quantized bitsandbytes models use blocksize 64, so we need to check the GPU
+ # Pre-quantized bitsandbytes models use blocksize 64.
from bitsandbytes.cextension import ROCM_WARP_SIZE_64
ALLOW_PREQUANTIZED_MODELS = not ROCM_WARP_SIZE_64
except Exception as e:
diff --git a/unsloth/disk_utils.py b/unsloth/disk_utils.py
index 43110475d33..684b58bd6a1 100644
--- a/unsloth/disk_utils.py
+++ b/unsloth/disk_utils.py
@@ -61,11 +61,9 @@ except ImportError:
_TRUE = ("1", "true", "yes", "on")
def is_kaggle_environment():
- # Kept byte for byte in step with unsloth_zoo.disk_utils: only a real
- # kernel sets KAGGLE_KERNEL_RUN_TYPE, and only the Kaggle image has a
- # /kaggle/working. KAGGLE_USERNAME / KAGGLE_KEY / KAGGLE_CONFIG_DIR
- # are what people export for the Kaggle CLI on their own machines,
- # and must never be mistaken for one.
+ # Kept in step with unsloth_zoo.disk_utils: only a real kernel sets KAGGLE_KERNEL_RUN_TYPE and
+ # only the Kaggle image has /kaggle/working, whereas KAGGLE_USERNAME / KAGGLE_KEY /
+ # KAGGLE_CONFIG_DIR are what people export for the Kaggle CLI on their own machines.
override = os.environ.get("UNSLOTH_IS_KAGGLE", None)
if override is not None:
return str(override).strip().lower() in _TRUE
@@ -100,10 +98,9 @@ except ImportError:
return None
def logical_numel(param, name = ""):
- # Packed storage cannot be unpacked without the zoo's knowledge of the
- # packing schemes, so this reports what `numel()` says, exactly as the
- # code did before it asked. Barely reachable: `model_16bit_bytes` is 0
- # here, and every caller returns before sizing anything.
+ # Packed storage cannot be unpacked without the zoo's knowledge of the packing schemes, so report
+ # what numel() says, exactly as the code did before it asked. Barely reachable: model_16bit_bytes
+ # is 0 here and every caller returns before sizing anything.
try:
return int(param.numel())
except Exception:
@@ -116,8 +113,8 @@ except ImportError:
return 0
def estimate_gguf_export_bytes(*args, **kwargs):
- # 0 means "unmeasurable", and every caller treats that as "do not
- # block". An old unsloth_zoo therefore behaves exactly as it did.
+ # 0 means "unmeasurable", and every caller treats that as "do not block", so an old unsloth_zoo
+ # behaves exactly as it did.
return 0
def kaggle_tmp_redirect(save_directory, *args, **kwargs):
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 7af9b08e8e9..c161babd18a 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -56,7 +56,7 @@ def Version(version):
raise ValueError(f"Could not parse version: {version}")
new_version = new_version.group(0).rstrip(".")
if new_version != version:
- new_version += ".1" # Add .1 for dev / alpha / beta / rc
+ new_version += ".1"
return TrueVersion(new_version)
except:
from inspect import getframeinfo, stack
@@ -145,9 +145,10 @@ def suppress_cuda_printf():
if not UNSLOTH_ENABLE_LOGGING:
import sys
- # Apply to stderr for FBGEMM and CUTLASS errors
+ # Applied to stderr for FBGEMM (pytorch/FBGEMM utils.py#L43-L52) and CUTLASS SM90-vs-SM100 MMA / arch / TMA errors.
sys.stderr = HidePrintMessage(sys.stderr)
- # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52
+ # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/ge
+ # mm/triton_gemm/utils.py#L43-L52
sys.stderr.add_filter("TMA benchmarks will be running")
# CUTLASS/FBGEMM MMA instruction error on SM90 vs SM100 (Blackwell) GPUs
# https://github.com/NVIDIA/cutlass/blob/main/include/cutlass/gemm/kernel/sm90_gemm_tma_warpspecialized.hpp
@@ -157,15 +158,14 @@ if not UNSLOTH_ENABLE_LOGGING:
# CUTLASS TMA-related errors when not targeting correct architecture
sys.stderr.add_filter("Trying to use tma without CUTE_ARCH_TMA")
# torchao logs a cosmetic "Skipping import of cpp extensions" WARNING on torch < 2.11. The
- # bnb-4bit / Unsloth paths don't use torchao's cpp kernels, so drop only that record rather
- # than raising the whole torchao logger to ERROR.
+ # bnb-4bit / Unsloth paths do not use its cpp kernels, so drop that one record rather than
+ # raising the whole torchao logger to ERROR.
logging.getLogger("torchao").addFilter(
HideLoggingMessage("Skipping import of cpp extensions due to incompatible torch version")
)
- # torch >= 2.11 path: torchao dlopens each prebuilt _C*.so and logs "Failed to load
- # .../_C*.so" when one can't (ABI tag mismatch in the wheel, e.g. a cp310 .so under a
- # cp312 runtime on Colab, or an arch-specific kernel the GPU lacks). It falls back to
- # non-cpp paths and Unsloth doesn't use these kernels, so drop the cosmetic record.
+ # torch >= 2.11: torchao dlopens each prebuilt _C*.so and logs a failure on an ABI tag
+ # mismatch (a cp310 .so under cp312) or a missing arch kernel, then falls back to non-cpp
+ # paths Unsloth does not use.
logging.getLogger("torchao").addFilter(HideLoggingMessage("Failed to load "))
# SyntaxWarning: invalid escape sequence '\.'
warnings.filterwarnings("ignore", message = "invalid escape sequence", category = SyntaxWarning)
@@ -193,7 +193,6 @@ if not UNSLOTH_ENABLE_LOGGING:
category = DeprecationWarning,
)
- # SWIG builtin type warnings (from bitsandbytes/triton SWIG bindings)
warnings.filterwarnings(
"ignore",
message = r"builtin type Swig.*has no __module__ attribute",
@@ -285,8 +284,8 @@ def fix_torch_check_is_size():
return
-# Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'
-# MUST do this at the start primarily due to tensorflow causing issues
+# Fix "AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'" first, mainly
+# because tensorflow causes issues.
def fix_message_factory_issue():
try:
import google.protobuf.message_factory
@@ -328,7 +327,7 @@ def fix_message_factory_issue():
pass
-# Fix Xformers performance issues since 0.0.25
+# Fix xformers performance issues since 0.0.25; see facebookresearch/xformers#1176 (comment 2545829591).
def fix_xformers_performance_issue():
spec = importlib.util.find_spec("xformers")
if spec is None:
@@ -359,18 +358,14 @@ def fix_xformers_performance_issue():
logger.info(f"Unsloth: Failed patching Xformers with error = {str(e)}")
-# flash-attn 4 ships `flash_attn/cute/` with NO `flash_attn/__init__.py`, so `flash_attn`
-# resolves as a namespace package with no `flash_attn.flash_attn_interface`. xformers gates on
-# `find_spec("flash_attn")` and then imports that submodule unguarded, so `import xformers.ops`
-# raises, `models/_utils.py` swallows it into `xformers = None`, HAS_XFORMERS goes False and
-# every fast-path model silently drops to SDPA. Measured on a B200 at seq_len 8192 with
-# Qwen3-0.6B + LoRA: 547 ms/step / 2.69 GB peak with xformers versus 2154 ms/step / 19.02 GB
-# peak on SDPA -- 3.9x slower, 7x memory, no message. flash-attn 2 has no wheel for torch 2.9 /
-# cp313, so flash-attn 4 is what a Blackwell user reaches for.
-#
-# The repair imports xformers ONCE with `flash_attn` hidden from `find_spec`, so xformers takes
-# the next branch of its own elif chain exactly as it would with no flash-attn at all, and the
-# working module is cached in `sys.modules`. Nothing is written to any third-party package.
+# flash-attn 4 ships flash_attn/cute/ with no __init__.py, so flash_attn is a namespace package
+# without flash_attn.flash_attn_interface. xformers gates on find_spec then imports that submodule
+# unguarded, so `import xformers.ops` raises, _utils.py swallows it into xformers = None,
+# HAS_XFORMERS goes False and every fast-path model drops to SDPA: measured on a B200 at seq_len
+# 8192, 547 ms/step and 2.69 GB peak against 2154 ms/step and 19.02 GB on SDPA. The repair imports
+# xformers ONCE with flash_attn hidden from find_spec, so it takes the next branch of its own elif
+# chain and the working module is cached in sys.modules. Nothing is written to any third-party
+# package.
_FLASH_ATTN_INTERFACE_NAME = "flash_attn_interface"
_FLASH_ATTN_INTERFACE_MODULE = "flash_attn." + _FLASH_ATTN_INTERFACE_NAME
_FA4_NAMESPACE_WARNED = [False]
@@ -464,13 +459,12 @@ def fix_flash_attn_4_namespace_shadow():
if _flash_attn_layout() != "flash_attn_4_only":
return
if importlib.util.find_spec("xformers") is None:
- # Nothing to protect: no xformers means SDPA regardless, and transformers'
- # `is_flash_attn_2_available()` is False here (the distribution is `flash-attn-4`, so
- # the metadata lookup for `flash_attn` misses).
+ # Nothing to protect: no xformers means SDPA anyway, and is_flash_attn_2_available() is False
+ # here, since the distribution is flash-attn-4 and the flash_attn metadata lookup misses.
return
if "xformers.ops.fmha.flash" in sys.modules:
- # Already imported successfully. A FAILED import leaves nothing in sys.modules, so this
- # does not mask the case we are here to fix.
+ # Already imported successfully. A FAILED import leaves nothing in sys.modules, so this does
+ # not mask the case we are here to fix.
return
real_find_spec = importlib.util.find_spec
@@ -480,9 +474,9 @@ def fix_flash_attn_4_namespace_shadow():
return None
return real_find_spec(name, package)
- # Process-global swap, scoped to this one import and restored in `finally`. Any other thread
- # calling find_spec("flash_attn*") in the ~1.0s window is told the package is absent, which
- # is the honest answer for the FA2 namespace xformers is asking about anyway.
+ # Process-global swap, scoped to this one import and restored in `finally`. Another thread
+ # calling find_spec("flash_attn*") in the ~1.0s window is told it is absent, which is honest
+ # for the FA2 namespace xformers asks about.
importlib.util.find_spec = _find_spec_without_flash_attn
try:
import xformers.ops # noqa: F401
@@ -669,7 +663,7 @@ def fix_transformers5_bare_annotation_configs():
original = PretrainedConfig.__dict__.get("__init_subclass__")
if original is None:
return
- # `__init_subclass__` is an implicit classmethod; unwrap to the function.
+ # __init_subclass__ is an implicit classmethod; unwrap to the function.
original_func = getattr(original, "__func__", original)
def __init_subclass__(cls, *args, **kwargs):
@@ -779,14 +773,12 @@ def _sdpa_mask_leaves_rows_fully_masked():
sdpa_mask = getattr(masking_utils, "sdpa_mask", None)
if sdpa_mask is None:
return False
- # Unwrap first: a second call must probe the ORIGINAL, or the patch we
- # already installed reports the bug as fixed and a reload silently drops it.
+ # Unwrap first: a second call must probe the ORIGINAL, or the installed patch reports the bug
+ # as fixed and a reload silently drops it.
sdpa_mask = getattr(sdpa_mask, "__wrapped__", sdpa_mask)
- # The query axis is named differently across the versions we support: 5.x
- # takes `q_length`, while 4.57.6 binds `sdpa_mask` to `sdpa_mask_recent_torch`,
- # which takes `cache_position` instead. Ask the signature rather than
- # guessing -- a probe that raises TypeError answers "no bug" for a reason
- # that has nothing to do with the bug.
+ # The query axis is named differently per version: 5.x takes q_length, while 4.57.6 binds
+ # sdpa_mask to sdpa_mask_recent_torch, which takes cache_position. Ask the signature, since a
+ # probe that raises TypeError answers "no bug" for an unrelated reason.
kwargs = {
"batch_size": 2,
"kv_length": 2,
@@ -809,9 +801,8 @@ def _sdpa_mask_leaves_rows_fully_masked():
mask = sdpa_mask(**kwargs)
if mask is None or mask.is_floating_point():
return False
- # Inside the guard as well: reading a mask's truth value is what fails
- # on a meta or otherwise unmaterialised tensor, and a probe that raises
- # would take `import unsloth` down with it.
+ # Inside the guard too: reading a mask's truth value is what fails on a meta or unmaterialised
+ # tensor, and a probe that raises would take `import unsloth` down with it.
return bool((~mask.any(dim = -1)).any())
except Exception:
# Signature moved under us. Patching blind would be worse than not.
@@ -869,21 +860,16 @@ def fix_transformers_fully_masked_rows():
def sdpa_mask(*args, **kwargs):
return _unmask_rows_attending_to_nothing(original(*args, **kwargs))
- # `functools.wraps` sets __wrapped__, but set it explicitly: the probe and
- # the tests both reach for it, and a future wraps-less edit must not
- # silently make the patch un-probeable and un-undoable.
+ # functools.wraps sets __wrapped__, but set it explicitly: the probe and the tests both read
+ # it, and a wraps-less edit must not make the patch un-probeable and un-undoable.
sdpa_mask.__wrapped__ = original
- # The mark travels ON the wrapper, so the guard above reads the live
- # binding and a reload that drops it is re-patched rather than skipped.
+ # The mark travels ON the wrapper, so the guard above reads the live binding and a reload that
+ # drops it is re-patched rather than skipped.
setattr(sdpa_mask, _SDPA_MASK_PATCH_FLAG, True)
try:
- # BOTH bindings, and they are not the same object. `eager_mask` calls
- # the module global by name at call time, so it picks this up; the
- # interface captured the original function in `_global_mapping` when
- # the class body ran, so it needs re-registering. Confirmed by search:
- # `sdpa_mask` is defined once and no other transformers module
- # references it.
+ # BOTH bindings, which are not the same object: eager_mask calls the module global by name at
+ # call time, while the interface captured the original in _global_mapping at class-body time.
masking_utils.sdpa_mask = sdpa_mask
interface = getattr(masking_utils, "ALL_MASK_ATTENTION_FUNCTIONS", None)
if interface is not None:
@@ -902,8 +888,7 @@ def fix_vllm_aimv2_issue():
spec = importlib.util.find_spec("vllm")
if spec is None:
return
- # A findable spec with unreadable dist metadata (broken/partial vllm install)
- # must not crash unsloth import; every other vllm probe here guards this too.
+ # A findable spec with unreadable dist metadata (broken/partial vllm install) must not crash `import unsloth`.
try:
vllm_version = importlib_version("vllm")
except Exception as e:
@@ -946,10 +931,9 @@ def fix_vllm_aimv2_issue():
logger.info(f"Unsloth: Failed patching vLLM with error = {str(e)}")
-# vLLM >= 0.22 (PR #35024) deleted `vllm.transformers_utils.tokenizer`, but an
-# older unsloth_zoo still imports it unguarded and crashes (issue #6385). Supply
-# a stub via a meta path finder appended AFTER the real finders, so it only
-# activates when vLLM no longer ships the module.
+# vLLM >= 0.22 (PR #35024) deleted vllm.transformers_utils.tokenizer, which an older
+# unsloth_zoo still imports unguarded (#6385). Stub it with a meta path finder appended AFTER
+# the real ones, so it only fires when vLLM no longer ships the module.
_VLLM_LORA_TOKENIZER_MODULE = "vllm.transformers_utils.tokenizer"
_VLLM_TOKENIZER_STUB_SENTINEL = "__unsloth_vllm_tokenizer_stub__"
@@ -1033,9 +1017,7 @@ def fix_vllm_guided_decoding_params():
if importlib.util.find_spec("vllm") is None:
return
- # GuidedDecodingParmas is renamed to StructuredOutputsParams in vLLM
- # https://github.com/vllm-project/vllm/pull/22772/files
- # trl still wants to use GuidedDecodingParams. This is a temporary patch till trl updates
+ # GuidedDecodingParams was renamed to StructuredOutputsParams in vLLM (vllm#22772); trl still wants the old name.
try:
import vllm
except (ImportError, OSError) as e:
@@ -1058,17 +1040,10 @@ def fix_vllm_guided_decoding_params():
def fix_trl_vllm_ascend():
- # transformers >= 4.48's `_is_package_available(name)` returns a tuple
- # (bool, version_or_None). TRL caches that tuple in module-level
- # `_*_available` flags and the matching `is_*_available()` accessors
- # return the tuple directly. A non-empty tuple is always truthy, so
- # `if is_X_available():` fires even when X is absent, triggering an
- # unconditional `import X` that fails. The surfaced case is
- # `vllm_ascend` (blocks `from trl import GRPOConfig, GRPOTrainer`
- # outside Huawei Ascend hosts); `llm_blender`, `deepspeed`, `joblib`
- # share the same shape. Coerce every tuple-cached flag in
- # trl.import_utils to bool; the existing accessors that just return
- # the cached value then naturally yield a bool.
+ # transformers >= 4.48's _is_package_available returns (bool, version_or_None), which TRL
+ # caches and returns directly. A non-empty tuple is truthy, so `if is_X_available():` fires
+ # for an absent X and triggers a failing import -- vllm_ascend blocked `from trl import
+ # GRPOConfig` off Ascend hosts. Coerce every tuple-cached flag to bool.
if importlib.util.find_spec("trl") is None:
return
try:
@@ -1084,7 +1059,6 @@ def fix_trl_vllm_ascend():
def ignore_logger_messages():
- # Ignore Environment variable `HF_TOKEN` is set
try:
from huggingface_hub._login import logger as huggingface_hub_logger
huggingface_hub_logger.addFilter(HideLoggingMessage("`HF_TOKEN`"))
@@ -1094,10 +1068,8 @@ def ignore_logger_messages():
def patch_ipykernel_hf_xet():
- # HF-XET == 1.1.10 and ipykernel == 7.0.0 / 7.0.1 causes issues
- # See https://github.com/huggingface/xet-core/issues/526
- # 2025-10-13T20:37:33.028737Z ERROR Python exception updating progress:, error: PyErr { type: <class 'LookupError'>, value: LookupError(<ContextVar name='shell_parent' at 0x7535b4cebd80>), traceback: Some(<traceback object at 0x753408489f40>) }, caller: "src/progress_update.rs:313"
- # at /home/runner/work/xet-core/xet-core/error_printer/src/lib.rs:28
+ # HF-XET 1.1.10 with ipykernel 7.0.0 / 7.0.1 raises LookupError on ContextVar 'shell_parent';
+ # see huggingface/xet-core#526.
if importlib.util.find_spec("hf_xet") is None:
return
if importlib.util.find_spec("ipykernel") is None:
@@ -1106,14 +1078,8 @@ def patch_ipykernel_hf_xet():
return
ipykernel_version = Version(importlib_version("ipykernel"))
- if (
- (Version(importlib_version("hf_xet")) == Version("1.1.10"))
- and (
- (ipykernel_version == Version("7.0.0"))
- or (
- ipykernel_version == Version("7.0.1")
- ) # 7.0.1 seems to also break with LookupError: <ContextVar name='shell_parent' at 0x7a9775143ec0>
- )
+ if (Version(importlib_version("hf_xet")) == Version("1.1.10")) and (
+ (ipykernel_version == Version("7.0.0")) or (ipykernel_version == Version("7.0.1"))
):
print(
"#### Unsloth: `hf_xet==1.1.10` and `ipykernel==7.0.0` or `ipykernel==7.0.1` breaks progress bars. Using ASCII progress bars.\n"
@@ -1125,8 +1091,7 @@ def patch_ipykernel_hf_xet():
def patch_trackio():
- # Set some environment variables to customize the Trackio dashboard for experiment tracking
- # See https://github.com/unslothai/notebooks/pull/110
+ # Customize the Trackio dashboard for experiment tracking; see unslothai/notebooks#110.
os.environ["TRACKIO_LOGO_LIGHT_URL"] = (
"https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20black%20text.png"
)
@@ -1149,28 +1114,21 @@ def patch_datasets():
)
-# psutil divides the pmgr "voltage-statesN-sram" IORegistry tables by 1e6 to
-# reach MHz, but Apple switched them from Hz to kHz on M4, so psutil <= 7.2.2
-# reads a 4.5 GHz M4 Pro back as 4 MHz (unslothai/unsloth#8519). Upstream fix is
-# giampaolo/psutil#2824, merged and unreleased; its heuristics are mirrored here.
-#
-# Unsloth's backend keeps the same correction in
-# studio/backend/utils/hardware/hardware.py, since the API server never imports
-# unsloth. Keep both in sync; delete both once a fixed psutil is our floor.
-#
+# psutil divides the pmgr voltage-states tables by 1e6 for MHz, but Apple switched them from
+# Hz to kHz on M4, so psutil <= 7.2.2 reads a 4.5 GHz M4 Pro as 4 MHz (#8519). Upstream fix
+# giampaolo/psutil#2824 is merged and unreleased; mirrored here and in
+# studio/backend/utils/hardware/hardware.py. Keep both in sync; delete once psutil is fixed.
# Apple clocks are 0.6-4.6 GHz, so a raw Hz entry sits above 1e8 and kHz below.
_APPLE_CPU_FREQ_UNIT_THRESHOLD = 100_000_000
_APPLE_MIN_PLAUSIBLE_CPU_MHZ = 500
_APPLE_MAX_PLAUSIBLE_CPU_MHZ = 20000
-# Below this a table is a GPU/NPU rail: above every Apple GPU peak so far, under
-# the slowest CPU cluster shipped (M1 E-core, 2064 MHz).
+# Below this a table is a GPU/NPU rail: above every Apple GPU peak so far, under the slowest
+# CPU cluster shipped (M1 E-core, 2064 MHz).
_APPLE_CPU_CLUSTER_MIN_PEAK_MHZ = 2000
_APPLE_VOLTAGE_STATES_KEY = re.compile(r"^voltage-states\d+-sram$")
-# Fixed for the life of the host, so probe once. The sentinel separates "not
-# probed yet" from "probed, unavailable".
+# Fixed for the life of the host, so probe once. The sentinel separates "not probed yet" from "probed, unavailable".
_apple_cpu_freq_range = "unprobed"
-# At import, not on first use: two threads reaching a lazy initialiser build a
-# lock each and then exclude nothing.
+# At import, not on first use: two threads reaching a lazy initialiser build a lock each and then exclude nothing.
_apple_cpu_freq_lock = threading.Lock()
@@ -1217,8 +1175,8 @@ def _apple_cpu_freq_range_mhz():
if _apple_cpu_freq_range != "unprobed":
return _apple_cpu_freq_range
- # Unlocked, every thread spawns its own ioreg and a slow failing probe landing
- # last overwrites a good reading with None.
+ # Unlocked, every thread spawns its own ioreg and a slow failing probe landing last overwrites
+ # a good reading with None.
with _apple_cpu_freq_lock:
if _apple_cpu_freq_range != "unprobed":
return _apple_cpu_freq_range
@@ -1257,9 +1215,9 @@ def _corrected_apple_cpu_freq(sample):
# macOS has no per-instant clock, so psutil reports the peak as `current`.
return sample._replace(current = peak, min = low, max = peak)
if not usable:
- return sample # 0, negative or NaN and no tables: nothing to say
- # No tables: recover the magnitude instead. psutil truncates in integer
- # arithmetic, so this lands on the GHz step (4 -> 4000 MHz), not the peak.
+ return sample
+ # No tables: recover the magnitude instead. psutil truncates in integer arithmetic, so this
+ # lands on the GHz step (4 -> 4000 MHz), not the peak.
return sample._replace(
current = current * 1000,
min = getattr(sample, "min", 0.0) * 1000,
@@ -1318,13 +1276,12 @@ def patch_psutil_cpu_freq():
try:
result = original_cpu_freq(*args, **kwargs)
except TypeError:
- # Arguments psutil does not take are the caller's mistake, not psutil
- # declining to answer. This function replaces psutil's globally, so
- # swallowing that would hide the error everywhere, not just here.
+ # Arguments psutil does not take are the caller's mistake, not psutil declining to answer.
+ # This replaces psutil's function globally, so swallowing would hide the error everywhere.
raise
except Exception:
- # psutil raises on M5, whose renumbered tables are not at the indexes
- # it hardcodes. With no tables either, its error is the honest answer.
+ # psutil raises on M5, whose renumbered tables are not at the indexes it hardcodes. With no
+ # tables either, its error is the honest answer.
stand_in = _from_tables(_percpu_requested(args, kwargs))
if stand_in is None:
raise
@@ -1354,8 +1311,8 @@ def check_fbgemm_gpu_version():
fbgemm_gpu_version = importlib_version("fbgemm_gpu_genai")
except:
return
- # We noticed some SegFault or bad alloc errors on lower versions of fbgemm_gpu.
- # Instead of raising an error, disable FBGEMM and fall back to Triton kernels.
+ # Lower fbgemm_gpu versions segfault or bad-alloc, so disable FBGEMM and fall back to Triton
+ # rather than raise.
if Version(fbgemm_gpu_version) < Version("1.4.0"):
os.environ["UNSLOTH_HAS_FBGEMM"] = "0"
logger.info(
@@ -1373,8 +1330,7 @@ def patch_enable_input_require_grads():
import inspect
from transformers import PreTrainedModel
- # Only patch the new variant that iterates over self.modules().
- # Ref: https://github.com/huggingface/transformers/pull/41993/files#diff-6b72b98c4c2dcfc6cc606843917733f5d858374fbc22a735ff483bbc0c1e63eaL1979-R1996
+ # Only patch the new variant that iterates over self.modules(); see huggingface/transformers#41993.
try:
original_source = inspect.getsource(PreTrainedModel.enable_input_require_grads)
except:
@@ -1399,8 +1355,7 @@ def patch_enable_input_require_grads():
try:
input_embeddings = module.get_input_embeddings()
except NotImplementedError:
- # Vision models may not implement get_input_embeddings (e.g. GLM
- # V4.6 skips only `self.visual`); skip them
+ # Vision models may not implement get_input_embeddings (GLM V4.6 skips only self.visual).
continue
if input_embeddings is None:
@@ -1459,9 +1414,8 @@ def patch_unsafe_trainer_rng_load():
"(CVE-2026-1839 / CVE-2025-32434); upgrade to torch >= 2.6."
)
- # Install one process-wide torch.load shim that stays inert unless the calling
- # thread is inside _load_rng_state, so we gate only at the real rng load with
- # no global-swap race and no effect on other torch.load callers.
+ # One process-wide torch.load shim, inert unless the calling thread is inside _load_rng_state,
+ # so the gate applies at the real rng load with no global-swap race.
if not getattr(torch.load, "_unsloth_rng_guard", False):
_orig_load = torch.load
_rng_active = threading.local()
@@ -1503,9 +1457,8 @@ def _is_custom_torch_build(raw_version_str):
local = raw_version_str.split("+", 1)[1]
if not local:
return False
- # Use fullmatch so the entire local identifier must match, not just a prefix.
- # cu/rocm require a trailing digit (e.g. cu124, rocm6.3). cpu/xpu are exact.
- # Case-insensitive since some builds may use uppercase.
+ # fullmatch, so the whole local identifier matches: cu/rocm need a trailing digit (cu124,
+ # rocm6.3), cpu/xpu are exact, and case-insensitive since some builds use uppercase.
return not re.fullmatch(r"cu\d[\d.]*|rocm\d[\d.]*|cpu|xpu", local, re.IGNORECASE)
@@ -1531,11 +1484,9 @@ _TORCHVISION_ABI_MARKERS = (
"torchvision.io.video",
"torchvision.io._video",
)
-# A loader failure is a torchvision break only when it names torchvision or the
-# torch libraries it links. The probe below imports torchvision where nothing
-# used to, so a box whose torchvision cannot load for an UNRELATED reason (a
-# missing CUDA library, say) must keep importing unsloth exactly as before
-# instead of being handed a hard "reinstall torchvision".
+# A loader failure means torchvision only when it names torchvision or the torch libraries it
+# links: the probe below imports torchvision where nothing used to, so a box broken for an
+# unrelated reason must keep importing unsloth, not get "reinstall torchvision".
_LOADER_FAILURE_MARKERS = ("undefined symbol", "cannot open shared object file")
_TORCH_LIBRARY_MARKERS = ("torchvision", "libtorch", "libc10", "_C.so", "c10::")
@@ -1556,23 +1507,14 @@ def _is_broken_torchvision_error(error) -> bool:
return False
-# PyPI carries exactly one torchvision build per release and it is one CUDA
-# family: its `_C.so` links libcudart, libc10_cuda and libtorch_cuda. Every
-# other build -- CPU, XPU, ROCm, and every CUDA family but PyPI's -- lives only
-# on download.pytorch.org, under an index named after torch's own local tag.
-# `--no-deps` keeps the installed torch, so an unqualified pin swaps a working
-# wheel for PyPI's and raises the very `operator torchvision::nms does not
-# exist` this command is handed out to clear. A tag we cannot turn into an
-# index (a vendor build: this repo's Radeon extras install
-# `torch 2.9.1+rocm7.2.0.lw.git7e1940d4` beside a repo.radeon.com torchvision;
-# or a source build) has no wheel to name at all, and neither does a
-# prerelease, whose stable-looking companion is synthesised from the release
-# numbers alone.
+# PyPI carries one torchvision build per release, for one CUDA family; every other build lives
+# on download.pytorch.org under an index named after torch's local tag. With --no-deps an
+# unqualified pin swaps a working wheel for PyPI's and raises the very
+# "operator torchvision::nms does not exist" it was handed out to clear.
_TORCH_BACKEND_INDEX = re.compile(r"cpu|xpu|cu\d+|rocm\d+(?:\.\d+)*", re.IGNORECASE)
-# conda keeps the backend in the build string (`py3.12_cuda12.4_cudnn9_0`) and
-# leaves the version plain, so a conda torch is indistinguishable from a PyPI
-# one by version alone. Its own ledger tells them apart:
-# `conda-meta/<name>-<version>-<build>.json`, one per installed package.
+# conda keeps the backend in the build string (py3.12_cuda12.4_cudnn9_0) and leaves the version
+# plain, so a conda torch looks like a PyPI one by version alone; its
+# conda-meta/<name>-<version>-<build>.json tells them apart.
_CONDA_TORCH_PACKAGES = ("pytorch", "pytorch-cpu", "pytorch-gpu", "libtorch")
@@ -1585,13 +1527,12 @@ def _torch_local_tag(torch_version_raw):
def _torch_is_conda_managed(torch_version_raw):
"""Did conda install this torch, rather than pip?"""
conda_meta = os.path.join(sys.prefix, "conda-meta")
- # A conda version never carries the `+tag`, but strip it so a pip torch
- # sitting in a conda prefix is still matched on its release numbers.
+ # A conda version never carries the +tag, but strip it so a pip torch inside a conda prefix is
+ # still matched on its release numbers.
version = (torch_version_raw or "").split("+", 1)[0]
if not version or not os.path.isdir(conda_meta):
return False
- # Pinned to this exact version, so an unrelated `pytorch-lightning-*.json`
- # cannot answer for torch.
+ # Pinned to this exact version, so an unrelated pytorch-lightning-*.json cannot answer for torch.
prefixes = tuple(f"{name}-{version}-" for name in _CONDA_TORCH_PACKAGES)
try:
entries = os.listdir(conda_meta)
@@ -1602,17 +1543,15 @@ def _torch_is_conda_managed(torch_version_raw):
def _has_no_matching_public_wheel(torch_version_raw):
try:
- # `.is_prerelease`, not a substring list: `2.11.0a1` and `2.11.0b2` are
- # prereleases that no `a0`/`b0` match would catch.
+ # .is_prerelease, not a substring list: 2.11.0a1 and 2.11.0b2 are prereleases no a0/b0 match would catch.
if TrueVersion(torch_version_raw).is_prerelease:
return True
except Exception:
return True
local = _torch_local_tag(torch_version_raw)
if not local:
- # An absent tag means PyPI for a pip install, but conda never writes one
- # either, and its torch may be CPU, ROCm or a CUDA family PyPI does not
- # ship. `--no-deps` would leave that torch beside PyPI's torchvision.
+ # An absent tag means PyPI for a pip install, but conda never writes one either, and its torch
+ # may be CPU, ROCm or a CUDA family PyPI does not ship.
return _torch_is_conda_managed(torch_version_raw)
return not _TORCH_BACKEND_INDEX.fullmatch(local)
@@ -1642,10 +1581,8 @@ def _torchvision_repair_command(required = None, torch_version_raw = None):
if required is None:
spec = "torchvision"
elif len(required) >= 3:
- # Exact, because the pair is exact: torchvision 0.22.0 requires torch
- # 2.7.0 and 0.22.1 requires torch 2.7.1. A `0.22.*` wildcard on a
- # torch 2.7.0 host resolves 0.22.1, and `--no-deps` then keeps the torch
- # that does not match it, rebuilding the mismatch the command repairs.
+ # Exact, because the pair is: torchvision 0.22.0 needs torch 2.7.0 and 0.22.1 needs 2.7.1, so
+ # a 0.22.* wildcard resolves 0.22.1 while --no-deps keeps the mismatched torch.
spec = f"torchvision=={required[0]}.{required[1]}.{required[2]}"
else:
spec = f"torchvision=={required[0]}.{required[1]}.*"
@@ -1711,8 +1648,7 @@ def torchvision_compatibility_check():
except Exception:
return
- # Known compatibility table (ground truth, takes precedence over formula).
- # See https://pytorch.org/get-started/previous-versions/
+ # Ground truth, takes precedence over the formula; see pytorch.org/get-started/previous-versions/
TORCH_TORCHVISION_COMPAT = {
(2, 9): (0, 24),
(2, 8): (0, 23),
@@ -1736,9 +1672,8 @@ def torchvision_compatibility_check():
if required is None:
return
- # Carry torch's own patch into the companion: the two move together
- # (2.7.0/0.22.0, 2.7.1/0.22.1), so the repair command can name one wheel
- # instead of a minor-wide range it cannot then satisfy under `--no-deps`.
+ # Carry torch's own patch into the companion: they move together (2.7.0/0.22.0, 2.7.1/0.22.1),
+ # so the repair names one wheel instead of a minor-wide range --no-deps cannot satisfy.
if len(torch_release) >= 3:
required = (required[0], required[1], torch_release[2])
@@ -1752,7 +1687,6 @@ def torchvision_compatibility_check():
_probe_torchvision_binary(torch_version_raw, torchvision_version_raw, required)
return
- # Version mismatch detected
message = (
f"Unsloth: torch=={torch_version_raw} requires "
f"torchvision>={required_tv_str}, "
@@ -1766,15 +1700,15 @@ def torchvision_compatibility_check():
torchvision_version_raw
)
- # Detect nightly/dev/alpha/beta/rc builds from the raw version string.
- # These often have version mismatches that are expected.
+ # Nightly/dev/alpha/beta/rc builds mismatch expectedly, so those and source builds only warn;
+ # a stable mismatch fails fast to prevent runtime operator errors.
_pre_tags = (".dev", "a0", "b0", "rc", "alpha", "beta", "nightly")
is_prerelease = any(t in torch_version_raw for t in _pre_tags) or any(
t in torchvision_version_raw for t in _pre_tags
)
- # Only downgrade to warning for custom/source or prerelease builds.
- # Stable mismatches should fail fast to prevent runtime operator errors.
+ # Only downgrade to warning for custom/source or prerelease builds. Stable mismatches should fail fast to
+ # prevent runtime operator errors.
if is_custom or is_prerelease:
reason = "custom/source build" if is_custom else "pre-release build"
logger.warning(
@@ -1816,8 +1750,8 @@ def _unsatisfied_transformers_requirements():
except Exception:
continue # Unparseable requirement line - ignore it, never guess.
- # extra = "" drops optional-extra requirements and inapplicable
- # python_version / sys_platform gates - packages the user is right not to have.
+ # extra = "" drops optional-extra requirements and inapplicable python_version / sys_platform
+ # gates: packages the user is right not to have.
if requirement.marker is not None:
try:
if not requirement.marker.evaluate({"extra": ""}):
@@ -1828,10 +1762,8 @@ def _unsatisfied_transformers_requirements():
try:
installed = importlib_version(requirement.name)
except PackageNotFoundError:
- # Absent, which `--no-deps` causes as readily as a stale version.
- # transformers checks its base requirements at its own root import and
- # raises PackageNotFoundError carrying the same misleading hint, so an
- # absent one belongs here - floor or no floor, it is not optional.
+ # Absent, which --no-deps causes as readily as a stale version: transformers checks its base
+ # requirements at root import and raises PackageNotFoundError with the same misleading hint.
unsatisfied.append((requirement.name, str(requirement.specifier), None))
continue
except Exception:
@@ -1841,8 +1773,8 @@ def _unsatisfied_transformers_requirements():
continue # Installed, and no floor it could fall below.
try:
- # Parse explicitly: SpecifierSet.contains() reports a non-PEP440 version
- # as "not contained" rather than raising, which would be a false positive.
+ # Parse explicitly: SpecifierSet.contains() reports a non-PEP440 version as "not contained"
+ # rather than raising, which would be a false positive.
installed_version = TrueVersion(installed)
except Exception:
continue # Not a PEP 440 version - we cannot judge it, so stay quiet.
@@ -1876,9 +1808,8 @@ def check_transformers_dependency_versions():
):
return
try:
- # find_spec RAISES ValueError, rather than returning None, for a transformers
- # in sys.modules with `__spec__` None or unset - a stub, or one mid-teardown.
- # Nothing here is worth failing `import unsloth` over.
+ # find_spec RAISES ValueError rather than returning None for a transformers in sys.modules with
+ # __spec__ None or unset (a stub, or one mid-teardown). Not worth failing the import over.
if importlib.util.find_spec("transformers") is None:
return
except Exception:
@@ -2064,11 +1995,11 @@ def fix_triton_compiled_kernel_missing_attrs():
except (ImportError, ModuleNotFoundError):
return
- # Only needed when the CompiledKernel class lacks num_ctas as a direct attr
- # but has metadata (triton >= 3.6.0 with torch < 2.10)
+ # Only needed when CompiledKernel lacks num_ctas as a direct attr but has metadata
+ # (triton >= 3.6.0 with torch < 2.10).
_ck_cls = triton_compiler.CompiledKernel
if hasattr(_ck_cls, "num_ctas"):
- return # Old triton with direct attrs -- no patch needed
+ return
_orig_init = _ck_cls.__init__
@@ -2121,9 +2052,8 @@ def fix_dynamo_config_thread_visibility():
mirrored_modules = ("torch._dynamo.config", "torch._inductor.config")
- # config.patch(...) and config.load_config(...) also assign via __setattr__, but
- # their writes are thread-local by design; a per-thread depth counter marks them
- # so they are not mirrored into the process-global default.
+ # config.patch() and config.load_config() also assign via __setattr__, but their writes are
+ # thread-local by design, so a per-thread depth counter keeps them out of the global default.
import threading
_scoped_depth = threading.local()
@@ -2141,7 +2071,7 @@ def fix_dynamo_config_thread_visibility():
def _patched_patch(self, *args, **kwargs):
ctx = original_patch(self, *args, **kwargs)
try:
- cls = type(ctx) # patch() builds a fresh ConfigPatch class each call
+ cls = type(ctx)
if not getattr(cls, "__unsloth_patch_wrapped__", False):
_enter0, _exit0 = cls.__enter__, cls.__exit__
@@ -2206,9 +2136,9 @@ def fix_dynamo_config_thread_visibility():
_patched_setattr.__unsloth_patched__ = True
ConfigModule.__setattr__ = _patched_setattr
- # No replay of existing overrides: unsloth installs this before it sets any
- # dynamo/inductor config, so the wrapper mirrors every later assignment. Replaying
- # would also bake a still-active config.patch override into the global default.
+ # No replay of existing overrides: unsloth installs this before setting any dynamo/inductor
+ # config, so the wrapper mirrors every later assignment, and replaying would bake a
+ # still-active config.patch override into the global default.
logger.info(
"Unsloth: Patched torch config modules so dynamo/inductor settings "
"(e.g. recompile_limit) apply across threads on torch >= 2.12."
@@ -2307,7 +2237,6 @@ def check_vllm_torch_sm100_compatibility():
This check runs early (before vLLM import) to provide a helpful error message
instead of a cryptic std::bad_alloc crash.
"""
- # vLLM installed? (without importing it)
if importlib.util.find_spec("vllm") is None:
return
@@ -2318,7 +2247,6 @@ def check_vllm_torch_sm100_compatibility():
except Exception:
return # Can't determine torch version, skip check
- # Any SM100 (Blackwell) GPU?
try:
import torch
@@ -2372,7 +2300,6 @@ def fix_vllm_pdl_blackwell():
if importlib.util.find_spec("vllm") is None:
return
- # Any SM100 (Blackwell) GPU? Fix applies globally via env var + monkey-patch.
try:
import torch
@@ -2399,13 +2326,12 @@ def fix_vllm_pdl_blackwell():
except (ImportError, OSError, ModuleNotFoundError, ValueError):
return False
- # PDL-related modules present?
has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils")
has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op")
has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op")
if not has_utils and not has_expand_op and not has_shrink_op:
- # Old vLLM version without PDL support - nothing to patch
+ # Old vLLM version without PDL support: nothing to patch.
return
# vLLM version already includes the fix?
@@ -2421,7 +2347,6 @@ def fix_vllm_pdl_blackwell():
except Exception as e:
logger.debug(f"Unsloth: vLLM version check failed ({e}), applying PDL workaround.")
- # Apply the PDL fix
os.environ["TRITON_DISABLE_PDL"] = "1"
def fake_supports_pdl(*args, **kwargs):
@@ -2435,8 +2360,7 @@ def fix_vllm_pdl_blackwell():
patched.append(name)
patched_names.add(name)
- # Patch the source module (utils.py) where supports_pdl is defined. It uses
- # @lru_cache, so clear the cache to avoid stale results.
+ # Patch the source module (utils.py) where supports_pdl is defined, and clear its @lru_cache to avoid stale results.
try:
utils_module = importlib.import_module("vllm.lora.ops.triton_ops.utils")
if hasattr(utils_module, "supports_pdl"):
@@ -2477,7 +2401,7 @@ def fix_vllm_pdl_blackwell():
f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - patched: {', '.join(patched)}"
)
else:
- # Just set the env var - vLLM might be an older version without supports_pdl
+ # Just set the env var: vLLM might be an older version without supports_pdl.
logger.info(f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})")
@@ -2586,8 +2510,8 @@ def disable_torchcodec_if_broken():
import warnings
warnings.warn(mismatch_hint, stacklevel = 2)
except Exception:
- # Warning filters promoted to errors must not abort the disable
- # fallback below (e.g. PYTHONWARNINGS=error, pytest -W error).
+ # Warning filters promoted to errors (PYTHONWARNINGS=error, pytest -W error) must not abort the
+ # disable fallback below.
pass
try:
import importlib.util
@@ -2597,7 +2521,7 @@ def disable_torchcodec_if_broken():
# RuntimeError on dlopen failure; OSError covers chained libavutil.so misses.
from torchcodec.decoders import AudioDecoder
except (ImportError, RuntimeError, OSError):
- # transformers: flip flag (<5) and/or rebind lru_cache'd func (>=5).
+ # transformers: flip the flag (<5) and/or rebind the lru_cache'd func (>=5).
try:
import transformers.utils.import_utils as tf_import_utils
@@ -2624,8 +2548,8 @@ def disable_torchcodec_if_broken():
except ImportError:
pass
- # Drop half-loaded entries and seat the absence sentinel. After this,
- # import torchcodec raises ModuleNotFoundError and find_spec returns None.
+ # Drop half-loaded entries and seat the absence sentinel: after this, `import torchcodec` raises
+ # ModuleNotFoundError and find_spec returns None.
for _stale in [
n
for n in list(sys.modules)
@@ -2675,8 +2599,7 @@ def disable_torchaudio_if_cuda_mismatched():
return
except (RuntimeError, OSError) as exc:
if "different CUDA versions" not in str(exc) and "torchaudio" not in str(exc).lower():
- # Some other failure. Not this function's business, and swallowing
- # it would hide a real one behind a message about CUDA versions.
+ # Some other failure: swallowing it would hide a real one behind a message about CUDA versions.
raise
try:
import warnings
@@ -2697,14 +2620,10 @@ def disable_torchaudio_if_cuda_mismatched():
tf_import_utils._torchaudio_available = False
except AttributeError:
pass
- # `speech` is transformers' composite backend and it is nothing but
- # torchaudio (`is_speech_available` returns `is_torchaudio_available()`).
- # On 4.x both read one module global, so setting the flag above covers
- # them. On 5.x each is separately `@lru_cache`d, so a `speech` answer
- # computed before this repair stays True and `requires_backends(...,
- # "speech")` waves callers on into a torchaudio that is now a None
- # sentinel -- a raw crash instead of the unavailable-backend path this
- # whole function exists to restore. Clear both caches.
+ # `speech` is transformers' composite backend and is nothing but torchaudio (is_speech_available
+ # returns is_torchaudio_available()). On 4.x both read one module global; on 5.x each is
+ # separately lru_cached, so an answer computed before this repair stays True and
+ # requires_backends(..., "speech") waves callers into a torchaudio that is now a None sentinel.
for _name in ("is_torchaudio_available", "is_speech_available"):
is_avail = getattr(tf_import_utils, _name, None)
if is_avail is None:
@@ -2753,21 +2672,19 @@ def disable_broken_wandb():
try:
import wandb
except Exception:
- # wandb is installed but broken - patch all checkers to skip it
+ # wandb is installed but broken: patch every checker to skip it.
logger.info(
"Unsloth: wandb is installed but broken (likely a protobuf version mismatch). "
"Disabling wandb to prevent import errors. To fix, run: pip install --upgrade wandb"
)
_wandb_false = lambda: False
- # Patch transformers' is_wandb_available (used by most trl trainers)
try:
import transformers.integrations.integration_utils as tf_integration
tf_integration.is_wandb_available = _wandb_false
except (ImportError, AttributeError):
pass
- # Patch accelerate's is_wandb_available. Patch both the source module and
- # the re-export namespace, since `from accelerate.utils import
- # is_wandb_available` reads accelerate.utils, not accelerate.utils.imports.
+ # Patch accelerate.utils.imports and the accelerate.utils re-export, since
+ # `from accelerate.utils import is_wandb_available` reads the latter.
try:
import accelerate.utils.imports as acc_imports
acc_imports.is_wandb_available = _wandb_false
@@ -2778,25 +2695,17 @@ def disable_broken_wandb():
acc_utils.is_wandb_available = _wandb_false
except (ImportError, AttributeError):
pass
- # Set env var as additional fallback
os.environ["WANDB_DISABLED"] = "true"
-# ---------------------------------------------------------------------------
-# peft 0.19.x + transformers 4.x drift
-# ---------------------------------------------------------------------------
-# peft 0.19.x's ``peft/utils/transformers_weight_conversion.py`` unconditionally
-# imports ``transformers.conversion_mapping`` and ``transformers.core_model_loading``
-# at module top. Neither submodule exists on transformers <5, so the import
-# explodes with ModuleNotFoundError -- silently swallowed by the bare except
-# in ``patch_peft_weight_converter_compatibility`` below. Fix: when (and only
-# when) the import is broken, stub the two missing submodules with the symbols
-# peft pulls at module top. The stubs are inert at runtime because peft itself
-# only calls into them behind ``if is_transformers_ge_v5:`` gates.
+# peft 0.19.x's transformers_weight_conversion.py imports transformers.conversion_mapping and
+# transformers.core_model_loading at module top; neither exists on transformers <5, so the
+# import raises ModuleNotFoundError, swallowed by the bare except below. Stub the two
+# submodules only when broken; peft calls them only behind `if is_transformers_ge_v5:`.
+
+# Stamped on stub modules so a second call is a strict no-op and third parties can introspect __unsloth_stub__.
# ---------------------------------------------------------------------------
-# Stamped on stub modules so a second call is a strict no-op and so third
-# parties can introspect ``__unsloth_stub__`` to detect our patch.
_UNSLOTH_STUB_SENTINEL = "__unsloth_stub__"
_PEFT_TENSOR_PARALLEL_FALLBACK_SYMBOLS = (
"ALL_PARALLEL_STYLES",
@@ -2962,12 +2871,11 @@ def _build_transformers_conversion_mapping_stub():
backfill a REAL module missing only some of them."""
mod = _make_peft_stub_module("transformers.conversion_mapping")
- # peft does ``.copy()`` + keyed assignment at module top; real dict suffices.
+ # peft does .copy() plus keyed assignment at module top; a real dict suffices.
mod._MODEL_TO_CONVERSION_PATTERN = {}
def get_checkpoint_conversion_mapping(model_type, *args, **kwargs):
- # ``None`` = peft's "no conversion registered"; both callsites
- # early-return on it.
+ # None is peft's "no conversion registered"; both callsites early-return on it.
return None
def get_model_conversion_mapping(model, *args, **kwargs):
@@ -2993,7 +2901,7 @@ def _install_transformers_conversion_mapping_stub():
try:
parent.conversion_mapping = mod
except Exception:
- # Frozen parent: sys.modules entry is enough for ``from ... import``.
+ # Frozen parent: the sys.modules entry is enough for `from ... import`.
pass
return mod
@@ -3095,8 +3003,8 @@ def _install_transformers_core_model_loading_stub():
return mod
-# Names peft's transformers_weight_conversion imports at module top level; a
-# real module missing ANY of them breaks that import as hard as an absent one.
+# Names peft's transformers_weight_conversion imports at module top; a real module missing ANY
+# of them breaks that import as hard as an absent one.
_PEFT_REQUIRED_SYMBOLS = {
"transformers.conversion_mapping": (
"_MODEL_TO_CONVERSION_PATTERN",
@@ -3153,8 +3061,8 @@ def _backfill_missing_peft_symbols(name):
return tuple(added)
-# An empty pattern is peft's own starting point; the rest stand in for real
-# upstream behaviour, so those are worth warning about.
+# An empty pattern is peft's own starting point; the rest stand in for real upstream
+# behaviour, so those are worth warning about.
_PEFT_INERT_BACKFILL_IS_FINE = frozenset(("_MODEL_TO_CONVERSION_PATTERN",))
@@ -3190,7 +3098,7 @@ def fix_peft_transformers_weight_conversion_import():
if importlib.util.find_spec("peft") is None:
return None
- # Already importable? Either we patched, or transformers is v5+.
+ # Already importable: either we patched, or transformers is v5+.
try:
importlib.import_module("peft.utils.transformers_weight_conversion")
return False
@@ -3203,7 +3111,7 @@ def fix_peft_transformers_weight_conversion_import():
):
return False
except ImportError as exc:
- # Older Python ImportError has no `.name`; string-match instead.
+ # Older Python ImportError has no .name; string-match instead.
msg = str(exc)
if (
"transformers.conversion_mapping" not in msg
@@ -3229,17 +3137,13 @@ def fix_peft_transformers_weight_conversion_import():
_install_transformers_core_model_loading_stub()
patched_any = True
- # Present but incomplete. transformers 5.x kept both modules and dropped
- # names peft still imports at module top -- `cannot import name
- # '_MODEL_TO_CONVERSION_PATTERN' from 'transformers.conversion_mapping'`.
- # The stubs above only fire when a module is ABSENT, so that case fell
- # through here and no-oped. Backfill the missing names onto the real
- # module: strictly additive, so a transformers that still defines them is
- # untouched, and nothing else on either side changes.
+ # Present but incomplete: transformers 5.x kept both modules and dropped names peft still
+ # imports at module top. The stubs above only fire when a module is ABSENT, so backfill the
+ # missing names onto the real module, which is strictly additive.
patched_any = _backfill_missing_conversion_symbols() or patched_any
- # An importable submodule can still lack individual symbols; backfill just
- # those names rather than replacing a real module wholesale.
+ # An importable submodule can still lack individual symbols; backfill just those rather than
+ # replacing a real module wholesale.
backfilled = {}
for _submodule in _PEFT_REQUIRED_SYMBOLS:
added = _backfill_missing_peft_symbols(_submodule)
@@ -3253,11 +3157,10 @@ def fix_peft_transformers_weight_conversion_import():
)
if not patched_any:
- # Real submodules present and complete; failure was for another reason.
+ # Real submodules present and complete; the failure was for another reason.
return False
- # Force a fresh import now that stubs are in place. Drop any cached
- # ``None`` entry first so importlib retries.
+ # Force a fresh import now that stubs are in place, dropping any cached None entry first so importlib retries.
pkg = "peft.utils.transformers_weight_conversion"
if pkg in sys.modules and sys.modules[pkg] is None:
del sys.modules[pkg]
@@ -3276,8 +3179,8 @@ def fix_peft_transformers_weight_conversion_import():
return True
-# What peft.utils.transformers_weight_conversion imports at module top. Kept
-# beside the stubs so the two lists cannot drift apart.
+# What peft.utils.transformers_weight_conversion imports at module top, kept beside the stubs
+# so the two lists cannot drift apart.
_PEFT_CONVERSION_SYMBOLS = {
"transformers.conversion_mapping": (
"_MODEL_TO_CONVERSION_PATTERN",
@@ -3296,23 +3199,18 @@ _PEFT_CONVERSION_SYMBOLS = {
),
}
-# Of those, the ones peft calls rather than merely imports. The stubs are
-# deliberately inert: on transformers <5 the whole module is ours and peft's
-# converter never runs. Landing an inert body on a REAL transformers is a
-# different matter -- peft would call it and get a wrong answer, so these are
-# replaced by a placeholder that says what is wrong instead.
+# Of those, the ones peft calls rather than merely imports. An inert body on a REAL
+# transformers would be called and answer wrongly, so those get a placeholder that says so.
_PEFT_CONVERSION_RUNTIME_SYMBOLS = frozenset(
(
"transformers.core_model_loading.dot_natural_key",
"transformers.core_model_loading.rename_source_key",
"transformers.core_model_loading.WeightRenaming",
"transformers.core_model_loading.WeightConverter",
- # `build_peft_weight_mapping` buckets its entries with
- # `isinstance(op, Concatenate)` / `isinstance(op, MergeModulelist)` and
- # builds `Transpose(dim0 = 0, dim1 = 1)` outright, so an inert stub does not
- # merely fail to help: the isinstance arms go quiet and the conversion is
- # skipped. `ConversionOps` stays import-only -- peft subclasses it at module
- # top and never asks about instances of it.
+ # build_peft_weight_mapping buckets entries with isinstance(op, Concatenate) /
+ # isinstance(op, MergeModulelist) and builds Transpose outright, so an inert stub silences the
+ # isinstance arms and the conversion is skipped. ConversionOps stays import-only: peft
+ # subclasses it and never asks about instances.
"transformers.core_model_loading.Concatenate",
"transformers.core_model_loading.MergeModulelist",
"transformers.core_model_loading.Transpose",
@@ -3339,12 +3237,9 @@ def _unsupported_conversion_symbol(qualified, donor_value = None):
"transformers that still exports it, or a peft that does not need it."
)
if isinstance(donor_value, type):
- # `isinstance` has to raise, not answer False. peft buckets its
- # conversion entries by type, and a placeholder that quietly matches
- # nothing drops the operations instead of reporting that it cannot do
- # the job. Subclassing still works: peft does `class PeftConcatenate
- # (Concatenate)` at module top, and creating a class does not construct
- # one.
+ # isinstance has to raise, not answer False: peft buckets conversion entries by type, and a
+ # placeholder that quietly matches nothing drops the operations instead of reporting that it
+ # cannot do the job. Subclassing still works, since creating a class constructs nothing.
class _RefusingMeta(type):
def __instancecheck__(cls, instance):
raise RuntimeError(message)
@@ -3369,23 +3264,14 @@ def _unsupported_conversion_symbol(qualified, donor_value = None):
return _refuse
-# The model types peft actually acts on, and what it converts them as. Only two
-# base patterns reach a rewrite -- `_MOE_TARGET_MODULE_MAPPING` and
-# `_MOE_FUSED_TARGETS` are keyed on `mixtral` and `qwen2_moe` alone, and
-# `_convert_peft_config_moe` returns early for anything else -- so this is the
-# whole set of lookups that must not be answered with a silent None.
-# Snapshotted from transformers `conversion_mapping._MODEL_TO_CONVERSION_PATTERN`,
-# because the case this file handles is that map being gone. Names are the
-# reason it is a list and not a rule: `deepseek_v3`, `dots1`, `longcat_flash`,
-# `minimax`, `mellum`, `qwen3_next`, `solar_open` and `flex_olmo` are all fused
-# MoE and none of them say so.
+# The model types peft acts on, snapshotted from conversion_mapping._MODEL_TO_CONVERSION_PATTERN
+# because the case handled here is that map being gone. A list and not a rule: deepseek_v3,
+# dots1, longcat_flash, minimax, mellum, qwen3_next, solar_open and flex_olmo are all fused MoE
+# and none of them say so.
_PEFT_MOE_CONVERSION_PATTERNS = {
- # The two base patterns map to themselves, and leaving them out was not the
- # harmless omission it looked like: `mixtral` says nothing about MoE, so the
- # substring hint answered the default for it -- the silent None this stand-in
- # exists to prevent -- and the drift test failed outright on transformers
- # 5.5.0, which pyproject permits. `qwen3_5_moe` is here for 5.3.0, where it
- # is a separate key; the hint would catch that one, but only by its name.
+ # The two base patterns map to themselves, and omitting them was not harmless: mixtral says
+ # nothing about MoE, so the substring hint answered the default and the drift test failed on
+ # transformers 5.5.0. qwen3_5_moe is here for 5.3.0, where it is a separate key.
"mixtral": "mixtral",
"qwen2_moe": "qwen2_moe",
"qwen3_5_moe": "qwen2_moe",
@@ -3415,13 +3301,10 @@ _PEFT_MOE_CONVERSION_PATTERNS = {
"solar_open": "qwen2_moe",
}
-# MoE-named model types whose conversion family is NOT one of the two fused ones,
-# so peft's `_convert_peft_config_moe` finds no `_MOE_TARGET_MODULE_MAPPING` entry
-# and returns without a rewrite. The substring hint below raised for all three
-# purely on the name, turning a load that works into a hard error. Every one is a
-# shipping model: `qwen3_5_moe_text` converts as `qwen3_5_text`, and both Granite
-# MoE variants as `granitemoe`. Checked before the hint, never instead of the
-# snapshot above, so a genuinely fused type still refuses.
+# MoE-named types whose conversion family is NOT one of the two fused ones, so peft's
+# _convert_peft_config_moe finds no mapping entry and returns without a rewrite. The substring
+# hint below raised for all three on the name alone. Checked before the hint, never instead of
+# the snapshot above.
_PEFT_MOE_NAMED_NOT_FUSED = frozenset(
(
"granitemoehybrid",
@@ -3430,17 +3313,10 @@ _PEFT_MOE_NAMED_NOT_FUSED = frozenset(
)
)
-# The other half of the same carve-out: MoE-named model types that are not in the
-# conversion map AT ALL. peft's `.get()` answers None for them and skips the target
-# rewrite, so a refusal here breaks an ordinary adapter load -- `qwen3_vl_moe` and
-# `lfm2_moe` are both supported models that the substring hint caught. Kept apart
-# from the set above so its upstream canary keeps comparing like with like: that
-# one is "in the map, mapped elsewhere", this one is "not in the map".
-#
-# Every `*moe*` / `*mixtral*` model type absent from `_MODEL_TO_CONVERSION_PATTERN`
-# in BOTH 5.3.0 and 5.5.0. A name absent from one only (`afmoe`, `qwen3_5_moe`)
-# stays fused: refusing a type that was fused costs an error message, answering
-# None for one is the silent mis-conversion this exists to prevent.
+# The other half of the carve-out: MoE-named types not in the conversion map AT ALL, where
+# peft's .get() answers None and skips the rewrite, so a refusal breaks an ordinary adapter
+# load (qwen3_vl_moe, lfm2_moe). A name absent from only one of 5.3.0 / 5.5.0 stays fused:
+# refusing a fused type costs a message, answering None is a silent mis-conversion.
_PEFT_MOE_NAMED_NOT_CONVERTED = frozenset(
(
"ernie4_5_vl_moe",
@@ -3455,9 +3331,8 @@ _PEFT_MOE_NAMED_NOT_CONVERTED = frozenset(
)
)
-# How many of those pairs a candidate map has to agree with before we believe it
-# is the conversion map under a new name. Three, so a coincidence does not pass
-# and a version that has renamed or dropped a handful of model types still does.
+# How many pairs a candidate map must match before we believe it is the conversion map under a
+# new name. Three: a coincidence does not pass, a few renamed types still do.
_CONVERSION_MAP_MATCHES = 3
@@ -3517,19 +3392,10 @@ class _UnavailableConversionPatternMap(dict):
"peft that does not need the conversion."
)
- # Only fused-MoE lookups are unsafe to answer with a silent None. peft reaches
- # `_convert_peft_config_moe` for ANY model type that has a checkpoint conversion
- # mapping, not just MoE ones, and for those a None is the correct answer: the function
- # returns without a MoE target rewrite, which is what it would do with the real map
- # too. Raising for all of them would break ordinary adapter loads to guard a case they
- # are not in.
- #
- # The snapshot is the list, not a rule over the name: eleven of the twenty-four fused
- # MoE model types say nothing about MoE in their names (`deepseek_v3`, `dots1`,
- # `longcat_flash`, `minimax`, `mellum`, `qwen3_next`, `solar_open`, `flex_olmo` among
- # them), so a substring test answered the default for exactly the checkpoints this
- # exists to protect. The substring hints stay on top of it, for a fused MoE model type
- # added after this snapshot that does follow the naming convention.
+ # Only fused-MoE lookups are unsafe to answer with a silent None: peft reaches
+ # _convert_peft_config_moe for any type with a conversion mapping, and there None is what the
+ # real map gives too, so raising for all would break ordinary adapter loads. The substring
+ # hints sit on top of the snapshot, for a fused type added later that follows the convention.
_MOE_HINTS = ("moe", "mixtral")
def _is_moe(self, key):
@@ -3587,8 +3453,8 @@ def _backfill_conversion_symbols_once(builders, added):
missing = [s for s in symbols if not hasattr(real, s)]
if not missing:
continue
- # Build the stub off to the side rather than installing it, so the real
- # module keeps its identity and everything else it exports.
+ # Build the stub off to the side rather than installing it, so the real module keeps its
+ # identity and everything else it exports.
saved = sys.modules.pop(name, None)
try:
donor = builders[name]()
@@ -3600,8 +3466,8 @@ def _backfill_conversion_symbols_once(builders, added):
for symbol in missing:
qualified = f"{name}.{symbol}"
if symbol == "_MODEL_TO_CONVERSION_PATTERN":
- # peft copies this and looks families up in it, so the stub's
- # empty dict silently drops every alias. Recover the real one.
+ # peft copies this and looks families up in it, so the stub's empty dict silently drops every
+ # alias. Recover the real one.
recovered = _recover_conversion_pattern_map(real)
if recovered is None:
logger.warning(
@@ -3610,12 +3476,10 @@ def _backfill_conversion_symbols_once(builders, added):
"targets for fused MoE checkpoints. Adapters for other "
"architectures are unaffected."
)
- # An empty dict is the one shape that fails SILENTLY. peft does
- # `_MODEL_TO_CONVERSION_PATTERN.copy()` at import and then
- # `.get(model_type, None)`, and a None makes `_convert_peft_config_moe`
- # return early, so every affected adapter loads with its legacy targets
- # unconverted and no message anywhere. Every other runtime symbol here is
- # backfilled as fail-on-use for exactly that reason; this one now matches.
+ # An empty dict is the one shape that fails SILENTLY: peft does
+ # _MODEL_TO_CONVERSION_PATTERN.copy() at import then .get(model_type, None), and a None makes
+ # _convert_peft_config_moe return early, so every affected adapter loads with legacy targets
+ # unconverted and no message anywhere.
setattr(
real,
symbol,
@@ -3624,9 +3488,8 @@ def _backfill_conversion_symbols_once(builders, added):
added.append(qualified)
continue
if qualified in _PEFT_CONVERSION_RUNTIME_SYMBOLS:
- # peft calls this one. The stub bodies exist to make the import
- # work on transformers <5, where peft's converter never runs;
- # on a real transformers it would run and answer wrongly.
+ # peft calls this one. The stub bodies exist to make the import work on transformers <5, where
+ # peft's converter never runs; on a real transformers it would run and answer wrongly.
setattr(
real,
symbol,
@@ -3651,15 +3514,10 @@ def _backfill_missing_conversion_symbols():
"transformers.core_model_loading": _install_transformers_core_model_loading_stub,
}
added = []
- # One pass is not enough when the two drifts coincide. conversion_mapping
- # imports names from core_model_loading at its own module top, so while
- # core_model_loading is still missing them, importing conversion_mapping
- # raises and the pass skips it. Backfilling core_model_loading later in the
- # same pass unblocks that import, but nothing comes back for it, and
- # _gpu_init calls this guard once, so an installation carrying both drifts
- # stayed broken. Repeat while a pass both adds a symbol and leaves a module
- # unimportable; each pass adds at least one symbol, so the bound is the
- # number of modules and there is no way to spin.
+ # One pass is not enough when the drifts coincide: conversion_mapping imports names from
+ # core_model_loading at module top, so while those are missing its import raises and the pass
+ # skips it, and _gpu_init calls this guard once. Repeat while a pass adds a symbol and leaves
+ # a module unimportable; each pass adds one, so the bound is the module count.
for _attempt in range(len(_PEFT_CONVERSION_SYMBOLS) + 1):
before = len(added)
skipped = _backfill_conversion_symbols_once(builders, added)
@@ -3828,8 +3686,8 @@ def _log_rocm_detection(message):
@functools.lru_cache(1)
def _is_rocm_torch_build() -> bool:
- # Most official ROCm wheels include a local version suffix like +rocmX.Y.
- # Some custom/source builds do not, so we fall back to runtime hints.
+ # Most official ROCm wheels carry a +rocmX.Y local version, but some custom or source builds do
+ # not, so fall back to runtime hints.
try:
torch_version_raw = str(importlib_version("torch")).lower()
if "rocm" in torch_version_raw:
@@ -3859,7 +3717,6 @@ def _is_rocm_torch_build() -> bool:
def _iter_amdgpu_asic_id_table_candidates():
- # Try torch-adjacent ids table paths first without importing torch.
try:
torch_spec = importlib.util.find_spec("torch")
except Exception:
@@ -3922,17 +3779,14 @@ def configure_amdgpu_asic_id_table_path():
return None
-# ---------------------------------------------------------------------------
-# bitsandbytes Windows ROCm fix: cextension.py runs get_rocm_gpu_arch()
-# (bnb >= 0.47) and get_rocm_warpsize() (0.49.x) at import, shelling out to
-# rocminfo / hipinfo.exe via PATH. Neither is on PATH on Windows (AMD torch
-# wheels put hipInfo.exe in venv Scripts), so every import logs ERROR +
-# WARNING, ROCM_GPU_ARCH becomes "unknown", and warp size defaults to 64:
-# wrong on RDNA (wave 32), breaking 4-bit blocksizes and
-# ALLOW_PREQUANTIZED_MODELS. Upstream fix unmerged (bitsandbytes#1969), so a
-# MetaPathFinder swaps both helpers for torch-device-props-first versions
-# right after bitsandbytes.cuda_specs executes, before cextension reads
-# them. Must run before `import unsloth_zoo` (imports bnb on ROCm).
+# bitsandbytes Windows ROCm fix: cextension.py calls get_rocm_gpu_arch() (bnb >= 0.47) and
+# get_rocm_warpsize() (0.49.x) at import, shelling out to rocminfo / hipInfo.exe via PATH. Neither
+# is on PATH on Windows (AMD torch wheels put hipInfo.exe in venv Scripts), so ROCM_GPU_ARCH
+# becomes "unknown" and warp size defaults to 64, wrong on RDNA (wave 32) and breaking 4-bit
+# blocksizes and ALLOW_PREQUANTIZED_MODELS. Upstream fix unmerged (bitsandbytes#1969), so a
+# MetaPathFinder swaps both helpers right after bitsandbytes.cuda_specs executes. Must run before
+# `import unsloth_zoo`.
+
# ---------------------------------------------------------------------------
_BNB_CUDA_SPECS_MODULE = "bitsandbytes.cuda_specs"
@@ -4032,7 +3886,7 @@ def _unsloth_get_rocm_gpu_arch():
props = _torch_rocm_device_props()
if props is not None:
try:
- # gcnArchName may carry feature flags, e.g. "gfx90a:sramecc+:xnack-"
+ # gcnArchName may carry feature flags, e.g. "gfx90a:sramecc+:xnack-".
arch = str(props.gcnArchName).split(":")[0].strip()
if arch.startswith("gfx"):
return arch
@@ -4136,9 +3990,8 @@ class _BnbCudaSpecsPatchLoader(importlib.abc.Loader):
def exec_module(self, module):
self._loader.exec_module(module)
- # Patch after the module body ran, before cextension calls it. The
- # finder stays on sys.meta_path (same lifecycle as the blockers
- # above) so importlib.reload(bitsandbytes.cuda_specs) re-patches.
+ # Patch after the module body ran, before cextension calls it. The finder stays on
+ # sys.meta_path so importlib.reload(bitsandbytes.cuda_specs) re-patches.
try:
_patch_bnb_cuda_specs_module(module)
except Exception as e:
@@ -4163,8 +4016,8 @@ class _BnbCudaSpecsPatchFinder(importlib.abc.MetaPathFinder):
):
if fullname != _BNB_CUDA_SPECS_MODULE:
return None
- # Delegate to remaining finders (editable installs, frozen apps)
- # and wrap the loader that would actually be used.
+ # Delegate to the remaining finders (editable installs, frozen apps) and wrap the loader that
+ # would actually be used.
spec = None
for finder in sys.meta_path:
if finder is self or getattr(finder, _BNB_ROCM_FIX_FINDER_SENTINEL, False):
@@ -4308,8 +4161,8 @@ def _is_broken_vllm_error(error) -> bool:
)
) or ("vllm" in message and "undefined symbol" in message):
return True
- # Forced extension load raises the bare loader error (no "vllm._C"
- # wrapper); match any .so failure as callers feed only vLLM imports.
+ # A forced extension load raises the bare loader error with no "vllm._C" wrapper, so match any
+ # .so failure; callers feed only vLLM imports.
if "cannot open shared object file" in message:
return True
current = getattr(current, "__cause__", None) or getattr(current, "__context__", None)
@@ -4319,8 +4172,8 @@ def _is_broken_vllm_error(error) -> bool:
_VLLM_RELEASES_URL = "https://github.com/vllm-project/vllm/releases"
_VLLM_INSTALL_DOCS_URL = "https://docs.vllm.ai/en/latest/getting_started/installation/gpu/"
-# A plain release version, e.g. "0.23.0". Anything else (rc / dev / post builds)
-# has no matching GitHub release asset, so we never name a wheel for it.
+# A plain release version, e.g. "0.23.0". Anything else (rc / dev / post) has no matching
+# GitHub release asset, so never name a wheel for it.
_VLLM_RELEASE_VERSION_RE = re.compile(r"^[0-9]+(?:\.[0-9]+)*$")
# Normalises platform.machine() onto the arch spelling used in wheel names.
@@ -4337,19 +4190,13 @@ def _both_arches(manylinux_tag):
return {"x86_64": manylinux_tag, "aarch64": manylinux_tag}
-# vLLM ships exactly one CUDA build per release as the unsuffixed "default" wheel
-# and the other under a "+cuXXX" local version tag. Which CUDA major is the default
-# flipped at 0.20.0 (release notes: "CUDA 13.0 default"), the "+cuXXX" tag has been
-# cu130 and then cu129, and the manylinux tag has moved manylinux1 -> 2_31 -> 2_35 ->
-# 2_34 -> 2_24 -> 2_28, differing between the two wheels of the same release. So the
-# asset name cannot be derived from a formula: vLLM's own docs document a
-# "+cu${CUDA_VERSION}" pattern that 404s (vllm-project/vllm#37847), and no release
-# has ever published a "+cu128" wheel.
-#
+# vLLM ships one CUDA build per release unsuffixed and the other under a +cuXXX tag, and which
+# major is default flipped at 0.20.0 while the manylinux tag moved and differs between the two
+# wheels of one release. The name cannot be derived: vLLM's documented "+cu${CUDA_VERSION}"
+# pattern 404s (vllm-project/vllm#37847) and no release published a +cu128 wheel.
# Each entry is (min_version, max_version, {cuda_major: (local_tag, {arch: manylinux})}),
-# transcribed from the actual release assets. A CUDA major or arch that is absent was
-# never published for that range. Extend the table when a new vLLM release lands; until
-# then newer versions fall back to the release page instead of a fabricated filename.
+# transcribed from the real release assets; an absent major or arch was never published for
+# that range. Extend on a new release; until then newer versions fall back to the release page.
_VLLM_WHEEL_ASSETS = (
(
"0.11.0",
@@ -4406,9 +4253,8 @@ _VLLM_WHEEL_ASSETS = (
),
)
-# From this release on, the default (unsuffixed) wheel is the CUDA 13 build and the
-# CUDA 12 build carries "+cu129", so we can still name the right variant for a release
-# newer than the table above even though its manylinux tag is unknown.
+# From this release on, the default wheel is the CUDA 13 build and the CUDA 12 one carries
+# +cu129, so the right variant can be named even when the manylinux tag is unknown.
_VLLM_CUDA13_DEFAULT_SINCE = "0.20.0"
@@ -4474,9 +4320,18 @@ def _get_vllm_cuda_mismatch_message(error):
# Detect what CUDA version is actually available on the system
system_cuda_display = None # Human-readable, e.g. "13.0"
system_cuda_major = None
+ # A random name is never reused, so a failed attempt would be litter rather than something the next run
+ # overwrites.
+ # torchao also hangs a handler off aten._grouped_mm at import time, and that operator only exists from torch
+ # 2.8. Same skew, different lookup: supplying the torch.nn.functional names above does not help it.
+ # Second guard for the same hazard, in case the alias is the file rather than the directory: never chain to
+ # this very hook.
+ # Ask the import system rather than probing for a filename, so the package (`sitecustomize/__init__.py`) and
+ # .pyc forms chain too.
+ # Debug mode calls find_device(...).type on gather/broadcast inputs
try:
import torch
- cuda_version = torch.version.cuda # e.g. "13.0" or "12.8"
+ cuda_version = torch.version.cuda
if cuda_version:
system_cuda_display = cuda_version
system_cuda_major = int(str(cuda_version).split(".")[0])
@@ -4664,8 +4519,8 @@ def _clear_vllm_modules():
sys.modules.pop(module_name, None)
-# vLLM's compiled extensions. A CUDA-major ABI break hits all of them, so
-# probing the eagerly-loaded _C and its siblings reliably trips it.
+# vLLM's compiled extensions: a CUDA-major ABI break hits all of them, so probing the eagerly
+# loaded _C and its siblings reliably trips it.
_VLLM_COMPILED_EXTENSIONS = (
"vllm._C",
"vllm._C_stable_libtorch",
@@ -4692,9 +4547,8 @@ def disable_broken_vllm(error = None):
try:
import vllm # noqa: F401
- # Lazy vLLM lets a bare `import vllm` succeed even when an extension
- # is ABI-broken; force-load each to surface the .so failure here.
- # A missing one raises ModuleNotFoundError (skipped below).
+ # Lazy vLLM lets a bare `import vllm` succeed with an ABI-broken extension; force-load each to
+ # surface the .so failure here. A missing one raises ModuleNotFoundError (skipped below).
for _ext in _VLLM_COMPILED_EXTENSIONS:
try:
importlib.import_module(_ext)
@@ -4839,8 +4693,7 @@ def maybe_set_windows_rocm_bnb_version():
if sys.platform != "win32":
return None
if os.environ.get("UNSLOTH_SKIP_BNB_ROCM_VERSION") == "1":
- # Real opt-out: drop our seeded default (marker present); explicit
- # user values carry no marker and are kept.
+ # Real opt-out: drop our seeded default (marker present); explicit user values carry no marker and are kept.
if os.environ.get("UNSLOTH_BNB_ROCM_VERSION_SOURCE") == "sitecustomize":
os.environ.pop("BNB_ROCM_VERSION", None)
os.environ.pop("UNSLOTH_BNB_ROCM_VERSION_SOURCE", None)
@@ -4888,7 +4741,7 @@ def patch_accelerate_recursively_apply():
if type(data).__name__ == "EmptyLogits":
cls = type(data)
if cls.__eq__ is object.__eq__:
- # Debug mode compares gathered metadata across ranks with ==
+ # Debug mode compares gathered metadata across ranks with ==.
cls.__eq__ = lambda self, other: type(other).__name__ == "EmptyLogits"
return data
return original_recursively_apply(func, data, *args, **kwargs)
@@ -4935,7 +4788,7 @@ def patch_accelerate_recursively_apply():
device = _search(data)
if device is None and found_sentinel:
- # Debug mode calls find_device(...).type on gather/broadcast inputs
+ # Debug mode calls find_device(...).type on gather/broadcast inputs.
try:
from accelerate.state import PartialState
return PartialState().device
@@ -4954,13 +4807,11 @@ def patch_accelerate_recursively_apply():
pass
-# The one ImportError worth answering False to. Matching "torchao" anywhere is
-# too wide: "No module named 'torchao.quantization'" and a missing
-# libtorchao_ops_cuda.so also say it, and both mean genuinely broken rather than
-# merely old, so they must keep propagating. Match the version complaint itself.
-# peft's current wording is the first alternative; the rest are how other
-# libraries phrase the same sentence, so an upstream reword does not silently
-# turn this back into a raise.
+# The one ImportError worth answering False to. Matching "torchao" anywhere is too wide: a
+# missing submodule ("No module named 'torchao.quantization'") or libtorchao_ops_cuda.so also
+# says it and both mean genuinely broken, so match the version complaint itself. peft's wording
+# is first; the rest are how other libraries phrase it, so a reword does not silently turn this
+# back into a raise.
_TORCHAO_STALE_VERSION_ERROR = re.compile(
r"incompatible version of torchao"
r"|torchao.{0,120}?only versions?\s+(?:above|below|>=|<=)"
@@ -4999,8 +4850,7 @@ def fix_peft_stale_torchao_import_error():
try:
return original(*args, **kwargs)
except ImportError as exc:
- # Only the version complaint; any other torchao import failure is
- # a real problem and must still surface.
+ # Only the version complaint; any other torchao import failure is a real problem and must still surface.
message = str(exc)
if _TORCHAO_STALE_VERSION_ERROR.search(message) is None:
raise
@@ -5023,9 +4873,9 @@ def fix_peft_stale_torchao_import_error():
except Exception:
return False
- # `from peft.import_utils import is_torchao_available` binds the original
- # into each importing module, so patching import_utils alone would leave
- # peft.tuners.lora.torchao, the actual caller, still raising.
+ # `from peft.import_utils import is_torchao_available` binds the original into each importing
+ # module, so patching import_utils alone leaves the real caller, peft.tuners.lora.torchao,
+ # still raising.
for mod_name, mod in tuple(sys.modules.items()):
if not mod_name.startswith("peft") or mod is None:
continue
@@ -5037,11 +4887,9 @@ def fix_peft_stale_torchao_import_error():
return patched
-# Every name torchao 0.18.0 imports from torch.nn.functional, from
-# grep -rh "from torch.nn.functional import" torchao/
-# scaled_grouped_mm is on the path of a plain `import torchao`.
-# scaled_dot_product_attention is listed for completeness only: it exists on
-# every supported torch, and the loop below skips symbols torch provides.
+# Every name torchao 0.18.0 imports from torch.nn.functional. scaled_grouped_mm is on the path
+# of a plain `import torchao`; scaled_dot_product_attention is listed for completeness, and the
+# loop below skips symbols torch provides.
_TORCHAO_TORCH_SYMBOLS = (
"ScalingType",
"SwizzleType",
@@ -5081,18 +4929,16 @@ def _make_torch_symbol_placeholder(name, detail):
return placeholder
-# The same skew one layer down: torchao 0.18 does `@implements([aten.
-# _grouped_mm.default])` at module scope in float8/float8_tensor.py, and that
-# op only arrived in torch 2.8, so older torch raises AttributeError on the
-# lookup. The torch.nn.functional names above do not help; this goes through
-# torch.ops, so the schema itself has to exist.
+# The same skew one layer down: torchao 0.18 does @implements([aten._grouped_mm.default]) at
+# module scope in float8/float8_tensor.py, and that op only arrived in torch 2.8, so older torch
+# raises AttributeError. This goes through torch.ops, so the schema itself has to exist.
_ATEN_GROUPED_MM_SCHEMA = (
"_grouped_mm(Tensor self, Tensor mat2, Tensor? offs=None, "
"Tensor? bias=None, ScalarType? out_dtype=None) -> Tensor"
)
-# Module level on purpose: a torch.library.Library deregisters everything it
-# defined once collected, so a local would undo itself.
+# Module level on purpose: a torch.library.Library deregisters everything it defined once
+# collected, so a local would undo itself.
_aten_grouped_mm_library = None
@@ -5150,8 +4996,7 @@ def _ensure_aten_grouped_mm(detail):
try:
import torch
- # FRAGMENT adds to a namespace someone else owns; DEF would try to
- # claim "aten" outright and be rejected.
+ # FRAGMENT adds to a namespace someone else owns; DEF would try to claim "aten" outright and be rejected.
library = torch.library.Library("aten", "FRAGMENT")
library.define(_ATEN_GROUPED_MM_SCHEMA)
library.impl("_grouped_mm", _refuse, "CompositeExplicitAutograd")
@@ -5219,9 +5064,8 @@ def fix_torchao_torch_symbol_skew():
torch_version,
)
- # The aten-op half of the same skew. Independent of the loop above: a torch
- # can have every functional symbol and still lack the operator, or vice
- # versa, so neither result gates the other.
+ # The aten-op half of the same skew, independent of the loop above: a torch can have every
+ # functional symbol and still lack the operator, or the reverse.
op_detail = (
f"torchao {torchao_version} registers a handler for it at "
f"import time, but torch {torch_version} does not provide it "
@@ -5239,15 +5083,10 @@ def fix_torchao_torch_symbol_skew():
return bool(patched)
-# vLLM inspects model architectures in a separate process, which imports
-# torchao itself, never sees a parent monkey-patch, and fails with the same
-# ImportError as the generic "Model architectures ['...'] failed to be
-# inspected".
-#
-# `sitecustomize` is the one hook that reaches a process we do not launch:
-# `site` imports it at interpreter startup off PYTHONPATH, which subprocesses
-# inherit. A `.pth` would also work, but only inside a real site directory,
-# which a library has no business writing into.
+# vLLM inspects architectures in a separate process that imports torchao itself, never sees a
+# parent monkey-patch, and fails the same way. sitecustomize is the one hook reaching a process
+# we do not launch: site imports it at startup off the inherited PYTHONPATH. A .pth would need
+# a real site directory, which a library has no business writing into.
_SUBPROCESS_FIX_DIRNAME = "unsloth_subprocess_import_fix"
@@ -5280,8 +5119,8 @@ def _subprocess_fix_directory():
raise RuntimeError(
"refusing a subprocess fix directory owned by another user: " + directory
)
- # chmod then re-read: some network and FUSE mounts ignore mode bits
- # and report success without changing anything.
+ # chmod then re-read: some network and FUSE mounts ignore mode bits and report success without
+ # changing anything.
if stat.S_IMODE(info.st_mode) & 0o022:
try:
os.chmod(directory, 0o700)
@@ -5531,8 +5370,8 @@ def _write_hook_atomically(target, source):
stream.write(payload)
os.replace(tmp, target) # atomic
except BaseException:
- # A random name is never reused, so a failed attempt would be
- # litter rather than something the next run overwrites.
+ # A random name is never reused, so a failed attempt is litter rather than something the next
+ # run overwrites.
try:
os.unlink(tmp)
except Exception:
@@ -5608,11 +5447,9 @@ def propagate_torchao_fix_to_subprocesses():
try:
import torch.nn.functional as F
- # Both halves of the skew must be absent for there to be nothing to do.
- # Today a torch missing the operator also misses the functional symbols
- # (2.8 vs 2.10), so the second check never fires alone; it keeps the
- # gate correct if that ever stops holding. Our own patches do not count
- # as torch being new enough: the in-process fix ran first.
+ # Both halves of the skew must be absent for there to be nothing to do. Today a torch missing
+ # the operator also misses the functional symbols (2.8 vs 2.10), so the second check never
+ # fires alone. Our own patches do not count: the in-process fix ran first.
if (
all(_torch_really_has(F, n) for n in _TORCHAO_TORCH_SYMBOLS)
and _aten_grouped_mm_library is None
@@ -5626,11 +5463,10 @@ def propagate_torchao_fix_to_subprocesses():
directory = _subprocess_fix_directory()
target = os.path.join(directory, "sitecustomize.py")
source = _subprocess_sitecustomize_source()
- # Rewrite only when it differs, so concurrent runs do not fight and a
- # reader never sees a truncated file. Matching contents only count as
- # evidence when the file is ours: see the helper. A directory in the
- # way makes os.replace raise, which the handler below turns into "no
- # subprocess fix" rather than into a hook we do not trust.
+ # Rewrite only when it differs, so concurrent runs do not fight and a reader never sees a
+ # truncated file. Matching contents count as evidence only when the file is ours. A directory
+ # in the way makes os.replace raise, which becomes "no subprocess fix" rather than a hook we
+ # do not trust.
if _existing_hook_is_trustworthy(target):
try:
existing = open(target, "r", encoding = "utf-8").read()
@@ -5648,14 +5484,12 @@ def propagate_torchao_fix_to_subprocesses():
)
return None
- # os.pathsep, not ":" -- Windows uses ";".
+ # os.pathsep, not ":": Windows uses ";".
current = os.environ.get("PYTHONPATH", "")
- # An empty component is an import location, not padding: it is what
- # `PYTHONPATH="$PYTHONPATH:/opt/lib"` leaves behind when PYTHONPATH was
- # unset, and CPython reads it as the cwd, so dropping it would take an
- # import location away from every descendant. A SET-BUT-EMPTY PYTHONPATH is
- # the opposite case: CPython ignores it entirely, so it must not become a
- # lone "" component, which would ADD the cwd instead.
+ # An empty component is an import location, not padding: PYTHONPATH="$PYTHONPATH:/opt/lib"
+ # leaves one when PYTHONPATH was unset, and CPython reads it as the cwd. A SET-BUT-EMPTY
+ # PYTHONPATH is the opposite: CPython ignores it, so it must not become a lone "" that ADDS
+ # the cwd.
parts = current.split(os.pathsep) if current else []
if directory not in parts:
os.environ["PYTHONPATH"] = os.pathsep.join([directory] + parts)
@@ -5668,13 +5502,10 @@ def propagate_torchao_fix_to_subprocesses():
return directory
-# torchao 0.18.0 moved `torchao/dtypes/nf4tensor.py` under
-# `quantization/quantize_/workflows/nf4/`, but torchtune (and xcodec2 through
-# it) still imports the old path and dies with ModuleNotFoundError.
-#
-# Same shape as the vLLM tokenizer stub above: a meta path finder APPENDED
-# after the real ones, so an older torchao that still ships the module wins,
-# and the alias resolves lazily so `import unsloth` pays nothing.
+# torchao 0.18.0 moved torchao/dtypes/nf4tensor.py under quantization/quantize_/workflows/nf4/,
+# but torchtune (and xcodec2 through it) still imports the old path. Same shape as the vLLM
+# tokenizer stub: a meta path finder APPENDED after the real ones, so an older torchao wins and
+# the alias resolves lazily.
_TORCHAO_NF4_OLD = "torchao.dtypes.nf4tensor"
_TORCHAO_NF4_NEW = "torchao.quantization.quantize_.workflows.nf4.nf4_tensor"
_TORCHAO_NF4_SENTINEL = "__unsloth_torchao_nf4_alias__"
@@ -5688,21 +5519,17 @@ class _TorchaoNF4AliasLoader(importlib.abc.Loader):
self.real_spec = None
def create_module(self, spec):
- # Return the RELOCATED module itself, not a stub with a hand-copied
- # surface: torchtune then sees whatever torchao actually ships, and
- # this cannot rot as symbols are added.
+ # Return the RELOCATED module itself, not a stub with a hand-copied surface: torchtune then
+ # sees whatever torchao ships, and this cannot rot as symbols are added.
module = importlib.import_module(_TORCHAO_NF4_NEW)
- # module_from_spec is about to overwrite this shared object's __spec__
- # with the old-name one (_bootstrap.py assigns __spec__ unconditionally,
- # unlike every other attribute), which would leave find_spec reporting
- # the old name for the new module and make reload run the no-op
- # exec_module below instead of the file.
+ # module_from_spec is about to overwrite this shared object's __spec__ with the old-name one
+ # (_bootstrap.py assigns it unconditionally), leaving find_spec reporting the old name and
+ # making reload run the no-op exec_module below instead of the file.
self.real_spec = getattr(module, "__spec__", None)
return module
def exec_module(self, module):
- # Already imported, so nothing to execute. Put back the __spec__
- # module_from_spec just clobbered.
+ # Already imported, so nothing to execute. Put back the __spec__ module_from_spec just clobbered.
if self.real_spec is not None:
try:
module.__spec__ = self.real_spec
@@ -5747,29 +5574,18 @@ def fix_torchao_nf4tensor_move():
sys.meta_path.append(_TorchaoNF4AliasFinder())
-# `datasets` fingerprints through dill, and `dill._dill._is_builtin_module`
-# pickles a module by reference only if its `__file__` starts with a sys
-# prefix, ends with an extension suffix, or contains the literal
-# `site-packages`. An install satisfying none of the three -- `pip install
-# --target <dir>`, a PYTHONPATH overlay, a Lambda layer, a vendored tree --
-# gets every package pickled BY VALUE. `Dataset.from_dict` then walks
-# `datasets/utils/_dill.py:_save_arrowTable` -> `create_arrowTable` -> its
-# globals -> the pyarrow MODULE, and dies on pyarrow's Cython `MonthDayNano`,
-# whose `__module__` is `builtins`:
-#
-# PicklingError: Can't pickle <class 'MonthDayNano'>:
-# it's not found as builtins.MonthDayNano
-#
-# Nothing in the traceback names the install layout, and every unsloth training
-# path builds a dataset. Reproduced on CPU against a byte-identical package
-# tree with the DIRECTORY NAME as the only variable: the plain `--target`
-# directory raised, a copy named `site-packages` returned a fingerprint.
+# `datasets` fingerprints through dill, and dill._dill._is_builtin_module pickles a module by
+# reference only if its __file__ starts with a sys prefix, ends with an extension suffix, or
+# contains the literal `site-packages`. An install matching none (pip install --target, a
+# PYTHONPATH overlay, a vendored tree) is pickled BY VALUE, and Dataset.from_dict then walks
+# datasets/utils/_dill.py:_save_arrowTable -> create_arrowTable -> its globals -> the pyarrow
+# MODULE, dying on pyarrow's Cython MonthDayNano, whose __module__ is `builtins`. Reproduced with
+# the DIRECTORY NAME as the only variable.
_DILL_FIX_SENTINEL = "_unsloth_dill_by_reference_fix"
_DILL_FIX_ENV = "UNSLOTH_DISABLE_DILL_FIX"
-# Never widened for these: dill's whole by-value contract is about the module
-# the user is working IN, and `python -m pkg` gives `__main__` a real `__spec__`
-# that would otherwise satisfy the rule below.
+# Never widened for these: dill's by-value contract is about the module the user works IN, and
+# `python -m pkg` gives __main__ a real __spec__ that would satisfy the rule below.
_DILL_NEVER_BY_REFERENCE = frozenset(("__main__", "__mp_main__"))
@@ -5785,12 +5601,9 @@ def _dill_path_pickles_by_value(origin):
real = os.path.realpath(origin)
except Exception:
return False
- # The LITERAL path only, as dill does: it reads `'site-packages' in
- # module.__file__` and resolves only for the prefix comparisons. Searching
- # `real` too would answer "not affected" for a symlink into a
- # site-packages-named directory dill still pickles by value, and since this
- # gate decides whether the live predicate is consulted at all, the original
- # PicklingError would simply stand.
+ # The LITERAL path only, as dill does: it reads 'site-packages' in module.__file__ and resolves
+ # only for the prefix comparisons. Searching `real` too would answer "not affected" for a
+ # symlink into a site-packages-named directory dill still pickles by value.
if "site-packages" in origin:
return False
for name in ("base_prefix", "base_exec_prefix", "exec_prefix", "prefix", "real_prefix"):
@@ -5835,9 +5648,9 @@ def _dill_install_root(origin):
except Exception:
return None
parent = os.path.dirname(real)
- # Any initializer, not just the source one: a bytecode-only deployment
- # gives `pyarrow/__init__.pyc`, and matching `__init__.py` exactly would
- # leave the root at `.../pyarrow`, where no sibling metadata is ever found.
+ # Any initializer, not just the source: a bytecode-only deployment gives pyarrow/__init__.pyc,
+ # and matching __init__.py exactly would leave the root at .../pyarrow, where no sibling
+ # metadata is found.
if os.path.splitext(os.path.basename(real))[0] == "__init__":
parent = os.path.dirname(parent)
return parent or None
@@ -5880,9 +5693,8 @@ def _dill_distribution_paths(root):
full = os.path.realpath(os.path.join(base, *rel.split("/")))
except Exception:
return
- # Inside the root, and not the metadata itself. `installed-files.txt`
- # entries are relative to the egg-info dir and start with `..`, so
- # containment is checked after resolving, never on the raw text.
+ # Inside the root, and not the metadata itself. installed-files.txt entries are relative to the
+ # egg-info dir and start with `..`, so containment is checked after resolving.
if not full.startswith(prefix):
return
head = full[len(prefix) :].split(os.sep, 1)[0]
@@ -5895,8 +5707,7 @@ def _dill_distribution_paths(root):
continue
meta = os.path.join(root, entry)
listed = False
- # RECORD is the one file a wheel install always leaves behind;
- # installed-files.txt is its egg-info equivalent.
+ # RECORD is the one file a wheel install always leaves behind; installed-files.txt is its egg-info equivalent.
for record, base in (("RECORD", root), ("installed-files.txt", meta)):
path = os.path.join(meta, record)
if not os.path.isfile(path):
@@ -5911,11 +5722,9 @@ def _dill_distribution_paths(root):
break
if listed:
continue
- # No file list anywhere. A `top_level.txt` name is honoured only where
- # it resolves to ONE file: `dill` -> `dill.py` is unambiguous, `google`
- # is a directory this metadata cannot account for. Declining the
- # package case costs the original loud PicklingError, not a silently
- # pinned fingerprint.
+ # No file list anywhere. A top_level.txt name is honoured only when it resolves to ONE file:
+ # `dill` -> dill.py is unambiguous, `google` is a directory the metadata cannot account for.
+ # Declining costs the original loud PicklingError, not a silently pinned fingerprint.
top_level = os.path.join(meta, "top_level.txt")
try:
with open(top_level, encoding = "utf-8") as f:
@@ -5966,9 +5775,8 @@ def _dill_module_is_importable_by_name(module, files = ()):
return False
if real in files:
return True
- # A deployment that compiles to adjacent bytecode and drops the sources
- # leaves RECORD naming `pkg/__init__.py` while the live spec points at
- # `pkg/__init__.pyc`. Same installed file, so the source answers for it.
+ # A deployment that compiles to adjacent bytecode and drops the sources leaves RECORD naming
+ # pkg/__init__.py while the live spec points at pkg/__init__.pyc. Same installed file.
stem, ext = os.path.splitext(real)
return ext in (".pyc", ".pyo") and stem + ".py" in files
@@ -5992,8 +5800,7 @@ def fix_dill_module_by_value_pickling():
if original is None or getattr(original, _DILL_FIX_SENTINEL, False):
return False
- # Ask dill itself before touching anything: the gate above is a copy of
- # dill's rule and a copy can go stale.
+ # Ask dill itself before touching anything: the gate above is a copy of dill's rule and a copy can go stale.
probe = None
roots = []
for name in ("datasets", "pyarrow"):
@@ -6017,15 +5824,10 @@ def fix_dill_module_by_value_pickling():
if probe is None or not roots:
return False
- # Every off-prefix path entry carrying installed metadata, not just the one
- # `datasets` or `pyarrow` happens to live in: a deployment can spread its
- # layers, and a transform reaching a dependency in a second layer hits the
- # very failure this patch prevents. Ownership is unchanged, so a wider
- # search admits more DEPENDENCIES and nothing else, and a root with no
- # metadata contributes nothing rather than falling back to "everything here
- # is a dependency" -- a loud crash beats a stale cache. Read once, at patch
- # time: re-scanning per pickled module would put directory listings inside
- # every fingerprint.
+ # Every off-prefix path entry carrying installed metadata, not just the one datasets or pyarrow
+ # lives in: layers can be spread, and a transform reaching a dependency in a second layer hits
+ # the very failure this prevents. A root with no metadata contributes nothing. Read once, at
+ # patch time: re-scanning per module would put directory listings in every fingerprint.
for entry in list(sys.path):
if not entry or not os.path.isdir(entry):
continue
@@ -6059,8 +5861,8 @@ def fix_dill_module_by_value_pickling():
setattr(_is_builtin_module, _DILL_FIX_SENTINEL, True)
_dill_module._is_builtin_module = _is_builtin_module
- # `dill.session` binds the name at import time, so patching the defining
- # module alone leaves that copy on the original.
+ # dill.session binds the name at import time, so patching the defining module alone leaves that
+ # copy on the original.
session = sys.modules.get("dill.session")
if session is not None and getattr(session, "_is_builtin_module", None) is original:
session._is_builtin_module = _is_builtin_module
diff --git a/unsloth/kernels/__init__.py b/unsloth/kernels/__init__.py
index 7ff472112f6..301efd28044 100644
--- a/unsloth/kernels/__init__.py
+++ b/unsloth/kernels/__init__.py
@@ -44,7 +44,9 @@ from .fast_lora import (
apply_lora_o,
fast_lora_forward,
)
-from .fp8 import * # Patch FbgmemFP8Linear/FP8Linear forwards before model creation, so compiled non-fast-inference models are covered too
+from .fp8 import * # Patch FbgmemFP8Linear/FP8Linear forwards before model creation, so compiled non-fast-inference
+
+# models are covered too
from .utils import (
fast_dequantize,
fast_gemv,
diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py
index 0d3bf08c356..6616e070604 100644
--- a/unsloth/kernels/cross_entropy_loss.py
+++ b/unsloth/kernels/cross_entropy_loss.py
@@ -77,7 +77,7 @@ def _cross_entropy_forward(
label_idx = tl.load(labels_ptr).to(tl.int32)
logits = tl.load(logits_ptr + col_offsets, mask = mask, other = -float("inf")).to(tl.float32)
- # Go logit scaling for Cohere: t * x
+ # Cohere logit scaling t * x; Gemma 2 softcapping t * tanh(1/t * x).
if DO_LOGIT_SCALING:
logits = LOGIT_SCALE * logits
# Do logit softcapping for Gemma 2: t * tanh(1/t * x)
@@ -162,10 +162,10 @@ def _chunked_cross_entropy_forward(
label_idx = tl.load(labels_ptr).to(tl.int32)
logits = tl.load(logits_ptr + col_offsets, mask = mask, other = -float("inf")).to(tl.float32)
- # Go logit scaling for Cohere: t * x
+ # Cohere logit scaling t * x; Gemma 2 softcapping t * tanh(1/t * x).
+ # Do logit scaling for Cohere
if DO_LOGIT_SCALING:
logits = LOGIT_SCALE * logits
- # Do logit softcapping for Gemma 2: t * tanh(1/t * x)
if DO_SOFTCAPPING:
logits = SOFTCAP * triton_tanh(logits / SOFTCAP)
@@ -173,14 +173,11 @@ def _chunked_cross_entropy_forward(
logsumexp = c + tl.log(tl.sum(tl.exp(logits - c), 0))
if chunk_idx == 0:
- # logsumexp(chunked_logsumexp) - x
- # Do the -x separately
+ # logsumexp(chunked_logsumexp) - x, with the -x done separately.
if label_idx != -100:
x = tl.load(logits_ptr + label_idx).to(tl.float32)
- # Go logit scaling for Cohere: t * x
if DO_LOGIT_SCALING:
x = LOGIT_SCALE * x
- # Do logit softcapping for Gemma 2: t * tanh(1/t * x)
if DO_SOFTCAPPING:
x = SOFTCAP * triton_tanh(x / SOFTCAP)
loss = -1.0 * x
@@ -244,7 +241,7 @@ def _cross_entropy_backward(
x = tl.load(logits_ptr + col_offsets, mask = mask, other = -float("inf")).to(tl.float32)
- # Do logit scaling for Cohere
+ # d/dx [s * x] = s for Cohere scaling; d/dx [t * tanh(1/t * x)] = 1 - tanh^2(1/t * x) for Gemma 2 softcapping.
if DO_LOGIT_SCALING:
# d/dx [s * x] = s
x = x * LOGIT_SCALE
@@ -272,7 +269,7 @@ def _cross_entropy_backward(
# d/dx [t * tanh(1/t * x)] = 1 - tanh^2(1/t * x)
y = y * (1.0 - partial * partial)
- # If y == 0: dC/dx = 0 ==> we already masked it to be = 0, so dloss = 0.
+ # If y == 0 then dC/dx = 0, and it is already masked to 0, so dloss = 0.
tl.store(logits_ptr + col_offsets, dloss * y, mask = mask)
@@ -310,7 +307,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
BLOCK_SIZE: int
num_warps: int
if n_chunks == 1:
- # For small vocabs <= 65336 like Llama, Mistral
+ # Small vocabs <= 65336 (Llama, Mistral) versus large vocabs like Gemma 256K below.
BLOCK_SIZE, num_warps = calculate_settings(vocab_size)
if is_cdna():
num_warps = num_warps // 2
@@ -363,9 +360,8 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
LOGIT_SCALE = logit_scaling,
num_warps = 32 if not is_cdna() else 16,
)
- # logsumexp(chunked_logsumexp) - x
- # Do the -x separately
- logsumexp = torch.logsumexp(logsumexp, dim = 1) # Row sum
+ # logsumexp(chunked_logsumexp) - x, with the -x done separately.
+ logsumexp = torch.logsumexp(logsumexp, dim = 1)
losses += logsumexp
losses.masked_fill_(labels == -100, 0) # Don't forget to mask padding out!
@@ -455,13 +451,12 @@ if (Version(torch.__version__) < Version("2.4.0")) and not hasattr(
fast_cross_entropy_loss = torch._disable_dynamo(fast_cross_entropy_loss)
-# Patch CE Losses in transformers
def patch_loss_functions(torch_compile = True):
_patch_loss_functions(fast_cross_entropy_loss, torch_compile = torch_compile)
- # Redirect LOSS_MAPPING aliases still pointing at stock ForCausalLMLoss
- # (e.g. ForConditionalGeneration for Qwen3.5, Csm...). unsloth_zoo also
- # does this; remove once the floor pin passes unslothai/unsloth-zoo#656.
+ # Redirect LOSS_MAPPING aliases still pointing at stock ForCausalLMLoss (e.g.
+ # ForConditionalGeneration for Qwen3.5, Csm). unsloth_zoo also does this; remove once the floor
+ # pin passes unslothai/unsloth-zoo#656.
try:
import transformers.loss.loss_utils as _lu
_unsloth_loss = _lu.LOSS_MAPPING.get("ForCausalLM")
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index f5d85e80885..05692e58925 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -164,48 +164,29 @@ class LoRA_MLP(torch.autograd.Function):
d_upA = torch.empty_like(upA)
d_upB = torch.empty_like(upB)
- # Down projection LoRA weights
- # d_downA = h.t() @ (dY @ downB.t())
- # d_downB = (downA.t() @ h.t()) @ dY
- # d_downA *= downS
- # d_downB *= downS
+ # d_downA = h.t() @ (dY @ downB.t()), d_downB = (downA.t() @ h.t()) @ dY, both scaled by downS.
d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)
d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)
- # Up projection LoRA weights
- # d_upA = X.t() @ (df @ upB.t())
- # d_upB = (upA.t() @ X.t()) @ df
- # d_upA *= upS
- # d_upB *= upS
+ # d_upA = X.t() @ (df @ upB.t()), d_upB = (upA.t() @ X.t()) @ df, both scaled by upS.
d_upA.addmm_(X.t(), df @ upB.t(), alpha = upS, beta = 0)
d_upB.addmm_(upA.t() @ X.t(), df, alpha = upS, beta = 0)
- # Gate projection LoRA weights
- # d_gateA = X.t() @ (de @ gateB.t())
- # d_gateB = (gateA.t() @ X.t()) @ de
- # d_gateA *= gateS
- # d_gateB *= gateS
+ # d_gateA = X.t() @ (de @ gateB.t()), d_gateB = (gateA.t() @ X.t()) @ de, both scaled by gateS.
d_gateA.addmm_(X.t(), de @ gateB.t(), alpha = gateS, beta = 0)
d_gateB.addmm_(gateA.t() @ X.t(), de, alpha = gateS, beta = 0)
- # dX = matmul_lora(df, upW.t(), upW_quant, upB, upA, upS)
- # dX += matmul_lora(de, gateW.t(), gateW_quant, gateB, gateA, gateS)
+ # dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.
upW = fast_dequantize(upW.t(), upW_quant)
dX = torch.matmul(df, upW.t(), out = X if ctx.inplace else None)
del upW
- # dX += df @ upB.to(dtype).t() @ (upS * upA.to(dtype).t())
dX.addmm_(df @ upB.t(), upA.t(), alpha = upS)
gateW = fast_dequantize(gateW.t(), gateW_quant)
- # dX += de @ gateW.t()
dX.addmm_(de, gateW.t())
del gateW
- # dX += de @ gateB.to(dtype).t() @ (gateS * gateA.to(dtype).t())
dX.addmm_(de @ gateB.t(), gateA.t(), alpha = gateS)
- # gateW, gateW_quant, gateA, gateB, gateS,
- # upW, upW_quant, upA, upB, upS,
- # downW, downW_quant, downA, downB, downS,
return (
dX.view(batch, seq_len, hd),
None,
@@ -387,9 +368,8 @@ class LoRA_QKV(torch.autograd.Function):
):
dtype = X.dtype
- # bitsandbytes 8-bit matmul expects 2D inputs.
- # TorchInductor/AOTAutograd fails on 3D tensors during backward,
- # so we explicitly flatten the sequence dimension.
+ # bitsandbytes 8-bit matmul expects 2D inputs, and TorchInductor/AOTAutograd fails on 3D tensors
+ # during backward, so flatten the sequence dimension explicitly.
orig_shape = X.shape
X_for_matmul = X
if X.dim() == 3:
@@ -459,8 +439,7 @@ class LoRA_QKV(torch.autograd.Function):
QA, QB, KA, KB, VA, VB = QA.t(), QB.t(), KA.t(), KB.t(), VA.t(), VB.t()
- ### Weight projection LoRA weights
- # See our blogpost for more details.
+ # Weight projection LoRA derivatives; see the Unsloth blogpost.
d_QA = torch.empty_like(QA)
d_QB = torch.empty_like(QB)
d_KA = torch.empty_like(KA)
@@ -468,57 +447,33 @@ class LoRA_QKV(torch.autograd.Function):
d_VA = torch.empty_like(VA)
d_VB = torch.empty_like(VB)
- # Q Projection
- # d_QA = X.t() @ (dQ @ QB.t())
- # d_QB = (QA.t() @ X.t()) @ dQ
- # d_QA *= QS
- # d_QB *= QS
+ # d_QA = X.t() @ (dQ @ QB.t()), d_QB = (QA.t() @ X.t()) @ dQ, both scaled by QS; K and V below are
+ # identical with their own scales.
d_QA.addmm_(X.t(), dQ @ QB.t(), alpha = QS, beta = 0)
d_QB.addmm_(QA.t() @ X.t(), dQ, alpha = QS, beta = 0)
- # K Projection
- # d_KA = X.t() @ (dK @ KB.t())
- # d_KB = (KA.t() @ X.t()) @ dK
- # d_KA *= KS
- # d_KB *= KS
d_KA.addmm_(X.t(), dK @ KB.t(), alpha = KS, beta = 0)
d_KB.addmm_(KA.t() @ X.t(), dK, alpha = KS, beta = 0)
- # V Projection
- # d_VA = X.t() @ (dV @ VB.t())
- # d_VB = (VA.t() @ X.t()) @ dV
- # d_VA *= VS
- # d_VB *= VS
d_VA.addmm_(X.t(), dV @ VB.t(), alpha = VS, beta = 0)
d_VB.addmm_(VA.t() @ X.t(), dV, alpha = VS, beta = 0)
- # Combine derivatives to find dX
- # dQ
+ # Combine the per-projection derivatives into dX.
QW = fast_dequantize(QW.t(), QW_quant)
dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace else None)
del QW
- # dX += (dQ @ QB.to(dtype).t() @ (QS * QA.to(dtype).t()))
dX.addmm_(dQ @ QB.t(), QA.t(), alpha = QS)
- # dK
KW = fast_dequantize(KW.t(), KW_quant)
- # dX += dK @ KW.t()
dX.addmm_(dK, KW.t())
del KW
- # dX += dK @ KB.to(dtype).t() @ (KS * KA.to(dtype).t())
dX.addmm_(dK @ KB.t(), KA.t(), alpha = KS)
- # dV
VW = fast_dequantize(VW.t(), VW_quant)
- # dX += dV @ VW.t()
dX.addmm_(dV, VW.t())
del VW
- # dX += dV @ VB.to(dtype).t() @ (VS * VA.to(dtype).t())
dX.addmm_(dV @ VB.t(), VA.t(), alpha = VS)
- # QW, QW_quant, QA, QB, QS,
- # KW, KW_quant, KA, KB, KS,
- # VW, VW_quant, VA, VB, VS,
return (
dX.view(batch, seq_len, hd),
None,
@@ -630,12 +585,7 @@ class LoRA_W(torch.autograd.Function):
d_A = torch.empty_like(A)
d_B = torch.empty_like(B)
- ### Weight projection LoRA weights
- # Weight projection
- # d_A = X.t() @ (dY @ B.t())
- # d_B = (A.t() @ X.t()) @ dY
- # d_A *= S
- # d_B *= S
+ # d_A = X.t() @ (dY @ B.t()), d_B = (A.t() @ X.t()) @ dY, both scaled by S.
d_A.addmm_(X.t(), dY @ B.t(), alpha = S, beta = 0)
d_B.addmm_(A.t() @ X.t(), dY, alpha = S, beta = 0)
@@ -643,10 +593,8 @@ class LoRA_W(torch.autograd.Function):
W = fast_dequantize(W.t(), W_quant)
dX = dY @ W.t()
del W
- # dX += dY @ B.to(dtype).t() @ (S * A.to(dtype).t())
dX.addmm_(dY @ B.t(), A.t(), alpha = S)
- # W, W_quant, A, B, S
return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None
@@ -692,8 +640,7 @@ def fast_lora_forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:
pass
result = self.base_layer(x, *args, **kwargs)
- # Per Tim Dettmers: for 4bit, defensively clone -- backprop can fail on a
- # manipulated view (may be fixed in newer PyTorch, untested).
+ # Per Tim Dettmers: for 4bit, defensively clone, since backprop can fail on a manipulated view.
result = result.clone()
for active_adapter in self.active_adapters:
diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.py
index 0e386ac4aa9..f4d4493aa72 100644
--- a/unsloth/kernels/flex_attention.py
+++ b/unsloth/kernels/flex_attention.py
@@ -46,25 +46,23 @@ if not HAS_FLEX_ATTENTION:
n_kv_heads = self.config.num_key_value_heads
n_groups = self.num_key_value_groups
+ # Grouped query attention
# Grouped query attention
K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)
V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)
K = K.reshape(bsz, n_heads, q_len, head_dim)
V = V.reshape(bsz, n_heads, q_len, head_dim)
- # See https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e
- # Gemma 9b should use 256 and not 224 (hs / nah). 27b uses the below
- # We default to using the config file itself
- # s = self.config.hidden_size // self.config.num_attention_heads
+ # Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,
+ # so default to the config. See google/gemma_pytorch commit 03e6575.
s = self.config.query_pre_attn_scalar
t = self.config.attn_logit_softcapping
- Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype) # Follow Keras exactly
+ Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype)
A = torch.matmul(Q, K.transpose(2, 3))
- A = t * torch.tanh(A / t) # Logit softcapping
+ A = t * torch.tanh(A / t)
A += causal_mask[:q_len, :q_len]
- # Much slower in torch compile!
- # A.masked_fill_(causal_mask[:q_len, :q_len], -float("inf"))
+ # Much slower under torch compile than the masked_fill_ it replaces.
A = torch.nn.functional.softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)
A = torch.matmul(A, V)
A = A.transpose(1, 2).contiguous()
@@ -74,9 +72,8 @@ if not HAS_FLEX_ATTENTION:
create_flex_attention_causal_mask = None
create_flex_attention_sliding_window_mask = None
else:
- # See https://github.com/pytorch-labs/attention-gym/blob/main/examples/flex_attn.ipynb
- # for more examples
- # BSD 3-Clause License Copyright (c) 2023, Driss Guessous, Horace He et al
+ # See pytorch-labs/attention-gym examples/flex_attn.ipynb. BSD 3-Clause License Copyright (c) 2023,
+ # Driss Guessous, Horace He et al.
import functools, math
def generate_tanh_softcap(t):
@@ -152,20 +149,17 @@ def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)
n_kv_heads = self.config.num_key_value_heads
n_groups = self.num_key_value_groups
- # Grouped query attention
K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)
V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, q_len, head_dim)
K = K.reshape(bsz, n_heads, q_len, head_dim)
V = V.reshape(bsz, n_heads, q_len, head_dim)
- # See https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e
- # Gemma 9b should use 256 and not 224 (hs / nah). 27b uses the below
- # We default to using the config file itself
- # s = self.config.hidden_size // self.config.num_attention_heads
+ # Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,
+ # so default to the config. See google/gemma_pytorch commit 03e6575.
s = self.config.query_pre_attn_scalar
t = self.config.attn_logit_softcapping
- Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype) # Follow Keras exactly
+ Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype)
A = torch_matmul(Q, K.transpose(2, 3))
# Logit softcapping
@@ -173,8 +167,7 @@ def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)
torch_tanh(A, out = A)
A *= t
A += causal_mask[:q_len, :q_len]
- # Much slower in torch compile!
- # A.masked_fill_(causal_mask[:q_len, :q_len], -float("inf"))
+ # Much slower under torch compile than the masked_fill_ it replaces.
A = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)
A = torch_matmul(A, V)
A = A.transpose(1, 2).contiguous()
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index a55eb2b0f83..73e007ff25b 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -83,8 +83,7 @@ def weight_dequant_kernel(x_ptr, s_ptr, y_ptr, M, N, BLOCK_SIZE: tl.constexpr):
n = tl.cdiv(N, BLOCK_SIZE)
offs_m = pid_m * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
offs_n = pid_n * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
- # tl.arange is int32, so offs_m * N overflows for tensors with more than
- # 2**31 elements (e.g. flattened MoE expert stacks); index in int64.
+ # tl.arange is int32, so offs_m * N overflows past 2**31 elements (flattened MoE expert stacks); index in int64.
offs = offs_m[:, None].to(tl.int64) * N + offs_n[None, :].to(tl.int64)
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
x = tl.load(x_ptr + offs, mask = mask).to(tl.float32)
@@ -128,26 +127,26 @@ def weight_dequant(
if x.shape[0] == s.shape[0]:
y = x.to(dtype) * s.to(dtype)
elif x.shape[1] == s.shape[0]:
- # sometimes, this is called with the transpose of the weight. Adjust for that.
+ # Sometimes called with the transpose of the weight.
y = x.t().to(dtype) * s.to(dtype)
y = y.t()
else:
raise ValueError(f"Incompatible shapes {x.shape = }, {s.shape = }")
return y
- # Block quantized weight: scale shape is (ceil(m/block_m), ceil(n/block_n))
else:
+ # Block quantized weight: scale shape is (ceil(m/block_m), ceil(n/block_n)).
return weight_dequant_block(x, s, dtype = dtype)
-# Copied from https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/inference/kernel.py
+# Copied from huggingface.co/deepseek-ai/DeepSeek-V3 inference/kernel.py
@triton.jit
def act_quant_kernel(x_ptr, y_ptr, s_ptr, BLOCK_SIZE: tl.constexpr):
pid = tl.program_id(axis = 0)
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
x = tl.load(x_ptr + offs).to(tl.float32)
s = tl.max(tl.abs(x)) / 448.0
- # All-zero row: keep scale at 1 so LoRA's zero dY doesn't become NaN
- # (a deviation from the original implementation).
+ # All-zero row: keep scale at 1 so LoRA's zero dY does not become NaN (a deviation from the original
+ # implementation).
s = 1.0 if s == 0 else s
y = x / s
y = y.to(y_ptr.dtype.element_ty)
@@ -170,23 +169,20 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, tor
return y, s
-# Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/quantization/fp8_kernel.py
+# Adapted from sgl-project/sglang python/sglang/srt/layers/quantization/fp8_kernel.py
@triton.jit
def _w8a8_block_fp8_matmul(
- # Pointers to inputs and output
A,
B,
C,
As,
Bs,
- # Shape for matmul
M,
N,
K,
# Block size for block-wise quantization
group_n,
group_k,
- # Stride for inputs and output
stride_am,
stride_ak,
stride_bk,
@@ -197,7 +193,6 @@ def _w8a8_block_fp8_matmul(
stride_As_k,
stride_Bs_k,
stride_Bs_n,
- # Meta-parameters
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
@@ -340,10 +335,9 @@ def torchao_block_matmul(
return out.to(output_dtype)
-# fbgemm <=1.3.0 silently corrupts blockwise outputs for some shapes
-# (fixed upstream in 1.4.0), so never use it for block FP8.
-# Preference: fbgemm (>=1.4.0) > torchao > triton (similar outputs/losses).
-# torchao is ~3x faster than the triton kernel but 15-30% slower than fbgemm (H100).
+# fbgemm <= 1.3.0 silently corrupts blockwise outputs for some shapes (fixed in 1.4.0), so never
+# use it for block FP8. Preference: fbgemm (>=1.4.0) > torchao > triton (similar outputs/losses);
+# torchao is ~3x faster than triton but 15-30% slower than fbgemm on H100.
fp8_block_matmul = (
torchao_block_matmul if torchao_blockwise_gemm is not None else w8a8_block_fp8_matmul_triton
)
@@ -355,20 +349,19 @@ def _blockwise_weight_dequant_any_shape(weight, weight_scale, block_size, out_dt
m, n = weight.shape
if weight_scale.dtype not in (torch.float32, torch.float16, torch.bfloat16):
weight_scale = weight_scale.to(torch.float32) # e.g. float8_e8m0fnu scales break triton
+ # Per-tensor quant: expand scalar to (ceil(m/128), ceil(n/128)) block shape
if weight_scale.numel() == 1:
- # Per-tensor scale: the normal forward stashes the un-expanded scalar,
- # which repeat_interleave cannot grow to (m, n). Scale directly.
+ # Per-tensor scale: the normal forward stashes the un-expanded scalar, which repeat_interleave
+ # cannot grow to (m, n).
return (weight.to(torch.float32) * weight_scale.float()).to(out_dtype)
if m % block_size[0] != 0 or n % block_size[1] != 0 or block_size[0] != block_size[1]:
- # Uneven tiling, or rectangular blocks. The triton kernel uses a single
- # BLOCK_SIZE for both axes and derives the column scale stride from it, so
- # it mis-indexes the scale when block_size[0] != block_size[1]. Expand the
- # per-block scales in torch, which handles both dimensions independently.
+ # Uneven tiling, or rectangular blocks: the triton kernel uses a single BLOCK_SIZE for both axes
+ # and derives the column scale stride from it, so it mis-indexes when block_size[0] != [1].
s_full = weight_scale.repeat_interleave(block_size[0], 0)[:m]
s_full = s_full.repeat_interleave(block_size[1], 1)[:, :n]
return (weight.to(torch.float32) * s_full).to(out_dtype)
- # Even tiling with square blocks: block-quant dequant with the real block size
- # (weight_dequant would silently default to 128 and dequantize wrongly).
+ # Even tiling with square blocks: pass the real block size, since weight_dequant would silently
+ # default to 128 and dequantize wrongly.
return weight_dequant_block(weight, weight_scale, block_size = block_size[0], dtype = out_dtype)
@@ -378,8 +371,8 @@ class FP8BlockQuantLinear(torch.autograd.Function):
m, n = weight.shape
if weight_scale.dtype not in (torch.float32, torch.float16, torch.bfloat16):
- # Upcast (e.g. e8m0) returns a fresh tensor and drops any Python
- # attribute, so carry block_size across the cast for the lookup below.
+ # Upcast (e.g. e8m0) returns a fresh tensor and drops any Python attribute, so carry block_size
+ # across the cast.
_scale_block_size = getattr(weight_scale, "block_size", None)
weight_scale = weight_scale.to(torch.float32) # e8m0 scales break triton dtype mapping
if _scale_block_size is not None:
@@ -388,7 +381,6 @@ class FP8BlockQuantLinear(torch.autograd.Function):
# Original scale, saved for backward before any transformation
original_weight_scale = weight_scale
- # Per-tensor quant: expand scalar to (ceil(m/128), ceil(n/128)) block shape
if weight_scale.numel() == 1:
block_size = [128, 128]
num_blocks_m = triton.cdiv(m, block_size[0])
@@ -404,7 +396,7 @@ class FP8BlockQuantLinear(torch.autograd.Function):
if triton.cdiv(m, block_size[0]) != p or triton.cdiv(n, block_size[1]) != q:
if triton.cdiv(m, block_size[0]) == q and triton.cdiv(n, block_size[1]) == p:
weight_scale = weight_scale.T
- original_weight_scale = weight_scale # Update for transposed case
+ original_weight_scale = weight_scale
else:
raise ValueError(
f"Weight shape {weight.shape} and scales shape {weight_scale.shape} is not compatible with block size {block_size}"
@@ -414,9 +406,8 @@ class FP8BlockQuantLinear(torch.autograd.Function):
weight = weight.contiguous()
if X.shape[-1] % block_size[1] != 0:
- # Hidden dim not divisible by the activation block: dequant + plain matmul.
- # Use the original (un-expanded) scale so a scalar per-tensor scale keeps
- # the fast scalar path in both forward and backward.
+ # Hidden dim not divisible by the activation block: dequant plus plain matmul, using the un-expanded
+ # scale so a scalar per-tensor scale keeps the fast path in forward and backward.
W_deq = _blockwise_weight_dequant_any_shape(
weight, original_weight_scale, block_size, X.dtype
)
@@ -435,7 +426,7 @@ class FP8BlockQuantLinear(torch.autograd.Function):
output_dtype = X.dtype,
)
ctx.weight = weight
- ctx.weight_scale = original_weight_scale # Save original for backward
+ ctx.weight_scale = original_weight_scale
ctx.block_size = block_size
return output.to(X.dtype)
@@ -466,15 +457,13 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
if weight.shape[0] == weight_scale.shape[0] and (
weight.shape[0] % 8 == 0 and weight.shape[1] % 8 == 0
):
- # The kernel needs weight dims divisible by 8 (else `cutlass cannot
- # implement`). Padding + f8f8bf16 is slower than dequant + bf16 matmul,
- # so f8f8bf16_rowwise runs only for proper, divisible-by-8 shapes.
+ # The kernel needs weight dims divisible by 8 (else "cutlass cannot implement"), and padding plus
+ # f8f8bf16 is slower than dequant plus bf16 matmul.
- # quantize_fp8_per_row squashes leading dims; save the shape first
+ # quantize_fp8_per_row squashes leading dims; save the shape first.
output_shape = (*x.shape[:-1], -1)
- # x_quantized/x_scale may land on a different device than x (FBGEMM
- # quantize.cu#L1237). Moving them here produces gibberish; move the
- # output instead. Compute runs on weight's device regardless.
+ # x_quantized/x_scale may land on a different device than x (FBGEMM quantize.cu#L1237); moving them
+ # here produces gibberish, so move the output instead.
x_quantized, x_scale = torch.ops.fbgemm.quantize_fp8_per_row(
x.view(-1, x.shape[-1]).contiguous(),
scale_ub = getattr(weight, "input_scale_ub", None),
@@ -497,8 +486,8 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
elif (
weight.shape[0] != weight_scale.shape[0] and weight.shape[1] == weight_scale.shape[0]
) or (weight.shape[0] % 8 != 0 or weight.shape[1] % 8 != 0):
- # Transposed weight/scale (backward dY@W) or non-divisible-by-8 shape
- # (e.g. Qwen 2.5 VL 7B gate proj 3420x1280): dequant is preferred.
+ # Transposed weight/scale (backward dY@W) or a non-divisible-by-8 shape (Qwen 2.5 VL 7B gate proj
+ # 3420x1280): dequant is preferred.
W_deq = weight_dequant(weight, weight_scale).T
output = torch_matmul(x, W_deq)
output = output + bias if bias is not None else output
@@ -553,14 +542,13 @@ class FP8_fbgemm_block_linear(torch.autograd.Function):
m, n = weight.shape
p, q = weight_scale.shape if weight_scale.ndim == 2 else (0, 0)
fits = triton.cdiv(m, bs_n) == p and triton.cdiv(n, bs_k) == q
- # numel() == 1 like fp8_linear and the dequant helper, but a (1, 1) that really
- # is this weight's grid still fits, so only a true per-tensor scale falls back.
+ # numel() == 1 like fp8_linear and the dequant helper, but a (1, 1) that really is this weight's
+ # grid still fits, so only a true per-tensor scale falls back.
per_tensor = weight_scale.numel() == 1 and not fits
if not per_tensor:
fits_transposed = triton.cdiv(n, bs_n) == p and triton.cdiv(m, bs_k) == q
- # Backward passes W.t() (fast_lora's downW.t()), so its block axes swap
- # too; at m == n only the stride tells the two grids apart. Transpose the
- # scale, not W, which X still has to matmul.
+ # Backward passes W.t() (fast_lora's downW.t()), so its block axes swap too; at m == n only the
+ # stride tells the two grids apart. Transpose the scale, not W, which X still has to matmul.
if fits_transposed and (not fits or (bs_n != bs_k and weight.stride(0) == 1)):
weight_scale = weight_scale.T
bs_n, bs_k = bs_k, bs_n
@@ -569,9 +557,8 @@ class FP8_fbgemm_block_linear(torch.autograd.Function):
f"Weight shape {weight.shape} and scales shape {weight_scale.shape} is not compatible with block size {bs_n, bs_k}"
)
- # f8f8bf16_blockwise takes only 128x128x128 blocks, float32 scale grids and
- # (measured on H800 / fbgemm 1.4.0) in_features % 16 == 0, out_features % 8
- # == 0. Anything else raises, so dequant + matmul as FP8BlockQuantLinear does.
+ # f8f8bf16_blockwise takes only 128x128x128 blocks, float32 scale grids and (measured on H800 /
+ # fbgemm 1.4.0) in_features % 16 == 0, out_features % 8 == 0; anything else raises.
kernel_supported = (
not per_tensor
and weight_scale.dtype == torch.float32
@@ -632,9 +619,8 @@ def fp8_fbgemm_block_linear(
def test_has_fbgemm():
- # Probe whether the faster FBGEMM works on this GPU. RTX 4090/5090 and
- # SM100 (Blackwell B200/B100) fail with CUTLASS SM90 kernels.
- # [TODO] Investigate with TorchAO why FBGEMM fails on consumer GPUs
+ # Probe whether the faster FBGEMM works on this GPU: RTX 4090/5090 and SM100 (Blackwell B200/B100)
+ # fail with CUTLASS SM90 kernels.
M, N, K = 128, 128, 128
xq = torch.ones(M, K, dtype = torch.float8_e4m3fn, device = "cuda")
wq = xq
@@ -665,6 +651,7 @@ def test_has_fbgemm():
if is_cutlass_cuda_error:
print("Unsloth: FBGEMM on the current GPU cannot load - will switch to Triton kernels")
+ # Row/channel FP8: 2D scale shaped (n, 1)
else:
print(
f"Unsloth: FBGEMM on the current GPU cannot load with error = {e} - will switch to Triton kernels"
@@ -681,11 +668,11 @@ if "UNSLOTH_HAS_FBGEMM" not in os.environ:
try:
import fbgemm_gpu
- # >=1.4.0 is fast and accurate (older versions corrupt some shapes whatever the
- # input values); ~15% faster than torchao. Probe it: consumer GPUs fail.
+ # >= 1.4.0 is fast and accurate (older versions corrupt some shapes whatever the input values) and
+ # ~15% faster than torchao, but consumer GPUs fail, so probe it.
if Version(fbgemm_gpu.__version__) >= Version("1.4.0"):
- # Suppress CUDA printf during probe: on Blackwell (SM100), FBGEMM's
- # SM90 CUTLASS kernel floods stdout with "Arch conditional MMA" before aborting.
+ # Suppress CUDA printf during the probe: on Blackwell (SM100) FBGEMM's SM90 CUTLASS kernel floods
+ # stdout with "Arch conditional MMA" before aborting.
from unsloth.import_fixes import suppress_cuda_printf
with suppress_cuda_printf():
_has_fbgemm = test_has_fbgemm()
@@ -706,10 +693,10 @@ def fp8_linear(
weight_scale,
bias = None,
):
- # Per-tensor (scalar scale) or block FP8 (2D scale, multiple columns)
+ # Per-tensor (scalar scale) or block FP8 (2D scale, multiple columns).
if weight_scale.numel() == 1 or (weight_scale.ndim == 2 and weight_scale.shape[1] > 1):
out = fp8_block_quant_linear(X, weight, weight_scale)
- # Row/channel FP8: 2D scale shaped (n, 1)
+ # Row/channel FP8: 2D scale shaped (n, 1).
else:
out = fbgemm_fp8_linear(X, weight, weight_scale, bias)
return out
@@ -728,11 +715,10 @@ if FbgemmFp8Linear is not None:
if FP8Linear is not None:
FP8Linear.forward = module_forward_patch(fp8_block_quant_linear, "weight_scale_inv")
-# FP8GroupedLinear's fused grouped matmul has no autograd formula, so training
-# backward fails. In training, use a custom autograd Function: dequant the frozen
-# fp8 weight for a differentiable bmm, saving only the fp8 weight + scale and
-# unwrapping TP shards; eval keeps the fused kernel. Gate on self.training (not
-# is_grad_enabled) so the grad-checkpoint no-grad forward and its recompute match.
+# FP8GroupedLinear's fused grouped matmul has no autograd formula, so training backward fails:
+# dequant the frozen fp8 weight for a differentiable bmm while eval keeps the fused kernel. Gate
+# on self.training, not is_grad_enabled, so the grad-checkpoint no-grad forward and its recompute
+# match.
if FP8GroupedLinear is not None:
_fp8_grouped_forward_orig = FP8GroupedLinear.forward
diff --git a/unsloth/kernels/geglu.py b/unsloth/kernels/geglu.py
index 0ed325afa0d..f7174c8efd7 100644
--- a/unsloth/kernels/geglu.py
+++ b/unsloth/kernels/geglu.py
@@ -21,7 +21,7 @@ from .utils import (
torch_gpu_device,
)
-# signed int32 max is 2**31-1 so num_elements cannot exceed 2**31
+# signed int32 max is 2**31-1, so num_elements cannot exceed 2**31.
NUM_INT32_ELEMENTS = 2**31
SAFE_INT32_BUFFER_MULTIPLIER = 4
BLOCK_SIZE = 1024
@@ -40,16 +40,14 @@ def _exact_forward_kernel(
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
- # f = 1/2 * e * (1 + erf(1/sqrt(2) * e))
- # h = f * up
+ # f = 1/2 * e * (1 + erf(1/sqrt(2) * e)), h = f * up.
e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)
- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)
+ g_row = tl.load(g + offsets, mask = mask, other = 0)
f_row = 0.5 * e_row * (tl.math.erf(tl.math.rsqrt(2.0) * e_row) + 1.0)
f_row = f_row.to(g_row.dtype) # Exact copy from HF
h_row = f_row * g_row
- # Store h
tl.store(h + offsets, h_row, mask = mask)
@@ -93,12 +91,11 @@ def _exact_backward_kernel(
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
- DW_row = tl.load(DW + offsets, mask = mask, other = 0) # .to(tl.float32)
+ DW_row = tl.load(DW + offsets, mask = mask, other = 0)
e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)
- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)
+ g_row = tl.load(g + offsets, mask = mask, other = 0)
- # Break e_row away for reuse
- # f = 1/2 * e * (1 + erf(1/sqrt(2) * e))
+ # Break e_row away for reuse f = 1/2 * e * (1 + erf(1/sqrt(2) * e))
f_partial_row = 0.5 * (tl.math.erf(tl.math.rsqrt(2.0) * e_row) + 1.0)
f_row = f_partial_row * e_row
@@ -110,17 +107,16 @@ def _exact_backward_kernel(
# dg = DW * g
dg_row = DW_row * g_row
- # df/de = 1/2 * (1 + erf(1/sqrt(2) * e)) + 1/sqrt(2*pi) * e * exp(-1/2 * e^2)
- t = 0.3989422804014327 # 1/sqrt(2*pi)
+ # df/de = 1/2 * (1 + erf(1/sqrt(2) * e)) + 1/sqrt(2*pi) * e * exp(-1/2 * e^2).
+ t = 0.3989422804014327
df_de = f_partial_row + t * e_row * tl.exp(-0.5 * e_row * e_row)
de_row = dg_row.to(tl.float32) * df_de
de_row = de_row.to(DW_row.dtype)
- # Store derivatives in buffers
- tl.store(DW + offsets, h_row, mask = mask) # h = f * g
- tl.store(e + offsets, df_row, mask = mask) # df = DW * f
- tl.store(g + offsets, de_row, mask = mask) # de
+ tl.store(DW + offsets, h_row, mask = mask)
+ tl.store(e + offsets, df_row, mask = mask)
+ tl.store(g + offsets, de_row, mask = mask)
def geglu_exact_backward_kernel(DW, e, g):
@@ -151,19 +147,16 @@ def _approx_forward_kernel(
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
- # f = 1/2 * e * (1 + tanh( sqrt(2/pi) * (x + 0.044715 * x^3 ) ))
- # f = 1/2 * e * (1 + tanh( sqrt(2/pi) * x * (1 + 0.044715 * x^2 ) ))
- # h = f * up
- s = 0.7978845608028654 # math.sqrt(2 / math.pi)
+ # f = 1/2 * e * (1 + tanh(sqrt(2/pi) * x * (1 + 0.044715 * x^2))), h = f * up.
+ s = 0.7978845608028654
e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)
- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)
+ g_row = tl.load(g + offsets, mask = mask, other = 0)
f_row = 0.5 * e_row * (triton_tanh(s * e_row * (1.0 + 0.044715 * e_row * e_row)) + 1.0)
f_row = f_row.to(g_row.dtype) # Exact copy from HF
h_row = f_row * g_row
- # Store h
tl.store(h + offsets, h_row, mask = mask)
@@ -211,19 +204,19 @@ def _approx_backward_kernel(
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
- DW_row = tl.load(DW + offsets, mask = mask, other = 0) # .to(tl.float32)
+ DW_row = tl.load(DW + offsets, mask = mask, other = 0)
e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)
- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)
+ g_row = tl.load(g + offsets, mask = mask, other = 0)
- # See https://www.desmos.com/calculator/nqprfoni6x
- s = 0.7978845608028654 # math.sqrt(2 / math.pi)
- a = s * e_row # a = sqrt(2 / pi) * x
- b = a * 0.044715 * e_row * e_row # b = a * 0.044715 * x^2
+ # See desmos.com/calculator/nqprfoni6x
+ s = 0.7978845608028654
+ a = s * e_row
+ b = a * 0.044715 * e_row * e_row
T = 1.0 + triton_tanh(a + b)
T2 = 0.5 * T
- # Q = 0.5 * -T * (T - 2.0) * (a + 3.0 * b)
+ # Q = 0.5 * -T * (T - 2.0) * (a + 3.0 * b), where a = sqrt(2/pi) * x and b = a * 0.044715 * x^2.
Q2 = -T2 * (T - 2.0) * (a + 3.0 * b)
- df_de = T2 + Q2 # 1/2 * (T + Q)
+ df_de = T2 + Q2
# f = 1/2 * e * (1 + tanh( sqrt(2/pi) * (x + 0.044715 * x^3 ) ))
f_row = T2 * e_row
@@ -238,10 +231,9 @@ def _approx_backward_kernel(
de_row = dg_row.to(tl.float32) * df_de
de_row = de_row.to(DW_row.dtype)
- # Store derivatives in buffers
- tl.store(DW + offsets, h_row, mask = mask) # h = f * g
- tl.store(e + offsets, df_row, mask = mask) # df = DW * f
- tl.store(g + offsets, de_row, mask = mask) # de
+ tl.store(DW + offsets, h_row, mask = mask)
+ tl.store(e + offsets, df_row, mask = mask)
+ tl.store(g + offsets, de_row, mask = mask)
def geglu_approx_backward_kernel(DW, e, g):
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index 4da6c8bf9a9..e3ad0fe5f79 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -45,17 +45,16 @@ def layernorm_forward(
r += row_idx
mu += row_idx
- # According to https://pytorch.org/torchtune/stable/_modules/torchtune/modules/layer_norm.html#Fp32LayerNorm, all modules
- # are in float32!
+ # Per torchtune's Fp32LayerNorm, all modules are in float32.
X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)
W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)
b_row = tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)
mean_X = tl.sum(X_row, axis = 0) / n_cols
- # (X[0] - mean) == -mean so we need to mask it out
+ # (X[0] - mean) == -mean, so mask it out.
XX = tl.where(mask, X_row - mean_X, 0)
row_var = tl.sum(XX * XX, axis = 0) / n_cols
- # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm
+ # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm.
eps_f32 = tl.full((), eps, tl.float32)
inv_var = tl.math.rsqrt(row_var + eps_f32)
tl.store(r, inv_var)
@@ -78,7 +77,7 @@ def layernorm_backward(
eps: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
- # Approximately follows https://github.com/karpathy/llm.c/blob/master/doc/layernorm/layernorm.md
+ # Approximately follows karpathy/llm.c doc/layernorm/layernorm.md
row_idx = tl.program_id(0)
col_offsets = tl.arange(0, BLOCK_SIZE)
mask = col_offsets < n_cols
@@ -88,8 +87,7 @@ def layernorm_backward(
r += row_idx
mu += row_idx
- # According to https://pytorch.org/torchtune/stable/_modules/torchtune/modules/layer_norm.html#Fp32LayerNorm, all modules
- # are in float32!
+ # Per torchtune's Fp32LayerNorm, all modules are in float32.
dY_row = tl.load(dY + col_offsets, mask = mask, other = 0).to(tl.float32)
X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)
W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)
@@ -197,7 +195,6 @@ def test_layernorm(
YY = torch.randn((bsz, seqlen, dim), dtype = dtype, device = "cuda", requires_grad = True)
Y.backward(YY)
correct_grad = X.grad.clone()
- # from unsloth.kernels import fast_layernorm
Y = fast_layernorm(layernorm, XX)
Y.backward(YY)
assert torch.dist(correct_grad, XX.grad).item() <= 0.1
diff --git a/unsloth/kernels/moe/autotune_cache.py b/unsloth/kernels/moe/autotune_cache.py
index 775c67d2ab9..150d0c8df34 100644
--- a/unsloth/kernels/moe/autotune_cache.py
+++ b/unsloth/kernels/moe/autotune_cache.py
@@ -73,7 +73,7 @@ def load_cached_config(cache_key: str) -> Optional[Dict[str, Any]]:
with open(cache_file, "r", encoding = "utf-8") as f:
cached_data = json.load(f)
- # Invalidate if device capability changed
+ # Invalidate if device capability changed.
current_device_capability = torch.cuda.get_device_capability()
if cached_data.get("device_capability") != current_device_capability:
logger.info("Device capability changed, invalidating cache")
@@ -167,11 +167,9 @@ def get_or_autotune_moe_kernels(
logger.info(f"Using in-memory cached MoE kernel configs: {cache_key}")
return _kernel_config_cache[cache_key]
- # Try to load from disk
if not force_autotune:
cached_data = load_cached_config(cache_key)
if cached_data is not None:
- # Reconstruct config objects from cache
try:
from .grouped_gemm.kernels.tuning import (
KernelConfigForward,
@@ -206,7 +204,6 @@ def get_or_autotune_moe_kernels(
_kernel_config_cache[cache_key] = configs
_autotune_completed[cache_key] = True
- # Save to disk
config_fwd, config_bwd_dx, config_bwd_dw = configs
save_cached_config(
cache_key,
@@ -244,7 +241,7 @@ def _run_moe_autotuning(
"""Run the actual auto-tuning for MoE kernels."""
device = "cuda"
- # Fixed token count avoids OOMs and seq_len dependency; we ignore the passed seq_len here
+ # Fixed token count avoids OOMs and a seq_len dependency, so the passed seq_len is ignored.
num_tokens = 4096
total_tokens = num_tokens * top_k
@@ -309,7 +306,6 @@ def _run_moe_autotuning(
use_tma_store = triton_config_fwd.kwargs.get("USE_TMA_STORE", False),
)
- # Autotune backward dX kernel
logger.info("Autotuning backward dX kernel...")
dummy_grad = torch.randn(total_tokens, 2 * intermediate_dim, device = device, dtype = dtype)
_ = grouped_gemm_dX(
@@ -324,6 +320,7 @@ def _run_moe_autotuning(
)
triton_config_bwd_dx = _autotuned_grouped_gemm_dX_kernel.best_config
+ # Safe Backward Configs: 64x64x256
config_bwd_dx = KernelConfigBackward_dX(
BLOCK_SIZE_M = triton_config_bwd_dx.kwargs["BLOCK_SIZE_M"],
BLOCK_SIZE_N = triton_config_bwd_dx.kwargs["BLOCK_SIZE_N"],
@@ -335,7 +332,6 @@ def _run_moe_autotuning(
use_tma_store = triton_config_bwd_dx.kwargs.get("USE_TMA_STORE", False),
)
- # Autotune backward dW kernel
logger.info("Autotuning backward dW kernel...")
_ = grouped_gemm_dW(
X = hidden_states,
@@ -376,7 +372,7 @@ def _get_heuristic_configs() -> Tuple[Any, Any, Any]:
KernelConfigBackward_dW,
)
- # Safe Forward Config: 64x128x128 (Fits A100 SMEM)
+ # Safe forward config 64x128x128 fits A100 SMEM; the backward pair below is 64x64x256.
config_fwd = KernelConfigForward(
BLOCK_SIZE_M = 64,
BLOCK_SIZE_N = 128,
@@ -390,7 +386,6 @@ def _get_heuristic_configs() -> Tuple[Any, Any, Any]:
use_tma_store = False,
)
- # Safe Backward Configs: 64x64x256
config_bwd_dx = KernelConfigBackward_dX(
BLOCK_SIZE_M = 64,
BLOCK_SIZE_N = 64,
diff --git a/unsloth/kernels/moe/benchmark/benchmark_fused_moe.py b/unsloth/kernels/moe/benchmark/benchmark_fused_moe.py
index 5c22800bdbf..dd68e7ffc50 100644
--- a/unsloth/kernels/moe/benchmark/benchmark_fused_moe.py
+++ b/unsloth/kernels/moe/benchmark/benchmark_fused_moe.py
@@ -59,7 +59,6 @@ def run_benchmark_forward(
X = torch.randn(bs, seqlen, hidden_size, dtype = dtype, device = device, requires_grad = True)
- # Forward
bench_forward_ref = lambda: ref_model(X) # noqa: E731
bench_forward_fused = lambda: tt_model(X) # noqa: E731
@@ -100,7 +99,7 @@ def run_benchmark_backward(
output, _ = ref_model(X)
- # Prevent autotuning forward pass
+ # Prevent autotuning the forward pass.
from grouped_gemm.kernels.forward import _autotuned_grouped_gemm_forward_kernel
_autotuned_grouped_gemm_forward_kernel.configs = _autotuned_grouped_gemm_forward_kernel.configs[
@@ -108,7 +107,6 @@ def run_benchmark_backward(
]
test_output, _ = tt_model(X_test)
- # Bench
grad_output = torch.randn_like(output)
bench_backward_ref = lambda: output.backward(grad_output, retain_graph = True) # noqa: E731
bench_backward_fused = lambda: test_output.backward(grad_output, retain_graph = True) # noqa: E731
@@ -140,7 +138,7 @@ def setup_model(
if isinstance(config, Qwen3MoeConfig):
ref_model = Qwen3MoeSparseMoeBlock(config).to(device, dtype)
- # Triton kernel grouped gemm version of MoE Block -- this is what we're testing
+ # Triton kernel grouped gemm version of the MoE block, the thing under test.
tt_model = Qwen3MoeFusedGroupedGEMMBlock.from_hf(
ref_model,
permute_x = permute_x,
@@ -302,7 +300,6 @@ if __name__ == "__main__":
mode = args.mode
if args.autotune:
- # logging.basicConfig(level=logging.INFO)
print(
f"Benchmarking {model_id} {mode}: seqlen={args.seqlen}, dtype={args.dtype}, permute_x={args.permute_x}, permute_y={args.permute_y}, autotune"
)
@@ -321,9 +318,8 @@ if __name__ == "__main__":
end_time = time.time()
print(f"Total time: {end_time - start_time:.4f} seconds")
- # NOTE: prefer the autotuner for now. The MoE block needs 2 forward
- # configs (gate_up_proj, down_proj) and 4 backward (dW + dX each); this
- # benchmark only supports 1 config at a time, so it's suboptimal here.
+ # Prefer the autotuner: the MoE block needs 2 forward configs (gate_up_proj, down_proj) and 4
+ # backward (dW + dX each), and this benchmark supports only 1 config at a time.
else:
assert False, "Use autotune for now"
kernel_configs = create_kernel_configs(args, args.permute_x, args.permute_y)
diff --git a/unsloth/kernels/moe/benchmark/utils.py b/unsloth/kernels/moe/benchmark/utils.py
index 7f119a7e6f4..13514a6873a 100644
--- a/unsloth/kernels/moe/benchmark/utils.py
+++ b/unsloth/kernels/moe/benchmark/utils.py
@@ -32,7 +32,7 @@ def create_merged_results(
test_config_cols = list(test_config_dict.keys())
for col in test_config_cols:
df[col] = test_config_dict[col]
- # Reorder columns so that test config cols are first
+ # Reorder columns so the test config columns come first.
df = df[test_config_cols + kernel_result_cols]
return df
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index 3ec5b80e2b5..a4b789594c7 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -33,17 +33,15 @@ ch.setFormatter(formatter)
logger.addHandler(ch)
-# Precompute TMA support to avoid graph breaks
-# TMA requires both:
-# 1. NVIDIA GPU with capability >= 9 (Hopper+)
-# 2. Triton version with TMA API (make_tensor_descriptor or _experimental_make_tensor_descriptor)
+# Precompute TMA support to avoid graph breaks: it needs an NVIDIA GPU with capability >= 9
+# (Hopper+) and a triton with the TMA API (make_tensor_descriptor).
def _check_tma_support():
if DEVICE_TYPE in ("xpu", "hip"):
return False
import triton.language as tl
gpu_supports_tma = torch.cuda.get_device_capability()[0] >= 9
- # Support both old experimental and new stable API names
+ # Support both the old experimental and the new stable API names.
triton_has_tma_api = hasattr(tl, "make_tensor_descriptor") or hasattr(
tl, "_experimental_make_tensor_descriptor"
)
@@ -52,7 +50,7 @@ def _check_tma_support():
_SUPPORTS_TMA = _check_tma_support()
-# Check if triton.set_allocator is available (Triton 3.0+)
+# triton.set_allocator is Triton 3.0+.
_HAS_SET_ALLOCATOR = hasattr(triton, "set_allocator")
@@ -124,7 +122,7 @@ def grouped_gemm_forward(
fuse_mul_post: bool = False,
# Autotuning -- overrides manual kernel params when True
autotune: bool = False,
- # Kernel tuning params (must be tuned, else poor performance)
+ # Kernel tuning params: must be tuned, else poor performance.
BLOCK_SIZE_M: int = 32,
BLOCK_SIZE_N: int = 32,
BLOCK_SIZE_K: int = 32,
@@ -133,9 +131,8 @@ def grouped_gemm_forward(
use_tma_load_w: bool = False,
use_tma_load_x: bool = False,
use_tma_store: bool = False,
- # software pipelining; no effect until loop is re-written
+ # Software pipelining; no effect until the loop is re-written.
flatten: bool = True,
- # debugging
debug: bool = False,
) -> torch.Tensor:
"""
@@ -175,7 +172,7 @@ def grouped_gemm_forward(
assert not (permute_y and use_tma_store), "Cannot use both TMA store and permute_y"
if use_tma_load_x:
- # TMA load for activations, TMA gather only supported on Blackwell+
+ # TMA load for activations; TMA gather is Blackwell+ only.
assert not permute_x, "Cannot use both use_tma_load_x and permute_x"
use_tma = use_tma_load_w or use_tma_load_x or use_tma_store
@@ -186,7 +183,7 @@ def grouped_gemm_forward(
use_tma_store = False
if use_tma or autotune:
- # Respect global persistent allocator if set
+ # Respect the global persistent allocator if set.
if _HAS_SET_ALLOCATOR and not getattr(triton, "_unsloth_allocator_set", False):
def alloc_fn(size: int, alignment: int, stream: int):
@@ -197,7 +194,6 @@ def grouped_gemm_forward(
if W.ndim == 3:
num_experts = W.shape[0]
N = W.shape[1]
- # K = W.shape[2]
else:
num_experts = m_sizes.shape[0]
N = W.shape[0] // num_experts
@@ -244,8 +240,6 @@ def grouped_gemm_forward(
print(f"DEBUG::GROUPED_GEMM {topk_weights.tolist()} {gather_indices.tolist()}")
y = torch.empty((total_tokens, N), device = X.device, dtype = X.dtype)
- # if total_tokens == 0 or N == 0:
- # return y
NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count
@@ -253,8 +247,6 @@ def grouped_gemm_forward(
return (NUM_SMS,)
if not autotune:
- # BLOCK_SIZE_K = min(K, BLOCK_SIZE_K)
- # BLOCK_SIZE_N = min(N, BLOCK_SIZE_N)
pass
if debug:
@@ -264,22 +256,18 @@ def grouped_gemm_forward(
print(f"DEBUG::GROUPED_GEMM {m_sizes.tolist()} {(gather_indices // topk).tolist()}")
kernel_args = {
- # Inputs
"x_ptr": X,
"w_ptr": W,
"m_sizes_ptr": m_sizes,
"gather_indices_ptr": gather_indices,
"topk_weights_ptr": topk_weights,
- # Output
"y_ptr": y,
- # Problem shapes
"NUM_TOKENS": num_tokens,
"NUM_EXPERTS": num_experts,
"TOPK": topk,
"N": N,
"K": K,
"NUM_SMS": NUM_SMS,
- # Gather / Scatter
"PERMUTE_X": permute_x,
"PERMUTE_Y": permute_y,
# TopK weight merging
@@ -364,10 +352,10 @@ def grouped_gemm_dX(
assert m_sizes.is_contiguous()
assert m_sizes.ndim == 1
+ # Preconditions
# Preconditions
assert not (permute_x and permute_y), "Cannot permute both X and Y"
- # Note that this is flipped from the forward pass
- # If we permuted y in the forward, we need to permute on load in the backward
+ # Flipped from the forward pass: a y permuted in the forward must be permuted on load in the backward.
assert not (permute_y and use_tma_load_dy), "Cannot use both TMA load and permute_y"
assert not (permute_x and use_tma_store), "Cannot use both TMA store and permute_x"
@@ -379,11 +367,10 @@ def grouped_gemm_dX(
use_tma_store = False
if use_tma or autotune:
- # Respect global persistent allocator if set
+ # Respect the global persistent allocator if set.
if _HAS_SET_ALLOCATOR and not getattr(triton, "_unsloth_allocator_set", False):
def alloc_fn(size: int, alignment: int, stream: int):
- # print(f"DEBUG::GROUPED_GEMM alloc_fn {size=} {alignment=} {stream=}")
return torch.empty(size, device = "cuda", dtype = torch.int8)
triton.set_allocator(alloc_fn)
@@ -400,13 +387,12 @@ def grouped_gemm_dX(
M_total, N_grad = dY.shape
N_total, K = W.shape
- # N = N_total // num_experts
assert N_grad == N, f"Grad_output N ({N_grad}) must match weight N ({N})"
assert M_total % topk == 0, f"M_total ({M_total}) must be divisible by topk ({topk})"
num_tokens = M_total // topk
- # the kernel only reads gather_indices under permute_x / permute_y, so it stays optional otherwise
+ # The kernel only reads gather_indices under permute_x / permute_y, so it stays optional otherwise.
if permute_x or permute_y:
assert (
gather_indices is not None
@@ -414,19 +400,17 @@ def grouped_gemm_dX(
total_tokens = gather_indices.shape[0] if gather_indices is not None else M_total
assert total_tokens == M_total, f"Total tokens ({total_tokens}) must match M_total ({M_total})"
- # Note that the output shape is [NUM_TOKENS * TOPK, K] even when `permute_x` is True since we need to accumulate gradients across all experts chosen by the token.
- # This will be done in a post-processing step reduction step.
+ # The output shape stays [NUM_TOKENS * TOPK, K] even under permute_x, since gradients must
+ # accumulate across all experts a token chose; reduced in a post-processing step.
output_shape = (total_tokens, K)
dX = torch.zeros(output_shape, device = dY.device, dtype = dY.dtype)
- NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count # if not debug else 1
+ NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count
def grid(META):
return (NUM_SMS,)
if not autotune:
- # BLOCK_SIZE_N = min(N_grad, BLOCK_SIZE_N)
- # BLOCK_SIZE_K = min(K, BLOCK_SIZE_K)
pass
if debug:
@@ -436,23 +420,20 @@ def grouped_gemm_dX(
print(f"DEBUG::GROUPED_GEMM {m_sizes.tolist()}")
kernel_args = {
- # Inputs
"dY_ptr": dY,
"w_ptr": W,
"gather_indices_ptr": gather_indices,
"m_sizes_ptr": m_sizes,
- # Output
"dX_ptr": dX,
- # Problem sizes
"NUM_EXPERTS": num_experts,
"NUM_TOKENS": num_tokens,
"TOPK": topk,
"N": N,
"K": K,
"NUM_SMS": NUM_SMS,
- # Gather / Scatter
"PERMUTE_X": permute_x,
"PERMUTE_Y": permute_y,
+ # Loop pipelining
"FLATTEN": flatten,
}
if not autotune:
@@ -529,7 +510,6 @@ def grouped_gemm_dW(
dY = dY.contiguous()
m_sizes = m_sizes.contiguous()
- # Preconditions
assert not (permute_x and permute_y), "Cannot permute both X and Y"
assert not (permute_y and use_tma_load_dy), "Cannot use both TMA load and permute_y"
assert not (permute_x and use_tma_load_x), "Cannot use both TMA load and permute_x"
@@ -542,7 +522,7 @@ def grouped_gemm_dW(
use_tma_store = False
if use_tma or autotune:
- # Respect global persistent allocator if set
+ # Respect the global persistent allocator if set.
if _HAS_SET_ALLOCATOR and not getattr(triton, "_unsloth_allocator_set", False):
def alloc_fn(size: int, alignment: int, stream: int):
@@ -566,7 +546,6 @@ def grouped_gemm_dW(
num_tokens = total_tokens // topk
num_experts = m_sizes.shape[0]
- # Get dimensions
_, K = X.shape
M_grad, N = dY.shape
@@ -575,8 +554,6 @@ def grouped_gemm_dW(
dW = torch.zeros((num_experts, N, K), device = X.device, dtype = X.dtype)
if not autotune:
- # BLOCK_SIZE_N = min(N, BLOCK_SIZE_N)
- # BLOCK_SIZE_K = min(K, BLOCK_SIZE_K)
pass
def grid(META):
@@ -603,24 +580,19 @@ def grouped_gemm_dW(
m_start += m_sizes[i]
kernel_args = {
- # Inputs
"x_ptr": X,
"dY_ptr": dY,
"m_sizes_ptr": m_sizes,
"gather_indices_ptr": gather_indices,
- # Output
"dW_ptr": dW,
- # Problem sizes
"NUM_TOKENS": num_tokens,
"TOPK": topk,
"NUM_EXPERTS": num_experts,
"N": N,
"K": K,
"NUM_SMS": NUM_SMS,
- # Gather / Scatter
"PERMUTE_X": permute_x,
"PERMUTE_Y": permute_y,
- # Loop pipelining
"FLATTEN": flatten,
}
@@ -683,7 +655,7 @@ class GroupedGemm(torch.autograd.Function):
ctx.dX_only = dX_only
ctx.dW_only = dW_only
- # NOTE: we don't save topk_weights for backward since we do not support training with fused_mul
+ # topk_weights is not saved for backward: training with fused_mul is unsupported.
ctx.save_for_backward(X, W, m_sizes, gather_indices)
fwd_config = {}
@@ -707,7 +679,7 @@ class GroupedGemm(torch.autograd.Function):
permute_x = permute_x,
permute_y = permute_y,
fuse_mul_post = fuse_mul_post,
- # Autotune -- this will override the manual kernel config if true
+ # Autotune overrides the manual kernel config when true.
autotune = autotune,
# Manual kernel config
**fwd_config,
@@ -760,7 +732,7 @@ class GroupedGemm(torch.autograd.Function):
topk = topk,
permute_x = permute_x,
permute_y = permute_y,
- # Autotune -- this will override the manual kernel config if true
+ # Autotune overrides the manual kernel config when true.
autotune = autotune,
# Manual kernel config
**bwd_dW_config,
@@ -788,7 +760,7 @@ class GroupedGemm(torch.autograd.Function):
topk = topk,
permute_x = permute_x,
permute_y = permute_y,
- # Autotune -- this will override the manual kernel config if true
+ # Autotune overrides the manual kernel config when true.
autotune = autotune,
# Manual kernel config
**bwd_dX_config,
@@ -802,19 +774,19 @@ class GroupedGemm(torch.autograd.Function):
return (
dX,
dW,
- None, # m_sizes
- None, # gather_indices
- None, # topk
- None, # permute_x
- None, # permute_y
- None, # topk_weights
- None, # fuse_mul_post
- None, # kernel_config_fwd
- None, # kernel_config_bwd_dX
- None, # kernel_config_bwd_dW
- None, # autotune
- None, # dX_only
- None, # dW_only
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
)
@@ -895,7 +867,6 @@ def grouped_gemm(
kernel_config_bwd_dW: KernelConfigBackward_dW = None,
autotune: bool = False,
is_first_gemm: bool = True,
- # Only for debugging
dX_only: bool = False,
dW_only: bool = False,
):
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py b/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py
index bea59a4548d..d6e04ca7688 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/autotuning.py
@@ -53,24 +53,23 @@ def _triton_supports_tma():
"""Check if current Triton version supports TMA API."""
import triton.language as tl
- # Check for both old experimental and new stable API names
+ # Both the old experimental and the new stable API names.
return hasattr(tl, "make_tensor_descriptor") or hasattr(
tl, "_experimental_make_tensor_descriptor"
)
-# TMA disabled for now: incompatible with permute_x/permute_y in the MoE
-# grouped GEMM passes. Re-enable once resolved.
-_TRITON_HAS_TMA = False # _triton_supports_tma()
+# TMA disabled for now: incompatible with permute_x/permute_y in the MoE grouped GEMM passes.
+_TRITON_HAS_TMA = False
def get_forward_configs(
BLOCK_M = DEFAULT_M_BLOCK_SIZES,
BLOCK_N = DEFAULT_N_BLOCK_SIZES,
BLOCK_K = DEFAULT_K_BLOCK_SIZES,
- TMA_LOAD_X = None, # Auto-detect if not specified
- TMA_LOAD_W = None, # Auto-detect if not specified
- TMA_STORE = False, # NOTE: TMA_STORE is disabled for now
+ TMA_LOAD_X = None,
+ TMA_LOAD_W = None,
+ TMA_STORE = False,
num_warps = DEFAULT_NUM_WARPS,
num_stages = DEFAULT_NUM_STAGES,
num_ctas = DEFAULT_NUM_CTAS,
@@ -149,14 +148,13 @@ def get_dX_kernel_configs(
BLOCK_M = DEFAULT_M_BLOCK_SIZES,
BLOCK_N = DEFAULT_N_BLOCK_SIZES,
BLOCK_K = DEFAULT_K_BLOCK_SIZES,
- TMA_LOAD_dY = None, # Auto-detect if not specified
- TMA_LOAD_W = None, # Auto-detect if not specified
- TMA_STORE = False, # NOTE: TMA_STORE is disabled for now
+ TMA_LOAD_dY = None,
+ TMA_LOAD_W = None,
+ TMA_STORE = False,
num_warps = DEFAULT_NUM_WARPS,
num_stages = DEFAULT_NUM_STAGES,
num_ctas = DEFAULT_NUM_CTAS,
):
- # Auto-detect TMA support
if TMA_LOAD_dY is None:
TMA_LOAD_dY = _TRITON_HAS_TMA
if TMA_LOAD_W is None:
@@ -232,11 +230,10 @@ def get_dW_kernel_configs(
num_warps = DEFAULT_NUM_WARPS,
num_stages = DEFAULT_NUM_STAGES,
num_ctas = DEFAULT_NUM_CTAS,
- TMA_LOAD_dY = None, # Auto-detect if not specified
- TMA_LOAD_X = None, # Auto-detect if not specified
+ TMA_LOAD_dY = None,
+ TMA_LOAD_X = None,
TMA_STORE = False,
):
- # Auto-detect TMA support
if TMA_LOAD_dY is None:
TMA_LOAD_dY = _TRITON_HAS_TMA
if TMA_LOAD_X is None:
@@ -344,7 +341,6 @@ def common_prune_criteria(config: triton.Config, kwargs: dict, dtype):
permute_y = kwargs["PERMUTE_Y"]
tokens_per_expert = num_tokens // num_experts
- # use_tma = [k for k in config.kwargs.keys() if k.startswith("USE_TMA_")]
MIN_BLOCK_SIZE_M = DEFAULT_M_BLOCK_SIZES[0]
if exceeds_smem_capacity(
num_stages, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, dtype, smem_size
@@ -354,8 +350,6 @@ def common_prune_criteria(config: triton.Config, kwargs: dict, dtype):
return True
if permute_x and permute_y:
return True
- # if not supports_tma() and any(use_tma):
- # return True
return False
@@ -382,7 +376,7 @@ def prune_kernel_configs_fwd(configs: list[triton.Config], args, **kwargs):
if common_prune_criteria(config, kwargs, dtype):
continue
if config.kwargs["USE_TMA_LOAD_X"] and kwargs["PERMUTE_X"]:
- # Dynamically disable TMA_LOAD_X for permuted X
+ # Dynamically disable TMA_LOAD_X for permuted X, and TMA_LOAD_dY for permuted Y.
config.kwargs["USE_TMA_LOAD_X"] = False
if config.kwargs["USE_TMA_STORE"] and kwargs["PERMUTE_Y"]:
continue
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
index b78ea58bd04..d6a9faebea2 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/backward.py
@@ -46,9 +46,9 @@ TODO:
@triton.jit
def _grouped_gemm_dX_kernel(
- dY_ptr, # [M_total, N]
- w_ptr, # [E, N, K]
- dX_ptr, # [M_total, K]
+ dY_ptr,
+ w_ptr,
+ dX_ptr,
gather_indices_ptr,
m_sizes_ptr,
# problem sizes
@@ -73,14 +73,12 @@ def _grouped_gemm_dX_kernel(
output_dtype = dX_ptr.dtype.element_ty
tidx = tl.program_id(0)
- # This removes the need for predication along N in the GEMM main loop
+ # Removes the need for predication along N in the GEMM main loop.
tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")
tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")
- # Create TMA descriptors for loading sorted tokens
- # When using TMA load, we don't permute_x, so shape should be [TOTAL_TOKENS, K]
- # Also, we are defining a single global descriptor with single block shape
- # Need to check that this does not result in errors when crossing expert boundaries
+ # A single global TMA descriptor with one block shape; TMA load never permutes x, so the shape is
+ # [TOTAL_TOKENS, K]. Unverified across expert boundaries.
if USE_TMA_LOAD_dY:
dY_desc = tl.make_tensor_descriptor(
dY_ptr,
@@ -110,16 +108,14 @@ def _grouped_gemm_dX_kernel(
m_end = m_start + m_size
if m_size > 0:
- # Advance n offset to the weights for that respective expert
n_start = expert_idx * N
- # N_start_offset = g.to(tl.int64) * N
- # tiles for this group's GEMM
+ # N_start_offset = g.to(tl.int64) * N tiles for this group's GEMM
num_m_tiles = tl.cdiv(m_size, BLOCK_SIZE_M)
num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
num_tiles_per_expert = num_m_tiles * num_k_tiles
if USE_TMA_STORE:
- # Need to define descript within loop to predicate store along M
+ # Define the descriptor inside the loop to predicate the store along M.
tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")
dX_desc = tl.make_tensor_descriptor(
dX_ptr,
@@ -128,8 +124,8 @@ def _grouped_gemm_dX_kernel(
block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],
)
- # Bounds are relative to tiles processed so far, so we only handle
- # this expert's tiles and never exceed the total across experts.
+ # Bounds are relative to tiles processed so far, so only this expert's tiles are handled and the
+ # total across experts is never exceeded.
while tidx >= processed_tiles and tidx < (processed_tiles + num_tiles_per_expert):
group_index = tidx - processed_tiles
@@ -140,7 +136,6 @@ def _grouped_gemm_dX_kernel(
if PERMUTE_X or PERMUTE_Y:
# These will be used for loading and storing in permuted order
gather_offsets = tile_m_idx * BLOCK_SIZE_M + m_block_range
- # indices_to_gather = m_start + gather_offsets
indices_to_gather = m_start + tl.max_contiguous(
tl.multiple_of(gather_offsets % m_size, BLOCK_SIZE_M),
BLOCK_SIZE_M,
@@ -155,35 +150,27 @@ def _grouped_gemm_dX_kernel(
row_mask = gather_offsets < m_size
row_mask = row_mask[:, None]
- # We only take into account the following two cases: (PERMUTE_X and NOT PERMUTE_Y) and (NOT PERMUTE_X and PERMUTE_Y)
- # Hence, we can make the following simplifying assumptions when loading and storing
- # Note the different strides between the two cases: the offsets for loading and storing are flipped and the strides must also be adjusted
+ # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so load/store offsets
+ # are simply flipped between the two cases, with the strides adjusted to match.
if PERMUTE_X:
- # Case where we permuted on load in the forward pass (typically first grouped GEMM in MoE MLP)
- load_a_idx = (
- indices_to_gather[:, None] * N
- ) # Load in contiguous (expert grouped) order
- store_idx = (
- expert_token_offsets * K
- ) # Permute on store from expert -> token order
+ # Permuted on load in the forward pass (typically the first grouped GEMM in the MoE
+ # MLP), so load
+ # contiguous and permute on store.
+ load_a_idx = indices_to_gather[:, None] * N
+ store_idx = expert_token_offsets * K
else:
- # Case where we permuted on store in the forward pass (typically second grouped GEMM in MoE MLP)
- load_a_idx = (
- expert_token_offsets * N
- ) # Permute on load from token -> expert order
- store_idx = indices_to_gather[:, None] * K # Store in contiguous order
+ # Permuted on store in the forward pass (typically the second grouped GEMM), so permute
+ # on load and
+ # store contiguous.
+ load_a_idx = expert_token_offsets * N
+ store_idx = indices_to_gather[:, None] * K
else:
- # # Position in full matrix - needed for TMA
- # m_offset = (M_start + (tile_m_idx * BLOCK_SIZE_M)).to(tl.int32)
- # k_offset = (tile_k_idx * BLOCK_SIZE_K).to(tl.int32)
- # Offsets *relative* to the *current* expert -- m_start will then advance to this expert's start token
+ # Offsets relative to the CURRENT expert; m_start then advances to this expert's start token.
offs_am = tile_m_idx * BLOCK_SIZE_M + m_block_range
- # [M, N] @ [N, K] -> [M, K] => Stride for A is N, stride for B is K
- # We need two additional offsets:
- # 1. For A, m_start to advance to this expert's start token
- # 2. For B, n_start to advance to this expert's weights since we are passing in an [E, N, K] weight matrix
+ # [M, N] @ [N, K] -> [M, K], so A strides by N and B by K, plus m_start for A's expert start token
+ # and n_start for B's slice of the [E, N, K] weight matrix.
row_offsets_a = m_start + offs_am[:, None]
load_a_idx = row_offsets_a * N
store_idx = row_offsets_a * K
@@ -195,13 +182,9 @@ def _grouped_gemm_dX_kernel(
offs_bk = tile_k_idx * BLOCK_SIZE_K + k_block_range
if not USE_TMA_LOAD_W:
row_offsets_b = n_start + n_block_range
- # offs_bn = n_start + n_block_range
- # row_offsets_b = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N)
w_ptrs = w_ptr + row_offsets_b[:, None] * K + offs_bk[None, :]
- # TODO: check whether predication along K is needed since we checked that K is divisible by BLOCK_SIZE_K in the forward kernel
- # col_mask = offs_bk[None, :] < K
- store_mask = row_mask # & col_mask
+ store_mask = row_mask
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_K), dtype = tl.float32)
@@ -214,30 +197,29 @@ def _grouped_gemm_dX_kernel(
dY = dY_desc.load([m_start + tile_m_idx * BLOCK_SIZE_M, n_offset])
if not USE_TMA_LOAD_W:
- w = tl.load(w_ptrs) # , mask=col_mask)
+ w = tl.load(w_ptrs)
else:
w = w_desc.load([expert_idx, n_offset, tile_k_idx * BLOCK_SIZE_K])
w = tl.reshape(w, (BLOCK_SIZE_N, BLOCK_SIZE_K))
- # TODO: check if predication along K is needed since we checked that K is divisible by BLOCK_SIZE_K in the forward kernel
# [M, N] @ [N, K] -> [M, K]
dY = dY.to(w.dtype)
- accumulator += tl.dot(dY, w) # NOTE: no transpose of b
+ accumulator += tl.dot(dY, w)
# Advance A along contiguous dimension
if not USE_TMA_LOAD_dY:
dY_ptrs += BLOCK_SIZE_N
- # Note we are no longer advancing B along contiguous dimension since weights are arranged as [N, K]
- # Instead, we need to stride by K to advance to the [N_BLOCK_SIZE, K_BLOCK_SIZE] tile
+ # B is no longer advanced along the contiguous dimension: weights are [N, K], so stride by K to
+ # reach the next [N_BLOCK_SIZE, K_BLOCK_SIZE] tile.
if not USE_TMA_LOAD_W:
w_ptrs += BLOCK_SIZE_N * K
dX = accumulator.to(output_dtype)
- # Writing out a BLOCK_M x BLOCK_K tile, so we need to stride by K
+ # A BLOCK_M x BLOCK_K tile is written out, so stride by K.
if USE_TMA_STORE:
- offset_m = tile_m_idx * BLOCK_SIZE_M # .to(tl.int32)
- offset_k = tile_k_idx * BLOCK_SIZE_K # .to(tl.int32)
+ offset_m = tile_m_idx * BLOCK_SIZE_M
+ offset_k = tile_k_idx * BLOCK_SIZE_K
dX_desc.store([m_start + offset_m, offset_k], dX)
else:
tl.store(
@@ -256,7 +238,7 @@ def _grouped_gemm_dX_kernel(
_autotuned_grouped_gemm_dX_kernel = triton.autotune(
configs = get_dX_kernel_configs(),
prune_configs_by = {"early_config_prune": prune_dX_configs},
- # NOTE: NUM_TOKENS removed from key to avoid recompilation for every sequence length
+ # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length.
key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],
)(_grouped_gemm_dX_kernel)
@@ -327,7 +309,7 @@ def _grouped_gemm_dW_kernel(
strides = [K, 1],
block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_K],
)
- # Output tiles per expert, since each expert weight matrix is [N, K]
+ # Output tiles per expert, since each expert weight matrix is [N, K].
num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)
num_k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
output_tiles_per_expert = num_n_tiles * num_k_tiles
@@ -336,7 +318,7 @@ def _grouped_gemm_dW_kernel(
block_range_n = tl.arange(0, BLOCK_SIZE_N)
block_range_k = tl.arange(0, BLOCK_SIZE_K)
- # NOTE: Important that N % BLOCK_SIZE_N == 0 and K % BLOCK_SIZE_K == 0 when using TMA store
+ # N % BLOCK_SIZE_N == 0 and K % BLOCK_SIZE_K == 0 are required when using TMA store.
if USE_TMA_STORE:
tl.static_assert(N % BLOCK_SIZE_N == 0, "N must be divisible by BLOCK_SIZE_N")
tl.static_assert(K % BLOCK_SIZE_K == 0, "K must be divisible by BLOCK_SIZE_K")
@@ -347,7 +329,7 @@ def _grouped_gemm_dW_kernel(
block_shape = [1, BLOCK_SIZE_N, BLOCK_SIZE_K],
)
- for tile_idx in range(tidx, output_tiles_per_expert, NUM_SMS): # , flatten=FLATTEN):
+ for tile_idx in range(tidx, output_tiles_per_expert, NUM_SMS):
# Output tile index
tile_n_idx = tile_idx % num_n_tiles
tile_k_idx = tile_idx // num_n_tiles
@@ -356,24 +338,23 @@ def _grouped_gemm_dW_kernel(
n_offset = tile_n_idx * BLOCK_SIZE_N
k_offset = tile_k_idx * BLOCK_SIZE_K
- # For storing
- # TODO: Check whether the k mask is needed since we statically check that K is divisible by BLOCK_SIZE_K in the forward kernel
- # ditto for n_mask
n_mask = block_range_n + n_offset < N
k_mask = block_range_k + k_offset < K
nk_mask = n_mask[:, None] & k_mask[None, :]
m_end = 0
for expert_idx in range(NUM_EXPERTS):
- # We need to instantiate a fresh accumulator for each expert
+ # A fresh accumulator per expert.
accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_K), dtype = acc_dtype)
m_start = m_end
- # Need to figure out why this cast is needed, otherwise compiler complains about mismatching types
+ # The cast is needed or the compiler complains about mismatching types; reason unknown.
m_size = tl.load(m_sizes_ptr + expert_idx).to(tl.int32)
m_end = m_start + m_size
- # NOTE: when storing the result, we need to offset by n_start since we are storing the result for this expert to the global [E, N, K] weight matrix
+ # Offset by n_start: the result goes into this expert's slice of the global [E, N, K] weight matrix.
+ # NOTE: when storing the result, we need to offset by n_start since we are storing the result for
+ # this expert to the global [E, N, K] weight matrix
n_start = expert_idx * N
store_row_offs = n_start + n_offset + block_range_n
@@ -403,15 +384,12 @@ def _grouped_gemm_dW_kernel(
if PERMUTE_X or PERMUTE_Y:
# These will be used for loading and storing in permuted order
- gather_offsets = (
- tile_m_idx + block_range_m
- ) # NOTE: tile_m_idx is already strided by BLOCK_SIZE_M
+ gather_offsets = tile_m_idx + block_range_m
indices_to_gather = m_start + tl.max_contiguous(
tl.multiple_of(gather_offsets % m_size, BLOCK_SIZE_M),
BLOCK_SIZE_M,
)
- # indices_to_gather = m_start + gather_offsets
expert_token_idx = tl.load(
gather_indices_ptr + indices_to_gather,
mask = indices_to_gather < TOTAL_TOKENS,
@@ -421,18 +399,16 @@ def _grouped_gemm_dW_kernel(
# Masks for permuted load and store
row_load_mask = gather_offsets < m_size
- # We only take into account the following two cases: (PERMUTE_X and NOT PERMUTE_Y) and (NOT PERMUTE_X and PERMUTE_Y)
- # Hence, we can make the following simplifying assumptions when loading and storing
- # Note the different strides between the two cases: the offsets for loading and storing are flipped and the strides must also be adjusted
+ # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so
+ # load/store offsets
+ # are flipped between the two cases, with the strides adjusted.
if PERMUTE_X:
x_row_load_idx = (
(expert_token_offsets // TOPK) * K
- ) # Permute on load from token -> expert order, divide by TOPK to index from original number of tokens
+ ) # Permute on load: token to expert order, /TOPK for the original count.
dY_row_load_idx = m_offsets[:, None] * N
else:
- x_row_load_idx = (
- indices_to_gather[:, None] * K
- ) # Load in contiguous order (no permutation on load)
+ x_row_load_idx = indices_to_gather[:, None] * K
dY_row_load_idx = expert_token_offsets * N
else:
@@ -460,21 +436,18 @@ def _grouped_gemm_dW_kernel(
)
accumulator += tl.dot(
- dY.T.to(x.dtype), # [BLOCK_N, BLOCK_M]
- x, # [BLOCK_M, BLOCK_K]
+ dY.T.to(x.dtype),
+ x,
)
y = accumulator.to(output_dtype)
if USE_TMA_STORE:
- # Need to expand dims to match [E, N, K] shape
+ # Expand dims to match the [E, N, K] shape.
y = tl.expand_dims(y, 0)
dW_desc.store([expert_idx, n_offset, k_offset], y)
else:
tl.store(
- dW_ptr
- # + (n_offset + offs_n)[:, None] * K
- + store_row_offs[:, None] * K
- + (k_offset + block_range_k)[None, :],
+ dW_ptr + store_row_offs[:, None] * K + (k_offset + block_range_k)[None, :],
y,
mask = nk_mask,
)
@@ -483,6 +456,6 @@ def _grouped_gemm_dW_kernel(
_autotuned_grouped_gemm_dW_kernel = triton.autotune(
configs = get_dW_kernel_configs(),
prune_configs_by = {"early_config_prune": prune_kernel_configs_backward_dW},
- # NOTE: NUM_TOKENS removed from key to avoid recompilation for every sequence length
+ # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length.
key = ["NUM_EXPERTS", "N", "K", "PERMUTE_X", "PERMUTE_Y"],
)(_grouped_gemm_dW_kernel)
diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
index cb1f22d0b85..0a4010e2881 100644
--- a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
+++ b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py
@@ -11,15 +11,8 @@ from .autotuning import (
)
-#
-# PERMUTE_X -> permute tokens so that they are ordered by expert
-# PERMUTE_Y -> permute output so that they are ordered by token
-# These are effectively the same thing: the former loads in permuted order, the latter stores in permuted order => we only need to define the permutation indices once
-# In the former, we use these row indices when loading X
-# For the latter, we use these row indices when storing Y
-# FUSE_MUL -> multiply routed outputs by their respective weights
-# topk_weights are in token order
-# Only account for the case when X is in expert order and we are permuting Y when fusing mul -- this precondition is checked in the interface
+# PERMUTE_X loads tokens in expert order, PERMUTE_Y stores output in token order: the same
+# permutation indices either way.
@triton.jit
def _grouped_gemm_forward_kernel(
x_ptr,
@@ -57,14 +50,11 @@ def _grouped_gemm_forward_kernel(
SHOULD_PERMUTE: tl.constexpr = PERMUTE_X or PERMUTE_Y
SHOULD_FUSE_MUL: tl.constexpr = FUSE_MUL_PRE or FUSE_MUL_POST
SHOULD_PERMUTE_OR_FUSE: tl.constexpr = SHOULD_PERMUTE or SHOULD_FUSE_MUL
- # tl.static_print("SHOULD_PERMUTE", PERMUTE_X, PERMUTE_Y, FUSE_MUL_PRE, FUSE_MUL_POST, SHOULD_PERMUTE, SHOULD_FUSE, SHOULD_PERMUTE_OR_FUSE)
tidx = tl.program_id(0)
output_dtype: tl.dtype = y_ptr.dtype.element_ty
- # Create TMA descriptors for loading sorted tokens
- # When using TMA load, we don't permute_x, so shape should be [TOTAL_TOKENS, K]
- # Also, we are defining a single global descriptor with single block shape
- # Need to check that this does not result in errors when crossing expert boundaries
+ # A single global TMA descriptor with one block shape; TMA load never permutes x, so the shape is
+ # [TOTAL_TOKENS, K]. Unverified across expert boundaries.
if USE_TMA_LOAD_X:
x_desc = tl.make_tensor_descriptor(
x_ptr,
@@ -98,10 +88,10 @@ def _grouped_gemm_forward_kernel(
num_n_tiles = tl.cdiv(N, BLOCK_SIZE_N)
num_tiles_per_expert = num_m_tiles * num_n_tiles
- # Need to create tma_store within loop since we need to predicate stores based on m_size
+ # The tma_store is created inside the loop so stores can be predicated on m_size.
if USE_TMA_STORE:
y_desc = tl.make_tensor_descriptor(
- y_ptr, # + m_start * N,
+ y_ptr,
shape = [m_end, N],
strides = [N, 1],
block_shape = [BLOCK_SIZE_M, BLOCK_SIZE_N],
@@ -129,21 +119,16 @@ def _grouped_gemm_forward_kernel(
expert_token_offsets = expert_token_idx[:, None]
# Masks for permuted load and store
-
row_mask = gather_offsets < m_size
row_mask = row_mask[:, None]
- # row_mask = indices_to_gather < m_end
- # row_mask = row_mask[:, None]
-
- # We only take into account the following two cases: (PERMUTE_X and NOT PERMUTE_Y) and (NOT PERMUTE_X and PERMUTE_Y)
- # Hence, we can make the following simplifying assumptions when loading and storing
- # Note the different strides between the two cases: the offsets for loading and storing are flipped and the strides must also be adjusted
+ # Only (PERMUTE_X and not PERMUTE_Y) and (not PERMUTE_X and PERMUTE_Y) occur, so load/store offsets
+ # are flipped between the two cases, with the strides adjusted.
if PERMUTE_X:
load_idx = (
(expert_token_offsets // TOPK) * K
- ) # Permute on load from token -> expert order, divide by TOPK to index from original number of tokens
- store_idx = indices_to_gather[:, None] * N # Store in contiguous order
+ ) # Permute on load from token to expert order, dividing by TOPK to index the original token count.
+ store_idx = indices_to_gather[:, None] * N
else:
off_am = tile_m_idx * BLOCK_SIZE_M
if not PERMUTE_Y:
@@ -157,19 +142,11 @@ def _grouped_gemm_forward_kernel(
if PERMUTE_Y:
if not USE_TMA_LOAD_X:
- load_idx = (
- indices_to_gather[:, None] * K
- ) # Load in contiguous order (no permutation on load)
- # offs_am = off_am + m_block_range
- # row_mask = offs_am[:, None] < m_size
- store_idx = (
- expert_token_offsets * N
- ) # Permute on store from expert -> token order
-
- # We always load topk weights in expert order
- # In the pre-multiplication case, we multiply permuted hidden states by weights before the first gemm
- # In the post-multiplication case, we multiply permuted hidden states by weights after the second gemm
- # In either case, the hidden states are grouped by expert, so we always permute on load of topk weights
+ load_idx = indices_to_gather[:, None] * K
+ store_idx = expert_token_offsets * N
+
+ # topk weights are always loaded in expert order: pre-multiplication scales hidden states before
+ # the first gemm and post-multiplication after the second, both grouped by expert.
if SHOULD_FUSE_MUL:
topk_load_idx = expert_token_offsets
@@ -215,10 +192,9 @@ def _grouped_gemm_forward_kernel(
y = accumulator.to(output_dtype)
- # NOTE: order of fusing multiplication is important
- # Fusing before accumulator dtype conversion results in numerical diffs
+ # Order of the fused multiplication matters: fusing before the accumulator dtype conversion changes
+ # the numerics.
if FUSE_MUL_POST:
- # Check for correct broadcasting
topk_weights = tl.load(topk_weights_ptr + topk_load_idx, mask = row_mask)
y *= topk_weights.to(output_dtype)
@@ -226,8 +202,8 @@ def _grouped_gemm_forward_kernel(
store_mask = row_mask & (offs_bn[None, :] < N)
if USE_TMA_STORE:
- offset_m = tile_m_idx * BLOCK_SIZE_M # .to(tl.int32)
- offset_n = tile_n_idx * BLOCK_SIZE_N # .to(tl.int32)
+ offset_m = tile_m_idx * BLOCK_SIZE_M
+ offset_n = tile_n_idx * BLOCK_SIZE_N
y_desc.store([m_start + offset_m, offset_n], y)
else:
tl.store(
@@ -243,8 +219,8 @@ def _grouped_gemm_forward_kernel(
_autotuned_grouped_gemm_forward_kernel = triton.autotune(
configs = get_forward_configs(),
prune_configs_by = {"early_config_prune": prune_kernel_configs_fwd},
- # NOTE: NUM_TOKENS removed from key to avoid recompilation for every sequence length
- # The kernel handles variable token counts via m_sizes and tile-based processing
+ # NUM_TOKENS is left out of the key to avoid recompiling for every sequence length; the kernel
+ # handles variable token counts via m_sizes and tile-based processing.
key = [
"NUM_EXPERTS",
"N",
diff --git a/unsloth/kernels/moe/grouped_gemm/reference/layers/llama4_moe.py b/unsloth/kernels/moe/grouped_gemm/reference/layers/llama4_moe.py
index a1d94d2c7d1..9170515f641 100644
--- a/unsloth/kernels/moe/grouped_gemm/reference/layers/llama4_moe.py
+++ b/unsloth/kernels/moe/grouped_gemm/reference/layers/llama4_moe.py
@@ -155,7 +155,7 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):
hidden_states = hidden_states.view(-1, hidden_dim)
if self.overlap_router_shared:
- # Marker for all prior ops on default stream
+ # Marker for all prior ops on the default stream.
self.default_event.record()
router_logits, routing_weights, selected_experts = self.run_router(hidden_states)
@@ -166,7 +166,8 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):
if self.overlap_router_shared:
with torch.cuda.stream(self.shared_expert_stream):
- # Ensure prior kernels on default stream complete
+ # Ensure prior kernels on the default stream complete, that hidden states stay valid on this stream,
+ # and that the shared expert stays valid on the default stream.
self.default_event.wait()
shared_expert_out = self.shared_expert(hidden_states)
@@ -189,8 +190,8 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):
hidden_states = hidden_states.sum(dim = 1)
hidden_states_after_weight_merge = hidden_states.view(-1, hidden_dim)
- # Token counts per expert + gather indices (token->expert order).
- # Auxiliary structs; not recorded in the autograd graph.
+ # Token counts per expert plus gather indices (token to expert order): auxiliary structs, not
+ # recorded in the autograd graph.
token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(
selected_experts
)
@@ -199,7 +200,6 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):
hidden_states = permute(hidden_states_after_weight_merge, gather_indices, self.top_k)
assert hidden_states.shape == (total_tokens, hidden_dim)
- # Start expert computation
first_gemm = torch_grouped_gemm(
X = hidden_states, W = self.experts.gate_up_proj, m_sizes = token_counts_by_expert
)
@@ -214,10 +214,8 @@ class Llama4GroupedGemmTextMoe(Llama4TextMoe):
)
assert second_gemm.shape == (total_tokens, hidden_dim)
- # Post-processing
hidden_states_unpermute = unpermute(second_gemm, gather_indices)
assert hidden_states_unpermute.shape == (total_tokens, hidden_dim)
- # grouped_gemm_out = hidden_states.view(batch_size, sequence_length, hidden_dim)
final_out = hidden_states_unpermute + shared_expert_out
@@ -331,7 +329,7 @@ class Llama4TritonTextMoe(Llama4GroupedGemmTextMoe):
hidden_states = hidden_states.view(-1, hidden_dim)
if self.overlap_router_shared:
- # Marker for all prior ops on default stream
+ # Marker for all prior ops on the default stream.
self.default_event.record()
router_logits, routing_weights, selected_experts = self.run_router(hidden_states)
@@ -342,7 +340,8 @@ class Llama4TritonTextMoe(Llama4GroupedGemmTextMoe):
if self.overlap_router_shared:
with torch.cuda.stream(self.shared_expert_stream):
- # Ensure prior kernels on default stream complete
+ # Ensure prior kernels on the default stream complete, that hidden states stay valid on this stream,
+ # and that the shared expert stays valid on the default stream.
self.default_event.wait()
shared_expert_out = self.shared_expert(hidden_states)
@@ -365,8 +364,8 @@ class Llama4TritonTextMoe(Llama4GroupedGemmTextMoe):
hidden_states = hidden_states.sum(dim = 1)
hidden_states = hidden_states.view(-1, hidden_dim)
- # Token counts per expert + gather indices (token->expert order).
- # Auxiliary structs; not recorded in the autograd graph.
+ # Token counts per expert plus gather indices (token to expert order): auxiliary structs, not
+ # recorded in the autograd graph.
token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(
selected_experts
)
@@ -375,7 +374,6 @@ class Llama4TritonTextMoe(Llama4GroupedGemmTextMoe):
hidden_states = permute(hidden_states, gather_indices, self.top_k)
assert hidden_states.shape == (total_tokens, hidden_dim)
- # Start expert computation
hidden_states = grouped_gemm(
X = hidden_states,
W = self.experts.gate_up_proj,
diff --git a/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py b/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py
index da3faa5f93c..47bd5548e9f 100644
--- a/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py
+++ b/unsloth/kernels/moe/grouped_gemm/reference/layers/qwen3_moe.py
@@ -44,7 +44,7 @@ class GroupedGEMMResult:
intermediate: torch.Tensor
second_gemm: torch.Tensor
hidden_states_unpermute: torch.Tensor
- hidden_states: torch.Tensor # final output
+ hidden_states: torch.Tensor
class Qwen3MoeGroupedGEMMBlock(torch.nn.Module):
@@ -70,10 +70,8 @@ class Qwen3MoeGroupedGEMMBlock(torch.nn.Module):
config.moe_intermediate_size,
)
- # gating
self.gate = torch.nn.Parameter(gate)
- # experts
self.gate_up_proj = torch.nn.Parameter(gate_up_proj, requires_grad = True)
self.down_proj = torch.nn.Parameter(down_proj, requires_grad = True)
self.act_fn = ACT2FN[config.hidden_act]
@@ -157,8 +155,8 @@ class Qwen3MoeGroupedGEMMBlock(torch.nn.Module):
router_logits, routing_weights, selected_experts = self.run_router(hidden_states)
- # Token counts per expert + gather indices (token->expert order).
- # Auxiliary structs; not recorded in the autograd graph.
+ # Token counts per expert plus gather indices (token to expert order): auxiliary structs, not
+ # recorded in the autograd graph.
token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(
selected_experts
)
@@ -167,7 +165,6 @@ class Qwen3MoeGroupedGEMMBlock(torch.nn.Module):
hidden_states = permute(hidden_states, gather_indices, self.top_k)
assert hidden_states.shape == (total_tokens, hidden_dim)
- # Start expert computation
first_gemm = torch_grouped_gemm(
X = hidden_states, W = self.gate_up_proj, m_sizes = token_counts_by_expert
)
@@ -274,13 +271,13 @@ class Qwen3MoeFusedGroupedGEMMBlock(Qwen3MoeGroupedGEMMBlock):
hidden_states = hidden_states.view(-1, hidden_dim)
router_logits, routing_weights, selected_experts = self.run_router(hidden_states)
- # Token counts per expert + gather indices (token->expert order).
- # Auxiliary structs; not recorded in the autograd graph.
+ # Token counts per expert plus gather indices (token to expert order): auxiliary structs, not
+ # recorded in the autograd graph.
token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(
selected_experts
)
- # When permute_x is set, the permute fuses into the first gemm prologue
+ # With permute_x set, the permute fuses into the first gemm's prologue.
if not self.permute_x:
hidden_states = permute(hidden_states, gather_indices, self.top_k)
# Start expert computation
diff --git a/unsloth/kernels/moe/grouped_gemm/reference/moe_block.py b/unsloth/kernels/moe/grouped_gemm/reference/moe_block.py
index 2412425c1a8..49c970a3f18 100644
--- a/unsloth/kernels/moe/grouped_gemm/reference/moe_block.py
+++ b/unsloth/kernels/moe/grouped_gemm/reference/moe_block.py
@@ -96,17 +96,15 @@ class Qwen3MoeFusedGroupedGEMMBlock(Qwen3MoeGroupedGEMMBlock):
hidden_states = hidden_states.view(-1, hidden_dim)
router_logits, routing_weights, selected_experts = self.run_router(hidden_states)
- # Pre-processing
- # 1. Compute tokens per expert and indices for gathering tokes from token order to expert order
- # NOTE: these are auxiliary data structs which don't need to be recorded in autograd graph
+ # Tokens per expert and the token-order to expert-order gather indices: auxiliary structs, not
+ # recorded in the autograd graph.
token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(
selected_experts
)
- # 2. permute_x -> permutation will be fused in prologue of first grouped gemm
+ # permute_x fuses the permutation into the first grouped gemm's prologue.
if not self.permute_x:
hidden_states = permute(hidden_states, gather_indices, self.top_k)
- # Start expert computation
hidden_states = grouped_gemm(
X = hidden_states,
W = self.gate_up_proj,
@@ -141,8 +139,7 @@ class Qwen3MoeFusedGroupedGEMMBlock(Qwen3MoeGroupedGEMMBlock):
dX_only = self.dX_only,
)
- # Post-processing
- # 1. Unpermute from expert order to token order
+ # Post-processing 1. Unpermute from expert order to token order
if not self.permute_y:
hidden_states = unpermute(hidden_states, gather_indices)
diff --git a/unsloth/kernels/moe/grouped_gemm/reference/moe_ops.py b/unsloth/kernels/moe/grouped_gemm/reference/moe_ops.py
index 41866de347c..510d89c96e4 100644
--- a/unsloth/kernels/moe/grouped_gemm/reference/moe_ops.py
+++ b/unsloth/kernels/moe/grouped_gemm/reference/moe_ops.py
@@ -20,7 +20,7 @@ def permute(X: torch.Tensor, gather_indices: torch.Tensor, topk: int):
"""
assert gather_indices.ndim == 1
X = X.view(-1, X.shape[-1])
- # Shortcut for topk == 1
+ # Shortcut for topk == 1.
if topk == 1:
return X[gather_indices]
@@ -94,7 +94,7 @@ def get_routing_indices(
min = 0,
max = num_experts,
)
- # token_indices_experts_sorted shape (bs*slen*top_k,)
+ # token_indices_experts_sorted has shape (bs * slen * top_k,).
gather_indices = torch.argsort(selected_experts.view(-1), stable = True)
if return_scatter_indices:
scatter_indices = gather_indices.argsort()
@@ -136,13 +136,11 @@ def torch_grouped_gemm(
if m_size > 0:
m_end = m_start + m_size
- # Extract group input
- # m_size x K
+ # Extract group input m_size x K
X_g = X[m_start:m_end]
- # N x K
W_g = W[g]
- # Y_g = X_g @ W_g.T -> [m_size, N]
+ # Y_g = X_g @ W_g.T -> [m_size, N].
W_g = W_g.T if transpose else W_g
Y_g = X_g @ W_g
diff --git a/unsloth/kernels/moe/tests/common.py b/unsloth/kernels/moe/tests/common.py
index f35a2fc9c34..b8574a82fb8 100644
--- a/unsloth/kernels/moe/tests/common.py
+++ b/unsloth/kernels/moe/tests/common.py
@@ -89,7 +89,7 @@ TOLERANCE = {
}
-# from https://github.com/triton-lang/triton/blob/main/bench/triton_bench/testing.py
+# From triton-lang/triton bench/triton_bench/testing.py
def assert_equal(ref, tri):
if isinstance(ref, torch.Tensor):
assert torch.all(ref == tri), f"tensors not equal {ref} != {tri}"
@@ -265,15 +265,13 @@ def remove_feature_flags(
return pruned_configs
-# Test Configs
-
TOPK = [1, 4]
NUM_EXPERTS = [4, 16]
TEST_MODEL_SIZES = [
- (32, 32), # Debug
- (128, 128), # Small
- (512, 512), # Medium
+ (32, 32),
+ (128, 128),
+ (512, 512),
]
SMALL_MODEL_CONFIGS = [
diff --git a/unsloth/kernels/moe/tests/moe_utils.py b/unsloth/kernels/moe/tests/moe_utils.py
index 3b87fe45a40..801557c96d6 100644
--- a/unsloth/kernels/moe/tests/moe_utils.py
+++ b/unsloth/kernels/moe/tests/moe_utils.py
@@ -34,13 +34,13 @@ def rebind_experts_to_shared_buffer(moe_block: Qwen3MoeSparseMoeBlock, config: Q
buffer_gate = torch.empty(num_experts, interm_size, hidden_size, device = device, dtype = dtype)
buffer_down = torch.empty(num_experts, hidden_size, interm_size, device = device, dtype = dtype)
- # Copy existing expert weights into buffers
+ # Rebind expert weights to views in the shared buffer
for i, expert in enumerate(moe_block.experts):
buffer_up[i].copy_(expert.up_proj.weight.data)
buffer_gate[i].copy_(expert.gate_proj.weight.data)
buffer_down[i].copy_(expert.down_proj.weight.data)
- # Rebind expert weights to views in the shared buffer
+ # Rebind expert weights to views in the shared buffer.
for i, expert in enumerate(moe_block.experts):
expert.up_proj.weight = torch.nn.Parameter(buffer_up[i])
expert.gate_proj.weight = torch.nn.Parameter(buffer_gate[i])
@@ -117,13 +117,11 @@ def check_gate_up_proj_grad(
assert ref_gate_proj_grad is not None
assert ref_up_proj_grad is not None
- # Extract gradients
test_gate_proj_grad = grouped_gemm_block.gate_up_proj.grad[i, :moe_intermediate_size]
test_up_proj_grad = grouped_gemm_block.gate_up_proj.grad[i, moe_intermediate_size:]
assert test_gate_proj_grad is not None
assert test_up_proj_grad is not None
- # Sanity check shapes
assert (
ref_gate_proj_grad.shape == test_gate_proj_grad.shape
), f"{ref_gate_proj_grad.shape} != {test_gate_proj_grad.shape}"
@@ -131,7 +129,6 @@ def check_gate_up_proj_grad(
ref_up_proj_grad.shape == test_up_proj_grad.shape
), f"{ref_up_proj_grad.shape} != {test_up_proj_grad.shape}"
- # Check gradients
diff = (ref_gate_proj_grad - test_gate_proj_grad).abs().max()
if not torch.allclose(ref_gate_proj_grad, test_gate_proj_grad, atol = atol, rtol = rtol):
print(f"expert {i} gate_proj_grad_diff: {diff.detach().cpu().item():.6f}")
@@ -199,7 +196,6 @@ def check_expert_grads(
ref_grads.shape == test_grads.shape
), f"{field}: {ref_grads.shape} != {test_grads.shape}"
- # Test each expert
for i in range(ref_grads.shape[0]):
ref_grad = ref_grads[i]
test_grad = test_grads[i]
@@ -208,7 +204,6 @@ def check_expert_grads(
ref_grad, test_grad, atol = atol, rtol = rtol
), f"{field}[{i}] diff: {diff.detach().cpu().item():.6f}"
- # Test all experts
diff = (ref_grads - test_grads).abs().max()
if verbose:
print(f"{field} diff: {diff.detach().cpu().item():.6f}")
@@ -238,7 +233,6 @@ def check_fwd(
rtol: float,
verbose: bool = False,
):
- # First check hidden states (output)
ref_output = ref_result.output
test_output = test_result.output
diff = (ref_output - test_output).abs().max()
@@ -248,7 +242,6 @@ def check_fwd(
ref_output, test_output, atol = atol, rtol = rtol
), f"output diff: {diff.detach().cpu().item():.6f}"
- # Check router logits
ref_router_logits = ref_result.router_logits
test_router_logits = test_result.router_logits
diff = (ref_router_logits - test_router_logits).abs().max()
@@ -272,8 +265,8 @@ def check_grouped_gemm_results(
test_value = getattr(fused_result, field.name)
diff = (ref_value - test_value).abs().max()
- # second_gemm in torch grouped gemm is not yet unpermuted so comparing the fused unpermuted second_gemm will result in error
- # instead the hidden_states_unpermute should match since hidden_states_unpermute for the fused result is the same as second_gemm
+ # The torch grouped gemm's second_gemm is not yet unpermuted, so compare hidden_states_unpermute
+ # instead, which matches the fused result's second_gemm.
if field.name == "second_gemm" and permute_y:
continue
@@ -404,18 +397,17 @@ class Qwen3MoeFusedGroupedGEMMBlock(Qwen3MoeGroupedGEMMBlock):
hidden_states = hidden_states.view(-1, hidden_dim)
router_logits, routing_weights, selected_experts = self.run_router(hidden_states)
- # Pre-processing: token counts per expert + token-order -> expert-order
- # gather indices (auxiliary, not recorded in the autograd graph).
+ # Token counts per expert plus token-order to expert-order gather indices: auxiliary, not recorded
+ # in the autograd graph.
token_counts_by_expert, gather_indices = self.get_token_counts_and_gather_indices(
selected_experts
)
- # permute_x fuses the permutation into the first grouped gemm's prologue
+ # permute_x fuses the permutation into the first grouped gemm's prologue.
if not self.permute_x:
hidden_states = permute(hidden_states, gather_indices, self.top_k)
assert hidden_states.shape == (total_tokens, hidden_dim)
- # Start expert computation
first_gemm = grouped_gemm(
X = hidden_states,
W = self.gate_up_proj,
diff --git a/unsloth/kernels/moe/tests/test_grouped_gemm.py b/unsloth/kernels/moe/tests/test_grouped_gemm.py
index 7ff47d07678..865df9f2007 100644
--- a/unsloth/kernels/moe/tests/test_grouped_gemm.py
+++ b/unsloth/kernels/moe/tests/test_grouped_gemm.py
@@ -43,8 +43,7 @@ from .common import (
SEED = 0
-# Only certain (permute_x, permute_y, use_W1) combinations are valid; see the
-# module string below for the full rationale.
+# Only certain (permute_x, permute_y, use_W1) combinations are valid; see the module string below.
def check_valid_config(
permute_x,
permute_y,
@@ -165,8 +164,8 @@ def _test_grouped_gemm_forward(
topk_weights, topk_ids = calculate_topk(
gating_output, topk, use_sigmoid = use_sigmoid, renormalize = renormalize
)
- topk_weights = topk_weights.view(-1) # num_tokens * topk
- topk_ids = topk_ids.view(-1) # num_tokens * topk
+ topk_weights = topk_weights.view(-1)
+ topk_ids = topk_ids.view(-1)
expert_token_counts, gather_indices = get_routing_indices(topk_ids, num_experts = E)
assert len(gather_indices) == total_tokens
@@ -189,7 +188,7 @@ def _test_grouped_gemm_forward(
else:
X_test = Xperm
- # No need to run all configs for tests, otherwise takes too long
+ # Running all configs would take too long.
if autotune:
from grouped_gemm.kernels.forward import _autotuned_grouped_gemm_forward_kernel
if num_autotune_configs is not None:
@@ -197,7 +196,6 @@ def _test_grouped_gemm_forward(
_autotuned_grouped_gemm_forward_kernel.configs[:num_autotune_configs]
)
- # Use autograd.Function interface
if use_autograd:
from grouped_gemm.interface import grouped_gemm
kernel_config_fwd = KernelConfigForward(
@@ -228,7 +226,6 @@ def _test_grouped_gemm_forward(
autotune = autotune,
is_first_gemm = use_W1,
)
- # Use manual interface
else:
test_output = grouped_gemm_forward(
X = X_test,
@@ -257,8 +254,8 @@ def _test_grouped_gemm_forward(
if permute_y:
ref_output = unpermute(ref_output, gather_indices)
if fuse_mul_post:
- # if we don't permute_y, then test output is permuted with topk weights applied
- # the ref output needs to be unpermuted before multiplying by topk weights since topk weights are in token order
+ # Without permute_y the test output is permuted with topk weights applied, so the reference output
+ # must be unpermuted before multiplying by topk weights, which are in token order.
if not permute_y:
ref_output = unpermute(ref_output, gather_indices)
test_output = unpermute(test_output, gather_indices)
@@ -269,7 +266,7 @@ def _test_grouped_gemm_forward(
), f"Grouped gemm forward failed: {(ref_output - test_output).abs().max().item():.6f}"
-# NOTE: Fuse multiplication of topk weights is only supported for inference and not training, although this may change in the future; not currently tested.
+# Fusing the topk-weight multiplication is inference-only, not training, and is not tested.
@pytest.mark.parametrize(
"kernel_config",
KERNEL_CONFIGS_FWD,
@@ -457,7 +454,7 @@ def _test_grouped_gemm_backward_dX(
if autotune and model_config.intermediate_size <= 128 and model_config.hidden_size <= 128:
pytest.skip("Skipping autotuning for small model configs")
- # Prevent OOM for large intermediate sizes
+ # Prevent OOM for large intermediate sizes.
if model_config.intermediate_size > 2048:
model_config.intermediate_size = 1024
if model_config.hidden_size > 2048:
@@ -502,8 +499,8 @@ def _test_grouped_gemm_backward_dX(
topk_weights, topk_ids = calculate_topk(
gating_output, topk, use_sigmoid = use_sigmoid, renormalize = renormalize
)
- topk_weights = topk_weights.view(-1) # num_tokens * topk
- topk_ids = topk_ids.view(-1) # num_tokens * topk
+ topk_weights = topk_weights.view(-1)
+ topk_ids = topk_ids.view(-1)
expert_token_counts, gather_indices = get_routing_indices(topk_ids, num_experts = E)
assert len(gather_indices) == total_tokens
@@ -512,7 +509,7 @@ def _test_grouped_gemm_backward_dX(
atol, rtol = TOLERANCE[X.dtype]
Xperm = permute(X, gather_indices, topk)
- # Need to retain grad otherwise grad is not propagated
+ # Grad is not propagated without retain_grad.
X.retain_grad()
W.retain_grad()
Xperm.retain_grad()
@@ -607,10 +604,9 @@ def _test_grouped_gemm_backward_dX(
test_output.backward(grad_output)
assert X_.grad is not None
- # NOTE:need to handle grad differenlty in this case due to errors arising to do how torch autograd handles unpermute and sum reduction
- # the grad of Xperm unpermuted and reduced across topk should match X_.grad
- # However, both will have a numerical difference with that of ref_grad
- # This is due to the fact that torch autograd handles unpermute and sum reduction differently see: https://discuss.pytorch.org/t/permute-unpermute-gradient/219557 else:
+ # Grad needs different handling here: Xperm's grad unpermuted and reduced across topk matches
+ # X_.grad, but both differ numerically from ref_grad because autograd handles unpermute and sum
+ # reduction differently. See discuss.pytorch.org/t/permute-unpermute-gradient/219557
if permute_x and use_W1:
X_grad_unperm = unpermute(Xperm.grad, gather_indices)
manual_grad_check = X_grad_unperm.view(num_tokens, topk, K).sum(dim = 1)
@@ -647,11 +643,10 @@ def _test_grouped_gemm_backward_dX(
num_warps = num_warps,
num_stages = num_stages,
flatten = flatten,
- # debug=True,
)
- # if permute_x and use_W1 (first grouped GEMM) then the kernel should have unpermuted the dX
- # therefore we need to unpermute the ref_grad to compare to the output of the kernel
+ # With permute_x and use_W1 (first grouped GEMM) the kernel has already unpermuted dX, so unpermute
+ # ref_grad to compare.
if permute_x and use_W1:
ref_grad = unpermute(ref_grad, gather_indices)
@@ -665,12 +660,11 @@ def _test_grouped_gemm_backward_dX(
), f"Grouped gemm manual backward_dX outputs mismatch: {diff:.6f}"
if permute_x and use_W1:
- # Show that reduction results in diffs
- # First calculate X.grad manually by backpropping through unpermuted ref_grad
+ # Show that reduction results in diffs First calculate X.grad manually by backpropping through
+ # unpermuted ref_grad
dX_ref_check = ref_grad.view(num_tokens, topk, K).sum(dim = 1)
# Do the same for the actual output of the kernel
dX_test_check = dX_test.view(num_tokens, topk, K).sum(dim = 1)
- # Show diffs for each combination
diff_ref_check = (X.grad - dX_ref_check).abs().max().item()
diff_test_check = (X.grad - dX_test_check).abs().max().item()
diff_check_test = (dX_ref_check - dX_test_check).abs().max().item()
@@ -679,8 +673,8 @@ def _test_grouped_gemm_backward_dX(
)
-# NOTE: We reduce the size of the Llama4 model configs to prevent OOM
-# Important to note that for the full model size (5120, 8192), the tests do result in diffs on the order of 1e-2.
+# The Llama4 model configs are shrunk to prevent OOM; at the full size (5120, 8192) the tests do
+# show diffs on the order of 1e-2.
@pytest.mark.parametrize(
"kernel_config",
KERNEL_CONFIGS_BWD_dX,
@@ -759,7 +753,7 @@ def test_grouped_gemm_backward_dX_autotune(
use_W1: bool,
num_autotune_configs: int,
):
- # TMA loads / stores will be autotuned
+ # TMA loads / stores are autotuned.
_test_grouped_gemm_backward_dX(
data_config = data_config,
model_config = model_config,
@@ -792,7 +786,7 @@ def test_grouped_gemm_backward_dX_autotune_autograd(
use_W1: bool,
num_autotune_configs: int,
):
- # TMA loads / stores will be autotuned
+ # TMA loads / stores are autotuned.
_test_grouped_gemm_backward_dX(
data_config = data_config,
model_config = model_config,
@@ -881,8 +875,8 @@ def _test_grouped_gemm_backward_dW(
topk_weights, topk_ids = calculate_topk(
gating_output, topk, use_sigmoid = use_sigmoid, renormalize = renormalize
)
- topk_weights = topk_weights.view(-1) # num_tokens * topk
- topk_ids = topk_ids.view(-1) # num_tokens * topk
+ topk_weights = topk_weights.view(-1)
+ topk_ids = topk_ids.view(-1)
expert_token_counts, gather_indices = get_routing_indices(topk_ids, num_experts = E)
assert len(gather_indices) == total_tokens
@@ -892,7 +886,7 @@ def _test_grouped_gemm_backward_dW(
Xperm = permute(X, gather_indices, topk)
Xperm_test = Xperm.detach().clone().requires_grad_(True)
- # Need to retain grad otherwise grad is not propagated
+ # Grad is not propagated without retain_grad.
X.retain_grad()
W.retain_grad()
Xperm.retain_grad()
@@ -903,8 +897,8 @@ def _test_grouped_gemm_backward_dW(
ref_output = torch_grouped_gemm(X = Xperm, W = W, m_sizes = expert_token_counts)
assert ref_output.shape == output_shape
- # if permute_y then the assumption is that the output of grouped_gemm was unpermuted on store
- # Therefore we have to unpermute before backpropping to ensure proper alignment
+ # With permute_y the grouped_gemm output was unpermuted on store, so unpermute before backpropping
+ # to keep the alignment.
if permute_y:
ref_output = unpermute(ref_output, gather_indices)
@@ -913,7 +907,6 @@ def _test_grouped_gemm_backward_dW(
assert X.grad is not None
assert W.grad is not None
- # Test backward kernel directly
X_ = X_test if permute_x else Xperm_test
if debug:
diff --git a/unsloth/kernels/moe/tests/test_llama4_moe.py b/unsloth/kernels/moe/tests/test_llama4_moe.py
index ccc04b692a7..56cc51d6a6c 100644
--- a/unsloth/kernels/moe/tests/test_llama4_moe.py
+++ b/unsloth/kernels/moe/tests/test_llama4_moe.py
@@ -32,7 +32,7 @@ LLAMA4_SCOUT_ID = "meta-llama/Llama-4-Scout-17B-16E"
SEED = 42
SEQ_LENS = [1024]
DTYPES = [torch.bfloat16]
-# Reduce the number of autotuning configs to prevent excessive runtime
+# Reduce the number of autotuning configs to prevent excessive runtime.
NUM_AUTOTUNE_CONFIGS = 50
@@ -162,7 +162,7 @@ def test_llama4_ref(
permute_x: bool,
permute_y: bool,
overlap_router_shared: bool,
- model_config: Llama4TextConfig, # test fixture
+ model_config: Llama4TextConfig,
bs: int = 1,
device = "cuda",
precision = ".6f",
@@ -177,10 +177,8 @@ def test_llama4_ref(
check_diff = partial(_check_diff, atol = atol, rtol = rtol, precision = precision, verbose = verbose)
check_grads = partial(_check_grads, atol = atol, rtol = rtol, precision = precision, verbose = verbose)
- # Reference op -- HF
llama4_ref = Llama4TextMoe(model_config).to(dtype = dtype, device = device)
- # Torch grouped gemm impl
llama4_gg_ref = Llama4GroupedGemmTextMoe(
model_config, overlap_router_shared = overlap_router_shared
).to(dtype = dtype, device = device)
diff --git a/unsloth/kernels/moe/tests/test_qwen3_moe.py b/unsloth/kernels/moe/tests/test_qwen3_moe.py
index c601ebcfc71..7c0675f0d9d 100644
--- a/unsloth/kernels/moe/tests/test_qwen3_moe.py
+++ b/unsloth/kernels/moe/tests/test_qwen3_moe.py
@@ -96,7 +96,7 @@ SEED = 42
SEQ_LENS = [1024]
DTYPES = [torch.bfloat16]
-# Reduce the number of autotuning configs to prevent excessive runtime
+# Reduce the number of autotuning configs to prevent excessive runtime.
NUM_AUTOTUNE_CONFIGS = 50
@@ -120,10 +120,9 @@ def test_qwen3_moe(
hidden_size = config.hidden_size
bs = 1
atol, rtol = TOLERANCES[dtype]
- # Reference op -- HF
moe_block = Qwen3MoeSparseMoeBlock(config).to(device, dtype)
- # Torch-native grouped gemm version of MoE Block -- for sanity checking
+ # Torch-native grouped gemm version of the MoE block, for sanity checking.
grouped_gemm_block = Qwen3MoeGroupedGEMMBlock.from_hf(moe_block).to(device, dtype)
grouped_gemm_block.check_weights(moe_block)
@@ -153,7 +152,7 @@ def test_qwen3_moe(
kernel_config_bwd_dW = None
kernel_config_bwd_dX = None
- # Triton kernel grouped gemm version of MoE Block -- this is what we're testing
+ # Triton kernel grouped gemm version of the MoE block, the thing under test.
fused_gemm_block = Qwen3MoeFusedGroupedGEMMBlock.from_hf(
moe_block,
permute_x = permute_x,
@@ -167,7 +166,6 @@ def test_qwen3_moe(
X = torch.randn(bs, seqlen, hidden_size, dtype = dtype, device = device, requires_grad = True)
- # Forward
ref_result = run_forward(moe_block, X, is_grouped_gemm = False)
grouped_result = run_forward(grouped_gemm_block, X, is_grouped_gemm = True)
fused_result = run_forward(fused_gemm_block, X, is_grouped_gemm = True)
@@ -178,15 +176,13 @@ def test_qwen3_moe(
char = "=",
num_chars = 100,
):
- # Sanity checks
-
with annotated_context("Checking HF vs torch grouped gemm MoE forward outputs..."):
check_fwd(ref_result, grouped_result, atol, rtol, verbose = False)
with annotated_context(
"Checking torch grouped gemm MoE vs fused grouped gemm MoE forward outputs..."
):
- # We implement a custom check for grouped gemm results to test each of the intermediate results for easier debugging
+ # Custom check over the grouped gemm intermediates, for easier debugging.
check_grouped_gemm_results(
grouped_result.grouped_gemm_result,
fused_result.grouped_gemm_result,
@@ -195,11 +191,9 @@ def test_qwen3_moe(
rtol = rtol,
verbose = False,
)
- # Actual test
with annotated_context("Checking HF vs fused grouped gemm MoE forward outputs..."):
check_fwd(ref_result, fused_result, atol, rtol, verbose = True)
- # Backward
grad_output = torch.randn_like(ref_result.output)
ref_backward_result = run_backward(
moe_block, grad_output, output = ref_result.output, X = ref_result.X
@@ -220,7 +214,6 @@ def test_qwen3_moe(
char = "=",
num_chars = 100,
):
- # Sanity checks
with annotated_context("Checking HF vs torch grouped gemm MoE grads..."):
check_grads(ref_backward_result, grouped_backward_result, atol, rtol, verbose = False)
with annotated_context(
@@ -234,7 +227,6 @@ def test_qwen3_moe(
verbose = False,
)
- # Actual test
with annotated_context("Checking HF vs fused grouped gemm MoE grads..."):
check_grads(ref_backward_result, fused_backward_result, atol, rtol, verbose = True)
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 5fc867e0cb3..13c8863332d 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -46,10 +46,10 @@ def _rms_layernorm_forward(
r += row_idx * r_row_stride
X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)
- W_row = tl.load(W + col_offsets, mask = mask, other = 0) # .to(tl.float32)
+ W_row = tl.load(W + col_offsets, mask = mask, other = 0)
row_var = tl.sum(X_row * X_row, axis = 0) / n_cols
- # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm
+ # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm.
eps_f32 = tl.full((), eps, tl.float32)
inv_var = tl.math.rsqrt(row_var + eps_f32)
tl.store(r, inv_var)
@@ -70,7 +70,6 @@ def _rms_layernorm_backward(
W_row_stride: tl.constexpr,
r,
r_row_stride: tl.constexpr,
- # dW, dW_row_stride,
n_cols: tl.constexpr,
eps: tl.constexpr,
GEMMA: tl.constexpr,
@@ -134,9 +133,8 @@ def _gemma_rms_layernorm_forward(
eps: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
- # Copies https://github.com/google-deepmind/gemma/blob/main/gemma/layers.py#L31
- # and https://github.com/keras-team/keras-nlp/blob/v0.8.2/keras_nlp/models/gemma/rms_normalization.py#L33
- # exactly. Essentially all in float32!
+ # Copies google-deepmind/gemma layers.py#L31 and keras-nlp gemma/rms_normalization.py#L33 exactly:
+ # essentially all in float32.
row_idx = tl.program_id(0)
col_offsets = tl.arange(0, BLOCK_SIZE)
mask = col_offsets < n_cols
@@ -149,7 +147,7 @@ def _gemma_rms_layernorm_forward(
W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)
row_var = tl.sum(X_row * X_row, axis = 0) / n_cols
- # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm
+ # Explicit float32 scalar to ensure correct type promotion on HIP/ROCm.
eps_f32 = tl.full((), eps, tl.float32)
inv_var = tl.math.rsqrt(row_var + eps_f32)
tl.store(r, inv_var)
@@ -214,7 +212,6 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
n_rows: int
n_cols: int
n_rows, n_cols = dY.shape
- # dW = X
dX = torch.empty_like(dY) if ctx.GEMMA else dY
with torch_gpu_device(dY.device):
@@ -229,7 +226,6 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
W.stride(0),
r,
r.stride(0),
- # dW, dW.stride(0),
n_cols,
ctx.eps,
GEMMA = ctx.GEMMA,
@@ -240,7 +236,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
return dX, None, None, None
-# [TODO] Unsure why RMS Layernorm is not torch.compiling properly
+# RMS Layernorm does not torch.compile properly; reason unknown.
@torch.compiler.disable
def fast_rms_layernorm(
layernorm,
@@ -320,7 +316,6 @@ def test_rms_layernorm(
YY = torch.randn((bsz, seqlen, dim), dtype = dtype, device = "cuda", requires_grad = True)
Y.backward(YY)
correct_grad = X.grad.clone()
- # from unsloth.kernels import fast_rms_layernorm
Y = fast_rms_layernorm(layernorm, XX)
Y.backward(YY)
assert torch.amax(correct_grad - XX.grad).item() <= 0.05
diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py
index 7acab02e8e3..282b2623bf8 100644
--- a/unsloth/kernels/rope_embedding.py
+++ b/unsloth/kernels/rope_embedding.py
@@ -138,19 +138,19 @@ def _rope_embedding(
)
if BACKWARD_PASS:
- # See our blog post for more info.
+ # See the Unsloth blog post for more info.
sin1 = -sin1
# [TODO] Autotune ROPE_GROUP_SIZE to be 1, 2, 4, 8
head_start = group_head_position * ROPE_GROUP_SIZE
head_end = min((head_start + ROPE_GROUP_SIZE), n_heads)
- # 10% Faster kernel from [HuyNguyen-hust](https://github.com/unslothai/unsloth/pull/238)
+ # 10% faster kernel from HuyNguyen-hust, unslothai/unsloth#238.
for k in range(head_start, head_end):
offs_q1 = row_position * Q_row_stride + k * head_dim + col_offsets
offs_q2 = row_position * Q_row_stride + k * head_dim + col_offsets + half_head_dim
- # For Gemma - sometimes RoPE must be done in float32 and not bfloat16
+ # Gemma sometimes needs RoPE done in float32 rather than bfloat16.
Q1 = tl.load(Q + offs_q1, mask = mask, other = 0).to(sin1.dtype)
Q2 = tl.load(Q + offs_q2, mask = mask, other = 0).to(sin1.dtype)
@@ -181,9 +181,9 @@ class Fast_RoPE_Embedding(torch.autograd.Function):
n_rows, n_cols = Q.shape
assert seq_len <= cos.shape[0]
- # [TODO] Changing blocksize to head_dim//2 seems to have
- # some concurrency / un-deterministic issues.
- BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2) # (head_dim//2)
+ # Changing blocksize to head_dim//2 showed concurrency / non-deterministic issues; group_size too
+ # large also hurts performance.
+ BLOCK_SIZE, num_warps = calculate_settings(head_dim // 2)
# group_size = 4 # 4 or 8, too large group_size can hurt performance.
div: int
@@ -261,7 +261,7 @@ class Fast_RoPE_Embedding(torch.autograd.Function):
)
-# [TODO] Unsure why RoPE Embedding is not torch.compiling properly
+# RoPE embedding does not torch.compile properly; reason unknown.
@torch.compiler.disable
def fast_rope_embedding(
Q,
@@ -289,12 +289,12 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
batch, n_heads_Q, seq_len, head_dim = Q.shape
_, n_heads_K, _, _ = K.shape
- # Inplace rotary embedding is generally fine
+ # Inplace rotary embedding is generally fine.
Q_out = Q.clone() if not Q.is_contiguous() else Q
K_out = K.clone() if not K.is_contiguous() else K
if has_indices:
- # TRL's rotary indices are always in int32, so casting is just for safety
+ # TRL's rotary indices are always int32, so the cast is only for safety.
rope_ptr = rope_indices.reshape(-1).to(dtype = torch.int32, device = Q.device)
else:
rope_ptr = cos.new_empty(1, dtype = torch.int32)
@@ -357,7 +357,7 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
rope_ptr = ctx.rope_indices if ctx.has_indices else ctx.cos.new_empty(1, dtype = torch.int32)
- # Inplace rotary embedding is generally fine
+ # Inplace rotary embedding is generally fine.
dQ_out = dQ.clone() if not dQ.is_contiguous() else dQ
dK_out = dK.clone() if not dK.is_contiguous() else dK
@@ -403,19 +403,17 @@ class Slow_RoPE_Embedding(torch.autograd.Function):
@staticmethod
def forward(ctx, Q, cos, sin, position_ids):
if position_ids is not None:
- # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
- cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]
- sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]
- cos = cos[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim]
- sin = sin[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim]
+ # The first two dimensions of cos and sin are always 1, so squeeze them.
+ cos = cos.squeeze(1).squeeze(0)
+ sin = sin.squeeze(1).squeeze(0)
+ cos = cos[position_ids].unsqueeze(2)
+ sin = sin[position_ids].unsqueeze(2)
- # Q * cos + rotate_half(Q) * sin
+ # Q * cos + rotate_half(Q) * sin, with the transposed rotate_half in the backward.
half = Q.shape[-1] // 2
RH_Q = torch.cat((-Q[..., half:], Q[..., :half]), dim = -1)
Q *= cos
Q.addcmul_(RH_Q, sin)
- # RH_Q *= sin
- # Q += RH_Q
ctx.save_for_backward(cos, sin)
return Q
@@ -427,8 +425,6 @@ class Slow_RoPE_Embedding(torch.autograd.Function):
RH_dY = torch.cat((dY[..., half:], -dY[..., :half]), dim = -1)
dY *= cos
dY.addcmul_(RH_dY, sin)
- # RH_dY *= sin
- # dY += RH_dY
return dY, None, None, None
diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py
index 6d74a868fea..213839aeee0 100644
--- a/unsloth/kernels/swiglu.py
+++ b/unsloth/kernels/swiglu.py
@@ -17,7 +17,7 @@ import triton.language as tl
import torch
from .utils import calculate_settings, torch_gpu_device
-# signed int32 max is 2**31-1 so num_elements cannot exceed 2**31
+# signed int32 max is 2**31-1, so num_elements cannot exceed 2**31.
NUM_INT32_ELEMENTS = 2**31
SAFE_INT32_BUFFER_MULTIPLIER = 4
BLOCK_SIZE = 1024
@@ -35,15 +35,14 @@ def _fg_kernel(e, g, h, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDEXING: tl.
mask = offsets < n_elements
e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)
- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)
+ g_row = tl.load(g + offsets, mask = mask, other = 0)
- # f = e * sigmoid(e)
- f_row = e_row * tl.sigmoid(e_row) # e_row / (1 + tl.exp(-e_row))
+ # f = e * sigmoid(e), h = f * g.
+ f_row = e_row * tl.sigmoid(e_row)
f_row = f_row.to(g_row.dtype) # Exact copy from HF
# h = f * g
h_row = f_row * g_row
- # Store h
tl.store(h + offsets, h_row, mask = mask)
@@ -83,13 +82,12 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDE
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
- DW_row = tl.load(DW + offsets, mask = mask, other = 0) # .to(tl.float32)
+ DW_row = tl.load(DW + offsets, mask = mask, other = 0)
e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32)
- g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32)
+ g_row = tl.load(g + offsets, mask = mask, other = 0)
- # e = e.float()
- # se = 1.0 / (1.0 + torch.exp(-e))
- se_row = tl.sigmoid(e_row) # 1.0 / (1.0 + tl.exp(-e_row))
+ # se = sigmoid(e), f = se * e, df = DW * f, dg = DW * g, de = dg * se * (1 + e * (1 - se)).
+ se_row = tl.sigmoid(e_row)
# f = (se * e).to(dtype)
f_row = se_row * e_row
f_row = f_row.to(DW_row.dtype)
@@ -103,10 +101,9 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE: tl.constexpr, LONG_INDE
de_row = dg_row.to(tl.float32) * se_row * (1.0 + e_row * (1.0 - se_row))
de_row = de_row.to(DW_row.dtype)
- # Store derivatives in buffers
- tl.store(DW + offsets, h_row, mask = mask) # h = f * g
- tl.store(e + offsets, df_row, mask = mask) # df = DW * f
- tl.store(g + offsets, de_row, mask = mask) # de
+ tl.store(DW + offsets, h_row, mask = mask)
+ tl.store(e + offsets, df_row, mask = mask)
+ tl.store(g + offsets, de_row, mask = mask)
def swiglu_DWf_DW_dfg_kernel(DW, e, g):
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 839eb9db84e..312baa66b53 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -33,7 +33,7 @@ from ..bnb_availability import native_kernels_ready
from .fp8 import weight_dequant, fp8_linear
import functools
-# torch.cuda.amp.custom_fwd is deprecated >= 2.4
+# torch.cuda.amp.custom_fwd is deprecated from 2.4.
import torch
torch_Tensor = torch.Tensor
@@ -49,12 +49,15 @@ else:
torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "cuda")
torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "cuda")
+# DEVICE_COUNT == 0 = no visible accelerator (e.g. CPU-only CI runner). The consumer functions below only index
+# these arrays during real GPU work, so empty containers are safe -- they just need to be defined so the module
+# imports cleanly.
if DEVICE_TYPE == "xpu":
torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "xpu")
torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "xpu")
-# tl.math.tanh now is libdevice.tanh
+# tl.math.tanh is now libdevice.tanh.
import triton
import triton.language as tl
@@ -68,7 +71,7 @@ if Version(triton.__version__) >= Version("3.0.0"):
else:
triton_tanh = tl.math.tanh
- # No casting in old Triton versions
+ # No casting in old Triton versions.
@triton.jit
def triton_cast(x, dtype):
return x.to(dtype)
@@ -137,15 +140,13 @@ HAS_CUDA_STREAM = False
try:
import bitsandbytes as bnb
- # If an earlier `import bitsandbytes` died inside __init__, CPython evicts only
- # the parent from sys.modules and keeps its submodules, so this retry re-executes
- # __init__ without rebinding `bnb.functional`. `import x.y as z` reads sys.modules
- # directly and survives that, plain attribute access does not.
+ # If an earlier `import bitsandbytes` died inside __init__, CPython evicts only the parent from
+ # sys.modules and keeps its submodules, so this retry re-executes __init__ without rebinding
+ # bnb.functional. `import x.y as z` reads sys.modules directly and survives that.
import bitsandbytes.functional as bnb_functional
except Exception:
- # device_type.py already degrades to 16bit/full finetuning when bnb is missing
- # (e.g. gfx906, whose generic wheel has no kernels). Keep the import working and
- # fail only if a 4bit path is actually entered.
+ # device_type.py already degrades to 16bit/full finetuning when bnb is missing (gfx906, whose
+ # generic wheel has no kernels), so keep the import working and fail only if a 4bit path is entered.
bnb = None
bnb_functional = None
@@ -178,19 +179,17 @@ else:
return nullcontext()
-# INTEL GPU Specific Logic
if DEVICE_TYPE == "xpu":
_gpu_getCurrentRawStream = torch._C._xpu_getCurrentRawStream
elif DEVICE_TYPE == "mlx":
def _gpu_getCurrentRawStream(_index = 0):
return 0
-# NVIDIA GPU Default Logic
elif hasattr(torch._C, "_cuda_getCurrentRawStream"):
_gpu_getCurrentRawStream = torch._C._cuda_getCurrentRawStream
else:
- # CPU-only torch wheel (no compiled CUDA backend). _get_tensor_stream
- # is only invoked during real GPU work, so a no-op binding is safe.
+ # CPU-only torch wheel (no compiled CUDA backend). _get_tensor_stream is only invoked during real
+ # GPU work, so a no-op binding is safe.
def _gpu_getCurrentRawStream(_index = 0):
return 0
@@ -202,16 +201,12 @@ def _get_tensor_stream(tensor: torch_Tensor) -> c_void_p:
return c_void_p(_gpu_getCurrentRawStream(tensor.device.index))
-# Get array of CUDA streams and other buffers
global CUDA_STREAMS
global XPU_STREAMS
global WEIGHT_BUFFERS
global ABSMAX_BUFFERS
-# DEVICE_COUNT == 0 = no visible accelerator (e.g. CPU-only CI runner).
-# The consumer functions below only index these arrays during real GPU
-# work, so empty containers are safe -- they just need to be defined so
-# the module imports cleanly.
+# DEVICE_COUNT == 0 means no visible accelerator (CPU-only CI runner).
if DEVICE_TYPE == "xpu":
if DEVICE_COUNT > 0:
_XPU_STREAMS = {
@@ -237,7 +232,6 @@ elif DEVICE_TYPE == "mlx":
WEIGHT_BUFFERS = []
ABSMAX_BUFFERS = []
else:
- # NVIDIA GPU Default Logic
if DEVICE_COUNT > 0:
_CUDA_STREAMS = {
(index := torch.cuda.device(i).idx): ctypes.c_void_p(
@@ -257,12 +251,11 @@ else:
WEIGHT_BUFFERS = []
ABSMAX_BUFFERS = []
-# Bitsandbytes operations
ctypes_c_int = ctypes.c_int
ctypes_c_int32 = ctypes.c_int32
-# Same verdict device_type.py used to clear ALLOW_BITSANDBYTES, applied to the binds
-# themselves. 0.45.5 leaves `functional.lib = None` when the native library fails to
-# load, so these lookups would kill `import unsloth` instead of degrading to 16bit.
+# Same verdict device_type.py used to clear ALLOW_BITSANDBYTES, applied to the binds themselves:
+# 0.45.5 leaves functional.lib = None when the native library fails to load, so these lookups
+# would kill `import unsloth` instead of degrading to 16bit.
if bnb is None or not native_kernels_ready(bnb, DEVICE_TYPE):
cdequantize_blockwise_fp32 = _bnb_required
cdequantize_blockwise_fp16_nf4 = _bnb_required
@@ -275,8 +268,7 @@ else:
cdequantize_blockwise_bf16_nf4 = bnb_functional.lib.cdequantize_blockwise_bf16_nf4
if DEVICE_TYPE == "xpu":
- # https://github.com/bitsandbytes-foundation/bitsandbytes/blob/c3b8de268fdb55a88f92feada23fc811a1e6877a/bitsandbytes/backends/xpu/ops.py#L115
- # for xpu, inference gemv using above link
+ # xpu inference gemv, per bitsandbytes backends/xpu/ops.py#L115.
cgemm_4bit_inference_naive_fp16 = bnb_functional.lib.cgemv_4bit_inference_fp16
cgemm_4bit_inference_naive_bf16 = bnb_functional.lib.cgemv_4bit_inference_bf16
else:
@@ -298,7 +290,6 @@ torch_float16 = torch.float16
torch_bfloat16 = torch.bfloat16
-# Check whether torchao can be imported to get Float8Tensor
if importlib.util.find_spec("torchao") is not None:
try:
from torchao.quantization import Float8Tensor
@@ -317,11 +308,10 @@ def QUANT_STATE(W):
return getattr(W, "quant_state", None)
-# fp8 weight dtypes. A `weight_scale` / `weight_scale_inv` should only be treated as a
-# quant state when the weight itself is still fp8. compressed-tensors layers expose an
-# already-dequantized bf16 weight at forward time while keeping a `weight_scale` around;
-# reading that as a quant state routes a bf16 weight into the bitsandbytes fast_gemv /
-# fast_dequantize path, which then reads a missing `absmax` and crashes.
+# fp8 weight dtypes: a weight_scale / weight_scale_inv is only a quant state while the weight
+# itself is still fp8. compressed-tensors layers expose an already-dequantized bf16 weight at
+# forward time while keeping a weight_scale, and reading that as a quant state routes it into
+# fast_gemv / fast_dequantize, which then reads a missing absmax and crashes.
_FP8_WEIGHT_DTYPES = tuple(
dtype
for dtype in (
@@ -336,21 +326,18 @@ def get_lora_parameters(proj):
"""Return (weight, weight quant_state, lora A, lora B, lora scale).
With QAT enabled, also fake-quantizes the base layer and lora weights.
"""
- # For DPO or disabled adapters
- base_layer = getattr(
- proj, "base_layer", proj
- ) # (proj.base_layer if hasattr(proj, "base_layer") else proj)
+ # For DPO or disabled adapters.
+ base_layer = getattr(proj, "base_layer", proj)
W = base_layer.weight
- # Optionally apply fake quantization to base layer weights for QAT
+ # Optionally apply fake quantization to base layer weights for QAT.
if hasattr(base_layer, "weight_fake_quantizer"):
weight_fake_quantizer = getattr(base_layer, "weight_fake_quantizer", None)
if weight_fake_quantizer is not None:
W = weight_fake_quantizer(W)
- # Get quant state for 4bit or FP8. Only fall back to a weight_scale(_inv) when the
- # weight is still fp8; a bf16 weight (e.g. a decompressed compressed-tensors layer)
- # must not carry a scale as its quant state or fast_gemv will crash on it.
+ # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed
+ # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.
W_quant = getattr(W, "quant_state", None)
if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:
W_quant = getattr(base_layer, "weight_scale_inv", None)
@@ -358,14 +345,12 @@ def get_lora_parameters(proj):
W_quant = getattr(base_layer, "weight_scale", None)
if getattr(base_layer, "quant_method", None) == "fp8":
- # we need to somehow store and pass this information :)
W.block_size = getattr(base_layer, "block_size", [128, 128])
- # A decompressed compressed-tensors layer keeps quant_method == "fp8" while its
- # weight is back to bf16, so it has no quant state to carry the block size.
+ # A decompressed compressed-tensors layer keeps quant_method == "fp8" while its weight is back to
+ # bf16, so it has no quant state to carry the block size.
if W_quant is not None:
W_quant.block_size = W.block_size
- # if not hasattr(proj, "disable_adapters") or proj.disable_adapters or proj.merged:
if getattr(proj, "disable_adapters", True) or proj.merged:
return W, W_quant, None, None, None
@@ -374,7 +359,7 @@ def get_lora_parameters(proj):
adapter = getattr(proj, "active_adapter", ("default"))
adapter = adapter[0]
- # Optionally apply fake quantization to lora weights for QAT
+ # Optionally apply fake quantization to lora weights for QAT.
lora_A_linear = proj.lora_A[adapter]
lora_B_linear = proj.lora_B[adapter]
A = lora_A_linear.weight
@@ -398,15 +383,12 @@ def get_lora_parameters(proj):
def get_lora_parameters_bias(proj):
- # For DPO or disabled adapters
- base_layer = getattr(
- proj, "base_layer", proj
- ) # (proj.base_layer if hasattr(proj, "base_layer") else proj)
+ # For DPO or disabled adapters.
+ base_layer = getattr(proj, "base_layer", proj)
W = base_layer.weight
- # Get quant state for 4bit or FP8. Only fall back to a weight_scale(_inv) when the
- # weight is still fp8; a bf16 weight (e.g. a decompressed compressed-tensors layer)
- # must not carry a scale as its quant state or fast_gemv will crash on it.
+ # Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed
+ # compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.
W_quant = getattr(W, "quant_state", None)
if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES:
W_quant = getattr(base_layer, "weight_scale_inv", None)
@@ -414,14 +396,12 @@ def get_lora_parameters_bias(proj):
W_quant = getattr(base_layer, "weight_scale", None)
if getattr(base_layer, "quant_method", None) == "fp8":
- # we need to somehow store and pass this information :)
W.block_size = getattr(base_layer, "block_size", [128, 128])
- # A decompressed compressed-tensors layer keeps quant_method == "fp8" while its
- # weight is back to bf16, so it has no quant state to carry the block size.
+ # A decompressed compressed-tensors layer keeps quant_method == "fp8" while its weight is back to
+ # bf16, so it has no quant state to carry the block size.
if W_quant is not None:
W_quant.block_size = W.block_size
- # if not hasattr(proj, "disable_adapters") or proj.disable_adapters or proj.merged:
if getattr(proj, "disable_adapters", True) or proj.merged:
return W, W_quant, None, None, None, base_layer.bias
@@ -451,7 +431,6 @@ def _maybe_fake_quantize_activations(X: torch.Tensor, proj: torch.nn.Module) ->
return X
-# INTEL GPU Specific Logic
if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
@torch.inference_mode
@@ -469,8 +448,7 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
if W.dtype == torch.float8_e4m3fn:
return weight_dequant(W, quant_state)
if type(quant_state) is not list:
- # New quant_state as a class
- # https://github.com/TimDettmers/bitsandbytes/pull/763/files
+ # New quant_state as a class, per TimDettmers/bitsandbytes#763.
absmax = quant_state.absmax
shape = quant_state.shape
dtype = quant_state.dtype
@@ -491,7 +469,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
XPU_STREAM = XPU_STREAMS[device_index]
n_elements_absmax = absmax.numel()
- # Create weight matrix
if use_global_buffer:
# Use same buffers for faster inference
size = shape[0] * shape[1]
@@ -530,7 +507,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
requires_grad = False,
)
- # NF4 dequantization of statistics
ptr_out_absmax = get_ptr(out_absmax)
with torch_gpu_device(device):
cdequantize_blockwise_fp32(
@@ -544,7 +520,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
)
out_absmax += offset
- # Dequantize W
fx = (
cdequantize_blockwise_fp16_nf4
if dtype == torch_float16
@@ -559,11 +534,10 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
ctypes_c_int(out.numel()),
XPU_STREAM,
)
- # Careful returning transposed data
+ # Careful returning transposed data.
is_transposed = True if W.shape[0] == 1 else False
return out.t() if is_transposed else out
-# NVIDIA GPU Default Logic
elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
@torch.inference_mode
@@ -580,8 +554,7 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
if W.dtype == torch.float8_e4m3fn:
return weight_dequant(W, quant_state)
if type(quant_state) is not list:
- # New quant_state as a class
- # https://github.com/TimDettmers/bitsandbytes/pull/763/files
+ # New quant_state as a class, per TimDettmers/bitsandbytes#763.
absmax = quant_state.absmax
shape = quant_state.shape
dtype = quant_state.dtype
@@ -604,9 +577,7 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
n_elements_absmax = absmax.numel()
- # Create weight matrix
if use_global_buffer:
- # Use same buffers for faster inference
size = shape[0] * shape[1]
global WEIGHT_BUFFERS
global ABSMAX_BUFFERS
@@ -644,7 +615,6 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
)
pass
- # NF4 dequantization of statistics
ptr_out_absmax = get_ptr(out_absmax)
with torch_gpu_device(device):
cdequantize_blockwise_fp32(
@@ -658,7 +628,6 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
)
out_absmax += offset
- # Dequantize W
fx = (
cdequantize_blockwise_fp16_nf4
if dtype == torch_float16
@@ -674,7 +643,7 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
CUDA_STREAM,
)
pass
- # Careful returning transposed data
+ # Careful returning transposed data.
is_transposed = True if W.shape[0] == 1 else False
return out.t() if is_transposed else out
@@ -695,7 +664,7 @@ else:
if W.dtype == torch.float8_e4m3fn:
return weight_dequant(W, quant_state)
if type(quant_state) is not list:
- # New quant_state as a class
+ # New quant_state as a class, per TimDettmers/bitsandbytes#763.
# https://github.com/TimDettmers/bitsandbytes/pull/763/files
absmax = quant_state.absmax
shape = quant_state.shape
@@ -707,7 +676,6 @@ else:
code2 = state2.code
blocksize2 = state2.blocksize
else:
- # Old quant_state as a list of lists
absmax, shape, dtype, blocksize, compressed_stats, _, _ = quant_state
offset, state2 = compressed_stats
absmax2, code2, blocksize2, _, _, _, _ = state2
@@ -716,7 +684,6 @@ else:
n_elements_absmax = absmax.numel()
device = W.device
- # Create weight matrix
if out is None:
out = torch_empty(shape, dtype = dtype, device = device, requires_grad = False)
else:
@@ -726,7 +693,6 @@ else:
n_elements_absmax, dtype = torch_float32, device = device, requires_grad = False
)
- # Do dequantization
ptr_out_absmax = get_ptr(out_absmax)
cdequantize_blockwise_fp32(
get_ptr(code2),
@@ -752,14 +718,13 @@ else:
ctypes_c_int(out.numel()),
)
- # Careful returning transposed data
+ # Careful returning transposed data.
is_transposed = True if W.shape[0] == 1 else False
return out.t() if is_transposed else out
pass
-# INTEL GPU Specific Logic
if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
def fast_gemv(
@@ -770,13 +735,10 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
):
if quant_state is None:
return torch_matmul(X, W, out = out)
- # For fast X @ W where seq_len == 1
- # From https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L1469
+ # Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763.
_, q_len, hd = X.shape
- # assert(q_len == 1)
if type(quant_state) is not list:
- # https://github.com/TimDettmers/bitsandbytes/pull/763/files
absmax = quant_state.absmax
shape = quant_state.shape
dtype = quant_state.dtype
@@ -796,7 +758,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
device_index = device.index
XPU_STREAM = XPU_STREAMS[device_index]
- # assert(dtype == X.dtype)
bout = shape[0]
if out is None:
@@ -809,9 +770,6 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
dtype = dtype,
device = device,
)
- # else:
- # assert(out.shape == (1, 1, bout,))
- # pass
if DEVICE_TYPE == "xpu":
m = 1
@@ -879,13 +837,10 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
):
if quant_state is None:
return torch_matmul(X, W, out = out)
- # For fast X @ W where seq_len == 1
- # From https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L1469
+ # Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763.
_, q_len, hd = X.shape
- # assert(q_len == 1)
if type(quant_state) is not list:
- # https://github.com/TimDettmers/bitsandbytes/pull/763/files
absmax = quant_state.absmax
shape = quant_state.shape
dtype = quant_state.dtype
@@ -906,7 +861,6 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
device_index = device.index
CUDA_STREAM = CUDA_STREAMS[device_index]
- # assert(dtype == X.dtype)
bout = shape[0]
if out is None:
@@ -919,9 +873,6 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
dtype = dtype,
device = device,
)
- # else:
- # assert(out.shape == (1, 1, bout,))
- # pass
n = 1
m = shape[0]
@@ -987,13 +938,10 @@ else:
):
if quant_state is None:
return torch_matmul(X, W, out = out)
- # For fast X @ W where seq_len == 1
- # From https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L1469
+ # Fast X @ W where seq_len == 1, from bitsandbytes functional.py#L1469 and TimDettmers/bitsandbytes#763.
_, q_len, hd = X.shape
- # assert(q_len == 1)
if type(quant_state) is not list:
- # https://github.com/TimDettmers/bitsandbytes/pull/763/files
absmax = quant_state.absmax
shape = quant_state.shape
dtype = quant_state.dtype
@@ -1009,7 +957,6 @@ else:
offset, state2 = compressed_stats
absmax2, code2, blocksize2, _, _, _, _ = state2
pass
- # assert(dtype == X.dtype)
bout = shape[0]
device = W.device
@@ -1023,9 +970,6 @@ else:
dtype = dtype,
device = device,
)
- # else:
- # assert(out.shape == (1, 1, bout,))
- # pass
n = 1
m = shape[0]
@@ -1100,7 +1044,6 @@ def fast_linear_forward(
W = fast_dequantize(W.t(), W_quant, use_global_buffer = True)
out = torch_matmul(X, W, out = out)
- # Add in LoRA weights
if lora_A is not None:
out_dim = out.shape[2]
dtype = X.dtype
@@ -1146,8 +1089,7 @@ def matmul_lora(
if isinstance(W, Float8Tensor):
assert W.ndim == 2
if W.block_size[0] == W.shape[0] and W.block_size[1] == 1:
- # Rowwise scaling becomes colwise after transpose, so this detects
- # the backward pass. TODO: avoid calling matmul_lora in backward.
+ # Rowwise scaling becomes colwise after transpose, so this detects the backward pass.
W = W.dequantize()
else:
W = W.contiguous()
@@ -1161,10 +1103,8 @@ def matmul_lora(
del W
if A is not None:
- # LoRA is enabled
A, B = A.t(), B.t()
XA = torch_matmul(X, A.to(dtype))
out.addmm_(XA, B.to(dtype), alpha = s)
- # out += (X @ A.to(dtype)) @ (s * B.to(dtype))
return out.view(batch, seq_len, -1) if reshape else out
diff --git a/unsloth/models/_attn_mask_compat.py b/unsloth/models/_attn_mask_compat.py
index df6d2c7e70b..b38bab35d7e 100644
--- a/unsloth/models/_attn_mask_compat.py
+++ b/unsloth/models/_attn_mask_compat.py
@@ -36,13 +36,11 @@ import torch
from transformers.utils.import_utils import is_torchdynamo_compiling
try:
- # `is_tracing` exists only from Transformers 5.0.0, so the fallback below is
- # the live path for all of 4.x, not just the 4.51.3 floor. The fallback
- # mirrors what 4.x `modeling_attn_mask_utils` computes inline
- # (`torch.jit.is_tracing() or isinstance(x, torch.fx.Proxy) or
- # is_torchdynamo_compiling()`), so the data-dependent `torch.all(...)`
- # branches stay skipped under JIT / FX / Dynamo and SDPA path selection is
- # unchanged.
+ # `is_tracing` exists only from Transformers 5.0.0, so the fallback below is the live path for all
+ # of 4.x, not just the 4.51.3 floor. It mirrors what 4.x modeling_attn_mask_utils computes inline
+ # (torch.jit.is_tracing() or isinstance(x, torch.fx.Proxy) or is_torchdynamo_compiling()), so the
+ # data-dependent torch.all(...) branches stay skipped under JIT / FX / Dynamo and SDPA path
+ # selection is unchanged.
from transformers.utils.import_utils import is_tracing # type: ignore[attr-defined]
except ImportError:
@@ -188,9 +186,9 @@ class AttentionMaskConverter:
expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
- # 0-dim tensor, not a Python float: a float literal lowers to an fp32
- # scalar and breaks ExecuTorch edge-dialect export on fp16/bf16 masks.
- # Matches upstream from 4.53.0 (huggingface/transformers#38637).
+ # 0-dim tensor, not a Python float: a float literal lowers to an fp32 scalar and breaks ExecuTorch
+ # edge-dialect export on fp16/bf16 masks. Matches upstream from 4.53.0
+ # (huggingface/transformers#38637).
inverted_mask = torch.tensor(1.0, dtype = dtype) - expanded_mask
return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
@@ -277,13 +275,10 @@ def _prepare_4d_causal_attention_mask_for_sdpa(
key_value_length = key_value_length,
)
- # Attend to all tokens in masked rows, e.g. the first rows under left
- # padding. Required by F.scaled_dot_product_attention's memory-efficient
- # path: https://github.com/pytorch/pytorch/issues/110213
- # Scoped to this branch to match upstream. At function scope it also runs
- # on the `attention_mask is None` path, where it is a no-op numerically
- # but materialises `to_causal_4d`'s stride-0 view into a dense
- # [bsz, 1, q, kv] tensor.
+ # Attend to all tokens in masked rows (the first rows under left padding), required by
+ # F.scaled_dot_product_attention's memory-efficient path (pytorch/pytorch#110213). Scoped to this
+ # branch to match upstream: at function scope it also runs on the attention_mask is None path,
+ # where it materialises to_causal_4d's stride-0 view into a dense [bsz, 1, q, kv] tensor.
if not is_tracing_ and expanded_4d_mask.device.type in ["cuda", "xpu"]:
expanded_4d_mask = AttentionMaskConverter._unmask_unattended(
expanded_4d_mask, min_dtype = torch.finfo(inputs_embeds.dtype).min
diff --git a/unsloth/models/_custom_dtype.py b/unsloth/models/_custom_dtype.py
index 5df5b719efe..0deb02ce82f 100644
--- a/unsloth/models/_custom_dtype.py
+++ b/unsloth/models/_custom_dtype.py
@@ -34,8 +34,7 @@ import os
import torch
-# A table instead of `eval`, so the field NAMES a dtype rather than being an
-# arbitrary expression.
+# A table instead of eval, so the field NAMES a dtype rather than being an arbitrary expression.
DTYPE_ALIASES = {
"None": None,
"none": None,
@@ -53,9 +52,8 @@ DTYPE_ALIASES = {
"fp32": torch.float32,
}
-# `unsloth_zoo==2026.8.15`, which this package's floor resolves to, still `eval`s the
-# dtype field, and `eval("fp16")` is a NameError. So a field this package accepts is
-# canonicalised to the one spelling both readers evaluate.
+# unsloth_zoo==2026.8.15, which this package's floor resolves to, still evals the dtype field, and
+# eval("fp16") is a NameError, so canonicalise to the one spelling both readers evaluate.
_CANONICAL_DTYPE_NAMES = {
None: "None",
torch.float16: "torch.float16",
diff --git a/unsloth/models/_uma_safetensors.py b/unsloth/models/_uma_safetensors.py
index 38d8b7d33a5..f53c934383d 100644
--- a/unsloth/models/_uma_safetensors.py
+++ b/unsloth/models/_uma_safetensors.py
@@ -92,10 +92,7 @@ def patch_unified_memory_safetensors_load():
return True
def _clone_move(tensor, device):
- # Clone into a regular CPU allocation to restore fast pinned-DMA, then
- # move. The clone transiently doubles the tensor's CPU footprint and can
- # OOM a low-memory UMA box; fall back to the direct, allocation-free move
- # (a genuine non-memory error re-raises identically from it).
+ # Clone into a regular CPU allocation to restore fast pinned-DMA, then move.
try:
return tensor.clone().to(device, non_blocking = False)
except (MemoryError, RuntimeError):
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 34883a18c31..f8b6b1ffa6f 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -45,8 +45,6 @@ __all__ = [
"create_boolean_mask",
"torch_amp_custom_fwd",
"torch_amp_custom_bwd",
- # "accelerate_old_send_to_device",
- # "accelerate_new_send_to_device",
"patch_gradient_accumulation_fix",
"apply_accepts_loss_kwargs_fix",
"patch_compiling_bitsandbytes",
@@ -274,12 +272,11 @@ def _unsloth_install_pretrain_detector(model):
return model
marker = getattr(model, "_unsloth_pretrain_marker", None)
if isinstance(marker, dict):
- # A live hook is already recording: keep it (no duplicates) and DON'T clear seen -- a
- # grad-enabled probe may have already flagged the poisoned cache, and a re-entrant
- # get_peft_model/patch_peft_model call must not erase that before train() resets.
+ # A live hook is already recording: keep it and do NOT clear seen, since a grad-enabled probe
+ # may already have flagged the poisoned cache and a re-entrant call must not erase that.
if "hook" in marker:
return model
- # Marker exists but its hook was torn down -> reinstall fresh, so reset seen.
+ # Marker exists but its hook was torn down, so reinstall fresh and reset seen.
marker["seen"] = False
else:
marker = {"seen": False}
@@ -289,8 +286,8 @@ def _unsloth_install_pretrain_detector(model):
return model
def _mark(_module, _inp):
- # Only a grad-enabled forward poisons the AOTAutograd backward-graph cache; a no-grad
- # probe builds no backward graph, so treat it as clean (avoids a needless dynamo reset).
+ # Only a grad-enabled forward poisons the AOTAutograd backward-graph cache; a no-grad probe
+ # builds no backward graph, so treat it as clean.
if torch.is_grad_enabled():
marker["seen"] = True
@@ -302,20 +299,17 @@ def _unsloth_install_pretrain_detector(model):
def _unsloth_reset_stray_compile_cache(self):
- # A manual forward/backward under torch.compile BEFORE trainer.train() (e.g. a grad-norm
- # probe) caches a forward + AOTAutograd backward graph in a one-off context; reusing it
- # poisons training with NaN/zero gradients. If such a forward was seen and compile is on,
- # drop the compiled-graph cache so training recompiles cleanly. No-op on the normal path.
- # Module-level (not just inside the RL trainer template) so the SFT auto-packing wrapper and
- # the plain-Trainer loop can import and run it too.
+ # A manual forward/backward under torch.compile BEFORE trainer.train() caches a forward plus
+ # AOTAutograd backward graph in a one-off context, and reusing it poisons training with
+ # NaN/zero gradients, so drop the cache. Module-level, so the SFT auto-packing wrapper and the
+ # plain-Trainer loop can run it too.
import os
model = getattr(self, "model", None)
if model is None:
return
# The detector hook can sit on any wrapper in the chain, and the probe may have run on a
- # different one than self.model, so walk the chain: detect a "seen" marker anywhere and
- # collect every marker to tear down below.
+ # different one than self.model, so walk the chain.
markers = []
seen = False
_curr = model
@@ -384,7 +378,7 @@ def apply_unsloth_gradient_checkpointing(use_gradient_checkpointing, max_seq_len
The effective use_gradient_checkpointing value (may change from "unsloth" to True)
"""
if use_gradient_checkpointing == "unsloth":
- # Offloading not worth it below ~512; standard gc is faster (crossover ~384-512).
+ # Offloading is not worth it below ~512; standard gc is faster (crossover ~384-512).
if max_seq_length < 512:
unpatch_unsloth_smart_gradient_checkpointing()
return True
@@ -392,39 +386,29 @@ def apply_unsloth_gradient_checkpointing(use_gradient_checkpointing, max_seq_len
patch_unsloth_smart_gradient_checkpointing(dtype = dtype)
return "unsloth"
elif use_gradient_checkpointing in (True, False):
- # User explicitly set True or False - unpatch any previous "unsloth" patching
+ # User explicitly set True or False, so unpatch any previous "unsloth" patching.
unpatch_unsloth_smart_gradient_checkpointing()
return use_gradient_checkpointing
return use_gradient_checkpointing
-# Models that don't work with flex_attention as the global Transformers
-# attention implementation:
-# GPT-OSS: training uses the custom flex sink patch, but inference intentionally
-# falls back to eager because flex decoding gives incorrect outputs.
-# Mllama: BlockMask Q_LEN!=KV_LEN ValueError on decode.
-# NemotronH: hybrid Mamba-2 + Transformer, raises NotImplementedError.
-# Gemma3N: timm vision wrappers don't support flex_attention.
-# ModernBERT: create_block_mask with _compile=True hits CUDA illegal memory
-# access on some GPU architectures (B200). Falls back to eager safely.
+# Models that break with flex_attention as the global attention implementation: GPT-OSS
+# inference (wrong decode outputs), Mllama (BlockMask Q_LEN != KV_LEN on decode), NemotronH
+# (hybrid Mamba-2), Gemma3N (timm vision wrappers), ModernBERT (CUDA illegal access on B200).
_FLEX_EXCLUDED_MODELS = ("gpt_oss", "mllama", "nemotron_h", "modernbert")
_FLEX_PREFERRED_MODELS = ("gemma3", "gemma3_text", "shieldgemma2")
_SDPA_EXCLUDED_MODELS = ("gpt_oss", "deepseek_v4")
-# The loader (loader.py) forces supports_sdpa=False for these because their bundled
-# SDPA modules are wrong. Kept here, not in loader.py, so _is_sdpa_excluded can honor
-# them without a loader -> _utils import cycle (loader.py already imports from _utils
-# and re-exports this name for callers like sentence_transformer.py). Entries are matched
-# as substrings against a comma-joined model_types string ending in a comma, so "gemma3,"
-# matches a distinct "gemma3" entry but not "gemma3n", and "gemma3_text" matches the
-# EmbeddingGemma text model.
+# The loader forces supports_sdpa=False for these, whose bundled SDPA modules are wrong; kept
+# here so _is_sdpa_excluded can honor them without a loader -> _utils cycle. Matched as
+# substrings of a comma-joined string, so "gemma3," matches gemma3 but not gemma3n.
DISABLE_SDPA_MODEL_NAMES = [
- "gemma3,", # Add comma bc gemma3 will match gemma3n
+ "gemma3,",
"gemma3_text", # Gemma3TextModel (EmbeddingGemma) - substring match, keep underscore
"gpt_oss",
]
_FLASH_EXCLUDED_MODELS = ("gpt_oss", "deepseek_v4")
-# deepseek_v4's custom attention is sdpa/flash-incompatible; force eager, and
-# excluded above so an explicit sdpa/flash request cannot re-enable the crash.
+# deepseek_v4's custom attention is sdpa/flash-incompatible: force eager, and it is excluded
+# above so an explicit request cannot re-enable the crash.
_EAGER_ONLY_PREFIXES = ("gemma3n", "deepseek_v4")
_FLASH_ATTENTION_MAX_HEAD_DIM = 256
_FLASH_ATTENTION_DISABLED_WARNED = set()
@@ -435,20 +419,16 @@ def _is_flex_excluded(model_type):
def _is_sdpa_disabled_by_name(model_type):
- # Mirror the loader's DISABLE_SDPA_MODEL_NAMES check: loader.py builds
- # model_types_all = ",".join(model_types) + "," and tests `name in model_types_all`.
- # Rebuild the same trailing-comma form for a single model_type so the match is
- # identical (e.g. "gemma3," matches "gemma3" but not "gemma3n", and "gemma3_text"
- # still matches "gemma3_text").
+ # Mirror the loader's DISABLE_SDPA_MODEL_NAMES check, which joins model_types with a trailing
+ # comma, so a single model_type matches identically.
model_types_all = model_type.lower() + ","
return any(name.lower() in model_types_all for name in DISABLE_SDPA_MODEL_NAMES)
def _is_sdpa_excluded(model_type):
- # SDPA is known-broken for these models, so an explicit sdpa request must not
- # re-enable it. Two sources: _SDPA_EXCLUDED_MODELS (resolver-level, e.g. gpt_oss)
- # and DISABLE_SDPA_MODEL_NAMES (loader-level, e.g. gemma3 / gemma3_text, which the
- # loader also forces to supports_sdpa=False).
+ # SDPA is known-broken for these, so an explicit sdpa request must not re-enable it. Two sources:
+ # _SDPA_EXCLUDED_MODELS (resolver, e.g. gpt_oss) and DISABLE_SDPA_MODEL_NAMES (loader, e.g.
+ # gemma3 / gemma3_text).
lowered = model_type.lower()
return lowered in _SDPA_EXCLUDED_MODELS or _is_sdpa_disabled_by_name(lowered)
@@ -468,10 +448,9 @@ def _is_eager_only(model_type):
return any(model_type.startswith(p) for p in _EAGER_ONLY_PREFIXES)
-# Ampere. Below it the flex HOP runs its eager `sdpa_dense` fallback, whose
-# backward does `softmax_scores.to(query.dtype) @ grad_out` -- casting the scores
-# but not `grad_out`. Such a card also forces fp16 here, so query is Half against
-# a Float grad and the matmul is refused.
+# Ampere. Below it the flex HOP runs its eager sdpa_dense fallback, whose backward casts
+# softmax_scores but not grad_out; such a card also forces fp16, so a Half query meets a Float
+# grad and the matmul is refused.
_FLEX_ATTENTION_MIN_CAPABILITY = (8, 0)
@@ -532,12 +511,10 @@ def _config_get(
return config.get(field_name, default)
return getattr(config, field_name, default)
except Exception:
- # transformers 5.x heterogeneous configs (Gemma 3n / Gemma 4, anything
- # with `per_layer_config`) raise AmbiguousGlobalPerLayerAttributeError,
- # not AttributeError, on a global read of a per-layer field, so a getattr
- # default does not cover it. This killed model load on transformers
- # 5.15.0. Not caught by name: the class does not exist on 4.x, and a
- # config is third-party code that may raise anything.
+ # transformers 5.x heterogeneous configs (Gemma 3n / Gemma 4, anything with per_layer_config)
+ # raise AmbiguousGlobalPerLayerAttributeError, not AttributeError, on a global read of a
+ # per-layer field, so a getattr default does not cover it; this killed model load on 5.15.0.
+ # Not caught by name: the class does not exist on 4.x.
return default
@@ -546,19 +523,16 @@ def _get_per_layer_values(config, field_name):
per_layer = _config_get(config, "per_layer_config", None)
if per_layer is None or isinstance(per_layer, (str, bytes)):
return []
- # Three shapes: a live `_PerLayerConfigView` (a Sequence, not a list/tuple);
- # `to_dict`/config.json, a mapping of zero-padded layer index to overrides
- # like `{"04": {"head_dim": 512}}`; and Unsloth's SimpleNamespace wrap of it.
- # Iterating a mapping walks the indices, not the overrides, so the probe
- # would answer 256 where the object form answers 512.
+ # Three shapes: a live _PerLayerConfigView (a Sequence), the to_dict/config.json mapping of
+ # zero-padded layer index to overrides, and Unsloth's SimpleNamespace wrap. Iterating a mapping
+ # walks the indices, so the probe would answer 256 where the object form answers 512.
if isinstance(per_layer, dict):
per_layer = list(per_layer.values())
else:
try:
iter(per_layer)
except TypeError:
- # Namespace form. `iter` not `__iter__`: the view may be an
- # old-style sequence with only `__getitem__`.
+ # `iter`, not `__iter__`: the view may be an old-style sequence with only __getitem__.
per_layer = list(vars(per_layer).values()) if hasattr(per_layer, "__dict__") else []
values = []
try:
@@ -627,9 +601,8 @@ def _collect_attention_head_dims(config):
"local_head_dim",
"kv_head_dim",
):
- # Per-layer first: on a heterogeneous config the global read refuses,
- # and no head dim reads as "no reason to disable Flash Attention" on
- # exactly the models whose layers may exceed its ceiling.
+ # Per-layer first: on a heterogeneous config the global read refuses, and no head dim reads as
+ # "no reason to disable Flash Attention" on exactly the models that may exceed its ceiling.
candidates = _get_per_layer_values(config, field_name)
candidates.append(_config_get(config, field_name, None))
for value in candidates:
@@ -699,14 +672,12 @@ def _disable_flash_attention_if_needed(
if disable_reason is None:
return attn_implementation
- # Only an implementation passed by the caller counts as an explicit request.
- # Values read from the config are synthesized by the loaders (the language path
- # seeds the config with attn_implementation="sdpa") or come from Transformers
- # defaults, so they must not be treated as a deliberate user choice.
+ # Only an implementation passed by the caller is an explicit request: config values are
+ # synthesized by the loaders or come from Transformers defaults.
explicit_request = attn_implementation
- # Off for a float32 load: with no flash-specific reason the config never steered the
- # choice, so a config-seeded "eager" must not drag an fp32 load from sdpa down to eager.
+ # Off for a float32 load: with no flash-specific reason the config never steered the choice, so
+ # a config-seeded "eager" must not drag an fp32 load from sdpa down to eager.
requested_attn_implementation = attn_implementation
if honor_config_attn_implementation:
if requested_attn_implementation is None:
@@ -719,13 +690,9 @@ def _disable_flash_attention_if_needed(
model_type = _config_get(config, "model_type", "")
- # The disable reason is flash-specific: honor an explicit non-flash request from
- # the caller instead of downgrading it. SDPA is honored unless the model's SDPA is
- # known-broken - _SDPA_EXCLUDED_MODELS (e.g. gpt_oss) or DISABLE_SDPA_MODEL_NAMES
- # (e.g. gemma3 / gemma3_text); flex_attention
- # is honored only when it is actually usable, since supports_flex_attention already
- # rejects the excluded/broken/unavailable configs. This keeps an explicit request
- # from selecting a backend the repo marks as wrong.
+ # The disable reason is flash-specific, so honor an explicit non-flash request. SDPA is honored
+ # unless known-broken, flex only when supports_flex_attention accepts it, so an explicit
+ # request cannot select a backend the repo marks as wrong.
if explicit_request == "sdpa" and not _is_sdpa_excluded(model_type.lower()):
return _set_attn_impl(config, "sdpa")
if explicit_request == "flex_attention" and supports_flex_attention:
@@ -796,13 +763,13 @@ def resolve_model_class(auto_model, config):
def _is_family_text_decoder(parent_model_type, text_model_type):
- # True only for the family's own text variant (gemma3 -> gemma3_text); a generic
- # reused decoder (llava -> llama) would load random weights, so keep the full model.
+ # True only for the family's own text variant (gemma3 -> gemma3_text); a generic reused decoder
+ # (llava -> llama) would load random weights, so keep the full model.
return bool(parent_model_type) and str(text_model_type).startswith(parent_model_type)
def _get_text_only_config(model_config, model_name):
- # Text sub-config of a vision-language config so FastLanguageModel skips the vision tower (PR #5816).
+ # Text sub-config of a vision-language config so FastLanguageModel skips the vision tower (#5816).
text_config = None
if hasattr(model_config, "get_text_config"):
text_config = model_config.get_text_config()
@@ -810,7 +777,7 @@ def _get_text_only_config(model_config, model_name):
text_config = getattr(model_config, "text_config", None)
if text_config is None:
raise ValueError(f"Cannot load {model_name} as text-only; use FastVisionModel")
- # Carry over quantization_config; copy first since get_text_config() shares the parent's object.
+ # Carry over quantization_config; copy first, since get_text_config() shares the parent's object.
qc = getattr(model_config, "quantization_config", None)
if qc is not None and getattr(text_config, "quantization_config", None) is None:
text_config = copy.copy(text_config)
@@ -819,8 +786,8 @@ def _get_text_only_config(model_config, model_name):
def _remap_text_only_skip_modules(qc):
- # Remap llm_int8_skip_modules off the VLM wrapper prefix (language_model.model.* ->
- # model.*) after text-only stripping, and drop vision/audio entries. See PR #5816.
+ # Remap llm_int8_skip_modules off the VLM wrapper prefix (language_model.model.* -> model.*)
+ # after text-only stripping, and drop vision/audio entries (#5816).
is_dict = isinstance(qc, dict)
skip = (
qc.get("llm_int8_skip_modules") if is_dict else getattr(qc, "llm_int8_skip_modules", None)
@@ -861,9 +828,9 @@ def _remap_text_only_skip_modules(qc):
def _get_text_only_key_mapping(parent_config, text_config):
- # transformers >=5 stopped auto-stripping the VLM wrapper prefix (base_model_prefix
- # changed language_model -> model), so remap the text weights onto the decoder keys.
- # None on tf <5 (still strips; a mapping would break the load) or non-family. See PR #5816.
+ # transformers >= 5 stopped stripping the VLM wrapper prefix (base_model_prefix went
+ # language_model -> model), so remap text weights onto the decoder keys. None on tf <5, which
+ # still strips and would break the load with a mapping (#5816).
if Version(transformers_version) < Version("5.0.0"):
return None
if not _is_family_text_decoder(
@@ -923,9 +890,8 @@ def resolve_attention_implementation(
if _is_eager_only(model_type):
attn_impl = _set_attn_impl(config, "eager")
elif prefers_flex_attention and supports_flex_attention:
- # Models in _FLEX_PREFERRED_MODELS (gemma3 family) prefer flex_attention
- # over flash. Caller can still override by passing
- # requested_attn_implementation="sdpa" (handled below).
+ # _FLEX_PREFERRED_MODELS (the gemma3 family) prefer flex_attention over flash; a caller can
+ # still override with requested_attn_implementation="sdpa".
attn_impl = _set_attn_impl(config, "flex_attention")
elif not flash_attention_disabled and HAS_FLASH_ATTENTION and supports_flash_attention:
attn_impl = _set_attn_impl(config, "flash_attention_2")
@@ -948,16 +914,14 @@ def resolve_attention_implementation(
elif supports_sdpa:
attn_impl = _set_attn_impl(config, "sdpa")
elif supports_flex_attention:
- # Flex is only a fallback for models that don't support SDPA
- # (e.g. some custom configurations). Without this fallback such
- # models would land on eager.
+ # Flex is only a fallback for models that do not support SDPA; without it such models would land on eager.
attn_impl = _set_attn_impl(config, "flex_attention")
else:
attn_impl = _set_attn_impl(config, "eager")
- # float32 only rules out flash, unlike config-driven reasons which say something about the
- # model itself. So an explicit non-flash request must skip the flash fallback ladder, which
- # answers differently: gemma3 + "sdpa" resolves to eager, but the ladder says flex_attention.
+ # float32 only rules out flash, unlike config-driven reasons, which say something about the
+ # model itself, so an explicit non-flash request skips the flash fallback ladder: gemma3 +
+ # "sdpa" resolves to eager, but the ladder says flex_attention.
reroute_request_through_flash_fallback = flash_attention_disabled and not (
float32_is_only_disable_reason
and not _is_flash_attention_requested(requested_attn_implementation)
@@ -977,15 +941,9 @@ def resolve_attention_implementation(
final_attn_impl = requested_attn_implementation
_set_attn_impl(config, final_attn_impl)
- # A caller who explicitly passes requested_attn_implementation="sdpa" keeps it even
- # on a conservatively unsupported model, mirroring _disable_flash_attention_if_needed
- # which honors an explicit sdpa request. The exception is a model whose SDPA is
- # known-broken - _SDPA_EXCLUDED_MODELS (e.g. gpt_oss) or DISABLE_SDPA_MODEL_NAMES
- # (e.g. gemma3 / gemma3_text, which the loader also forces to supports_sdpa=False):
- # an explicit request must not re-enable it, so it still downgrades to eager, just
- # like flex falls back for _FLEX_EXCLUDED_MODELS. A synthesized/default sdpa
- # (requested is None, so the value came from the model resolution above or the
- # config) also downgrades.
+ # An explicit "sdpa" is kept even on a conservatively unsupported model, except where SDPA is
+ # known-broken, which still downgrades to eager just as flex falls back for
+ # _FLEX_EXCLUDED_MODELS. A synthesized default sdpa (requested is None) also downgrades.
honor_explicit_sdpa = requested_attn_implementation == "sdpa" and not _is_sdpa_excluded(
model_type
)
@@ -1029,7 +987,6 @@ def _run_temporary_patches(phase):
_run_temporary_patches("init")
# =============================================
-# Disable some warnings which can get annoying
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "torch")
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "torch")
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "huggingface_hub")
@@ -1051,8 +1008,8 @@ logging.getLogger("transformers.tokenization_utils_base").setLevel(logging.CRITI
TORCHAO_MSG = "Error: torchao not found, please install with `pip install torchao`"
-# Artifacts a Transformers/PEFT load never reads (ONNX/TF/Flax/CoreML/GGUF/training state), skipped
-# when prewarming so a mixed-format repo is not pulled in full.
+# Artifacts a Transformers/PEFT load never reads (ONNX/TF/Flax/CoreML/GGUF/training state),
+# skipped when prewarming so a mixed-format repo is not pulled in full.
_PREFETCH_IGNORE_PATTERNS = (
"*.onnx",
"onnx/*",
@@ -1076,8 +1033,8 @@ _PREFETCH_IGNORE_PATTERNS = (
)
-# Repo-root tokenizer / config / processor files from_pretrained reads from root even when weights
-# load from a subfolder. Exact names (no wildcard) so they match only root-level files.
+# Repo-root tokenizer / config / processor files from_pretrained reads from root even when
+# weights load from a subfolder. Exact names, so they match only root-level files.
_ROOT_AUX_PREFETCH_PATTERNS = (
"config.json",
"generation_config.json",
@@ -1090,7 +1047,8 @@ _ROOT_AUX_PREFETCH_PATTERNS = (
"vocab.txt",
"merges.txt",
"spiece.model",
- # More VOCAB_FILES_NAMES the slow tokenizer may fetch (DeBERTa-v2, Whisper, Mistral, XLM-R/mBART, Marian, FSMT/XLM, GPT-2).
+ # More VOCAB_FILES_NAMES the slow tokenizer may fetch (DeBERTa-v2, Whisper, Mistral,
+ # XLM-R/mBART, Marian, FSMT/XLM, GPT-2).
"spm.model",
"normalizer.json",
"tokenizer.model.v3",
@@ -1110,24 +1068,23 @@ _ROOT_AUX_PREFETCH_PATTERNS = (
"preprocessor_config.json",
"processor_config.json",
"video_preprocessor_config.json", # Qwen2.5-VL-style video processors
- # trust_remote_code auto_map can name any module, so warm every *.py (tiny; none in a non-remote repo).
+ # trust_remote_code auto_map can name any module, so warm every *.py.
"*.py",
"*.tiktoken", # tiktoken vocab (e.g. Qwen's qwen.tiktoken)
)
-# Files a PEFT adapter load reads: config + weights (glob covers sharded adapters). Any merged
-# full-model weights the repo also ships match none of these.
+# Files a PEFT adapter load reads: config plus weights, the glob covering sharded adapters. Any
+# merged full-model weights the repo ships match none of these.
_ADAPTER_PREFETCH_PATTERNS = (
"adapter_config.json",
"adapter_model*",
)
-# Weight files in a SUBDIRECTORY. A bare root load reads only root weights, so ignoring these drops
-# alternate-precision/experimental dirs (fp16/, experimental/). "*/*" spans "/" (HF fnmatch), so nested
-# weights match while root "model.safetensors" is kept. Only applied when weights_at_root (diffusion
-# keeps weights in subfolders).
+# Weight files in a SUBDIRECTORY: a bare root load reads only root weights, so ignoring these
+# drops alternate-precision dirs. "*/*" spans "/" in HF fnmatch, so nested weights match while
+# root model.safetensors is kept. Only under weights_at_root: diffusion uses subfolders.
_SUBDIR_WEIGHT_IGNORE_PATTERNS = (
"*/*.safetensors",
"*/*.bin",
@@ -1147,8 +1104,8 @@ def _in_requested_load_scope(filename, subfolder):
return "/" not in filename # root load: no directory component
-# .safetensors training-state files that are NOT model weights (e.g. optimizer.safetensors next to a
-# real pytorch_model.bin); counting them as "model safetensors present" would drop the needed .bin.
+# .safetensors training-state files that are NOT model weights (optimizer.safetensors beside a
+# real pytorch_model.bin); counting them as model safetensors would drop the needed .bin.
_NON_MODEL_WEIGHT_STEMS = frozenset(
{
"optimizer",
@@ -1169,7 +1126,7 @@ def _is_model_weight_safetensors(filename):
return False
if name.startswith("adapter_"):
return False
- # Stem before first dot: "optimizer.safetensors" -> "optimizer" (real shards kept); rng_state via prefix.
+ # Stem before the first dot: "optimizer.safetensors" -> "optimizer" (real shards kept); rng_state via prefix.
stem = name.split(".", 1)[0].lower()
if stem in _NON_MODEL_WEIGHT_STEMS or stem.startswith("rng_state"):
return False
@@ -1352,20 +1309,19 @@ def _prefetch_ignore_patterns(
or (from_flax and pattern == "*.msgpack")
)
]
- # Drop the format the load will not read (the other doubles the download); skipped for a whole
- # multi-component snapshot (see docstring).
+ # Drop the format the load will not read, since the other doubles the download; skipped for a
+ # whole multi-component snapshot.
whole_multi_component = not weights_at_root and not (
isinstance(subfolder, str) and subfolder.strip("/")
)
if whole_multi_component:
pass
elif st_module_paths and (from_tf or from_flax or use_safetensors is not None):
- # An explicit format request cannot be scoped: no repo listing is fetched on these branches, and
- # the globs span "/", so they would prune a declared ST module's only weight. Keep both formats,
- # the same trade whole_multi_component already makes.
+ # An explicit format request cannot be scoped: no repo listing is fetched here and the globs
+ # span "/", so they would prune a declared ST module's only weight. Keep both formats.
pass
elif from_tf or from_flax:
- # TF / Flax loads never read the PyTorch formats; drop safetensors and .bin.
+ # TF / Flax loads never read the PyTorch formats.
ignore_patterns.extend(
(
"*.safetensors",
@@ -1375,10 +1331,10 @@ def _prefetch_ignore_patterns(
)
)
elif use_safetensors is True:
- # Explicit safetensors: load never reads .bin (no model_info call needed).
+ # Explicit safetensors: the load never reads .bin, so no model_info call is needed.
ignore_patterns.extend(("*.bin", "*.bin.index.json"))
elif use_safetensors is False:
- # Explicit .bin: load never reads safetensors.
+ # Explicit .bin: the load never reads safetensors.
ignore_patterns.extend(("*.safetensors", "*.safetensors.index.json"))
else:
# Auto: skip .bin only once in-scope safetensors are confirmed (best-effort; any failure keeps both).
@@ -1395,8 +1351,8 @@ def _prefetch_ignore_patterns(
.siblings
or []
)
- # Count only in-scope model-weights safetensors (not adapters/sidecars): variant-matching if
- # a variant is requested, else canonical, proving the .bin redundant.
+ # Count only in-scope model-weight safetensors, not adapters or sidecars, variant-matching when
+ # a variant is requested, else canonical, to prove the .bin redundant.
has_safetensors = any(
_is_model_weight_safetensors(sibling.rfilename)
and _in_requested_load_scope(sibling.rfilename, subfolder)
@@ -1459,12 +1415,11 @@ def maybe_prefetch_hf_snapshot(
if not isinstance(model_name, str) or not model_name:
return False
- # Local path: nothing to download. Expand ~ first (os.path.exists does not).
+ # Local path: nothing to download. Expand ~ first, which os.path.exists does not.
model_path = os.path.expanduser(model_name)
if os.path.isdir(model_path) or os.path.exists(model_path):
return False
- # Looks local but not yet on disk (e.g. an uncreated output dir): not a Hub repo id, so leave it
- # for from_pretrained rather than download it.
+ # Looks local but is not yet on disk (an uncreated output dir): not a Hub repo id, so leave it for from_pretrained.
if (
os.path.isabs(model_path)
or model_name.startswith(("~", "./", "../", ".\\", "..\\"))
@@ -1481,9 +1436,9 @@ def maybe_prefetch_hf_snapshot(
if fast_inference: # vLLM has its own download path
return False
- # A sentence-transformers repo can hold weights at the root AND in declared module subfolders
- # (2_Dense/...). Probed once here and shared, since both the subdir prune below and the redundant
- # format prune span "/" and would drop those weights.
+ # A sentence-transformers repo can hold weights at the root AND in declared module subfolders.
+ # Probed once and shared, since the subdir prune and the format prune both span "/" and would
+ # drop those weights.
st_module_paths = ()
if (
weights_at_root
@@ -1497,7 +1452,7 @@ def maybe_prefetch_hf_snapshot(
cache_dir = cache_dir,
)
# tokenizer-only / adapter-only warms allow-list exact files below, so the weight-format ignore
- # list (and its auto-branch model_info call) is skipped.
+ # list and its model_info call are skipped.
ignore_patterns = (
None
if tokenizer_only or adapter_only or gguf_file
@@ -1514,12 +1469,12 @@ def maybe_prefetch_hf_snapshot(
st_module_paths = st_module_paths,
)
)
- # Narrow the warm to what the load reads (skip extra checkpoints/precisions); every branch still warms
- # root tokenizer/config/custom-code so those never fall in-process.
+ # Narrow the warm to what the load reads; every branch still warms root tokenizer, config and
+ # custom code so those never fall in-process.
allow_patterns = None
if gguf_file:
- # gguf_file=NAME reads exactly that GGUF, but the static ignore list drops *.gguf; so warm just
- # that file (plus root aux), under <subfolder>/ if set.
+ # gguf_file=NAME reads exactly that GGUF, but the static ignore list drops *.gguf, so warm that
+ # file plus root aux.
_gguf_path = (
f"{subfolder.strip('/')}/{gguf_file}"
if isinstance(subfolder, str) and subfolder.strip("/")
@@ -1530,11 +1485,11 @@ def maybe_prefetch_hf_snapshot(
# A distinct tokenizer repo: warm only tokenizer / config / vocab files, never its weights.
allow_patterns = list(_ROOT_AUX_PREFETCH_PATTERNS)
elif adapter_only:
- # A PEFT adapter load reads only adapter_config.json + adapter_model.* (plus root aux), not any
- # merged weights the repo may also publish.
+ # A PEFT adapter load reads only adapter_config.json, adapter_model.* and root aux, not any
+ # merged weights the repo may publish.
allow_patterns = [*_ADAPTER_PREFETCH_PATTERNS, *_ROOT_AUX_PREFETCH_PATTERNS]
- # PeftModel reads one format (safetensors when present): explicit use_safetensors wins, else
- # prefer safetensors when shipped (best-effort; any failure keeps both).
+ # PeftModel reads one format: an explicit use_safetensors wins, else prefer safetensors when
+ # the repo ships them.
if use_safetensors is False:
ignore_patterns = [
"adapter_model*.safetensors",
@@ -1545,15 +1500,12 @@ def maybe_prefetch_hf_snapshot(
):
ignore_patterns = ["adapter_model*.bin", "adapter_model*.bin.index.json"]
elif isinstance(subfolder, str) and subfolder.strip("/"):
- # subfolder=X: load resolves every weight under X/, so warm that subfolder (plus root aux).
+ # subfolder=X: the load resolves every weight under X/, so warm that subfolder plus root aux.
allow_patterns = [f"{subfolder.strip('/')}/*", *_ROOT_AUX_PREFETCH_PATTERNS]
elif weights_at_root:
- # A bare load reads only root weights: drop subdir weights (fp16/, checkpoint dirs) while keeping
- # subdir configs. Diffusion leaves weights_at_root False.
- #
- # Except when the repo is both: a sentence-transformers model with a root weight AND weighted
- # module subfolders (2_Dense/...), which the ST load reads too. Dropping those turned a
- # satisfiable request into one that could never succeed. See _st_weighted_subfolder_paths.
+ # A bare load reads only root weights, so drop subdir weights and keep subdir configs. Except
+ # a sentence-transformers model with a root weight AND weighted module subfolders, which the ST
+ # load also reads: dropping those made the request unsatisfiable.
if not st_module_paths:
ignore_patterns = [*(ignore_patterns or []), *_SUBDIR_WEIGHT_IGNORE_PATTERNS]
try:
@@ -1590,7 +1542,7 @@ class HideLoggingMessage(logging.Filter):
return not (self.text in x.getMessage())
-# Replace warning messages (analogous to HideLoggingMessage but for warnings.warn)
+# Replace warning messages, analogous to HideLoggingMessage but for warnings.warn.
class ReplaceWarningMessage:
"""
Intercepts warnings.warn calls and replaces matching messages with Unsloth branded ones.
@@ -1698,7 +1650,7 @@ if not UNSLOTH_ENABLE_LOGGING:
except:
pass
-# The speedups for torchdynamo mostly come with GPU Ampere or higher and which is not detected here.
+# The torchdynamo speedups mostly need Ampere or higher, which is not detected here.
from transformers.training_args import logger as transformers_training_args_logger
transformers_training_args_logger.addFilter(HideLoggingMessage("The speedups"))
@@ -1717,7 +1669,7 @@ transformers_trainer_logger.addFilter(HideLoggingMessage("No label_names"))
transformers_trainer_logger.addFilter(HideLoggingMessage("The tokenizer has new"))
del transformers_trainer_logger
-# Strip the "[transformers] " prefix transformers>=5 adds; keep the messages.
+# Strip the "[transformers] " prefix transformers >= 5 adds; keep the messages.
try:
try:
import transformers.utils.logging as _tf_log
@@ -1732,7 +1684,6 @@ try:
except Exception:
pass
-# Using the default loss: `ForCausalLMLoss`.
try:
from transformers.modeling_utils import logger as transformers_modeling_utils_logger
transformers_modeling_utils_logger.addFilter(HideLoggingMessage("ForCausalLMLoss"))
@@ -1740,7 +1691,6 @@ try:
except:
pass
-# The model weights are not tied. Please use the `tie_weights` method before using the `infer_auto_device` function.
try:
from accelerate.utils.modeling import logger as accelerate_utils_modeling_logger
accelerate_utils_modeling_logger.addFilter(HideLoggingMessage("The model weights are not tied"))
@@ -1748,7 +1698,6 @@ try:
except:
pass
-# Setting `pad_token_id` to `eos_token_id`
try:
from transformers.generation.utils import (
logger as transformers_generation_utils_logger,
@@ -1763,7 +1712,18 @@ try:
except:
pass
-# The following generation flags are not valid and may be ignored:
+# Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl"
+# Hide HF Hub unauthenticated request warnings
+# We detected that you are using `from_pretrained` with a meta device context manager or
+# `torch.set_default_device('meta')
+# `use_cache=True` is incompatible with gradient checkpointing
+# Using a slow image processor as `use_fast`
+# UserWarning: Logical operators 'and' and 'or' are deprecated for non-scalar tensors; please use '&' or '|'
+# instead Will be fixed in torch 2.8.1 https://github.com/pytorch/pytorch/issues/158463
+# You passed `quantization_config` or equivalent parameters
+# MXFP4 quantization requires triton >= 3.4.0
+# Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed.
+# Gemma4 It is strongly recommended to train Gemma4 models with the `eager`
try:
from transformers.generation.configuration_utils import (
logger as configuration_logger,
@@ -1773,7 +1733,6 @@ try:
except:
pass
-# Gemma3 It is strongly recommended to train Gemma3 models with the `eager`
try:
from transformers.models.gemma3.modeling_gemma3 import logger as gemma3_logger
gemma3_logger.addFilter(HideLoggingMessage("strongly recommended"))
@@ -1781,7 +1740,6 @@ try:
except:
pass
-# Gemma4 It is strongly recommended to train Gemma4 models with the `eager`
try:
from transformers.models.gemma4.modeling_gemma4 import logger as gemma4_logger
gemma4_logger.addFilter(HideLoggingMessage("strongly recommended"))
@@ -1789,7 +1747,6 @@ try:
except:
pass
-# Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed.
try:
from huggingface_hub.file_download import logger as hub_logger
hub_logger.addFilter(HideLoggingMessage("hf_xet"))
@@ -1797,7 +1754,6 @@ try:
except:
pass
-# MXFP4 quantization requires triton >= 3.4.0
try:
from transformers.quantizers.quantizer_mxfp4 import logger as mxfp4_logger
mxfp4_logger.addFilter(HideLoggingMessage("requires triton"))
@@ -1805,7 +1761,6 @@ try:
except:
pass
-# You passed `quantization_config` or equivalent parameters
try:
warnings.filterwarnings(
action = "ignore",
@@ -1816,8 +1771,7 @@ try:
except:
pass
-# UserWarning: Logical operators 'and' and 'or' are deprecated for non-scalar tensors; please use '&' or '|' instead
-# Will be fixed in torch 2.8.1 https://github.com/pytorch/pytorch/issues/158463
+# Logical operators 'and'/'or' deprecated for non-scalar tensors; fixed in torch 2.8.1, pytorch/pytorch#158463.
try:
warnings.filterwarnings(
action = "ignore",
@@ -1828,7 +1782,6 @@ try:
except:
pass
-# Using a slow image processor as `use_fast`
try:
from transformers.processing_utils import logger as processing_utils_logger
processing_utils_logger.addFilter(HideLoggingMessage("`use_fast`"))
@@ -1836,7 +1789,6 @@ try:
except:
pass
-# Using a slow image processor as `use_fast`
try:
from transformers.models.auto.image_processing_auto import (
logger as processing_utils_logger,
@@ -1846,7 +1798,6 @@ try:
except:
pass
-# `use_cache=True` is incompatible with gradient checkpointing
try:
from transformers.trainer import logger as trainer_logger
trainer_logger.addFilter(HideLoggingMessage("`use_cache=True`"))
@@ -1854,7 +1805,6 @@ try:
except:
pass
-# `use_cache=True` is incompatible with gradient checkpointing
try:
from transformers.utils.generic import logger as trainer_logger
trainer_logger.addFilter(HideLoggingMessage("`use_cache=True`"))
@@ -1862,7 +1812,6 @@ try:
except:
pass
-# We detected that you are using `from_pretrained` with a meta device context manager or `torch.set_default_device('meta')
try:
from transformers.modeling_utils import logger as modeling_utils_logger
modeling_utils_logger.addFilter(HideLoggingMessage("anti-pattern"))
@@ -1870,13 +1819,12 @@ try:
except:
pass
-# Errors out on
-# Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint
+# Errors out on "Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint".
from transformers.modeling_utils import logger as transformers_logger
-# Faster safetensors loads on UMA (integrated) GPUs; lazy gate keeps this import
-# fork-safe (no CUDA init). No-op off-UMA. Opt out: UNSLOTH_DISABLE_UMA_CLONE_LOAD=1.
+# Faster safetensors loads on UMA GPUs; the lazy gate keeps this import fork-safe (no CUDA
+# init). No-op off-UMA, opt out with UNSLOTH_DISABLE_UMA_CLONE_LOAD=1.
from ._uma_safetensors import patch_unified_memory_safetensors_load
patch_unified_memory_safetensors_load()
@@ -1947,7 +1895,6 @@ try:
except:
pass
-# Hide HF Hub unauthenticated request warnings
try:
from huggingface_hub.utils._http import logger as hf_http_logger
hf_http_logger.addFilter(HideLoggingMessage("You are sending unauthenticated requests"))
@@ -1955,7 +1902,7 @@ try:
except:
pass
-# Replace PEFT target_parameters warning with Unsloth branded message for MoE models
+# Replace the PEFT target_parameters warning with an Unsloth-branded message for MoE models.
ReplaceWarningMessage.add_rule(
match_text = "target_parameters",
replacement = (
@@ -1965,7 +1912,6 @@ ReplaceWarningMessage.add_rule(
category = RuntimeWarning,
)
-# Patch get_model_param_count to record correct 4bit / 8bit
from transformers.trainer_pt_utils import is_deepspeed_zero3_enabled
@@ -2013,13 +1959,12 @@ transformers.trainer_pt_utils.get_model_param_count = get_model_param_count
import transformers.trainer
transformers.trainer.get_model_param_count = get_model_param_count
-# =============================================
-# =============================================
-# Edits all Config files to enable RoPE Scaling for all models
+# Edits all Config files to enable RoPE Scaling for all models.
# Transformers had to update for Mistral Nemo 12b since Attention is (5120, 4096) now.
+# =============================================
def patch_mistral_nemo_config(config):
if "head_dim (" not in config:
add_head_dim = (
@@ -2038,14 +1983,13 @@ def patch_mistral_nemo_config(config):
try:
- # Some Config files use layer_type_validation
- # for eg Gemma-2, so we must import it to stop errors.
+ # Some Config files use layer_type_validation (Gemma-2), so it must be imported to stop errors.
from transformers.configuration_utils import layer_type_validation
except:
pass
try:
- # Transformers 5.0+ uses RotaryEmbeddingConfigMixin as a base class for configs
+ # Transformers 5.0+ uses RotaryEmbeddingConfigMixin as a base class for configs.
from transformers.modeling_rope_utils import RotaryEmbeddingConfigMixin
except:
pass
@@ -2068,11 +2012,10 @@ model_architectures = [
"falcon_h1",
]
-# Transformers 5.x uses class-level annotations with @strict, @auto_docstring,
-# and interval() in config classes. exec(inspect.getsource(...)) fails because
-# those symbols are not in scope. Skip the exec-based config patching for 5.x
-# since those configs already use rope_parameters (the v5 replacement for
-# rope_scaling).
+# Transformers 5.x config classes use class-level annotations with @strict, @auto_docstring and
+# interval(), so exec(inspect.getsource(...)) fails with those symbols out of scope. Skip the
+# exec-based patching there; those configs already use rope_parameters, the v5 replacement for
+# rope_scaling.
_skip_config_exec_patch = Version(transformers_version) >= Version("5.0.0")
for model_name in model_architectures:
@@ -2080,7 +2023,7 @@ for model_name in model_architectures:
break
config_filepath = f"transformers.models.{model_name}.configuration_{model_name}"
model_filepath = f"transformers.models.{model_name}.modeling_{model_name}"
- config_filename = f"{model_name.title().replace('_', '')}Config" # qwen3 arch folder is qwen3_moe but config is Qwen3Config. Need to remove underscore(_) for now
+ config_filename = f"{model_name.title().replace('_', '')}Config" # qwen3_moe -> Qwen3Config: drop the underscore.
try:
exec(f"from {config_filepath} import {config_filename}", globals())
except:
@@ -2106,7 +2049,7 @@ for model_name in model_architectures:
config,
)
- # Just for Mistral Nemo
+ # Just for Mistral Nemo.
if model_name == "mistral":
if Version(transformers_version) <= Version("4.42.4"):
config = patch_mistral_nemo_config(config)
@@ -2117,10 +2060,9 @@ for model_name in model_architectures:
exec(f"{config_filepath}.{config_filename} = {config_filename}", globals())
except Exception:
continue
-# =============================================
+# torch.cuda.amp.custom_fwd is deprecated from 2.4.
# =============================================
-# torch.cuda.amp.custom_fwd is deprecated >= 2.4
torch_version = torch.__version__
if DEVICE_TYPE in ("cuda", "hip"):
if Version(torch_version) < Version("2.4.0"):
@@ -2135,32 +2077,12 @@ elif DEVICE_TYPE == "xpu":
else:
torch_amp_custom_fwd = torch.amp.custom_fwd(device_type = "xpu")
torch_amp_custom_bwd = torch.amp.custom_bwd(device_type = "xpu")
-# =============================================
-# =============================================
-# Fix KeyError: 'Cache only has 0 layers, attempted to access layer with index 0'
-# import transformers.cache_utils
-# if hasattr(transformers.cache_utils, "DynamicCache") and \
-# transformers.cache_utils.DynamicCache.__getitem__.__name__ != "__cache_utils_getitem__":
-
-# source = inspect.getsource(transformers.cache_utils.DynamicCache.__getitem__)
-# start = source.find("def")
-# spaces = start*" "
-# source = source.split("\n")
-# source = "\n".join(x[start:] for x in source)
-# where = source.find("raise KeyError")
-# source = source[:where] + \
-# f"if len(self) == 0:\n{spaces}{spaces}"\
-# " raise RuntimeError('Unsloth: You must call `FastLanguageModel.for_inference(model)` before doing inference for Unsloth models.')\n" + \
-# f"{spaces}{spaces}else:\n{spaces}{spaces}{spaces}" + source[where:]
-# source = source.replace("__getitem__", "__cache_utils_getitem__", 1)
-# exec(source)
-# transformers.cache_utils.DynamicCache.__getitem__ = __cache_utils_getitem__
-# pass
-# =============================================
+# Fix KeyError: 'Cache only has 0 layers, attempted to access layer with index 0'.
+
+# Weird Databricks errors.
# =============================================
-# Weird Databricks errors
from transformers.utils import is_openai_available
if is_openai_available():
@@ -2176,16 +2098,13 @@ if is_openai_available():
transformers.utils.is_openai_available = _is_openai_available
# =============================================
-# Get Flash Attention v2 if Ampere (RTX 30xx, A100)
from transformers import AutoTokenizer
from transformers.utils.import_utils import _is_package_available
def _package_available(pkg_name: str) -> bool:
- # transformers >= 5.x makes `_is_package_available` always return a
- # `(exists, version)` tuple, which is truthy even when the package is
- # absent; older versions returned a plain bool. Normalise to a bool so
- # callers don't take "package present" branches for missing packages.
+ # transformers >= 5.x makes _is_package_available always return an (exists, version) tuple,
+ # truthy even when absent; older versions returned a plain bool.
result = _is_package_available(pkg_name)
if isinstance(result, tuple):
return bool(result[0])
@@ -2197,11 +2116,9 @@ HAS_FLASH_ATTENTION = False
HAS_FLASH_ATTENTION_SOFTCAPPING = False
if DEVICE_TYPE == "cuda" and not torch.cuda.is_available():
- # UNSLOTH_ALLOW_CPU=1 keeps DEVICE_TYPE "cuda" on driverless hosts, so ask
- # whether a device is present before asking what it can do. The three flags
- # keep the defaults set just above: no device means no bfloat16 support to
- # claim (False only costs float32, True fails at the first cast) and Flash
- # Attention is a CUDA kernel that could not run here anyway.
+ # UNSLOTH_ALLOW_CPU=1 keeps DEVICE_TYPE "cuda" on driverless hosts, so ask whether a device is
+ # present before asking what it can do. No device means no bfloat16 to claim (False costs only
+ # float32, True fails at the first cast) and no Flash Attention either.
SUPPORTS_BFLOAT16 = False
HAS_FLASH_ATTENTION = False
HAS_FLASH_ATTENTION_SOFTCAPPING = False
@@ -2212,7 +2129,7 @@ elif DEVICE_TYPE == "cuda":
if major_version >= 8:
SUPPORTS_BFLOAT16 = True
if _package_available("flash_attn"):
- # Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl"
+ # Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl" and for softcapping; see #1437.
try:
try:
# See https://github.com/unslothai/unsloth/issues/1437
@@ -2238,7 +2155,6 @@ elif DEVICE_TYPE == "cuda":
"Using Xformers instead. No performance changes will be seen."
)
- # Stop Flash Attention from importing!
import transformers.utils.import_utils
transformers.utils.import_utils.is_flash_attn_2_available = (
@@ -2257,6 +2173,7 @@ elif DEVICE_TYPE == "cuda":
elif DEVICE_TYPE == "hip":
SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported()
if _package_available("flash_attn"):
+ # Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl" and for softcapping; see #1437.
# Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl"
try:
try:
@@ -2283,7 +2200,6 @@ elif DEVICE_TYPE == "hip":
"Using Xformers instead. No performance changes will be seen."
)
- # Stop Flash Attention from importing!
import transformers.utils.import_utils
transformers.utils.import_utils.is_flash_attn_2_available = lambda *args, **kwargs: (
@@ -2297,9 +2213,7 @@ elif DEVICE_TYPE == "hip":
elif DEVICE_TYPE == "xpu":
SUPPORTS_BFLOAT16 = True
-# =============================================
-# Get Xformers
-# Silence xformers CUDA mismatch warnings before import
+# Silence xformers CUDA mismatch warnings before import.
try:
_xformers_logger = logging.getLogger("xformers")
_xformers_logger.setLevel(logging.ERROR)
@@ -2309,11 +2223,10 @@ except:
try:
from xformers import __version__ as xformers_version
- # Xformers <= 0.0.32.post2 has a broken FA3 dispatch on Blackwell/RTX 50x GPUs.
- # The FA3 check used `capability >= (9, 0)` which matches SM 10.0/11.0/12.0,
- # causing sm_90a kernels to be attempted on non-Hopper GPUs (CUDA error in
- # flash_fwd_launch_template.h:188). Fixed in 0.0.33 with `<= (9, 0)`.
- # See https://github.com/facebookresearch/xformers/issues/1329
+ # Xformers <= 0.0.32.post2 dispatches FA3 wrongly on Blackwell/RTX 50x: `capability >= (9, 0)`
+ # matches SM 10.0/11.0/12.0 and runs sm_90a kernels on non-Hopper GPUs (CUDA error in
+ # flash_fwd_launch_template.h:188). Fixed in 0.0.33 with `<= (9, 0)`; see
+ # facebookresearch/xformers#1329.
if DEVICE_TYPE == "cuda":
major_version, minor_version = torch.cuda.get_device_capability()
if (f"{major_version}.{minor_version}" in ("10.0", "11.0", "12.0")) and (
@@ -2328,8 +2241,8 @@ try:
"```\n"
)
- # Temporarily disable 0.0.27 and higher - inference issues
- if False: # Version(xformers_version) >= Version("0.0.27"):
+ # Temporarily disable 0.0.27 and higher: inference issues.
+ if False:
raise ImportError(
"Unsloth: If you are in Colab, we updated the top cell install instructions - please change it to below "
"then press Disconnect Runtime and then Restart it.\n"
@@ -2366,7 +2279,7 @@ try:
from xformers._cpp_lib import _register_extensions
try:
- _register_extensions() # Check if C++ modules are loaded correctly
+ _register_extensions()
except Exception as error:
raise ImportError(
"Unsloth: Xformers was not installed correctly.\n"
@@ -2390,11 +2303,10 @@ except Exception as e:
xformers_attention = None
xformers_version = None
-# Check TRL version
from trl import __version__ as trl_version
-# Unsloth now supports all TRL versions!
-if False: # Version(trl_version) >= Version("0.9.0"):
+# Unsloth now supports all TRL versions.
+if False:
raise ImportError(
"Unsloth: If you are in Colab, we updated the top cell install instructions - please change it to below "
"then press Disconnect Runtime and then Restart it.\n"
@@ -2408,43 +2320,23 @@ if False: # Version(trl_version) >= Version("0.9.0"):
"Please downgrade TRL via `pip install --force-reinstall trl"
)
+# Fix new Xformers versions "TypeError: Multiple dispatch failed for 'torch._ops.aten.to.dtype_layout'".
+
+# Transformers 4.46 breaks dynamic caching; this is a hack.
# =============================================
-# Fix new Xformers versions TypeError: Multiple dispatch failed for 'torch._ops.aten.to.dtype_layout'
-# accelerate_old_send_to_device = None
-# accelerate_new_send_to_device = None
-# if xformers_version is not None and Version(xformers_version) >= Version("0.0.27"):
-# import accelerate.utils.operations
-# if hasattr(accelerate.utils.operations, "send_to_device") and \
-# accelerate.utils.operations.send_to_device.__name__ != "_fixed_send_to_device":
-# accelerate_old_send_to_device = accelerate.utils.operations.send_to_device
-# from accelerate.utils.operations import *
-# send_to_device = inspect.getsource(accelerate.utils.operations.send_to_device)
-# send_to_device = re.sub(
-# r"([ ]{4,})return tensor\.to\(device\)",
-# r"\1try: return tensor.to(device)\n\1except: return tensor",
-# send_to_device,
-# ).replace("def send_to_device", "def _fixed_send_to_device")
-# exec(send_to_device)
-# # accelerate.utils.operations.send_to_device = _fixed_send_to_device
-# accelerate_new_send_to_device = _fixed_send_to_device
-# pass
-# pass
-
-# Transformers 4.46 breaks dynamic caching. This is a hack
import transformers.generation.configuration_utils
if hasattr(transformers.generation.configuration_utils, "ALL_CACHE_IMPLEMENTATIONS"):
if type(transformers.generation.configuration_utils.ALL_CACHE_IMPLEMENTATIONS) is list:
if "dynamic" not in transformers.generation.configuration_utils.ALL_CACHE_IMPLEMENTATIONS:
transformers.generation.configuration_utils.ALL_CACHE_IMPLEMENTATIONS.append("dynamic")
-# =============================================
+# Torch compile settings.
# =============================================
-# Torch compile settings
UNSLOTH_COMPILE_DEBUG = os.environ.get("UNSLOTH_COMPILE_DEBUG", "0") == "1"
UNSLOTH_COMPILE_MAXIMUM = os.environ.get("UNSLOTH_COMPILE_MAXIMUM", "0") == "1"
UNSLOTH_COMPILE_IGNORE_ERRORS = os.environ.get("UNSLOTH_COMPILE_IGNORE_ERRORS", "1") == "1"
-# Just remove max_autotune_gemm warning
+# Just remove the max_autotune_gemm warning.
from torch._inductor.runtime.hints import DeviceProperties
@@ -2499,7 +2391,7 @@ del accelerate
def patch_regional_compilation():
- # Regional torch 2.5 Recompilation - weirdly very slow??
+ # Regional torch 2.5 recompilation is weirdly very slow, and only works on torch 2.5.
if torch.nn.ModuleList.__name__ == "UnslothModuleList":
return
# Only works for torch 2.5
@@ -2533,8 +2425,6 @@ def patch_regional_compilation():
# =============================================
-
-
def prepare_model_for_kbit_training(
model: Any,
use_gradient_checkpointing: Optional = True,
@@ -2552,14 +2442,16 @@ def prepare_model_for_kbit_training(
)
+# LoraLayer.update_layer downcasts PEFT layers to float16; mixed precision needs float32.
# =============================================
-# Weirdly LoraLayer.update_layer downcasts PEFT layers to float16??
-# For mixed precision, we need it to be in float32 not float16.
from peft import __version__ as peft_version
from peft.utils.integrations import dequantize_module_weight
if Version(peft_version) < Version("0.12.0"):
+ # Fix up incorrect downcasting of LoRA weights
from peft.tuners.lora.layer import LoraLayer
+
+ # Prefer filesystem markers (harder to misidentify) before env-key matching
try:
source = inspect.getsource(LoraLayer.update_layer)
text = "if weight is not None:\n"
@@ -2574,7 +2466,6 @@ if Version(peft_version) < Version("0.12.0"):
source = source.replace("def update_layer", "def LoraLayer_update_layer")
exec(source, globals())
- # Fix up incorrect downcasting of LoRA weights
from peft.tuners.lora.layer import LoraLayer
LoraLayer.update_layer = LoraLayer_update_layer
@@ -2630,18 +2521,16 @@ import psutil
def _get_statistics(statistics = None, force_download = True):
- # We log some basic stats about which environment is being used.
- # We simply download a README.md file from HF - all data is made public.
- # This is simply so we can check if some envs are broken or not.
- # You can disable this by commenting the below out
+ # Basic stats on which environment is in use: a README.md is downloaded from HF, all data
+ # public, so broken envs can be detected. Disable with UNSLOTH_DISABLE_STATISTICS.
n_cpus = psutil.cpu_count(logical = False)
keynames = "\n" + "\n".join(os.environ.keys())
- # Check modelscope for down detection
global USE_MODELSCOPE
USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1"
+ # Fallback to env-key detection
if statistics is None:
- # Prefer filesystem markers (harder to misidentify) before env-key matching
+ # Prefer filesystem markers (harder to misidentify) before env-key matching.
try:
from pathlib import Path
if Path("/kaggle/working").exists():
@@ -2653,7 +2542,6 @@ def _get_statistics(statistics = None, force_download = True):
except Exception:
pass
- # Fallback to env-key detection
if statistics is None:
if "\nKAGGLE_" in keynames:
statistics = "kaggle"
@@ -2667,10 +2555,8 @@ def _get_statistics(statistics = None, force_download = True):
statistics = "aws"
elif "\nAZURE_" in keynames:
statistics = "azure"
- # elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp"
elif "\nINVOCATION_ID" in keynames:
statistics = "lambda"
- # else: statistics = "other"
else:
def try_vllm_check():
@@ -2732,15 +2618,12 @@ def _get_statistics(statistics = None, force_download = True):
)
except Exception:
logger.debug("Unsloth: stats_check failed with an exception.")
- # Don't retry without a time limit — would freeze offline
+ # Do not retry without a time limit: it would freeze offline.
def get_statistics(local_files_only = False):
- # We log some basic stats about which environment is being used.
- # This is also to check if HuggingFace is down or not!
- # We simply download a README.md file from HF - all data is made public.
- # This is simply so we can check if some envs are broken or not.
- # You can disable this by setting UNSLOTH_DISABLE_STATISTICS
+ # Basic stats on which environment is in use, and whether HuggingFace is down: a README.md is
+ # downloaded from HF, all data public. Disable with UNSLOTH_DISABLE_STATISTICS.
import os
if (
@@ -2798,33 +2681,16 @@ def get_statistics(local_files_only = False):
# =============================================
-# Fixes Bitsandbytes to remove missing warnings
from transformers.utils.quantization_config import (
BitsAndBytesConfig,
QuantizationMethod,
)
-# Importing this module twice in one process must not raise.
-#
-# This block reads BitsAndBytesConfig.__init__'s source, rewrites it, and (at
-# the bottom of the section) installs the result over the original. The
-# replacement is built with exec(), so it has no file on disk. On a SECOND
-# import, inspect.getsource() is therefore handed our own exec'd function,
-# linecache finds nothing to read for it, and the import dies with
-#
-# OSError: could not get source code
-#
-# taking every later `from unsloth...` in that process with it. Anything that
-# drops unsloth from sys.modules and imports it again reaches this: on a clean
-# tree, `pytest tests/vllm_compat/test_extended_module_imports.py
-# tests/python/test_vision_lora_targeting.py` is 30 failures, and it is only
-# luck of the xdist scheduling that a full run does not usually pair them.
-#
-# The patch is idempotent by nature -- when it has already run, the wanted end
-# state is the one already in place -- so recognising our own handiwork and
-# standing down is the fix. The marker is set on the function rather than
-# tracked in a module global because a re-import gets fresh globals but the same
-# transformers class object.
+# Importing this module twice must not raise. This block rewrites BitsAndBytesConfig.__init__
+# via exec(), so it has no file on disk: on a SECOND import inspect.getsource() is handed our
+# own function and dies with "OSError: could not get source code", taking every later
+# `from unsloth...` with it. The patch is idempotent, so recognising our own work and standing
+# down is the fix; the marker lives on the function, since a re-import gets fresh globals.
_bnb_init = BitsAndBytesConfig.__init__
if getattr(_bnb_init, "__unsloth_patched__", False):
@@ -2834,8 +2700,7 @@ else:
try:
BitsAndBytesConfig__init__ = inspect.getsource(_bnb_init)
except (OSError, TypeError):
- # Someone else replaced __init__ with something sourceless. Leaving
- # theirs alone is strictly better than failing the import.
+ # Someone else replaced __init__ with something sourceless; leaving theirs alone beats failing the import.
BitsAndBytesConfig__init__ = None
if BitsAndBytesConfig__init__ is None:
@@ -2873,13 +2738,12 @@ if DEVICE_COUNT == 1 and int(os.environ.get("WORLD_SIZE", "1")) <= 1:
accelerate.accelerator.Accelerator.distributed_type = property(lambda self: DistributedType.NO)
-# to move multiple tensors to the same device
def move_to_device(target_device, *tensors):
"""Move tensors to target_device (returns same objects if already there)."""
if isinstance(target_device, int):
target_device = torch.device(target_device)
elif isinstance(target_device, str):
- # if string we expect it to be a device name like "cuda:0"
+ # If a string, expect a device name like "cuda:0".
target_device = torch.device(target_device)
elif isinstance(target_device, torch.device):
pass
@@ -2897,9 +2761,9 @@ def move_to_device(target_device, *tensors):
import transformers.utils.quantization_config
transformers.utils.quantization_config.BitsAndBytesConfig.__init__ = _BitsAndBytesConfig__init__
-# =============================================
-# Offloading to disk for modules (lm_head, embed_tokens)
+# Offloading to disk for modules (lm_head, embed_tokens).
+# =============================================
import pickle
@@ -2921,7 +2785,7 @@ def offload_to_disk(
pickle_module = pickle,
pickle_protocol = pickle.HIGHEST_PROTOCOL,
)
- # We must use weights_only = False due to pickling
+ # weights_only = False is required due to pickling.
offloaded_W = torch.load(filename, map_location = "cpu", mmap = True, weights_only = False)
offloaded_W._offloaded_file_location = filename
return offloaded_W
@@ -2953,7 +2817,7 @@ def offload_output_embeddings(model, temporary_location: str = "_unsloth_tempora
return
-# Fixes a weird Torch 2.3 bug which says T4s have bfloat16
+# Fixes a weird Torch 2.3 bug which says T4s have bfloat16.
def is_bfloat16_supported():
return SUPPORTS_BFLOAT16
@@ -3009,7 +2873,7 @@ def is_vLLM_available():
return _package_available("vllm")
-# Patches models to add RoPE Scaling
+# Patches models to add RoPE Scaling.
def patch_linear_scaling(
model_name = "gemma2",
rope_module = None,
@@ -3032,7 +2896,7 @@ def patch_linear_scaling(
try:
function = inspect.getsource(attention_module.__init__)
except:
- # Most likely already patched!
+ # Most likely already patched.
return None, None
where = function.find("def")
function = function.split("\n")
@@ -3082,7 +2946,7 @@ def patch_linear_scaling(
return init_name, function
-# Patches for Llama-3 LlamaExtendedRotaryEmbedding
+# Patches for Llama-3 LlamaExtendedRotaryEmbedding.
def patch_llama_rope_scaling(
model_name = "llama",
rope_module = None,
@@ -3111,7 +2975,7 @@ def patch_llama_rope_scaling(
try:
function = inspect.getsource(attention_module.__init__)
except:
- # Most likely already patched!
+ # Most likely already patched.
return None, None
where = function.find("def")
function = function.split("\n")
@@ -3185,7 +3049,6 @@ def patch_llama_rope_scaling(
def create_boolean_mask(n = 4096, sliding_window = 2048):
- # Creates a boolean mask for attention
mask = torch.ones(n, n, dtype = torch.bool)
if sliding_window == 0:
return torch.triu(mask, diagonal = 1, out = mask)
@@ -3242,12 +3105,10 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):
if "num_items_in_batch" in kwargs:
num_items_in_batch = kwargs["num_items_in_batch"]
if num_items_in_batch is None:
- # Remove it since the model does not support it!
kwargs.pop("num_items_in_batch")
elif "num_items_in_batch" not in inputs:
inputs["num_items_in_batch"] = num_items_in_batch
- # Get gradient accumulation steps if possible
if (
num_items_in_batch is None
and getattr(getattr(self, "args", self), "gradient_accumulation_steps", 1) != 1
@@ -3264,8 +3125,8 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):
"Using gradient accumulation will be very slightly less accurate.\n"
"Read more on gradient accumulation issues here: https://unsloth.ai/blog/gradient"
)
- # Gemma3 multimodal models in transformers 5.x require token_type_ids during training.
- # For text-only SFT, token_type_ids should be all zeros (no image tokens).
+ # Gemma3 multimodal models on transformers 5.x require token_type_ids during training; for
+ # text-only SFT they are all zeros.
if "token_type_ids" not in inputs and "input_ids" in inputs:
_inner = model
for _attr in ("base_model", "model", "model"):
@@ -3277,7 +3138,7 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):
_has_ccm = _mod is not None and hasattr(_mod, "create_causal_mask_mapping")
if _has_ccm and _inner.training:
inputs["token_type_ids"] = torch.zeros_like(inputs["input_ids"])
- # Gemma4 uses mm_token_type_ids (not token_type_ids) for VLM masking
+ # Gemma4 uses mm_token_type_ids, not token_type_ids, for VLM masking.
if "mm_token_type_ids" not in inputs and "input_ids" in inputs:
_inner = model
for _attr in ("base_model", "model", "model"):
@@ -3295,8 +3156,7 @@ def _unsloth_pre_compute_loss(self, model, inputs, *args, **kwargs):
def patch_gradient_accumulation_fix(Trainer):
- # Fixes gradient accumulation
- # Fixes Output 0 of UnslothFusedLossBackward is a view and is being modified inplace.
+ # Fixes "Output 0 of UnslothFusedLossBackward is a view and is being modified inplace" and gradient accumulation.
import inspect
if hasattr(Trainer, "get_batch_samples"):
@@ -3312,16 +3172,15 @@ def patch_gradient_accumulation_fix(Trainer):
if Trainer.get_batch_samples.__name__ != "_unsloth_get_batch_samples":
Trainer.get_batch_samples = _unsloth_get_batch_samples
- # Also fix passing in num_items_in_batch
if not hasattr(Trainer, "_old_compute_loss"):
- # Fix transformers 4.57.0 causing `Output 0 of UnslothFusedLossBackward is a view and is being modified inplace.`
+ # Fix transformers 4.57.0 raising "Output 0 of UnslothFusedLossBackward is a view and is being
+ # modified inplace".
function = inspect.getsource(Trainer.compute_loss)
if "loss *=" in function or "loss*=" in function:
where = function.find("def")
function = function.split("\n")
function = "\n".join(x[where:] for x in function)
- # Import all variables that need importing
import transformers.trainer
items_in_trainer = dir(transformers.trainer)
@@ -3336,7 +3195,9 @@ def patch_gradient_accumulation_fix(Trainer):
globals(),
)
- # Replace loss*= with loss = loss *
+ # Replace loss *= with loss = loss *.
+ # Fix when num_items_in_batch is nothing
+ # https://github.com/huggingface/transformers/pull/35207
function = re.sub(
r"loss[\s]{0,}\*\=",
"loss = loss *",
@@ -3354,7 +3215,6 @@ def patch_gradient_accumulation_fix(Trainer):
"`pip install --upgrade --no-cache-dir --no-deps unsloth transformers git+https://github.com/huggingface/trl.git`"
)
- # Also fix up loss scaling ie negate loss *= self.args.gradient_accumulation_steps
if not (
Trainer.training_step.__name__ == "_unsloth_training_step"
or "num_items_in_batch" not in inspect.signature(Trainer.training_step).parameters
@@ -3364,7 +3224,6 @@ def patch_gradient_accumulation_fix(Trainer):
function = function.split("\n")
function = "\n".join(x[where:] for x in function)
- # Import all variables that need importing
import transformers.trainer
items_in_trainer = dir(transformers.trainer)
@@ -3377,23 +3236,21 @@ def patch_gradient_accumulation_fix(Trainer):
globals(),
)
- # Accelerate does / self.args.gradient_accumulation_steps internally, so if we already
- # summed it up and did the division before hand, we have to negate it.
+ # Accelerate divides by args.gradient_accumulation_steps internally, so a sum already divided
+ # here has to be negated.
function = function.replace(
"loss *= self.args.gradient_accumulation_steps",
"if num_items_in_batch is not None: loss *= self.args.gradient_accumulation_steps",
)
function = function.replace("def training_step", "def _unsloth_training_step", 1)
- # Fix 4.47.0 issue where num_items_in_batch was removed
- # See https://github.com/huggingface/transformers/pull/35121
+ # Fix 4.47.0 removing num_items_in_batch (huggingface/transformers#35121) and the case where it
+ # is nothing (huggingface/transformers#35207).
function = function.replace(
"if self.model_accepts_loss_kwargs:",
"if False:",
)
- # Fix when num_items_in_batch is nothing
- # https://github.com/huggingface/transformers/pull/35207
function = re.sub(
r"else:\n"
r"([\s]{4,})self\.accelerator\.backward\(loss, \*\*kwargs\)\n"
@@ -3409,12 +3266,10 @@ def patch_gradient_accumulation_fix(Trainer):
exec(function, globals())
Trainer.training_step = _unsloth_training_step
- # Settle any deferred compile-mode switch at the start of every step.
- # On recompile-limit exhaustion unsloth_zoo defers the switch to eager
- # instead of flipping mid-call: non-reentrant checkpointing packs the
- # forward compiled and would recompute it eagerly, aborting the backward
- # with "Something went unexpectedly wrong in activation checkpoint".
- # Between steps nothing is half-packed, so the switch is free.
+ # Settle any deferred compile-mode switch at the start of every step: on recompile-limit
+ # exhaustion unsloth_zoo defers the switch to eager rather than flipping mid-call, since
+ # non-reentrant checkpointing packs the forward compiled and would recompute it eagerly,
+ # aborting the backward. Between steps nothing is half-packed.
if not getattr(Trainer, "_unsloth_settles_eager_fallbacks", False):
try:
from unsloth_zoo.temporary_patches.utils import (
@@ -3437,10 +3292,9 @@ def patch_gradient_accumulation_fix(Trainer):
Trainer.training_step = _unsloth_training_step_settling_fallbacks
Trainer._unsloth_settles_eager_fallbacks = True
- # Wrap Trainer.__init__: (1) pre-init, shadow accepts_loss_kwargs on whatever
- # model was passed in (covers PEFT wrapping done after FastModel.from_pretrained);
- # (2) post-init, clamp accelerator GA to 1 for the transformers 5.0-5.5
- # GradientAccumulationPlugin regression. No-op on 4.x and 5.6+. See #4982.
+ # Wrap Trainer.__init__: pre-init, shadow accepts_loss_kwargs on the model passed in (covers
+ # PEFT wrapping after from_pretrained); post-init, clamp accelerator GA to 1 for the
+ # transformers 5.0-5.5 GradientAccumulationPlugin regression. No-op on 4.x and 5.6+ (#4982).
if not getattr(Trainer, "_unsloth_init_wrapped_for_accelerate_gas", False):
_original_trainer_init = Trainer.__init__
@@ -3476,7 +3330,7 @@ def patch_gradient_accumulation_fix(Trainer):
def _unsloth_compile_cache_leaves():
- # Accepts `UNSLOTH_COMPILE_LOCATION` overrides (the env var unsloth_zoo honors).
+ # Accepts UNSLOTH_COMPILE_LOCATION overrides, the env var unsloth_zoo honors.
leaves = {"unsloth_compiled_cache", "unsloth_cache", "unsloth_compiled"}
loc = os.environ.get("UNSLOTH_COMPILE_LOCATION", "") or ""
loc = loc.rstrip("/\\")
@@ -3486,8 +3340,8 @@ def _unsloth_compile_cache_leaves():
def _forward_is_unsloth_compiled(model):
- # True iff forward was installed from the Unsloth compile cache directory.
- # __module__ stays as the transformers module, so check co_filename.
+ # True iff forward was installed from the Unsloth compile cache directory. __module__ stays the
+ # transformers module, so check co_filename.
leaves = _unsloth_compile_cache_leaves()
def check(m):
@@ -3522,8 +3376,8 @@ def _forward_is_unsloth_compiled(model):
def _find_concrete_accepts_loss_kwargs(model):
- # Walk wrapper chain for first class that declares accepts_loss_kwargs in its
- # own __mro__ dict. Avoids PEFT __getattr__ forwarding and our own shadow.
+ # Walk the wrapper chain for the first class declaring accepts_loss_kwargs in its own __mro__
+ # dict, avoiding PEFT __getattr__ forwarding and our own shadow.
seen = set()
m = model
for _ in range(6):
@@ -3545,8 +3399,8 @@ def _find_concrete_accepts_loss_kwargs(model):
def _shadow_accepts_loss_kwargs(model, value):
- # Set the attribute at every wrapper level so HF's hasattr check resolves
- # regardless of where accelerator / peft unwrap lands.
+ # Set the attribute at every wrapper level so HF's hasattr check resolves wherever accelerator
+ # or peft unwrap lands.
seen = set()
m = model
for _ in range(8):
@@ -3566,10 +3420,9 @@ def _shadow_accepts_loss_kwargs(model, value):
def apply_accepts_loss_kwargs_fix(model):
- # Shadow the correct accepts_loss_kwargs on the model so HF Trainer picks it
- # up via hasattr(unwrapped_model, ...). Replaces the old Trainer.__init__
- # source rewrite. Priority: compiled forward -> True; else first class attr
- # in wrapper chain; else leave HF default. Issue #4982.
+ # Shadow accepts_loss_kwargs on the model so HF Trainer sees it via hasattr(unwrapped_model).
+ # Priority: compiled forward -> True, else the first class attr in the chain, else the HF
+ # default (#4982).
if _forward_is_unsloth_compiled(model):
_shadow_accepts_loss_kwargs(model, True)
return "True (Unsloth compiled forward)"
@@ -3627,10 +3480,9 @@ def unsloth_compile_transformers(
return_logits = False,
unsloth_force_compile = False,
):
- # Again here, not only at import: a dotenv or config loader can populate
- # `UNSLOTH_FORCE_CUSTOM_DTYPE` after this package was imported, and the compiler
- # below reaches the older `unsloth_zoo` reader that still `eval`s the dtype field.
- # Idempotent, so a registered or already sanitized value is unchanged.
+ # Again here, not only at import: a dotenv or config loader can set UNSLOTH_FORCE_CUSTOM_DTYPE
+ # after this package was imported, and the compiler reaches the older zoo reader that evals the
+ # dtype field. Idempotent.
from ._custom_dtype import neutralize_inherited_custom_dtype
neutralize_inherited_custom_dtype()
@@ -3655,8 +3507,8 @@ def unsloth_compile_transformers(
supports_sdpa = [True]
- # Run patches BEFORE compiler so class replacements (e.g. GptOssTopKRouter,
- # GptOssExperts) are in place before the compiler caches references to them.
+ # Run patches BEFORE the compiler so class replacements (GptOssTopKRouter, GptOssExperts) are
+ # in place before it caches references to them.
_run_temporary_patches("pre_compile")
for model_type in model_types:
@@ -3688,13 +3540,12 @@ def unsloth_compile_transformers(
return_logits = return_logits,
supports_sdpa = supports_sdpa,
)
- # Redo patches which override compiler
+ # Redo patches which override the compiler.
_run_temporary_patches("post_compile")
return model_types, supports_sdpa[0]
-# We need an empty logits flag to warn people logits will not be returned anymore unless asked ie
-# os.environ['UNSLOTH_RETURN_LOGITS'] = '1'
+# An empty logits flag warns that logits are no longer returned unless os.environ['UNSLOTH_RETURN_LOGITS'] = '1'.
LOGITS_ERROR_STRING = (
"Unsloth: Logits are empty from 2024.11 onwards. To get raw logits again, please "
'set the environment variable `UNSLOTH_RETURN_LOGITS` to `"1" BEFORE starting to train ie before `trainer.train()`. For example:\n'
@@ -3730,7 +3581,8 @@ class EmptyLogits:
return LOGITS_ERROR_STRING
def __reduce__(self):
- # Stateless pickling so gather_object works on the sentinel
+ # Stateless pickling so gather_object works on the sentinel, and gathered copies compare equal
+ # in accelerate debug mode.
return (type(self), ())
def __eq__(self, other):
@@ -3749,8 +3601,8 @@ for j, function in enumerate(functions):
exec(f"EMPTY_LOGITS.{function} = raise_{j}", globals(), locals())
except:
continue
-# The loop above stomps pickle hooks with stubs returning None, which breaks
-# gather_object on EMPTY_LOGITS in distributed runs. Restore default pickling.
+# The loop above stomps pickle hooks with stubs returning None, breaking gather_object on
+# EMPTY_LOGITS in distributed runs; restore default pickling.
for function in ("__reduce__", "__reduce_ex__", "__getstate__", "__setstate__"):
try:
delattr(EMPTY_LOGITS, function)
@@ -3828,7 +3680,7 @@ def fast_inference_setup(model_name, model_config):
patch_vllm()
if model_name.endswith("unsloth-bnb-4bit"):
if not vllm_dynamic_quant_supported(model_name, model_config):
- # Instead use -bnb-4bit variant
+ # Instead use the -bnb-4bit variant.
logger.warning_once(
f"Unsloth: Switching from Unsloth dynamic quant to normal quant since\n"
f"we do not yet support fast inference for {model_name}"
@@ -3866,14 +3718,16 @@ def patch_peft_fast_inference(model):
model.fast_generate = model.model.fast_generate
model.fast_generate_batches = model.model.fast_generate_batches
- # load_lora copies into vLLM's own adapter tensors, so it needs an engine.
+ # load_lora copies into vLLM's own adapter tensors, so it needs an engine; with one, the Zoo
+ # helper stays, since vLLM reads the adapter back through its own loader.
from unsloth_zoo.vllm_utils import load_lora
model.load_lora = functools.partial(load_lora, model)
- # An engine keeps the Zoo helper it has always had: vLLM reads the saved
- # adapter back through its own LoRA loader, so what that file may carry
- # is its call, not one to change here.
+ # Without an engine there was no `save_lora` at all, and `save_lora` needs none: it is `save_pretrained`
+ # over the adapter. Gating it on the engine gave `fast_inference = False` GRPO runs `AttributeError:
+ # 'Lfm2ForCausalLM' object has no attribute 'save_lora'`. Set only when absent, so a model carrying its
+ # own keeps it.
if not hasattr(model, "save_lora"):
try:
from unsloth_zoo.vllm_utils import save_lora
@@ -3882,11 +3736,9 @@ def patch_peft_fast_inference(model):
if save_lora is not None:
model.save_lora = functools.partial(save_lora, model)
- # Without an engine there was no `save_lora` at all, and `save_lora` needs
- # none: it is `save_pretrained` over the adapter. Gating it on the engine
- # gave `fast_inference = False` GRPO runs `AttributeError:
- # 'Lfm2ForCausalLM' object has no attribute 'save_lora'`.
- # Set only when absent, so a model carrying its own keeps it.
+ # Without an engine there was no save_lora at all, and it needs none (it is save_pretrained
+ # over the adapter): gating it on the engine gave fast_inference = False GRPO runs an
+ # AttributeError. Set only when absent.
if not hasattr(model, "save_lora"):
model.save_lora = functools.partial(save_lora_adapter, model)
@@ -3913,7 +3765,7 @@ except:
class TorchAOConfig:
qat_scheme: Optional[str] = "int4"
- # Each (config, filter_fn) pair defines a quantization rule
+ # Each (config, filter_fn) pair defines a quantization rule.
base_config_and_filter_fns: List[
Tuple["AOBaseConfig", Optional[Callable[[torch.nn.Module, str], bool]]]
] = field(
@@ -3925,7 +3777,6 @@ class TorchAOConfig:
]
)
- # Optional transformation to apply before quantization setup
prequantization_transform: Optional[Callable[[torch.nn.Module], None]] = None
@@ -3936,34 +3787,30 @@ def _untie_input_output_embeddings(model: torch.nn.Module) -> None:
Model is modified in-place.
"""
- # 1) Persist setting in config
if hasattr(model.config, "tie_word_embeddings"):
model.config.tie_word_embeddings = False
- # 2) Find input and output embeddings
in_emb = model.get_input_embeddings()
out_proj = model.get_output_embeddings() or getattr(model, "lm_head", None)
if out_proj is None:
raise AttributeError("Couldn't locate output projection (lm_head).")
- # (Optional) sanity: shapes should match [vocab, hidden]
assert (
out_proj.weight.shape == in_emb.weight.shape
), f"Shape mismatch: out_proj {out_proj.weight.shape} vs in_emb {in_emb.weight.shape}"
- # 3) Only clone if they are actually tied (shared storage)
+ # Clone only if actually tied (shared storage), then prevent automatic re-tying and verify no
+ # shared storage remains.
if out_proj.weight.data_ptr() == in_emb.weight.data_ptr():
with torch.no_grad():
W = in_emb.weight.detach().clone()
out_proj.weight = torch.nn.Parameter(W) # new storage, keeps dtype/device
- # 4) Prevent future automatic re-tying
def _no_tie(self):
return
model.tie_weights = _no_tie.__get__(model, model.__class__)
- # 5) Verify no shared storage
assert out_proj.weight.data_ptr() != in_emb.weight.data_ptr(), "Embeddings still tied!"
@@ -3989,7 +3836,7 @@ def _convert_torchao_model(model):
for base_config, filter_fn in model._torchao_config.base_config_and_filter_fns:
quantize_(model, QATConfig(base_config, step = "convert"), filter_fn = filter_fn)
- # Default filter function used for quantize_
+ # Default filter function used for quantize_.
if filter_fn is None:
if "_default" in module_to_fqn_dict:
raise ValueError("Cannot use multiple default quantization configs")
@@ -4034,8 +3881,8 @@ def _prepare_model_for_qat(
except ImportError:
raise ImportError(TORCHAO_MSG)
- # Gemma3 models have issues with int8 embedding quantization due to their
- # large vocabulary size (262144). Auto-switch to int4 weight-only instead.
+ # Gemma3 int8 embedding quantization struggles with its 262144-token vocabulary, so auto-switch
+ # to int4 weight-only.
if qat_scheme == "int8-int4":
model_types = get_transformers_model_type(model.config)
is_gemma3 = any("gemma3" in mt or "gemma_3" in mt for mt in model_types)
@@ -4129,10 +3976,8 @@ def _prepare_model_for_qat(
except ImportError:
raise ImportError(TORCHAO_MSG)
- # IntxWeightOnlyConfig already defaults to
- # `mapping_type = MappingType.SYMMETRIC`, so we intentionally do not
- # import `MappingType` here. Matches the upstream Cactus runtime
- # int8 / per-group-32 / symmetric weight-only configuration.
+ # IntxWeightOnlyConfig already defaults to MappingType.SYMMETRIC, so MappingType is deliberately
+ # not imported. Matches the upstream Cactus int8 / per-group-32 / symmetric configuration.
group_size = 32
base_config = IntxWeightOnlyConfig(
weight_dtype = torch.int8,
@@ -4143,13 +3988,8 @@ def _prepare_model_for_qat(
and m.in_features >= group_size
and m.in_features % group_size == 0
)
- # Warn if any Linear layer is skipped by the cactus filter because
- # its in_features is not divisible by `group_size`. torchao's
- # PerGroup(32) quantizer rejects non-divisible widths at
- # `quantize_()` time, so the filter excludes those layers to keep
- # the QAT prepare step from crashing. Surface that silently-skipped
- # coverage gap to the user so they know some Linears will stay in
- # full precision during training.
+ # torchao's PerGroup(32) quantizer rejects in_features not divisible by group_size, so the
+ # filter excludes those Linears; warn, or the gap is silent and they stay full precision.
skipped_cactus_layers = [
name
for name, module in model.named_modules()
@@ -4178,7 +4018,6 @@ def _prepare_model_for_qat(
else:
torchao_config = qat_scheme
- # Save Torchao metadata everywhere
inner_model = model
while hasattr(inner_model, "model"):
inner_model._torchao_config = torchao_config
@@ -4194,7 +4033,7 @@ def _prepare_model_for_qat(
def patch_hf_quantizer():
- # To tell hf trainer that the quantized model is trainable
+ # Tell the HF trainer that the quantized model is trainable.
def make_trainable(self):
return True
@@ -4228,20 +4067,20 @@ patch_hf_quantizer()
def verify_fp8_support_if_applicable(model_config):
quant_method = get_quant_type(model_config)
if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE != "cuda":
+ # In case of block quantized, we allow L4 because we fall back to torchao kernels.
raise ValueError(
f"Unsloth: FP8 quantization is only supported on CUDA GPUs. You are using {DEVICE_TYPE}."
)
- # [TODO] Need to add FP8 support for Intel XPUs
if DEVICE_TYPE == "cuda":
major_version, minor_version = torch.cuda.get_device_capability()
if quant_method == "fbgemm_fp8" and major_version < 9:
- # While L4 does support FP8 as data type, it doesn't have fbgemm (package) support yet. So we restrict it.
+ # L4 has FP8 as a data type but no fbgemm support yet, except block quantized, where torchao
+ # kernels are the fallback.
raise ValueError(
f"Unsloth: FBGEMM FP8 quantization is only supported on H100 and higher GPUs. L4 is not supported. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details."
)
if quant_method == "fp8" and major_version * 10 + minor_version < 89:
- # In case of block quantized, we allow L4 because we fall back to torchao kernels.
raise ValueError(
f"Unsloth: FP8 quantization is only supported on L4 and higher GPUs with compute capability 8.9 or higher. You are using {torch.cuda.get_device_name()}. Refer to https://developer.nvidia.com/cuda-gpus for more details."
)
@@ -4285,9 +4124,6 @@ def hf_login(token: Optional[str] = None) -> Optional[str]:
# =============================================
-# MoE (Mixture of Experts) Detection and LoRA Utilities
-
-
def is_moe_model(model) -> bool:
"""
Detect if a model is a Mixture of Experts (MoE) model.
@@ -4300,17 +4136,14 @@ def is_moe_model(model) -> bool:
"""
config = getattr(model, "config", model)
- # Different MoE models use different config attribute names:
- # - Qwen3-MoE: num_experts
- # - GLM4-MoE: n_routed_experts, num_local_experts
- # - Mixtral: num_local_experts
+ # MoE configs name the count differently: num_experts (Qwen3-MoE), n_routed_experts /
+ # num_local_experts (GLM4-MoE), num_local_experts (Mixtral).
num_experts = None
for attr in ("num_experts", "n_routed_experts", "num_local_experts"):
num_experts = getattr(config, attr, None)
if num_experts is not None:
break
- # Check text_config for VL models
if num_experts is None and hasattr(config, "text_config"):
for attr in ("num_experts", "n_routed_experts", "num_local_experts"):
num_experts = getattr(config.text_config, attr, None)
@@ -4361,8 +4194,8 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]:
return {target_modules}
is_regex = re.search(r"[*+?()[\]{}|\\^$]", target_modules) is not None
- # Key detection on the mlp/ffn/experts path segment (absent from an
- # attention-only regex), never on q/k/v/o leaves alone.
+ # Key detection on the mlp/ffn/experts path segment, absent from an attention-only regex, never
+ # on q/k/v/o leaves alone.
targets_mlp_path = any(
tag in target_modules for tag in ("mlp", "ffn", "feed_forward", "experts")
)
@@ -4372,13 +4205,12 @@ def _moe_target_set_from_string(target_modules: str) -> set[str]:
named = {name for name in _MOE_BROAD_MLP_TARGETS if name in target_modules}
if named:
return named
- # A generic projection under an mlp path (e.g. ".*mlp.*proj"): any proj
- # occurrence that is not an attention leaf name.
+ # A generic projection under an mlp path (".*mlp.*proj"): any proj occurrence that is not an attention leaf name.
if re.search(r"(?<![qkvo]_)(?<!out_)(?<!in_)proj", target_modules):
return set(_MOE_BROAD_MLP_TARGETS)
- # The auto regex on fused-expert models lists only attention Linears as
- # leaves; its mlp tag block is the remaining MLP-intent signal. A regex
- # like "(mlp|self_attn).(q_proj|o_proj)" has neither and stays attention-only.
+ # The auto regex on fused-expert models lists only attention Linears, so its mlp tag block is
+ # the remaining MLP-intent signal; "(mlp|self_attn).(q_proj|o_proj)" has neither and stays
+ # attention-only.
if "mlp|feed_forward|ffn|dense" in target_modules:
return set(_MOE_BROAD_MLP_TARGETS)
@@ -4410,7 +4242,6 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[str
return None
config = getattr(model, "config", model)
- # Get num_experts from various possible config attributes
num_experts = None
for attr in ("num_experts", "n_routed_experts", "num_local_experts"):
num_experts = getattr(config, attr, None)
@@ -4448,10 +4279,9 @@ def get_moe_target_parameters(model, target_modules = None) -> Optional[List[str
alternate_name = "experts.down_proj",
)
- # gate_up_proj combines gate_proj and up_proj; also match the fused name directly.
- # Only target a fused expert Parameter that exists: per-expert Linear layouts
- # (e.g. gpt-oss bnb-4bit) have no fused Parameter and are handled by
- # get_moe_target_modules, so skip them rather than pass PEFT a dead path.
+ # gate_up_proj combines gate_proj and up_proj. Target only a fused expert Parameter that
+ # exists: per-expert Linear layouts (gpt-oss bnb-4bit) have none and go through
+ # get_moe_target_modules, so skip them rather than hand PEFT a dead path.
if "gate_proj" in target_set or "up_proj" in target_set or "gate_up_proj" in target_set:
if _moe_parameter_exists(model, gate_up_name):
moe_params.append(gate_up_name)
@@ -4512,11 +4342,9 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:
if not hasattr(model, "named_modules"):
return []
- # Scope the returned suffixes to the requested projection leaves, matching
- # get_moe_target_parameters: gate_proj/up_proj/gate_up_proj map to the fused
- # gate_up ModuleList (e.g. gate_up_projs); down_proj maps to the down ModuleList
- # (e.g. down_projs). A down-only (or gate/up-only) request must not pull in the
- # other projection.
+ # Scope the suffixes to the requested leaves, matching get_moe_target_parameters:
+ # gate/up/gate_up map to the fused gate_up ModuleList and down_proj to the down one, so a
+ # down-only request must not pull in the other projection.
want_gate_up = bool(target_set & {"gate_proj", "up_proj", "gate_up_proj"})
want_down = "down_proj" in target_set
@@ -4525,10 +4353,9 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:
if not isinstance(module, torch.nn.ModuleList) or len(module) == 0:
continue
parent, _, leaf = name.rpartition(".")
- # ModuleList directly under an ``experts`` container, holding only Linear
- # leaves (bnb Linear4bit / Linear8bitLt subclass nn.Linear). After PEFT has
- # wrapped the experts the child is a LoRA layer whose ``base_layer`` is the
- # Linear, so accept that too (keeps this idempotent across a re-wrapped model).
+ # ModuleList directly under an `experts` container holding only Linear leaves (bnb Linear4bit
+ # subclasses nn.Linear). After PEFT wraps the experts the child is a LoRA layer whose
+ # base_layer is the Linear, so accept that too and stay idempotent.
if not parent.endswith("experts"):
continue
if not all(
@@ -4545,7 +4372,7 @@ def get_moe_target_modules(model, target_modules = None) -> List[str]:
continue
if is_gate_up and not want_gate_up:
continue
- # One entry per expert index; ``leaf.<i>`` matches expert i in every layer.
+ # One entry per expert index; leaf.<i> matches expert i in every layer.
for expert_index in range(len(module)):
targets.add(f"{leaf}.{expert_index}")
@@ -4735,10 +4562,9 @@ def _drop_tied_output_module(model, modules_to_save, ensure_weight_tying):
or not EMBEDDING_MODULES <= _embedding_leaves(modules_to_save)
):
return modules_to_save
- # Bare leaf, not the caller's spelling: peft 0.18 ties `tied_weight_keys` minus whole
- # `modules_to_save` entries, so a qualified `model.embed_tokens` matches nothing there
- # and it reconstructs no output module, leaving the head frozen. PEFT resolves the
- # bare name by suffix on every version, so normalizing is safe.
+ # Bare leaf, not the caller's spelling: peft 0.18 ties tied_weight_keys minus whole
+ # modules_to_save entries, so a qualified model.embed_tokens matches nothing and no output
+ # module is reconstructed, leaving the head frozen. PEFT resolves bare names by suffix.
kept = [x for x in modules_to_save if _embedding_leaf(x) != "lm_head"]
normalized = []
for entry in kept:
@@ -4748,8 +4574,8 @@ def _drop_tied_output_module(model, modules_to_save, ensure_weight_tying):
elif _embedding_name_is_unambiguous(model, entry):
normalized.append(leaf)
else:
- # Rewriting would widen the scope to another subtree's embedding. Keep
- # both entries: two copies beats a silently frozen head.
+ # Rewriting would widen the scope to another subtree's embedding, so keep both entries: two
+ # copies beats a silently frozen head.
return modules_to_save
return normalized
@@ -4798,15 +4624,15 @@ def make_fast_generate_wrapper(original_generate):
@functools.wraps(original_generate)
def _fast_generate_wrapper(*args, **kwargs):
def _has_sampling_params(a):
- # SamplingParams passed directly or inside a positional list/tuple
+ # SamplingParams passed directly or inside a positional list/tuple.
return type(a).__name__ == "SamplingParams" or (
isinstance(a, (list, tuple))
and any(type(i).__name__ == "SamplingParams" for i in a)
)
def _is_vllm_prompt(a):
- # str prompt, a vLLM prompt dict (prompt / prompt_token_ids / prompt_embeds /
- # multi_modal_data), or a list/tuple of those
+ # A str prompt, a vLLM prompt dict (prompt / prompt_token_ids / prompt_embeds /
+ # multi_modal_data), or a list or tuple of those.
head = a[0] if isinstance(a, (list, tuple)) and len(a) > 0 else a
return isinstance(head, str) or (
isinstance(head, dict)
@@ -4816,7 +4642,7 @@ def make_fast_generate_wrapper(original_generate):
)
)
- # vLLM-only; also catch SamplingParams passed positionally (fast_generate(prompt, params))
+ # vLLM-only; also catches SamplingParams passed positionally.
if "sampling_params" in kwargs or any(_has_sampling_params(a) for a in args):
raise ValueError(
"Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). "
@@ -4830,12 +4656,9 @@ def make_fast_generate_wrapper(original_generate):
"Since `fast_inference=False`, LoRA weights are already merged into the model."
)
- # A vLLM-style prompt (string, {"prompt":..., "multi_modal_data":...} dict, or a list/tuple
- # of either) only works under vLLM; tokenize first when fast_inference=False. A positional
- # arg may be HF token ids, so check it conservatively with _is_vllm_prompt. The `prompts` /
- # `prompt_token_ids` / `prompt_embeds` keywords are vLLM-only names that HuggingFace generate
- # does not accept, so any of them being present is a vLLM-style call (even a bare token list,
- # or an explicit None from a defaulted kwargs dict), hence membership rather than a value check.
+ # A vLLM-style prompt only works under vLLM, so tokenize first when fast_inference=False. A
+ # positional arg may be HF token ids, so check conservatively. The prompts / prompt_token_ids /
+ # prompt_embeds keywords are vLLM-only names HF generate rejects, hence a membership test.
vllm_prompt_kwarg = any(
k in kwargs for k in ("prompts", "prompt_token_ids", "prompt_embeds")
)
@@ -4851,21 +4674,15 @@ def make_fast_generate_wrapper(original_generate):
" output = model.fast_generate(**inputs.to('cuda'), max_new_tokens=64, temperature=1.0)"
)
- # Call original generate
return original_generate(*args, **kwargs)
return _fast_generate_wrapper
-# Fix llm_int8_skip_modules not being respected for VLMs with dynamic quantization.
-# Dynamic quant checkpoints (eg gemma-3-4b-it-unsloth-bnb-4bit) encode skip paths as
-# "language_model.model.layers.*", but the live module tree surfaces them as
-# "model.language_model.layers.*". This prefix mismatch causes should_convert_module
-# to miss the skip list, so modules meant to stay in 16-bit get wrapped in Linear4bit
-# without a quant_state, producing "Skipping ... no quant_state found" warnings.
-# We patch should_convert_module to expand both the module name and the skip patterns
-# into all equivalent alias forms before delegating to the original matcher.
-# Ref: https://github.com/unslothai/unsloth/issues/4208
+# llm_int8_skip_modules was ignored for VLMs with dynamic quantization: those checkpoints spell
+# skip paths "language_model.model.layers.*" while the live tree has
+# "model.language_model.layers.*", so should_convert_module missed them and 16-bit modules were
+# wrapped in Linear4bit with no quant_state. Expand both into all aliases first (#4208).
import transformers.quantizers.quantizers_utils as _quantizers_utils
if (
diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py
index 0ac2bed66b5..d21a4e9327d 100644
--- a/unsloth/models/cohere.py
+++ b/unsloth/models/cohere.py
@@ -75,7 +75,7 @@ def fast_layernorm_inference(
return XX.to(X.dtype)
-# QK norm in Cohere
+# QK norm in Cohere.
def CohereAttention_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -90,7 +90,6 @@ def CohereAttention_fast_forward(
*args,
**kwargs,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
- # Clear inference
if hasattr(self, "paged_attention"):
del self.paged_attention_K
del self.paged_attention_V
@@ -123,14 +122,13 @@ def CohereAttention_fast_forward(
if past_key_value is not None:
kv_seq_len += past_key_value[0].shape[-2]
- # Extend RoPE dynamically to fit in VRAM
+ # Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.
if position_embeddings:
cos, sin = position_embeddings
else:
cos, sin = self.rotary_emb.get_cached(kv_seq_len, Q.device.index)
rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")
- # Useful for LongRoPE
Q, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)
if past_key_value is not None:
@@ -138,7 +136,6 @@ def CohereAttention_fast_forward(
V = torch.cat([past_key_value[1], V], dim = 2)
past_key_value = (K, V) if use_cache else None
- # Attention module
use_varlen = seq_info is not None and past_key_value is None
backend = select_attention_backend(use_varlen)
attention_config = AttentionConfig(
@@ -152,8 +149,8 @@ def CohereAttention_fast_forward(
"softmax_scale": getattr(self, "softmax_scale", None),
},
)
- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse
- # (KV cache / padding mask) raises. None => byte-identical default.
+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /
+ # padding mask) raises. None means the byte-identical default.
_pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)
context = AttentionContext(
bsz = bsz,
@@ -176,7 +173,7 @@ def CohereAttention_fast_forward(
return attn_output, attn_weights, past_key_value
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L590
+# Ported from transformers models/llama/modeling_llama.py#L590
def CohereDecoderLayer_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -191,14 +188,13 @@ def CohereDecoderLayer_fast_forward(
*args,
**kwargs,
):
- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:
+ if use_cache and hasattr(self, "_flag_for_generation"):
out_weight = torch.empty(
self.input_layernorm.weight.shape,
dtype = torch.float32,
device = f"{DEVICE_TYPE_TORCH}:0",
)
- # Self Attention
residual = hidden_states
hidden_states = fast_layernorm_inference(self.input_layernorm, hidden_states, out_weight)
hidden_states_attention, self_attn_weights, present_key_value = self.self_attn(
@@ -213,7 +209,6 @@ def CohereDecoderLayer_fast_forward(
**kwargs,
)
- # Fully Connected
hidden_states_mlp = fast_swiglu_inference(self.mlp, hidden_states)
residual += hidden_states_attention
residual += hidden_states_mlp
@@ -233,7 +228,6 @@ def CohereDecoderLayer_fast_forward(
**kwargs,
)
- # Fully Connected
hidden_states_mlp = self.mlp(hidden_states)
hidden_states = residual + hidden_states_attention + hidden_states_mlp
@@ -247,7 +241,7 @@ def CohereDecoderLayer_fast_forward(
from math import sqrt as math_sqrt
-KV_CACHE_INCREMENT = 256 # KV Cache update size
+KV_CACHE_INCREMENT = 256
torch_nn_functional_softmax = torch.nn.functional.softmax
torch_matmul = torch.matmul
@@ -270,15 +264,12 @@ def CohereAttention_fast_forward_inference(
n_groups = self.num_key_value_groups
n_kv_heads = self.config.num_key_value_heads
head_dim = self.head_dim
- # assert(n_kv_heads * n_groups == n_heads)
hidden_size = self.config.hidden_size
attention_size = n_heads * head_dim
seq_len = K1.shape[-2]
kv_seq_len = seq_len + 1
- # Prefill phase
- # if not hasattr(self, "paged_attention"):
if do_prefill:
self.paged_attention = torch.empty(
(KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),
@@ -301,7 +292,7 @@ def CohereAttention_fast_forward_inference(
(bsz, n_heads, 1, head_dim), dtype = dtype, device = f"{DEVICE_TYPE_TORCH}:0"
)
- # Mistral Nemo 12b has weird dimensions
+ # Mistral Nemo 12b has weird dimensions.
if attention_size != hidden_size:
self.temp_O = torch.empty(
(bsz, 1, hidden_size), dtype = dtype, device = f"{DEVICE_TYPE_TORCH}:0"
@@ -316,7 +307,7 @@ def CohereAttention_fast_forward_inference(
)
self.scalar = 1.0 / math_sqrt(self.head_dim)
self.half_head_dim = head_dim // 2
- # Cohere has QK layernorms
+ # Cohere has QK layernorms.
if self.use_qk_norm:
self.q_norm_out_weight = torch.empty(
self.q_norm.weight.shape,
@@ -355,10 +346,8 @@ def CohereAttention_fast_forward_inference(
Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight)
Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight)
- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
cos = cos[position_ids].unsqueeze(1)
@@ -372,18 +361,13 @@ def CohereAttention_fast_forward_inference(
Qn *= cos
Qn.addcmul_(RH_Q, sin)
- RH_K = RH_Q[
- :, :n_kv_heads, :, :
- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")
+ RH_K = RH_Q[:, :n_kv_heads, :, :]
RH_K[:, :, :, :h] = Kn[:, :, :, h:]
RH_K[:, :, :, h:] = Kn[:, :, :, :h]
RH_K[:, :, :, :h].neg_()
Kn *= cos
Kn.addcmul_(RH_K, sin)
- # New KV cache
- # Kn = torch.cat([K1, Kn], dim = 2)
- # Vn = torch.cat([V1, Vn], dim = 2)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
@@ -393,14 +377,14 @@ def CohereAttention_fast_forward_inference(
sliding_window = getattr(self.config, "sliding_window", None)
if sliding_window is not None and kv_seq_len > sliding_window:
start = kv_seq_len - sliding_window
- Knn = Kn[:, :, start:, :] # .contiguous()
- Vnn = Vn[:, :, start:, :] # .contiguous()
+ Knn = Kn[:, :, start:, :]
+ Vnn = Vn[:, :, start:, :]
if attention_mask is not None:
attention_mask = attention_mask[..., start:]
else:
Knn, Vnn = Kn, Vn
- # Grouped query attention
+ # Grouped query attention.
_, _, cached_len, _ = Knn.shape
if n_groups != 1:
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
@@ -408,14 +392,12 @@ def CohereAttention_fast_forward_inference(
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)
- # Attention
if bsz == 1:
- Qn *= (
- self.scalar
- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963
- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows
+ Qn *= self.scalar
+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805
+ # (comment 2153349963).
A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)
+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
A = torch_matmul(A, Vnn, out = Qn)
else:
A = scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = False)
@@ -425,8 +407,7 @@ def CohereAttention_fast_forward_inference(
return A, (Kn, Vn)
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825
-# @torch.inference_mode
+# Ported from transformers models/llama/modeling_llama.py#L825
def CohereModel_fast_forward_inference(
self,
input_ids,
@@ -455,7 +436,7 @@ def CohereModel_fast_forward_inference(
seq_len,
sliding_window = getattr(self.config, "sliding_window", None),
)
- # Pre-convert to bool once for all layers (avoids per-layer .eq(0))
+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).
if attention_mask is not None and attention_mask.dtype != torch.bool:
attention_mask = attention_mask.eq(0)
else:
diff --git a/unsloth/models/diffusion.py b/unsloth/models/diffusion.py
index 679046de4ac..8df1a65d1a5 100644
--- a/unsloth/models/diffusion.py
+++ b/unsloth/models/diffusion.py
@@ -38,22 +38,22 @@ from .loader_utils import (
__all__ = ["FastDiffusionModel", "DIFFUSION_MODEL_TYPES", "is_diffusion_model_type"]
-# transformers model_type strings routed to this slow path
+# transformers model_type strings routed to this slow path.
DIFFUSION_MODEL_TYPES = ("diffusion_gemma", "diffusion_gemma4")
-# Default LoRA targets: standard nn.Linear modules in the shared Gemma-4 backbone. The 128 MoE experts
-# are fused 3D Parameters (gate_up_proj/down_proj), not nn.Linear, so PEFT LoRA cannot target them.
+# Default LoRA targets are the standard nn.Linear modules in the shared Gemma-4 backbone: the 128
+# MoE experts are fused 3D Parameters, not nn.Linear, so PEFT LoRA cannot target them.
DIFFUSION_LORA_TARGETS = [
"q_proj",
"k_proj",
"v_proj",
- "o_proj", # attention
+ "o_proj",
"gate_proj",
"up_proj",
- "down_proj", # dense (non-expert) MLP
+ "down_proj",
]
-# Vision tower uses a custom Linear with the same suffix names; exclude it so only the text path is wrapped.
+# The vision tower uses a custom Linear with the same suffix names; exclude it so only the text path is wrapped.
DIFFUSION_LORA_EXCLUDE = r".*(vision_tower|embed_vision).*"
@@ -133,9 +133,8 @@ def _load_diffusion_config(
from transformers import DiffusionGemma4Config
aliased = DiffusionGemma4Config.from_dict(cd)
- # Only this object knows the rewrite happened; the checkpoint on disk still says
- # `diffusion_gemma`, so anything rebuilding the config from `model_name` -- the
- # planner -- hits the AutoConfig failure just caught here.
+ # Only this object knows the rewrite happened: the checkpoint on disk still says diffusion_gemma,
+ # so anything rebuilding the config from model_name hits the AutoConfig failure just caught.
aliased._unsloth_legacy_alias = True
return aliased
@@ -197,8 +196,8 @@ class FastDiffusionModel:
model_cls = _resolve_diffusion_model_class(config)
- # Prefetch the whole repo root so the weight load is a cache hit. No subfolder: the pipeline
- # loads every component subfolder, so narrowing would leave unet/vae/text_encoder to Xet.
+ # Prefetch the whole repo root so the weight load is a cache hit. No subfolder: the pipeline loads
+ # every component subfolder, so narrowing would leave unet/vae/text_encoder to Xet.
maybe_prefetch_hf_snapshot(
model_name,
token = token,
@@ -208,14 +207,14 @@ class FastDiffusionModel:
fast_inference = False,
force_download = kwargs.get("force_download", False),
use_safetensors = kwargs.get("use_safetensors"),
- # Forward variant (e.g. "fp16") so the warm keeps variant weights.
+ # Forward the variant (e.g. "fp16") so the warm keeps variant weights.
variant = kwargs.get("variant"),
)
- # Optional bitsandbytes quant. The MoE experts (3D Parameters) are not nn.Linear so bnb skips
- # them; only attention + dense MLP Linears quantize, lm_head/embeddings stay full precision.
- # Before the plan: the skip list becomes `modules_to_not_convert`, and sizing
- # those at 4 bits while they load in compute dtype OOMs a tight map.
+ # Optional bitsandbytes quant: the MoE experts (3D Parameters) are not nn.Linear so bnb skips
+ # them, and lm_head/embeddings stay full precision. Before the plan, since the skip list becomes
+ # modules_to_not_convert and sizing those at 4 bits while they load in compute dtype OOMs a tight
+ # map.
qcfg = None
if load_in_4bit or load_in_8bit:
from transformers import BitsAndBytesConfig
@@ -236,12 +235,8 @@ class FastDiffusionModel:
else:
qcfg = BitsAndBytesConfig(load_in_8bit = True)
- # Same leaf-level resolution as llama.py and vision.py: an unresolved "unsloth"
- # becomes torch.device("unsloth") in transformers and raises instead of loading.
- #
- # A legacy `diffusion_gemma` checkpoint loads only because the config above rewrites
- # the type in memory. The planner takes a name and rebuilds from the checkpoint, so
- # it hits the same error and would report it as a generic planning failure.
+ # Same leaf-level resolution as llama.py and vision.py: an unresolved "unsloth" becomes
+ # torch.device("unsloth") in transformers and raises instead of loading.
device_map = resolve_unsloth_device_map(
requested_device_map(device_map),
model_name,
@@ -256,8 +251,8 @@ class FastDiffusionModel:
device_map_planner_kwargs,
kwargs,
),
- # This leaf popped `local_files_only` off kwargs above, so the helper gets the
- # resolved value rather than the caller's raw mapping.
+ # This leaf popped local_files_only off kwargs above, so the helper gets the resolved value rather
+ # than the caller's raw mapping.
**planner_hub_kwargs(
{
"cache_dir": cache_dir,
@@ -291,7 +286,7 @@ class FastDiffusionModel:
# Match the load's weight format to the warm (None/auto already matches).
if kwargs.get("use_safetensors") is not None:
load_kwargs["use_safetensors"] = kwargs["use_safetensors"]
- # Forward variant to the real load so it reads the warmed variant weights.
+ # Forward the variant to the real load so it reads the warmed variant weights.
if kwargs.get("variant") is not None:
load_kwargs["variant"] = kwargs["variant"]
@@ -312,7 +307,7 @@ class FastDiffusionModel:
if not return_tokenizer:
return model, None
- # Prefer the processor (chat template + tokenizer); fall back to a bare tokenizer. Returned as
+ # Prefer the processor (chat template plus tokenizer), falling back to a bare tokenizer, returned as
# "tokenizer" to match the Unsloth (model, tokenizer) contract.
try:
tokenizer = AutoProcessor.from_pretrained(
@@ -354,10 +349,8 @@ class FastDiffusionModel:
if target_modules is None:
target_modules = DIFFUSION_LORA_TARGETS
- # NOTE: use_dora (and any other LoraConfig kwarg outside this allowlist,
- # e.g. use_rslora) is silently dropped here. Unsloth does not reach this
- # path today, so it's untested/unsupported on diffusion models; add it
- # to the allowlist below if that changes.
+ # use_dora, and any other LoraConfig kwarg outside this allowlist, is silently dropped: Unsloth does
+ # not reach this path today, so it is untested on diffusion models.
lora_kwargs = dict(
r = r,
lora_alpha = lora_alpha,
diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py
index d0c9e09b23f..f50eaabb329 100644
--- a/unsloth/models/falcon_h1.py
+++ b/unsloth/models/falcon_h1.py
@@ -84,7 +84,6 @@ def FalconH1Attention_fast_forward(
*args,
**kwargs,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
- # Clear inference
if hasattr(self, "paged_attention"):
del self.paged_attention_K
del self.paged_attention_V
@@ -108,7 +107,7 @@ def FalconH1Attention_fast_forward(
V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)
seq_info = get_packed_info_from_kwargs(kwargs, hidden_states.device)
- # Falcon H1 multiplies key states by a multiplier
+ # Falcon H1 multiplies key states by a multiplier.
K = K * self.config.key_multiplier
Q = Q.transpose(1, 2)
@@ -118,7 +117,7 @@ def FalconH1Attention_fast_forward(
if past_key_value is not None:
kv_seq_len += past_key_value[0].shape[-2]
- # Extend RoPE dynamically to fit in VRAM
+ # Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.
if position_embeddings and kv_seq_len <= position_embeddings[0].shape[0]:
cos, sin = position_embeddings
else:
@@ -135,7 +134,6 @@ def FalconH1Attention_fast_forward(
V = torch.cat([past_key_value[1], V], dim = 2)
past_key_value = (K, V) if use_cache else None
- # Attention module
window = (-1, -1)
use_varlen = (
attention_mask is None
@@ -229,15 +227,12 @@ def FalconH1Attention_fast_forward_inference(
n_groups = self.num_key_value_groups
n_kv_heads = self.config.num_key_value_heads
head_dim = self.head_dim
- # assert(n_kv_heads * n_groups == n_heads)
hidden_size = self.config.hidden_size
attention_size = n_heads * head_dim
seq_len = K1.shape[-2]
kv_seq_len = seq_len + 1
- # Prefill phase
- # if not hasattr(self, "paged_attention"):
device = hidden_states.device
if do_prefill:
self.paged_attention = torch.empty(
@@ -253,7 +248,7 @@ def FalconH1Attention_fast_forward_inference(
self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)
self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
- # Mistral Nemo 12b has weird dimensions
+ # Mistral Nemo 12b has weird dimensions.
if attention_size != hidden_size:
self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
else:
@@ -282,25 +277,17 @@ def FalconH1Attention_fast_forward_inference(
Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0])
Kn.mul_(self.config.key_multiplier)
Vn = fast_linear_forward(self.v_proj, Xn, out = self.temp_KV[1])
- Qn = Qn.view(
- bsz, 1, n_heads, head_dim
- ) # .transpose(1, 2) # we will transpose after normalisation
- Kn = Kn.view(
- bsz, 1, n_kv_heads, head_dim
- ) # .transpose(1, 2) # we will transpose after normalisation
+ Qn = Qn.view(bsz, 1, n_heads, head_dim)
+ Kn = Kn.view(bsz, 1, n_kv_heads, head_dim)
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
Qn = Qn.transpose(1, 2)
Kn = Kn.transpose(1, 2)
- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
-
- # Need to do it prior 2 steps before hitting full on short KV cache
- # or else error
+ # Must happen 2 steps before hitting full on a short KV cache, or it errors.
self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)
cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
cos = cos[position_ids].unsqueeze(1)
@@ -310,22 +297,17 @@ def FalconH1Attention_fast_forward_inference(
RH_Q = self.RH_Q
RH_Q[:, :, :, :h] = Qn[:, :, :, h:]
RH_Q[:, :, :, h:] = Qn[:, :, :, :h]
- RH_Q[:, :, :, :h].neg_() # torch.neg(RH_Q[:,:,:,:h], out = RH_Q[:,:,:,:h])
+ RH_Q[:, :, :, :h].neg_()
Qn *= cos
Qn.addcmul_(RH_Q, sin)
- RH_K = RH_Q[
- :, :n_kv_heads, :, :
- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")
+ RH_K = RH_Q[:, :n_kv_heads, :, :]
RH_K[:, :, :, :h] = Kn[:, :, :, h:]
RH_K[:, :, :, h:] = Kn[:, :, :, :h]
- RH_K[:, :, :, :h].neg_() # torch.neg(RH_K[:,:,:,:h], out = RH_K[:,:,:,:h])
+ RH_K[:, :, :, :h].neg_()
Kn *= cos
Kn.addcmul_(RH_K, sin)
- # New KV cache
- # Kn = torch.cat([K1, Kn], dim = 2)
- # Vn = torch.cat([V1, Vn], dim = 2)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
@@ -335,14 +317,14 @@ def FalconH1Attention_fast_forward_inference(
sliding_window = getattr(self.config, "sliding_window", None)
if sliding_window is not None and kv_seq_len > sliding_window:
start = kv_seq_len - sliding_window
- Knn = Kn[:, :, start:, :] # .contiguous()
- Vnn = Vn[:, :, start:, :] # .contiguous()
+ Knn = Kn[:, :, start:, :]
+ Vnn = Vn[:, :, start:, :]
if attention_mask is not None:
attention_mask = attention_mask[..., start:]
else:
Knn, Vnn = Kn, Vn
- # Grouped query attention
+ # Grouped query attention.
_, _, cached_len, _ = Knn.shape
if bsz == 1 or not SDPA_HAS_GQA and n_groups != 1:
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
@@ -350,14 +332,12 @@ def FalconH1Attention_fast_forward_inference(
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)
- # Attention
if bsz == 1:
- Qn *= (
- self.scalar
- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963
- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows
+ Qn *= self.scalar
+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805
+ # (comment 2153349963).
A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)
+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
A = torch_matmul(A, Vnn, out = Qn)
else:
if SDPA_HAS_GQA:
@@ -374,7 +354,7 @@ def FalconH1Attention_fast_forward_inference(
return A, (Kn, Vn)
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/falcon_h1/modeling_falcon_h1.py
+# Ported from transformers models/falcon_h1/modeling_falcon_h1.py
def FalconH1DecoderLayer_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -433,7 +413,6 @@ def FalconH1DecoderLayer_fast_forward(
hidden_states += residual
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm_inference(self.pre_ff_layernorm, hidden_states)
hidden_states = fast_swiglu_inference(self.feed_forward, hidden_states)
@@ -466,10 +445,9 @@ def FalconH1DecoderLayer_fast_forward(
hidden_states = mamba_hidden_states + attention_hidden_states
- # residual connection after attention + Mamba
+ # Residual connection after attention + Mamba.
hidden_states = residual + hidden_states
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm(self.pre_ff_layernorm, hidden_states)
hidden_states = self.feed_forward(hidden_states)
@@ -487,7 +465,7 @@ def _FalconH1_fast_forward_inference(
attention_fast_forward_inference = FalconH1Attention_fast_forward_inference,
mlp_fast_forward_inference = fast_swiglu_inference,
):
- # Makes attention and MLP customisable (e.g. qwen3/cohere custom attention).
+ # Makes attention and MLP customisable (qwen3/cohere custom attention).
def FalconH1Model_fast_forward_inference_custom(
self,
input_ids,
@@ -537,7 +515,7 @@ def _FalconH1_fast_forward_inference(
next_decoder_cache = []
for idx, decoder_layer in enumerate(self.model.layers):
- residual.copy_(X) # residual = X
+ residual.copy_(X)
X = fast_rms_layernorm_inference(
decoder_layer.input_layernorm,
X,
@@ -565,7 +543,7 @@ def _FalconH1_fast_forward_inference(
X += residual
- residual.copy_(X) # residual = X
+ residual.copy_(X)
X = fast_rms_layernorm_inference(
decoder_layer.pre_ff_layernorm,
X,
@@ -602,7 +580,8 @@ def _FalconH1_fast_forward_inference(
return FalconH1Model_fast_forward_inference_custom
-# Separate prepare_inputs_for_generation for Hybrid FalconH1
+# Separate prepare_inputs_for_generation for Hybrid FalconH1, which has its own cache type
+# FalconHybridMambaAttentionDynamicCache.
def _fast_prepare_inputs_for_generation(
self,
input_ids,
@@ -617,11 +596,9 @@ def _fast_prepare_inputs_for_generation(
# Overwritten -- has a unique cache type, `FalconHybridMambaAttentionDynamicCache`
empty_past_kv = past_key_values is None
- # If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens
- # Exception 1: when passing input_embeds, input_ids may be missing entries
- # Exception 2: some generation methods do special slicing of input_ids, so we don't need to do it here
- # Exception 3: with synced GPUs cache_position may go out of bounds, but we only want dummy token in that case.
- # (we can't check exception 3 while compiling)
+ # Slice input_ids through cache_position to keep only unprocessed tokens, except: with
+ # input_embeds input_ids may be missing entries; some generation methods slice input_ids
+ # themselves; and with synced GPUs cache_position may go out of bounds.
if not empty_past_kv:
if (
inputs_embeds is not None # Exception 1
@@ -634,16 +611,6 @@ def _fast_prepare_inputs_for_generation(
input_ids.shape[1] != cache_position.shape[0]
): # Default case (the "else", a no op, is Exception 2)
input_ids = input_ids[:, cache_position]
- # TODO: Wire up Cache to work for inference.
- # else:
- # past_key_values = FalconHybridMambaAttentionDynamicCache(
- # self.config,
- # input_ids.shape[0],
- # self.dtype,
- # devices=[
- # self.model.layers[i].mamba.conv1d.weight.device for i in range(self.config.num_hidden_layers)
- # ],
- # )
if attention_mask is not None and position_ids is None:
# create position_ids on the fly for batch generation
@@ -652,7 +619,7 @@ def _fast_prepare_inputs_for_generation(
if not empty_past_kv:
position_ids = position_ids[:, -input_ids.shape[1] :]
- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+ # inputs_embeds are only used in the 1st generation step.
if inputs_embeds is not None and empty_past_kv:
model_inputs = {"inputs_embeds": inputs_embeds}
else:
@@ -712,17 +679,14 @@ class FastFalconH1Model(FastLlamaModel):
PeftModelForCausalLM.forward = PeftModel_fast_forward
fix_prepare_inputs_for_generation(FalconH1ForCausalLM)
- # Solves https://github.com/unslothai/unsloth/issues/168
- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.
- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
- # https://github.com/huggingface/transformers/pull/27931
- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,
+ # huggingface/transformers#27931), so the old rotary embeddings are retained.
import transformers.models.falcon_h1.modeling_falcon_h1
transformers.models.falcon_h1.modeling_falcon_h1.FalconH1RotaryEmbedding = (
LlamaRotaryEmbedding
)
- # Avoids the float64 RMSNorm compile kernel that fails on Intel Arc DG2 (issue #6555).
+ # Avoids the float64 RMSNorm compile kernel that fails on Intel Arc DG2 (#6555).
patch_falcon_h1_rms_layernorm()
return
diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py
index df05176e0ba..05725726c38 100644
--- a/unsloth/models/gemma.py
+++ b/unsloth/models/gemma.py
@@ -63,23 +63,18 @@ torch_nn_functional_gelu = torch.nn.functional.gelu
def fast_geglu_inference(self, X):
- # gate = self.gate_proj(X)
- # up = self.up_proj(X)
bsz, _, hd = X.shape
- # mlp_size = self.config.intermediate_size
- # temp = torch.empty((2, bsz, 1, mlp_size), dtype = X.dtype, device = "cuda:0")
- gate = fast_linear_forward(self.gate_proj, X) # , out = temp[0])
- up = fast_linear_forward(self.up_proj, X) # , out = temp[1])
+ gate = fast_linear_forward(self.gate_proj, X)
+ up = fast_linear_forward(self.up_proj, X)
gate = torch_nn_functional_gelu(gate, approximate = "tanh")
gate *= up
- # X = self.down_proj(gate)
down = fast_linear_forward(self.down_proj, gate, out = up[:, :, :hd])
return down
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L590
+# Ported from transformers models/llama/modeling_llama.py#L590
def GemmaDecoderLayer_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -93,14 +88,13 @@ def GemmaDecoderLayer_fast_forward(
*args,
**kwargs,
):
- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:
+ if use_cache and hasattr(self, "_flag_for_generation"):
out_weight = torch.empty(
self.input_layernorm.weight.shape,
dtype = torch.float32,
device = f"{DEVICE_TYPE_TORCH}:0",
)
- # Self Attention
residual = hidden_states
hidden_states = fast_rms_layernorm_inference_gemma(
self.input_layernorm, hidden_states, out_weight
@@ -118,7 +112,6 @@ def GemmaDecoderLayer_fast_forward(
)
hidden_states += residual
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm_inference_gemma(
self.post_attention_layernorm, hidden_states, out_weight
@@ -141,7 +134,6 @@ def GemmaDecoderLayer_fast_forward(
)
hidden_states = residual + hidden_states
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states, gemma = True)
hidden_states = self.mlp(hidden_states)
@@ -158,8 +150,7 @@ def GemmaDecoderLayer_fast_forward(
from math import sqrt as math_sqrt
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825
-# @torch.inference_mode
+# Ported from transformers models/llama/modeling_llama.py#L825
def GemmaModel_fast_forward_inference(
self,
input_ids,
@@ -179,8 +170,7 @@ def GemmaModel_fast_forward_inference(
input_ids = input_ids[:, : self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config)))
- # 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32
- # 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32
+ # 3072**0.5 is 55.5000 in bfloat16 against 55.4256 in float32, and 2048**0.5 is 45.2500 against 45.2548.
hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype)
bsz, q_len, hd = hidden_states.shape
@@ -193,11 +183,11 @@ def GemmaModel_fast_forward_inference(
hidden_states,
seq_len,
)
- # Pre-convert to bool once for all layers (avoids per-layer .eq(0))
+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).
if attention_mask is not None and attention_mask.dtype != torch.bool:
attention_mask = attention_mask.eq(0)
- # Compute rotary_seq_len once to avoid per-layer GPU-CPU sync from .item()
+ # Compute rotary_seq_len once to avoid a per-layer GPU-CPU sync from .item().
rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)
next_decoder_cache = []
@@ -242,27 +232,24 @@ def GemmaModel_fast_forward_inference(
)
-# Follows line by line https://github.com/google-deepmind/gemma/blob/main/gemma/positional_embeddings.py#L45
-# Formulates cos and sin differently from Llama!
+# Follows google-deepmind/gemma positional_embeddings.py#L45 line by line, which formulates cos and
+# sin differently from Llama.
class GemmaFixedRotaryEmbedding(torch.nn.Module):
- # Fixes https://github.com/huggingface/transformers/pull/28837
- # https://github.com/microsoft/DeepSpeed/issues/4932
- # The precision of RoPE buffers is not correct, so we cast to int64.
+ # RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.
def __init__(
self,
dim = None,
max_position_embeddings = 2048,
base = 10000,
device = None,
- config = None, # [TODO] Hack to pass in config - need to remove later
+ config = None,
):
super().__init__()
- # In transformers 5.0+, RotaryEmbedding(config) passes config as first positional arg (dim)
+ # In transformers 5.0+, RotaryEmbedding(config) passes config as the first positional arg (dim).
if config is None and dim is not None and hasattr(dim, "max_position_embeddings"):
config = dim
dim = None
if config is not None:
- # [TODO] Hack to pass in config - need to remove later
base = _get_rope_theta(config, default = base)
partial_rotary_factor = (
config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
@@ -275,7 +262,7 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
self.dim = dim
self.max_position_embeddings = max_position_embeddings
self.base = base
- # Dynamic RoPE we first set it to a max of 4 * 8192 tokens then we iteratively grow this
+ # Dynamic RoPE starts at a max of 4 * 8192 tokens and grows iteratively in increments of 8192.
self.current_rope_size = min(4 * 8192, self.max_position_embeddings)
self.multi_gpu_cos_cached = [None] * DEVICE_COUNT
self.multi_gpu_sin_cached = [None] * DEVICE_COUNT
@@ -288,7 +275,6 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
dtype = torch.get_default_dtype(),
)
- # dummy so that patch_utils doesn't fail for now
self.cos_cached = torch.empty(
1, device = get_current_device(), dtype = torch.get_default_dtype()
)
@@ -297,8 +283,8 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
)
def _set_cos_sin_cache(self, seq_len, device, dtype):
- # Note: on the original Llama codebase, these tensors are created on the target device (and not on CPU) and
- # in FP32. They are applied (multiplied) in FP32 as well.
+ # The original Llama codebase creates these on the target device in FP32 and multiplies in FP32; the
+ # difference here is explicit division t/x rather than t * (1/x).
self.current_rope_size = seq_len
# The difference is we do division explicitly instead of t * (1/x) ie we do t/x.
@@ -311,9 +297,9 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
radians_new = radians_new.squeeze(0)
emb = torch.cat((radians_new, radians_new), dim = -1)
- # We must do RoPE in float32!
- cos = emb.cos().to(device = device, non_blocking = True) # , dtype = dtype)
- sin = emb.sin().to(device = device, non_blocking = True) # , dtype = dtype)
+ # RoPE must be done in float32.
+ cos = emb.cos().to(device = device, non_blocking = True)
+ sin = emb.sin().to(device = device, non_blocking = True)
self.multi_gpu_cos_cached[device.index] = cos
self.multi_gpu_sin_cached[device.index] = sin
return cos, sin
@@ -324,7 +310,6 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
position_ids = None,
seq_len = None,
):
- # x: [bs, num_attention_heads, seq_len, head_size]
if seq_len is not None and seq_len > self.current_rope_size:
self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)
@@ -358,9 +343,7 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):
"""LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
- # Fixes https://github.com/huggingface/transformers/pull/28837
- # https://github.com/microsoft/DeepSpeed/issues/4932
- # The precision of RoPE buffers is not correct, so we cast to int64.
+ # RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.
def __init__(
self,
dim = None,
@@ -368,7 +351,7 @@ class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):
base = 10000,
device = None,
scaling_factor = 1.0,
- config = None, # [TODO] Hack to pass in config - need to remove later
+ config = None,
):
self.scaling_factor = scaling_factor
super().__init__(
@@ -380,8 +363,8 @@ class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):
)
def _set_cos_sin_cache(self, seq_len, device, dtype):
- # Note: on the original Llama codebase, these tensors are created on the target device (and not on CPU) and
- # in FP32. They are applied (multiplied) in FP32 as well.
+ # The original Llama codebase creates these on the target device in FP32 and multiplies in FP32; the
+ # difference here is explicit division t/x rather than t * (1/x).
self.current_rope_size = seq_len
# The difference is we do division explicitly instead of t * (1/x) ie we do t/x.
@@ -395,9 +378,9 @@ class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):
radians_new = radians_new.squeeze(0)
emb = torch.cat((radians_new, radians_new), dim = -1)
- # We must do RoPE in float32!
- cos = emb.cos().to(device = device, non_blocking = True) # , dtype = dtype)
- sin = emb.sin().to(device = device, non_blocking = True) # , dtype = dtype)
+ # RoPE must be done in float32.
+ cos = emb.cos().to(device = device, non_blocking = True)
+ sin = emb.sin().to(device = device, non_blocking = True)
self.multi_gpu_cos_cached[device.index] = cos
self.multi_gpu_sin_cached[device.index] = sin
return cos, sin
@@ -424,11 +407,8 @@ class FastGemmaModel(FastLlamaModel):
PeftModelForCausalLM.forward = PeftModel_fast_forward
fix_prepare_inputs_for_generation(GemmaForCausalLM)
- # Solves https://github.com/unslothai/unsloth/issues/168
- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.
- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
- # https://github.com/huggingface/transformers/pull/27931
- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,
+ # huggingface/transformers#27931), so the old rotary embeddings are retained.
import transformers.models.gemma.modeling_gemma
transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding = GemmaFixedRotaryEmbedding
@@ -440,18 +420,15 @@ class FastGemmaModel(FastLlamaModel):
tokenizer,
correct_dtype = None,
):
- # Gemma does not downcast RoPE
+ # Gemma does not downcast RoPE.
model, tokenizer = patch_model_and_tokenizer(
model, tokenizer, downcast_rope = False, correct_dtype = correct_dtype
)
- # Add 1 to weight
- # return output * (1 + self.weight)
- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/gemma/modeling_gemma.py#L89
+ # Gemma returns output * (1 + self.weight); see transformers models/gemma/modeling_gemma.py#L89.
from transformers.models.gemma.modeling_gemma import GemmaRMSNorm
- # Freeze all parameters except LoRA
- # We do this first since += 1 seems to not be liked by requires_grad = True
+ # Freeze all parameters except LoRA first, since += 1 does not agree with requires_grad = True.
for name, param in model.named_parameters():
if ".lora_A." in name or ".lora_B." in name:
param.requires_grad_(True)
@@ -461,15 +438,11 @@ class FastGemmaModel(FastLlamaModel):
# Patch RMS Layernorm
for name, module in model.named_modules():
if isinstance(module, GemmaRMSNorm):
- # Must be in float32
- # https://github.com/keras-team/keras-nlp/blob/v0.8.2/keras_nlp/models/gemma/rms_normalization.py#L36
- # module = module.to(torch.float32)
- # Leave + 1 to Triton kernel itself
- # module.weight += 1.0 # return output * (1 + self.weight)
+ # Must be in float32 (keras-nlp gemma/rms_normalization.py#L36); the +1 is left to the Triton kernel
+ # itself.
if not hasattr(module, "variance_epsilon"):
module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon
- # Clear deleted GPU items
import gc
for _ in range(3):
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index 00849b62afc..2d1ab52bc91 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -69,7 +69,7 @@ if HAS_FLASH_ATTENTION_SOFTCAPPING:
from flash_attn import flash_attn_func
-# Logit softcapping
+# Logit softcapping.
def Gemma2Attention_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -83,7 +83,6 @@ def Gemma2Attention_fast_forward(
*args,
**kwargs,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
- # Clear inference
if hasattr(self, "paged_attention"):
del self.paged_attention_K
del self.paged_attention_V
@@ -117,7 +116,7 @@ def Gemma2Attention_fast_forward(
rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")
if rope_position_ids is not None:
- # Useful for LongRoPE
+ # Useful for LongRoPE.
cos_var, sin_var = self.rotary_emb.get_cached(kv_seq_len, device_index)
Q, K = fast_rope_embedding(Q, K, cos_var, sin_var, rope_position_ids)
else:
@@ -128,7 +127,7 @@ def Gemma2Attention_fast_forward(
V = torch.cat([past_key_value[1], V], dim = 2)
past_key_value = (K, V) if use_cache else None
- # Only enable if the attention_mask is True
+ # Only enable if the attention_mask is True.
use_sliding_window = kwargs.get("use_sliding_window")
has_sliding_window = (
use_sliding_window
@@ -169,10 +168,8 @@ def Gemma2Attention_fast_forward(
},
)
- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse
- # (KV cache / padding mask) raises. None => byte-identical default. gemma2 is
- # sliding-window and softcapped: the engage gate caps spans at the window and
- # excludes softcap models entirely, so PG never engages here.
+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /
+ # padding mask) raises. None means the byte-identical default.
_pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)
context = AttentionContext(
bsz = bsz,
@@ -201,7 +198,7 @@ def Gemma2Attention_fast_forward(
return A, None, past_key_value
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L590
+# Ported from transformers models/llama/modeling_llama.py#L590
def Gemma2DecoderLayer_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -215,14 +212,13 @@ def Gemma2DecoderLayer_fast_forward(
*args,
**kwargs,
):
- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:
+ if use_cache and hasattr(self, "_flag_for_generation"):
out_weight = torch.empty(
self.input_layernorm.weight.shape,
dtype = torch.float32,
device = f"{DEVICE_TYPE_TORCH}:0",
)
- # Self Attention
residual = hidden_states
hidden_states = fast_rms_layernorm_inference_gemma(
self.input_layernorm, hidden_states, out_weight
@@ -244,7 +240,6 @@ def Gemma2DecoderLayer_fast_forward(
)
hidden_states += residual
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm_inference_gemma(
self.pre_feedforward_layernorm, hidden_states, out_weight
@@ -271,7 +266,6 @@ def Gemma2DecoderLayer_fast_forward(
hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states, gemma = True)
hidden_states = residual + hidden_states
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm(
self.pre_feedforward_layernorm, hidden_states, gemma = True
@@ -292,7 +286,7 @@ def Gemma2DecoderLayer_fast_forward(
from math import sqrt as math_sqrt
-KV_CACHE_INCREMENT = 256 # KV Cache update size
+KV_CACHE_INCREMENT = 256
torch_nn_functional_softmax = torch.nn.functional.softmax
torch_matmul = torch.matmul
torch_tanh = torch.tanh
@@ -317,7 +311,6 @@ def Gemma2Attention_fast_forward_inference(
n_groups = self.num_key_value_groups
n_kv_heads = self.config.num_key_value_heads
head_dim = self.head_dim
- # assert(n_kv_heads * n_groups == n_heads)
hidden_size = self.config.hidden_size
attention_size = n_heads * head_dim
@@ -325,8 +318,6 @@ def Gemma2Attention_fast_forward_inference(
kv_seq_len = seq_len + 1
device = hidden_states.device
- # Prefill phase
- # if not hasattr(self, "paged_attention"):
if do_prefill:
self.paged_attention = torch.empty(
(KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),
@@ -340,16 +331,14 @@ def Gemma2Attention_fast_forward_inference(
self.temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)
self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)
self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
- # Only for Gemma2
+ # Only for Gemma2.
self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
self.attention = torch.empty(
(bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device
)
- # See https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e
- # Gemma 9b should use 256 and not 224 (hs / nah). 27b uses the below
- # We default to using the config file itself
- # s = self.config.hidden_size // self.config.num_attention_heads
+ # Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,
+ # so default to the config. See google/gemma_pytorch commit 03e6575.
self.scalar = 1.0 / math_sqrt(self.config.query_pre_attn_scalar)
# self.scalar = 1.0 / math_sqrt(self.config.hidden_size // self.config.num_attention_heads)
self.half_head_dim = head_dim // 2
@@ -376,10 +365,8 @@ def Gemma2Attention_fast_forward_inference(
Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
cos = cos[position_ids].unsqueeze(1)
@@ -393,18 +380,13 @@ def Gemma2Attention_fast_forward_inference(
Qn *= cos
Qn.addcmul_(RH_Q, sin)
- RH_K = RH_Q[
- :, :n_kv_heads, :, :
- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")
+ RH_K = RH_Q[:, :n_kv_heads, :, :]
RH_K[:, :, :, :h] = Kn[:, :, :, h:]
RH_K[:, :, :, h:] = Kn[:, :, :, :h]
RH_K[:, :, :, :h].neg_()
Kn *= cos
Kn.addcmul_(RH_K, sin)
- # New KV cache
- # Kn = torch.cat([K1, Kn], dim = 2)
- # Vn = torch.cat([V1, Vn], dim = 2)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
@@ -414,12 +396,12 @@ def Gemma2Attention_fast_forward_inference(
sliding_window = self.config.sliding_window
if use_sliding_window and kv_seq_len > sliding_window:
start = kv_seq_len - sliding_window
- Knn = Kn[:, :, start:, :] # .contiguous()
- Vnn = Vn[:, :, start:, :] # .contiguous()
+ Knn = Kn[:, :, start:, :]
+ Vnn = Vn[:, :, start:, :]
else:
Knn, Vnn = Kn, Vn
- # Grouped query attention
+ # Grouped query attention.
_, _, cached_len, _ = Knn.shape
if n_groups != 1:
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
@@ -427,29 +409,25 @@ def Gemma2Attention_fast_forward_inference(
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)
- # Attention
- # [TODO] Gemma2 uses manual matmul for all batch sizes since SDPA lacks
- # softcapping (tanh logit scaling). If PyTorch adds a softcap param to
- # SDPA, consider SDPA for bsz > 1 to match the llama/qwen3 pattern.
- Qn *= (
- self.scalar
- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963
- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows
+ # Gemma2 uses manual matmul for all batch sizes since SDPA lacks softcapping (tanh logit scaling);
+ # if PyTorch adds a softcap param, consider SDPA for bsz > 1 to match llama/qwen3.
+ Qn *= self.scalar
+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805 (comment 2153349963).
A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
- # Softcapping must happen BEFORE the mask is applied.
- # Reference: google-deepmind/gemma _modules.py and transformers gemma2 eager_attention_forward
+ # Softcapping must happen BEFORE the mask is applied; see google-deepmind/gemma _modules.py and
+ # transformers gemma2 eager_attention_forward.
A *= self.reciprocal_t
A.tanh_()
- A *= self.t # Logit softcapping
+ A *= self.t
if attention_mask is not None and isinstance(attention_mask, torch.Tensor):
- # Slice mask to match K/V when sliding window is active
+ # Slice mask to match K/V when sliding window is active.
if attention_mask.shape[-1] != A.shape[-1]:
attention_mask = attention_mask[:, :, :, -A.shape[-1] :]
A += attention_mask
- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)
+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
A = torch_matmul(A, Vnn, out = Qn)
A = A.transpose(1, 2)
A = A.reshape(bsz, 1, attention_size)
@@ -457,8 +435,7 @@ def Gemma2Attention_fast_forward_inference(
return A, (Kn, Vn)
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825
-# @torch.inference_mode
+# Ported from transformers models/llama/modeling_llama.py#L825
def Gemma2Model_fast_forward_inference(
self,
input_ids,
@@ -478,8 +455,7 @@ def Gemma2Model_fast_forward_inference(
input_ids = input_ids[:, : self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config)))
- # 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32
- # 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32
+ # 3072**0.5 is 55.5000 in bfloat16 against 55.4256 in float32, and 2048**0.5 is 45.2500 against 45.2548.
hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype)
bsz, q_len, hd = hidden_states.shape
@@ -507,8 +483,7 @@ def Gemma2Model_fast_forward_inference(
GA = attention_mask
next_decoder_cache = []
for idx, decoder_layer in enumerate(self.model.layers):
- # For pipeline parallelism, we need to move all tensors to the same device
- # note that this movement is once per GPU in PP
+ # For pipeline parallelism every tensor must be on the same device; this movement happens once per GPU in PP.
device_index = getattr(decoder_layer, "_per_layer_device_index", 0)
hidden_states, position_ids = move_to_device(device_index, hidden_states, position_ids)
@@ -582,11 +557,8 @@ class FastGemma2Model(FastLlamaModel):
PeftModelForCausalLM.forward = PeftModel_fast_forward
fix_prepare_inputs_for_generation(Gemma2ForCausalLM)
- # Solves https://github.com/unslothai/unsloth/issues/168
- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.
- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
- # https://github.com/huggingface/transformers/pull/27931
- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,
+ # huggingface/transformers#27931), so the old rotary embeddings are retained.
import transformers.models.gemma2.modeling_gemma2
transformers.models.gemma2.modeling_gemma2.Gemma2RotaryEmbedding = GemmaFixedRotaryEmbedding
@@ -598,36 +570,28 @@ class FastGemma2Model(FastLlamaModel):
tokenizer,
correct_dtype = None,
):
- # Gemma does not downcast RoPE
+ # Gemma does not downcast RoPE.
model, tokenizer = patch_model_and_tokenizer(
model, tokenizer, downcast_rope = False, correct_dtype = correct_dtype
)
- # Add 1 to weight
- # return output * (1 + self.weight)
- # https://github.com/huggingface/transformers/blob/main/src/transformers/models/gemma/modeling_gemma.py#L89
+ # Gemma returns output * (1 + self.weight); see transformers models/gemma/modeling_gemma.py#L89.
from transformers.models.gemma2.modeling_gemma2 import Gemma2RMSNorm
- # Freeze all parameters except LoRA
- # We do this first since += 1 seems to not be liked by requires_grad = True
+ # Freeze all parameters except LoRA first, since += 1 does not agree with requires_grad = True.
for name, param in model.named_parameters():
if ".lora_A." in name or ".lora_B." in name:
param.requires_grad_(True)
else:
param.requires_grad_(False)
- # Patch RMS Layernorm
for name, module in model.named_modules():
if isinstance(module, Gemma2RMSNorm):
- # Must be in float32
- # https://github.com/keras-team/keras-nlp/blob/v0.8.2/keras_nlp/models/gemma/rms_normalization.py#L36
- # module = module.to(torch.float32)
- # Leave + 1 to Triton kernel itself
- # module.weight += 1.0 # return output * (1 + self.weight)
+ # Must be in float32 (keras-nlp gemma/rms_normalization.py#L36); the +1 is left to the Triton kernel
+ # itself.
if not hasattr(module, "variance_epsilon"):
module.variance_epsilon = module.eps # Gemma doesn't use variance_epsilon
- # Clear deleted GPU items
import gc
for _ in range(3):
diff --git a/unsloth/models/glm4_moe.py b/unsloth/models/glm4_moe.py
index 503afbce4b6..146d3b03f9c 100644
--- a/unsloth/models/glm4_moe.py
+++ b/unsloth/models/glm4_moe.py
@@ -43,7 +43,7 @@ import torch.nn.functional as F
from typing import Optional, Tuple
from ..kernels import fast_rms_layernorm
-# grouped_gemm expects its parent dir on sys.path
+# grouped_gemm expects its parent dir on sys.path.
HAS_GROUPED_GEMM = False
try:
import sys
@@ -55,8 +55,8 @@ try:
if _moe_path not in sys.path:
sys.path.insert(0, _moe_path)
- # Import first to apply the TMA compatibility shim (patches triton.language
- # for both old and new TMA API names)
+ # Import first to apply the TMA compatibility shim, which patches triton.language for both old and
+ # new TMA API names.
import grouped_gemm # noqa: F401 - triggers TMA compatibility shim
from grouped_gemm.interface import grouped_gemm
@@ -72,7 +72,6 @@ except ImportError as e:
warnings.warn(f"Grouped GEMM not available: {e}. MoE will use fallback implementation.")
-# Import transformers GLM4 MoE Lite classes
try:
from transformers.models.glm4_moe_lite.modeling_glm4_moe_lite import (
Glm4MoeLiteAttention,
@@ -89,7 +88,7 @@ try:
except ImportError:
HAS_GLM4_MOE = False
- # Create dummy classes for type checking
+ # Dummy classes for type checking.
class Glm4MoeLiteAttention:
pass
@@ -137,7 +136,7 @@ def Glm4MoeLiteMoE_fast_forward(self, hidden_states):
router_logits = self.gate(hidden_states) # [num_tokens, n_routed_experts]
topk_indices, topk_weights = self.route_tokens_to_experts(router_logits)
- # Sigmoid router returns fp32; cast weights to hidden_states dtype (e.g. bf16)
+ # The sigmoid router returns fp32; cast weights to the hidden_states dtype.
topk_weights = topk_weights.to(hidden_states.dtype)
with torch.no_grad():
@@ -146,7 +145,7 @@ def Glm4MoeLiteMoE_fast_forward(self, hidden_states):
)
if HAS_GROUPED_GEMM:
- # Under autocast hidden_states may be fp32 while weights are bf16
+ # Under autocast hidden_states may be fp32 while weights are bf16.
hidden_states = hidden_states.to(self.experts.gate_up_proj.dtype)
# gate_up_proj: [num_tokens, hidden_dim] -> [total_tokens, 2*intermediate_dim]
@@ -162,7 +161,7 @@ def Glm4MoeLiteMoE_fast_forward(self, hidden_states):
is_first_gemm = True,
)
- # Activation: SiLU(gate) * up
+ # SiLU(gate) * up.
gate, up = intermediate.chunk(2, dim = -1)
intermediate = torch_nn_functional_silu(gate) * up
@@ -179,7 +178,7 @@ def Glm4MoeLiteMoE_fast_forward(self, hidden_states):
is_first_gemm = False,
)
- # Merge topk weights: [num_tokens, top_k, hidden_dim] -> [num_tokens, hidden_dim]
+ # Merge topk weights: [num_tokens, top_k, hidden_dim] -> [num_tokens, hidden_dim].
hidden_states = (
expert_output.view(num_tokens, self.top_k, hidden_dim) * topk_weights.unsqueeze(-1)
).sum(dim = 1)
@@ -242,10 +241,9 @@ def Glm4MoeLiteNaiveMoe_fast_forward(
with torch.no_grad():
token_counts_by_expert, gather_indices = get_routing_indices(top_k_index, self.num_experts)
- # Under autocast hidden_states may be fp32 while weights are bf16
+ # Under autocast hidden_states may be fp32 while weights are bf16.
hidden_states = hidden_states.to(self.gate_up_proj.dtype)
- # First grouped GEMM: gate_up_proj
intermediate = grouped_gemm(
X = hidden_states,
W = self.gate_up_proj,
@@ -262,7 +260,6 @@ def Glm4MoeLiteNaiveMoe_fast_forward(
gate, up = intermediate.chunk(2, dim = -1)
intermediate = self.act_fn(gate) * up
- # Second grouped GEMM: down_proj
expert_output = grouped_gemm(
X = intermediate,
W = self.down_proj,
@@ -315,13 +312,11 @@ def Glm4MoeLiteDecoderLayer_fast_forward(
)
hidden_states = residual + hidden_states
- # MLP/MoE
residual = hidden_states
hidden_states = fast_rms_layernorm_inference(self.post_attention_layernorm, hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
else:
- # Training path
residual = hidden_states
hidden_states = fast_rms_layernorm(self.input_layernorm, hidden_states)
hidden_states, _ = self.self_attn(
@@ -336,7 +331,6 @@ def Glm4MoeLiteDecoderLayer_fast_forward(
)
hidden_states = residual + hidden_states
- # MLP/MoE
residual = hidden_states
hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)
hidden_states = self.mlp(hidden_states)
@@ -370,15 +364,13 @@ class FastGLM47Model(FastLlamaModel):
"Please upgrade with: pip install --upgrade transformers"
)
- # Patch MoE forward with grouped GEMM (TMA compat handled by
- # grouped_gemm/__init__.py)
+ # Patch MoE forward with grouped GEMM; TMA compat is handled by grouped_gemm/__init__.py.
if HAS_GROUPED_GEMM:
Glm4MoeLiteNaiveMoe.forward = Glm4MoeLiteNaiveMoe_fast_forward
Glm4MoeLiteMoE.forward = Glm4MoeLiteMoE_fast_forward
- # Attention/rope/decoder/model forwards are NOT patched: GLM4 uses MLA
- # with different projection names and lacks extend_rope_embedding, so the
- # Llama-compatible infrastructure doesn't apply.
+ # Attention/rope/decoder/model forwards are NOT patched: GLM4 uses MLA with different projection
+ # names and lacks extend_rope_embedding, so the Llama-compatible infrastructure does not apply.
return
@@ -397,7 +389,7 @@ class FastGLM47Model(FastLlamaModel):
trust_remote_code = False,
**kwargs,
):
- # Used by loader, not passed to model
+ # Used by the loader, not passed to the model.
kwargs.pop("unsloth_force_compile", None)
return FastLlamaModel.from_pretrained(
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 1f4afa37697..6cb8f1a3195 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -32,8 +32,8 @@ from .llama import (
)
from .mistral import *
-# Without bnb, peft stops exporting its 4bit LoRA layer too. Both names only feed
-# isinstance checks, so placeholders nothing can match are exact stand-ins.
+# Without bnb, peft stops exporting its 4bit LoRA layer too. Both names only feed isinstance checks,
+# so placeholders nothing can match are exact stand-ins.
try:
from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit
from peft.tuners.lora import Linear4bit as Peft_Linear4bit
@@ -92,7 +92,6 @@ def GraniteAttention_fast_forward(
*args,
**kwargs,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
- # Clear inference
if hasattr(self, "paged_attention"):
del self.paged_attention_K
del self.paged_attention_V
@@ -125,7 +124,7 @@ def GraniteAttention_fast_forward(
cos, sin = position_embeddings
rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")
if rope_position_ids is not None:
- # Useful for LongRoPE
+ # Useful for LongRoPE.
Q, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)
else:
Q, K = fast_rope_embedding(Q, K, cos, sin)
@@ -135,7 +134,6 @@ def GraniteAttention_fast_forward(
V = torch.cat([past_key_value[1], V], dim = 2)
past_key_value = (K, V) if use_cache else None
- # Attention module
use_varlen = attention_mask is None and seq_info is not None and past_key_value is None
backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
@@ -172,8 +170,8 @@ def GraniteAttention_fast_forward(
},
)
- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse
- # (KV cache / padding mask) raises. None => byte-identical default.
+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /
+ # padding mask) raises. None means the byte-identical default.
_pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)
context = AttentionContext(
bsz = bsz,
@@ -216,7 +214,7 @@ def GraniteDecoderLayer_fast_forward(
else self.config.residual_multiplier
)
- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:
+ if use_cache and hasattr(self, "_flag_for_generation"):
residual = hidden_states
hidden_states = fast_rms_layernorm_inference(self.input_layernorm, hidden_states)
hidden_states, self_attn_weights, present_key_value = self.self_attn(
@@ -234,7 +232,6 @@ def GraniteDecoderLayer_fast_forward(
)
hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm_inference(self.post_attention_layernorm, hidden_states)
hidden_states = fast_swiglu_inference(self.mlp, hidden_states)
@@ -256,7 +253,6 @@ def GraniteDecoderLayer_fast_forward(
)
hidden_states = torch.add(residual, hidden_states, alpha = residual_multiplier)
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)
hidden_states = self.mlp(hidden_states)
@@ -272,7 +268,7 @@ def GraniteDecoderLayer_fast_forward(
from math import sqrt as math_sqrt
-KV_CACHE_INCREMENT = 256 # KV Cache update size
+KV_CACHE_INCREMENT = 256
torch_nn_functional_softmax = torch.nn.functional.softmax
torch_matmul = torch.matmul
torch_tanh = torch.tanh
@@ -301,7 +297,6 @@ def GraniteAttention_fast_forward_inference(
n_groups = self.num_key_value_groups
n_kv_heads = self.config.num_key_value_heads
head_dim = self.head_dim
- # assert(n_kv_heads * n_groups == n_heads)
hidden_size = self.config.hidden_size
attention_size = n_heads * head_dim
@@ -309,8 +304,6 @@ def GraniteAttention_fast_forward_inference(
kv_seq_len = seq_len + 1
device = hidden_states.device
- # Prefill phase
- # if not hasattr(self, "paged_attention"):
if do_prefill:
self.paged_attention = torch.empty(
(KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),
@@ -351,10 +344,8 @@ def GraniteAttention_fast_forward_inference(
Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
cos, sin = position_embeddings
- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
cos, sin = cos[position_ids], sin[position_ids]
@@ -367,24 +358,19 @@ def GraniteAttention_fast_forward_inference(
Qn *= cos
Qn.addcmul_(RH_Q, sin)
- RH_K = RH_Q[
- :, :n_kv_heads, :, :
- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")
+ RH_K = RH_Q[:, :n_kv_heads, :, :]
RH_K[:, :, :, :h] = Kn[:, :, :, h:]
RH_K[:, :, :, h:] = Kn[:, :, :, :h]
RH_K[:, :, :, :h].neg_()
Kn *= cos
Kn.addcmul_(RH_K, sin)
- # New KV cache
- # Kn = torch.cat([K1, Kn], dim = 2)
- # Vn = torch.cat([V1, Vn], dim = 2)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)
- # Grouped query attention
+ # Grouped query attention.
_, _, cached_len, _ = Kn.shape
if bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):
Kn = Kn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
@@ -392,7 +378,6 @@ def GraniteAttention_fast_forward_inference(
Kn = Kn.reshape(bsz, n_heads, cached_len, head_dim)
Vn = Vn.reshape(bsz, n_heads, cached_len, head_dim)
- # Attention
if bsz == 1:
Qn *= self.scaling
A = torch_matmul(Qn, Kn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
@@ -428,8 +413,7 @@ def GraniteAttention_fast_forward_inference(
return A, (Kn, Vn)
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825
-# @torch.inference_mode
+# Ported from transformers models/llama/modeling_llama.py#L825
def GraniteModel_fast_forward_inference(
self,
input_ids,
@@ -456,7 +440,7 @@ def GraniteModel_fast_forward_inference(
hidden_states,
seq_len,
)
- # Pre-convert to bool once for all layers (avoids per-layer .eq(0))
+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).
if attention_mask is not None and attention_mask.dtype != torch.bool:
attention_mask = attention_mask.eq(0)
else:
@@ -510,9 +494,8 @@ class GraniteRotaryEmbedding(LlamaRotaryEmbedding):
def patched_init(original_init):
def new_init(self, *args, **kwargs):
- # GraniteModel_fast_forward_inference can't reach residual_multiplier/config,
- # so stash the whole config here to pass it around. See:
- # https://github.com/huggingface/transformers/blob/e5fd865ebae062b7cf03a81b8c6affeb39f30bec/src/transformers/models/granite/modeling_granite.py#L243
+ # GraniteModel_fast_forward_inference cannot reach residual_multiplier/config, so stash the whole
+ # config here to pass it around. See transformers models/granite/modeling_granite.py#L243
config = kwargs.get("config", args[0] if args else None)
if config is not None:
self.config = config
@@ -555,8 +538,8 @@ class FastGraniteModel(FastLlamaModel):
tokenizer,
correct_dtype = None,
):
- # Torch.compile fails on embedding matrix??
- # Workaround randomnly fixes it for torch versions < 2.2
+ # Torch.compile fails on the embedding matrix; this workaround randomly fixes it for torch < 2.2,
+ # and the same is done for lm_head.
model.model.embed_tokens = torch.nn.Embedding.from_pretrained(
model.model.embed_tokens.weight
)
@@ -570,7 +553,7 @@ class FastGraniteModel(FastLlamaModel):
lm_head.out_features = lm_head.weight.shape[0]
model.lm_head = lm_head
- # Granite has tied weights! This means lm_head == embed_tokens
+ # Granite has tied weights, so lm_head == embed_tokens.
if model.model.embed_tokens.weight.data_ptr() != model.lm_head.weight.data_ptr():
lm_head = torch.nn.Linear(1, 1, bias = None)
del lm_head.weight
@@ -579,8 +562,8 @@ class FastGraniteModel(FastLlamaModel):
lm_head.out_features = lm_head.weight.shape[0]
model.lm_head = lm_head
- # Also patch all dtypes - BnB seems to not allocate the correct type?
- # BnB default dtype seems to be float16!
+ # BnB does not allocate the correct type: its default dtype is float16, so patch all dtypes. See
+ # TimDettmers/bitsandbytes#763.
correct_dtype = lm_head.weight.dtype
for name, module in model.named_modules():
@@ -594,7 +577,7 @@ class FastGraniteModel(FastLlamaModel):
else:
# https://github.com/TimDettmers/bitsandbytes/pull/763/files
quant_state.dtype = correct_dtype
- # Downcast RoPE embedding to correct data type
+ # Downcast RoPE embedding to the correct data type.
if name.endswith("rotary_emb") or hasattr(module, "cos_cached"):
if hasattr(module, "cos_cached") and (module.cos_cached.dtype != correct_dtype):
module.cos_cached = module.cos_cached.to(correct_dtype)
@@ -606,7 +589,6 @@ class FastGraniteModel(FastLlamaModel):
module.short_cos_cached = module.short_cos_cached.to(correct_dtype)
module.short_sin_cached = module.short_sin_cached.to(correct_dtype)
- # Clear deleted GPU items
import gc
for _ in range(3):
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index a4d47d27736..dc3993f2927 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -85,7 +85,7 @@ from ..device_type import (
)
transformers_version = Version(transformers_version)
-# Transformers moved rotary embeddings out of all attention layers
+# Transformers moved rotary embeddings out of all attention layers.
IS_ATTENTION_REFACTOR = transformers_version > Version("4.47.1")
try:
from transformers.modeling_layers import GradientCheckpointingLayer
@@ -105,7 +105,6 @@ from ..kernels import *
from ..tokenizer_utils import *
from .vision import FastBaseModel
-# Final patching code
from transformers.models.llama.modeling_llama import (
LlamaAttention,
LlamaDecoderLayer,
@@ -141,15 +140,14 @@ import types
try:
from huggingface_hub.utils import get_token
except:
- # Old HF Hub versions <= 0.0.25
+ # Old HF Hub versions <= 0.0.25.
from huggingface_hub.utils._token import get_token
from triton import __version__ as triton_version
-# Not `xformers is not None`: attention_dispatch probes the install and turns HAS_XFORMERS
-# off when the library imports but has no kernel that runs here. Recomputing it from the
-# import alone left the model code on the xFormers path the dispatcher had already left --
-# and Mistral answers "xFormers" by skipping the 4D sliding-window mask, so every sequence
-# longer than config.sliding_window attended to the whole causal history on SDPA instead.
+# Not `xformers is not None`: attention_dispatch turns HAS_XFORMERS off when the library imports
+# but has no kernel that runs here. Recomputing from the import alone left the model on the
+# xFormers path, and Mistral then skipped the 4D sliding-window mask, so every sequence longer
+# than config.sliding_window attended to the whole causal history on SDPA.
BlockDiagonalCausalMask = xformers.attn_bias.BlockDiagonalCausalMask if HAS_XFORMERS else None
@@ -167,9 +165,9 @@ def original_apply_o(self, X):
from math import sqrt as math_sqrt
-KV_CACHE_INCREMENT = 512 # KV Cache update size
+KV_CACHE_INCREMENT = 512
torch_nn_functional_softmax = torch.nn.functional.softmax
-# SDPA has GQA internally
+# SDPA has GQA internally.
SDPA_HAS_GQA = "enable_gqa" in scaled_dot_product_attention.__doc__
from peft.utils.other import ModulesToSaveWrapper
@@ -195,16 +193,13 @@ def _offload_frozen_module_for_training(
new_dtype = module.modules_to_save.default.weight.dtype
if new_dtype == torch.float16:
- # See https://github.com/unslothai/unsloth/pull/1200
- # Tesla T4 must use float32 and not float16
+ # Tesla T4 must use float32 and not float16; see #1200.
new_dtype = torch.float32
- # `device_type` is a bare type like "cuda", and `.to("cuda")` resolves to the
- # CURRENT device, which is cuda:0, so on a split model it drags the trainable
- # copy off its own card and the forward then mixes devices. Prefer the index
- # the copy already has; under the default `use_gradient_checkpointing =
- # "unsloth"` it has none, because the disk offload above rebuilt it on CPU,
- # so fall back to the placement recorded before that. Only then the bare type.
+ # `device_type` is bare ("cuda") and .to("cuda") resolves to the CURRENT device, so on a split
+ # model it drags the trainable copy off its card. Prefer the index the copy already has; under
+ # use_gradient_checkpointing="unsloth" it has none (rebuilt on CPU), so fall back to the
+ # placement recorded before that, then the bare type.
wanted_type = torch.device(device_type).type
target_device = device_type
for candidate in (module.modules_to_save.default.weight.device, original_device):
@@ -215,13 +210,11 @@ def _offload_frozen_module_for_training(
module.modules_to_save.default.to(device = target_device, dtype = new_dtype, non_blocking = True)
module.modules_to_save.default.requires_grad_(True)
- # [TODO] Move old module to CPU - should be disk!
if offload_device is not None:
module.original_module.to(device = offload_device, non_blocking = True)
module.original_module.requires_grad_(False)
-# Fix new HF's inference code
def _fast_prepare_inputs_for_generation(
self,
input_ids,
@@ -232,7 +225,7 @@ def _fast_prepare_inputs_for_generation(
past_key_values = kwargs.get("past_key_values", None)
original_attention_mask = attention_mask
- # Only use inputs_embeds on the first step (no cache). Fixes issue #3798.
+ # Only use inputs_embeds on the first step (no cache); fixes #3798.
use_inputs_embeds = inputs_embeds is not None and past_key_values is None
if input_ids is not None and input_ids.numel() > 0:
@@ -246,12 +239,10 @@ def _fast_prepare_inputs_for_generation(
device = "cuda" if torch.cuda.is_available() else "cpu"
if past_key_values is not None:
- # Check for uninitialized DynamicCache
if len(past_key_values) == 0:
past_key_values = None
kwargs["past_key_values"] = None
use_inputs_embeds = inputs_embeds is not None
- # New since 4.56
elif hasattr(past_key_values, "get_seq_length") and past_key_values.get_seq_length() == 0:
past_key_values = None
kwargs["past_key_values"] = None
@@ -272,7 +263,7 @@ def _fast_prepare_inputs_for_generation(
if hasattr(past_key_values, "get_seq_length"):
past_len = int(past_key_values.get_seq_length())
else:
- # legacy tuple cache: (layer, (K,V))
+ # Legacy tuple cache: (layer, (K, V)).
past_len = int(past_key_values[0][0].shape[-2])
max_cache_len = None
@@ -283,7 +274,6 @@ def _fast_prepare_inputs_for_generation(
m = past_key_values.get_max_length()
max_cache_len = int(m) if m is not None else None
- # ensure cache_position
cache_position = kwargs.get("cache_position", None)
if cache_position is None:
kwargs["cache_position"] = torch.arange(
@@ -296,7 +286,6 @@ def _fast_prepare_inputs_for_generation(
if hasattr(cache_position, "device") and cache_position.device != device:
kwargs["cache_position"] = cache_position.to(device)
- # Get to the base model
base_model = self
if hasattr(base_model, "base_model_prefix"):
base_model = getattr(base_model, base_model.base_model_prefix)
@@ -309,7 +298,7 @@ def _fast_prepare_inputs_for_generation(
sig = inspect.signature(inspect.unwrap(fn))
return "device" in sig.parameters
except:
- # transformers <= 4.51.3 includes device arg but > 4.51.3 does not
+ # transformers <= 4.51.3 includes the device arg; later versions do not.
return transformers_version < Version("4.52.0")
base_model._unsloth_mask_needs_device = _check_needs_device(
@@ -373,7 +362,6 @@ def _fast_prepare_inputs_for_generation(
def fix_prepare_inputs_for_generation(module):
- # Fix prepare_inputs_for_generation
if hasattr(module, "prepare_inputs_for_generation"):
module.prepare_inputs_for_generation = _fast_prepare_inputs_for_generation
@@ -427,15 +415,12 @@ def LlamaAttention_fast_forward_inference(
n_groups = self.num_key_value_groups
n_kv_heads = self.config.num_key_value_heads
head_dim = self.head_dim
- # assert(n_kv_heads * n_groups == n_heads)
hidden_size = self.config.hidden_size
attention_size = n_heads * head_dim
seq_len = K1.shape[-2]
kv_seq_len = seq_len + 1
- # Prefill phase
- # if not hasattr(self, "paged_attention"):
device = hidden_states.device
if do_prefill:
self.paged_attention = torch.empty(
@@ -451,7 +436,7 @@ def LlamaAttention_fast_forward_inference(
self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)
self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
- # Mistral Nemo 12b has weird dimensions
+ # Mistral Nemo 12b has weird dimensions.
if attention_size != hidden_size:
self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
else:
@@ -483,15 +468,11 @@ def LlamaAttention_fast_forward_inference(
Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
-
- # Need to do it prior 2 steps before hitting full on short KV cache
- # or else error
+ # Must be done 2 steps before hitting full on a short KV cache, or it errors.
if position_ids.dim() == 1:
position_ids = position_ids[:, None]
- # Transformers 5.x accumulates position_ids as [batch, full_seq_len] across
- # decode steps; single-token inference only needs the last position.
+ # Transformers 5.x accumulates position_ids as [batch, full_seq_len] across decode steps;
+ # single-token inference only needs the last position.
if position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
position_ids = position_ids.to(Qn.device)
@@ -509,40 +490,34 @@ def LlamaAttention_fast_forward_inference(
RH_Q = self.RH_Q
RH_Q[:, :, :, :h] = Qn[:, :, :, h:]
RH_Q[:, :, :, h:] = Qn[:, :, :, :h]
- RH_Q[:, :, :, :h].neg_() # torch.neg(RH_Q[:,:,:,:h], out = RH_Q[:,:,:,:h])
+ RH_Q[:, :, :, :h].neg_()
Qn *= cos
Qn.addcmul_(RH_Q, sin)
- RH_K = RH_Q[
- :, :n_kv_heads, :, :
- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")
+ RH_K = RH_Q[:, :n_kv_heads, :, :]
RH_K[:, :, :, :h] = Kn[:, :, :, h:]
RH_K[:, :, :, h:] = Kn[:, :, :, :h]
- RH_K[:, :, :, :h].neg_() # torch.neg(RH_K[:,:,:,:h], out = RH_K[:,:,:,:h])
+ RH_K[:, :, :, :h].neg_()
Kn *= cos
Kn.addcmul_(RH_K, sin)
- # New KV cache
- # Kn = torch.cat([K1, Kn], dim = 2)
- # Vn = torch.cat([V1, Vn], dim = 2)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)
- # Handle sliding windows
+ # Handle sliding windows, from transformers models/mistral/modeling_mistral.py#L193
sliding_window = getattr(self.config, "sliding_window", None)
if sliding_window is not None and kv_seq_len > sliding_window:
- # From https://github.com/huggingface/transformers/blob/main/src/transformers/models/mistral/modeling_mistral.py#L193
start = kv_seq_len - sliding_window
- Knn = Kn[:, :, start:, :] # .contiguous()
- Vnn = Vn[:, :, start:, :] # .contiguous()
+ Knn = Kn[:, :, start:, :]
+ Vnn = Vn[:, :, start:, :]
if attention_mask is not None:
attention_mask = attention_mask[..., start:]
else:
Knn, Vnn = Kn, Vn
- # Grouped query attention
+ # Grouped query attention.
_, _, cached_len, _ = Knn.shape
if bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
@@ -550,27 +525,23 @@ def LlamaAttention_fast_forward_inference(
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)
- # when qlen==vlen and attn_mask is None, we should use causal attention
+ # When qlen == vlen and attn_mask is None, causal attention is the right choice.
Q_len = Qn.shape[-2]
K_len = Knn.shape[-2]
if attention_mask is None and Q_len == K_len:
is_causal = True
else:
is_causal = False
- # Attention
if bsz == 1:
- Qn *= (
- self.scalar
- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963
- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows
+ Qn *= self.scalar
+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805.
A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)
+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
A = torch_matmul(A, Vnn, out = Qn)
- # --- attention_mask fixup for SDPA if user passes 2D padding mask
+ # attention_mask fixup for SDPA when the user passes a 2D padding mask.
else:
if attention_mask is not None and attention_mask.dim() == 2:
attention_mask = attention_mask[:, None, None, :].to(torch.bool)
- # is it more appropriate to use _prepare_4d_causal_attention_mask_for_sdpa?
elif (
attention_mask is not None
and attention_mask.dim() == 4
@@ -609,11 +580,7 @@ def fast_swiglu_inference(
gate_multiplier = None,
down_multiplier = None,
):
- # gate = self.gate_proj(X)
- # up = self.up_proj(X)
bsz, _, hd = X.shape
- # mlp_size = self.config.intermediate_size
- # temp = torch.empty((2, bsz, 1, mlp_size), dtype = X.dtype, device = "cuda:0")
gate = fast_linear_forward(self.gate_proj, X, out = temp_gate)
@@ -625,7 +592,6 @@ def fast_swiglu_inference(
gate = torch_nn_functional_silu(gate, inplace = True)
gate *= up
- # X = self.down_proj(gate)
down = fast_linear_forward(self.down_proj, gate, out = up[:, :, :hd])
if down_multiplier is not None:
@@ -684,7 +650,7 @@ def fast_rms_layernorm_inference_gemma(
return XX.to(X.dtype)
-# Normal layernorm with mean removal
+# Normal layernorm with mean removal.
@torch.compile(fullgraph = False, dynamic = True, options = torch_compile_options)
def fast_layernorm_compiled(layernorm, X):
old_dtype = X.dtype
@@ -699,7 +665,7 @@ def fast_layernorm_compiled(layernorm, X):
return X.to(old_dtype)
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L320
+# Ported from transformers models/llama/modeling_llama.py#L320
def LlamaAttention_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -714,7 +680,6 @@ def LlamaAttention_fast_forward(
*args,
**kwargs,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
- # Clear inference
if hasattr(self, "paged_attention"):
del self.paged_attention_K
del self.paged_attention_V
@@ -754,11 +719,6 @@ def LlamaAttention_fast_forward(
if rope_position_ids is None and seq_info is not None:
rope_position_ids = kwargs.get("position_ids")
- # Q, K = (
- # fast_rope_embedding(Q, K, cos, sin)
- # if rope_position_ids is None
- # else inplace_rope_embedding(Q, K, cos, sin, rope_position_ids)
- # )
Q, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)
if past_key_value is not None:
@@ -766,11 +726,10 @@ def LlamaAttention_fast_forward(
V = torch.cat([past_key_value[1], V], dim = 2)
past_key_value = (K, V) if use_cache else None
- # Attention module
use_varlen = seq_info is not None and past_key_value is None
backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
- # should dropout be hardcoded to 0.0?
+ # Should dropout be hardcoded to 0.0?
config = AttentionConfig(
backend = backend,
n_kv_heads = n_kv_heads,
@@ -778,9 +737,9 @@ def LlamaAttention_fast_forward(
flash_dense_kwargs = {"causal": True},
flash_varlen_kwargs = {"dropout_p": 0.0, "causal": True},
)
- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward (same route
- # as packed_seq_lengths); misuse (KV cache / padding mask) raises. None => byte-identical
- # default. Reuse of this forward also carries the branch to qwen2 & gemma.
+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward (same route as
+ # packed_seq_lengths); misuse (KV cache / padding mask) raises. None means the byte-identical
+ # default. Reuse of this forward carries the branch to qwen2 and gemma.
_pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)
context = AttentionContext(
bsz = bsz,
@@ -802,7 +761,7 @@ def LlamaAttention_fast_forward(
return attn_output, attn_weights, past_key_value
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L590
+# Ported from transformers models/llama/modeling_llama.py#L590
def LlamaDecoderLayer_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -847,7 +806,6 @@ def LlamaDecoderLayer_fast_forward(
)
hidden_states += residual
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm_inference(self.post_attention_layernorm, hidden_states)
hidden_states = fast_swiglu_inference(self.mlp, hidden_states)
@@ -869,7 +827,6 @@ def LlamaDecoderLayer_fast_forward(
)
hidden_states = residual + hidden_states
- # Fully Connected
residual = hidden_states
hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)
hidden_states = self.mlp(hidden_states)
@@ -883,7 +840,7 @@ def LlamaDecoderLayer_fast_forward(
return outputs
-# https://github.com/unslothai/unsloth/issues/404#issuecomment-2323473452
+# See unslothai/unsloth#404 (comment 2323473452).
__DTYPE_MAP = {
"float32": torch.float32,
torch.float32: torch.float32,
@@ -894,7 +851,7 @@ __DTYPE_MAP = {
}
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825
+# Ported from transformers models/llama/modeling_llama.py#L825
def LlamaModel_fast_forward(
self,
input_ids: Optional[torch.LongTensor] = None,
@@ -923,7 +880,6 @@ def LlamaModel_fast_forward(
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
- # retrieve input_ids and inputs_embeds
if input_ids is not None and inputs_embeds is not None:
raise ValueError(
"Unsloth: You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time"
@@ -965,8 +921,8 @@ def LlamaModel_fast_forward(
past_key_values_length = past_key_values[0][0].shape[2]
seq_length_with_past = seq_length_with_past + past_key_values_length
- # We already handle KV cache position_ids ourselves.
- if False: # (past_key_values_length != 0):
+ # KV cache position_ids are handled here already.
+ if False:
position_ids = torch.arange(
past_key_values_length,
seq_length + past_key_values_length,
@@ -975,7 +931,7 @@ def LlamaModel_fast_forward(
)
position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
elif position_ids is not None:
- position_ids = position_ids.view(-1, seq_length).to(torch.int32) # .long()
+ position_ids = position_ids.view(-1, seq_length).to(torch.int32)
else:
position_ids = None
@@ -983,13 +939,13 @@ def LlamaModel_fast_forward(
if position_ids.shape[0] != batch_size:
position_ids = position_ids.repeat((batch_size, 1))
- # Embed positions
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
inputs_embeds = inputs_embeds.to(_get_dtype(dtype_from_config(self.config)))
- # Normalized from Gemma
+ # Normalized from Gemma: cast to bfloat16/float16 to match it exactly, since 3072**0.5 is
+ # 55.5000 in bfloat16 against 55.4256 in float32.
IS_GEMMA = self.config.model_type.startswith("gemma")
IS_GEMMA2 = self.config.model_type.startswith("gemma2")
IS_COHERE = self.config.model_type.startswith("cohere")
@@ -999,14 +955,10 @@ def LlamaModel_fast_forward(
train_embed_tokens = self.embed_tokens.weight.requires_grad
if IS_GEMMA:
- # Match Gemma exactly by casting to bfloat16 / float16
- # inputs_embeds *= math_sqrt(self.config.hidden_size)
- # Ie 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32
- # & 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32
normalizer = torch.tensor(math_sqrt(self.config.hidden_size), dtype = inputs_embeds.dtype)
if train_embed_tokens:
- # Careful we must not do an inplace op!
+ # Careful: this must not be an inplace op.
inputs_embeds = inputs_embeds * normalizer
else:
inputs_requires_grad = inputs_embeds.requires_grad
@@ -1016,12 +968,10 @@ def LlamaModel_fast_forward(
elif inputs_requires_grad:
inputs_embeds.requires_grad_(False)
inputs_embeds *= normalizer
- # inputs_embeds *= math_sqrt(self.config.hidden_size)
if inputs_requires_grad:
inputs_embeds.requires_grad_(True)
- # Fix up attention mask by setting elements to 0
- # Specifically for DPO
+ # Fix up the attention mask by setting elements to 0, specifically for DPO.
if (
getattr(self, "_has_no_labels", False) is True
and (attention_mask is not None)
@@ -1030,8 +980,7 @@ def LlamaModel_fast_forward(
and (not train_embed_tokens)
and self.training
):
- # Careful for inference the attention_mask is size (1, kv_seq_len)
- # Whilst the input_embeds is size (1, 1, 4096)
+ # Careful: for inference the attention_mask is size (1, kv_seq_len) while input_embeds is (1, 1, 4096).
inputs_requires_grad = inputs_embeds.requires_grad
if not inputs_embeds.is_leaf:
inputs_embeds = inputs_embeds.detach()
@@ -1043,16 +992,12 @@ def LlamaModel_fast_forward(
if inputs_requires_grad:
inputs_embeds.requires_grad_(True)
- # Ignore attention_mask
if attention_mask is None:
padding_mask = None
elif self.training:
attention_mask = None
padding_mask = None
else:
- # if 0 in attention_mask:
- # padding_mask = attention_mask
- # else:
padding_mask = None
attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
@@ -1062,9 +1007,7 @@ def LlamaModel_fast_forward(
past_key_values_length,
sliding_window = getattr(self.config, "sliding_window", None),
)
- # Must NOT convert to bool - weirdly this causes stuff to error out!
- # if attention_mask is not None:
- # attention_mask = attention_mask.to(torch.bool)
+ # Must NOT convert to bool; that weirdly causes errors.
hidden_states = inputs_embeds
if IS_GRANITE or IS_FALCON_H1: # granite has embedding multiplier
@@ -1072,30 +1015,23 @@ def LlamaModel_fast_forward(
if past_key_values is None and self.training:
use_cache = False
- # if use_cache:
- # logger.warning_once(
- # "Unsloth: `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`"
- # )
- # use_cache = False
- # decoder layers
all_hidden_states = () if output_hidden_states else None
all_self_attns = () if output_attentions else None
next_decoder_cache = () if use_cache else None
- # Gradient checkpointing methods (ie sqrt)
+ # Gradient checkpointing methods (ie sqrt).
if hasattr(self, "_gradient_checkpointing_boundaries"):
boundaries = self._gradient_checkpointing_boundaries
else:
boundaries = None
- # Check checkpointing method
gradient_checkpointing = False
if self.gradient_checkpointing and self.training and not use_cache:
gradient_checkpointing = True
- # Gemma2 has alternating SWA and global attn
+ # Gemma2 has alternating SWA and global attn.
use_static_mask = True
dynamic_SWA_mask = None
dynamic_GA_mask = None
@@ -1104,12 +1040,9 @@ def LlamaModel_fast_forward(
self.SWA_mask = True
self.GA_mask = False
elif attention_mask is not None:
- # Fixes https://github.com/unslothai/unsloth/issues/853
- # Unsloth needs a 2D mask, not a [2, 1, n, n] mask!
+ # Unsloth needs a 2D mask, not [2, 1, n, n] (#853), converted to float not bool
+ # (pytorch/pytorch#103749).
- # https://github.com/pytorch/pytorch/issues/103749
- # Need to convert to float and not using bool
- # attention_mask = (1.0 - attention_mask.float()) * torch.finfo(inputs_embeds.dtype).min
dynamic_SWA_mask = _prepare_4d_causal_attention_mask_for_sdpa(
attention_mask,
(batch_size, seq_length),
@@ -1128,16 +1061,13 @@ def LlamaModel_fast_forward(
elif not hasattr(self, "SWA_mask"):
if HAS_FLEX_ATTENTION:
- # Use Flex Attention instead!
self.SWA_mask = create_flex_attention_sliding_window_mask(
self.max_seq_length, self.config.sliding_window
)
self.GA_mask = create_flex_attention_causal_mask(self.max_seq_length)
else:
- n = self.max_seq_length # self.config.max_position_embeddings
- # masked_fill is making stuff slower!
- # self. GA_mask = create_boolean_mask(n = n, sliding_window = 0)
- # self.SWA_mask = create_boolean_mask(n = n, sliding_window = self.config.sliding_window)
+ n = self.max_seq_length
+ # masked_fill makes things slower.
self.SWA_mask = (
AttentionMaskConverter(
is_causal = True,
@@ -1174,8 +1104,8 @@ def LlamaModel_fast_forward(
IS_ATTENTION_REFACTOR
and (hasattr(self, "rotary_emb") or not hasattr(self.layers[0].self_attn, "rotary_emb"))
) or IS_GRANITE:
- # position_embeddings is mandatory on main: https://github.com/huggingface/transformers/pull/34858
- # granite always had the attention refactor, so let it always use this path.
+ # position_embeddings is mandatory on main (huggingface/transformers#34858), and granite always
+ # had the attention refactor, so it always takes this path.
self.rotary_emb.extend_rope_embedding(hidden_states, self.config.max_position_embeddings)
position_embeddings = self.rotary_emb.get_cached(
self.config.max_position_embeddings, hidden_states.device.index
@@ -1183,7 +1113,6 @@ def LlamaModel_fast_forward(
else:
position_embeddings = None
- # Go through every layer!
for idx, decoder_layer in enumerate(self.layers):
if output_hidden_states:
all_hidden_states += (hidden_states,)
@@ -1244,7 +1173,6 @@ def LlamaModel_fast_forward(
if output_attentions:
all_self_attns += (layer_outputs[1],)
- # Final layernorm
if use_cache:
if IS_FALCON_H1:
hidden_states = fast_rms_layernorm_inference(self.final_layernorm, hidden_states)
@@ -1277,7 +1205,7 @@ def LlamaModel_fast_forward(
)
-# https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825
+# Ported from transformers models/llama/modeling_llama.py#L825
def _LlamaModel_fast_forward_inference(
attention_fast_forward_inference = LlamaAttention_fast_forward_inference,
mlp_fast_forward_inference = fast_swiglu_inference,
@@ -1300,7 +1228,6 @@ def _LlamaModel_fast_forward_inference(
X = X.to(_get_dtype(dtype_from_config(self.config)))
bsz, q_len, hd = X.shape
assert q_len == 1
- # Get saved buffers to reduce memory movement
residual = torch.empty(
(bsz, q_len, hd), dtype = torch.float32, device = f"{DEVICE_TYPE_TORCH}:0"
)
@@ -1327,13 +1254,13 @@ def _LlamaModel_fast_forward_inference(
seq_len,
sliding_window = getattr(self.config, "sliding_window", None),
)
- # Pre-convert to bool once for all layers (avoids per-layer .eq(0))
+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).
if attention_mask is not None and attention_mask.dtype != torch.bool:
attention_mask = attention_mask.eq(0)
else:
attention_mask = None
- # Compute rotary_seq_len once to avoid per-layer GPU-CPU sync from .item()
+ # Compute rotary_seq_len once to avoid a per-layer GPU-CPU sync from .item().
rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)
next_decoder_cache = []
@@ -1341,7 +1268,7 @@ def _LlamaModel_fast_forward_inference(
for idx, decoder_layer in enumerate(self.model.layers):
device_index = getattr(decoder_layer, "_per_layer_device_index", 0)
X, residual, position_ids = move_to_device(device_index, X, residual, position_ids)
- residual.copy_(X) # residual = X
+ residual.copy_(X)
X = fast_rms_layernorm_inference(
decoder_layer.input_layernorm,
X,
@@ -1360,7 +1287,7 @@ def _LlamaModel_fast_forward_inference(
)
X += residual
- residual.copy_(X) # residual = X
+ residual.copy_(X)
X = fast_rms_layernorm_inference(
decoder_layer.post_attention_layernorm,
X,
@@ -1395,7 +1322,7 @@ def _LlamaModel_fast_forward_inference(
return LlamaModel_fast_forward_inference_custom
-# For ensuring backwards compatibility, we create LlamaModel_fast_forward_inference that is consumed by other models
+# LlamaModel_fast_forward_inference exists for backwards compatibility and is consumed by other models.
LlamaModel_fast_forward_inference = _LlamaModel_fast_forward_inference()
@@ -1441,7 +1368,6 @@ def CausalLM_fast_forward(fast_forward_inference):
else self.config.output_hidden_states
)
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
- # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
self.model._has_no_labels = labels is None
outputs = self.model(
input_ids = input_ids,
@@ -1471,12 +1397,11 @@ def CausalLM_fast_forward(fast_forward_inference):
else:
num_logits_to_keep = max(num_logits_to_keep, logits_to_keep)
- # Move items to same device as lm_head
hidden_states = hidden_states.to(lm_head_device)
if labels is not None:
labels = labels.to(lm_head_device)
- # Output last hidden states without logits if asked
+ # Output last hidden states without logits if asked.
if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":
if num_logits_to_keep != 0:
hidden_states = hidden_states[:, -num_logits_to_keep:, :]
@@ -1495,9 +1420,9 @@ def CausalLM_fast_forward(fast_forward_inference):
logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(dtype))
else:
RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"
- # < 1024 Normal Unsloth uses less VRAM!
+ # Below 1024 normal Unsloth uses less VRAM.
if bsz * q_len <= 1024 and not RETURN_LOGITS:
- # Use unsloth_fused_ce_loss which actually calculates the best chunk size to reduce VRAM usage
+ # unsloth_fused_ce_loss calculates the best chunk size to reduce VRAM usage.
RETURN_LOGITS = False
if not RETURN_LOGITS and labels is not None:
@@ -1508,24 +1433,17 @@ def CausalLM_fast_forward(fast_forward_inference):
if self.config.model_type == "falcon_h1":
hidden_states = hidden_states * self.config.lm_head_multiplier
- # Packed-boundary guard on raw labels (the fused kernel shifts internally).
- # This branch RETURNS, so mask_packed_sequence_boundaries() below is dead on
- # packed training paths: it needs UNSLOTH_RETURN_LOGITS=1, which itself forces
- # packing off (unsloth/trainer.py). Out-of-place, no-op without packed_seq_lengths.
+ # Packed-boundary guard on raw labels (the fused kernel shifts internally). This branch RETURNS,
+ # so mask_packed_sequence_boundaries() below is dead on packed paths: it needs
+ # UNSLOTH_RETURN_LOGITS=1, which forces packing off. Out-of-place, no-op without
+ # packed_seq_lengths.
labels = mask_packed_boundary_labels(
labels,
kwargs.get("packed_seq_lengths"),
)
- ### DISABLED since T4 breaks
- # OutOfResources: out of resource: shared memory, Required: 98304, Hardware limit: 65536. Reducing block sizes or `num_stages` may help.
- # loss = fused_linear_cross_entropy(
- # hidden_states = hidden_states,
- # lm_weight = lm_head,
- # labels = labels,
- # num_items_in_batch = n_items,
- # logit_softcapping = logit_softcapping,
- # )
+ # fused_linear_cross_entropy is disabled: T4 raises OutOfResources (needs 98304 shared memory
+ # against a 65536 limit).
loss = unsloth_fused_ce_loss(
trainer = None,
hidden_states = hidden_states,
@@ -1540,8 +1458,7 @@ def CausalLM_fast_forward(fast_forward_inference):
logit_softcapping = logit_softcapping,
)
if not return_dict:
- # Fused CE never materializes `logits`; use EMPTY_LOGITS
- # like the return_dict branch below (fixes #2068).
+ # Fused CE never materializes logits; use EMPTY_LOGITS like the return_dict branch below (#2068).
output = (EMPTY_LOGITS,) + outputs[1:]
return (loss,) + output if loss is not None else output
@@ -1558,12 +1475,9 @@ def CausalLM_fast_forward(fast_forward_inference):
logits = logits.to(_get_dtype(dtype_from_config(self.config)))
loss = None
- # Which field carries the scale is per family (cohere multiplies by logit_scale,
- # granite divides by logits_scaling, falcon_h1 multiplies by lm_head_multiplier).
- # The planner sizes the head's card from the same answer, so a new family is taught
- # once, not twice.
- # granite: https://github.com/huggingface/transformers/blob/4d1d0f29a493098e6bc6b904b82e29cb331827f5/src/transformers/models/granite/modeling_granite.py#L1103
- # cohere: https://github.com/huggingface/transformers/blob/4d1d0f29a493098e6bc6b904b82e29cb331827f5/src/transformers/models/cohere/modeling_cohere.py#L1176
+ # Which field carries the scale is per family (cohere logit_scale, granite logits_scaling,
+ # falcon_h1 lm_head_multiplier). The planner sizes the head's card from the same answer, so a
+ # new family is taught once, not twice.
if detect_logit_transforms is not None:
_transforms = detect_logit_transforms(self.config)
logit_softcapping = _transforms["logit_softcapping"]
@@ -1580,10 +1494,6 @@ def CausalLM_fast_forward(fast_forward_inference):
if labels is not None:
shift_logits = logits
- # if not hasattr(self, "extra_ignored_labels"):
- # # Fixes https://github.com/unslothai/unsloth/issues/10
- # self.extra_ignored_labels = torch.full((self.max_seq_length, 1), -100, device = "cuda:0")
- # pass
shift_labels = torch.empty_like(labels)
shift_labels[..., :-1] = labels[..., 1:]
shift_labels[..., -1] = -100
@@ -1591,7 +1501,6 @@ def CausalLM_fast_forward(fast_forward_inference):
shift_labels,
kwargs.get("packed_seq_lengths"),
)
- # shift_labels = torch.hstack((labels[..., 1:], self.extra_ignored_labels[:labels.shape[0]]))
n_items = kwargs.get("num_items_in_batch", None)
if n_items is None:
n_items = kwargs.get("n_items", None)
@@ -1774,9 +1683,9 @@ def _compute_config_rope_inv_freq(config, rope_scaling):
original_rope_scaling = rope_scaling
rope_scaling = _rope_scaling_as_dict(rope_scaling)
rope_type = rope_scaling.get("rope_type", None) or rope_scaling.get("type", None)
- # "default"/unset means unscaled RoPE. transformers >=5 reports
- # rope_type="default" for every plain config and dropped "default" from
- # ROPE_INIT_FUNCTIONS, so compute it directly instead of warning per load.
+ # "default"/unset means unscaled RoPE: transformers >= 5 reports rope_type="default" for every
+ # plain config and dropped "default" from ROPE_INIT_FUNCTIONS, so compute it directly instead
+ # of warning per load.
if rope_type in (None, "default"):
return _vanilla_inv_freq_from_config(config).to(dtype = torch.float32, device = "cpu"), 1.0
try:
@@ -1809,31 +1718,25 @@ def _compute_config_rope_inv_freq(config, rope_scaling):
return None, 1.0
-# Solves https://github.com/unslothai/unsloth/issues/168
-# Static KV Cache was introduced in 4.38.0, causing training to be much slower.
-# Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
-# https://github.com/huggingface/transformers/pull/27931
-# https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
+# Static KV Cache landed in 4.38.0 and made training much slower (#168,
+# huggingface/transformers#27931), so the old rotary embeddings are retained.
class LlamaRotaryEmbedding(torch.nn.Module):
- # Fixes https://github.com/huggingface/transformers/pull/28837
- # https://github.com/microsoft/DeepSpeed/issues/4932
- # The precision of RoPE buffers is not correct, so we cast to int64.
+ # RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.
def __init__(
self,
dim = None,
max_position_embeddings = 2048,
base = 10000,
device = None,
- config = None, # [TODO] Hack to pass in config - need to remove later
+ config = None,
):
super().__init__()
# cos/sin multiplier (1.0 except yarn / longrope); set before any cache build.
self.attention_scaling = 1.0
- # Base-class-from-config path (modern transformers): derive inv_freq like
- # transformers so config.rope_scaling is not dropped (#2405). Scaled
- # subclasses are excluded to avoid double-scaling.
+ # Base-class-from-config path (modern transformers): derive inv_freq like transformers so
+ # config.rope_scaling is not dropped (#2405). Scaled subclasses excluded to avoid
+ # double-scaling.
if config is not None:
- # [TODO] Hack to pass in config - need to remove later
base = _get_rope_theta(config, default = base)
partial_rotary_factor = (
config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
@@ -1849,7 +1752,7 @@ class LlamaRotaryEmbedding(torch.nn.Module):
self.base = base
# Kept so the v5 rope repair can rebuild the scaled inv_freq (#2405).
self._unsloth_rope_config = config
- # Dynamic RoPE we first set it to a max of 4 * 8192 tokens then we iteratively grow this
+ # Dynamic RoPE starts at a max of 4 * 8192 tokens and grows iteratively in increments of 8192.
self.current_rope_size = min(4 * 8192, self.max_position_embeddings)
self.multi_gpu_cos_cached = [None] * DEVICE_COUNT
self.multi_gpu_sin_cached = [None] * DEVICE_COUNT
@@ -1857,7 +1760,6 @@ class LlamaRotaryEmbedding(torch.nn.Module):
inv_freq = self._unsloth_recompute_inv_freq()
self.register_buffer("inv_freq", inv_freq, persistent = False)
- # Build here to make `torch.jit.trace` work.
for device_idx in range(DEVICE_COUNT):
self._set_cos_sin_cache(
seq_len = self.current_rope_size,
@@ -1865,7 +1767,6 @@ class LlamaRotaryEmbedding(torch.nn.Module):
dtype = torch.get_default_dtype(),
)
- # dummy so that patch_utils doesn't fail for now
self.cos_cached = torch.empty(
1, device = get_current_device(), dtype = torch.get_default_dtype()
)
@@ -1878,8 +1779,8 @@ class LlamaRotaryEmbedding(torch.nn.Module):
return inv_freq
def _unsloth_recompute_inv_freq(self):
- # Config scaling (llama3/yarn) first, else vanilla + subclass scaling.
- # Shared by __init__ and the v5 rope repair so they cannot diverge.
+ # Config scaling (llama3/yarn) first, else vanilla plus subclass scaling. Shared by __init__ and
+ # the v5 rope repair so they cannot diverge.
config = getattr(self, "_unsloth_rope_config", None)
config_inv_freq = None
rope_scaling = getattr(config, "rope_scaling", None) if config is not None else None
@@ -1901,8 +1802,7 @@ class LlamaRotaryEmbedding(torch.nn.Module):
return t
def _set_cos_sin_cache(self, seq_len, device, dtype):
- # Note: on the original Llama codebase, these tensors are created on the target device (and not on CPU) and
- # in FP32. They are applied (multiplied) in FP32 as well.
+ # The original Llama codebase creates these on the target device in FP32 and multiplies in FP32.
self.current_rope_size = seq_len
t = torch.arange(
self.current_rope_size, device = self.inv_freq.device, dtype = torch.int64
@@ -1910,10 +1810,10 @@ class LlamaRotaryEmbedding(torch.nn.Module):
t = self._apply_time_scaling(t)
freqs = torch.outer(t, self.inv_freq)
- # Different from paper, but it uses a different permutation in order to obtain the same calculation
+ # Different from the paper, but it uses a different permutation to obtain the same calculation.
emb = torch.cat((freqs, freqs), dim = -1)
- # Applied here so attention_scaling survives extend_rope_embedding rebuilds;
- # default 1.0 keeps unscaled paths bit-identical.
+ # Applied here so attention_scaling survives extend_rope_embedding rebuilds; the 1.0 default
+ # keeps unscaled paths bit-identical.
cos = (emb.cos() * self.attention_scaling).to(dtype = dtype, device = device, non_blocking = True)
sin = (emb.sin() * self.attention_scaling).to(dtype = dtype, device = device, non_blocking = True)
self.multi_gpu_cos_cached[device.index] = cos
@@ -1926,7 +1826,6 @@ class LlamaRotaryEmbedding(torch.nn.Module):
position_ids = None,
seq_len = None,
):
- # x: [bs, num_attention_heads, seq_len, head_size]
if seq_len is not None and seq_len > self.current_rope_size:
self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)
@@ -1948,7 +1847,7 @@ class LlamaRotaryEmbedding(torch.nn.Module):
def extend_rope_embedding(self, x, seq_len):
if seq_len <= self.current_rope_size:
return
- # Iteratively grow by increments of 8192
+ # Iteratively grow by increments of 8192.
self.current_rope_size = ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192
for device_idx in range(DEVICE_COUNT):
self._set_cos_sin_cache(
@@ -1959,9 +1858,7 @@ class LlamaRotaryEmbedding(torch.nn.Module):
class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):
"""LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
- # Fixes https://github.com/huggingface/transformers/pull/28837
- # https://github.com/microsoft/DeepSpeed/issues/4932
- # The precision of RoPE buffers is not correct, so we cast to int64.
+ # RoPE buffer precision is wrong, so cast to int64; see huggingface/transformers#28837 and microsoft/DeepSpeed#4932.
def __init__(
self,
dim = None,
@@ -1969,7 +1866,7 @@ class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):
base = 10000,
device = None,
scaling_factor = 1.0,
- config = None, # [TODO] Hack to pass in config - need to remove later
+ config = None,
):
self.scaling_factor = scaling_factor
super().__init__(
@@ -1985,8 +1882,7 @@ class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):
return t / self.scaling_factor
-# See https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/rotary_embedding.py#L736
-# For Llama 3.1
+# For Llama 3.1; see vllm model_executor/layers/rotary_embedding.py#L736
class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):
def __init__(
self,
@@ -1994,7 +1890,7 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):
max_position_embeddings = 2048,
base = 10000,
device = None,
- config = None, # [TODO] Hack to pass in config - need to remove later
+ config = None,
):
super().__init__(
dim = dim,
@@ -2004,12 +1900,10 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):
config = config,
)
- # From https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/api/model.py#L41
+ # From meta-llama/llama-models llama3_1/api/model.py#L41
def _apply_inv_freq_scaling(self, freqs: torch.Tensor):
- # llama3 factors from config; Llama-3.1 defaults when built without one
- # (legacy codegen path). Hardcoding 8 is wrong for e.g. Llama-3.2 (32).
- # v5 renames rope_scaling -> rope_parameters; read either so the factor
- # survives even if the rope_scaling back-compat shim is dropped.
+ # llama3 factors come from the config, with Llama-3.1 defaults when built without one: hardcoding
+ # 8 is wrong for Llama-3.2 (32). v5 renames rope_scaling to rope_parameters, so read either.
config = getattr(self, "_unsloth_rope_config", None)
rope_scaling = _rope_scaling_as_dict(
getattr(config, "rope_scaling", None) or getattr(config, "rope_parameters", None) or {}
@@ -2038,7 +1932,7 @@ class LlamaExtendedRotaryEmbedding(LlamaRotaryEmbedding):
class LongRopeRotaryEmbedding(torch.nn.Module):
- # For Phi 3.5 128K https://huggingface.co/microsoft/Phi-3.5-mini-instruct/blob/main/modeling_phi3.py
+ # For Phi 3.5 128K; see microsoft/Phi-3.5-mini-instruct modeling_phi3.py
def __init__(
self,
dim = None,
@@ -2048,7 +1942,7 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
short_factor = None,
long_factor = None,
device = None,
- config = None, # [TODO] Hack to pass in config - need to remove later
+ config = None,
):
super().__init__()
assert short_factor is not None
@@ -2056,7 +1950,6 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
assert type(original_max_position_embeddings) is int
if config is not None:
- # [TODO] Hack to pass in config - need to remove later
base = _get_rope_theta(config, default = base)
partial_rotary_factor = (
config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
@@ -2069,15 +1962,14 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
self.max_position_embeddings = max_position_embeddings
self.original_max_position_embeddings = original_max_position_embeddings
self.base = base
- # Dynamic RoPE we first set it to a max of 4 * 8192 tokens then we iteratively grow this
+ # Dynamic RoPE starts at a max of 4 * 8192 tokens and grows iteratively.
self.current_rope_size = min(original_max_position_embeddings, self.max_position_embeddings)
self.multi_gpu_short_cos_cached = [None] * DEVICE_COUNT
self.multi_gpu_short_sin_cached = [None] * DEVICE_COUNT
self.multi_gpu_long_cos_cached = [None] * DEVICE_COUNT
self.multi_gpu_long_sin_cached = [None] * DEVICE_COUNT
- # Long RoPE similar to RoPE except short sequences have 1 cos / sin
- # and long sequences have another cos / sin
+ # Long RoPE is like RoPE except short sequences have one cos/sin and long sequences another.
inv_freq_shape = (
torch.arange(0, self.dim, 2, dtype = torch.int64, device = "cpu").float() / self.dim
)
@@ -2086,7 +1978,7 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
short_inv_freq = 1.0 / (short_factor * self.base**inv_freq_shape)
long_inv_freq = 1.0 / (long_factor * self.base**inv_freq_shape)
- # Phi-3 Scale factor
+ # Phi-3 scale factor.
scale = self.max_position_embeddings / self.original_max_position_embeddings
if scale <= 1.0:
scaling_factor = 1.0
@@ -2096,12 +1988,10 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
)
self.scaling_factor = scaling_factor
- # Short and long inv_freq
self.register_buffer("short_inv_freq", short_inv_freq, persistent = False)
self.register_buffer("long_inv_freq", long_inv_freq, persistent = False)
- # Build here to make `torch.jit.trace` work.
- # Initialize short sequences cache for all devices
+ # Build here to make torch.jit.trace work, initializing the short-sequence cache for all devices.
dtype = torch.bfloat16 if is_bfloat16_supported() else torch.float16
t = torch.arange(
original_max_position_embeddings,
@@ -2122,7 +2012,6 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
self.multi_gpu_short_cos_cached[device_idx] = cos_cached
self.multi_gpu_short_sin_cached[device_idx] = sin_cached
- # dummy so that patch_utils doesn't fail for now
self.short_cos_cached = torch.empty(
1, device = get_current_device(), dtype = torch.get_default_dtype()
)
@@ -2137,14 +2026,12 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
)
def _set_cos_sin_cache(self, seq_len, device, dtype):
- # Note: on the original Llama codebase, these tensors are created on the target device (and not on CPU) and
- # in FP32. They are applied (multiplied) in FP32 as well.
+ # The original Llama codebase creates these on the target device in FP32 and multiplies in FP32.
self.current_rope_size = seq_len
t = torch.arange(
self.current_rope_size, device = self.long_inv_freq.device, dtype = torch.int64
).float()
- # Long sequences
freqs = torch.outer(t, self.long_inv_freq)
emb = torch.cat((freqs, freqs), dim = -1)
cos_cached = (emb.cos() * self.scaling_factor).to(
@@ -2163,7 +2050,6 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
position_ids = None,
seq_len = None,
):
- # x: [bs, num_attention_heads, seq_len, head_size]
if seq_len is not None and seq_len > self.current_rope_size:
self._set_cos_sin_cache(seq_len = seq_len, device = x.device, dtype = x.dtype)
@@ -2198,7 +2084,7 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
def extend_rope_embedding(self, x, seq_len):
if seq_len <= self.current_rope_size:
return
- # Iteratively grow by increments of 8192
+ # Iteratively grow by increments of 8192.
self.current_rope_size = ((seq_len // 8192) + ((seq_len % 8192) != 0)) * 8192
for device_idx in range(DEVICE_COUNT):
self._set_cos_sin_cache(
@@ -2207,10 +2093,9 @@ class LongRopeRotaryEmbedding(torch.nn.Module):
def unsloth_fast_generate(self, *args, **kwargs):
- # Restore training mode after generation if we started in it
restore_training_mode = self.training
- # Snapshot the real GC mode (e.g. "unsloth") before for_inference clears it,
- # so the restore preserves it rather than collapsing to a plain bool.
+ # Snapshot the real GC mode (e.g. "unsloth") before for_inference clears it, so the restore
+ # preserves it rather than collapsing to a plain bool.
use_gradient_checkpointing = next(
(v for v in (getattr(m, "gradient_checkpointing", False) for m in self.modules()) if v),
False,
@@ -2218,9 +2103,8 @@ def unsloth_fast_generate(self, *args, **kwargs):
FastLlamaModel.for_inference(self)
- # Unpack BatchEncoding passed as input_ids (old notebooks do
- # generate(input_ids=tokenizer(...))). v5 generate() calls .shape on it and
- # crashes; unpack into separate kwargs so v4 and v5 both work.
+ # Unpack a BatchEncoding passed as input_ids (old notebooks do generate(input_ids=tokenizer(...))):
+ # v5 generate() calls .shape on it and crashes.
_maybe_encoding = kwargs.get("input_ids", None)
if (
_maybe_encoding is not None
@@ -2244,16 +2128,11 @@ def unsloth_fast_generate(self, *args, **kwargs):
"You will need to do long context extension by increasing the `max_seq_length` in `FastLanguageModel.from_pretrained`."
)
- # Must patch accelerate for Xformers
- # if accelerate_new_send_to_device is not None:
- # import accelerate.utils.operations
- # accelerate.utils.operations.send_to_device = accelerate_new_send_to_device
- # pass
+ # Must patch accelerate for Xformers.
- # For newer HF
kwargs["cache_implementation"] = "dynamic"
- # transformers 4.50 renamed num_logits_to_keep -> logits_to_keep; pop both,
- # re-emit under the spelling forward() accepts.
+ # transformers 4.50 renamed num_logits_to_keep to logits_to_keep; pop both and re-emit under the
+ # spelling forward() accepts.
_provided_num = kwargs.pop("num_logits_to_keep", None)
_provided_logits = kwargs.pop("logits_to_keep", None)
_provided = _provided_logits if _provided_logits is not None else _provided_num
@@ -2262,7 +2141,7 @@ def unsloth_fast_generate(self, *args, **kwargs):
_has_new = "logits_to_keep" in _fwd_params
_has_old = "num_logits_to_keep" in _fwd_params
except (TypeError, ValueError):
- # Opaque forward: keep the caller's spelling, default to new.
+ # Opaque forward: keep the caller's spelling, defaulting to the new one.
_has_old = _provided_num is not None and _provided_logits is None
_has_new = not _has_old
if _has_new:
@@ -2270,28 +2149,21 @@ def unsloth_fast_generate(self, *args, **kwargs):
elif _has_old:
kwargs["num_logits_to_keep"] = _provided if _provided is not None else 1
- # Remove token_type_ids
kwargs.pop("token_type_ids", None)
- # Check pad_token
model_eos_token_id = getattr(self.config, "eos_token_id", None)
if model_eos_token_id is not None and hasattr(model_eos_token_id, "__iter__"):
model_eos_token_id = model_eos_token_id[0]
kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)
- # Mixed precision autocast
+ # Mixed precision autocast.
with (
_get_inference_mode_context_manager(self),
torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),
):
output = self._old_generate(*args, **kwargs)
- # Return accelerate back
- # if accelerate_new_send_to_device is not None:
- # accelerate.utils.operations.send_to_device = accelerate_old_send_to_device
- # pass
-
if restore_training_mode:
FastLlamaModel.for_training(
self,
@@ -2348,11 +2220,8 @@ class FastLlamaModel:
PeftModelForCausalLM.forward = PeftModel_fast_forward
fix_prepare_inputs_for_generation(LlamaForCausalLM)
- # Solves https://github.com/unslothai/unsloth/issues/168
- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.
- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
- # https://github.com/huggingface/transformers/pull/27931
- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,
+ # huggingface/transformers#27931), so the old rotary embeddings are retained.
import transformers.models.llama.modeling_llama
transformers.models.llama.modeling_llama.LlamaRotaryEmbedding = LlamaRotaryEmbedding
@@ -2417,11 +2286,10 @@ class FastLlamaModel:
raise RuntimeError(
"Unsloth: `unsloth_vllm_standby` is True, but environment variable `UNSLOTH_VLLM_STANDBY` is not set to 1!"
)
- # Only vLLM cannot take a revision; fast_inference may have just been cleared
- # above, and a num_labels load stays in-process, so both can honour the pin.
+ # Only vLLM cannot take a revision, and fast_inference may have just been cleared above while a
+ # num_labels load stays in-process, so both can honour the pin. vLLM fetches the default branch,
+ # so pinning only the config and tokenizer would mix two refs in one model.
if _vllm_will_load_weights(fast_inference, num_labels) and revision is not None:
- # vLLM fetches the default branch, so pinning only the config and tokenizer
- # would mix two refs in one model.
logger.warning_once(
f"Unsloth: Ignoring revision = `{revision}` since vLLM loads weights from "
"the default branch. Use `fast_inference = False` to load a pinned revision."
@@ -2455,7 +2323,6 @@ class FastLlamaModel:
print(statistics)
- # Warn about fast transfers
if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ:
old_hf_transfer = os.environ["HF_HUB_ENABLE_HF_TRANSFER"]
if old_hf_transfer in ("False", "false"):
@@ -2472,7 +2339,7 @@ class FastLlamaModel:
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
model_patcher.pre_patch()
- # For debugging - we use a download counter to see if environments are not breaking or if HF is down
+ # A download counter, to see whether environments are breaking or HF is down.
get_statistics(kwargs.get("local_files_only", False))
if dtype is None:
@@ -2480,22 +2347,16 @@ class FastLlamaModel:
elif dtype == torch.bfloat16 and not SUPPORTS_BFLOAT16:
logger.warning_once("Device does not support bfloat16. Will change to float16.")
dtype = torch.float16
- # elif dtype == torch.float16 and SUPPORTS_BFLOAT16:
- # logger.warning_once("Device supports bfloat16 but you selected float16. Will change to bfloat16.")
- # dtype = torch.bfloat16
assert dtype == torch.float16 or dtype == torch.bfloat16 or dtype == torch.float32
- # RoPE Scaling
- # Respect a user-provided config so it is the single config object used
- # everywhere below; otherwise HF would receive it again through **kwargs
- # alongside our own config= and fail with a duplicate-kwarg TypeError.
+ # Respect a user-provided config so it is the single config object used everywhere below; else
+ # HF gets it again through **kwargs alongside our config= and fails with a duplicate kwarg.
user_config = kwargs.pop("config", None)
if user_config is not None:
model_config = user_config
- # model_name may have been remapped to a prequantized repo whose
- # checkpoint needs its quantization_config; graft it onto the user
- # config or the 4bit weights load without their quant state.
+ # model_name may have been remapped to a prequantized repo whose checkpoint needs its
+ # quantization_config; graft it on, or the 4bit weights load without their quant state.
if getattr(model_config, "quantization_config", None) is None:
_checkpoint_config = AutoConfig.from_pretrained(
model_name,
@@ -2527,8 +2388,8 @@ class FastLlamaModel:
)
# Prefetch the repo (killable child) so the weight load is a cache hit. Runs after the
- # AutoConfig/model-class check so an unsupported repo fails on its small config fetch.
- # Warm the revision the load uses, or the repo downloads twice.
+ # AutoConfig/model-class check so an unsupported repo fails on its small config fetch, and warms
+ # the revision the load uses or the repo downloads twice.
_prefetched = maybe_prefetch_hf_snapshot(
model_name,
token = token,
@@ -2543,21 +2404,20 @@ class FastLlamaModel:
use_safetensors = kwargs.get("use_safetensors"),
from_tf = kwargs.get("from_tf", False),
from_flax = kwargs.get("from_flax", False),
- # Bare load reads only ROOT weights; skip subdir weights. Ignored when a subfolder is set.
+ # A bare load reads only ROOT weights, so skip subdir weights; ignored when a subfolder is set.
weights_at_root = True,
variant = kwargs.get("variant"), # forward so the warm keeps the variant .bin
gguf_file = kwargs.get(
"gguf_file"
), # forward so the warm fetches the GGUF (else ignored)
)
- # Child did the forced download; clear the flag so the load reuses the warm cache.
+ # The child did the forced download; clear the flag so the load reuses the warm cache.
if _prefetched and kwargs.get("force_download", False):
kwargs["force_download"] = False
- # Tokenizer always loads in-process. Resolve the cache_dir the tokenizer load will actually
- # use, mirroring load_correct_tokenizer: without an explicit cache_dir, Colab/Kaggle route to
- # a special tokenizer cache (huggingface_tokenizers_cache / Kaggle tmp), NOT the HF-default
- # cache the base snapshot warmed. So the base warm does not cover the tokenizer there.
+ # The tokenizer always loads in-process, and without an explicit cache_dir Colab/Kaggle route it
+ # to a special tokenizer cache rather than the one the base snapshot warmed, so resolve the
+ # cache_dir the tokenizer load will actually use.
from ..tokenizer_utils import (
IS_COLAB_ENVIRONMENT,
IS_KAGGLE_ENVIRONMENT,
@@ -2573,9 +2433,8 @@ class FastLlamaModel:
_tokenizer_cache_dir = "huggingface_tokenizers_cache"
elif IS_KAGGLE_ENVIRONMENT:
_tokenizer_cache_dir = os.path.join(KAGGLE_TMP, "huggingface_tokenizers_cache")
- # Warm the tokenizer repo into the cache the load will use whenever the base warm did not
- # cover it: a distinct tokenizer repo, fast_inference (base warm skipped), or a tokenizer
- # cache_dir that differs from the base-warm cache_dir (Colab/Kaggle special cache).
+ # Warm the tokenizer repo into the cache the load will use whenever the base warm did not cover
+ # it: a distinct tokenizer repo, fast_inference (base warm skipped), or a differing cache_dir.
_warm_tokenizer_repo = (
isinstance(_tokenizer_repo, str)
and bool(_tokenizer_repo)
@@ -2603,7 +2462,7 @@ class FastLlamaModel:
pass
has_rope_scaling = True
- # If max_seq_length is not specified, use maximum from config
+ # If max_seq_length is not specified, use the maximum from the config.
if max_seq_length is None:
max_seq_length = model_max_seq_length
@@ -2642,24 +2501,20 @@ class FastLlamaModel:
)
from unsloth_zoo.utils import get_quant_type
- # Extract load_in_8bit from kwargs if provided
load_in_8bit = kwargs.get("load_in_8bit", False)
- # Check and disable bitsandbytes loading if model has non-bitsandbytes quantization
+ # Disable bitsandbytes loading if the model has non-bitsandbytes quantization.
load_in_4bit, load_in_8bit, _ckpt_quant_method = check_and_disable_bitsandbytes_loading(
model_config, load_in_4bit = load_in_4bit, load_in_8bit = load_in_8bit
)
- # Correct UNSLOTH_MODEL_NAME's bnb tokens now that the effective bnb state is known
- # (the per-load env was built before remap/disable). gpt-oss only; no-op otherwise.
+ # Correct UNSLOTH_MODEL_NAME's bnb tokens now the effective bnb state is known (the per-load env
+ # was built before remap/disable). gpt-oss only.
sync_unsloth_model_name_bnb_flags(load_in_4bit, load_in_8bit)
- # `num_labels` sends the load to AutoModelForSequenceClassification, whose `score`
- # replaces the `lm_head` the planner named off the repo's config, so `dispatch_model`
- # refuses the map: "does not give any device for ... score.weight".
+ # num_labels sends the load to AutoModelForSequenceClassification, whose `score` replaces the
+ # lm_head the planner named off the repo's config, so dispatch_model refuses the map. The weights
+ # load against user_config while the planner rebuilds the repo's from model_name.
_planner_skip_reason = None
- # The weights load against `user_config`; the planner rebuilds the repo's from
- # `model_name`. A changed `num_hidden_layers` or `vocab_size` then gets a map for a
- # different model, which dispatches short or budgets short.
if user_config is not None:
_planner_skip_reason = (
"a caller-supplied config may not describe the repo the planner rebuilds"
@@ -2673,15 +2528,12 @@ class FastLlamaModel:
or "num_labels loads a task head the repo config does not describe"
)
- # A prequantized checkpoint is always sized by the list in its own config.json:
- # `merge_quantization_configs` overlays loading attributes for GPTQ / AWQ /
- # AutoRound / FbgemmFp8 / CompressedTensors / Mxfp4 and never for bitsandbytes, so
- # neither the merge below (#5027) nor any argument here reaches the plan. Modules
- # the load then keeps dense are charged at 4bit, budgeting the head device short.
- #
- # Worth refusing over only for mamba, whose `out_proj` stacks are GiBs of error. The
- # other reachable gap is MoE routers, ~70 MiB even on Qwen3-235B-A22B (4096 x 128
- # experts x 94 layers), inside the planner's own 256 MiB safety margin.
+ # A prequantized checkpoint is always sized by its own config.json: merge_quantization_configs
+ # overlays loading attributes for GPTQ / AWQ / AutoRound / FbgemmFp8 / CompressedTensors / Mxfp4
+ # and never for bitsandbytes, so neither the merge below (#5027) nor any argument here reaches the
+ # plan. Worth refusing over only for mamba, whose out_proj stacks are GiBs of error; the other
+ # reachable gap is MoE routers, ~70 MiB even on Qwen3-235B-A22B, inside the planner's 256 MiB
+ # margin.
if _planner_skip_reason is None and IS_FALCON_H1 and _ckpt_quant_method == "bitsandbytes":
_bundled = getattr(model_config, "quantization_config", None)
_bundled_skip = (
@@ -2695,9 +2547,8 @@ class FastLlamaModel:
"load adds, so the plan would size dense weights at 4bit"
)
- # Here, not in loader.py: the mapper up there can still substitute the repo (a
- # -bnb-4bit name resolving to its 16-bit twin), so a plan sized for the name the
- # caller gave is the wrong plan for the one actually loaded.
+ # Here, not in loader.py: the mapper there can still substitute the repo (a -bnb-4bit name
+ # resolving to its 16-bit twin), so a plan sized for the caller's name is the wrong plan.
device_map = resolve_unsloth_device_map(
requested_device_map(device_map),
model_name,
@@ -2709,11 +2560,11 @@ class FastLlamaModel:
trust_remote_code = trust_remote_code,
**planner_hub_kwargs(kwargs),
revision = revision,
- # The dtype the load gets, not the checkpoint's: `from_pretrained` overrides
- # config.json, and planning the wrong one mis-sizes weights 2x either way.
+ # The dtype the load gets, not the checkpoint's: from_pretrained overrides config.json, and
+ # planning the wrong one mis-sizes weights 2x either way.
**add_dtype_kwargs(dtype),
- # The caller's own config, still untouched in kwargs here, overrides the
- # flags: loader.py clears them whenever it forwards one.
+ # The caller's own config, still untouched in kwargs here, overrides the flags: loader.py clears
+ # them whenever it forwards one.
**planner_quantization_kwargs(
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
@@ -2729,7 +2580,7 @@ class FastLlamaModel:
if load_in_4bit:
llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES.copy()
if IS_FALCON_H1:
- # we cannot quantize out_proj layer due to mamba kernels: https://github.com/tiiuae/Falcon-H1/issues/13#issuecomment-2918671274
+ # out_proj cannot be quantized due to mamba kernels; see tiiuae/Falcon-H1#13 (comment 2918671274).
llm_int8_skip_modules.append("out_proj")
bnb_config = BitsAndBytesConfig(
load_in_4bit = True,
@@ -2738,10 +2589,9 @@ class FastLlamaModel:
bnb_4bit_compute_dtype = dtype,
llm_int8_skip_modules = llm_int8_skip_modules,
)
- # Pre-quantized checkpoints (e.g. unsloth/Qwen3-4B-bnb-4bit) use the
- # quantization_config baked into config.json, ignoring our runtime
- # BitsAndBytesConfig. Merge our skip list into the bundled config so
- # task heads like `score` stay in compute dtype. See unslothai/unsloth#5027.
+ # Pre-quantized checkpoints use the quantization_config baked into config.json, ignoring our
+ # runtime BitsAndBytesConfig, so merge our skip list into it to keep task heads like `score` in
+ # compute dtype (#5027).
if _ckpt_quant_method == "bitsandbytes" and _ckpt_qcfg is not None:
if isinstance(_ckpt_qcfg, dict):
_ckpt_skip = list(_ckpt_qcfg.get("llm_int8_skip_modules") or [])
@@ -2759,12 +2609,11 @@ class FastLlamaModel:
except Exception:
pass
- # https://huggingface.co/togethercomputer/LLaMA-2-7B-32K/discussions/12
- # RoPE Scaling's max_position_embeddings must be updated
+ # RoPE Scaling's max_position_embeddings must be updated; see togethercomputer/LLaMA-2-7B-32K discussion 12.
max_position_embeddings = max(max_seq_length, model_max_seq_length)
kwargs.pop("attn_implementation", None) # No need since we auto call it
- # Cannot be None, since HF now checks for the config
+ # Cannot be None, since HF now checks for the config.
if load_in_4bit:
kwargs["quantization_config"] = bnb_config
@@ -2773,13 +2622,12 @@ class FastLlamaModel:
raise_handler = RaiseUninitialized()
try:
if num_labels is not None:
- # Transformers 5.x @strict config classes reject unexpected kwargs
- # like num_labels and max_position_embeddings. Set on the config
- # object directly and pass config= instead.
+ # Transformers 5.x @strict config classes reject unexpected kwargs like num_labels and
+ # max_position_embeddings, so set them on the config object and pass config= instead.
set_task_config_attr(model_config, "num_labels", num_labels)
if max_position_embeddings is not None:
model_config.max_position_embeddings = max_position_embeddings
- # Pop config-level attrs that would be rejected by @strict model init
+ # Pop config-level attrs that would be rejected by @strict model init.
for _cfg_key in ("id2label", "label2id", "rope_scaling"):
_cfg_val = kwargs.pop(_cfg_key, None)
if _cfg_val is not None:
@@ -2791,16 +2639,14 @@ class FastLlamaModel:
model_name,
config = model_config,
device_map = device_map,
- # torch_dtype = dtype, # transformers changed torch_dtype to dtype
- # quantization_config = bnb_config,
token = token,
trust_remote_code = trust_remote_code,
attn_implementation = preferred_attn_impl,
revision = revision,
**kwargs,
)
- # Defensive: ensure the task head is in a floating dtype, guarding
- # against any path leaving it as integer storage. See unslothai/unsloth#5027.
+ # Defensive: ensure the task head is in a floating dtype, guarding against any path leaving it
+ # as integer storage (#5027).
for _head_name in ("score", "classifier", "qa_outputs"):
_head = getattr(model, _head_name, None)
if (
@@ -2819,13 +2665,13 @@ class FastLlamaModel:
offload_embedding = False,
fast_inference = fast_inference,
)
- # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200).
+ # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200), reading
+ # scales from the same revision as the weights.
_restore_dropped_fp8_scales(
model,
model_name,
local_files_only = kwargs.get("local_files_only", False),
token = token,
- # Read scales from the same revision as the weights.
revision = revision,
subfolder = kwargs.get("subfolder"),
cache_dir = kwargs.get("cache_dir"),
@@ -2833,9 +2679,8 @@ class FastLlamaModel:
)
elif not fast_inference:
if user_config is not None:
- # Transformers 5.x @strict model init rejects extra kwargs next
- # to config=; set the override on the config and pass the single
- # config object through so user overrides reach the actual load.
+ # Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override
+ # on the config and pass the single config object through.
if max_position_embeddings is not None:
model_config.max_position_embeddings = max_position_embeddings
model = AutoModelForCausalLM.from_pretrained(
@@ -2852,8 +2697,6 @@ class FastLlamaModel:
model = AutoModelForCausalLM.from_pretrained(
model_name,
device_map = device_map,
- # torch_dtype = dtype, # transformers changed torch_dtype to dtype
- # quantization_config = bnb_config,
token = token,
max_position_embeddings = max_position_embeddings,
trust_remote_code = trust_remote_code,
@@ -2861,7 +2704,6 @@ class FastLlamaModel:
revision = revision,
**kwargs,
)
- # Attach dispatch hooks for bnb multi-device loads.
from unsloth.models.vision import _attach_bnb_multidevice_hooks
_attach_bnb_multidevice_hooks(
@@ -2871,13 +2713,13 @@ class FastLlamaModel:
offload_embedding = False,
fast_inference = False,
)
- # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200).
+ # Re-apply block-fp8 weight_scale_inv tensors transformers dropped on load (#6200), reading
+ # scales from the same revision as the weights.
_restore_dropped_fp8_scales(
model,
model_name,
local_files_only = kwargs.get("local_files_only", False),
token = token,
- # Read scales from the same revision as the weights.
revision = revision,
subfolder = kwargs.get("subfolder"),
cache_dir = kwargs.get("cache_dir"),
@@ -2920,10 +2762,8 @@ class FastLlamaModel:
load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]
pass
- # Load vLLM first
llm = load_vllm(**load_vllm_kwargs)
- # Convert to HF format
_, quant_state_dict = get_vllm_state_dict(
llm,
config = model_config,
@@ -2938,10 +2778,9 @@ class FastLlamaModel:
model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)
finally:
raise_handler.remove()
- # Return old flag
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer
- # Counteract saved tokenizers
+ # Counteract saved tokenizers.
tokenizer_name = model_name if tokenizer_name is None else tokenizer_name
# Route the tokenizer load to the custom cache_dir the prefetch warmed.
_tokenizer_cache_kwargs = {}
@@ -2961,12 +2800,10 @@ class FastLlamaModel:
model, tokenizer = patch_tokenizer(model, tokenizer)
model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)
- # Patch up QKV / O and MLP
for idx, layer in enumerate(model.model.layers):
layer.self_attn.apply_qkv = original_apply_qkv
layer.self_attn.apply_o = original_apply_o
- # Patch Trainer
from transformers.trainer import Trainer
try:
@@ -2998,8 +2835,7 @@ class FastLlamaModel:
spaces = re.search(r"\n([\s\t]{1,})", original_debug).group(0)[1:]
front_spaces = re.match(r"([\s\t]{1,})", inner_training_loop).group(0)
- # Cannot use \\ since it will cause a SyntaxWarning in Python 3.12
- # Instead use chr(92) == \\
+ # Cannot use a backslash escape (SyntaxWarning in Python 3.12), so use chr(92).
debug_info = """debug_info = \\
f"==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = {len(set(p.device for p in model.parameters()))}\\n"\\
f" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\n"\\
@@ -3042,11 +2878,10 @@ class FastLlamaModel:
"is_torch_tpu_available()",
"False",
)
- # Wire the stray-forward compile-cache reset into the plain Trainer path: get_peft_model
- # arms the pre-train detector for every LoRA model, but only the TRL SFT/RL wrappers run
- # the reset. A grad-enabled probe before a bare transformers.Trainer.train() would
- # otherwise keep the poisoned Dynamo cache and leave the detector hook installed. Anchored
- # on the first body statement; a no-op (and harmless) if upstream drops that line.
+ # Wire the stray-forward compile-cache reset into the plain Trainer path: get_peft_model arms
+ # the detector for every LoRA model, but only the TRL SFT/RL wrappers run the reset, so a
+ # grad-enabled probe before a bare Trainer.train() would keep the poisoned Dynamo cache.
+ # Anchored on the first body statement; a harmless no-op if upstream drops that line.
inner_training_loop = inner_training_loop.replace(
"self.accelerator.free_memory()",
"self.accelerator.free_memory()\n"
@@ -3059,18 +2894,15 @@ class FastLlamaModel:
exec(inner_training_loop, globals())
Trainer._inner_training_loop = _fast_inner_training_loop
- # Save max_seq_length
model.max_seq_length = max_seq_length
m = model
while hasattr(m, "model"):
m.max_seq_length = max_seq_length
m = m.model
m.max_seq_length = max_seq_length
- # Save to modules as well
for module in model.modules():
module.max_seq_length = max_seq_length
- # We check the tokenizer first for errors
if fix_tokenizer:
tokenizer = check_tokenizer(
model = model,
@@ -3083,27 +2915,24 @@ class FastLlamaModel:
)
patch_saving_functions(tokenizer)
- # Fix up config for transformers uploading PEFT
- # Not necessary anymore since we require transformers>=4.37!
+ # Not necessary since we requires transformers >= 4.37
if False:
name = model.config._name_or_path
if name.startswith("unsloth/") and name.endswith("-bnb-4bit"):
name = name[: len(name) - len("-bnb-4bit")]
model.config.update({"_name_or_path": name})
- # Log Unsloth version for future fastpaths for inference
+ # Log the Unsloth version for future inference fastpaths.
model.config.update({"unsloth_version": __version__})
- # Add save modules
patch_saving_functions(model)
Trainer._inner_training_loop = _fast_inner_training_loop
- # Fix gradient accumulation. See issue #4982.
+ # Fix gradient accumulation; see #4982.
apply_accepts_loss_kwargs_fix(model)
patch_gradient_accumulation_fix(Trainer)
- # Save tokenizer for inference purposes
- tokenizer.padding_side = "left" # Force inference
+ tokenizer.padding_side = "left"
internal_model = model
while hasattr(internal_model, "model"):
internal_model._saved_temp_tokenizer = tokenizer
@@ -3113,13 +2942,12 @@ class FastLlamaModel:
# Prevent Transformers Trainer from auto-wrapping Unsloth LoRA models in DP.
_mark_unsloth_disable_data_parallel(model)
- # For transformers > 4.47.1, we need to add rotary_emb to all attention layers
+ # For transformers > 4.47.1, rotary_emb must be added to all attention layers.
if IS_ATTENTION_REFACTOR or hasattr(model.model, "rotary_emb"):
rotary_emb = model.model.rotary_emb
for layer in model.model.layers:
layer.self_attn.rotary_emb = rotary_emb
- # Add for_inference and for_training
model.for_training = functools.partial(FastLlamaModel.for_training, model)
model.for_inference = functools.partial(FastLlamaModel.for_inference, model)
m = model
@@ -3128,16 +2956,14 @@ class FastLlamaModel:
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
- # Patch generate
is_classification = "Classification" in str(type(model))
if not is_classification and model.generate.__name__ != "unsloth_fast_generate":
model._old_generate = model.generate
unsloth_fast_generate.__doc__ = model._old_generate.__doc__
model.generate = types.MethodType(unsloth_fast_generate, model)
- # Zero weight[padding_idx] only for embeddings NOT tied to lm_head: when
- # tied, zeroing the row forces pad logit = 0, which beats the (negative)
- # logits of real tokens (e.g. Gemma) and makes the decoder emit <pad>.
- # Skip if eos_token == pad_token to avoid zeroing the EOS embedding.
+ # Zero weight[padding_idx] only for embeddings NOT tied to lm_head: when tied, zeroing the row
+ # forces pad logit = 0, which beats the negative logits of real tokens (Gemma) and makes the
+ # decoder emit <pad>. Skipped when eos_token == pad_token.
eos_token_id = getattr(tokenizer, "eos_token_id", None) if tokenizer is not None else None
pad_token_id = getattr(tokenizer, "pad_token_id", None) if tokenizer is not None else None
if tokenizer is not None and eos_token_id != pad_token_id:
@@ -3151,7 +2977,6 @@ class FastLlamaModel:
and getattr(module, "padding_idx", None) is not None
):
if module.padding_idx < module.weight.shape[0]:
- # Skip if tied to lm_head
if (
lm_head_weight is not None
and module.weight.data_ptr() == lm_head_weight.data_ptr()
@@ -3159,9 +2984,9 @@ class FastLlamaModel:
continue
module.weight[module.padding_idx] = 0
- # LAST: `post_patch` replaces the embedding modules and the QKV/MLP
- # patching below it replaces the forwards a hook wraps, so an earlier
- # attach is lost. Skipped under vLLM, which owns the weights.
+ # LAST: post_patch replaces the embedding modules and the QKV/MLP patching below replaces the
+ # forwards a hook wraps, so an earlier attach is lost. Skipped under vLLM, which owns the
+ # weights.
if not fast_inference:
try:
from unsloth.models.vision import _repair_dispatch_hooks
@@ -3222,7 +3047,6 @@ class FastLlamaModel:
**kwargs,
):
if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1":
- # Check for other PEFT args in kwargs
for peft_arg, flag in (
("finetune_vision_layers", False),
("finetune_language_layers", True),
@@ -3256,13 +3080,12 @@ class FastLlamaModel:
)
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")
- # Full finetuning still compiles, so a stray pre-train forward can poison the
- # cache; install the detector here too (it is idempotent).
+ # Full finetuning still compiles, so a stray pre-train forward can poison the cache; install the
+ # detector here too (idempotent).
_unsloth_install_pretrain_detector(model)
return model
transformers_set_seed(random_state)
- # Apply gradient checkpointing with smart heuristics
max_seq = getattr(model, "max_seq_length", 512)
dtype = model.get_input_embeddings().weight.dtype
use_gradient_checkpointing = apply_unsloth_gradient_checkpointing(
@@ -3277,7 +3100,6 @@ class FastLlamaModel:
if isinstance(model, PeftModelForCausalLM) or isinstance(
model, PeftModelForSequenceClassification
):
- # Check if exactly the same and then pass through!
assert hasattr(model, "peft_config")
peft_config = model.peft_config["default"].to_dict()
@@ -3295,18 +3117,15 @@ class FastLlamaModel:
for param in check_parameters:
check_all = check_all and (peft_config[param] == eval(param))
- # use_dora arrives via **kwargs, not a named parameter of this
- # function, so it can't go in check_parameters (which uses
- # eval(param) and needs each entry to be a bound local name).
- # Compare it explicitly so a DoRA request against an existing
- # plain-LoRA adapter doesn't silently pass through unchanged.
+ # use_dora arrives via **kwargs, not as a named parameter, so it cannot go in check_parameters
+ # (which evals each entry as a bound local); compare it explicitly, or a DoRA request against an
+ # existing plain-LoRA adapter silently passes through unchanged.
check_all = check_all and (
bool(peft_config.get("use_dora", False)) == bool(kwargs.get("use_dora", False))
)
- # Everything the stored adapter trains, wherever PEFT filed it: tying moves
- # the counterpart to modules_to_tie, which to_dict() drops, so read the
- # config object.
+ # Everything the stored adapter trains, wherever PEFT filed it: tying moves the counterpart to
+ # modules_to_tie, which to_dict() drops, so read the config object.
requested_modules_to_save = modules_to_save
modules_to_save = list(peft_config["modules_to_save"] or [])
old_target_modules = list(peft_config["target_modules"]) + modules_to_save
@@ -3314,8 +3133,8 @@ class FastLlamaModel:
getattr(model.peft_config["default"], "modules_to_tie", None) or []
)
- # The new side is what THIS call asks for, through the same redirect. Built
- # from the stored lists instead, a narrowed request would compare equal.
+ # The new side is what THIS call asks for, through the same redirect; built from the stored
+ # lists instead, a narrowed request would compare equal.
requested_targets, requested_saved, _ = _redirect_embedding_targets(
target_modules,
requested_modules_to_save,
@@ -3323,13 +3142,13 @@ class FastLlamaModel:
)
new_target_modules = list(requested_targets) + list(requested_saved or [])
if isinstance(model, PeftModelForSequenceClassification):
- # PeftModelForSequenceClassification.__init__ extends modules_to_save with
- # these unconditionally, so the stored config names modules the caller
- # never asked for. Mirror it, or an unchanged request looks different.
+ # PeftModelForSequenceClassification.__init__ extends modules_to_save with these
+ # unconditionally, so the stored config names modules the caller never asked for. Mirror it, or
+ # an unchanged request looks different.
new_target_modules += ["classifier", "score"]
- # Tying is not in check_parameters and leaves both lists looking the same, so
- # compare it separately. On what it actually does, not what was asked: on an
- # untied model the request changes nothing either way.
+ # Tying is not in check_parameters and leaves both lists looking the same, so compare it
+ # separately, on what it does rather than what was asked: on an untied model the request changes
+ # nothing either way.
_requested_ties = _effective_weight_tying(
model,
requested_saved,
@@ -3338,17 +3157,15 @@ class FastLlamaModel:
_stored_ties = bool(getattr(model.peft_config["default"], "modules_to_tie", None))
check_all = check_all and (_requested_ties == _stored_ties)
if _requested_ties:
- # PEFT files the counterpart under modules_to_tie, which the old side
- # counts, so name it here even when the caller listed only one side.
+ # PEFT files the counterpart under modules_to_tie, which the old side counts, so name it here
+ # even when the caller listed only one side.
new_target_modules += list(EMBEDDING_MODULES)
- # Per-expert Linear MoE experts (e.g. gpt-oss bnb-4bit) were auto-added to the
- # saved target_modules when the adapter was first created. Recompute them so a
- # repeat get_peft_model call with the same args stays idempotent instead of
- # tripping the mismatch below. No-op for non per-expert-Linear models.
+ # Per-expert Linear MoE experts (gpt-oss bnb-4bit) were auto-added to the saved target_modules
+ # at creation, so recompute them and keep a repeat get_peft_model call idempotent.
new_target_modules += get_moe_target_modules(model, target_modules)
- # Now check! Fold away PEFT's model.embed_tokens alias, and only that one:
- # layers.0.q_proj is a real target and is not layers.1.q_proj.
+ # Fold away PEFT's model.embed_tokens alias, and only that one: layers.0.q_proj is a real target
+ # and is not layers.1.q_proj.
def _leaf_name(module):
return _embedding_leaf(module) or module
@@ -3362,11 +3179,8 @@ class FastLlamaModel:
)
if check_all:
- # Simply pass through!
logger.warning("Unsloth: Already have LoRA adapters! We shall skip this step.")
- # Offload!
- # [TODO] First offload lm_head and embed_tokens to CPU (should be disk!!)
if "embed_tokens" in new_target_modules:
print("Unsloth: Training embed_tokens in mixed precision to save VRAM")
@@ -3381,8 +3195,8 @@ class FastLlamaModel:
model.get_output_embeddings(), DEVICE_TYPE_TORCH
)
- # Pre-wrapped PEFT model passes through here; still arm the detector so an RL
- # trainer can reset a compile cache poisoned by a pre-train forward.
+ # A pre-wrapped PEFT model passes through here; still arm the detector so an RL trainer can
+ # reset a compile cache poisoned by a pre-train forward.
_unsloth_install_pretrain_detector(model)
# This branch returns before patch_peft_model, so record here too;
# apply_unsloth_gradient_checkpointing above already re-patched global state to match (#4735).
@@ -3449,7 +3263,6 @@ class FastLlamaModel:
assert type(use_rslora) is bool
if use_rslora:
if not SUPPORTS_RSLORA:
- # We manually check for PEFT
import peft
raise RuntimeError(
f"Unsloth: Your PEFT version of {peft.__version__} does not support `use_rslora`.\n"
@@ -3477,8 +3290,8 @@ class FastLlamaModel:
train_lm_head = False
train_embed_tokens = False
final_modules = []
- # LoRA on these never trains (see _redirect_embedding_targets); modules_to_save
- # does, and is what embedding_learning_rate matches.
+ # LoRA on these never trains (see _redirect_embedding_targets); modules_to_save does, and is
+ # what embedding_learning_rate matches.
target_modules, modules_to_save, _moved_embedding_modules = _redirect_embedding_targets(
target_modules,
modules_to_save,
@@ -3521,10 +3334,9 @@ class FastLlamaModel:
"Beware - your finetuning might be noticeably slower!"
)
- # Check if we added new tokens!
if hasattr(model, "_need_to_train_embeddings"):
- # Check if embed_tokens/lm_head are already being trained
- # (either as LoRA targets in final_modules or via modules_to_save)
+ # embed_tokens/lm_head may already be trained, either as LoRA targets in final_modules or via
+ # modules_to_save.
_embed_already_trained = train_embed_tokens or "embed_tokens" in final_modules
_lm_head_already_trained = train_lm_head or "lm_head" in final_modules
if not _lm_head_already_trained or not _embed_already_trained:
@@ -3533,7 +3345,7 @@ class FastLlamaModel:
"train the lm_head and embed_tokens.\nWe must turn it on for you."
)
- # Only add to modules_to_save if not already a LoRA target
+ # Only add to modules_to_save if not already a LoRA target.
if not _embed_already_trained:
train_embed_tokens = True
if modules_to_save is None:
@@ -3548,18 +3360,8 @@ class FastLlamaModel:
elif "lm_head" not in modules_to_save:
modules_to_save.append("lm_head")
- # Check for Llama-3
- # if hasattr(model._saved_temp_tokenizer, "_using_llama3_template"):
- # if not train_embed_tokens and not train_lm_head:
- # raise RuntimeError("")
-
- # First fix untrained tokens
- # Wrong - can cause reserved tokens to pop out!!
- # if train_embed_tokens or train_lm_head:
- # fix_untrained_tokens(model, eps = 1e-16)
- # pass
+ # Fixing untrained tokens here is wrong: it can cause reserved tokens to pop out.
- # Check modules_to_save
if modules_to_save is not None:
for module in modules_to_save:
# By leaf: PEFT resolves model.embed_tokens to the same module.
@@ -3577,7 +3379,6 @@ class FastLlamaModel:
vllm_engine = None
if hasattr(model, "vllm_engine"):
- # Fast inference!
vllm_engine = model.vllm_engine
vllm_fast_generate = model.fast_generate
vllm_fast_generate_batches = model.fast_generate_batches
@@ -3592,15 +3393,15 @@ class FastLlamaModel:
"Unsloth: Currently fast inference does not work with using biases for LoRA."
)
- # Does not get lora yet, so get name from model, not base model
+ # Does not get lora yet, so take the name from model, not base model.
is_classification = "Classification" in str(type(model))
- # Auto-detect MoE models and populate target_parameters for expert layers
+ # Auto-detect MoE models and populate target_parameters for expert layers.
if target_parameters is None:
target_parameters = get_moe_target_parameters(model, target_modules)
- # Per-expert Linear expert layouts (e.g. gpt-oss bnb-4bit) are Linear modules,
- # not fused Parameters, so target them via target_modules. No-op otherwise.
+ # Per-expert Linear expert layouts (gpt-oss bnb-4bit) are Linear modules, not fused Parameters,
+ # so target them via target_modules.
_moe_module_targets = get_moe_target_modules(model, target_modules)
if _moe_module_targets:
_added = [t for t in _moe_module_targets if t not in final_modules]
@@ -3635,7 +3436,7 @@ class FastLlamaModel:
ensure_weight_tying = ensure_weight_tying,
**kwargs,
)
- # Older PEFT has no `ensure_weight_tying`; passing it would TypeError.
+ # Older PEFT has no ensure_weight_tying; passing it would TypeError.
if "ensure_weight_tying" not in inspect.signature(LoraConfig).parameters:
del arguments["ensure_weight_tying"]
if not SUPPORTS_LOFTQ:
@@ -3646,7 +3447,6 @@ class FastLlamaModel:
_saved_temp_tokenizer = model._saved_temp_tokenizer
lora_config = LoraConfig(**arguments)
- # First offload lm_head and embed_tokens to disk
input_embeddings_device = model.get_input_embeddings().weight.device
if is_classification:
output_embeddings_device = model.score.weight.device
@@ -3658,7 +3458,6 @@ class FastLlamaModel:
print("Unsloth: Offloading input_embeddings to disk to save VRAM")
offload_input_embeddings(model, temporary_location)
- # Remove old items to save VRAM
for _ in range(3):
gc.collect()
clean_gpu_cache()
@@ -3667,7 +3466,6 @@ class FastLlamaModel:
print("Unsloth: Offloading output_embeddings to disk to save VRAM")
offload_output_embeddings(model, temporary_location)
- # Remove old items to save VRAM
for _ in range(3):
gc.collect()
clean_gpu_cache()
@@ -3684,10 +3482,8 @@ class FastLlamaModel:
)
except Exception as _exc:
logger.warning_once(f"Unsloth: could not lift adapter hooks: {_exc}")
- # Fix LoraConfig.auto_mapping is None
fix_lora_auto_mapping(model)
- # Apply QAT + LoRA if specified
if qat_scheme is not None:
print("Unsloth: Applying QAT to mitigate quantization degradation")
model = FastLlamaModel._prepare_for_qat(model, qat_scheme)
@@ -3707,7 +3503,6 @@ class FastLlamaModel:
if not hasattr(source_module, "weight"):
return
weight = source_module.weight
- # Remove existing registration to avoid "attribute already exists"
if "weight" in getattr(target_module, "_parameters", {}):
target_module._parameters.pop("weight")
if hasattr(target_module, "weight"):
@@ -3720,7 +3515,8 @@ class FastLlamaModel:
)
target_module.register_parameter("weight", weight)
- # Tie trainable copies created by ModulesToSaveWrapper first (these are used in forward)
+ # Tie the trainable copies created by ModulesToSaveWrapper first, since those are used in
+ # forward, then the original_module references if present.
if hasattr(input_embeddings, "modules_to_save") and hasattr(
output_embeddings, "modules_to_save"
):
@@ -3732,7 +3528,6 @@ class FastLlamaModel:
input_embeddings.modules_to_save.default,
)
- # Tie original_module references as well if present
if hasattr(input_embeddings, "original_module") and hasattr(
output_embeddings, "original_module"
):
@@ -3767,7 +3562,6 @@ class FastLlamaModel:
original_device = output_embeddings_device,
)
- # Patch tokenizer to pad to the right
internal_model = model
while hasattr(internal_model, "model"):
if hasattr(internal_model, "_saved_temp_tokenizer"):
@@ -3778,14 +3572,12 @@ class FastLlamaModel:
# Prevent Transformers Trainer from auto-wrapping Unsloth LoRA models in DP.
_mark_unsloth_disable_data_parallel(model)
- # Clear deleted GPU items
for _ in range(3):
gc.collect()
clean_gpu_cache()
patch_peft_fast_inference(model)
- # Add for_inference and for_training
model.for_training = functools.partial(FastLlamaModel.for_training, model)
model.for_inference = functools.partial(FastLlamaModel.for_inference, model)
m = model
@@ -3793,18 +3585,17 @@ class FastLlamaModel:
m.for_training = functools.partial(FastBaseModel.for_training, m)
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
- # Detect a stray pre-train forward so train() can drop the torch.compile
- # graph cache it would otherwise poison (see prepare_for_training_mode).
+ # Detect a stray pre-train forward so train() can drop the torch.compile graph cache it would
+ # otherwise poison (see prepare_for_training_mode).
_unsloth_install_pretrain_detector(model)
model = _exclude_rope_inv_freq_from_ddp(model)
return model
@staticmethod
def patch_peft_model(model, use_gradient_checkpointing = "unsloth"):
- # Persist the effective GC mode so the trainer restores it verbatim: for_inference()
- # clears the module flags every GRPO step, and a plain TrainingArguments defaults it to
- # False, which would otherwise silently disable it at train time (#4735). Recorded here,
- # not in get_peft_model, so adapters loaded via loader.py's from_pretrained path are covered.
+ # Persist the effective GC mode so the trainer restores it verbatim: for_inference() clears the
+ # module flags every GRPO step, and TrainingArguments defaults it to False, which would silently
+ # disable it at train time (#4735). Recorded here so loader.py's from_pretrained path is covered.
model._unsloth_gradient_checkpointing = use_gradient_checkpointing
if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1":
return FastBaseModel.patch_peft_model(
@@ -3816,7 +3607,6 @@ class FastLlamaModel:
):
raise TypeError("Unsloth: Your model needs to call `.get_peft_model` first!")
- # Get activation function
model_type = model.config.model_type
if model_type == "llama":
@@ -3848,26 +3638,21 @@ class FastLlamaModel:
use_reentrant = True,
)
- # Fix up config for transformers uploading PEFT
for active_adapter in model.peft_config.keys():
- # Not necessary since we requires transformers >= 4.37
if False:
name = model.peft_config[active_adapter].base_model_name_or_path
if name.startswith("unsloth/") and name.endswith("-bnb-4bit"):
name = name[: len(name) - len("-bnb-4bit")]
model.peft_config[active_adapter].base_model_name_or_path = name
pass
- # Add revision to enable future fast inference paths
- # [TODO] Bugs out!see https://github.com/unslothai/unsloth/issues/492
- # model.peft_config[active_adapter].revision = f"unsloth"
+ # Adding a revision to enable future fast inference paths bugs out; see #492.
from transformers.trainer import Trainer
if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":
raise RuntimeError("Unsloth: Unsuccessfully patched Trainer! Please file a bug report!")
- # Fix loftq issues
- # loftq_config must not = None, but rather {}
+ # loftq_config must not be None, but rather {}.
all_configs = model.peft_config
for key, current_config in all_configs.items():
if hasattr(current_config, "loftq_config") and current_config.loftq_config is None:
@@ -3876,7 +3661,6 @@ class FastLlamaModel:
current_config = current_config.__class__(**new_args)
all_configs[key] = current_config
- # Do patching
n_mlp = 0
n_qkv = 0
n_o = 0
@@ -3885,11 +3669,10 @@ class FastLlamaModel:
model.active_adapters[0] if hasattr(model, "active_adapters") else model.active_adapter
)
- # Get dropout and bias
lora_dropout = model.peft_config[active_adapter].lora_dropout
bias = model.peft_config[active_adapter].bias
- # We also do not inplace edit QKV for Cohere!
+ # QKV is not inplace-edited for Cohere.
_apply_lora_mlp = (
functools.partial(apply_lora_mlp, inplace = False)
if model_type == "cohere"
@@ -3899,10 +3682,8 @@ class FastLlamaModel:
if lora_dropout == 0 and bias == "none":
for idx, layer in enumerate(model.model.model.layers):
if model_type != "falcon_h1":
- # LoRAMLP.apply doesn't have functionality for gate and down multipliers yet.
- # Don't patch falcon h1 for the time being.
+ # LoRAMLP.apply has no gate/down multiplier support yet, so falcon h1 is not patched for now.
- # MLP patching
mlp_module = layer.mlp
gate_proj = mlp_module.gate_proj
up_proj = mlp_module.up_proj
@@ -3919,9 +3700,9 @@ class FastLlamaModel:
and (len(getattr(up_proj, "lora_magnitude_vector", []) or []) == 0)
and (len(getattr(down_proj, "lora_magnitude_vector", []) or []) == 0)
):
- # https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module
+ # See stackoverflow.com/questions/50599045 on replacing a function within a class of a module.
if hasattr(mlp_module, "_unsloth_forward"):
- # then we've patched the mlp to use TiledMLP
+ # Then the mlp has been patched to use TiledMLP.
mlp_module._unsloth_forward = types.MethodType(
_apply_lora_mlp, mlp_module
)
@@ -3934,7 +3715,6 @@ class FastLlamaModel:
"are not enabled or a bias term (like in Qwen) is used."
)
- # QKV attention patching
q_proj = layer.self_attn.q_proj
k_proj = layer.self_attn.k_proj
v_proj = layer.self_attn.v_proj
@@ -3960,7 +3740,6 @@ class FastLlamaModel:
"are not enabled or a bias term (like in Qwen) is used."
)
- # O attention patching
o_proj = layer.self_attn.o_proj
if (
hasattr(o_proj, "lora_A")
@@ -3981,21 +3760,18 @@ class FastLlamaModel:
)
patch_saving_functions(model)
- # Patch cross entropy loss labels
- # Fixes https://github.com/unslothai/unsloth/issues/10
+ # Patch cross entropy loss labels; fixes #10.
max_seq_length = model.max_seq_length
- # extra_ignored_labels = torch.full((max_seq_length, 1), -100, device = "cuda:0")
- # model.model.extra_ignored_labels = extra_ignored_labels
internal_model = model
while hasattr(internal_model, "model"):
internal_model.max_seq_length = max_seq_length
internal_model = internal_model.model
internal_model.max_seq_length = max_seq_length
- # Save to modules as well
+ # Since transformers 4.53, must turn on explicitly
+ # Since transformers 4.53, must turn off explicitly
for module in model.modules():
module.max_seq_length = max_seq_length
- # Patch tokenizer to pad to the right
internal_model = model
while hasattr(internal_model, "model"):
if hasattr(internal_model, "_saved_temp_tokenizer"):
@@ -4004,14 +3780,12 @@ class FastLlamaModel:
if hasattr(internal_model, "_saved_temp_tokenizer"):
internal_model._saved_temp_tokenizer.padding_side = "right"
- # Clear deleted GPU items
for _ in range(3):
gc.collect()
clean_gpu_cache()
patch_peft_fast_inference(model)
- # Add for_inference and for_training
model.for_training = functools.partial(FastLlamaModel.for_training, model)
model.for_inference = functools.partial(FastLlamaModel.for_inference, model)
m = model
@@ -4019,8 +3793,6 @@ class FastLlamaModel:
m.for_training = functools.partial(FastBaseModel.for_training, m)
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
- # Detect a stray pre-train forward so train() can drop the torch.compile
- # graph cache it would otherwise poison (see prepare_for_training_mode).
_unsloth_install_pretrain_detector(model)
return model
@@ -4036,10 +3808,8 @@ class FastLlamaModel:
m.gradient_checkpointing = False
if hasattr(m, "training"):
m.training = False
- # Pad tokenizer to the left
if hasattr(m, "_saved_temp_tokenizer"):
m._saved_temp_tokenizer.padding_side = "left"
- # Set a flag for generation!
m._flag_for_generation = True
m = model
@@ -4049,12 +3819,11 @@ class FastLlamaModel:
_for_inference(m)
model.eval() # to turn off training on modules deeper in
- # Since transformers 4.53, must turn off explicitly
+ # Since transformers 4.53, this must be turned off explicitly.
for module in model.modules():
if hasattr(module, "gradient_checkpointing"):
module.gradient_checkpointing = False
- # Also disable training for embeddings for NEFTune
if hasattr(model, "get_input_embeddings"):
embeddings = model.get_input_embeddings()
if hasattr(embeddings, "training"):
@@ -4064,8 +3833,8 @@ class FastLlamaModel:
if hasattr(embeddings, "training"):
embeddings.training = False
- # Restore use_cache values that prepare_model_for_training disabled
- # for gradient checkpointing (older unsloth_zoo has no restore helper)
+ # Restore use_cache values that prepare_model_for_training disabled for gradient checkpointing
+ # (older unsloth_zoo has no restore helper).
try:
from unsloth_zoo.training_utils import restore_use_cache
restore_use_cache(model)
@@ -4080,7 +3849,6 @@ class FastLlamaModel:
"Unsloth: I think you're passing a tokenizer, not the model to for_training!"
)
- # Delete all fast inference loras
for param in model.parameters():
if hasattr(param, "_fast_lora"):
del param._fast_lora
@@ -4090,13 +3858,11 @@ class FastLlamaModel:
m.gradient_checkpointing = use_gradient_checkpointing
if hasattr(m, "training"):
m.training = True
- # Pad tokenizer to the left
if hasattr(m, "_saved_temp_tokenizer"):
m._saved_temp_tokenizer.padding_side = "right"
- # Set a flag for generation!
if hasattr(m, "_flag_for_generation"):
try:
- # A PEFT wrapper delegates the read but owns nothing to delete
+ # A PEFT wrapper delegates the read but owns nothing to delete.
del m._flag_for_generation
except AttributeError:
pass
@@ -4108,12 +3874,11 @@ class FastLlamaModel:
_for_training(m)
model.train() # to turn on training on modules deeper in
- # Since transformers 4.53, must turn on explicitly
+ # Since transformers 4.53, this must be turned on explicitly.
for module in model.modules():
if hasattr(module, "gradient_checkpointing"):
module.gradient_checkpointing = use_gradient_checkpointing
- # Also re-enable training for embeddings for NEFTune
if hasattr(model, "get_input_embeddings"):
embeddings = model.get_input_embeddings()
if hasattr(embeddings, "training"):
@@ -4123,8 +3888,8 @@ class FastLlamaModel:
if hasattr(embeddings, "training"):
embeddings.training = True
- # Re-disable use_cache if prepare_model_for_training had disabled it
- # and for_inference restored it (record only exists after a disable)
+ # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored
+ # it; the record only exists after a disable.
if (
use_gradient_checkpointing
and getattr(model, "_unsloth_use_cache_originals", None) is not None
@@ -4140,7 +3905,7 @@ class FastLlamaModel:
from .rl import PatchFastRL
# Auto-enable grouped-GEMM MoE (tf<5 ModuleList experts) on built / PEFT'd models. Wrap the
-# loader leaves before PatchFastRL so downstream patchers see the wrapped versions. Guarded.
+# loader leaves before PatchFastRL so downstream patchers see the wrapped versions.
try:
from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moe
FastLlamaModel.from_pretrained = staticmethod(
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 3ceecbdce70..0313cc75c2b 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -21,9 +21,8 @@ from ._utils import (
USE_MODELSCOPE,
get_transformers_model_type,
hf_login,
- # Single source of truth is _utils.py; re-exported here so callers doing
- # `from unsloth.models.loader import DISABLE_SDPA_MODEL_NAMES` keep working and so
- # _is_sdpa_excluded (in _utils) can honor it without a loader -> _utils cycle.
+ # Single source of truth is _utils.py; re-exported so callers importing it from here keep
+ # working and so _is_sdpa_excluded can honor it without a loader -> _utils cycle.
DISABLE_SDPA_MODEL_NAMES,
)
from ._custom_dtype import register_custom_dtype
@@ -59,7 +58,7 @@ except:
try:
from huggingface_hub.utils import get_token
except:
- # For older versions of huggingface_hub
+ # For older versions of huggingface_hub.
from huggingface_hub.utils._token import get_token
import importlib.util
from ..device_type import (
@@ -72,7 +71,7 @@ from ..device_type import (
ALLOW_BITSANDBYTES,
)
-# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!
+# huggingface/transformers#26037 allows 4 bit loading.
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from unsloth_zoo.tiled_mlp import patch_tiled_mlp
@@ -90,8 +89,7 @@ SUPPORTS_FALCON_H1 = transformers_version >= Version("4.53.0")
SUPPORTS_GEMMA3N = transformers_version >= Version("4.53.0")
SUPPORTS_GPTOSS = transformers_version >= Version("4.55.0")
SUPPORTS_GEMMA4 = transformers_version >= Version("5.5.0")
-# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq).
-# See _fix_rope_inv_freq() below for details.
+# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq); see _fix_rope_inv_freq() below.
_NEEDS_ROPE_FIX = transformers_version >= Version("5.0.0")
if SUPPORTS_GEMMA:
from .gemma import FastGemmaModel
@@ -121,12 +119,11 @@ from ._utils import (
maybe_prefetch_hf_snapshot,
)
-# Source of truth is unsloth_zoo.model_lists. Re-exported so callers doing
-# `from unsloth.models.loader import FORCE_FLOAT32` keep working. The fallback
-# list is also unioned in so a newer unsloth still forces float32 for these
-# archs when paired with an older unsloth_zoo that predates them (upgrade skew).
+# Source of truth is unsloth_zoo.model_lists, re-exported for callers importing FORCE_FLOAT32
+# from here. The fallback list is unioned in so a newer unsloth still forces float32 for these
+# archs against an older zoo.
_FORCE_FLOAT32_FALLBACK = [
- "gemma3,", # Add comma bc gemma3 will match gemma3n
+ "gemma3,",
"gemma3text", # Gemma3TextModel (EmbeddingGemma, standalone text-only Gemma3)
"gemma3n",
"gemma4", # Gemma4 (gemma4 / gemma4_text): float16 NaNs grad norms in the backward
@@ -145,7 +142,7 @@ for _mt in _FORCE_FLOAT32_FALLBACK:
FORCE_FLOAT32.append(_mt)
global DISABLE_COMPILE_MODEL_NAMES
-# Must be alphabetically sorted for each entry
+# Must be alphabetically sorted for each entry.
def _strip_unsloth_bnb_4bit_suffix(model_name: str) -> str:
@@ -283,8 +280,8 @@ DISABLE_COMPILE_MODEL_NAMES = [
]
# Architectures with gated-deltanet (linear attention) layers. Unsloth bundles the
-# flash-linear-attention Triton kernels (unsloth_zoo/_vendored/fla), so no install is
-# needed; transformers uses the much slower pure PyTorch path only when they can't be enabled.
+# flash-linear-attention Triton kernels, so no install is needed; transformers falls back to
+# the much slower pure PyTorch path only when they cannot be enabled.
FLA_MODEL_TYPE_PREFIXES = ("qwen3_next", "qwen3_5", "kimi_linear", "olmo_hybrid")
_fla_advised = False
@@ -316,11 +313,9 @@ def _maybe_advise_fla_install(model_types):
except Exception:
return
_fla_advised = True
- # Prefer unsloth_zoo's specific reason when it disabled the kernels on purpose.
- # The generic text below blames CUDA / torch / triton minimums, all of which are
- # satisfied on e.g. an H100 that hit the fla #640 Triton miscompile, so it would
- # send that user looking in the wrong place. try/except keeps an unsloth pinned
- # against an older unsloth_zoo working.
+ # Prefer unsloth_zoo's reason when it disabled the kernels on purpose: the generic text blames
+ # CUDA / torch / triton minimums, all satisfied on an H100 that hit the fla #640 Triton
+ # miscompile, so it would point that user in the wrong direction.
try:
from unsloth_zoo.temporary_patches.fla_vendor import fla_unavailable_reason
reason = fla_unavailable_reason()
@@ -350,9 +345,8 @@ def _fix_rope_inv_freq(model):
return model
for name, module in model.named_modules():
- # Unsloth's LlamaRotaryEmbedding and subclasses (Extended, LinearScaling,
- # Granite). Native v5 rotary classes (Gemma3, etc.) have original_inv_freq
- # which v5's _init_weights() uses to restore inv_freq, so they are fine.
+ # Unsloth's LlamaRotaryEmbedding and subclasses (Extended, LinearScaling, Granite). Native v5
+ # rotary classes carry original_inv_freq, which v5's _init_weights() uses to restore inv_freq.
if (
hasattr(module, "inv_freq")
and hasattr(module, "base")
@@ -381,7 +375,7 @@ def _fix_rope_inv_freq(model):
dtype = torch.get_default_dtype(),
)
- # LongRopeRotaryEmbedding (Phi-3.5 style with short_inv_freq + long_inv_freq)
+ # LongRopeRotaryEmbedding (Phi-3.5 style with short_inv_freq + long_inv_freq).
elif (
hasattr(module, "short_inv_freq")
and hasattr(module, "long_inv_freq")
@@ -456,11 +450,11 @@ class FastLanguageModel(FastLlamaModel):
qat_scheme = None,
load_in_fp8 = False, # fp8 LoRA (True, False, 'block')
unsloth_tiled_mlp = False,
- text_only = False, # Skip vision/audio towers and load only the text decoder
+ text_only = False,
*args,
**kwargs,
):
- # Respect user-provided quantization_config (e.g. BitsAndBytesConfig)
+ # Respect a user-provided quantization_config (e.g. BitsAndBytesConfig).
quantization_config = kwargs.get("quantization_config", None)
if quantization_config is not None:
if isinstance(quantization_config, dict):
@@ -476,9 +470,8 @@ class FastLanguageModel(FastLlamaModel):
load_in_8bit = True
load_in_4bit = False
- # Login to allow private models
- # Before normalization: dtype is also derived below from a 4bit config's
- # compute dtype, which is not a request for the whole model.
+ # Login to allow private models. Before normalization: dtype is also derived below from a 4bit
+ # config's compute dtype, which is not a request for the whole model.
user_float32 = _requested_float32(dtype)
token = hf_login(token)
# Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
@@ -495,16 +488,15 @@ class FastLanguageModel(FastLlamaModel):
if isinstance(bnb_compute_dtype, torch.dtype):
dtype = bnb_compute_dtype
- # Distributed-safe device placement for quantized models.
- # In multi-GPU (torchrun), each rank must load the model on its own device
- # to avoid Accelerate device relocation errors with quantized weights.
+ # Distributed-safe placement for quantized models: under torchrun each rank must load on its
+ # own device, else Accelerate raises device relocation errors on quantized weights.
is_quantized = load_in_4bit or load_in_8bit or load_in_fp8
device_map = requested_device_map(device_map)
if is_quantized and isinstance(device_map, str):
distributed_device_map, is_dist = prepare_device_map()
if is_dist:
- # One whole model per rank; sharding one across the ranks' GPUs as well
- # would have every rank fighting for the same cards.
+ # One whole model per rank; sharding one across the ranks' GPUs too would have every rank
+ # fighting for the same cards.
device_map = distributed_device_map
# @_offline_aware_load already forced offline when needed; delegations inherit it.
@@ -526,12 +518,12 @@ class FastLanguageModel(FastLlamaModel):
use_gradient_checkpointing = use_gradient_checkpointing,
resize_model_vocab = resize_model_vocab, # [TODO] No effect
revision = revision,
- return_logits = False, # Return logits
+ return_logits = False,
fullgraph = True, # No graph breaks
use_exact_model_name = use_exact_model_name,
offload_embedding = offload_embedding,
float32_mixed_precision = float32_mixed_precision,
- # Pass vLLM/inference parameters
+ # Pass vLLM/inference parameters.
fast_inference = fast_inference,
gpu_memory_utilization = gpu_memory_utilization,
float8_kv_cache = float8_kv_cache,
@@ -564,7 +556,7 @@ class FastLanguageModel(FastLlamaModel):
)
if DEVICE_TYPE_TORCH == "cuda":
for i in range(DEVICE_COUNT):
- # [TODO] DGX Spark vLLM breaks
+ # DGX Spark vLLM breaks.
if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
print(
"Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"
@@ -573,14 +565,12 @@ class FastLanguageModel(FastLlamaModel):
fast_inference = False
break
- # bitsandbytes unusable (absent, or unstable as on some AMD stacks). This is
- # a capability check, so it is not gated on use_exact_model_name: that only
- # suppresses repo-name remapping and cannot make bitsandbytes available.
+ # bitsandbytes unusable (absent, or unstable on some AMD stacks). A capability check, so not
+ # gated on use_exact_model_name, which only suppresses repo-name remapping.
if not ALLOW_BITSANDBYTES:
- # A user-supplied config sets load_in_4bit/8bit above and is forwarded
- # in kwargs, so clearing the flags alone still rebuilds the bnb
- # quantizer downstream. Only drop it when it asks for bnb: a GPTQ /
- # AWQ / fp8 / torchao config must pass through untouched.
+ # A user config sets load_in_4bit/8bit above and is forwarded in kwargs, so clearing the flags
+ # alone still rebuilds the bnb quantizer downstream. Drop it only when it asks for bnb: a
+ # GPTQ / AWQ / fp8 / torchao config must pass through untouched.
_quant_cfg = kwargs.get("quantization_config", None)
if isinstance(_quant_cfg, dict):
_wants_bnb = bool(
@@ -603,8 +593,8 @@ class FastLanguageModel(FastLlamaModel):
"Unsloth: `bitsandbytes` is unavailable here - disabling 4bit/8bit. "
"16bit LoRA and full finetuning still work."
)
- # 8bit is bitsandbytes too: leaving either set sends the request on to
- # Transformers, which builds the bnb quantizer and fails there.
+ # 8bit is bitsandbytes too: leaving either set sends the request to Transformers, which builds
+ # the bnb quantizer and fails there.
load_in_4bit = False
load_in_8bit = False
if _wants_bnb:
@@ -637,28 +627,26 @@ class FastLanguageModel(FastLlamaModel):
else:
assert new_model_name is not None
model_name = new_model_name
- # If mapper resolved to a pre-quantized FP8 model, disable
- # on-the-fly quantization to avoid double quantization
+ # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid
+ # quantizing twice.
if load_in_fp8 != False and new_model_name != old_model_name:
load_in_fp8 = False
# Only this block honours use_exact_model_name; the transforms below do not.
mapper_moved_name = model_name != old_model_name
- # Check if pre-quantized models are allowed
- # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
+ # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2, and our pre-quants use blocksize = 64.
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
- # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set
- # Say so: dropping the flags in silence surfaces much later as an
- # out-of-memory failure whose message never mentions quantization.
+ # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set. Say
+ # so: dropping the flags silently resurfaces as an OOM whose message never mentions
+ # quantization.
if model_name.lower().endswith("-bf16") and (
load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))
):
- # A user quantization_config stays in **kwargs and still quantizes.
- # `load_in_4bit` defaults to True, so a set flag is not proof of a
- # request; an explicit `load_in_16bit` IS one, for exactly this load.
+ # A user quantization_config stays in **kwargs and still quantizes. load_in_4bit defaults to
+ # True, so a set flag is no proof of a request; an explicit load_in_16bit is.
if (
not load_in_16bit
and (load_in_4bit or load_in_8bit or load_in_fp8 != False)
@@ -684,13 +672,12 @@ class FastLanguageModel(FastLlamaModel):
base_revision = _revision_for_resolved_repo(
revision, model_name, old_model_name, mapper_moved_name
)
- # The PeftConfig probe reads the adapter repo, which peft loads in-process, so it
- # keeps the ref even when vLLM takes the base model's away just after.
+ # The PeftConfig probe reads the adapter repo, which peft loads in-process, so it keeps the ref
+ # even when vLLM takes the base model's away just after.
adapter_revision = base_revision
- # vLLM fetches the default branch, so the pin is dead for the weights. Drop it before
- # the probe: model_types picks the architecture class off that config, so a ref the
- # weights are not at dispatches the wrong one. llama.py's predicate spares the
- # in-process pre-Volta and num_labels fallbacks, which can still honour the pin.
+ # vLLM fetches the default branch, so the pin is dead for the weights. Drop it before the
+ # probe: model_types picks the architecture class off that config, so a ref the weights are not
+ # at dispatches the wrong one. llama.py's predicate spares the in-process fallbacks.
if base_revision is not None and _vllm_will_load_weights(
fast_inference, kwargs.get("num_labels")
):
@@ -700,7 +687,6 @@ class FastLanguageModel(FastLlamaModel):
)
base_revision = None
- # First check if it's a normal model via AutoConfig
from huggingface_hub.utils import (
disable_progress_bars,
enable_progress_bars,
@@ -769,7 +755,7 @@ class FastLanguageModel(FastLlamaModel):
# Old transformers versions check
both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32
- # Error out if both LoRA and normal model config exists.
+ # Error out if both LoRA and normal model config exist.
if both_exist:
raise RuntimeError(
"Unsloth: Your repo has a LoRA adapter and a base model.\n"
@@ -793,8 +779,7 @@ class FastLanguageModel(FastLlamaModel):
f"AutoConfig error: {autoconfig_error}\n\n"
f"PeftConfig error: {peft_error}\n\n"
)
- # Chain an offline-related cause if either probe had one, so @_offline_aware_load
- # still retries from cache (e.g. adapter repo: permanent AutoConfig 404 + transient PeftConfig).
+ # Chain an offline-related cause if either probe had one, so @_offline_aware_load still retries from cache.
_cause = next(
(
e
@@ -812,12 +797,11 @@ class FastLanguageModel(FastLlamaModel):
if len(model_types) == 1:
model_type = model_types[0]
else:
- # Leave as tuple if more than one arch
+ # Leave as tuple if more than one arch.
model_type = model_types
# New transformers need to check manually.
if SUPPORTS_LLAMA32 and is_model and is_peft:
- # Check if folder exists locally
if os.path.isdir(model_name):
exist_adapter_config = os.path.exists(
os.path.join(model_name, "adapter_config.json")
@@ -825,14 +809,11 @@ class FastLanguageModel(FastLlamaModel):
exist_config = os.path.exists(os.path.join(model_name, "config.json"))
both_exist = exist_adapter_config and exist_config
else:
- # Both AutoConfig and PeftConfig loaded successfully from this
- # remote repo, so both config.json and adapter_config.json
- # definitely exist -- no need for an extra HfFileSystem network call.
+ # Both AutoConfig and PeftConfig loaded from this remote repo, so config.json and
+ # adapter_config.json both exist and no extra HfFileSystem call is needed.
both_exist = True
- # Get base model for PEFT:
if is_peft:
- # Check base model again for PEFT
model_name = peft_config.base_model_name_or_path
if not use_exact_model_name:
model_name = get_model_name(
@@ -842,21 +823,19 @@ class FastLanguageModel(FastLlamaModel):
token = token,
trust_remote_code = trust_remote_code,
)
- # Check if pre-quantized models are allowed
- # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
+ # Are pre-quantized models allowed? AMD Instinct GPUs need blocksize = 128 on bitsandbytes
+ # < 0.49.2, and our pre-quants use 64.
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
- # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set
- # Say so: dropping the flags in silence surfaces much later as an
- # out-of-memory failure whose message never mentions quantization.
+ # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set. Say
+ # so: dropping the flags silently resurfaces as an OOM that never mentions quantization.
if model_name.lower().endswith("-bf16") and (
load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))
):
- # A user quantization_config stays in **kwargs and still quantizes.
- # `load_in_4bit` defaults to True, so a set flag is not proof of a
- # request; an explicit `load_in_16bit` IS one, for exactly this load.
+ # A user quantization_config stays in **kwargs and still quantizes. load_in_4bit defaults to
+ # True, so a set flag is no proof of a request; an explicit load_in_16bit is.
if (
not load_in_16bit
and (load_in_4bit or load_in_8bit or load_in_fp8 != False)
@@ -947,21 +926,7 @@ class FastLanguageModel(FastLlamaModel):
f"to obtain the latest transformers build, then restart this session."
)
dispatch_model = FastQwen3Model if model_type == "qwen3" else FastQwen3MoeModel
- # elif model_type == "falcon_h1":
- # dispatch_model = FastFalconH1Model
- # if not SUPPORTS_FALCON_H1:
- # raise ImportError(
- # f"Unsloth: Your transformers version of {transformers_version} does not support FalconH1.\n"\
- # f"The minimum required version is 4.50.3.\n"\
- # f'Try `pip install --upgrade "transformers>=4.50.3"`\n'\
- # f"to obtain the latest transformers build, then restart this session."\
- # )
- # Temporary disable optimized Cohere until errors match
- # elif model_type == "cohere":
- # dispatch_model = FastCohereModel
- # Temporary disable optimized Granite until errors match
- # elif model_type == "granite":
- # dispatch_model = FastGraniteModel
+ # Optimized Cohere and Granite paths are disabled until their errors match.
else:
delegated, tokenizer = FastModel.from_pretrained(
model_name = old_model_name,
@@ -980,12 +945,12 @@ class FastLanguageModel(FastLlamaModel):
use_gradient_checkpointing = use_gradient_checkpointing,
resize_model_vocab = resize_model_vocab, # [TODO] No effect
revision = revision,
- return_logits = False, # Return logits
+ return_logits = False,
fullgraph = True, # No graph breaks
use_exact_model_name = use_exact_model_name,
offload_embedding = offload_embedding,
float32_mixed_precision = float32_mixed_precision,
- # Pass vLLM/inference parameters
+ # Pass vLLM/inference parameters.
fast_inference = fast_inference,
gpu_memory_utilization = gpu_memory_utilization,
float8_kv_cache = float8_kv_cache,
@@ -1001,19 +966,18 @@ class FastLanguageModel(FastLlamaModel):
)
return _mark_requested_float32(delegated, user_float32), tokenizer
- # Apply gradient checkpointing with smart heuristics
use_gradient_checkpointing = apply_unsloth_gradient_checkpointing(
use_gradient_checkpointing, max_seq_length, dtype
)
- # Keep the local checkpoint dir as tokenizer when self-sufficient (see _resolve_checkpoint_tokenizer_name).
+ # Keep the local checkpoint dir as tokenizer when self-sufficient; see _resolve_checkpoint_tokenizer_name.
tokenizer_name = _resolve_checkpoint_tokenizer_name(old_model_name, kwargs)
if fast_inference:
fast_inference, model_name = fast_inference_setup(model_name, model_config)
- # model_name can move once more here. Skip for PEFT: model_name is then the base
- # model, and `revision` names the adapter that PeftModel.from_pretrained loads below.
+ # model_name can move once more here. Skip for PEFT: model_name is then the base model, and
+ # `revision` names the adapter PeftModel.from_pretrained loads below.
if not is_peft:
base_revision = _revision_for_resolved_repo(
base_revision, model_name, old_model_name, mapper_moved_name
@@ -1027,17 +991,16 @@ class FastLanguageModel(FastLlamaModel):
load_in_4bit_kwargs = False
load_in_8bit_kwargs = False
- # Mirror FastModel: bitsandbytes < 0.46.0 needs dynamo disabled.
- # Best effort: never crash the load (old unsloth_zoo without the
- # zoo #710 fix raises NameError here on Python 3.13).
+ # Mirror FastModel: bitsandbytes < 0.46.0 needs dynamo disabled. Best effort, never crash the
+ # load (an old zoo without the #710 fix raises NameError here on Python 3.13).
try:
patch_compiling_bitsandbytes()
except Exception as e:
print(f"Unsloth: Could not patch bitsandbytes for torch.compile - {e}")
- # The optimized path has never carried `offload_embedding`, so a request for one is
- # dropped rather than honoured; say so instead of leaving the caller to infer it
- # from memory use. `"auto"` stays quiet: it promises a decision, and off is one.
+ # The optimized path never carried offload_embedding, so a request for one is dropped rather
+ # than honoured; say so instead of leaving the caller to infer it from memory use. "auto"
+ # stays quiet: it promises a decision, and off is one.
if offload_embedding != OFFLOAD_EMBEDDING_AUTO and offload_embedding:
print(
"Unsloth: Not offloading embeddings; the optimized path for this "
@@ -1075,8 +1038,8 @@ class FastLanguageModel(FastLlamaModel):
if resize_model_vocab is not None:
model.resize_token_embeddings(resize_model_vocab)
- # `resize_token_embeddings` rebuilds the embedding, dropping `_hf_hook`;
- # this is the last module swap of all, after every repair above.
+ # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module
+ # swap of all, after every repair above.
try:
from unsloth.models.vision import _repair_dispatch_hooks
_repaired = _repair_dispatch_hooks(model)
@@ -1106,11 +1069,11 @@ class FastLanguageModel(FastLlamaModel):
)
if load_in_4bit:
- # Fix up bitsandbytes config, but respect user-provided quantization_config
+ # Fix up the bitsandbytes config, but respect a user-provided quantization_config.
if quantization_config is None:
- # `load_in_4bit` is the requested flag, not the effective one: a non-bnb
- # checkpoint (MXFP4/gptq/awq) had bnb disabled by check_and_disable, so stamping a
- # synthetic bnb config would corrupt its real one. Only stamp bnb/unquantized.
+ # load_in_4bit is the requested flag, not the effective one: a non-bnb checkpoint
+ # (MXFP4/gptq/awq) had bnb disabled by check_and_disable, so a synthetic bnb config would
+ # corrupt its real one.
try:
from unsloth_zoo.utils import get_quant_type
_stamp_bnb = get_quant_type(model.config) in (None, "bitsandbytes")
@@ -1119,14 +1082,14 @@ class FastLanguageModel(FastLlamaModel):
if _stamp_bnb:
compute_dtype = dtype_from_config(model.config)
quantization_config = {
- # Sometimes compute_dtype is not a string!!
+ # compute_dtype is sometimes not a string.
"bnb_4bit_compute_dtype": compute_dtype,
"bnb_4bit_quant_type": "nf4",
"bnb_4bit_use_double_quant": True,
"llm_int8_enable_fp32_cpu_offload": False,
"llm_int8_has_fp16_weight": False,
- # Whatever the load really used. None here would describe a layout
- # that never existed, and saving it makes the adapter unreloadable.
+ # Whatever the load really used. None here describes a layout that never existed, and saving it
+ # makes the adapter unreloadable.
"llm_int8_skip_modules": _loaded_skip_modules(model.config),
"llm_int8_threshold": 6.0,
"load_in_4bit": True,
@@ -1144,9 +1107,9 @@ class FastLanguageModel(FastLlamaModel):
_tag_model_with_fp8_torchao_config(model, fp8_mode)
if is_peft:
- # From https://github.com/huggingface/peft/issues/184
- # Now add PEFT adapters
- # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet.
+ # From huggingface/peft#184
+ # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet, and it always
+ # loads in-process, so warm it even under fast_inference.
_prefetched = maybe_prefetch_hf_snapshot(
old_model_name,
token = token,
@@ -1156,15 +1119,15 @@ class FastLanguageModel(FastLlamaModel):
# Adapter always loads in-process via PeftModel, so warm it even under fast_inference.
fast_inference = False,
force_download = kwargs.get("force_download", False),
- # Leave use_safetensors auto (inheriting base format could skip a safetensors-only
- # adapter). adapter_only restricts the warm to the adapter files + root aux.
+ # Leave use_safetensors auto, since inheriting the base format could skip a safetensors-only
+ # adapter; adapter_only restricts the warm to adapter files plus root aux.
adapter_only = True,
)
- # Child did the forced download; clear the flag so the load reuses the warm cache.
+ # The child did the forced download; clear the flag so the load reuses the warm cache.
if _prefetched and kwargs.get("force_download", False):
kwargs["force_download"] = False
- # Forward cache_dir so the load reads the warmed adapter. No subfolder (that targets the
- # base checkpoint; adapters live at the root).
+ # Forward cache_dir so the load reads the warmed adapter. No subfolder: that targets the base
+ # checkpoint, and adapters live at the root.
peft_load_kwargs = {}
if kwargs.get("cache_dir") is not None:
peft_load_kwargs["cache_dir"] = kwargs["cache_dir"]
@@ -1178,15 +1141,14 @@ class FastLanguageModel(FastLlamaModel):
trust_remote_code = trust_remote_code,
**peft_load_kwargs,
)
- # Patch it as well!
model = dispatch_model.patch_peft_model(model, use_gradient_checkpointing)
try:
from .vision import _lift_endpoint_hooks_onto_adapters
_lift_endpoint_hooks_onto_adapters(model)
except Exception:
pass # never block loading on a placement nicety
- # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back
- # to the original loop, an attention-only adapter keeps the grouped path. Guarded.
+ # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back to the
+ # original loop, an attention-only adapter keeps the grouped path.
try:
from unsloth_zoo.temporary_patches.moe_grouped_modulelist import (
auto_enable_grouped_moe,
@@ -1195,8 +1157,7 @@ class FastLanguageModel(FastLlamaModel):
except Exception:
pass # optional speedup; never block model loading
- # Patch Tiled MLP
- # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}""
+ # Patch Tiled MLP; enable by setting UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}".
patch_tiled_mlp_choice = os.environ.get(
"UNSLOTH_TILED_MLP", "arctic" if unsloth_tiled_mlp else "0"
)
@@ -1205,8 +1166,8 @@ class FastLanguageModel(FastLlamaModel):
model = _fix_rope_inv_freq(model)
model = _exclude_rope_inv_freq_from_ddp(model)
- # This path never sets UNSLOTH_FORCE_FLOAT32, so answer False rather than
- # leave the trainer reading whatever a later load writes to the environment.
+ # This path never sets UNSLOTH_FORCE_FLOAT32, so answer False rather than let the trainer read
+ # whatever a later load writes to the environment.
model = _mark_forced_float32(model, False)
# Full finetuning delegates to FastModel above, so this path is always LoRA.
model = _mark_full_finetuning(model, False)
@@ -1275,7 +1236,7 @@ class FastModel(FastBaseModel):
use_gradient_checkpointing = "unsloth",
resize_model_vocab = None, # [TODO] No effect
revision = None,
- return_logits = False, # Return logits
+ return_logits = False,
fullgraph = True, # No graph breaks
use_exact_model_name = False,
auto_model = None,
@@ -1295,12 +1256,12 @@ class FastModel(FastBaseModel):
load_in_fp8 = False, # fp8 LoRA (True, False, 'block')
unsloth_tiled_mlp = False,
target_parameters = None, # For MoE expert parameters
- text_only = False, # Skip vision/audio towers and load only the text decoder
+ text_only = False,
*args,
**kwargs,
):
user_config = kwargs.pop("config", None)
- # Respect user-provided quantization_config (e.g. BitsAndBytesConfig)
+ # Respect a user-provided quantization_config (e.g. BitsAndBytesConfig).
quantization_config = kwargs.get("quantization_config", None)
if quantization_config is not None:
if isinstance(quantization_config, dict):
@@ -1316,9 +1277,8 @@ class FastModel(FastBaseModel):
load_in_8bit = True
load_in_4bit = False
- # Login to allow private models
- # Before normalization: dtype is also derived below from a 4bit config's
- # compute dtype, which is not a request for the whole model.
+ # Login to allow private models. Before normalization: dtype is also derived below from a 4bit
+ # config's compute dtype, which is not a request for the whole model.
user_float32 = _requested_float32(dtype)
token = hf_login(token)
if whisper_language is not None:
@@ -1348,9 +1308,9 @@ class FastModel(FastBaseModel):
assert load_in_fp8 in (True, False, "block")
patch_compiled_autograd()
- # Same best-effort wrapper as the FastLanguageModel path: unsloth_zoo's
- # patch imports bitsandbytes unconditionally, so on a host without it this
- # raised before the capability fallback below could take the 16bit path.
+ # Same best-effort wrapper as the FastLanguageModel path: unsloth_zoo's patch imports
+ # bitsandbytes unconditionally, so on a host without it this raised before the capability
+ # fallback could take the 16bit path.
try:
patch_compiling_bitsandbytes()
except Exception as e:
@@ -1365,14 +1325,11 @@ class FastModel(FastBaseModel):
load_in_fp8 = False
load_in_16bit = False
- # bitsandbytes unusable (absent, or unstable as on some AMD stacks). This is
- # a capability check, so it is not gated on use_exact_model_name: that only
- # suppresses repo-name remapping and cannot make bitsandbytes available.
+ # bitsandbytes unusable (absent, or unstable on some AMD stacks). A capability check, so not
+ # gated on use_exact_model_name, which only suppresses repo-name remapping.
if not ALLOW_BITSANDBYTES:
- # A user-supplied config sets load_in_4bit/8bit above and is forwarded
- # in kwargs, so clearing the flags alone still rebuilds the bnb
- # quantizer downstream. Only drop it when it asks for bnb: a GPTQ /
- # AWQ / fp8 / torchao config must pass through untouched.
+ # A user config sets load_in_4bit/8bit above and is forwarded in kwargs, so clearing the flags
+ # alone still rebuilds the bnb quantizer downstream. Drop it only when it asks for bnb.
_quant_cfg = kwargs.get("quantization_config", None)
if isinstance(_quant_cfg, dict):
_wants_bnb = bool(
@@ -1395,8 +1352,8 @@ class FastModel(FastBaseModel):
"Unsloth: `bitsandbytes` is unavailable here - disabling 4bit/8bit. "
"16bit LoRA and full finetuning still work."
)
- # 8bit is bitsandbytes too: leaving either set sends the request on to
- # Transformers, which builds the bnb quantizer and fails there.
+ # 8bit is bitsandbytes too: leaving either set sends the request to Transformers, which builds
+ # the bnb quantizer and fails there.
load_in_4bit = False
load_in_8bit = False
if _wants_bnb:
@@ -1422,16 +1379,15 @@ class FastModel(FastBaseModel):
if qat_scheme == "phone-deployment":
qat_scheme = "int8-int4"
- # Distributed-safe device placement for quantized models.
- # In multi-GPU (torchrun), each rank must load the model on its own device
- # to avoid Accelerate device relocation errors with quantized weights.
+ # Distributed-safe placement for quantized models: under torchrun each rank must load on its
+ # own device, else Accelerate raises device relocation errors.
is_quantized = load_in_4bit or load_in_8bit or load_in_fp8
device_map = requested_device_map(device_map)
if is_quantized and isinstance(device_map, str):
distributed_device_map, is_dist = prepare_device_map()
if is_dist:
- # One whole model per rank; sharding one across the ranks' GPUs as well
- # would have every rank fighting for the same cards.
+ # One whole model per rank; sharding one across the ranks' GPUs too would have every rank
+ # fighting for the same cards.
device_map = distributed_device_map
if fast_inference:
@@ -1442,7 +1398,7 @@ class FastModel(FastBaseModel):
)
if DEVICE_TYPE_TORCH == "cuda":
for i in range(DEVICE_COUNT):
- # [TODO] DGX Spark vLLM breaks
+ # DGX Spark vLLM breaks.
if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
print(
"Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"
@@ -1474,28 +1430,26 @@ class FastModel(FastBaseModel):
else:
assert new_model_name is not None
model_name = new_model_name
- # If mapper resolved to a pre-quantized FP8 model, disable
- # on-the-fly quantization to avoid double quantization
+ # If the mapper resolved to a pre-quantized FP8 model, disable on-the-fly quantization to avoid
+ # quantizing twice.
if load_in_fp8 != False and new_model_name != old_model_name:
load_in_fp8 = False
# Only this block honours use_exact_model_name; the transforms below do not.
mapper_moved_name = model_name != old_model_name
- # Check if pre-quantized models are allowed
- # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
+ # Are pre-quantized models allowed? AMD Instinct GPUs need blocksize = 128 on bitsandbytes
+ # < 0.49.2, and our pre-quants use 64.
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
- # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set
- # Say so: dropping the flags in silence surfaces much later as an
- # out-of-memory failure whose message never mentions quantization.
+ # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set. Say
+ # so: dropping the flags silently resurfaces as an OOM that never mentions quantization.
if model_name.lower().endswith("-bf16") and (
load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))
):
- # A user quantization_config stays in **kwargs and still quantizes.
- # `load_in_4bit` defaults to True, so a set flag is not proof of a
- # request; an explicit `load_in_16bit` IS one, for exactly this load.
+ # A user quantization_config stays in **kwargs and still quantizes. load_in_4bit defaults to
+ # True, so a set flag is no proof of a request; an explicit load_in_16bit is.
if (
not load_in_16bit
and (load_in_4bit or load_in_8bit or load_in_fp8 != False)
@@ -1513,7 +1467,6 @@ class FastModel(FastBaseModel):
load_in_fp8 = False
load_in_16bit = True
- # Check modelscope
if USE_MODELSCOPE and not os.path.exists(model_name):
from modelscope import snapshot_download
model_name = snapshot_download(model_name)
@@ -1522,13 +1475,12 @@ class FastModel(FastBaseModel):
base_revision = _revision_for_resolved_repo(
revision, model_name, old_model_name, mapper_moved_name
)
- # The PeftConfig probe reads the adapter repo, which peft loads in-process, so it
- # keeps the ref even when vLLM takes the base model's away just after.
+ # The PeftConfig probe reads the adapter repo, which peft loads in-process, so it keeps the ref
+ # even when vLLM takes the base model's away just after.
adapter_revision = base_revision
- # vLLM fetches the default branch, so the pin is dead for the weights. Drop it here,
- # not at dispatch: model_types, auto_model and the text-only decision all come off the
- # config probed below, so a ref the weights are not at dispatches the wrong model.
- # Same predicate as FastBaseModel, whose own guard is then a no-op here.
+ # vLLM fetches the default branch, so the pin is dead for the weights. Drop it here, not at
+ # dispatch: model_types, auto_model and the text-only decision all come off the config probed
+ # below, so a ref the weights are not at dispatches the wrong model.
if base_revision is not None and fast_inference and is_vLLM_available():
logger.warning_once(
f"Unsloth: Ignoring revision = `{base_revision}` since vLLM loads weights "
@@ -1536,7 +1488,6 @@ class FastModel(FastBaseModel):
)
base_revision = None
- # First check if it's a normal model via AutoConfig
from huggingface_hub.utils import (
disable_progress_bars,
enable_progress_bars,
@@ -1555,8 +1506,8 @@ class FastModel(FastBaseModel):
# @_offline_aware_load already forced offline when needed; nested calls inherit it.
local_files_only = kwargs.get("local_files_only", False)
- # Text-diffusion slow-path dispatch, factored so both the normal route (below) and the
- # legacy-config fallback (in the AutoConfig except handler) share one call site.
+ # Text-diffusion slow-path dispatch, factored so the normal route and the legacy-config
+ # fallback share one call site.
def _dispatch_diffusion():
model, tokenizer = FastDiffusionModel.from_pretrained(
model_name = model_name,
@@ -1573,9 +1524,8 @@ class FastModel(FastBaseModel):
revision = base_revision,
**kwargs,
)
- # Returns before the FORCE_FLOAT32 scan and no diffusion type is on that
- # list, so False. Stamped, not left unset, or the trainer reads whatever
- # UNSLOTH_FORCE_FLOAT32 an earlier load wrote.
+ # Returns before the FORCE_FLOAT32 scan and no diffusion type is on that list, so False.
+ # Stamped, not left unset, or the trainer reads whatever an earlier load wrote.
model = _mark_forced_float32(model, False)
model = _mark_full_finetuning(model, full_finetuning)
return _mark_requested_float32(model, user_float32), tokenizer
@@ -1596,10 +1546,9 @@ class FastModel(FastBaseModel):
except Exception as error:
autoconfig_error = str(error)
autoconfig_exc = error
- # Legacy text-diffusion configs use model_type "diffusion_gemma", which current
- # transformers does not register by name (it ships "diffusion_gemma4"). AutoConfig
- # raises before we can dispatch; route straight to the diffusion slow path, whose
- # loader aliases the legacy type to the gemma4 classes.
+ # Legacy text-diffusion configs use model_type "diffusion_gemma", which current transformers
+ # does not register (it ships "diffusion_gemma4"), so AutoConfig raises before dispatch. Route
+ # straight to the diffusion slow path, whose loader aliases the legacy type.
if "diffusion_gemma" in autoconfig_error and is_diffusion_model_type("diffusion_gemma"):
return _dispatch_diffusion()
if "architecture" in autoconfig_error:
@@ -1637,7 +1586,7 @@ class FastModel(FastBaseModel):
is_peft = False
# Old transformers versions check
both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32
- # Error out if both LoRA and normal model config exists.
+ # Error out if both LoRA and normal model config exist.
if both_exist:
raise RuntimeError(
"Unsloth: Your repo has a LoRA adapter and a base model.\n"
@@ -1661,8 +1610,7 @@ class FastModel(FastBaseModel):
f"AutoConfig error: {autoconfig_error}\n\n"
f"PeftConfig error: {peft_error}\n\n"
)
- # Chain an offline-related cause if either probe had one, so @_offline_aware_load
- # still retries from cache (e.g. adapter repo: permanent AutoConfig 404 + transient PeftConfig).
+ # Chain an offline-related cause if either probe had one, so @_offline_aware_load still retries from cache.
_cause = next(
(
e
@@ -1680,27 +1628,20 @@ class FastModel(FastBaseModel):
model_types_all = ",".join(model_types) + ","
_maybe_advise_fla_install(model_types)
+ # Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom
+ # block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile
+ # patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.
# ---- Text-diffusion models (e.g. DiffusionGemma) take a transformers-only slow path. ----
- # These use a custom block-diffusion `generate` and a novel backbone, so we skip Unsloth's
- # autoregressive kernel/compile patching and load the unmodified HF model (bit-identical to
- # naive transformers), keeping only 4bit/8bit + PEFT LoRA conveniences.
if is_diffusion_model_type(model_types):
return _dispatch_diffusion()
- # Save model types and loading method
lowered_model_name = model_name.lower()
- # Build UNSLOTH_MODEL_NAME fresh from THIS load's model types + flags; do not prepend the
- # inherited os.environ value (a stale "_load_in_4bit_" from an earlier load, e.g. across a
- # save->reload subprocess, would push gpt-oss onto the BnB router patch when later loading
- # a 16bit checkpoint -> "weights not initialized"). Only the type tokens and the load flags
- # below are consumed downstream; the raw model name/path is excluded so a path containing a
- # flag sentinel cannot be misread.
- #
- # Encode the EFFECTIVE bnb state: a non-bnb checkpoint (MXFP4/gptq/awq) has load_in_4bit
- # disabled later by check_and_disable, so recording the requested flag here would route a
- # native MXFP4 gpt-oss onto the BnB router patch. This is only an EARLY best-effort (an
- # adapter-only PEFT repo has model_config=None here, and the base may be remapped); the
- # authoritative correction is sync_unsloth_model_name_bnb_flags(...) after check_and_disable.
+ # Build UNSLOTH_MODEL_NAME fresh from THIS load's types and flags: prepending the inherited
+ # value would let a stale "_load_in_4bit_" push gpt-oss onto the BnB router patch on a later
+ # 16bit load. The raw model name is excluded so a path containing a sentinel is not misread.
+ # Encode the EFFECTIVE bnb state: a non-bnb checkpoint has load_in_4bit disabled later by
+ # check_and_disable, so the requested flag would route a native MXFP4 gpt-oss onto the BnB
+ # router patch. Early best-effort; sync_unsloth_model_name_bnb_flags is authoritative.
try:
from unsloth_zoo.utils import get_quant_type
_bnb_compatible_quant = get_quant_type(model_config) in (None, "bitsandbytes")
@@ -1717,18 +1658,15 @@ class FastModel(FastBaseModel):
string += "load_in_fp8"
os.environ["UNSLOTH_MODEL_NAME"] = string
- # Check versions
LATEST = "\nPlease use transformers via `pip install --no-deps git+https://github.com/huggingface/transformers.git`"
NIGHTLY = (
'\nPlease use nightly transformers via pip install --upgrade "transformers>=4.49.0"`'
)
- # Pixtral
if "pixtral" in model_types_all and transformers_version < Version("4.49.0"):
raise RuntimeError("Unsloth: Pixtral only works on transformers >= 4.49.0." + LATEST)
- # Qwen 2.5
elif "qwen2_5" in model_types_all and transformers_version < Version("4.49.0"):
raise RuntimeError("Unsloth: Qwen 2.5 only works on transformers >= 4.49.0." + LATEST)
- # Gemma 4 must be before Gemma 3N and Gemma 3
+ # Gemma 4 must come before Gemma 3N, and Gemma 3N before Gemma 3.
elif "gemma4" in model_types_all:
if not SUPPORTS_GEMMA4:
raise RuntimeError("Unsloth: Gemma 4 requires transformers >= 5.5.0" + LATEST)
@@ -1748,54 +1686,46 @@ class FastModel(FastBaseModel):
";"
"from unsloth_zoo.temporary_patches.gemma3n import patch_Gemma3nConv_Embed_forwards; patch_Gemma3nConv_Embed_forwards()"
)
- # Set norms to float32 since anyways they get upcasted to float32
- # common in both gemma-3 and gemma-3n
+ # Set norms to float32 since they get upcasted anyway; common to gemma-3 and gemma-3n.
+ # Granite-4 rms norms are stored as 16 bit, but we upcast
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
- # Gemma 3
elif "gemma3" in model_types_all:
if transformers_version < Version("4.50.0.dev0"):
raise RuntimeError(
"Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY
)
- # Set norms to float32 since anyways they get upcasted to float32
- # common in both gemma-3 and gemma-3n
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
- # ROCm/HIP: Gemma3 compiled forward produces NaN on RDNA GPUs
- # (gfx1100, gfx1101, gfx1102, gfx1150, gfx1151, etc.).
- # Disable torch.compile for model forward; loss compilation is fine.
- # See https://github.com/unslothai/unsloth/issues/3385
+ # ROCm/HIP: the Gemma3 compiled forward gives NaN on RDNA GPUs (gfx1100-gfx1151), so disable
+ # torch.compile for the model forward; loss compilation is fine (#3385).
from unsloth.kernels.utils import is_rdna
if is_rdna():
os.environ["UNSLOTH_COMPILE_DISABLE"] = "partial"
- # Cohere
elif "cohere2" in model_types_all and transformers_version < Version("4.50.0.dev0"):
raise RuntimeError(
"Unsloth: Cohere's Command model only works on transformers >= 4.50.0." + NIGHTLY
)
- # Sesame
elif "csm" in model_types_all:
os.environ["UNSLOTH_COMPILE_DISABLE"] = "partial" # Inference is too slow
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Sesame fails
+ # Set down projection compute dtype to be float32 for float16 machines Set norms to float32 since
+ # anyways they get upcasted to float32
register_custom_dtype(
"all;torch.float32;torch.float16;"
"if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)"
";"
)
- # Granite 4
elif "granitemoehybrid" in model_types_all:
- # Granite-4 rms norms are stored as 16 bit, but we upcast
+ # Granite-4 rms norms are stored as 16 bit, but we upcast.
+ # Set norms to float32 since anyways they get upcasted to float32
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
- # OLMo 2
elif "olmo2" in model_types_all and transformers_version < Version("4.50.0.dev0"):
raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY)
- # OLMo 3
elif "olmo3" in model_types_all and transformers_version < Version("4.57.0.dev0"):
raise RuntimeError("Unsloth: OLMo-3 only works on transformers >= 4.57.0." + LATEST)
elif "falcon_h1" in model_types_all:
- # Falcon must use float32 Triton ie TRITON_F32_DEFAULT = 'ieee'
- # since Mamba kernels error out on using lower precision
+ # Falcon must use float32 Triton (TRITON_F32_DEFAULT = 'ieee') since Mamba kernels error on lower precision.
register_custom_dtype(
"float16;torch.float32;torch.float16;"
"if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16)"
@@ -1803,8 +1733,8 @@ class FastModel(FastBaseModel):
"os.environ['TRITON_F32_DEFAULT'] = 'ieee'"
)
elif "nemotron_h" in model_types_all:
- # NemotronH (hybrid Mamba-2 + Transformer) uses same Mamba kernels as Falcon-H1
- # Mamba kernels need float32 Triton precision
+ # NemotronH (hybrid Mamba-2 + Transformer) uses the same Mamba kernels as Falcon-H1, which need
+ # float32 Triton precision.
register_custom_dtype(
"float16;torch.float32;torch.float16;"
"if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16)"
@@ -1813,13 +1743,10 @@ class FastModel(FastBaseModel):
)
elif "gpt_oss" in model_types_all:
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
- # Use the EFFECTIVE bnb state, not the raw flag: a native MXFP4 checkpoint loaded
- # with the default load_in_4bit=True (e.g. openai/gpt-oss-20b by exact name) has
- # bnb disabled later by check_and_disable, so the raw flag would wrongly pick the
- # BnB dtype path. Mirrors the _load_in_4bit_ token gate above.
+ # Use the EFFECTIVE bnb state, not the raw flag: a native MXFP4 checkpoint loaded with the
+ # default load_in_4bit=True has bnb disabled later, so the raw flag picks the wrong dtype path.
if not (load_in_4bit and _bnb_compatible_quant):
- # Only upcast MoE biases for MXFP4, not BnB
- # Set norms to float32 since anyways they get upcasted to float32
+ # Only upcast MoE biases for MXFP4, not BnB.
register_custom_dtype(
"all;None;None;"
"x = 'gate_up_proj_bias'\n"
@@ -1833,8 +1760,6 @@ class FastModel(FastBaseModel):
";"
)
else:
- # Set down projection compute dtype to be float32 for float16 machines
- # Set norms to float32 since anyways they get upcasted to float32
register_custom_dtype(
"torch.float16;torch.bfloat16;torch.float16;"
"if ('down_projs' in name) and hasattr(module, 'weight') and "
@@ -1845,7 +1770,6 @@ class FastModel(FastBaseModel):
"module._pre_set_compute_dtype = torch.float32\n"
";"
)
- # Set norms to float32 since anyways they get upcasted to float32
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
else:
for check_model_name in DISABLE_COMPILE_MODEL_NAMES:
@@ -1860,12 +1784,11 @@ class FastModel(FastBaseModel):
break
if auto_model is not None:
- # All other models need to disable static cache
+ # All other models need to disable static cache.
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
# New transformers need to check manually.
if SUPPORTS_LLAMA32 and is_model and is_peft:
- # Check if folder exists locally
if os.path.isdir(model_name):
exist_adapter_config = os.path.exists(
os.path.join(model_name, "adapter_config.json")
@@ -1873,32 +1796,27 @@ class FastModel(FastBaseModel):
exist_config = os.path.exists(os.path.join(model_name, "config.json"))
both_exist = exist_adapter_config and exist_config
else:
- # Both AutoConfig and PeftConfig loaded successfully from this
- # remote repo, so both config.json and adapter_config.json
- # definitely exist -- no need for an extra HfFileSystem network call.
+ # Both AutoConfig and PeftConfig loaded from this remote repo, so config.json and
+ # adapter_config.json both exist and no extra HfFileSystem call is needed.
both_exist = True
- # Get base model for PEFT:
if is_peft:
- # Check base model again for PEFT
model_name = peft_config.base_model_name_or_path
if not use_exact_model_name:
model_name = get_model_name(model_name, load_in_4bit)
- # Check if pre-quantized models are allowed
- # AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
+ # Are pre-quantized models allowed? AMD Instinct GPUs need blocksize = 128 on bitsandbytes
+ # < 0.49.2, and our pre-quants use 64.
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
model_name = _strip_unsloth_bnb_4bit_suffix(model_name)
- # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set
- # Say so: dropping the flags in silence surfaces much later as an
- # out-of-memory failure whose message never mentions quantization.
+ # '-bf16' hub repos load bf16; a local dir keeps the requested quant unless 16bit is set. Say
+ # so: dropping the flags silently resurfaces as an OOM that never mentions quantization.
if model_name.lower().endswith("-bf16") and (
load_in_16bit or not os.path.isdir(os.path.expanduser(model_name))
):
- # A user quantization_config stays in **kwargs and still quantizes.
- # `load_in_4bit` defaults to True, so a set flag is not proof of a
- # request; an explicit `load_in_16bit` IS one, for exactly this load.
+ # A user quantization_config stays in **kwargs and still quantizes. load_in_4bit defaults to
+ # True, so a set flag is no proof of a request; an explicit load_in_16bit is.
if (
not load_in_16bit
and (load_in_4bit or load_in_8bit or load_in_fp8 != False)
@@ -1943,7 +1861,7 @@ class FastModel(FastBaseModel):
if model_type_arch != "siglip":
break
for disable_name in FORCE_FLOAT32:
- # add comma to model_types_all matching in case of exact match for end
+ # Add a comma to model_types_all so an exact match at the end still matches.
if (
disable_name.lower() == model_type_arch.lower().replace("-", "").replace("_", "")
or disable_name.lower() in model_types_all
@@ -1952,7 +1870,6 @@ class FastModel(FastBaseModel):
do_forced_float32 = True
dtype = torch.bfloat16 # Change to bfloat16 loading
break
- # Apply gradient checkpointing with smart heuristics
use_gradient_checkpointing = apply_unsloth_gradient_checkpointing(
use_gradient_checkpointing, max_seq_length, dtype
)
@@ -1995,9 +1912,8 @@ class FastModel(FastBaseModel):
if model_type in model_types_all:
supports_sdpa = False
- # Keep the local checkpoint dir as tokenizer when self-sufficient (see
- # _resolve_checkpoint_tokenizer_name). A VLM also needs local processor files, else
- # we fall back to the base repo so its cached processor loads.
+ # Keep the local checkpoint dir as tokenizer when self-sufficient; a VLM also needs local
+ # processor files, else fall back to the base repo so its cached processor loads.
_ckpt_arch = getattr(model_config, "architectures", None) or []
_ckpt_is_vlm = any(x.endswith("ForConditionalGeneration") for x in _ckpt_arch) or hasattr(
model_config, "vision_config"
@@ -2006,8 +1922,7 @@ class FastModel(FastBaseModel):
old_model_name, kwargs, require_processor = _ckpt_is_vlm
)
- # Capture task intent before text_only can replace a parent VLM config
- # with its nested text config.
+ # Capture task intent before text_only can replace a parent VLM config with its nested text config.
task_config_attrs = _get_user_task_config_attrs(user_config)
for _cfg_key in ("num_labels", "id2label", "label2id", "problem_type"):
_cfg_val = kwargs.get(_cfg_key, None)
@@ -2017,7 +1932,6 @@ class FastModel(FastBaseModel):
for _cfg_key, _cfg_val in task_config_attrs.items():
set_task_config_attr(model_config, _cfg_key, _cfg_val)
- # Check if VLM
architectures = getattr(model_config, "architectures", None)
if architectures is None:
architectures = []
@@ -2028,8 +1942,8 @@ class FastModel(FastBaseModel):
if load_text_only:
if hasattr(model_config, "vision_config"):
text_config = _get_text_only_config(model_config, old_model_name)
- # Skip the vision tower only for families with their own text decoder (Gemma 3);
- # others would load random weights, so keep the full model (use FastVisionModel).
+ # Skip the vision tower only for families with their own text decoder (Gemma 3); others would
+ # load random weights, so keep the full model.
text_class = resolve_model_class(AutoModelForCausalLM, text_config)
if text_class is None or not _is_family_text_decoder(
getattr(model_config, "model_type", ""),
@@ -2041,16 +1955,16 @@ class FastModel(FastBaseModel):
f"Loading {old_model_name} as text-only; vision/audio towers skipped. "
"Use FastVisionModel for multimodal inputs."
)
- # Remap VLM text weights (tf >=5) while model_config is still the parent. #5816
+ # Remap VLM text weights (tf >= 5) while model_config is still the parent (#5816).
_apply_text_only_key_mapping(kwargs, model_config, text_config)
model_config = text_config
is_vlm = False
- # model_config is no longer the repo's config, so anything rebuilding
- # it from model_name (the device-map planner) sees a different model.
+ # model_config is no longer the repo's config, so anything rebuilding it from model_name (the
+ # device-map planner) sees a different model.
text_only_decoder = True
else:
is_vlm = False
- # If num_labels is set, use AutoModelForSequenceClassification
+ # If num_labels is set, use AutoModelForSequenceClassification.
for _cfg_key, _cfg_val in task_config_attrs.items():
set_task_config_attr(model_config, _cfg_key, _cfg_val)
if auto_model is None:
@@ -2058,16 +1972,10 @@ class FastModel(FastBaseModel):
from transformers import AutoModelForSequenceClassification
auto_model = AutoModelForSequenceClassification
elif is_vlm:
- # Check if the model's auto_map supports the VLM auto class.
- # Some repo-code VL models register only a generic auto class and not
- # AutoModelForImageTextToText/AutoModelForVision2Seq: Nemotron-VL uses
- # AutoModelForCausalLM, DeepSeek-OCR uses AutoModel. Calling the VLM auto
- # class on those raises "Unrecognized configuration class ... for
- # AutoModelForImageTextToText", so fall back to whatever generic class the
- # repo actually registered. Match the CONCRETE class name we would pass
- # (AutoModelForVision2Seq aliases to AutoModelForImageTextToText on tf>=5),
- # since transformers resolves remote code by that exact name -- a config
- # that only registers the legacy key must still take the generic fallback.
+ # Some repo-code VL models register only a generic auto class (Nemotron-VL uses
+ # AutoModelForCausalLM, DeepSeek-OCR AutoModel), so the VLM auto class raises "Unrecognized
+ # configuration class". Fall back to what the repo registered, matching the CONCRETE class
+ # name, since transformers resolves remote code by that exact name.
_auto_map = getattr(model_config, "auto_map", {}) or {}
_vlm_class_name = AutoModelForVision2Seq.__name__
_has_vlm_class = _vlm_class_name in _auto_map
@@ -2087,8 +1995,8 @@ class FastModel(FastBaseModel):
load_in_4bit_kwargs = False
load_in_8bit_kwargs = False
- # FastBaseModel remaps again via fast_inference_setup. Skip for PEFT: model_name is
- # then the base model, and `revision` names the adapter PeftModel loads below.
+ # FastBaseModel remaps again via fast_inference_setup. Skip for PEFT: model_name is then the
+ # base model, and `revision` names the adapter PeftModel loads below.
if not is_peft:
base_revision = _revision_for_resolved_repo(
base_revision, model_name, old_model_name, mapper_moved_name
@@ -2121,17 +2029,16 @@ class FastModel(FastBaseModel):
whisper_task = whisper_task,
auto_config = model_config,
auto_config_from_caller = user_config is not None,
- # `resize_token_embeddings` below replaces the embedding module and hooks do
- # not travel to the replacement, so an offload installed during the load would
- # leave a CPU embedding feeding a GPU decoder. An explicit request is left
- # alone, since that combination was already this caller's to get wrong.
+ # resize_token_embeddings below replaces the embedding module and hooks do not follow, so an
+ # offload installed during the load would leave a CPU embedding feeding a GPU decoder. An
+ # explicit request is left alone.
offload_embedding = (
False
if resize_model_vocab is not None and offload_embedding == OFFLOAD_EMBEDDING_AUTO
else offload_embedding
),
float32_mixed_precision = float32_mixed_precision,
- # Pass vLLM/inference parameters
+ # Pass vLLM/inference parameters.
fast_inference = fast_inference,
gpu_memory_utilization = gpu_memory_utilization,
float8_kv_cache = float8_kv_cache,
@@ -2147,8 +2054,8 @@ class FastModel(FastBaseModel):
if resize_model_vocab is not None:
model.resize_token_embeddings(resize_model_vocab)
- # `resize_token_embeddings` rebuilds the embedding, dropping `_hf_hook`;
- # this is the last module swap of all, after every repair above.
+ # resize_token_embeddings rebuilds the embedding and drops _hf_hook, so this is the last module
+ # swap of all, after every repair above.
try:
from unsloth.models.vision import _repair_dispatch_hooks
_repaired = _repair_dispatch_hooks(model)
@@ -2178,11 +2085,10 @@ class FastModel(FastBaseModel):
)
if load_in_4bit:
- # Fix up bitsandbytes config, but respect user-provided quantization_config
+ # Fix up the bitsandbytes config, but respect a user-provided quantization_config.
if quantization_config is None:
- # `load_in_4bit` is the requested flag, not the effective one: a non-bnb
- # checkpoint (MXFP4/gptq/awq) had bnb disabled by check_and_disable, so stamping a
- # synthetic bnb config would corrupt its real one. Only stamp bnb/unquantized.
+ # load_in_4bit is the requested flag, not the effective one: a non-bnb checkpoint had bnb
+ # disabled by check_and_disable, so a synthetic bnb config would corrupt its real one.
try:
from unsloth_zoo.utils import get_quant_type
_stamp_bnb = get_quant_type(model.config) in (None, "bitsandbytes")
@@ -2191,14 +2097,14 @@ class FastModel(FastBaseModel):
if _stamp_bnb:
compute_dtype = dtype_from_config(model.config)
quantization_config = {
- # Sometimes compute_dtype is not a string!!
+ # compute_dtype is sometimes not a string.
"bnb_4bit_compute_dtype": compute_dtype,
"bnb_4bit_quant_type": "nf4",
"bnb_4bit_use_double_quant": True,
"llm_int8_enable_fp32_cpu_offload": False,
"llm_int8_has_fp16_weight": False,
- # Whatever the load really used. None here would describe a layout
- # that never existed, and saving it makes the adapter unreloadable.
+ # Whatever the load really used. None here describes a layout that never existed, and saving it
+ # makes the adapter unreloadable.
"llm_int8_skip_modules": _loaded_skip_modules(model.config),
"llm_int8_threshold": 6.0,
"load_in_4bit": True,
@@ -2216,13 +2122,10 @@ class FastModel(FastBaseModel):
_tag_model_with_fp8_torchao_config(model, fp8_mode)
if is_peft:
- # From https://github.com/huggingface/peft/issues/184
- # Now add PEFT adapters
+ # From huggingface/peft#184
- # Gemma4 ClippableLinear wraps nn.Linear -- PEFT can't inject LoRA
- # on it directly. Monkey-patch PEFT to target the inner .linear
- # child instead (same patch as vision.py training path).
- # See https://github.com/huggingface/peft/issues/3129
+ # Gemma4 ClippableLinear wraps nn.Linear and PEFT cannot inject LoRA on it directly, so patch
+ # PEFT to target the inner .linear child (same patch as vision.py). See huggingface/peft#3129.
_clippable_linear_cls = None
try:
from transformers.models.gemma4.modeling_gemma4 import (
@@ -2270,7 +2173,8 @@ class FastModel(FastBaseModel):
_LoraModel._create_and_replace = _patched_car
- # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet.
+ # Warm the adapter repo: PeftModel downloads it in-process and can hang on Xet, and it always
+ # loads in-process, so warm it even under fast_inference.
_prefetched = maybe_prefetch_hf_snapshot(
old_model_name,
token = token,
@@ -2280,15 +2184,15 @@ class FastModel(FastBaseModel):
# Adapter always loads in-process via PeftModel, so warm it even under fast_inference.
fast_inference = False,
force_download = kwargs.get("force_download", False),
- # Leave use_safetensors auto (inheriting base format could skip a safetensors-only
- # adapter). adapter_only restricts the warm to the adapter files + root aux.
+ # Leave use_safetensors auto, since inheriting the base format could skip a safetensors-only
+ # adapter; adapter_only restricts the warm to adapter files plus root aux.
adapter_only = True,
)
- # Child did the forced download; clear the flag so the load reuses the warm cache.
+ # The child did the forced download; clear the flag so the load reuses the warm cache.
if _prefetched and kwargs.get("force_download", False):
kwargs["force_download"] = False
- # Forward cache_dir so the load reads the warmed adapter. No subfolder (that targets the
- # base checkpoint; adapters live at the root).
+ # Forward cache_dir so the load reads the warmed adapter. No subfolder: that targets the base
+ # checkpoint, and adapters live at the root.
peft_load_kwargs = {}
if kwargs.get("cache_dir") is not None:
peft_load_kwargs["cache_dir"] = kwargs["cache_dir"]
@@ -2304,11 +2208,10 @@ class FastModel(FastBaseModel):
**peft_load_kwargs,
)
finally:
- # Always restore original PEFT method, even if loading fails
+ # Always restore the original PEFT method, even if loading fails.
if _clippable_linear_cls is not None:
_LoraModel._create_and_replace = _original_car
- # Patch it as well!
model = FastBaseModel.post_patch_model(
model, use_gradient_checkpointing, trust_remote_code = trust_remote_code
)
@@ -2317,8 +2220,8 @@ class FastModel(FastBaseModel):
_lift_endpoint_hooks_onto_adapters(model)
except Exception:
pass # never block loading on a placement nicety
- # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back
- # to the original loop, an attention-only adapter keeps the grouped path. Guarded.
+ # Re-evaluate grouped MoE now the adapter is attached: an expert-LoRA block falls back to the
+ # original loop, an attention-only adapter keeps the grouped path.
try:
from unsloth_zoo.temporary_patches.moe_grouped_modulelist import (
auto_enable_grouped_moe,
@@ -2327,13 +2230,11 @@ class FastModel(FastBaseModel):
except Exception:
pass # optional speedup; never block model loading
- # Apply QAT if specified
if qat_scheme is not None:
print("Unsloth: Applying QAT to mitigate quantization degradation")
model = FastModel._prepare_for_qat(model, qat_scheme)
- # Patch Tiled MLP
- # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}""
+ # Patch Tiled MLP; enable by setting UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}".
patch_tiled_mlp_choice = os.environ.get(
"UNSLOTH_TILED_MLP", "arctic" if unsloth_tiled_mlp else "0"
)
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 31c60ce6d2e..ac169165e71 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -34,16 +34,15 @@ from .mapper import (
_add_lower_only,
)
-# The alias helpers a fetched mapper.py may call, resolved to the INSTALLED
-# implementations. `_get_new_mapper` reads such calls as data: it takes the table and
-# the two literal strings out of the AST and applies them with these, so the fetched
-# text never supplies behaviour.
+# The alias helpers a fetched mapper.py may call, resolved to the INSTALLED implementations:
+# _get_new_mapper reads such calls as data, taking the table and the two literal strings out of
+# the AST, so the fetched text never supplies behaviour.
_MAPPER_HELPERS = {
"_add_with_lower": _add_with_lower,
"_add_lower_only": _add_lower_only,
}
-# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!
+# huggingface/transformers#26037 allows 4 bit loading.
from transformers import __version__ as transformers_version
from unsloth.models._utils import TorchAOConfig
from unsloth_zoo.utils import Version, get_quant_type
@@ -66,7 +65,7 @@ BAD_MAPPINGS = {
def _get_torchao_fp8_config(fp8_mode):
- # Lazy import so a broken optional vLLM install doesn't break `import unsloth`.
+ # Lazy import so a broken optional vLLM install does not break `import unsloth`.
from unsloth_zoo.vllm_utils import _get_torchao_fp8_config as _impl
return _impl(fp8_mode)
@@ -120,10 +119,8 @@ def prepare_device_map():
UNSLOTH_DEVICE_MAP = "unsloth"
-# Same planner, different answer when it declines. Every veto path ends in "sequential",
-# which fills the first device to its whole free budget -- right for a loader default,
-# wrong for a caller that picked several cards on their combined capacity. Naming the
-# fallback spares such a caller enumerating veto reasons that are unsloth's and can grow.
+# Same planner, different answer when it declines: every veto path ends in "sequential", which
+# fills the first device to its whole free budget, wrong for a caller that picked several cards.
UNSLOTH_BALANCED_DEVICE_MAP = "unsloth_balanced"
_PLANNED_DEVICE_MAPS = {UNSLOTH_DEVICE_MAP: "sequential", UNSLOTH_BALANCED_DEVICE_MAP: "balanced"}
@@ -159,9 +156,8 @@ def planner_hub_kwargs(loader_kwargs):
hub["cache_dir"] = loader_kwargs["cache_dir"]
if _get_effective_local_files_only(loader_kwargs):
hub["local_files_only"] = True
- # Resolving a remote class is a third lookup, and the planner honours `code_revision`
- # for it (`_HUB_KWARGS` in unsloth_zoo's `device_map_planner`). Left out, the plan is
- # built from the default revision's code and can name a tree the model does not have.
+ # Resolving a remote class is a third lookup, and the planner honours code_revision for it; left
+ # out, the plan is built from the default revision's code and can name a tree the model lacks.
if loader_kwargs.get("code_revision") is not None:
hub["code_revision"] = loader_kwargs["code_revision"]
return hub
@@ -243,9 +239,8 @@ def planner_quantization_kwargs(
try:
from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES
except Exception:
- # Built on every quantized load, planning or not, so an older unsloth_zoo must
- # not turn a 4bit load into an ImportError. One without the shared list predates
- # the planner that consumes it, so this plan was going to decline anyway.
+ # Built on every quantized load, planning or not, so an older unsloth_zoo must not turn a 4bit load
+ # into an ImportError. One without the shared list predates the planner that consumes it.
return kwargs
kwargs["llm_int8_skip_modules"] = SKIP_QUANTIZATION_MODULES + list(extra_skip_modules or [])
return kwargs
@@ -281,8 +276,8 @@ def planner_class_mismatch_reason(loaded_class, planned_class):
return f"the load builds {loaded_class.__name__}, not the planned {planned_class.__name__}"
-# accelerate's `max_memory` spellings, in the order it tries them: GiB/MiB/KiB are binary,
-# GB/MB/KB are decimal, and a lowercase trailing `b` on a decimal unit means bits, not bytes.
+# accelerate's max_memory spellings, in the order it tries them: GiB/MiB/KiB are binary, GB/MB/KB
+# decimal, and a lowercase trailing `b` on a decimal unit means bits.
_SIZE_UNITS = (
("GIB", 2**30, False),
("MIB", 2**20, False),
@@ -320,7 +315,7 @@ def _as_bytes(size):
amount = int(float(size[: -len(unit)]) * scale)
except ValueError:
return None
- # Bits, if they spelled the unit "Gb" rather than "GB". Binary units have no such form.
+ # Bits, if they spelled the unit "Gb" rather than "GB"; binary units have no such form.
if has_bit_form and size.endswith("b"):
amount //= 8
return amount if amount >= 0 else None
@@ -350,7 +345,7 @@ def resolve_unsloth_device_map(
`skip_reason` is the caller's veto, for when only the caller can tell the planner
would describe a different model than the load builds.
"""
- # `isinstance` first: a caller's explicit dict is unhashable, so `in` alone raises.
+ # isinstance first: a caller's explicit dict is unhashable, so `in` alone raises.
if not isinstance(device_map, str) or device_map not in _PLANNED_DEVICE_MAPS:
return device_map
_declined = _PLANNED_DEVICE_MAPS[device_map]
@@ -366,8 +361,8 @@ def resolve_unsloth_device_map(
if full_finetuning:
return _fallback("full finetuning does not use the quantized planner")
if is_distributed():
- # Every rank already owns the whole model on its own card; splitting on top of
- # that puts every rank on every card, a different execution model, not a bigger one.
+ # Every rank already owns the whole model on its own card; splitting on top of that puts every rank
+ # on every card, a different execution model.
return _fallback("each rank of a distributed launch owns its own device")
if DEVICE_TYPE_TORCH != "cuda":
return _fallback(f"the planner has no memory budgets for {DEVICE_TYPE_TORCH}")
@@ -376,23 +371,19 @@ def resolve_unsloth_device_map(
except Exception as error:
return _fallback(f"the devices could not be counted ({error})")
- # Popped, not forwarded: `max_memory` is a named parameter of the planner, so a copy
- # left in `planner_kwargs` raises `TypeError: got multiple values for keyword argument`,
- # which the handler below turns into a silent "sequential", losing the cap and the plan.
- #
- # A caller's mapping replaces the measured one rather than editing it: its keys are the
- # devices they will let the load use. That is accelerate's reading too --
- # `_init_infer_auto_device_map` takes `devices = list(max_memory.keys())` and
- # `get_max_memory` never widens a supplied mapping -- so `{0: ..., 1: ...}` on a
- # four-GPU host means GPUs 2 and 3 are somebody else's.
+ # Popped, not forwarded: max_memory is a named parameter of the planner, so a copy left in
+ # planner_kwargs raises TypeError, which the handler below turns into a silent "sequential".
+ # A caller's mapping replaces the measured one rather than editing it: its keys are the devices
+ # they will let the load use, as accelerate reads it too, so {0: ..., 1: ...} on a four-GPU host
+ # means GPUs 2 and 3 are somebody else's.
planner_kwargs = dict(planner_kwargs or {})
requested_memory = planner_kwargs.pop("max_memory", None)
requested_memory = dict(requested_memory) if requested_memory else None
- # Read before probing: `mem_get_info` initialises a CUDA context on each device it
- # touches, and a withheld card is likely busy with the workload it was withheld for.
- # One that refuses (ECC error, MIG parent, Exclusive_Process) would also drop the plan
- # to "sequential" over a device this load was never going to use.
+ # Read before probing: mem_get_info initialises a CUDA context on each device it touches, and a
+ # withheld card is likely busy with the workload it was withheld for. One that refuses (ECC error,
+ # MIG parent, Exclusive_Process) would also drop the plan to "sequential" over a device this load
+ # was never going to use.
if requested_memory is None:
probe = list(range(device_count))
else:
@@ -411,8 +402,8 @@ def resolve_unsloth_device_map(
except Exception as error:
return _fallback(f"the planner is unavailable ({error})")
- # Free, not total: this process's context and anything else resident on the card make
- # total an overcommit. Guarded because a card can still refuse mid-probe.
+ # Free, not total: this process's context and anything else resident make total an overcommit.
+ # Guarded, because a card can still refuse mid-probe.
try:
max_memory = {index: torch.cuda.mem_get_info(index)[0] for index in probe}
except Exception as error:
@@ -424,12 +415,12 @@ def resolve_unsloth_device_map(
measured = max_memory.get(device)
budget = _as_bytes(written)
if budget is None:
- # Nothing to compare against: what we measured, or for a device we never
- # measured (cpu, disk) their value untouched for the planner to read.
+ # Nothing to compare against: what we measured, or for a device we never measured (cpu, disk) their
+ # value untouched for the planner to read.
budgets[device] = measured if measured is not None else written
else:
- # Under what is actually free: they may know of reservations we cannot
- # measure, but planning above free is how a plan OOMs on dispatch.
+ # Under what is actually free: they may know of reservations we cannot measure, but planning above
+ # free is how a plan OOMs on dispatch.
budgets[device] = budget if measured is None else min(measured, budget)
max_memory = budgets
@@ -465,7 +456,7 @@ def __get_model_name(
if load_in_fp8 != False:
if load_in_fp8 == True and (os.environ.get("UNSLOTH_HAS_FBGEMM", "0") == "1"):
if lower_model_name in FLOAT_TO_FP8_ROW_MAPPER:
- # Faster row scaling only works if FBGEMM works!
+ # Faster row scaling only works if FBGEMM works; otherwise use the slower blockwise type.
return FLOAT_TO_FP8_ROW_MAPPER[lower_model_name]
elif lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER:
# Otherwise we use the slower blockwise type
@@ -473,8 +464,8 @@ def __get_model_name(
else:
if lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER:
return FLOAT_TO_FP8_BLOCK_MAPPER[lower_model_name]
- # No pre-quantized model found. vllm >= 0.12.0 quantizes to FP8 on the
- # fly (return original name); older vllm falls through to offline quant.
+ # No pre-quantized model found. vllm >= 0.12.0 quantizes to FP8 on the fly (returning the original
+ # name); older vllm falls through to offline quant.
if importlib.util.find_spec("vllm") is not None:
import vllm
if Version(vllm.__version__) >= Version("0.12.0"):
@@ -494,10 +485,6 @@ def __get_model_name(
elif not load_in_4bit and lower_model_name in INT_TO_FLOAT_MAPPER:
new_model_name = INT_TO_FLOAT_MAPPER[lower_model_name]
- # logger.warning_once(
- # f"Unsloth: You passed in `{model_name}` which is a 4bit model, yet you set\n"\
- # f"`load_in_4bit = False`. We shall load `{new_model_name}` instead."
- # )
return new_model_name
elif not load_in_4bit and lower_model_name in MAP_TO_UNSLOTH_16bit:
@@ -510,10 +497,6 @@ def __get_model_name(
return model_name
new_model_name = FLOAT_TO_INT_MAPPER[lower_model_name]
- # logger.warning_once(
- # f"Unsloth: You passed in `{model_name}` and `load_in_4bit = True`.\n"\
- # f"We shall load `{new_model_name}` for 4x faster loading."
- # )
return new_model_name
return None
@@ -527,12 +510,12 @@ def _get_new_mapper():
new_mapper = (
"https://raw.githubusercontent.com/unslothai/unsloth/main/unsloth/models/mapper.py"
)
- # Capped WHILE reading, since `requests.get` buffers the whole body first, and
- # the deadline is total because `timeout` is per-read.
+ # Capped WHILE reading, since requests.get buffers the whole body first, and the deadline is total
+ # because timeout is per-read.
byte_cap = 1_000_000
deadline = time.monotonic() + 10
chunks, total = [], 0
- # Redirects by hand: `requests` drains each intermediate body inside `get`.
+ # Redirects by hand: requests drains each intermediate body inside get.
url = new_mapper
for _ in range(5):
response = requests.get(url, timeout = 3, stream = True, allow_redirects = False)
@@ -546,10 +529,9 @@ def _get_new_mapper():
if time.monotonic() > deadline:
return {}, {}, {}, {}, {}
else:
- # `read1`: the timeout is per SOCKET READ, so a trickling peer
- # resets it forever and the deadline is never reached.
+ # read1: the timeout is per SOCKET READ, so a trickling peer resets it forever and the deadline is
+ # never reached. requests only decompresses inside iter_content.
raw = response.raw
- # `requests` only decompresses inside `iter_content`.
try:
raw.decode_content = True
except AttributeError:
@@ -561,7 +543,7 @@ def _get_new_mapper():
if read_once is not None:
chunk = read_once(65_536)
else:
- # Older urllib3 has no `read1`; one byte returns as promptly.
+ # Older urllib3 has no read1; one byte returns as promptly.
chunk = raw.read(1)
if not chunk:
break
@@ -576,18 +558,17 @@ def _get_new_mapper():
else:
return {}, {}, {}, {}, {}
new_mapper = b"".join(chunks).decode(encoding, errors = "replace")
- # Never exec the response: that is arbitrary code execution inside every
- # `from_pretrained` that hits an unmapped name. Only `__INT_TO_FLOAT_MAPPER` is
- # data, and the tables are returned rather than written into this module.
+ # Never exec the response: that is arbitrary code execution inside every from_pretrained that hits
+ # an unmapped name.
import ast
- # `ast.parse` builds the whole tree before any literal-only check runs, so this
- # is no defence against exhaustion (python/cpython#95588); the byte cap above is.
+ # ast.parse builds the whole tree before any literal-only check runs, so this is no defence against
+ # exhaustion (python/cpython#95588); the byte cap above is.
tree = ast.parse(new_mapper)
# Every module-level literal dict, in source order; chosen below.
literal_bindings = []
for index, node in enumerate(tree.body):
- # `AnnAssign` too, or an annotation upstream turns the probe off silently.
+ # AnnAssign too, or an annotation upstream turns the probe off silently.
if isinstance(node, ast.AnnAssign):
targets = [node.target.id] if isinstance(node.target, ast.Name) else []
value = node.value
@@ -619,9 +600,8 @@ def _get_new_mapper():
found = literal
return found
- # Statements that really RUN at import: `ast.walk` reaches into function bodies
- # and dead branches, which would fabricate a mapping. Local, because the tests
- # run this body in a bare namespace.
+ # Statements that really RUN at import: ast.walk reaches into function bodies and dead branches,
+ # which would fabricate a mapping. Local, because the tests run this body in a bare namespace.
def _constant_truth(test):
"""True/False for a statically decidable condition, else None.
@@ -658,7 +638,8 @@ def _get_new_mapper():
shadowed = frozenset(),
class_body = False,
):
- # What ENDED the suite: `"return"` leaves the function, `True` the suite.
+ # What ENDED the suite: "return" leaves the function, True the suite, and only the statements ABOVE
+ # it still see the module global.
for statement in body:
if class_body:
# Only the statements ABOVE it still see the module global.
@@ -699,7 +680,7 @@ def _get_new_mapper():
yield from _executed_nodes(children, shadowed)
for handler in getattr(statement, "handlers", []) or []:
yield from _executed_nodes(handler.body, shadowed)
- # Only the non-suite parts; a `Lambda` is excluded at the seed too.
+ # Only the non-suite parts; a Lambda is excluded at the seed too.
pending = [
child
for child in ast.iter_child_nodes(statement)
@@ -734,8 +715,7 @@ def _get_new_mapper():
continue
pending.append(child)
- # The aliases a newer mapper.py adds live inside `build_mappers`, which the
- # installed builder cannot know.
+ # The aliases a newer mapper.py adds live inside build_mappers, which the installed builder cannot know.
def _calls_the_builder(node):
"""Whether this executed node IS the `build_mappers(...)` call.
@@ -750,8 +730,8 @@ def _get_new_mapper():
)
builder_body = []
- # From the EXECUTED nodes, with the statement each runs in: a source table
- # rebound after the call is not what the module exports.
+ # From the EXECUTED nodes, with the statement each runs in: a source table rebound after the call is
+ # not what the module exports.
exported_names = (
"INT_TO_FLOAT_MAPPER",
"FLOAT_TO_INT_MAPPER",
@@ -900,8 +880,8 @@ def _get_new_mapper():
if source_table is None:
source_table = _binding_at(source_name, builder_index)
- # Not the end of the probe: a row-only FP8 repo cannot be expressed through the
- # source table at all. A body that adds nothing is reported so, at the end.
+ # Not the end of the probe: a row-only FP8 repo cannot be expressed through the source table at all,
+ # so a body that adds nothing is reported so at the end.
empty_base = not source_table
tables = build_mappers(source_table or {})
# Restored at the call below, which REPLACES all five tables.
@@ -916,8 +896,8 @@ def _get_new_mapper():
"FLOAT_TO_FP8_ROW_MAPPER": tables[4],
}
- # Only the helper CALLS: the builder binds its own `INT_TO_FLOAT_MAPPER = {}`,
- # which the whole-name rule below would read as a clear.
+ # Only the helper CALLS: the builder binds its own INT_TO_FLOAT_MAPPER = {}, which the whole-name
+ # rule below would read as a clear.
builder_additions = [
(node, shadowed)
for node, shadowed in _executed_nodes(builder_body)
@@ -925,8 +905,8 @@ def _get_new_mapper():
and isinstance(node.func, ast.Name)
and node.func.id in _MAPPER_HELPERS
]
- # In EXECUTION order, so a rebind after the call still empties the table.
- # `rebuilt` stands where the build assigns the exports; matched by identity.
+ # In EXECUTION order, so a rebind after the call still empties the table; `rebuilt` stands where the
+ # build assigns the exports, matched by identity.
rebuilt = object()
ordered = []
for node, shadowed in _executed_nodes(tree.body):
@@ -944,8 +924,7 @@ def _get_new_mapper():
table.clear()
table.update(original)
elif isinstance(node, (ast.Assign, ast.AnnAssign)):
- # An annotated subscript assigns as the plain form does; a bare one
- # binds nothing.
+ # An annotated subscript assigns as the plain form does; a bare one binds nothing.
if isinstance(node, ast.AnnAssign):
if node.value is None:
continue
@@ -964,11 +943,11 @@ def _get_new_mapper():
continue
if isinstance(replacement, dict):
if not replacement and not builder_called:
- # An INITIALISER, not a clear: a mapper.py with no
- # `build_mappers` writes `X = {}` and fills all five
- # from a module-scope loop the installed builder
- # reproduces. Reading it as a clear emptied every
- # table and the upgrade notice stopped firing.
+ # An INITIALISER, not a clear: a mapper.py with no build_mappers writes X = {}
+ # and fills all five
+ # from a module-scope loop the installed builder reproduces, so reading it as a
+ # clear emptied
+ # every table and stopped the upgrade notice firing.
continue
table.clear()
table.update(replacement)
@@ -1012,8 +991,7 @@ def _get_new_mapper():
and isinstance(node.func, ast.Attribute)
and node.func.attr == "update"
):
- # Only a named receiver and a literal mapping; the keyword form cannot
- # express keys with a slash.
+ # Only a named receiver and a literal mapping; the keyword form cannot express keys with a slash.
if not isinstance(node.func.value, ast.Name):
continue
if node.func.value.id in shadowed:
@@ -1028,8 +1006,8 @@ def _get_new_mapper():
if isinstance(additions, dict):
table.update(additions)
elif isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
- # An alias not derivable from the source table, applied with the
- # INSTALLED helper from literal arguments only.
+ # An alias not derivable from the source table, applied with the INSTALLED helper from literal
+ # arguments only.
helper = _MAPPER_HELPERS.get(node.func.id)
if helper is None:
continue
@@ -1106,7 +1084,7 @@ def get_model_name(
float_to_int = FLOAT_TO_INT_MAPPER,
map_to_unsloth_16bit = MAP_TO_UNSLOTH_16bit,
)
- # Remap "bad" names (e.g. oversized dynamic quants or MoEs)
+ # Remap "bad" names (oversized dynamic quants or MoEs).
if (
new_model_name is not None
and type(new_model_name) is str
@@ -1114,9 +1092,8 @@ def get_model_name(
):
new_model_name = BAD_MAPPINGS[new_model_name.lower()]
elif new_model_name is None and model_name.lower() in BAD_MAPPINGS:
- # Some bad names (e.g. the `-unsloth-bnb-4bit` dynamic quants) are keys
- # of the mappers, not values, so the resolver returns None for them and
- # the remap above is skipped; remap the input name directly instead.
+ # Some bad names (the -unsloth-bnb-4bit dynamic quants) are keys of the mappers, not values, so the
+ # resolver returns None and the remap above is skipped; remap the input name directly.
new_model_name = BAD_MAPPINGS[model_name.lower()]
if (
@@ -1125,7 +1102,6 @@ def get_model_name(
and model_name[0].isalnum()
and not _env_says_offline() # offline: skip the remote (raw GitHub) mapper refresh
):
- # Try checking if a new Unsloth version allows it!
(
NEW_INT_TO_FLOAT_MAPPER,
NEW_FLOAT_TO_INT_MAPPER,
@@ -1140,8 +1116,8 @@ def get_model_name(
int_to_float = NEW_INT_TO_FLOAT_MAPPER,
float_to_int = NEW_FLOAT_TO_INT_MAPPER,
map_to_unsloth_16bit = NEW_MAP_TO_UNSLOTH_16bit,
- # the fp8 probe has to look at the FETCHED tables too, or a new fp8 repo would
- # miss both here and in the installed tables and skip the upgrade message
+ # The fp8 probe has to look at the FETCHED tables too, or a new fp8 repo would miss both here and in
+ # the installed tables and skip the upgrade message.
fp8_block = NEW_FP8_BLOCK_MAPPER,
fp8_row = NEW_FP8_ROW_MAPPER,
)
@@ -1190,7 +1166,7 @@ def _offline_quantize_to_fp8(
for x in (getattr(config, "architectures", None) or [])
)
is_vlm = is_vlm or hasattr(config, "vision_config")
- # Decide text-only before the cache name so the fp8 artifact and its path stay in sync. #5816
+ # Decide text-only before the cache name so the fp8 artifact and its path stay in sync (#5816).
text_config = None
if text_only and hasattr(config, "vision_config"):
from ._utils import (
@@ -1209,11 +1185,11 @@ def _offline_quantize_to_fp8(
is_vlm = False
temp_dir = tempfile.gettempdir()
- # Cache text-only and full-VLM artifacts separately so neither reuses the other. #5816
+ # Cache text-only and full-VLM artifacts separately so neither reuses the other (#5816).
cache_name = model_name.split("/")[-1] + "-fp8-" + fp8_mode
if revision is not None:
- # Sanitizing is lossy (`release/v1` and `release.v1` collapse), so a digest of the
- # raw ref rides along and two refs never share an artifact.
+ # Sanitizing is lossy (release/v1 and release.v1 collapse), so a digest of the raw ref rides along
+ # and two refs never share an artifact.
digest = hashlib.sha256(revision.encode("utf-8")).hexdigest()[:12]
readable = re.sub(r"[^0-9A-Za-z_-]", "_", revision)[:40]
cache_name += "-rev-" + readable + "-" + digest
@@ -1336,6 +1312,7 @@ def _load_fp8_weight_map(
if is_local and os.path.exists(_local_path(index_file)):
index_path = _local_path(index_file)
elif not is_local:
+ # Errors after this point are per-tensor: warn and continue, never abort or hide them.
try:
index_path = _remote_path(index_file)
except Exception:
@@ -1407,7 +1384,7 @@ def _match_fp8_module(module_by_name, base):
if "language_model.model." in base:
candidates.append(base.replace("language_model.model.", "model.language_model.", 1))
if base.startswith("language_model."):
- candidates.append("model." + base) # add model. prefix
+ candidates.append("model." + base)
for candidate in candidates:
if candidate in module_by_name:
return module_by_name[candidate]
@@ -1441,8 +1418,8 @@ def _restore_dropped_fp8_scales(
# A variant load reads variant-named files; skip to avoid applying default scales to them.
if variant:
return (0, 0)
- # No fp8 params means the checkpoint was dequantized on purpose (e.g. load_in_16bit);
- # re-applying a scale would corrupt those already-correct 16bit weights, so do nothing.
+ # No fp8 params means the checkpoint was dequantized on purpose (load_in_16bit), and re-applying a
+ # scale would corrupt those already-correct 16bit weights.
if not any(p.dtype in _FP8_DTYPES for p in model.parameters()):
return (0, 0)
weight_map = _load_fp8_weight_map(
@@ -1471,8 +1448,8 @@ def _restore_dropped_fp8_scales(
if not isinstance(weight, torch.Tensor) or weight.ndim != 2:
continue
if weight.device.type == "meta":
- # Disk-offloaded layer: weight lives on meta until forward, so it cannot be
- # scaled in place here. Count and warn rather than silently leave it unscaled.
+ # Disk-offloaded layer: the weight lives on meta until forward, so it cannot be scaled in place
+ # here. Count and warn rather than silently leave it unscaled.
offloaded += 1
continue
if weight.dtype in _FP8_DTYPES:
@@ -1510,9 +1487,8 @@ def _restore_dropped_fp8_scales(
scale = scale.to(weight.device)
with torch.no_grad():
if out_features % bs0 == 0 and in_features % bs1 == 0:
- # Memory-frugal path: multiply block views in place against the broadcast
- # fp32 scale, avoiding a full expanded scale and fp32 copy that could OOM.
- # The in-place multiply promotes to fp32, matching the fallback exactly.
+ # Memory-frugal path: multiply block views in place against the broadcast fp32 scale, avoiding a
+ # full expanded scale and fp32 copy that could OOM.
module.weight.data.view(out_blocks, bs0, in_blocks, bs1).mul_(
scale[:, None, :, None]
)
@@ -1569,8 +1545,8 @@ def check_and_disable_bitsandbytes_loading(
if quant_method is None or quant_method == "bitsandbytes":
return load_in_4bit, load_in_8bit, quant_method
- # Model has a non-bitsandbytes quantization config (e.g., compressed-tensors, gptq, awq)
- # We should disable BOTH bitsandbytes loading to avoid config conflicts
+ # A non-bitsandbytes quantization config (compressed-tensors, gptq, awq) means BOTH bitsandbytes
+ # loading flags must be disabled to avoid config conflicts.
if load_in_4bit or load_in_8bit:
if verbose:
print(
@@ -1620,7 +1596,6 @@ def _get_fp8_mode_and_check_settings(
else:
fp8_mode = load_in_fp8
- # Check user settings
if fp8_mode not in ["row", "block"]:
raise ValueError(f"Unsloth: `load_in_fp8` can only be 'row' or 'block', got '{fp8_mode}'")
if full_finetuning:
@@ -1646,8 +1621,7 @@ def _get_fp8_mode_and_check_settings(
"Unsloth: On the fly `load_in_fp8` requires torch 2.9.0+. Try `unsloth/Qwen3-8B` instead."
)
- # Check if torchao has this PR: https://github.com/pytorch/ao/pull/3158,
- # which will be released in 0.15.0.
+ # Check if torchao has pytorch/ao#3158, released in 0.15.0.
if importlib.util.find_spec("torchao") is None:
raise ValueError(
"Unsloth: Please install torchao for on the fly float8 to work! Try `unsloth/Qwen3-8B` instead."
@@ -1662,7 +1636,7 @@ def _get_fp8_mode_and_check_settings(
if Version(torchao.__version__) < Version("0.15.0"):
raise ValueError(error_message)
- # If fbgemm_gpu_genai is installed and old, disable FBGEMM and use Triton instead
+ # If fbgemm_gpu_genai is installed and old, disable FBGEMM and use Triton kernels instead.
if (
importlib.util.find_spec("fbgemm_gpu") is not None
and importlib.util.find_spec("fbgemm_gpu.experimental") is not None
@@ -1679,18 +1653,12 @@ def _get_fp8_mode_and_check_settings(
return fp8_mode
-# Rotary inv_freq buffers are deliberately kept on CPU - Unsloth pre-builds a
-# cos/sin cache per GPU instead (see LlamaRotaryEmbedding.multi_gpu_cos_cached)
-# so the GPU-resident lookup never needs to move the tiny inv_freq tensor itself.
-# torch.nn.parallel.DistributedDataParallel ignores device entirely when it
-# broadcasts buffers across ranks, so a CPU buffer crashes NCCL's
-# _broadcast_coalesced with "No backend type associated with device type cpu".
-# Telling DDP to skip these specific buffers avoids that crash without moving
-# inv_freq to GPU (which would break the per-GPU cache design) and without
-# disabling buffer broadcast for every other module (the user's workaround).
-# Re-run this after wrapping with PEFT too - the buffers' fully qualified
-# names change once they sit under a PeftModel (eg "base_model.model...").
-# https://github.com/unslothai/unsloth/issues/6656
+# Rotary inv_freq buffers are deliberately kept on CPU: Unsloth pre-builds a cos/sin cache per GPU
+# instead (LlamaRotaryEmbedding.multi_gpu_cos_cached). torch.nn.parallel.DistributedDataParallel
+# ignores device when it broadcasts buffers across ranks, so a CPU buffer crashes NCCL's
+# _broadcast_coalesced with "No backend type associated with device type cpu"; telling DDP to skip
+# these buffers avoids that without moving inv_freq to GPU. Re-run after wrapping with PEFT, since
+# the fully qualified names change under a PeftModel ("base_model.model...") (#6656).
_ROTARY_INV_FREQ_BUFFER_NAMES = ("inv_freq", "short_inv_freq", "long_inv_freq")
@@ -1707,16 +1675,15 @@ def _exclude_rope_inv_freq_from_ddp(model):
from torch.nn.parallel import DistributedDataParallel
DistributedDataParallel._set_params_and_buffers_to_ignore_for_model(model, ignored)
except Exception:
- # Private PyTorch API - fall back to setting the attribute DDP reads
- # directly if it ever moves or changes signature.
+ # Private PyTorch API: fall back to setting the attribute DDP reads directly if it ever moves or
+ # changes signature.
model._ddp_params_and_buffers_to_ignore = ignored
return model
-# =============================================================================
-# Offline loading - single source of truth (shared by vision.py, loader.py and
-# the Unsloth exporter). Decide offline ONCE at the load boundary and force it
-# ONCE around the whole load, so every nested HF call inherits it.
+# Offline loading, shared by vision.py, loader.py and the Unsloth exporter: decide offline ONCE at
+# the load boundary and force it ONCE around the whole load, so nested HF calls inherit it.
+
# =============================================================================
_OFFLINE_ENV_VALUES = {"1", "true", "yes", "on"}
@@ -1737,17 +1704,16 @@ def _get_effective_local_files_only(kwargs):
return _env_says_offline()
-# Attribute stamped on a tokenizer/processor that was loaded local-only, so a later
-# save still knows. transformers takes local_files_only as an explicit from_pretrained
-# parameter and never copies it into tokenizer.init_kwargs, and _offline_aware_load
-# restores the offline env vars when the load window closes, so without this stamp an
-# explicit local_files_only = True load is invisible by the time we save (issue #7481).
+# Stamped on a tokenizer/processor loaded local-only so a later save still knows: transformers
+# takes local_files_only as an explicit from_pretrained parameter and never copies it into
+# tokenizer.init_kwargs (#7481). The load's cache_dir and ref travel with it, since saving
+# otherwise derives a cache from HF_HUB_CACHE and drops the branch from name_or_path.
_LOCAL_FILES_ONLY_ATTR = "_unsloth_local_files_only"
-# The load's cache_dir travels with it too: saving derives one from HF_HUB_CACHE /
-# HF_HOME, which does not see a caller-supplied cache.
+# The load's cache_dir travels with it too: saving derives one from HF_HUB_CACHE / HF_HOME, which does not see a
+# caller-supplied cache.
_LOADED_CACHE_DIR_ATTR = "_unsloth_loaded_cache_dir"
-# So does the ref it was read at: saving restores sentencepiece assets from
-# tokenizer.name_or_path, which names the repo but not the branch.
+# So does the ref it was read at: saving restores sentencepiece assets from tokenizer.name_or_path, which names
+# the repo but not the branch.
_LOADED_REVISION_ATTR = "_unsloth_loaded_revision"
@@ -1756,6 +1722,8 @@ def _mark_loaded_revision(result, revision):
if revision is None:
return result
for obj in result if isinstance(result, (tuple, list)) else (result,):
+ # Objects that reject new attributes (__slots__) are skipped.
+ # Skip objects that reject new attributes (__slots__).
try:
targets = (obj, getattr(obj, "tokenizer", None))
except Exception:
@@ -1763,7 +1731,6 @@ def _mark_loaded_revision(result, revision):
for target in targets:
if target is None:
continue
- # Skip objects that reject new attributes (__slots__).
try:
setattr(target, _LOADED_REVISION_ATTR, str(revision))
except Exception:
@@ -1781,15 +1748,14 @@ def _mark_loaded_local_files_only(result, cache_dir = None):
"""Stamp a load's local-only mode and cache_dir onto the returned objects."""
for obj in result if isinstance(result, (tuple, list)) else (result,):
try:
- # A processor keeps the tokenizer that _has_tokenizer_model unwraps to,
- # so stamp both (a wrapped model can raise from its own __getattr__).
+ # A processor keeps the tokenizer that _has_tokenizer_model unwraps to, so stamp both; a wrapped
+ # model can raise from its own __getattr__.
targets = (obj, getattr(obj, "tokenizer", None))
except Exception:
targets = (obj,)
for target in targets:
if target is None:
continue
- # Objects that reject new attributes (__slots__) are skipped.
try:
setattr(target, _LOCAL_FILES_ONLY_ATTR, True)
if cache_dir:
@@ -1825,7 +1791,8 @@ def _is_offline_related_error(exc):
# Match network failures by type (locale independent), not just message wording.
_net_types = [ConnectionError, TimeoutError, socket.gaierror, urllib.error.URLError]
_offline_fnf_types = () # FileNotFoundError subclasses that count as offline
- # urllib HTTPError is a URLError subclass: judge by status (5xx offline, 4xx propagates).
+ # urllib HTTPError is a URLError subclass, so judge by status (5xx offline, 4xx propagates).
+ # TLS/cert failures are security-sensitive (MITM, expired CA) and are never offline-retried.
_http_types = (urllib.error.HTTPError,)
# TLS/cert failures are security-sensitive (MITM, expired CA): never offline-retry them.
_ssl_types = [ssl.SSLError]
@@ -1883,7 +1850,7 @@ def _is_offline_related_error(exc):
"connection refused",
"we couldn't connect to",
"proxyerror",
- # Raw socket.gaierror DNS wording (Linux / macOS)
+ # Raw socket.gaierror DNS wording (Linux / macOS).
"name or service not known",
"temporary failure in name resolution",
"nodename nor servname provided",
@@ -1892,14 +1859,14 @@ def _is_offline_related_error(exc):
cur = exc
while cur is not None and id(cur) not in seen:
seen.add(id(cur))
- # TLS/cert failure (corporate MITM, expired CA): security-sensitive, never retry from
- # cache. Skip this node; a deeper cause in the chain may still be a genuine outage.
+ # TLS/cert failure (corporate MITM, expired CA): security-sensitive, never retry from cache. Skip
+ # this node; a deeper cause in the chain may still be a genuine outage.
if isinstance(cur, _ssl_types) or isinstance(getattr(cur, "reason", None), _ssl_types):
cur = cur.__cause__ or cur.__context__
continue
is_fnf = isinstance(cur, FileNotFoundError) and not isinstance(cur, _offline_fnf_types)
- # urllib HTTPError is a URLError (net type) but must be judged by status code below,
- # unlike LocalEntryNotFoundError (an HfHubHTTPError that is always offline).
+ # urllib HTTPError is a URLError (net type) but must be judged by status code below, unlike
+ # LocalEntryNotFoundError, an HfHubHTTPError that is always offline.
if (
isinstance(cur, _net_types)
and not is_fnf
@@ -1910,10 +1877,10 @@ def _is_offline_related_error(exc):
code = _http_status(cur)
if code is not None and 500 <= code < 600:
return True
- # No status -> wording fallback (coded 4xx already decided above).
+ # No status, so fall back to wording; a coded 4xx was already decided above.
if code is None and not is_fnf and any(w in str(cur).lower() for w in _wording):
return True
- # OSError wording fallback (HTTP status already decided above).
+ # OSError wording fallback; the HTTP status was already decided above.
elif isinstance(cur, OSError) and not is_fnf:
if any(w in str(cur).lower() for w in _wording):
return True
@@ -1921,17 +1888,20 @@ def _is_offline_related_error(exc):
return False
-# Process-wide HF offline state; the depth counter lets nested windows share one
-# flip (first entrant saves originals, last exit restores). Lock guards flip/restore.
+# Process-wide HF offline state; the depth counter lets nested windows share one flip (first entrant saves
+# originals, last exit restores). Lock guards flip/restore.
_force_offline_lock = _threading.RLock()
_force_offline_depth = 0
-_force_offline_saved = [] # in-process module attributes
-_force_offline_saved_env = {} # HF offline env-var originals
+_force_offline_saved = []
+_force_offline_saved_env = {}
def _reset_hf_sessions():
"""Clear hub's per-thread cached Sessions so the next rebuilds against the current
offline flag. On hub 0.x the offline adapter is baked in at Session creation. Best-effort."""
+ # Snapshot in-process constants BEFORE forcing the env: a module first imported here would otherwise
+ # initialize its constant from the just-set "1" and we would save (then restore) True, pinning the process
+ # offline after the window.
try:
from huggingface_hub.utils._http import reset_sessions
except Exception:
@@ -1956,9 +1926,6 @@ def _force_hf_offline():
if _force_offline_depth == 0:
saved = []
saved_env = {}
- # Snapshot in-process constants BEFORE forcing the env: a module first imported
- # here would otherwise initialize its constant from the just-set "1" and we would
- # save (then restore) True, pinning the process offline after the window.
try:
import huggingface_hub.constants as _hfc
if hasattr(_hfc, "HF_HUB_OFFLINE"):
@@ -1983,6 +1950,7 @@ def _force_hf_offline():
pass
_force_offline_saved = saved
_force_offline_saved_env = saved_env
+ # Drop offline-mounted sessions so later online calls rebuild for the network.
# Rebuild cached sessions so they pick up the offline adapter.
_reset_hf_sessions()
_force_offline_depth += 1
@@ -2004,7 +1972,6 @@ def _force_hf_offline():
else:
os.environ[_k] = _v
_force_offline_saved_env = {}
- # Drop offline-mounted sessions so later online calls rebuild for the network.
_reset_hf_sessions()
@@ -2029,11 +1996,11 @@ def _restore_progress_bars(were_disabled):
pass
-# Every way a cache miss reaches the caller once offline mode has skipped Transformers'
-# own "does not appear to have a file named" raise: the resolved path stays None and the
-# next line dereferences it, so the message names the None and never the cache. Same set
-# the Unsloth training worker matches (studio/backend/core/training/worker.py, #7845):
-# weights come out as `endswith`, tokenizers/processors as any of the other four.
+# Every way a cache miss reaches the caller once offline mode has skipped Transformers' own "does not appear to
+# have a file named" raise: the resolved path stays None and the next line dereferences it, so the message names
+# the None and never the cache. Same set the Unsloth training worker matches
+# (studio/backend/core/training/worker.py, #7845): weights come out as `endswith`, tokenizers/processors as any
+# of the other four.
_EMPTY_CACHE_ARTIFACTS = (
"'nonetype' object has no attribute 'endswith'",
"'nonetype' object has no attribute 'readlines'",
@@ -2059,8 +2026,8 @@ def _empty_cache_artifact(exc):
while exc is not None and id(exc) not in seen:
seen.add(id(exc))
text = str(exc).lower()
- # Gate on the None: the same TypeError wording about a real path (`not 'int'`)
- # is a caller bug, not an empty cache.
+ # Gate on the None: the same TypeError wording about a real path (`not 'int'`) is a caller bug, not an
+ # empty cache.
if ("nonetype" in text or "path 'none'" in text) and any(
artifact in text for artifact in _EMPTY_CACHE_ARTIFACTS
):
@@ -2118,25 +2085,25 @@ def _offline_aware_load(fn):
if _get_effective_local_files_only(kwargs):
kwargs["local_files_only"] = True
with _force_hf_offline():
- # Stamp inside the window: the env vars are restored on exit, so the
- # request has to travel on the objects themselves to reach saving.
+ # Stamp inside the window: the env vars are restored on exit, so the request has to travel on
+ # the objects themselves to reach saving.
return _mark_loaded_local_files_only(fn(*args, **kwargs), kwargs.get("cache_dir"))
- _pb_were_disabled = _progress_bars_were_disabled() # restore before any retry
+ _pb_were_disabled = _progress_bars_were_disabled()
try:
return fn(*args, **kwargs)
except Exception as e:
- # Skip if not network-related, or already retried by a nested decorator
- # (else outer layers reload the whole model again).
+ # Skip if not network-related, or already retried by a nested decorator (else outer layers reload
+ # the whole model again).
if not _is_offline_related_error(e) or getattr(e, "_unsloth_offline_retried", False):
raise
- # Holding `e` holds its frames, and those frames hold the half-built model,
- # so the collect below could not free it and a large VLM OOMed on the reload
- # the retry exists to make. A wrapper's cause/context carries tracebacks over
- # the SAME frames, so the whole chain has to be released, not just the top.
+ # Holding `e` holds its frames, and those frames hold the half-built model, so the collect below
+ # could not free it and a large VLM OOMed on the reload the retry exists to make. A wrapper's
+ # cause/context carries tracebacks over the SAME frames, so the whole chain has to be released, not
+ # just the top.
online_error = e
_release_traceback_locals(online_error)
- # Retry OUTSIDE the except so the failed attempt's traceback (a partial model)
- # is freed before reallocating, else a large VLM can OOM on the second load.
+ # Retry OUTSIDE the except so the failed attempt's traceback (a partial model) is freed before
+ # reallocating, else a large VLM can OOM on the second load.
try:
gc.collect()
if torch.cuda.is_available():
@@ -2152,8 +2119,8 @@ def _offline_aware_load(fn):
with _force_hf_offline():
return fn(*args, **kwargs)
except Exception as e:
- # A real retry failure (corrupt checkpoint, OOM) is news and goes out as
- # itself; only the empty-cache artifact is worth replacing.
+ # A real retry failure (corrupt checkpoint, OOM) is news and goes out as itself; only the empty-
+ # cache artifact is worth replacing.
if not _empty_cache_artifact(e):
# Tag so an enclosing _offline_aware_load skips its own redundant retry.
try:
@@ -2161,22 +2128,21 @@ def _offline_aware_load(fn):
except Exception:
pass
raise
- # The retry can load a whole cached model and only then trip over a missing
- # tokenizer file, and this error outlives the call on the online one, so its
- # frames would keep that model resident for as long as the caller holds it.
+ # The retry can load a whole cached model and only then trip over a missing tokenizer file, and this
+ # error outlives the call on the online one, so its frames would keep that model resident for as
+ # long as the caller holds it.
_release_traceback_locals(e)
retry_error = e
- # Report the ONLINE error: this retry only runs because of it, and its own
- # failure names an empty cache badly (offline mode skips Transformers'
- # "does not appear to have a file named" raise, so the user saw
+ # Report the ONLINE error: this retry only runs because of it, and its own failure names an empty cache
+ # badly (offline mode skips Transformers' "does not appear to have a file named" raise, so the user saw
# `AttributeError: 'NoneType' ... 'endswith'`).
try:
online_error._unsloth_offline_retried = True
except Exception:
pass
- # Raise OUTSIDE the handler above: inside it, Python would overwrite
- # `__context__` with the cache miss, and that chain is often the only thing
- # that still makes the online error classifiable as network-related.
+ # Raise OUTSIDE the handler above: inside it, Python would overwrite `__context__` with the cache miss,
+ # and that chain is often the only thing that still makes the online error classifiable as network-
+ # related.
if online_error.__cause__ is None and online_error.__context__ is None:
# No chain to lose, so chain the retry on where it also prints.
raise online_error from retry_error
@@ -2215,8 +2181,8 @@ def _resolve_hub_repo_local_dir(
token = None,
cache_dir = None,
revision = None,
- # Default closed: a "resolve local dir" helper must not download. False here
- # means five filenames each retried with backoff before it gives up.
+ # Default closed: a "resolve local dir" helper must not download. False here means five filenames each
+ # retried with backoff before it gives up.
local_files_only = True,
filenames = (
"tokenizer_config.json",
diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py
index 71095b796db..6fa681f41d5 100644
--- a/unsloth/models/mapper.py
+++ b/unsloth/models/mapper.py
@@ -1116,8 +1116,8 @@ __INT_TO_FLOAT_MAPPER = \
"unsloth/Magistral-Small-2509-bnb-4bit",
),
},
- # No Unsloth 16bit repo exists at this size, so this is a 1-tuple naming the
- # real upstream, same as the other 70B and 405B rows.
+ # No Unsloth 16bit repo exists at this size, so this is a 1-tuple naming the real upstream, like the
+ # other 70B and 405B rows.
"unsloth/Apertus-70B-Instruct-2509-unsloth-bnb-4bit" : (
"swiss-ai/Apertus-70B-Instruct-2509",
),
@@ -1311,7 +1311,6 @@ __INT_TO_FLOAT_MAPPER = \
"google/functiongemma-270m-it",
"unsloth/functiongemma-270m-it-unsloth-bnb-4bit",
),
- # Ministral 3 models
"unsloth/Ministral-3-3B-Instruct-2512-unsloth-bnb-4bit" : {
"8" : (
"mistralai/Ministral-3-3B-Instruct-2512",
@@ -1426,7 +1425,7 @@ def build_mappers(__INT_TO_FLOAT_MAPPER):
if type(values) is dict:
assert "16" in values
float16_values = values["16"]
- # Float8 and other quantized types
+ # Float8 and other quantized types.
if "8" in values:
float8_values = values["8"]
assert len(float8_values) == 3
@@ -1455,14 +1454,14 @@ def build_mappers(__INT_TO_FLOAT_MAPPER):
for value in values:
FLOAT_TO_INT_MAPPER[value] = key
- # Map to Unsloth version for 16bit versions
+ # Map to the Unsloth version for 16bit.
if len(values) == 2:
if values[0].startswith("unsloth"):
_add_with_lower(MAP_TO_UNSLOTH_16bit, values[1], values[0])
_add_with_lower(MAP_TO_UNSLOTH_16bit, block, values[0])
_add_with_lower(MAP_TO_UNSLOTH_16bit, row, values[0])
elif len(values) == 3:
- # Dynamic Unsloth quantization
+ # Dynamic Unsloth quantization.
if values[0].startswith("unsloth"):
_add_with_lower(MAP_TO_UNSLOTH_16bit, values[1], values[0])
_add_with_lower(MAP_TO_UNSLOTH_16bit, values[2], values[0])
@@ -1470,7 +1469,6 @@ def build_mappers(__INT_TO_FLOAT_MAPPER):
_add_with_lower(MAP_TO_UNSLOTH_16bit, row, values[0])
pass
- # Get lowercased
lowered_key = key.lower()
INT_TO_FLOAT_MAPPER[lowered_key] = values[0].lower()
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index 41b5a7c3b6b..4da95393af5 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -68,7 +68,6 @@ def MistralAttention_fast_forward(
*args,
**kwargs,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
- # Clear inference
if hasattr(self, "paged_attention"):
del self.paged_attention_K
del self.paged_attention_V
@@ -96,7 +95,7 @@ def MistralAttention_fast_forward(
if past_key_value is not None:
kv_seq_len += past_key_value[0].shape[-2]
- # Extend RoPE dynamically to fit in VRAM
+ # Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.
self.rotary_emb.extend_rope_embedding(V, seq_len = kv_seq_len)
cos, sin = self.rotary_emb.get_cached(kv_seq_len, Q.device.index)
@@ -109,11 +108,10 @@ def MistralAttention_fast_forward(
V = torch.cat([past_key_value[1], V], dim = 2)
past_key_value = (K, V) if use_cache else None
- # Attention module
sw_cfg = getattr(self.config, "sliding_window", None)
- # A non-positive window means "no local attention", the same as absent. Passing 0 through
- # made window_size (0, 0) for flash and an all-false SDPA mask, since the lower bound
- # q_pos - (0 - 1) sits above the causal upper bound.
+ # A non-positive window means "no local attention", the same as absent: passing 0 through made
+ # window_size (0, 0) for flash and an all-false SDPA mask, since q_pos - (0 - 1) sits above the
+ # causal upper bound.
if sw_cfg is None or sw_cfg == "null" or (isinstance(sw_cfg, int) and sw_cfg <= 0):
sw = kv_seq_len
else:
@@ -133,8 +131,8 @@ def MistralAttention_fast_forward(
"softmax_scale": getattr(self, "softmax_scale", None),
},
)
- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse
- # (KV cache / padding mask) raises. None => byte-identical default.
+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /
+ # padding mask) raises. None means the byte-identical default.
_pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)
context = AttentionContext(
bsz = bsz,
@@ -146,11 +144,10 @@ def MistralAttention_fast_forward(
seq_info = seq_info,
attention_mask = attention_mask,
causal_mask = causal_mask,
- # The window the flash path already gets through window_size. SDPA needs it too, and
- # not only in the branch above: training takes `elif self.training: pass`, so no 4D
- # mask is synthesized, and with xformers off and flash absent the local window was the
- # one thing nothing carried -- every sequence past config.sliding_window silently
- # attended its whole causal history.
+ # The window the flash path already gets through window_size. SDPA needs it too, and not only in
+ # the branch above: training takes `elif self.training: pass`, so no 4D mask is synthesized, and
+ # with xformers off and flash absent every sequence past config.sliding_window silently attended
+ # its whole causal history.
sliding_window = None if window_size == (-1, -1) else sw,
prefix_seg_info = _pg_seg,
)
@@ -182,6 +179,7 @@ def MistralForCausalLM_fast_forward(
) -> Union[Tuple, CausalLMOutputWithPast]:
# PrefixGrouper brings its own mask: a synthesized causal attention_mask would trip
# resolve_prefix_seg_info on the no-xFormers path and force a fallback.
+ # Not using xformers - need to create attention masks
if (
causal_mask is None
and past_key_values is None
@@ -191,7 +189,7 @@ def MistralForCausalLM_fast_forward(
sliding_window = getattr(self.config, "sliding_window", None)
if HAS_XFORMERS:
- # Always create causal mask for xformers
+ # Always create a causal mask for xformers.
if sliding_window is None or sliding_window == "null" or sliding_window <= 0:
causal_mask = xformers.attn_bias.LowerTriangularMask()
elif q_len <= sliding_window:
@@ -201,29 +199,27 @@ def MistralForCausalLM_fast_forward(
[q_len] * bsz
).make_local_attention(window_size = sliding_window)
- # If attention_mask exists, it will be handled in the attention forward
+ # If attention_mask exists, it is handled in the attention forward.
elif self.training:
- # LlamaModel_fast_forward's DPO embed-masking block needs the 2D
- # attention_mask; it nulls the mask before attention anyway, so
- # leaving it 2D is safe and avoids a 4D conversion that crashes DPO.
+ # LlamaModel_fast_forward's DPO embed-masking block needs the 2D attention_mask; it nulls the mask
+ # before attention anyway, so leaving it 2D avoids a 4D conversion that crashes DPO.
pass
else:
- # Not using xformers - need to create attention masks
if (
sliding_window is None
or sliding_window == "null"
or sliding_window <= 0
or q_len <= sliding_window
):
- # Fully causal mask
+ # Fully causal mask.
causal_mask_values = torch.triu(
torch.full((q_len, q_len), -torch.inf, device = input_ids.device),
diagonal = 1,
)
else:
- # Sliding window attention
+ # Sliding window attention.
q_indices = torch.arange(q_len, device = input_ids.device).view(-1, 1)
k_indices = torch.arange(q_len, device = input_ids.device).view(1, -1)
@@ -239,7 +235,7 @@ def MistralForCausalLM_fast_forward(
attention_mask = causal_mask_values[None, None, :, :].expand(bsz, 1, q_len, q_len)
else:
if attention_mask.dim() == 2:
- # Convert 0/1 padding mask to additive format: 1->0 (keep), 0->-inf (mask)
+ # Convert the 0/1 padding mask to additive form: 1 -> 0 (keep), 0 -> -inf (mask).
padding_mask = torch.where(
attention_mask[:, None, None, :].bool(),
0.0,
@@ -261,7 +257,6 @@ def MistralForCausalLM_fast_forward(
)
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
- # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
self.model._has_no_labels = labels is None
if past_key_values is not None:
@@ -298,15 +293,14 @@ def MistralForCausalLM_fast_forward(
if labels is not None:
labels = labels.to(lm_head_device)
- # Merge legacy / new spellings before branching so the decode-time
- # last-token slice fires on the normal path too. Skip int max() if
- # either is a tensor (HF selective-decode form).
+ # Merge legacy / new spellings before branching so the decode-time last-token slice fires on the
+ # normal path too. Skip int max() if either is a tensor (HF selective-decode form).
if isinstance(num_logits_to_keep, torch.Tensor) or isinstance(logits_to_keep, torch.Tensor):
num_logits_to_keep = 0
else:
num_logits_to_keep = max(num_logits_to_keep, logits_to_keep)
- # If we are in GRPO mode, return raw hidden states
+ # In GRPO mode, return raw hidden states.
if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":
if num_logits_to_keep != 0:
hidden_states = hidden_states[:, -num_logits_to_keep:, :]
@@ -325,9 +319,9 @@ def MistralForCausalLM_fast_forward(
logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(lm_head.dtype))
else:
RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"
- # < 1024 Normal Unsloth uses less VRAM!
+ # Below 1024 normal Unsloth uses less VRAM.
if bsz * q_len <= 1024 and not RETURN_LOGITS:
- # Use unsloth_fused_ce_loss which actually calculates the best chunk size to reduce VRAM usage
+ # unsloth_fused_ce_loss calculates the best chunk size to reduce VRAM usage.
RETURN_LOGITS = False
if not RETURN_LOGITS and labels is not None:
@@ -336,20 +330,13 @@ def MistralForCausalLM_fast_forward(
n_items = kwargs.get("n_items", None)
logit_softcapping = getattr(self.config, "final_logit_softcapping", 0)
- # Packed-boundary guard, see llama.py. This branch returns, so
- # mask_packed_sequence_boundaries() below is never reached.
+ # Packed-boundary guard, see llama.py. This branch returns, so mask_packed_sequence_boundaries()
+ # below is never reached.
labels = mask_packed_boundary_labels(
labels,
kwargs.get("packed_seq_lengths"),
)
- # loss = fused_linear_cross_entropy(
- # hidden_states = hidden_states,
- # lm_weight = lm_head,
- # labels = labels,
- # num_items_in_batch = n_items,
- # logit_softcapping = logit_softcapping,
- # )
loss = unsloth_fused_ce_loss(
trainer = None,
hidden_states = hidden_states,
@@ -364,8 +351,7 @@ def MistralForCausalLM_fast_forward(
logit_softcapping = logit_softcapping,
)
if not return_dict:
- # Fused CE never materializes `logits`; use EMPTY_LOGITS
- # like the return_dict branch below (fixes #2068).
+ # Fused CE never materializes logits; use EMPTY_LOGITS like the return_dict branch below (#2068).
output = (EMPTY_LOGITS,) + outputs[1:]
return (loss,) + output if loss is not None else output
@@ -384,11 +370,6 @@ def MistralForCausalLM_fast_forward(
loss = None
if labels is not None:
shift_logits = logits
- # if not hasattr(self, "extra_ignored_labels"):
- # # Fixes https://github.com/unslothai/unsloth/issues/10
- # self.extra_ignored_labels = torch.full((self.max_seq_length, 1), -100, device = "cuda:0")
- # pass
- # shift_labels = torch.hstack((labels[..., 1:], self.extra_ignored_labels[:labels.shape[0]]))
shift_labels = torch.empty_like(labels)
shift_labels[..., :-1] = labels[..., 1:]
shift_labels[..., -1] = -100
@@ -444,10 +425,9 @@ class FastMistralModel(FastLlamaModel):
scaled_rope_module = LlamaLinearScalingRotaryEmbedding,
attention_module = MistralAttention,
)
- # Just for Mistral Nemo models!
+ # Just for Mistral Nemo models.
if function is not None and init_name is not None:
function = patch_mistral_nemo_attention(function)
- # if True:#init_name is not None:
exec(function, globals())
MistralAttention.__init__ = eval(init_name)
MistralAttention.forward = MistralAttention_fast_forward
@@ -459,11 +439,8 @@ class FastMistralModel(FastLlamaModel):
PeftModelForCausalLM.forward = PeftModel_fast_forward
fix_prepare_inputs_for_generation(MistralForCausalLM)
- # Solves https://github.com/unslothai/unsloth/issues/168
- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.
- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
- # https://github.com/huggingface/transformers/pull/27931
- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,
+ # huggingface/transformers#27931), so the old rotary embeddings are retained.
import transformers.models.mistral.modeling_mistral
transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding = LlamaRotaryEmbedding
diff --git a/unsloth/models/qwen2.py b/unsloth/models/qwen2.py
index 61af235b523..dbbd5b2a062 100644
--- a/unsloth/models/qwen2.py
+++ b/unsloth/models/qwen2.py
@@ -57,11 +57,8 @@ class FastQwen2Model(FastLlamaModel):
PeftModelForCausalLM.forward = PeftModel_fast_forward
fix_prepare_inputs_for_generation(Qwen2ForCausalLM)
- # Solves https://github.com/unslothai/unsloth/issues/168
- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.
- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
- # https://github.com/huggingface/transformers/pull/27931
- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py
+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,
+ # huggingface/transformers#27931), so the old rotary embeddings are retained.
import transformers.models.qwen2.modeling_qwen2
transformers.models.qwen2.modeling_qwen2.Qwen2RotaryEmbedding = LlamaRotaryEmbedding
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index 986c1fa115b..d41410c7a2a 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -77,7 +77,6 @@ def Qwen3Attention_fast_forward(
*args,
**kwargs,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
- # Clear inference
if hasattr(self, "paged_attention"):
del self.paged_attention_K
del self.paged_attention_V
@@ -96,17 +95,13 @@ def Qwen3Attention_fast_forward(
assert n_kv_heads * n_groups == n_heads
Q, K, V = self.apply_qkv(self, hidden_states)
- Q = Q.view(
- bsz, q_len, n_heads, head_dim
- ) # .transpose(1, 2) # we will transpose after normalisation
- K = K.view(
- bsz, q_len, n_kv_heads, head_dim
- ) # .transpose(1, 2) # we will transpose after normalisation
+ Q = Q.view(bsz, q_len, n_heads, head_dim)
+ K = K.view(bsz, q_len, n_kv_heads, head_dim)
V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)
seq_info = get_packed_info_from_kwargs(kwargs, hidden_states.device)
- # Qwen3 adds QKNorm (the only difference from Qwen2). A compiled norm
- # mismatches Transformers' numbers, so use fast_rms_layernorm. TODO: investigate.
+ # Qwen3 adds QKNorm, the only difference from Qwen2. A compiled norm mismatches Transformers'
+ # numbers, so use fast_rms_layernorm.
Q = fast_rms_layernorm(self.q_norm, Q)
K = fast_rms_layernorm(self.k_norm, K)
@@ -117,7 +112,7 @@ def Qwen3Attention_fast_forward(
if past_key_value is not None:
kv_seq_len += past_key_value[0].shape[-2]
- # Extend RoPE dynamically to fit in VRAM
+ # Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.
if position_embeddings and kv_seq_len <= position_embeddings[0].shape[0]:
cos, sin = position_embeddings
else:
@@ -126,7 +121,6 @@ def Qwen3Attention_fast_forward(
cos, sin = rotary_emb.get_cached(kv_seq_len, Q.device.index)
rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")
- # Useful for LongRoPE
Q, K = fast_rope_embedding(Q, K, cos, sin, rope_position_ids)
if past_key_value is not None:
@@ -134,7 +128,6 @@ def Qwen3Attention_fast_forward(
V = torch.cat([past_key_value[1], V], dim = 2)
past_key_value = (K, V) if use_cache else None
- # Attention module
use_varlen = seq_info is not None and past_key_value is None
backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
attention_config = AttentionConfig(
@@ -148,8 +141,8 @@ def Qwen3Attention_fast_forward(
"softmax_scale": getattr(self, "softmax_scale", None),
},
)
- # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse
- # (KV cache / padding mask) raises. None => byte-identical default.
+ # PrefixGrouper seg table rides in **kwargs from the GRPO logprob forward; misuse (KV cache /
+ # padding mask) raises. None means the byte-identical default.
_pg_seg = resolve_prefix_seg_info(kwargs, past_key_value, attention_mask)
context = AttentionContext(
bsz = bsz,
@@ -200,15 +193,12 @@ def Qwen3Attention_fast_forward_inference(
n_groups = self.num_key_value_groups
n_kv_heads = self.config.num_key_value_heads
head_dim = self.head_dim
- # assert(n_kv_heads * n_groups == n_heads)
hidden_size = self.config.hidden_size
attention_size = n_heads * head_dim
seq_len = K1.shape[-2]
kv_seq_len = seq_len + 1
- # Prefill phase
- # if not hasattr(self, "paged_attention"):
device = hidden_states.device
if do_prefill:
self.paged_attention = torch.empty(
@@ -224,7 +214,7 @@ def Qwen3Attention_fast_forward_inference(
self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)
self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
- # Mistral Nemo 12b has weird dimensions
+ # Mistral Nemo 12b has weird dimensions.
if attention_size != hidden_size:
self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
else:
@@ -252,12 +242,8 @@ def Qwen3Attention_fast_forward_inference(
Qn = fast_linear_forward(self.q_proj, Xn, out = self.temp_QA[0])
Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0])
Vn = fast_linear_forward(self.v_proj, Xn, out = self.temp_KV[1])
- Qn = Qn.view(
- bsz, 1, n_heads, head_dim
- ) # .transpose(1, 2) # we will transpose after normalisation
- Kn = Kn.view(
- bsz, 1, n_kv_heads, head_dim
- ) # .transpose(1, 2) # we will transpose after normalisation
+ Qn = Qn.view(bsz, 1, n_heads, head_dim)
+ Kn = Kn.view(bsz, 1, n_kv_heads, head_dim)
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
Qn = fast_rms_layernorm_inference(self.q_norm, Qn)
@@ -266,14 +252,10 @@ def Qwen3Attention_fast_forward_inference(
Qn = Qn.transpose(1, 2)
Kn = Kn.transpose(1, 2)
- # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
- # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
-
- # Need to do it prior 2 steps before hitting full on short KV cache
- # or else error
+ # Must be done 2 steps before hitting full on a short KV cache, or it errors.
self.rotary_emb.extend_rope_embedding(Vn, seq_len + 2)
cos, sin = self.rotary_emb.get_cached(kv_seq_len, Qn.device.index)
- # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last
+ # Transformers 5.x: position_ids may be [batch, full_seq_len]; slice to last.
if position_ids.dim() >= 2 and position_ids.shape[-1] > 1:
position_ids = position_ids[:, -1:]
cos = cos[position_ids].unsqueeze(1)
@@ -283,22 +265,17 @@ def Qwen3Attention_fast_forward_inference(
RH_Q = self.RH_Q
RH_Q[:, :, :, :h] = Qn[:, :, :, h:]
RH_Q[:, :, :, h:] = Qn[:, :, :, :h]
- RH_Q[:, :, :, :h].neg_() # torch.neg(RH_Q[:,:,:,:h], out = RH_Q[:,:,:,:h])
+ RH_Q[:, :, :, :h].neg_()
Qn *= cos
Qn.addcmul_(RH_Q, sin)
- RH_K = RH_Q[
- :, :n_kv_heads, :, :
- ] # torch.empty((n_kv_heads, 1, head_dim), dtype = dtype, device = "cuda:0")
+ RH_K = RH_Q[:, :n_kv_heads, :, :]
RH_K[:, :, :, :h] = Kn[:, :, :, h:]
RH_K[:, :, :, h:] = Kn[:, :, :, :h]
- RH_K[:, :, :, :h].neg_() # torch.neg(RH_K[:,:,:,:h], out = RH_K[:,:,:,:h])
+ RH_K[:, :, :, :h].neg_()
Kn *= cos
Kn.addcmul_(RH_K, sin)
- # New KV cache
- # Kn = torch.cat([K1, Kn], dim = 2)
- # Vn = torch.cat([V1, Vn], dim = 2)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
@@ -308,14 +285,14 @@ def Qwen3Attention_fast_forward_inference(
sliding_window = getattr(self.config, "sliding_window", None)
if sliding_window is not None and kv_seq_len > sliding_window:
start = kv_seq_len - sliding_window
- Knn = Kn[:, :, start:, :] # .contiguous()
- Vnn = Vn[:, :, start:, :] # .contiguous()
+ Knn = Kn[:, :, start:, :]
+ Vnn = Vn[:, :, start:, :]
if attention_mask is not None:
attention_mask = attention_mask[..., start:]
else:
Knn, Vnn = Kn, Vn
- # when qlen==vlen and attn_mask is None, we should use causal attention
+ # When qlen == vlen and attn_mask is None, causal attention is the right choice.
Q_len = Qn.shape[-2]
K_len = Knn.shape[-2]
if attention_mask is not None and attention_mask.dim() == 2:
@@ -340,7 +317,7 @@ def Qwen3Attention_fast_forward_inference(
# Avoid SDPA GQA drift for batched masked decode.
use_sdpa_gqa = False
- # Grouped query attention
+ # Grouped query attention.
_, _, cached_len, _ = Knn.shape
if bsz == 1 or ((not use_sdpa_gqa) and n_groups != 1):
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
@@ -348,14 +325,12 @@ def Qwen3Attention_fast_forward_inference(
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)
- # Attention
if bsz == 1:
- Qn *= (
- self.scalar
- ) # See https://github.com/ggerganov/llama.cpp/issues/7805#issuecomment-2153349963
- # It seems like doing (Q * scalar) @ K is better than (Q @ K) * scalar to stop overflows
+ Qn *= self.scalar
+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805
+ # (comment 2153349963).
A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
- A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32) # .to(A.dtype)
+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
A = torch_matmul(A, Vnn, out = Qn)
else:
if use_sdpa_gqa:
@@ -400,8 +375,8 @@ class FastQwen3Model(FastLlamaModel):
PeftModelForCausalLM.forward = PeftModel_fast_forward
fix_prepare_inputs_for_generation(Qwen3ForCausalLM)
- # Retain old rotary embeddings; static KV cache (transformers 4.38.0)
- # slowed training. See unslothai/unsloth#168 and transformers#27931.
+ # Retain the old rotary embeddings: the static KV cache (transformers 4.38.0) slowed training.
+ # See unslothai/unsloth#168 and huggingface/transformers#27931.
import transformers.models.qwen3.modeling_qwen3
transformers.models.qwen3.modeling_qwen3.Qwen3RotaryEmbedding = LlamaRotaryEmbedding
diff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.py
index 872cb60a2e7..279b74b99dc 100644
--- a/unsloth/models/qwen3_moe.py
+++ b/unsloth/models/qwen3_moe.py
@@ -34,16 +34,7 @@ from transformers.models.qwen3_moe.modeling_qwen3_moe import (
)
# For Pytorch 2.1.1
-# TODO: Transformers moved to `attention_interface`. So we might not need these anymore
-# try:
-# from transformers.models.qwen3_moe.modeling_qwen3_moe import (
-# Qwen3SdpaAttention,
-# Qwen3FlashAttention2,
-# )
-# except:
-# Qwen3SdpaAttention = Qwen3Attention
-# Qwen3FlashAttention2 = Qwen3Attention
-# pass
+# Transformers moved to attention_interface, so the Sdpa/FlashAttention2 aliases may no longer be needed.
from unsloth_zoo.utils import Version, _get_dtype
@@ -56,7 +47,7 @@ def Qwen3MoeSparseMoeBlock_fast_forward(
temp_gate = None,
temp_up = None,
):
- # adapted from https://github.com/huggingface/transformers/pull/36878/files#diff-0855b77fc27ad9449158a1c74953f909b011c00de7125f7c8e68d0ff209c092aR356-R370
+ # Adapted from huggingface/transformers#36878 (modeling change R356-R370).
bsz, seq_len, hd = X.shape
X = X.view(-1, hd)
@@ -68,11 +59,11 @@ def Qwen3MoeSparseMoeBlock_fast_forward(
routing_weights = torch_nn_functional_softmax(router_logits, dim = -1, dtype = torch.float32)
routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim = -1)
routing_weights /= routing_weights.sum(dim = -1, keepdim = True)
- # cast back to the input dtype
+ # Cast back to the input dtype.
routing_weights = routing_weights.to(X.dtype)
final_X = torch.zeros((bsz * seq_len, hd), dtype = torch.float32, device = X.device)
- # One-hot the selected experts into a mask to index which expert is used
+ # One-hot the selected experts into a mask indexing which expert is used.
expert_mask = torch.nn.functional.one_hot(
selected_experts, num_classes = self.num_experts
).permute(2, 1, 0)
@@ -81,13 +72,13 @@ def Qwen3MoeSparseMoeBlock_fast_forward(
expert_layer = self.experts[expert_idx]
idx, top_x = torch.where(expert_mask[expert_idx])
- # Index hidden states for this expert and scale by routing_weights
+ # Index hidden states for this expert and scale by routing_weights.
current_state = X[None, top_x].reshape(-1, hd)
current_X = (
expert_layer(current_state) * routing_weights[top_x, idx, None]
) # Qwen3MoeMLP.forward = fast_swiglu_inference speeds this up
- # index_add_ needs a tensor index, so use top_x
+ # index_add_ needs a tensor index, so use top_x.
final_X.index_add_(0, top_x, current_X.to(X.dtype))
final_X = final_X.reshape(bsz, seq_len, hd)
return final_X, router_logits
@@ -110,7 +101,7 @@ def Qwen3MoeDecoderLayer_fast_forward(
):
residual = hidden_states
- if use_cache and hasattr(self, "_flag_for_generation"): # past_key_value is not None:
+ if use_cache and hasattr(self, "_flag_for_generation"):
residual = hidden_states
hidden_states = fast_rms_layernorm_inference(self.input_layernorm, hidden_states)
hidden_states, self_attn_weights, present_key_value = self.self_attn(
@@ -127,7 +118,7 @@ def Qwen3MoeDecoderLayer_fast_forward(
)
hidden_states += residual
- # MoE Router MLP
+ # MoE Router MLP.
residual = hidden_states
hidden_states = fast_rms_layernorm_inference(self.post_attention_layernorm, hidden_states)
hidden_states, router_logits = Qwen3MoeSparseMoeBlock_fast_forward(self.mlp, hidden_states)
@@ -148,7 +139,7 @@ def Qwen3MoeDecoderLayer_fast_forward(
)
hidden_states = residual + hidden_states
- # MoE Router MLP
+ # MoE Router MLP.
residual = hidden_states
hidden_states = fast_rms_layernorm(self.post_attention_layernorm, hidden_states)
hidden_states, router_logits = self.mlp(hidden_states)
@@ -177,8 +168,6 @@ class FastQwen3MoeModel(FastQwen3Model):
exec(function, globals())
Qwen3MoeAttention.__init__ = eval(init_name)
Qwen3MoeAttention.forward = Qwen3Attention_fast_forward
- # Qwen3SdpaAttention .forward = Qwen3Attention_fast_forward
- # Qwen3FlashAttention2 .forward = Qwen3Attention_fast_forward
Qwen3MoeSparseMoeBlock.forward = Qwen3MoeSparseMoeBlock_fast_forward
Qwen3MoeMLP.forward = fast_swiglu_inference # This is analogous to Dense models' MLP
Qwen3MoeDecoderLayer.forward = Qwen3MoeDecoderLayer_fast_forward
@@ -187,11 +176,8 @@ class FastQwen3MoeModel(FastQwen3Model):
PeftModelForCausalLM.forward = PeftModel_fast_forward
fix_prepare_inputs_for_generation(Qwen3MoeForCausalLM)
- # Solves https://github.com/unslothai/unsloth/issues/168
- # Static KV Cache was introduced in 4.38.0, causing training to be much slower.
- # Inference can now be CUDAGraphed, but we shall retain the old rotary embeddings.
- # https://github.com/huggingface/transformers/pull/27931
- # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py\
+ # Static KV Cache landed in 4.38.0 and made training much slower (#168,
+ # huggingface/transformers#27931), so the old rotary embeddings are retained.
import transformers.models.qwen3_moe.modeling_qwen3_moe
transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = (
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 65cb2621cd4..64fba41b818 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -43,13 +43,13 @@ from .rl_replacements import (
torch_compile_options = {
"epilogue_fusion": True,
- "max_autotune": False, # Disable Triton mm kernels
+ "max_autotune": False,
"shape_padding": True,
"trace.enabled": False,
"triton.cudagraphs": False,
}
-# vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it)
+# vLLM compatibility shim: TRL expects GuidedDecodingParams even when vLLM does not provide it.
try:
import vllm.sampling_params as _unsloth_vllm_sp
if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"):
@@ -74,13 +74,11 @@ except Exception:
except Exception:
trl_version = Version("0.0.0")
-# Get PyTorch version for feature detection
try:
torch_version = Version(torch.__version__.split("+")[0].split("a")[0].split("b")[0])
except Exception:
torch_version = Version("0.0.0")
-# Get transformers version for feature detection
try:
from transformers import __version__ as _transformers_version_raw
transformers_version = Version(_transformers_version_raw)
@@ -155,7 +153,7 @@ def PatchRL(FastLanguageModel):
try:
from trl.models import unwrap_model_for_generation
except ImportError:
- # Local fallback -- TRL removed or moved this symbol
+ # Local fallback: TRL removed or moved this symbol.
from contextlib import contextmanager as _cm
@_cm
@@ -188,10 +186,8 @@ def PatchRL(FastLanguageModel):
@contextmanager
def unsloth_unwrap_model_for_generation(model, *args, **kwargs):
- # why: snapshot before TRL's unwrap context manager, which calls
- # gradient_checkpointing_disable() before yielding; preserve the actual
- # mode value (e.g. "unsloth") rather than collapsing it to a bool, so
- # the finally restore matches the caller's configured GC mode.
+ # Snapshot before TRL's unwrap CM, which calls gradient_checkpointing_disable() before
+ # yielding. Keep the mode value (e.g. "unsloth"), not a bool, so the finally restore matches.
use_gradient_checkpointing = next(
(
v
@@ -201,11 +197,9 @@ def PatchRL(FastLanguageModel):
False,
)
with unwrap_model_for_generation(model, *args, **kwargs) as unwrapped_model:
- # Put the model in inference mode.
FastLanguageModel.for_inference(model)
- # We must use .clone for Unsloth since we force inference_mode
- # Rather we should have used no_grad
+ # .clone is required because inference_mode is forced here; no_grad would have been the better choice.
original_generate = unwrapped_model.generate
def generate_with_clone(*args, **kwargs):
@@ -219,7 +213,6 @@ def PatchRL(FastLanguageModel):
try:
yield unwrapped_model
finally:
- # Restore generate and return
unwrapped_model.generate = original_generate
FastLanguageModel.for_training(
model,
@@ -255,9 +248,8 @@ def PatchRL(FastLanguageModel):
if len(self.label_names) == 0
else all(inputs.get(k) is not None for k in self.label_names)
)
- # For CLIP-like models capable of returning loss values.
- # If `return_loss` is not specified or being `None` in `inputs`, we check if the default value of `return_loss`
- # is `True` in `model.forward`.
+ # For CLIP-like models capable of returning loss values: if return_loss is unset in inputs, check
+ # whether model.forward defaults it to True.
return_loss = inputs.get("return_loss", None)
if return_loss is None:
return_loss = self.can_return_loss
@@ -270,7 +262,7 @@ def PatchRL(FastLanguageModel):
else:
ignore_keys = []
- # labels may be popped when computing the loss (label smoothing for instance) so we grab them first.
+ # labels may be popped when computing the loss (label smoothing), so grab them first.
if has_labels or loss_without_labels:
labels = nested_detach(tuple(inputs.get(name) for name in self.label_names))
if len(labels) == 1:
@@ -278,8 +270,8 @@ def PatchRL(FastLanguageModel):
else:
labels = None
- # Force logits during eval, but restore the user's prior setting after
- # so an explicit UNSLOTH_RETURN_LOGITS="1" is not silently turned off.
+ # Force logits during eval, but restore the user's prior setting after so an explicit
+ # UNSLOTH_RETURN_LOGITS="1" is not silently turned off.
_old_return_logits = os.environ.get("UNSLOTH_RETURN_LOGITS", "0")
os.environ["UNSLOTH_RETURN_LOGITS"] = "1"
with torch.no_grad():
@@ -317,7 +309,6 @@ def PatchRL(FastLanguageModel):
logits = tuple(v for k, v in outputs.items() if k not in ignore_keys)
else:
logits = outputs
- # TODO: this needs to be fixed and made cleaner later.
if self.args.past_index >= 0:
self._past = outputs[self.args.past_index - 1]
os.environ["UNSLOTH_RETURN_LOGITS"] = _old_return_logits
@@ -570,12 +561,10 @@ pass
'''
-# Marks a config class generated by Unsloth. The generated class is renamed to
-# the TRL name it stands in for (see _patch_config_pickle_identity), so the
-# "already patched" checks cannot go by __name__ alone any more.
+# Marks an Unsloth-generated config class. It is renamed to the TRL name it stands in for,
+# so the "already patched" checks cannot go by __name__ alone.
_UNSLOTH_PATCHED_CONFIG_FLAG = "_unsloth_patched_rl_config"
-# Set on the PRISTINE config class, pointing at the Unsloth subclass that has
-# taken over its module attribute.
+# Set on the PRISTINE config class, pointing at the Unsloth subclass that has taken over its module attribute.
_UNSLOTH_CONFIG_PICKLE_TARGET = "_unsloth_config_pickle_target"
@@ -638,14 +627,12 @@ def _patch_config_pickle_identity(pristine_config, patched_config):
home_module = importlib.import_module(home_module_name)
except Exception:
return
- # The rebinding above walks the trl.trainer.<x>_trainer -> <x>_config naming
- # convention. The pristine class's own __module__ is what pickle consults,
- # which is the same module for every stock TRL layout and the correct one for
- # the trl.experimental wrappers the convention misses.
+ # The rebinding above follows the trl.trainer.<x>_trainer -> <x>_config convention, but
+ # pickle consults the pristine class's own __module__, which is also right for
+ # trl.experimental wrappers the convention misses.
current = getattr(home_module, qualname, None)
if current is not pristine_config and current is not patched_config:
- # Something else owns the name. Renaming the patched class now would
- # only move the failure, so leave both classes as they are.
+ # Something else owns the name; renaming the patched class would only move the failure.
return
try:
setattr(home_module, qualname, patched_config)
@@ -763,13 +750,9 @@ def _column_names(dataset):
source = dataset
try:
iterator = iter(dataset)
- # `iterator is dataset` catches a bare generator and misses an
- # `IterableDataset` whose `__iter__` hands back one stored generator:
- # that is not the dataset, but it is still the only pass there is. Two
- # `iter()` calls returning the same object covers both, and a
- # `datasets.IterableDataset` restarts, so it answers False and is
- # rewound rather than chained. Both calls happen before anything is
- # read.
+ # `iterator is dataset` misses an IterableDataset whose __iter__ returns one stored
+ # generator. Two iter() calls giving the same object catches both; a datasets.IterableDataset
+ # restarts, so it answers False and is rewound.
single_pass = iterator is dataset or iterator is iter(dataset)
row = next(iterator, None)
if single_pass and row is not None:
@@ -779,11 +762,8 @@ def _column_names(dataset):
row = None
if isinstance(row, dict):
names.update(row.keys())
- # The row too: it was read either way, and on a one-shot stream it is the
- # ONLY row anything may look at. Without it `_sliceable_per_token` had no
- # widths to compare and fell back to `input_ids` alone, leaving `labels`
- # and `attention_mask` at their overlength size -- rows whose supervision
- # no longer lines up with the tokens they describe.
+ # Keep the probed row: on a one-shot stream it is the only row anything may see. Without it
+ # _sliceable_per_token had no widths and cut input_ids alone, leaving labels overlength.
return tuple(names), source, (row if isinstance(row, dict) else None)
@@ -804,13 +784,9 @@ class _CappedBase:
self._per_token = tuple(per_token)
def _slice(self, row):
- # Per value, per row, exactly as the `map` path does it.
- # `_sliceable_per_token` judges alignment from ONE probed row, and an
- # optional column that is a list there can be None -- or a different
- # width -- further in. Slicing that raised inside the dataloader, which
- # is a failure the caller would not have had without the cap. Only
- # `input_ids` is cut unconditionally: it is the column the cap exists
- # for, and the width every other one is measured against.
+ # Per value, per row, like the map path: _sliceable_per_token judges from ONE row, and an
+ # optional column that is a list there can be None later, which raised in the dataloader.
+ # Only input_ids is cut unconditionally.
if not isinstance(row, dict):
return row
try:
@@ -850,11 +826,9 @@ class _CappedBase:
yield cut
def __getattr__(self, attribute):
- # Anything else the trainer asks for (column_names, features, ...) is
- # the wrapped split's own answer. Never a dunder, and never our own
- # state: a DataLoader worker pickles the split, and `__setstate__`
- # looked up before `__init__` has run would recurse on `_inner`
- # forever.
+ # Everything else (column_names, features, ...) is the wrapped split's answer. Never a dunder
+ # nor our own state: a DataLoader worker pickles the split, and __setstate__ before __init__
+ # would recurse on _inner forever.
inner = self.__dict__.get("_inner")
if inner is None or attribute.startswith("__"):
raise AttributeError(attribute)
@@ -870,11 +844,8 @@ class _CappedRows(_CappedBase):
def __init__(self, inner, cut, supervision, per_token):
super().__init__(inner, cut, supervision, per_token)
- # No supervision columns means `_keep` is True for every row, so the
- # index would be `range(len(inner))` -- and building it read and
- # transformed every item, which for a `with_transform` split is a whole
- # extra tokenization pass before the dataloader can even start, plus an
- # O(n) list. Stay lazy and let `__getitem__` map straight through.
+ # With no supervision columns _keep is True for every row, so building the index would
+ # transform every item -- a whole extra tokenization pass for a with_transform split.
self._index = (
None
if not self._supervision
@@ -927,10 +898,8 @@ def _is_stream(dataset):
_SCAN_ROWS = 1024
-# A producer that truncates every row itself can say so and be believed without a
-# scan: scanning a lazily-tokenizing split (`with_transform`) would tokenize every
-# row in `__init__`, the exact eager pass it exists to avoid. Only a cap at or
-# below the enforced one counts.
+# Believe a producer's own truncation claim: scanning a with_transform split tokenizes every
+# row in __init__, the eager pass it avoids. Only a cap at or below the enforced one counts.
_TRUNCATION_ATTESTATION_ATTR = "_unsloth_truncated_to"
@@ -1071,8 +1040,7 @@ def _first_row_without_consuming(dataset):
return dataset[0]
except Exception:
return None
- # A stream whose `iter` hands back the same exhausting generator cannot
- # spare a row, and nothing here chains it back.
+ # A stream whose iter hands back the same exhausting generator cannot spare a row, and nothing here chains it back.
probe = iter(dataset)
if probe is dataset or probe is iter(dataset):
return None
@@ -1118,35 +1086,25 @@ def _sliceable_per_token(
A row that cannot be read without costing it leaves `input_ids` alone: the
column the cap exists for, and the one every other is measured against.
"""
- # `input_ids` first, and the rest in a fixed order: `names` arrives as a set,
- # and the `map` path reads the width off `input_ids` as it walks this list,
- # so a run that happened to order `labels` ahead of it sliced the labels
- # without ever comparing them to anything.
+ # input_ids first, then a fixed order: `names` is a set, and the map path reads the width off
+ # input_ids as it walks this list, so labels first sliced them against nothing.
known = [c for c in _PER_TOKEN if c in names]
- # A user-defined per-token field -- `loss_mask`, `token_weights`, a custom
- # model input -- is not in the allow-list, so it kept its full length while
- # `input_ids` was cut. A custom collator (the case where padding-free is off
- # and this wrapper still runs) then gets mismatched lengths and either fails
- # or supervises the wrong tokens. Judge those the way the row check below
- # judges everything else, by alignment, but only accept a flat vector of
- # scalars: that is what a per-token field is, and it keeps `messages` or a
- # column of strings that happens to be as long as the row out of the slice.
+ # A custom per-token field (loss_mask, token_weights) is not in the allow-list, so it stayed
+ # full length while input_ids was cut and a custom collator got mismatched lengths. Judge by
+ # alignment, but only a flat vector of scalars, keeping `messages` out of the slice.
custom = sorted(c for c in names if c not in _PER_TOKEN)
per_token = known + custom
if len(known) < 2 and not custom:
return known
- # `probed` is the row `_column_names` already read. Preferring it is what
- # lets a one-shot stream align every per-token column: reading another row
- # would cost the caller that example, so without it this fell back to
- # `input_ids` alone and left the masks and labels overlength.
+ # `probed` is the row _column_names already read. Preferring it is what lets a one-shot stream align
+ # every per-token column: reading another row would cost the caller that example.
row = probed if isinstance(probed, dict) else _first_row_without_consuming(dataset)
if not isinstance(row, dict):
return ["input_ids"] if "input_ids" in names else []
try:
width = len(row.get("input_ids"))
except Exception:
- # Nothing to measure against, so a custom column has no evidence at all
- # behind it. Fall back to the named ones only.
+ # Nothing to measure against, so a custom column has no evidence behind it; fall back to the named ones only.
return known
kept = []
for name in per_token:
@@ -1154,11 +1112,8 @@ def _sliceable_per_token(
if name in custom and not _is_token_vector(value, width):
continue
try:
- # As long as `input_ids`, which is what makes the FIRST axis the
- # token axis: a `[seq_len, channels]` field slices correctly there,
- # and a channel-major one (`position_ids` under mrope is
- # `[3, seq_len]`) fails this check and is left alone, which is the
- # shape the nested test used to be aimed at.
+ # As long as input_ids, which makes the FIRST axis the token axis: [seq_len, channels] slices
+ # right, and a channel-major position_ids ([3, seq_len] under mrope) fails and is left alone.
if len(value) != width:
continue
except Exception:
@@ -1249,9 +1204,8 @@ def _pin_pristine_sft_loss_type(config_cls):
else:
return False
field.default = "nll"
- # The class attribute is the other copy of the default: dataclasses seeds it
- # at class creation and a later subclass reads the field, so leave the two
- # agreeing rather than half-patched.
+ # The class attribute is the other copy of the default: dataclasses seeds it at class creation and a
+ # later subclass reads the field, so leave the two agreeing rather than half-patched.
setattr(config_cls, "loss_type", "nll")
return True
@@ -1294,14 +1248,9 @@ def _wrap_sft_evaluate_cap(trainer_cls):
does the same and the two must agree.
"""
columns = ["labels"] if "labels" in names else []
- # The TRAINER's resolved value first. It is the one the collator uses,
- # and it is present whenever TRL resolved it -- including the cases the
- # generated block never runs in: a TRL whose guard did not match, or
- # padding-free off from the start. Falling straight through to this
- # split's own schema then read False off a late pre-tokenized split
- # carrying only `input_ids` and `completion_mask`, kept the rows whose
- # completion was cut away entirely, and the collator turned them into
- # all -100, i.e. a NaN eval loss.
+ # The TRAINER's resolved value first: the collator uses it, and it is set whenever TRL
+ # resolved it. The split's own schema read False off a pre-tokenized eval split with only
+ # input_ids + completion_mask, so cut-away rows survived as all -100, i.e. a NaN eval loss.
only = getattr(args, "_unsloth_resolved_completion_only", None)
if only is None:
only = getattr(args, "_unsloth_completion_only_loss", None)
@@ -1322,68 +1271,34 @@ def _wrap_sft_evaluate_cap(trainer_cls):
drop_unsupervised = True,
packs_late = False,
):
- # `evaluate()` caps the split, stores it on the trainer, and Transformers
- # then calls `get_eval_dataloader`, which this module also wraps -- so the
- # paired wrappers reach here twice for one call. Re-capping is a no-op at
- # best, and over a one-shot stream it is destructive: `_column_names` and
- # `_sliceable_per_token` each read a row to probe it, and `_CappedStream`
- # hands out a fresh generator over the SAME exhausted source rather than
- # rewinding, so the second pass eats the first rows instead of replaying
- # them. Our own wrapper, capped the same way, is already the answer.
+ # evaluate() caps the split and Transformers then calls get_eval_dataloader, which is also
+ # wrapped, so both reach here in one call. Re-capping is destructive over a one-shot stream:
+ # each probe reads a row and _CappedStream re-opens the SAME exhausted source.
if _cap_still_holds(dataset, cap, drop_unsupervised):
return dataset
names, dataset, probed = _column_names(dataset)
if "input_ids" not in names:
- # No tokens here yet, so there is nothing to cut. TRL does not
- # tokenize a late split either -- `_prepare_dataset` runs only from
- # `__init__` -- but that is its own gap and not one a truncation can
- # close.
+ # No tokens here yet, so there is nothing to cut. TRL does not tokenize a late split either
+ # (_prepare_dataset runs only from __init__), but that is its own gap.
return _mark_capped(dataset, cap, drop_unsupervised)
- # `eval_packing` is consulted only where the packer can actually reach the
- # split, which is why `packs_late` is passed in per entry point rather than
- # read here. Up to TRL 1.6 nothing packs a late split: `evaluate()` and
- # `predict()` are the base Trainer's, they call `get_eval_dataloader` /
- # `get_test_dataloader` straight through, and `_prepare_dataset` never runs
- # again, so skipping the cap there handed the collator the raw overlength
- # rows with `max_length` already cleared. From TRL 1.7.0 the opposite is
- # true for `evaluate` alone: it prepares the split itself under
- # `eval_packing`, and every strategy owns the overflow -- `wrapped`
- # concatenates the token stream before chunking, `bfd_split` splits an
- # overlength example into more chunks -- so cutting rows at the cap first
- # throws that away. `predict` and the two dataloader builders stay the base
- # Trainer's on every TRL, so they always cap.
+ # eval_packing is consulted only where the packer reaches the split, hence packs_late per
+ # entry point. Up to TRL 1.6 nothing packs a late split; from 1.7.0 `evaluate` prepares it
+ # itself and the strategy owns the overflow, so capping rows first throws that away.
if packs_late and _eval_packing_on(args):
- # Left FOR the packer, so it is not capped -- and must not be marked
- # as if it were. `packs_late` is read off the class, but TRL only
- # actually prepares a split that was PASSED to `evaluate`:
- # if not self._skip_prepare_dataset and eval_dataset is not None
- # and not isinstance(eval_dataset, str):
- # (trl 1.9.2 sft_trainer.py:1675, same line since 1.7.0). A split
- # stored on the trainer, one named by a string key, and any run with
- # `skip_prepare_dataset` therefore reach `get_eval_dataloader` as
- # this very object with no packer having touched it -- and that
- # builder, which does cap, then read the mark through
- # `_cap_still_holds` and handed the collator the overlength rows.
- # Where TRL really does pack, it hands back a NEW object, so nothing
- # is lost by leaving this one unmarked.
+ # Left FOR the packer, so it is not capped and must not be marked as if it were. TRL only
+ # prepares a split PASSED to evaluate, so a stored split or string key arrives here untouched;
+ # where it really packs, it hands back a NEW object.
return dataset
- # A packed split carries document lengths, not tokens. Slicing `input_ids`
- # under a `seq_lengths` that still describes the longer row makes
- # padding-free build position ids for tokens the row no longer has, which
- # is worse than not cutting: the construction-time cap refuses this shape
- # too.
+ # A packed split carries document lengths, not tokens: slicing input_ids under a seq_lengths that
+ # still describes the longer row builds position ids for tokens the row no longer has.
if "seq_lengths" in names:
return _mark_capped(dataset, cap, drop_unsupervised)
try:
- # TRL slices [-max_length:] for `keep_end`, and so does the
- # construction-time cap. Always keeping the prefix evaluates the wrong
- # half of every long row for callers whose completion sits at the tail.
+ # TRL slices [-max_length:] for keep_end, and so does the construction-time cap; always keeping the
+ # prefix evaluates the wrong half of every long row.
mode = getattr(args, "truncation_mode", "keep_start")
- # keep_start and keep_end are the only two slices there are, and a
- # third value silently became keep_start here, cutting every late
- # row from the side the caller asked us not to. The construction
- # path already refuses those; this one has to as well, and refusing
- # means handing the split back untouched so the caller still has it.
+ # keep_start and keep_end are the only two slices there are, and a third value silently became
+ # keep_start here. Refusing means handing the split back untouched so the caller still has it.
if mode not in ("keep_start", "keep_end"):
print(
f"Unsloth: `truncation_mode = {mode}` is not one of "
@@ -1392,39 +1307,25 @@ def _wrap_sft_evaluate_cap(trainer_cls):
return dataset
cut = slice(-cap, None) if mode == "keep_end" else slice(None, cap)
per_token = _sliceable_per_token(dataset, names, cap, probed)
- # Never on the predict path. Dropping rows is right for a loss, which
- # is meaningless over a row with no supervised token, and wrong for
- # `predict`, whose contract is one prediction per row IN ORDER: a
- # caller zipping the output back onto its own dataframe would silently
- # get a shorter, shifted column.
+ # Never on the predict path: dropping rows is right for a loss and wrong for predict, whose contract
+ # is one prediction per row IN ORDER.
supervision = _supervision_columns(args, names) if drop_unsupervised else []
- # A stream has no length and cannot be rewound, so there is no prefix
- # scan to skip the work with, and indexing it does not even fail
- # loudly: on datasets 4.x `dataset[0]` reads 0 as a COLUMN name and
- # returns an IterableColumn, whose len() then raised TypeError into
- # the catch below and handed the eval call its uncapped stream back.
- # map() is lazy and applies to every row it will ever yield.
+ # A stream has no length, cannot be rewound, and on datasets 4.x dataset[0] reads 0 as a
+ # COLUMN name rather than failing. Use map(), which is lazy over every row it will yield.
overlength = True
if not _is_stream(dataset):
try:
overlength = max(len(r) for r in dataset["input_ids"]) > cap
except Exception:
- # The scan only exists to skip a pointless map. A split with no
- # column access (a custom map-style dataset) cannot answer it, and
- # that is not a reason to hand it back uncapped.
+ # The scan only exists to skip a pointless map; a split with no column access cannot answer it, and
+ # that is no reason to hand it back uncapped.
pass
- # A split already under the cap still goes through the supervision
- # filter below. Being short is not the same as being supervised: a row
- # whose labels are all -100, or whose active mask is all zeros, is a
- # NaN loss whether or not anything had to be cut off it, and the
- # construction-time cap filters those rows unconditionally too.
- # Returning early on the length alone was the one way the two paths
- # disagreed.
+ # A split already under the cap still goes through the supervision filter below: being short is not
+ # the same as being supervised, and a row whose labels are all -100 is a NaN loss either way.
if not overlength and not supervision:
return _mark_capped(dataset, cap, drop_unsupervised)
- # A split we cannot rewrite is capped on read instead. `map` belongs to
- # `datasets`, and a `with_transform` split has it but recreates its rows
- # on every read, so mapping writes a table nobody reads.
+ # A split we cannot rewrite is capped on read instead: map belongs to datasets, and a with_transform
+ # split has it but recreates its rows on every read, so mapping writes a table nobody reads.
transform = str((getattr(dataset, "format", None) or {}).get("type", "")).lower()
if (
not hasattr(dataset, "map")
@@ -1444,12 +1345,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):
_cut = cut,
_cols = tuple(per_token),
):
- # Per value, not per column. `_sliceable_per_token` judges by ONE
- # row, so an optional column that is a list there and None (or a
- # different width) three rows later raised inside `map` -- and
- # the broad catch below then returned the UNCAPPED split, losing
- # the `input_ids` truncation too. `input_ids` is always cut; an
- # auxiliary value that cannot take the same slice is left alone.
+ # Per value, not per column: _sliceable_per_token judges by ONE row, so an optional column
+ # that is None three rows later raised inside map and the catch returned the UNCAPPED split.
out = {}
width = None
for name in _cols:
@@ -1466,17 +1363,9 @@ def _wrap_sft_evaluate_cap(trainer_cls):
return out
new = dataset if not overlength else dataset.map(_slice_row)
- # Truncating can leave a row with every label at -100, or a mask that is
- # now all zeros, which the collator turns into all -100. A batch of
- # those has no supervised token and reports a NaN loss. TRL filters them
- # right after its own truncation; here `args.max_length` is already None
- # so TRL does not, and the construction-time cap filters them for the
- # splits it saw. Masks are applied one after another onto the same
- # labels, so what survives is their intersection.
- # One intersection over labels AND every active mask, not a filter each:
- # the collator applies the masks ONTO the labels, so a row whose only
- # supervised label sits where the mask is zero passes both filters
- # separately and still goes out all -100.
+ # A truncated row can end all -100, or with an all-zero mask the collator makes all -100, and
+ # such a batch reports a NaN loss. Intersect labels AND every active mask, not one filter
+ # each: masks are applied ONTO the labels, so separate filters still pass an all -100 row.
if supervision:
kept = new.filter(
lambda e, c = tuple(supervision): any(
@@ -1484,9 +1373,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):
for v in zip(*[e[n] for n in c])
)
)
- # Hand back the caller's own split when the filter dropped
- # nothing: a copy of an unchanged dataset is a new object for the
- # trainer to cache and reload for no reason at all.
+ # Hand back the caller's own split when the filter dropped nothing: a copy of an unchanged dataset
+ # is a new object for the trainer to cache and reload for no reason.
try:
new = new if len(kept) == len(new) else kept
except Exception:
@@ -1529,8 +1417,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):
column each time. Keep the answer, keyed on the split object and holding a
reference to it, so a later split cannot inherit its `id()`.
"""
- # Carried onto args because `_cap` only ever sees those. `is not None`
- # rather than truthiness: False is TRL's answer just as much as True.
+ # Carried onto args because _cap only ever sees those. `is not None` rather than truthiness: False
+ # is TRL's answer just as much as True.
resolved = getattr(trainer, "completion_only_loss", None)
if resolved is not None:
try:
@@ -1547,17 +1435,14 @@ def _wrap_sft_evaluate_cap(trainer_cls):
trainer._unsloth_eval_cap_memo = memo
except Exception:
return _cap(dataset, cap, trainer.args, drop_unsupervised, packs_late)
- # `truncation_mode` shapes the SLICE, so it belongs in the key: without
- # it, evaluating once with keep_start and again with keep_end handed
- # back the cached prefixes for both.
+ # truncation_mode shapes the SLICE, so it belongs in the key: without it, evaluating once with
+ # keep_start and again with keep_end handed back the cached prefixes for both.
key = (
id(dataset),
drop_unsupervised,
getattr(trainer.args, "truncation_mode", "keep_start"),
- # `evaluate` and `get_eval_dataloader` share `drop_unsupervised` and
- # see the same object in one call, but only the first may skip the cut
- # under `eval_packing`, so without this the second reused the first's
- # answer.
+ # evaluate and get_eval_dataloader share drop_unsupervised and see the same object in one call, but
+ # only the first may skip the cut under eval_packing.
packs_late,
)
seen = memo.get(key)
@@ -1566,13 +1451,9 @@ def _wrap_sft_evaluate_cap(trainer_cls):
return seen[2]
capped = _cap(dataset, cap, trainer.args, drop_unsupervised, packs_late)
memo[key] = (dataset, cap, capped, token)
- # Bounded, because every entry pins BOTH the original split and the
- # capped copy for the trainer's whole lifetime -- deliberately, so a
- # later split cannot inherit a freed `id()`. A caller that builds a
- # fresh validation subset each epoch therefore accumulated Arrow tables
- # until the host ran out. Evicting the least recently used costs nothing
- # real: a run evaluates over a handful of splits, which is what the
- # bound leaves room for.
+ # Bounded: every entry pins both the original split and the capped copy for the trainer's
+ # life, so a later split cannot inherit a freed id(). A fresh subset per epoch otherwise
+ # accumulated Arrow tables until the host ran out.
while len(memo) > _EVAL_CAP_MEMO_MAX:
memo.pop(next(iter(memo)))
return capped
@@ -1584,20 +1465,14 @@ def _wrap_sft_evaluate_cap(trainer_cls):
drop_unsupervised = True,
packs_late = False,
):
- # `evaluate(eval_dataset = "validation")` is the supported way to pick one
- # split out of a stored dict: `get_eval_dataloader` resolves it as
- # `self.eval_dataset[eval_dataset]`. Capping the KEY is a no-op, so the
- # split it names reached the collator uncapped. Cap it where it is stored
- # and hand the key straight back.
+ # evaluate(eval_dataset = "validation") picks one split out of a stored dict, and capping the
+ # KEY is a no-op, so the split it names reached the collator uncapped.
if isinstance(given, str):
stored = getattr(trainer, "eval_dataset", None)
if isinstance(stored, dict) and given in stored:
capped = _cap_cached(trainer, stored[given], cap, drop_unsupervised, packs_late)
- # Staged for the caller to swap in and OUT, never written
- # through. Overwriting `stored[given]` destroyed the uncapped
- # original, so a later `truncation_mode = "keep_end"` -- which
- # the memo key now distinguishes on purpose -- could only re-cap
- # the saved prefix and never produce the suffix asked for.
+ # Staged for the caller to swap in and OUT: overwriting stored[given] destroyed the uncapped
+ # original, so a later truncation_mode = "keep_end" could only re-cap the saved prefix.
if capped is not stored[given]:
trainer._unsloth_pending_split_swap = (stored, given, capped)
return given
@@ -1615,14 +1490,9 @@ def _wrap_sft_evaluate_cap(trainer_cls):
def wrapped(self, *args, **kwargs):
cap = getattr(getattr(self, "args", None), "max_seq_length", None)
retained = getattr(getattr(self, "args", None), "max_length", None)
- # A retained `max_length` is NOT proof the cap is enforced
- # downstream. It is exactly what the construction block leaves
- # behind when it turns padding-free OFF instead of clearing the
- # cap, and TRL's collator does not truncate rows that already carry
- # `input_ids`. `_prepare_dataset` runs only from `__init__`, so a
- # split handed over later is never prepared either: skipping on a
- # retained `max_length` let an overlength late split reach the model
- # with nothing enforcing anything. Cap to whichever value is set.
+ # A retained max_length does not prove the cap is enforced: it is what the construction block
+ # leaves when it turns padding-free OFF, TRL's collator never truncates rows carrying
+ # input_ids, and _prepare_dataset runs only from __init__.
if retained is not None:
cap = retained
if not cap:
@@ -1638,8 +1508,8 @@ def _wrap_sft_evaluate_cap(trainer_cls):
swap = getattr(self, "_unsloth_pending_split_swap", None)
if swap is None:
return original(self, *args, **kwargs)
- # A named split: swap the capped copy in for this call only, so
- # the caller keeps the uncapped original for the next mode.
+ # A named split: swap the capped copy in for this call only, so the caller keeps the uncapped
+ # original for the next mode.
container, key, replacement = swap
self._unsloth_pending_split_swap = None
previous = container[key]
@@ -1648,22 +1518,16 @@ def _wrap_sft_evaluate_cap(trainer_cls):
return original(self, *args, **kwargs)
finally:
container[key] = previous
- # `evaluate()` with no split passed falls back to the one stored on
- # the trainer, and a caller can install or replace that after
- # construction -- which is exactly where the constructor's cap can no
- # longer see it. Every eval during training comes through here too.
+ # evaluate() with no split falls back to the one stored on the trainer, which a caller can install
+ # after construction, where the constructor's cap can no longer see it.
stored = getattr(self, keyword, None) if keyword == "eval_dataset" else None
if stored is None:
return original(self, *args, **kwargs)
capped = _cap_splits(self, stored, cap, drop_unsupervised, packs_late)
if capped is stored:
return original(self, *args, **kwargs)
- # Swapped onto the trainer rather than passed down as an argument,
- # because Trainer.evaluate recurses over a dict of splits by NAME when
- # nothing was passed:
- # eval_dataset = _eval_dataset if override else eval_dataset_name
- # Passing the dict makes that an override, which changes what
- # `get_eval_dataloader` is handed and what it caches.
+ # Swapped onto the trainer rather than passed down: Trainer.evaluate recurses over a dict of
+ # splits by NAME when nothing was passed, and passing the dict makes that an override.
setattr(self, keyword, capped)
try:
return original(self, *args, **kwargs)
@@ -1673,17 +1537,11 @@ def _wrap_sft_evaluate_cap(trainer_cls):
wrapped._unsloth_eval_cap_wrapped = True
return wrapped
- # `predict` keeps every row: its contract is one prediction per row in order.
- #
- # The two dataloader builders are public API and neither goes through
- # `evaluate`/`predict`, so a caller doing `get_eval_dataloader(late)` reached
- # the padding-free collator with `args.max_length` already cleared and
- # nothing capping the split. Same wrapper, same argument position; the
- # `drop_unsupervised` split follows the method it serves, since
- # `get_test_dataloader` feeds `predict`.
- # Only `evaluate` can hand its split to TRL's own prep, and only from 1.7.0.
- # Read once, before anything is wrapped, so the probe sees TRL's method rather
- # than ours.
+ # predict keeps every row: one prediction per row, in order. The two dataloader builders are
+ # public API and bypass evaluate/predict, so get_eval_dataloader(late) met the padding-free
+ # collator with max_length already cleared and nothing capping the split.
+ # Only evaluate can hand its split to TRL's own prep, and only from 1.7.0. Read once, before
+ # anything is wrapped, so the probe sees TRL's method rather than ours.
prepares_late = _trl_prepares_late_evals(trainer_cls)
for name, keyword, drop_unsupervised, packs_late in (
("evaluate", "eval_dataset", True, prepares_late),
@@ -1700,17 +1558,14 @@ def _wrap_sft_evaluate_cap(trainer_cls):
_UNSLOTH_RETURN_HIDDEN_STATES_SUPPORT_MARKER = "__UNSLOTH_SUPPORTS_RETURN_HIDDEN_STATES__"
_UNSLOTH_GRPO_HIDDEN_STATES_WRAPPED_ATTR = "_unsloth_grpo_hidden_states_forward_wrapped"
_UNSLOTH_GRPO_HIDDEN_STATES_WARNING_ATTR = "_unsloth_grpo_hidden_states_warning_issued"
-# Whether the *most recent* forward handed back real logits instead of hidden
-# states. The warning attribute above is warn-once bookkeeping and is never
-# cleared, so it answers "ever degraded", not "degraded on the call the caller
-# is about to dispatch on".
+# Whether the MOST RECENT forward handed back real logits instead of hidden states: the warning
+# attribute above is warn-once bookkeeping and is never cleared, so it answers "ever degraded".
_UNSLOTH_GRPO_HIDDEN_STATES_DEGRADED_ATTR = "_unsloth_grpo_hidden_states_degraded"
def _module_returns_logits(module):
- # get_output_embeddings() is None on the decoder bodies (Qwen2Model, ...) and the
- # head module on the *ForCausalLM wrappers, so it finds the head owner by behaviour
- # rather than by a model-name list.
+ # get_output_embeddings() is None on the decoder bodies and the head module on the *ForCausalLM
+ # wrappers, so it finds the head owner by behaviour rather than by a model-name list.
if module is None:
return False
get_output_embeddings = getattr(module, "get_output_embeddings", None)
@@ -1734,10 +1589,9 @@ def _grpo_hidden_states_wrap_target(model):
child = getattr(model, attr, None)
if child is None or child is model or not hasattr(child, "forward"):
continue
- # Descend only into an adapter that still owns the head. A bare *ForCausalLM
- # (what TRL builds GRPO's ref_model as) also has a `.model`, but that is its
- # decoder body, and wrapping it leaves the head above running untouched: the
- # fallback becomes a silent no-op returning [B, T, vocab], not [B, T, hidden].
+ # Descend only into an adapter that still owns the head: a bare *ForCausalLM (TRL's GRPO
+ # ref_model) also has .model, but that is its decoder body, so wrapping it returns
+ # [B, T, vocab] and the fallback is a silent no-op.
if not _module_returns_logits(child):
continue
return child
@@ -1813,12 +1667,9 @@ def _get_num_logits_to_keep(forward_signature, args, kwargs):
def _warn_grpo_hidden_states_fallback_once(model, message):
- # The degradation flag is per call: whether the tensor this forward is about
- # to return is real logits. Degradation is not a property of the model -- a
- # forward that splats **kwargs into a sub-module only reached by some inputs
- # (a vision tower, say) raises for the batches that reach it and succeeds for
- # the rest -- so a sticky flag would keep routing later, genuine hidden
- # states through the raw-logits helper, skipping the lm_head matmul.
+ # The degradation flag is per call: a forward that splats **kwargs into a sub-module raises
+ # only for the batches that reach it, so a sticky flag would send real hidden states through
+ # the raw-logits helper.
setattr(model, _UNSLOTH_GRPO_HIDDEN_STATES_DEGRADED_ATTR, True)
if getattr(model, _UNSLOTH_GRPO_HIDDEN_STATES_WARNING_ATTR, False):
return
@@ -1861,9 +1712,8 @@ def _minimise_logits_kwarg(forward_signature, args, forward_kwargs):
bound = forward_signature.bind_partial(*args, **forward_kwargs)
except TypeError:
return None
- # A forward given `labels` positionally lands it in `bound.arguments` and
- # never in `forward_kwargs`, so the lookup above cannot see it, and the loss
- # the model then computes for itself would be one position wide.
+ # A forward given labels positionally lands it in bound.arguments and never in forward_kwargs, so
+ # the lookup above cannot see it and the loss the model computes would be one position wide.
if bound.arguments.get("labels") is not None:
return None
accepts_var_keyword = any(
@@ -1874,13 +1724,8 @@ def _minimise_logits_kwarg(forward_signature, args, forward_kwargs):
declared = name in forward_signature.parameters
if not declared and not accepts_var_keyword:
continue
- # Passing it positionally already and then also by keyword is a
- # TypeError. Give up entirely rather than moving to the next name: a
- # caller who bound a width positionally has said what it wants, and
- # reaching for the OTHER spelling would either fight that width or, on a
- # forward that only understands the one it was handed and swallows the
- # rest in `**kwargs`, be accepted and ignored -- no saving, and a
- # non-None return arms the re-run below for nothing.
+ # Positionally and by keyword is a TypeError. Give up rather than try the OTHER spelling,
+ # which would fight the caller's width or be swallowed by **kwargs and silently ignored.
if name in bound.arguments and name not in forward_kwargs:
return None
forward_kwargs[name] = 1
@@ -1915,8 +1760,8 @@ def _drop_spare_hidden_states(outputs):
elif hasattr(outputs, "hidden_states"):
outputs.hidden_states = None
except Exception:
- # A frozen or exotic output object is not worth failing the step over;
- # the caller has already taken the layer it needs.
+ # A frozen or exotic output object is not worth failing the step over; the caller has already taken
+ # the layer it needs.
logger.debug(
"Unsloth: could not drop spare GRPO hidden states.",
exc_info = True,
@@ -1951,10 +1796,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):
model_name = type(target_model).__name__
def wrapped_forward(*args, **kwargs):
- # accelerate's extract_model_from_parallel(keep_fp32_wrapper = False), which the
- # GRPO loop calls every step, rebinds an instance-level forward as
- # MethodType(forward, model), so the module arrives as a leading positional
- # argument. `original_forward` is already bound, so drop it.
+ # accelerate's extract_model_from_parallel(keep_fp32_wrapper = False), called every GRPO step,
+ # rebinds the forward as MethodType, so the module arrives as a leading positional argument;
+ # original_forward is already bound, so drop it.
while len(args) != 0 and args[0] is target_model:
args = args[1:]
if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") != "1":
@@ -1962,10 +1806,8 @@ def _install_grpo_hidden_states_forward_wrapper(model):
setattr(target_model, _UNSLOTH_GRPO_HIDDEN_STATES_DEGRADED_ATTR, True)
return original_forward(*args, **kwargs)
- # copy: _drop_forward_kwargs_consumed_positionally hands `kwargs` straight
- # back when there is nothing to drop, so mutating it below would poison the
- # caller's dict and make the fallback calls further down re-send the very
- # kwargs the model just rejected.
+ # Copy: _drop_forward_kwargs_consumed_positionally returns kwargs unchanged when there is
+ # nothing to drop, so mutating it would poison the caller's dict for the fallback calls.
forward_kwargs = dict(
_drop_forward_kwargs_consumed_positionally(forward_signature, args, kwargs)
)
@@ -1988,6 +1830,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):
)
return original_forward(*args, **kwargs)
+ # TRL 0.26+: Config may be in a separate *_config.py module
+ # Thin wrapper fallback: walk the Trainer's MRO to find Config in the real implementation module (e.g.,
+ # trl.experimental.bco)
try:
outputs = original_forward(*args, **forward_kwargs)
except TypeError as error:
@@ -1996,12 +1841,8 @@ def _install_grpo_hidden_states_forward_wrapper(model):
if not rejected_hidden_states(message):
raise
return forward_without_hidden_states()
- # The signature advertised the parameter but the forward refuses
- # the value. Retry without our minimisation rather than losing
- # hidden states entirely over an optimisation -- and send the retry
- # through the same fallback, since a forward that splats **kwargs
- # into a sub-module can refuse the logits limiter and the hidden
- # states one after the other.
+ # The signature advertised the parameter but the forward refuses the value: retry without the
+ # minimisation, through the same fallback, rather than lose hidden states over it.
forward_kwargs.pop(logits_kwarg, None)
logits_kwarg = None
try:
@@ -2019,12 +1860,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):
)
if logits_kwarg is None:
return outputs
- # `outputs.logits` is the return value now, and we asked for a single
- # position because we meant to throw the logits away. GRPO drops the
- # last one and slices the completion window out of what is left, so
- # one position leaves it nothing. Put the caller's own limit back --
- # absent means every position, which is what a model that ignores
- # `logits_to_keep` would have returned anyway -- and re-run.
+ # outputs.logits is the return value now, and one position was asked for only to throw the
+ # logits away; GRPO drops the last and slices the completion window, so restore the caller's
+ # own limit and re-run.
if logits_kwarg in kwargs:
forward_kwargs[logits_kwarg] = kwargs[logits_kwarg]
else:
@@ -2034,11 +1872,9 @@ def _install_grpo_hidden_states_forward_wrapper(model):
hidden_states = hidden_states[-1]
if num_logits_to_keep != 0:
hidden_states = hidden_states[:, -num_logits_to_keep:, :]
- # Only the last layer is ever read. Every earlier layer is still hanging
- # off `outputs`, and on a multi-GPU dispatch accelerate's
- # AlignDevicesHook.post_forward walks the whole returned object and
- # copies every tensor in it to the input device, so keeping them costs a
- # cross-device copy per layer as well as the memory.
+ # Only the last layer is read, and accelerate's AlignDevicesHook.post_forward copies every
+ # tensor in the returned object to the input device, so keeping the rest costs a cross-device
+ # copy per layer as well as the memory.
_drop_spare_hidden_states(outputs)
_note_grpo_hidden_states_success(target_model)
return _replace_outputs_logits(outputs, hidden_states)
@@ -2108,17 +1944,14 @@ def _backport_vision_dataset_gate(RLTrainer_source):
def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
- # Defensive wrapper: matches patch_trl_rl_trainers()'s try/except so
- # direct callers don't see exceptions from the impl on TRL versions
- # that rename or move classes (e.g. TRL 1.x trl.experimental).
+ # Defensive wrapper matching patch_trl_rl_trainers()'s try/except, so direct callers do not see
+ # exceptions from the impl on TRL versions that rename or move classes.
try:
return _patch_trl_rl_trainers_impl(trainer_file)
except Exception as e:
- # Warning, not info. The impl RETURNS for the benign case this swallow
- # exists for (a trainer this TRL does not ship), so anything reaching
- # here means the module imported and generation itself failed, and the
- # run silently falls back to trl's trainer, losing Unsloth's
- # compute_loss, bf16/fp16 fixup and dataset handling at once.
+ # Warning, not info: the impl RETURNS for the benign case, so reaching here means generation
+ # failed and the run silently falls back to trl's trainer, losing Unsloth's compute_loss,
+ # bf16/fp16 fixup and dataset handling.
logger.warning_once(
f"Unsloth: Could not build the patched trl.trainer.{trainer_file}, "
f"so training will use trl's own trainer instead: "
@@ -2128,7 +1961,6 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
- # Patch for vLLM and Unsloth PEFT
import trl
import trl.trainer
@@ -2138,7 +1970,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
logger.info(f"Unsloth: Could not import trl.trainer.{trainer_file}: {error}")
return
- # Get SFTTrainer and SFTConfig names
name = [
x
for x in dir(trainer)
@@ -2161,7 +1992,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
return
if len(config) != 1:
- # TRL 0.26+: Config may be in a separate *_config.py module
+ # TRL 0.26+: the Config may be in a separate *_config.py module, or reachable by walking the
+ # Trainer's MRO to the real implementation module (trl.experimental.bco).
config_module_name = trainer_file.replace("_trainer", "_config")
try:
config_mod = eval(f"trl.trainer.{config_module_name}")
@@ -2176,8 +2008,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
except Exception:
pass
if len(config) != 1 and len(name) == 1:
- # Thin wrapper fallback: walk the Trainer's MRO to find Config
- # in the real implementation module (e.g., trl.experimental.bco)
try:
_temp_cls = eval(f"trl.trainer.{trainer_file}.{name[0]}")
for _parent in _temp_cls.__mro__[1:]:
@@ -2204,7 +2034,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
return
- # Get SFTTrainer, SFTConfig
RLTrainer_name = name[0]
RLConfig_name = config[0]
try:
@@ -2218,7 +2047,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
try:
RLConfig = eval(f"trl.trainer.{trainer_file}.{RLConfig_name}")
except Exception:
- # TRL 0.26+: Config may be in a separate *_config.py module
+ # TRL 0.26+: the Config may be in a separate *_config.py module, or loadable from the parent trainer's module.
try:
config_module_name = trainer_file.replace("_trainer", "_config")
RLConfig = eval(f"trl.trainer.{config_module_name}.{RLConfig_name}")
@@ -2244,7 +2073,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
logger.info(f"Unsloth: Could not load {RLConfig_name}")
return
- # Check name
if RLTrainer.__name__.startswith("Unsloth"):
print(f"Unsloth: {RLTrainer.__name__} is already patched.")
return
@@ -2252,10 +2080,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
print(f"Unsloth: {RLConfig.__name__} is already patched.")
return
- # TRL 0.26+: Resolve thin wrappers to their experimental parent class.
- # Thin wrappers are deprecation shims in trl.trainer that just forward
- # *args/**kwargs to the real implementation in trl.experimental.
- # Only resolve if a parent class actually lives in a trl.experimental module.
+ # TRL 0.26+: resolve thin wrappers (trl.trainer shims forwarding to trl.experimental) to their
+ # parent class, and only when that parent really lives in a trl.experimental module.
_trainer_resolved_module = None
try:
_trainer_src = inspect.getsource(RLTrainer)
@@ -2287,14 +2113,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
_parent_mod = inspect.getmodule(_parent)
if _parent_mod is None:
continue
- # Only resolve to a parent that lives in trl.experimental
if "trl.experimental" in _parent_mod.__name__:
RLConfig = _parent
break
except Exception:
pass
- # Get old source
old_RLTrainer_source = inspect.getsource(RLTrainer)
old_RLConfig_source = inspect.getsource(RLConfig)
@@ -2307,7 +2131,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
# Fix _deprecate_arguments not getting imported so stop __ but not _
imports = [x for x in all_imports if not x.startswith("__")]
- # Get default arguments
EMPTY = inspect.Parameter.empty
processed = []
for RLobject in [RLTrainer, RLConfig]:
@@ -2345,11 +2168,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
)
- # Process RLTrainer first
arguments, call_args = processed[0]
RLTrainer_post = ""
- # Add tokenizer if not seen
if "tokenizer" not in parameters and "processing_class" in parameters:
arguments += f",\n{' ' * 8}tokenizer = None"
call_args = call_args.replace(
@@ -2357,7 +2178,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"processing_class = tokenizer if tokenizer is not None else processing_class",
)
- # Edit bf16, fp16 by checking model's dtype/torch_dtype directly
+ # Edit bf16, fp16 by checking the model's dtype/torch_dtype directly.
extra_args = ""
if "args" in call_args and "model" in call_args:
mixed_precision = (
@@ -2366,22 +2187,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"use_fp16 = getattr(args, 'fp16', False)\n"
"if type(use_fp16) is not bool: use_fp16 = False\n"
"force_float32 = False\n"
- # device-aware bf16 check (CUDA/XPU/HIP), so V100/T4 never pick bf16
- # but AMD/Intel are unaffected; fall back on older unsloth_zoo.
+ # Device-aware bf16 check (CUDA/XPU/HIP), so V100/T4 never pick bf16 while AMD/Intel are unaffected;
+ # fall back on older unsloth_zoo.
"try:\n"
" from unsloth_zoo.device_type import device_is_bf16_supported as _bf16_supported\n"
"except Exception:\n"
" _bf16_supported = torch.cuda.is_bf16_supported\n"
- # FORCE_FLOAT32 models (Gemma3, gpt_oss, ...) cannot use float16. On a GPU without
- # bf16 (V100/T4) keep them in float32 so they never autocast to fp16. On a bf16 GPU,
- # full finetuning can still use bf16 autocast (master weights stay float32), which is
- # faster and uses less memory; LoRA/QLoRA keep float32 when forced.
- # Stamped by from_pretrained: the env is process wide, so a LoRA model
- # loaded after this one would cost this trainer its bfloat16.
+ # FORCE_FLOAT32 models (Gemma3, gpt_oss) cannot use float16: without bf16 keep float32, with
+ # bf16 full finetuning may still autocast. Stamped by from_pretrained, since the env is
+ # process wide and a later load would otherwise answer for this trainer.
"full_finetuning = getattr(model, '_unsloth_full_finetuning', None)\n"
"if full_finetuning is None: full_finetuning = os.environ.get('UNSLOTH_ENABLE_FULL_FINETUNING', '0') == '1'\n"
- # Stamped by from_pretrained: the env is process wide, so a forced family
- # loaded earlier would answer here for a model that is not forced at all.
+ # Stamped by from_pretrained: the env is process wide, so a forced family loaded earlier would
+ # answer here for a model that is not forced at all.
"model_forced_float32 = getattr(model, '_unsloth_forced_float32', None)\n"
"if model_forced_float32 is None: model_forced_float32 = os.environ.get('UNSLOTH_FORCE_FLOAT32', '0') == '1'\n"
"if model_forced_float32 and not (full_finetuning and _bf16_supported()):\n"
@@ -2395,8 +2213,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"float16 = dtype == torch.float16\n"
"bfloat16 = dtype == torch.bfloat16\n"
"float32 = dtype == torch.float32\n"
- # Set only when the caller passed dtype = torch.float32 themselves: a
- # request, not a side effect of upcasting, and immune to a second load.
+ # Set only when the caller passed dtype = torch.float32 themselves: a request, not a side effect of
+ # upcasting, and immune to a second load.
"user_float32 = bool(getattr(model, '_unsloth_user_float32', False))\n"
"if full_finetuning:\n"
" if bfloat16 and use_fp16: use_fp16 = False\n"
@@ -2411,10 +2229,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" if hasattr(args, 'mixed_precision'): args.mixed_precision = 'no'\n"
" # args.mixed_precision is a new argument which needs to be set now\n"
"elif (not use_bf16 and not use_fp16) and mixed_precision_dtype == 'float32' and float32 and user_float32 and not _bf16_supported():\n"
- # Without bf16 the only autocast is float16, whose narrower exponent range
- # overflows float32 values to inf then NaN. Gated on the explicit request:
- # full finetuning upcasts to float32 itself, and float16 autocast over
- # float32 master weights is the normal V100/T4 recipe (see #4082).
+ # Without bf16 the only autocast is float16, whose exponent range overflows float32 to inf
+ # then NaN. Gated on the explicit request: fp16 autocast over fp32 master weights is the
+ # normal V100/T4 recipe (#4082).
" print('Unsloth: Model is in float32 and this GPU has no bfloat16 support, so training stays in float32. Pass fp16 = True to force float16 mixed precision instead.')\n"
" args.fp16 = False\n"
" args.bf16 = False\n"
@@ -2445,10 +2262,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += mixed_precision
- # Check if per_device_eval_batch_size (default 8) bigger than bsz
- # Also use FP16 / BF16 evaluation
+ # Check if per_device_eval_batch_size (default 8) is bigger than bsz, and use FP16 / BF16 evaluation.
if "args" in call_args:
- # Check eval_dataset first
if "eval_dataset" in call_args:
check_eval_dataset = (
"if getattr(args, 'eval_dataset', None) is not None and "
@@ -2458,7 +2273,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += check_eval_dataset
- # Check if gradient accumulation bug fix is applied
check_ga = (
"ga_steps = getattr(args, 'gradient_accumulation_steps', None)\n"
"if ga_steps is not None and ga_steps > 1:\n"
@@ -2492,7 +2306,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += eval_changes
- # Force logits to be produced if preprocess_logits_for_metrics or compute_metrics is used
+ # Force logits to be produced if preprocess_logits_for_metrics or compute_metrics is used.
if "model" in call_args:
logits_check = (
"_output_logits = False\n"
@@ -2515,7 +2329,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += warnings_issued_check
- # Check max_seq_length
if "model" in call_args:
length_check = (
"if 'max_seq_length' not in locals() and not hasattr(args, 'max_seq_length'):\n"
@@ -2534,7 +2347,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += length_check
- # At this point max_seq_length might be set, but trl is moving to max_length
+ # max_seq_length may be set here, but trl is moving to max_length.
if trainer_file == "sft_trainer":
max_length_check = (
"if 'max_length' not in locals() and not hasattr(args, 'max_length'):\n"
@@ -2558,31 +2371,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" print('Unsloth: We did not find `max_seq_length` or `max_length` in the model or args. We will set it to 1024.')\n"
" args.max_length = 1024\n"
)
- # TRL >= 1.0.0 refuses padding-free without packing while `max_length` is
- # set, since it cannot truncate a flattened batch. Unsloth auto-enables
- # padding-free and the block above always writes `args.max_length`, so
- # essentially every SFT user tripped that guard.
- #
- # Keep the length that block resolved (`max_seq_length`, capped by the
- # model, wins) and move it to wherever it will actually be enforced:
- # * Unsloth's dataset prep will tokenize -> park it in `max_seq_length`,
- # which prep reads, and hand TRL the None it asks for.
- # * it will not (`skip_prepare_dataset`, or rows that already carry
- # `input_ids` / `labels`) -> nothing would truncate, so turn
- # padding-free off and keep `max_length` for TRL's own collator.
- #
- # The copy is unconditional on purpose: no TRL from 0.22.2 to 1.9.2
- # declares `max_seq_length` on SFTConfig (only UnslothSFTConfig re-adds
- # it), so a hasattr() gate would skip it for every pristine
- # `trl.SFTConfig` and the clear below would drop the cap. `args` is a
- # dataclass instance, so the assignment just adds the attribute and
- # `to_dict()` never sees it.
- #
- # It must be None, not 0: TRL's guard reads `args.max_length is not None`,
- # and rl_replacements.py normalises the None back to 0 inside the Zoo.
- # The schema comes off the first yielded row like the Zoo does, falling
- # back to `column_names` (a `with_transform` dataset reports its backing
- # columns there while yielding `input_ids`).
+ # TRL >= 1.0.0 refuses padding-free without packing while max_length is set, and Unsloth
+ # auto-enables padding-free, so nearly every SFT user tripped that guard. Move the resolved
+ # length to where it is enforced: max_seq_length when prep tokenizes, else padding-free off.
+ # Unconditional copy: no TRL from 0.22.2 to 1.9.2 declares max_seq_length on SFTConfig, so a
+ # hasattr() gate would skip every pristine config and the clear below would drop the cap.
+ # Must be None, not 0: TRL's guard reads `args.max_length is not None`.
if "`max_length` is not enforced" in old_RLTrainer_source:
max_length_check += (
"if getattr(args, 'padding_free', False) is True and not getattr(args, 'packing', False) "
@@ -2596,20 +2390,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" _unsloth_skip_prepare = True\n"
" except Exception:\n"
" pass\n"
- # Metadata first and a row only as a fallback, the other way round
- # from before: reading a row off a one-shot training stream consumes
- # it, and nothing here chains it back, so the run began at row 2.
- # `iter(x) is iter(x)` marks the streams that cannot spare one --
- # the same signal the cap scan and the schema probe use.
- # A `with_transform` split reports its BACKING columns, so a
- # transform yielding `input_ids` over a stored `text` answered
- # "raw" and the cap was cleared for rows nothing then truncates.
- # Its rows are rebuilt on every read, so probing one is free.
- # An unprobeable stream cannot be ruled tokenized either, and
- # `True` there clears the cap on the same guess: refuse instead
- # and keep `max_length` for TRL's collator.
- # One rule, read twice: the late `_unsloth_pretokenized` probe asks the
- # same question of an eval split and must not trust its columns either.
+ # Metadata first, a row only as fallback: reading one off a one-shot stream consumes it, and
+ # `iter(x) is iter(x)` marks those. A with_transform split reports BACKING columns, so one
+ # yielding input_ids over stored `text` read "raw"; probing its rows is free.
" def _unsloth_is_transformed(_ds):\n"
" _f = getattr(_ds, 'format', None)\n"
" _f = _f.get('type') if isinstance(_f, dict) else None\n"
@@ -2632,23 +2415,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" _unsloth_prep_truncates = False\n"
" except Exception:\n"
" _unsloth_prep_truncates = False\n"
- # Already-tokenized rows are not a dead end. TRL's own _prepare_dataset
- # truncates them (sft_trainer.py: `elif args.max_length is not None:
- # truncate_dataset(...)`), and the LM collator it builds passes no
- # max_length, so truncation there is the ONLY thing enforcing the cap. The
- # Zoo's replacement returns pre-tokenized rows untouched, so doing it here
- # restores TRL's contract rather than inventing one, and padding-free can
- # stay on. skip_prepare_dataset is the exception: the user asked for the
- # dataset to be left alone, and TRL skips truncation there too.
- # Only a MATERIALISED tokenized dataset. A with_transform dataset yields
- # input_ids on access while column_names still says ["text"], so mapping it
- # would truncate the wrong thing; that case keeps the fallback below.
- # One predicate, applied per DATASET, because the train split and each eval
- # split can be in different shapes. Truncating is only safe for a materialised
- # tokenized table: raw conversational rows (messages: list[dict]) are per-row
- # sequences too and would be sliced into corrupted turns, and a with_transform
- # dataset recreates its rows on read, so map() writes the backing table while
- # the reader keeps handing back overlength input_ids.
+ # Already-tokenized rows are not a dead end: TRL's _prepare_dataset truncates them and its LM
+ # collator passes no max_length, so that truncation is the only thing enforcing the cap. The
+ # Zoo returns pre-tokenized rows untouched. skip_prepare_dataset is the exception.
+ # Only a MATERIALISED tokenized dataset, judged per DATASET since splits differ: with_transform
+ # yields input_ids while column_names still says ["text"], raw conversational rows would be
+ # sliced into corrupted turns, and a transform's rows are rebuilt on every read.
" def _unsloth_truncatable(_ds):\n"
" if _ds is None or not hasattr(_ds, 'map'): return False\n"
" try:\n"
@@ -2660,67 +2432,36 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" _cols = getattr(_ds, 'column_names', None)\n"
" if isinstance(_cols, dict):\n"
" _cols = [_c for _v in _cols.values() for _c in (_v or [])]\n"
- # A packed split is out: TRL skips truncation when packing
- # (`if args.max_length is not None and not packing`), and cutting
- # `input_ids` under a `seq_lengths` that still describes the old
- # row is worse than not cutting at all.
+ # A packed split is out: TRL skips truncation when packing, and cutting input_ids under a
+ # seq_lengths that still describes the old row is worse than not cutting.
" if 'seq_lengths' in (_cols or ()): return False\n"
" return bool(_cols) and 'input_ids' in _cols\n"
" except Exception:\n"
" return False\n"
- # Read a row BACK. Every predicate above is a guess about what the dataset
- # will do; this is the one check that observes it. A split with no
- # `input_ids` is raw, so prep tokenizes it with the cap and it is fine.
+ # Read a row BACK: every predicate above is a guess about what the dataset will do, and this is the
+ # one check that observes it.
" _unsloth_cap = args.max_length\n"
- # TRL slices [-max_length:] for `keep_end`, which callers use when
- # the completion sits at the tail of a long prompt. Consuming
- # `max_length` while always keeping the prefix trained on the wrong
- # half of every row.
+ # TRL slices [-max_length:] for keep_end, which callers use when the completion sits at the tail of
+ # a long prompt; always keeping the prefix trained on the wrong half of every row.
" _unsloth_truncation_mode = getattr(args, 'truncation_mode', 'keep_start') or 'keep_start'\n"
- # keep_start and keep_end are the only two slices there are. TRL's SFT path
- # never reads this attribute at all -- it belongs to the preference trainers --
- # so nothing downstream would catch a third value, and mapping it to the
- # default would silently cut from the side the caller asked us not to. Refuse
- # the enforcement claim instead, the same answer this block gives for any
- # split it cannot honour.
+ # keep_start and keep_end are the only two slices, and TRL's SFT path never reads this
+ # attribute, so a third value would go uncaught. Refuse the enforcement claim instead.
" _unsloth_keep_end = _unsloth_truncation_mode == 'keep_end'\n"
" _unsloth_known_mode = _unsloth_truncation_mode in ('keep_start', 'keep_end')\n"
" _unsloth_slice = slice(-_unsloth_cap, None) if _unsloth_keep_end else slice(None, _unsloth_cap)\n"
- # Resolved here, one level out from the truncation block, because the fallback
- # below reads it even when `skip_prepare_dataset` skips that block entirely.
- # `eval_packing` is separate from `packing`:
- # packing = args.packing if args.eval_packing is None else args.eval_packing
- # (sft_trainer.py), so packing = False with eval_packing = True reaches this
- # branch, which is gated on `not args.packing`. TRL then PACKS the eval split
- # rather than truncating it, and every strategy owns the overflow: `wrapped`
- # concatenates the stream before chunking, `bfd_split` splits an overlength
- # example into more chunks. Cutting rows at the cap first throws that away.
+ # Resolved outside the truncation block, since the fallback reads it even under
+ # skip_prepare_dataset. eval_packing is separate from packing, so packing=False with
+ # eval_packing=True lands here and TRL packs the eval split instead of truncating it.
" _unsloth_eval_packing = getattr(args, 'packing', False) if getattr(args, 'eval_packing', None) is None else getattr(args, 'eval_packing')\n"
" _unsloth_completion_only = getattr(args, 'completion_only_loss', None)\n"
- # Column names first, and a row only if reading one is free. On a
- # one-shot stream this probe consumed the first TRAINING example --
- # nothing chains it back here, so the run started at row 2. The
- # same `iter(x) is iter(x)` signal the cap scan uses marks those,
- # and an unreadable schema resolves to False, which is what TRL
- # itself answers for a split with no `prompt`/`completion`.
- # A `with_transform` split answers `column_names` with its BACKING
- # table, so a transform storing `text` but yielding `prompt` /
- # `completion` resolved to False here while TRL, which reads a
- # yielded row, resolved True and applied `completion_mask`. The
- # cap filters then kept rows whose completion was cut away
- # entirely, and the collator turned those into all -100.
- # Same rule as the two schema probes above: transformed columns
- # are not evidence, so ignore them and read a row instead.
+ # Column names first, a row only if free: on a one-shot stream this probe ate the first
+ # TRAINING example. A with_transform split answers with its BACKING table, so one yielding
+ # prompt/completion read False here while TRL read True and applied completion_mask.
" if _unsloth_completion_only is None:\n"
" try:\n"
- # A `set_format(columns = [...], output_all_columns = False)`
- # split yields only the named columns while `column_names`
- # still answers with the whole backing table. Reading the
- # table resolved completion-only True off a `completion` the
- # rows never hand over, TRL resolved False from a yielded
- # row, and the cap then filtered on a `completion_mask` the
- # collator ignores -- dropping valid rows, up to emptying the
- # split. The format's own column list is what is yielded.
+ # A set_format(output_all_columns = False) split yields only the named columns while
+ # column_names still lists the whole backing table, so completion-only resolved True off a
+ # completion the rows never hand over. The format's own column list is what is yielded.
" _unsloth_fmt = getattr(train_dataset, 'format', None)\n"
" _unsloth_fmt = _unsloth_fmt if isinstance(_unsloth_fmt, dict) else {}\n"
" _unsloth_shown = _unsloth_fmt.get('columns')\n"
@@ -2737,22 +2478,17 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" except Exception:\n"
" _unsloth_train_sample = {}\n"
" _unsloth_completion_only = ('prompt' in _unsloth_train_sample and 'completion' in _unsloth_train_sample)\n"
- # Parked on args so the late evaluate()/predict() cap reads the SAME value.
- # It resolves from the train schema, which a split handed over later cannot
- # see, and disagreeing with the collator is what leaves an all -100 row in.
+ # Parked on args so the late evaluate()/predict() cap reads the SAME value: it resolves from
+ # the train schema, and disagreeing with the collator leaves an all -100 row in.
" args._unsloth_completion_only_loss = _unsloth_completion_only\n"
- # EVERY row, not the first. A short row 0 in front of a long
- # row 5000 read as "within the cap", and in the fallback branch
- # nothing downstream truncates it. A map-style split is read in
- # full; a stream cannot be rewound, so a bounded prefix is all
- # there is and the check says so rather than pretending.
+ # EVERY row, not the first: a short row 0 before a long row 5000 read as within the cap. A
+ # map-style split is read in full; a stream cannot be rewound, so a bounded prefix is all
+ # there is and the check says so.
" _UNSLOTH_SCAN_ROWS = 1024\n"
" def _unsloth_within_cap(_ds):\n"
" if _ds is None: return True\n"
- # Believe a producer's own truncation claim: scanning a
- # `with_transform` split would tokenize every row in `__init__`,
- # the eager pass it exists to avoid. Read from `__dict__` so a
- # wrapper does not inherit the inner split's claim.
+ # Believe a producer's own truncation claim: scanning a with_transform split tokenizes every
+ # row in __init__. Read from __dict__ so a wrapper does not inherit the inner split's claim.
" _unsloth_own = getattr(_ds, '__dict__', None)\n"
" if isinstance(_unsloth_own, dict):\n"
" _unsloth_claim = _unsloth_own.get('_unsloth_truncated_to')\n"
@@ -2761,49 +2497,33 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" try:\n"
" try: _n = len(_ds)\n"
" except Exception: _n = None\n"
- # A single-pass stream cannot be scanned at all: reading it here IS
- # consuming it, and the trainer would then get an exhausted split
- # (or one short by up to 1024 rows). Two `iter()` calls handing back
- # the same object is what says so -- true for a bare generator and
- # for a `torch.utils.data.IterableDataset` that returns shared
- # iterator state, false for a `datasets.IterableDataset`, which
- # restarts. Unverifiable, so answer as the prefix case below does:
- # not proven within the cap, which drops the enforcement claim and
- # leaves every row where it is.
+ # A single-pass stream cannot be scanned: reading it here IS consuming it, and two iter()
+ # calls returning the same object say so (a datasets.IterableDataset restarts and does not).
+ # Unverifiable, so answer as the prefix case: not proven within the cap.
" _unsloth_rows = iter(_ds)\n"
" if _unsloth_rows is iter(_ds): return False\n"
" _seen = 0\n"
" for _row in _unsloth_rows:\n"
" if 'input_ids' not in _row: return True\n"
" if len(_row['input_ids']) > _unsloth_cap: return False\n"
- # An unexhausted stream is UNVERIFIED, not verified. Calling the
- # first 1024 fitting rows proof let a later overlength row through,
- # and in the fallback branch nothing truncates a pre-tokenized row.
- # A stream that can be rewritten is capped by the lazy map above and
- # never reaches this; one that cannot is refused.
+ # An unexhausted stream is UNVERIFIED, not verified: treating the first 1024 fitting rows as proof
+ # let a later overlength row through, and nothing truncates a pre-tokenized row here.
" _seen += 1\n"
" if _n is None and _seen >= _UNSLOTH_SCAN_ROWS: return False\n"
" except Exception:\n"
" return False\n"
" return True\n"
- # Each eval split counts. A tokenized eval split in a shape the truncation
- # cannot rewrite (with_transform, or an iterable with no column_names) is
- # left alone above, and prep will not re-tokenize rows that already carry
- # `input_ids`, so consuming `max_length` on the strength of the train split
- # alone let evaluation run over the cap.
+ # Each eval split counts: one the truncation cannot rewrite is left alone above, and prep
+ # never re-tokenizes rows carrying input_ids, so trusting the train split left eval uncapped.
" def _unsloth_splits_within_cap(_ev):\n"
" _splits = list(_ev.values()) if isinstance(_ev, dict) else [_ev]\n"
" return all(_unsloth_within_cap(_s) for _s in _splits)\n"
- # Not train-only. `_unsloth_prep_truncates` is decided from the train
- # split, so a raw train beside a pre-tokenized eval set skipped this
- # whole block, consumed `max_length`, and left evaluation uncapped:
- # prep does not re-tokenize rows that already carry `input_ids`.
+ # Not train-only: _unsloth_prep_truncates is decided from the train split, so a raw train beside a
+ # pre-tokenized eval set skipped this whole block and left evaluation uncapped.
" if not _unsloth_skip_prepare:\n"
- # TRL forwards its preparation map_kwargs; honour the same setting so a large
- # pre-tokenized dataset is not rewritten single-process. Resolved through the
- # same helper as every other map site: the config layer writes "run
- # in-process" as `dataset_num_proc = 1`, and datasets >= 4.1 builds a Pool(1)
- # for it, forking a tokenizer worker on the host that asked for none.
+ # Honour TRL's preparation map_kwargs so a large pre-tokenized dataset is not rewritten
+ # single-process, through the same helper as every map site: the config layer writes serial as
+ # dataset_num_proc = 1, and datasets >= 4.1 builds a Pool(1) for it.
" _unsloth_map_kw = {}\n"
" try:\n"
" try:\n"
@@ -2814,12 +2534,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" except Exception:\n"
" _unsloth_nproc = None\n"
" if _unsloth_nproc: _unsloth_map_kw['num_proc'] = _unsloth_nproc\n"
- # Same rule as TRL's truncate_dataset: slice every per-row list column, so
- # input_ids, labels, attention_mask and the masks stay aligned. Written out
- # rather than imported, because trl.data_utils drags in the processor stack
- # and an ImportError there would silently drop the cap. A torch/numpy format
- # hands batched map() tensors, where `if _col` raises on the ambiguous truth
- # value, so ask for a per-row sequence and exclude str/bytes.
+ # Same rule as TRL's truncate_dataset: slice every per-row list column so input_ids, labels,
+ # attention_mask and the masks stay aligned. Written out rather than imported, since
+ # trl.data_utils drags in the processor stack and an ImportError would drop the cap.
" def _unsloth_is_sequence_column(_col):\n"
" try:\n"
" if len(_col) == 0: return False\n"
@@ -2827,24 +2544,15 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" except Exception:\n"
" return False\n"
" if isinstance(_first, (str, bytes)): return False\n"
- # len(), not hasattr('__len__'): under set_format('torch') a scalar
- # column batches to a 1-D tensor whose element is 0-dim, which HAS
- # __len__ and raises on it. The later len(_v) then threw TypeError,
- # the outer catch restored the overlength dataset, and a truncatable
- # run died on "cannot be enforced".
+ # len(), not hasattr('__len__'): under set_format('torch') a scalar column batches to a 1-D
+ # tensor whose element is 0-dim, so len(_v) threw TypeError and the catch restored the
+ # overlength dataset.
" try: len(_first)\n"
" except Exception: return False\n"
" return True\n"
- # Per-token columns only, matched by row length against `input_ids`.
- # A packed split carries `seq_lengths` -- document lengths, not tokens
- # -- and slicing that by the cap left it stale, so padding-free built
- # position ids for more tokens than the row now holds.
- # Per VALUE, because `_unsloth_is_sequence_column` judges the
- # column from its first row. An optional field that is a list
- # there and None (or a scalar) further in raised TypeError out
- # of `len`, the enclosing handler restored the overlength split,
- # and a truncatable run died on "cannot be enforced". The late
- # cap validates each row for the same reason.
+ # Per-token columns only, matched by row length against input_ids: a packed split's
+ # seq_lengths are document lengths, and slicing them left padding-free building position ids
+ # for tokens the row no longer has. Per VALUE, since a list row 0 can be None later.
" def _unsloth_cut_value(_v, _r):\n"
" try:\n"
" if len(_v) != len(_r): return _v\n"
@@ -2861,38 +2569,22 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" else:\n"
" _out[_k] = [_unsloth_cut_value(_v, _r) for _v, _r in zip(_col, _ids)]\n"
" return _out\n"
- # A stream has no length, and `IterableDataset.map` takes no `num_proc`:
- # passing one raised TypeError, the catch below restored the original,
+ # A stream has no length, and IterableDataset.map takes no num_proc: passing one raised TypeError
# and the run died on "cannot be enforced" instead of being truncated.
" def _unsloth_is_stream(_ds):\n"
" try: return not hasattr(_ds, '__len__')\n"
" except Exception: return True\n"
- # One split, capped and then checked. A stream's map is lazy and applies
- # to EVERY row it will ever yield, which is a stronger guarantee than the
- # bounded prefix scan: reading 1024 rows and calling the rest verified let
- # a later overlength row through, since nothing else truncates a
- # pre-tokenized stream.
- # Enforcement, not observation. A `with_transform` split that happens
- # to sit under the cap is not enforced -- it rebuilds its rows on every
- # read -- so it keeps the old answer: hold `max_length` and turn
- # padding-free off. Only a split we rewrote, or a raw one the tokenizer
- # pass will cut, counts.
- # Schema first, and a row only when one is free. `next(iter(_ds))`
- # on a single-pass stream is a row the run then trains without:
- # if it reads raw the split is declared safe and training starts
- # at row 2, if it reads tokenized construction rejects a stream
- # the caller still owns and has already been mutated. The
- # `iter(x) is iter(x)` test is the one the cap scan and the
- # schema probe already use; an unprobeable stream answers True,
- # which holds `max_length` rather than clearing it on a guess.
+ # One split, capped and then checked. A stream's map is lazy and applies to EVERY row it will ever
+ # yield, a stronger guarantee than the bounded prefix scan.
+ # Enforcement, not observation: a with_transform split under the cap is not enforced, since it
+ # rebuilds its rows on every read, so it keeps max_length and turns padding-free off.
+ # Schema first, a row only when free: next(iter(_ds)) on a single-pass stream is a row the run
+ # then trains without. An unprobeable stream answers True, holding max_length.
" def _unsloth_pretokenized(_ds):\n"
" try:\n"
- # Columns first, EXCEPT for a transform: a `with_transform` split reports
- # its backing table, so one storing `text` and yielding overlength
- # `input_ids` answered "raw" here. `_unsloth_truncatable` already refuses
- # to rewrite it, and this answer then marked it safe anyway and cleared
- # `max_length`, leaving padding-free with rows nothing truncates. Its rows
- # are rebuilt on every read, so probing one below costs nothing.
+ # Columns first, EXCEPT for a transform: a with_transform split reports its backing table, so
+ # one storing `text` and yielding overlength input_ids read "raw" and cleared max_length. Its
+ # rows are rebuilt on every read, so probing one costs nothing.
" _cols = None if _unsloth_is_transformed(_ds) else getattr(_ds, 'column_names', None)\n"
" if isinstance(_cols, dict):\n"
" _cols = [_c for _v in _cols.values() for _c in (_v or [])]\n"
@@ -2902,13 +2594,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" _row = next(_probe, None)\n"
" except Exception: return True\n"
" return isinstance(_row, dict) and 'input_ids' in _row\n"
- # Every rank reaches this before TRL's `_prepare_dataset`, and
- # TRL runs its own preparation maps under `main_process_first`.
- # Without the same window, eight ranks each start `num_proc`
- # workers against one Arrow cache -- 64 processes doing the same
- # work, and writing the same cache files at the same time. One
- # rank materialises, the rest read what it wrote. A single
- # process gets a no-op context manager, so nothing changes there.
+ # Every rank reaches this before TRL's _prepare_dataset, which runs its own maps under
+ # main_process_first. Without the same window, eight ranks start num_proc workers each against
+ # one Arrow cache. A single process gets a no-op context manager.
" def _unsloth_rank_first():\n"
" try:\n"
" from accelerate import PartialState\n"
@@ -2924,91 +2612,47 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" if not _unsloth_truncatable(_ds): return _ds, not _unsloth_pretokenized(_ds)\n"
" _kw = {} if _unsloth_is_stream(_ds) else _unsloth_map_kw\n"
" _new = _ds.map(_unsloth_truncate_rows, batched = True, **_kw)\n"
- # TRL filters these immediately after truncating: a row whose
- # prompt alone fills the cap has every label at -100 and
- # contributes no loss, so leaving them in feeds batches with no
- # supervised tokens.
- # `labels` is only one of the three ways a row says which tokens
- # are supervised. A completion-only or assistant-only row carries
- # `completion_mask` / `assistant_masks` instead, and truncating a
- # long prompt can leave that mask all zeros; TRL's collator turns
- # those into all -100 and a batch made of them has no supervised
- # token at all, which is a NaN loss rather than a small one.
- # A mask is supervision when TRL will actually apply it, and the
- # two masks are not symmetric there. DataCollatorForLanguageModeling
- # guards `completion_mask` behind `self.completion_only_loss` but
- # applies `assistant_masks` on presence alone:
- # if self.completion_only_loss and "completion_mask" in examples[0]:
- # if "assistant_masks" in examples[0]:
- # (trl/trainer/sft_trainer.py, checked on 0.25.1 and v1.9.2). TRL only
- # ever BUILDS that column under `assistant_only_loss`, which is why the
- # flag looks like the gate, but a pre-tokenized split -- the only kind
- # this branch handles -- carries whichever columns the caller put there.
- # So gating on the flag left an all-zero assistant mask in place, and
- # the collator turned the row into all -100: no supervised token, and a
- # batch of them is a NaN loss.
- # A None `completion_only_loss` is NOT "on". TRL resolves it from
- # the dataset shape:
- # if args.completion_only_loss is None:
- # self.completion_only_loss = "prompt" in dataset_sample and "completion" in dataset_sample
- # (sft_trainer.py:736). A pre-tokenized split has neither column, so
- # the effective mode is False and the collator ignores the mask
- # entirely; treating None as enabled deleted rows that still had
- # valid full-sequence supervision, and could empty the split.
+ # TRL filters these right after truncating: a row whose prompt fills the cap is all -100 and
+ # contributes no loss. labels is only one of three supervision signals; completion-only and
+ # assistant-only rows carry completion_mask / assistant_masks instead.
+ # A mask is supervision when TRL will apply it, and the two are not symmetric:
+ # DataCollatorForLanguageModeling gates completion_mask on completion_only_loss but applies
+ # assistant_masks on presence, so gating on the flag left an all-zero mask and an all -100 row.
+ # A None completion_only_loss is NOT "on": TRL resolves it from the dataset shape (prompt plus
+ # completion), and a pre-tokenized split has neither, so treating None as enabled deleted rows
+ # with valid full-sequence supervision.
" _unsloth_cols = getattr(_new, 'column_names', None) or ()\n"
- # One mode for every split, resolved from the TRAIN sample, because that is
- # what the collator uses:
- # dataset_sample = next(iter(train_dataset))
- # if args.completion_only_loss is None:
- # self.completion_only_loss = "prompt" in dataset_sample and "completion" in dataset_sample
- # (sft_trainer.py). Resolving it per split disagreed with the collator whenever
- # the schemas differ: prompt/completion training data makes the collator apply
- # `completion_mask`, while a pre-tokenized eval split carrying only
- # `input_ids` and `completion_mask` read here as full-sequence loss, so rows
- # whose mask truncated to all zeros survived and went all -100 at eval.
+ # One mode for every split, from the TRAIN sample, because that is what the collator uses.
+ # Per-split resolution disagreed with it whenever the schemas differ, so rows whose mask
+ # truncated to all zeros survived and went all -100 at eval.
" _unsloth_masks = []\n"
" if _unsloth_completion_only and 'completion_mask' in _unsloth_cols:\n"
" _unsloth_masks.append('completion_mask')\n"
" if 'assistant_masks' in _unsloth_cols:\n"
" _unsloth_masks.append('assistant_masks')\n"
" try:\n"
- # `labels` is unconditional: it IS the supervision.
- # One intersection over labels AND every active mask, not a filter each.
- # The collator applies the masks ONTO the labels, so a row whose only
- # supervised label sits where the mask is zero passes both filters
- # separately and still goes out all -100.
+ # labels is unconditional: it IS the supervision. Intersect labels AND every active mask
+ # rather than filtering each, since masks are applied one after another onto the labels; zip
+ # stops at the shorter, which is what an intersection means for a ragged pair.
" _unsloth_supervision = (['labels'] if 'labels' in _unsloth_cols else []) + _unsloth_masks\n"
- # The masks are applied one after another onto the same labels, so
- # what survives is their INTERSECTION. Filtering each on its own kept
- # rows whose two masks light up in different positions, which TRL then
- # labels all -100 -- the very rows this filter exists to drop. zip
- # stops at the shorter, which is what an intersection means for a
- # ragged pair.
+ # The masks are applied one after another onto the same labels, so what survives is their
+ # INTERSECTION. Filtering each on its own kept rows whose two masks light up in different
+ # positions, which TRL then labels all -100 -- the very rows this filter exists to drop. zip
+ # stops at the shorter, which is what an intersection means for a ragged pair.
" if _unsloth_supervision:\n"
" _new = _new.filter(lambda _e, _c = tuple(_unsloth_supervision): any(all((_x != -100) if _n == 'labels' else _x for _n, _x in zip(_c, _v)) for _v in zip(*[_e[_n] for _n in _c])), **_kw)\n"
" except Exception:\n"
" pass\n"
- # Recorded, not raised: the caller wraps these calls in a broad
- # `except Exception` that would turn a raise into "could not
- # truncate". The raise happens past that handler.
+ # Recorded, not raised: the caller wraps these calls in a broad `except Exception` that would turn a
+ # raise into "could not truncate", so the raise happens past that handler.
" try:\n"
" if _unsloth_supervision and len(_new) == 0: _unsloth_emptied.append(1)\n"
" except TypeError:\n"
" pass\n"
" return _new, (True if _unsloth_is_stream(_new) else _unsloth_within_cap(_new))\n"
- # Resolved BEFORE the try, because the fallback below needs it even when
- # the try never ran. `eval_packing` is separate from `packing`:
- # packing = args.packing if args.eval_packing is None else args.eval_packing
- # (sft_trainer.py), so packing = False with eval_packing = True reaches this
- # branch, which is gated on `not args.packing`. TRL then PACKS the eval split
- # rather than truncating it, and every strategy owns the overflow: `wrapped`
- # concatenates the stream before chunking, `bfd_split` splits an overlength
- # example into more chunks. Cutting rows at the cap first throws that away.
- # So the packer keeps the split, and the enforcement claim is dropped rather
- # than the tokens: `max_length` stays and padding-free turns off. That is
- # required anyway, since packing raises on a None `max_length`, and it has to
- # hold even with no eval split at construction, because one handed to
- # `evaluate()` later would find `max_length` already cleared.
+ # Resolved BEFORE the try, since the fallback needs it even when the try never ran.
+ # eval_packing is separate from packing, so TRL may PACK the eval split instead of truncating
+ # it: keep max_length and turn padding-free off, which packing requires anyway.
" _unsloth_emptied = []\n"
" _unsloth_orig_train = train_dataset\n"
" _unsloth_orig_eval = eval_dataset if 'eval_dataset' in locals() else None\n"
@@ -3016,38 +2660,20 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" _unsloth_capped = _unsloth_known_mode\n"
" if not _unsloth_known_mode:\n"
" print('Unsloth: `truncation_mode = ' + str(_unsloth_truncation_mode) + '` is not one of keep_start / keep_end, so `max_length` is not being enforced here.')\n"
- # A raw train split is tokenized with the cap by prep, so leave it alone.
- # `and`, like the eval splits below. A plain assignment threw away the
- # unknown-mode refusal seeded above, so a `truncation_mode` this cannot
- # honour was warned about and then silently served as keep_start, with
- # `max_length` cleared and padding-free left on.
- # `_unsloth_known_mode` too: seeding `_unsloth_capped` false only drops the
- # ENFORCEMENT claim. The slice still ran, with `_unsloth_keep_end` false for
- # any unknown value, so the fallback scanned an already-trimmed split, found
- # it within the cap and merely turned padding-free off -- every row silently
- # cut from the start right after warning that the mode could not be honoured.
- # Leaving the split alone lets that scan see the real lengths and say so.
+ # A raw train split is tokenized with the cap by prep, so leave it alone. `and`, not a plain
+ # assignment, which threw away the unknown-mode refusal seeded above and served an unhonoured
+ # truncation_mode as keep_start.
+ # _unsloth_known_mode too: clearing _unsloth_capped only drops the ENFORCEMENT claim while the
+ # slice still ran, so the fallback scanned an already-trimmed split. Leaving the split alone
+ # lets that scan see the real lengths.
" if _unsloth_known_mode and not _unsloth_prep_truncates:\n"
" train_dataset, _unsloth_split_ok = _unsloth_cap_split(train_dataset)\n"
" _unsloth_capped = _unsloth_capped and _unsloth_split_ok\n"
- # An eval split TRL will PACK must not be truncated first. The branch
- # is gated on `not args.packing`, but `eval_packing` is resolved
- # separately:
- # packing = args.packing if args.eval_packing is None else args.eval_packing
- # (sft_trainer.py), so packing = False with eval_packing = True reaches
- # here. TRL then packs the eval split instead of truncating it
- # (`if packing: ... elif args.max_length is not None: truncate_dataset`),
- # and the wrapped strategy concatenates the whole token stream before
- # chunking it, so cutting each row at the cap first throws away every
- # token past it and evaluates on a truncated corpus.
- # Leaving it uncut also means the cap is not enforced for that split, and
- # packing needs `max_length` anyway ("When packing is enabled,
- # `max_length` can't be `None`"), so this drops the enforcement claim
- # rather than the split: `max_length` stays and padding-free turns off,
- # which is the same answer this block already gives for a split it
- # cannot rewrite.
- # Each eval split on its own: a raw one stays raw for the tokenizer pass
- # that follows, and only a materialised tokenized one is cut.
+ # An eval split TRL will PACK must not be truncated first: the branch is gated on
+ # `not args.packing` while eval_packing resolves separately, and the wrapped strategy
+ # concatenates the stream before chunking. Drop the enforcement claim, not the split.
+ # Each eval split on its own: a raw one stays raw for the tokenizer pass that follows, and only a
+ # materialised tokenized one is cut.
" if _unsloth_eval_packing or not _unsloth_known_mode:\n"
" _unsloth_capped = False\n"
" elif 'eval_dataset' in locals() and eval_dataset is not None:\n"
@@ -3061,36 +2687,27 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" eval_dataset, _unsloth_split_ok = _unsloth_cap_split(eval_dataset)\n"
" _unsloth_capped = _unsloth_capped and _unsloth_split_ok\n"
" _unsloth_prep_truncates = _unsloth_capped\n"
- # The splits that WERE rewritten keep their truncation: it is the cap
- # the caller asked for, applied exactly as TRL's own truncate_dataset
- # would. Rolling them back because a different split cannot be
- # rewritten put an overlength train set back and turned a healthy run
- # into "cannot be enforced". Only the claim of enforcement is dropped.
+ # Splits that WERE rewritten keep their truncation: it is the cap the caller asked for, as
+ # TRL's truncate_dataset would apply it. Rolling them back for a sibling that cannot be
+ # rewritten put an overlength train set back; only the claim of enforcement is dropped.
" if not _unsloth_capped:\n"
" print('Unsloth: `max_length` cannot be enforced for every split here, so padding-free batching is being turned off instead.')\n"
" except Exception as _unsloth_truncate_error:\n"
" train_dataset = _unsloth_orig_train\n"
" if 'eval_dataset' in locals(): eval_dataset = _unsloth_orig_eval\n"
- # The flag is decided from the train split, so a failure while capping
- # an eval split would otherwise leave it reading "cap enforced".
+ # The flag is decided from the train split, so a failure while capping an eval split would otherwise
+ # leave it reading "cap enforced".
" _unsloth_prep_truncates = False\n"
# Never silent: a swallowed failure here reads as the cap being enforced.
" print('Unsloth: could not truncate the pre-tokenized dataset to `max_length` (' + str(_unsloth_truncate_error) + ').')\n"
- # Outside the handler on purpose. Every row losing its supervised
- # tokens means the cap sits below where the supervision starts, and
- # an empty split is not something to hand onwards: every TRL 1.x
- # reads `next(iter(train_dataset))` in `__init__` to resolve
- # `completion_only_loss` and `_is_vision_dataset`, so it comes out
- # as a bare `StopIteration` naming nothing at all.
+ # Outside the handler on purpose: if every row loses its supervised tokens the cap sits below
+ # where supervision starts, and every TRL 1.x reads next(iter(train_dataset)) in __init__, so
+ # an empty split surfaces as a bare StopIteration naming nothing.
" if _unsloth_emptied:\n"
" raise ValueError('Unsloth: truncating to `max_length = ' + str(args.max_length) + '` left every row with no supervised token, so there is nothing to train on. The supervised part of your rows starts past that length: raise `max_length`, or set `truncation_mode = \"keep_end\"` if the completion sits at the end of each row.')\n"
- # A producer that truncates every row enforces the cap just as
- # `truncate_dataset` would, so keep padding-free rather than pay the
- # fallback for a guarantee already held. Unsloth's online tokenization
- # is this shape: a `with_transform` view capped on read, invisible
- # without reading -- and reading it all is the eager pass it avoids.
- # Not under `eval_packing`: that split is meant to be overlength and
- # the packer needs the `max_length` this branch clears.
+ # A producer that truncates every row enforces the cap as truncate_dataset would, so keep
+ # padding-free rather than pay the fallback; Unsloth's online tokenization is this shape. Not
+ # under eval_packing: that split is overlength on purpose.
" if not _unsloth_prep_truncates and not _unsloth_eval_packing:\n"
" def _unsloth_attests(_ds):\n"
" if _ds is None: return True\n"
@@ -3108,19 +2725,13 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" args.max_length = None\n"
" max_length = None\n"
" else:\n"
- # Turning padding-free off keeps `max_length` for TRL's collator, and that
- # collator does not truncate: for rows that already carry `input_ids`
- # nothing downstream enforces the cap. Before this block existed TRL's own
- # guard made the same configuration a hard error, so an observed overlength
- # row must stay one rather than become a silently uncapped run.
- # `skip_prepare_dataset` used to exempt this, which made it the one
- # way to get a silently uncapped run: TRL then neither truncates nor
- # builds its collator with a truncation length, so the oversized rows
- # reach the model. The check is what decides, not the flag.
- # An eval split left for the packer is overlength ON PURPOSE, so scanning it
- # here turned a working eval-packing run into a hard error and denied
- # `wrapped` / `bfd_split` the overflow they exist to handle. The train split
- # is still scanned: nothing packs that one.
+ # Turning padding-free off keeps max_length for TRL's collator, which does not truncate, so
+ # rows already carrying input_ids are unenforced. TRL's own guard used to make this a hard
+ # error, so an observed overlength row must stay one rather than run silently uncapped.
+ # skip_prepare_dataset used to exempt this, the one way to get a silently uncapped run: TRL
+ # then neither truncates nor gives its collator a truncation length.
+ # An eval split left for the packer is overlength ON PURPOSE, so scanning it here turned a working
+ # eval-packing run into a hard error. The train split is still scanned: nothing packs that one.
" _unsloth_scan_eval = None if _unsloth_eval_packing else (eval_dataset if 'eval_dataset' in locals() else None)\n"
" if not (_unsloth_within_cap(train_dataset) and _unsloth_splits_within_cap(_unsloth_scan_eval)):\n"
" raise ValueError('Unsloth: `max_length = ' + str(args.max_length) + '` cannot be enforced. Your dataset already carries `input_ids` and holds rows longer than that, and nothing downstream truncates pre-tokenized rows. Truncate it yourself before passing it in, or drop `max_length`.')\n"
@@ -3129,7 +2740,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += max_length_check
- # Enable for training and move padding side of tokenizer to right
+ # Sync chat_template from processing_class to vLLM's tokenizer This fixes base models that have custom chat
+ # templates applied after loading
if "model" in call_args:
training_check = (
"if model is not None and hasattr(model, 'for_training'):\n"
@@ -3143,7 +2755,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += training_check
- # Check data collator if it's correct!
if "data_collator" in call_args and "train_dataset" in call_args:
data_collator_check = (
"__tokenizer = processing_class if 'processing_class' in locals() else tokenizer\n"
@@ -3168,10 +2779,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += data_collator_check
- # Also check if .pad exists -> if not, and is VLM, then change it!
- # Only swap LM/Seq2Seq collators; leave preference collators
- # (DPODataCollatorWithPadding etc.) alone so ORPO/DPO/CPO/KTO keep
- # their own prompt/chosen/rejected handling.
+ # Also swap when .pad is missing on a VLM. LM/Seq2Seq collators only: preference collators
+ # (DPODataCollatorWithPadding etc.) keep their own prompt/chosen/rejected handling.
pad_check = (
"if not isinstance(data_collator, UnslothVisionDataCollator):\n"
" if not hasattr(__tokenizer, 'pad') and hasattr(__tokenizer, 'tokenizer'):\n"
@@ -3190,7 +2799,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += pad_check
- # Check NEFTune
if "model" in call_args:
neftune_check = (
"if hasattr(self, 'neftune_hook_handle'):\n"
@@ -3202,7 +2810,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
RLTrainer_post += neftune_check
- # Add accelerator scaler to model
if "model" in call_args:
accelerator_check = (
"if hasattr(self, 'accelerator'):\n"
@@ -3216,7 +2823,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
RLTrainer_post += accelerator_check
- # Add enabling and disabling training modes
if "model" in call_args:
training_check = (
"if hasattr(self, 'train'):\n"
@@ -3225,8 +2831,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
RLTrainer_post += training_check
- # Sync chat_template from processing_class to vLLM's tokenizer
- # This fixes base models that have custom chat templates applied after loading
+ # Sync chat_template from processing_class to vLLM's tokenizer, which fixes base models that have
+ # custom chat templates applied after loading.
if "model" in call_args:
vllm_chat_template_sync = (
"if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n"
@@ -3238,14 +2844,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
RLTrainer_post += vllm_chat_template_sync
- # Edit optional metrics
other_metrics_processor = ""
if trainer_file in RL_METRICS_CHANGES:
process_extra_args = RL_METRICS_CHANGES[trainer_file]
for process_extra_arg in process_extra_args:
other_metrics_processor += process_extra_arg(old_RLTrainer_source, old_RLConfig_source)
- # Add statistics as well!
extra_args += (
"other_metrics = []\n"
f"{other_metrics_processor}\n"
@@ -3253,13 +2857,11 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
f"PatchRLStatistics('{trainer_file}', other_metrics)\n"
)
- # Patch optional args
if trainer_file in RL_EXTRA_ARGS:
process_extra_args = RL_EXTRA_ARGS[trainer_file]
for process_extra_arg in process_extra_args:
extra_args += process_extra_arg(call_args, extra_args)
- # Create RLTrainer args
extra_args = extra_args.split("\n")
extra_args = "\n".join(" " * 8 + x for x in extra_args)
RLTrainer_post = RLTrainer_post.split("\n")
@@ -3268,18 +2870,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
RLTrainer_extra_args = extra_args
RLTrainer_call_args = call_args
- # Fix RLConfig next
arguments, call_args = processed[1]
extra_args = ""
- # Edit GA / bsz and weight_decay
replacements = {
"output_dir": None,
"logging_nan_inf_filter": False,
"per_device_train_batch_size": 4,
"gradient_accumulation_steps": 2,
- # LoRA decays A and B toward 0 so effective W = W_init + (alpha/r) * B @ A is pulled toward W_init, not 0 as in full FT.
- # 0.001 keeps a small Frobenius prior |A|_F^2 + |B|_F^2 without measurably dragging the merged adapter back to base.
+ # LoRA decays A and B toward 0, so W = W_init + (alpha/r) * B @ A is pulled to W_init, not 0
+ # as in full FT; 0.001 keeps a small Frobenius prior without dragging the adapter to base.
"weight_decay": 0.001,
"seed": 3407,
"optim": "adamw_8bit",
@@ -3290,8 +2890,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"logging_steps": 1,
"max_seq_length": None,
"num_generations": 8,
- # "steps_per_generation" : 1, # Otherwise defaults to ga_steps which is wrong
- # "generation_batch_size" : None, # Useless. If steps_per_generation set, generation_batch_size clashes
+ # steps_per_generation would otherwise default to ga_steps, which is wrong, and
+ # generation_batch_size clashes with it.
"top_k": None,
"vllm_mode": "colocate",
"generation_kwargs": {},
@@ -3303,12 +2903,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"auto_find_batch_size": False, # Auto /2 batch size - too many people complained so removing
"dataloader_pin_memory": True,
"padding_free": None, # None = user didn't set it, allows auto-enable detection
- # Might fail so disable for now
- # "dataloader_persistent_workers" : True, # Keeps dataloader in RAM
- # "dataloader_prefetch_factor" : 2,
- # "dataloader_num_workers" : 2, # Default is 0 means 1
+ # Might fail, so persistent dataloader workers / prefetch are disabled for now.
}
- # warmup_ratio deprecated in transformers >= 5.0; warmup_steps accepts float
+ # warmup_ratio is deprecated in transformers >= 5.0; warmup_steps accepts a float.
if transformers_version >= Version("5.0.0"):
replacements["warmup_steps"] = 0.1
else:
@@ -3320,19 +2917,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
y = f"{k} = {y},\n"
arguments = re.sub(x, y, arguments)
- # Fix GRPO beta default as 0.001 TRL used to be 0.04, now 0.00!
- # https://github.com/huggingface/trl/pull/3516
- # https://verl.readthedocs.io/en/latest/examples/config.html
+ # GRPO beta default is 0.001: TRL used 0.04 and now 0.00. See huggingface/trl#3516 and the verl docs.
if trainer_file == "grpo_trainer":
replacements = {
"loss_type": "bnpo", # Default GRPO paper
"beta": 0.001, # Recommended as seen in verl
"auto_find_batch_size": False, # Cannot work on GRPO
- # [TODO] See https://fengyao.notion.site/off-policy-rl
- # https://github.com/huggingface/trl/pull/3867 (August 7th)
+ # See fengyao.notion.site/off-policy-rl and huggingface/trl#3867.
"vllm_importance_sampling_correction": False,
- # TRL >= 1.7.0 enables the MoE router aux loss by default (0.001); the optimized
- # GRPO forward does not compute it, so default off. Opt in via router_aux_loss_coef > 0.
+ # TRL >= 1.7.0 enables the MoE router aux loss by default (0.001), but the optimized GRPO forward
+ # does not compute it, so default it off; opt in via router_aux_loss_coef > 0.
"router_aux_loss_coef": 0.0,
}
for k, v in replacements.items():
@@ -3341,15 +2935,13 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
y = f"{k} = {y},\n"
arguments = re.sub(x, y, arguments)
- # TRL >= 1.7.0 defaults SFT to loss_type="chunked_nll" (trl#5846), which patches
- # the lm_head and calls the backbone directly. Two problems:
- # 1. It bypasses the model forward, so unsloth_fused_ce_loss never runs. Ours
- # chunks too and peaks 1.7-3.7GB lower on gemma-3-4b at 141-8192 tokens.
- # 2. It divides by num_items_in_batch ignoring model_accepts_loss_kwargs, so
- # on models setting that flag False (gemma3, qwen-vl, paligemma, glm4v)
- # training_step divides by grad-accum again: loss and grads scaled 1/GA.
- # Explicit loss_type= still wins. Keep scoped to sft_trainer: loss_type is an
- # unrelated field in DPO/KTO/GRPO and the global dict above is applied to all.
+ # TRL >= 1.7.0 defaults SFT to loss_type="chunked_nll" (trl#5846), which patches the lm_head and
+ # calls the backbone directly, so unsloth_fused_ce_loss never runs (ours chunks too and peaks
+ # 1.7-3.7GB lower on gemma-3-4b at 141-8192 tokens), and it divides by num_items_in_batch ignoring
+ # model_accepts_loss_kwargs, so on models setting that flag False (gemma3, qwen-vl, paligemma,
+ # glm4v) training_step divides by grad-accum again and loss and grads are scaled 1/GA. Explicit
+ # loss_type= still wins. Kept scoped to sft_trainer, since loss_type is an unrelated field in
+ # DPO/KTO/GRPO.
if trainer_file == "sft_trainer":
replacements = {"loss_type": "nll"}
for k, v in replacements.items():
@@ -3358,7 +2950,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
y = f"{k} = {y},\n"
arguments = re.sub(x, y, arguments)
- # Warn on too large or too small learning rate
if "learning_rate" in call_args:
learning_rate_check = (
"if learning_rate < 1e-7: print(f'Unsloth: Your learning rate of `{learning_rate}` is too small and less than 1e-7! "
@@ -3368,8 +2959,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += learning_rate_check
- # Fix num_train_epochs = None causing TypeError in Trainer.__init__
- # Trainer does `args.num_train_epochs > 0` which fails when None
+ # Fix num_train_epochs = None causing a TypeError in Trainer.__init__, which does `args.num_train_epochs > 0`.
if "num_train_epochs" in call_args:
num_train_epochs_check = (
"if num_train_epochs is None:\n"
@@ -3377,7 +2967,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += num_train_epochs_check
- # Check if max_seq_length is NOT defined (max_length is now default)
+ # Check whether max_seq_length is NOT defined; max_length is now the default.
if "max_seq_length" not in call_args and "max_length" in call_args:
max_seq_length_pre = """max_seq_length : Optional[int] = field(
default = None,
@@ -3390,9 +2980,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
max_seq_length_call = ""
max_seq_length_post = ""
- # Add output_dir saving
if "output_dir" in call_args:
- # Default checks
saving_check = (
"if output_dir is None and save_strategy == 'steps' and save_steps == 500:\n"
" output_dir = 'unsloth_training_checkpoints'\n"
@@ -3400,22 +2988,13 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += saving_check
- # Edit dataset_num_proc
- # The policy lives in unsloth_zoo.dataset_num_proc: it had drifted into four
- # inline copies, two wrong (stdlib `multiprocessing` asked about a start
- # method `datasets` takes from `multiprocess`, and `1` used as the serial
- # sentinel when datasets >= 4.1 builds a Pool(1) for it). The zoo rather than
- # unsloth, so generated source never imports back into its generator;
- # unsloth.dataset_num_proc is the fallback for an older zoo, and the
- # ladder is guarded so a stale generated file degrades to the caller's value.
- # serial_as_none depends on who reads the value back. Only SFT has a
- # downstream auto-sizer: unsloth_zoo.sft_prepare_dataset reads a config
- # `None` as "auto-size me", so serial has to be written as `1` there and the
- # map() call site (rl_replacements.py) turns it back into None. DPO, KTO,
- # CPO, ORPO, Reward and PRM hand args.dataset_num_proc straight to
- # Dataset.map, where nothing can inflate a None but a `1` is a Pool(1) on
- # datasets >= 4.1 -- one worker with its own tokenizer copy, on a host the
- # memory clamp had just declared too small for workers.
+ # The worker-count policy lives in unsloth_zoo.dataset_num_proc: it had drifted into four
+ # inline copies, two wrong (stdlib multiprocessing start method, and `1` as the serial
+ # sentinel when datasets >= 4.1 builds a Pool(1)). In the zoo so generated source never
+ # imports back into its generator.
+ # serial_as_none depends on the reader: unsloth_zoo.sft_prepare_dataset reads a config None as
+ # "auto-size me", so SFT writes serial as 1 and the map() call site turns it back. DPO, KTO,
+ # CPO, ORPO, Reward and PRM pass it straight to Dataset.map, where 1 is a Pool(1).
if "dataset_num_proc" in call_args:
_serial_as_none = "False" if trainer_file == "sft_trainer" else "True"
num_proc_check = (
@@ -3432,7 +3011,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += num_proc_check
- # Add padding if flex attention is added
if "pad_to_multiple_of" in call_args:
pad_to_multiple_of = (
"if os.environ.get('UNSLOTH_ENABLE_FLEX_ATTENTION', '0') == '1':\n"
@@ -3444,10 +3022,11 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += pad_to_multiple_of
- # Check for loss_type = dr_grpo and scale_rewards for GRPO
+ # Check for loss_type = dr_grpo and scale_rewards for GRPO; DAPO uses per-token loss, so BNPO loss
+ # is used. See huggingface/trl#3130 (comment 2746947835).
if "loss_type" in call_args and "scale_rewards" in call_args:
- # See https://github.com/huggingface/trl/issues/3130#issuecomment-2746947835
- # DAPO uses per token loss so BNPO loss used
+ # See https://github.com/huggingface/trl/issues/3130#issuecomment-2746947835 DAPO uses per token loss so
+ # BNPO loss used
check_dr_grpo = (
"if loss_type.lower() == 'dr_grpo':\n"
" loss_type = 'dr_grpo'\n"
@@ -3472,7 +3051,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += check_dr_grpo
- # Check GRPO num_generations mismatch
+ # Check the GRPO num_generations mismatch; if world size is not set by accelerate or torchrun at this point it is 1.
if (
"per_device_train_batch_size" in call_args
and "num_generations" in call_args
@@ -3506,7 +3085,7 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += check_num_generations
- # Check temperature must not be <= 0. Also stop if >= 10
+ # Temperature must not be <= 0, and stop if >= 10.
if "temperature" in call_args:
check_temperature = (
"if temperature <= 0:\n"
@@ -3517,22 +3096,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
extra_args += check_temperature
- # Edit config with anything extra
if trainer_file in RL_CONFIG_CHANGES:
process_extra_args = RL_CONFIG_CHANGES[trainer_file]
for process_extra_arg in process_extra_args:
extra_args += process_extra_arg(old_RLTrainer_source, old_RLConfig_source)
- # Create RLConfig args
extra_args = extra_args.split("\n")
extra_args = "\n".join(" " * 8 + x for x in extra_args)
RLConfig_arguments = arguments
RLConfig_extra_args = extra_args
RLConfig_call_args = call_args
- # TRL 0.27.0+ forces use_reentrant=False in gradient_checkpointing_kwargs.
- # Unsloth gradient checkpointing requires use_reentrant=True, so we remove
- # the setting after super().__init__() when it gets auto-applied.
+ # TRL 0.27.0+ forces use_reentrant=False in gradient_checkpointing_kwargs, but Unsloth gradient
+ # checkpointing requires True, so remove the setting after super().__init__() applies it.
RLConfig_post = ""
if trl_version >= Version("0.27.0"):
RLConfig_post = (
@@ -3542,14 +3118,12 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" del self.gradient_checkpointing_kwargs['use_reentrant']\n"
)
- # Patch vLLM and other functions
RLTrainer_extras = patch_functions(
RLTrainer, trainer_file, RLTrainer_name, all_imports, imports
)
if RLTrainer_extras is None:
RLTrainer_extras = f"_Unsloth{RLTrainer_name} = {RLTrainer_name}"
- # Create full module
exec(f"from trl.trainer import ({RLTrainer_name}, {RLConfig_name},)")
__RLTrainer_doc__ = eval(f"trl.trainer.{RLTrainer_name}").__doc__
if __RLTrainer_doc__ is None:
@@ -3558,17 +3132,14 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
if __RLConfig_doc__ is None:
__RLConfig_doc__ = ""
- # Get all pre-modules
if trainer_file in RL_PRE_ITEMS:
RL_pre = "\n".join(RL_PRE_ITEMS[trainer_file])
else:
RL_pre = ""
- # Check if SamplingParams is in there
if "SamplingParams" in old_RLTrainer_source:
RL_pre = RL_pre + "\n" + inspect.getsource(vLLMSamplingParams)
- # Selective log softmax and other functions
selective_log_softmax_code = inspect.getsource(selective_log_softmax)
grpo_selective_log_softmax_code = inspect.getsource(grpo_selective_log_softmax)
calculate_pad_tokens_in_prompt_code = inspect.getsource(calculate_pad_tokens_in_prompt)
@@ -3578,7 +3149,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
align_completion_tool_mask_code = inspect.getsource(align_completion_tool_mask)
autotune_batch_and_chunks_code = inspect.getsource(autotune_batch_and_chunks)
sanitize_logprob_code = inspect.getsource(sanitize_logprob)
- # Get final source code
RLTrainer_source = RLTrainer_replacement.format(
RLTrainer_name = RLTrainer_name,
__RLTrainer_doc__ = __RLTrainer_doc__,
@@ -3611,19 +3181,19 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
if RLTrainer_name == "GRPOTrainer":
- # Base torch_compile_options shared by all device types
+ # Base torch_compile_options shared by all device types; CUDA adds its own, and XPU / HIP / others
+ # use the base only.
base_options = """torch_compile_options = {
"epilogue_fusion" : True,
"max_autotune" : False,
"shape_padding" : True,
"trace.enabled" : False,"""
- # Generate torch_compile_options based on device type
if DEVICE_TYPE == "cuda":
# CUDA-specific options (added to base options)
cuda_options = """
"triton.enable_persistent_tma_matmul": torch.cuda.get_device_capability()[0] >= 9,"""
- # cutlass options were added in PyTorch 2.8.0
+ # cutlass options were added in PyTorch 2.8.0.
if torch_version >= Version("2.8.0"):
cuda_options += """
"cuda.cutlass_epilogue_fusion_enabled": torch.cuda.get_device_capability()[0] >= 9,
@@ -3646,10 +3216,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
RLTrainer_source = re.sub(pattern, new_options, RLTrainer_source, flags = re.DOTALL)
if trl_version >= Version("1.4.0"):
- # The `elif is_peft_model(model) and args.beta != 0.0:` ref-adapter block
- # was introduced in TRL 1.4.0 and is used through 1.7.x. Remove only that
- # block, anchored on the final ref_param copy so we do NOT also swallow the
- # following gradient-checkpointing enable_input_require_grads() block.
+ # The `elif is_peft_model(model) and args.beta != 0.0:` ref-adapter block exists from TRL
+ # 1.4.0 through 1.7.x. Anchored on the final ref_param copy so the following
+ # enable_input_require_grads() block is not swallowed.
peft_pattern = (
r"\s*elif is_peft_model\(model\) and args\.beta != 0\.0:"
r".*?"
@@ -3665,9 +3234,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
if trl_version >= Version("1.7.0"):
- # router_aux_loss_coef / aux_loss_enabled were added in TRL 1.7.0. Unsloth's
- # optimized GRPO forward cannot compute the MoE router aux loss, so reject
- # explicit opt-in (router_aux_loss_coef > 0) at init rather than silently ignoring it.
+ # router_aux_loss_coef / aux_loss_enabled arrived in TRL 1.7.0, and the optimized GRPO forward
+ # cannot compute the MoE router aux loss, so reject an explicit opt-in at init.
RLTrainer_source = RLTrainer_source.replace(
"self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0",
"self.aux_loss_enabled = is_moe and args.router_aux_loss_coef != 0.0\n"
@@ -3703,11 +3271,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
flags = re.DOTALL,
)
- # Remove TRL 0.26.0's unconditional bfloat16 cast of trainable params. It
- # hardcodes bfloat16 for QLoRA, ignoring the user's dtype and breaking
- # GradScaler with fp16=True. Unsloth already handles adapter dtype via
- # patch_model_and_tokenizer, so the block is unnecessary (and already a
- # no-op for GRPO, whose peft init block is removed above).
+ # Remove TRL 0.26.0's unconditional bfloat16 cast of trainable params: it ignores the user's
+ # dtype and breaks GradScaler with fp16=True. patch_model_and_tokenizer already handles it.
RLTrainer_source = RLTrainer_source.replace(
'if getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False):',
"if False:",
@@ -3727,11 +3292,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
)
RLTrainer_source = RLTrainer_source.replace(original_text, new_text)
- # Do NOT override _is_vlm -- let TRL detect VLM models naturally
- # (forcing _is_vlm=False errors on vision datasets in TRL 0.27.1+).
- # But some notebooks pass a bare tokenizer as processing_class, so TRL
- # sets _is_vlm=False even for VLMs; add an architecture-based override
- # before the validation check.
+ # Do NOT override _is_vlm: forcing it False errors on vision datasets in TRL 0.27.1+. A bare
+ # tokenizer as processing_class makes TRL set it False even for VLMs, so add an
+ # architecture-based override before the validation check.
_vlm_check_original = (
' self._is_vision_dataset = "image" in dataset_sample or "images" in dataset_sample\n'
" if self._is_vision_dataset and not self._is_vlm:"
@@ -3750,9 +3313,8 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
if _vlm_check_original in RLTrainer_source:
RLTrainer_source = RLTrainer_source.replace(_vlm_check_original, _vlm_check_patched)
- # TRL 0.22.x keys off _is_vlm, not _is_vision_dataset (0.24.0+), so the
- # vision-only signature columns never overlap the tokenized ones. Merge
- # both sets; _remove_unused_columns ignores extras.
+ # TRL 0.22.x keys off _is_vlm, not _is_vision_dataset (0.24.0+), so the vision-only signature
+ # columns never overlap the tokenized ones. Merge both sets; _remove_unused_columns ignores extras.
_sig_vlm_old = 'self._signature_columns = ["messages", "prompt", "completion", "images"]'
_sig_vlm_new = (
'self._signature_columns = ["messages", "prompt", "completion", "images",'
@@ -3762,16 +3324,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
RLTrainer_source = _backport_vision_dataset_gate(RLTrainer_source)
- # Inject model reference before _prepare_dataset for dynamic
- # token_type_ids detection in sft_prepare_dataset
+ # Inject the model reference before _prepare_dataset for dynamic token_type_ids detection in
+ # sft_prepare_dataset.
_prep_pattern = r"([ \t]*)train_dataset = self\._prepare_dataset\("
_prep_replacement = (
r"\1self._unsloth_model_ref = model\n\1train_dataset = self._prepare_dataset("
)
RLTrainer_source = re.sub(_prep_pattern, _prep_replacement, RLTrainer_source, count = 1)
- # Silence TRL's noisy batch_size=1 + padding-free warning (handles both
- # the original "anihilate" typo and the corrected "annihilate" spelling)
+ # Silence TRL's noisy batch_size=1 + padding-free warning, handling both the original "anihilate"
+ # typo and the corrected spelling.
for _typo in ("anihilate", "annihilate"):
_idx = RLTrainer_source.find(_typo)
if _idx == -1:
@@ -3792,28 +3354,20 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
RLTrainer_source = RLTrainer_source[:_line_start] + RLTrainer_source[_block_end:]
break
- # TRL turns a plain `TrainingArguments` into its own config with
- # `args = <X>Config(**dict_args)`. That name resolves through the generated
- # module's globals, which are imported from TRL before the config is patched
- # below, so the conversion hands back a PRISTINE config: no unsloth fields on
- # it (#3931 arrives this way too), and a class that no longer answers to its
- # own module attribute, so the first checkpoint save cannot pickle it. Build
- # the Unsloth subclass instead. Only the construction is rewritten - the
- # `isinstance(args, <X>Config)` guard next to it still wants the pristine
- # class, which the subclass satisfies.
+ # TRL converts a plain TrainingArguments with `args = <X>Config(**dict_args)`, resolved
+ # through the generated module's globals, so it hands back a PRISTINE config: no unsloth
+ # fields (#3931), and a class the first checkpoint save cannot pickle. Only the construction
+ # is rewritten; the isinstance guard still wants the pristine class.
RLTrainer_source = RLTrainer_source.replace(
f"args = {RLConfig_name}(**dict_args)",
f"args = Unsloth{RLConfig_name}(**dict_args)",
)
- # Remove multiple doc strings
if __RLConfig_doc__ != "" and RLTrainer_source.count(__RLTrainer_doc__) == 2:
RLTrainer_source = RLTrainer_source.replace(__RLTrainer_doc__, "", 1)
- # Remove multiple newlines
RLTrainer_source = re.sub(r"[\n]{3,}", "\n", RLTrainer_source)
- # Create new function
_resolved_module = _trainer_resolved_module or _config_resolved_module
_model_location = (
_resolved_module.__name__ if _resolved_module is not None else f"trl.trainer.{trainer_file}"
@@ -3829,7 +3383,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
if trainer_file == "grpo_trainer":
_patch_resume_from_checkpoint_memory(patched_trainer)
- # Patch Trainer
exec(
f"trl.{RLTrainer_name} = created_module.Unsloth{RLTrainer_name}",
locals(),
@@ -3846,7 +3399,6 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
globals(),
)
- # Patch Config
exec(
f"trl.{RLConfig_name} = created_module.Unsloth{RLConfig_name}",
locals(),
@@ -3863,13 +3415,18 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
globals(),
)
_displaced_config = None
+ # TRL 1.0.0+ wraps generation in: with torch.no_grad(), disable_gradient_checkpointing(self.model, ...): The
+ # toggle only suppresses a cosmetic PyTorch warning; under no_grad it has no functional effect. But on exit
+ # it calls gradient_checkpointing_enable(), overwriting Unsloth's custom "unsloth" wrapper -- for Gemma-4
+ # this corrupts forward numerics and blows GRPO KL divergence up to ~10^12 at step 1. Replacing the context
+ # manager with a no-op preserves Unsloth's wrapper. trl < 1.0.0 (no disable_gradient_checkpointing): early
+ # return. trl >= 1.0.0: noop is correct; only loss is the cosmetic warning.
try:
config_module_name = trainer_file.replace("_trainer", "_config")
config_module = importlib.import_module(f"trl.trainer.{config_module_name}")
if hasattr(config_module, RLConfig_name):
- # Remember what this attribute held: on the TRL releases that put a
- # thin wrapper here it is a class of its own, and its instances need
- # the same pickling fallback the pristine class gets.
+ # Remember what this attribute held: on the TRL releases that put a thin wrapper here it is a class
+ # of its own, and its instances need the same pickling fallback the pristine class gets.
_displaced_config = getattr(config_module, RLConfig_name)
setattr(
config_module,
@@ -3915,38 +3472,31 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
init = inspect.getsource(RLTrainer.__init__)
old_init = init
- # Remove brackets in comments since it interferes ie (...)
+ # Remove brackets in comments, replacing (...) with [...], since they interfere.
comments = re.findall(r"\#[^\n]{1,}\n", init)
bracketed_comments = [x for x in comments if "(" in x or ")" in x]
- # Replace with [...] instead
for bracketed_comment in bracketed_comments:
init = init.replace(
bracketed_comment,
bracketed_comment.replace("(", "[").replace(")", "]"),
)
- # Remove peft_config
init = init.replace("elif peft_config is None:", "elif False:")
init = init.replace("elif peft_config is not None:", "elif False:")
init = init.replace("if peft_config is None:", "if False:")
init = init.replace("if peft_config is not None:", "if False:")
init = init.replace("get_peft_model(model, peft_config)", "model")
- # New TRL 0.20.0
init = init.replace(
"if peft_config is not None or (is_peft_available() and isinstance(model, PeftModel)):",
"if False:",
)
- # New TRL 0.20.0
init = init.replace("model = self._prepare_peft_model(model, peft_config, args)\n", "pass\n")
- # TRL 0.22.0+ uses prepare_peft_model as a standalone function
+ # TRL 0.22.0+ uses prepare_peft_model as a standalone function.
init = init.replace("model = prepare_peft_model(model, peft_config, args)", "pass")
- # Skip add_adapter("ref") for reference model computation
- # Unsloth: We comment out the "ref" adapter creation because:
- # 1. We want to use the original BASE MODEL as the reference model, not the SFT/LoRA model
- # 2. PEFT doesn't allow multiple adapters when target_parameters is used (MoE models)
- # When "ref" is not in peft_config, GRPO/RLOO fallback uses disable_adapter()
- # which gives the base model logits - exactly what we want
+ # Skip add_adapter("ref"): the BASE model is the wanted reference, and PEFT forbids multiple
+ # adapters under target_parameters (MoE). Without "ref", GRPO/RLOO falls back to
+ # disable_adapter(), which is exactly the base model logits.
add_adapter_block_pattern = (
r"([ \t]*)" # Capture leading indentation
r"if\s+is_peft_available\(\)\s+and\s+is_peft_model\(model\)\s+and\s+args\.beta\s*!=\s*0\.0\s*:"
@@ -3960,11 +3510,10 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
indent = match.group(1)
lines = full_match.split("\n")
commented_lines = []
- # Add explanation comment first
commented_lines.append(
f"{indent}# Unsloth: Commented out - use base model as reference, not SFT/LoRA model"
)
- # Comment out each line - insert # after leading whitespace to preserve indentation
+ # Comment out each line by inserting # after the leading whitespace, to preserve indentation.
for line in lines:
if line.strip():
stripped = line.lstrip()
@@ -3976,9 +3525,8 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
init = re.sub(add_adapter_block_pattern, comment_out_block, init, flags = re.DOTALL)
- # Set use_vllm if not set
if "args.use_vllm" in init and "model" in init and "args" in init:
- # .*? matches first match. .+? matches final match.
+ # .*? matches the first match, .+? the final one.
replacer = re.findall(
r"def __init__\(.*?\).*?\:\n",
init,
@@ -3995,16 +3543,13 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
+ " " * 16
+ "args.use_vllm = True\n"
)
- # " " * 16 + "args.vllm_importance_sampling_correction = True\n" + \
- # " " * 16 + "args.vllm_importance_sampling_cap = 2.0\n"
if "grpo" in trainer_file and trl_version >= Version("0.18.0"):
- # If model has vllm_engine, then use vllm in colocate mode. Donot wait for server
+ # If the model has a vllm_engine, use vllm in colocate mode and do not wait for a server.
vllm_setter += " " * 12 + "args.vllm_mode='colocate'\n"
if trl_version >= Version("0.23.0"):
- # Align TRL sleep mode with the engine's actual enable_sleep_mode
- # (the vision standby gate may have disabled it); fall back to the
- # standby env var when the engine cannot be introspected.
+ # Align TRL sleep mode with the engine's actual enable_sleep_mode (the vision standby gate may have
+ # disabled it); fall back to the standby env var when the engine cannot be introspected.
vllm_setter += (
" " * 12
+ "_unsloth_esm = getattr(getattr(getattr(getattr(model.vllm_engine, 'llm_engine', None), 'vllm_config', None), 'model_config', None), 'enable_sleep_mode', None)\n"
@@ -4016,8 +3561,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
init = init.replace(replacer, replacer + vllm_setter)
- # breakpoint()
-
vllm_part = re.findall(
r"(\n[\s]{8}" r"if (self|args)\.use_vllm\:.*?" r"\n[\s]{8}" "else:\n)",
init,
@@ -4026,15 +3569,11 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
if len(vllm_part) == 1:
vllm_part, args = vllm_part[0][0], vllm_part[0][1]
- # Strip all comments
new_vllm_part = re.sub(
r"^\s*\#[^\n]*\n?", "", vllm_part, flags = re.MULTILINE
) # to also remove whole comment line instead of just starting at #
- new_vllm_part = re.sub(
- r"\s*\#.*$", "", new_vllm_part, flags = re.MULTILINE
- ) # remove comments that occur after code
+ new_vllm_part = re.sub(r"\s*\#.*$", "", new_vllm_part, flags = re.MULTILINE)
- # Get SamplingParams
sampling_params = re.findall(
r"\n[\s]{4,}(self\.[^\s]{1,}[\s]{0,}\=[\s]{0,}SamplingParams\(.+?\))",
new_vllm_part,
@@ -4043,14 +3582,13 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
if len(sampling_params) == 1:
sampling_params = sampling_params[0]
- # Fix guided_decoding
sampling_params = sampling_params.replace(
"guided_decoding=guided_decoding,",
"guided_decoding="
'GuidedDecodingParams(backend="outlines", regex=args.vllm_guided_decoding_regex) '
'if getattr(args, "vllm_guided_decoding_regex", None) is not None else None,',
)
- # Replace with our vLLM engine when sharing weights
+ # Replace with our vLLM engine when sharing weights.
sampling_params = (
" " * 12
+ "if getattr(getattr(model, 'vllm_engine', None), 'shared_weights', False): "
@@ -4059,7 +3597,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
+ sampling_params
)
- # count the indentation of last line of sampling_params.
splitted_sampling_params = sampling_params.split("\n")
if len(splitted_sampling_params) >= 2:
last_line = splitted_sampling_params[-1]
@@ -4067,9 +3604,8 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
last_prev_indentation = len(last_prev_line) - len(last_prev_line.lstrip())
last_indentation = len(last_line) - len(last_line.lstrip())
- # Add extra arguments to SamplingParams
extra = "**getattr(getattr(args, 'vllm_sampling_params', vLLMSamplingParams()), '_set_kwargs', {})"
- # Backwards replace
+ # Backwards replace.
to_replace = (
",\n"
+ " " * last_prev_indentation
@@ -4079,7 +3615,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
+ ")"
)
sampling_params = to_replace.join(sampling_params.rsplit(")", 1))
- # Strip multiple commas
sampling_params = re.sub(r"[\,][\s]{0,}\,", ",", sampling_params)
new_vllm_part = (
@@ -4087,8 +3622,8 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
)
if trl_version >= Version("0.18.0"):
- # Guard LLM init - use existing vLLM engine when sharing weights,
- # otherwise keep the original LLM() creation for sync/reload path
+ # Guard LLM init: use the existing vLLM engine when sharing weights, otherwise keep the original
+ # LLM() creation for the sync/reload path.
vllm_llm_init_pattern = r"(?P<indent>[ \t]*)self\.llm\s*=\s*LLM\(.*?\)*\)\s*?\n(?!,)"
def guard_llm_init(match):
@@ -4110,7 +3645,7 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
init = init.replace(vllm_part, new_vllm_part)
- # Search for vLLM calling in all child functions
+ # Search for vLLM calls in all child functions.
functions = dir(RLTrainer)
RLTrainer_source = inspect.getsource(RLTrainer)
functions = [x for x in functions if f"def {x}" in RLTrainer_source]
@@ -4136,7 +3671,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
continue
original_source = source
- # Check for function
for edit_function in edit_functions:
source = edit_function(function, source)
@@ -4153,28 +3687,25 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
source,
)
- # llm_model = self.llm.llm_engine.model_executor.driver_worker.model_runner.model
source = re.sub(
r"(\n[\s]{4,}).+?model_executor\.driver_worker.+?\n",
r"\n\1pass\n",
source,
)
- # llm_model.load_weights(model.state_dict().items())
source = re.sub(
r"(\n[\s]{4,}).+?load_weights\(.+?\n",
r"\n\1pass\n",
source,
)
- # .state_dict()
source = re.sub(
r"\.state_dict\(\)",
r"",
source,
)
- # Replace self.llm.generate and self.llm.chat with lora_request (only when sharing weights)
+ # Replace self.llm.generate and self.llm.chat with lora_request, only when sharing weights.
if "CUDA_VISIBLE_DEVICES" in os.environ:
lora_name = (
trainer_file
@@ -4192,19 +3723,14 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
+ r" else None)",
source,
)
- # All these are to fix multiple commas before lora_request (in case the original code ends with something like ",)")
- # https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py#L1388 for eg has such an ending
+ # Fix multiple commas before lora_request, in case the original code ends with ",)" as trl's
+ # grpo_trainer.py#L1388 does.
source = re.sub(r"\,[\s]{1,}\,[\s]{0,}lora_request", ", lora_request", source)
source = re.sub(r"[\s]{1,}\,[\s]{0,}lora_request", ", lora_request", source)
source = re.sub(r"[\,]{1,}[\s]{0,}lora_request", ", lora_request", source)
- # Prefer using unsloth's sampling params and fallback to trl's if not found
- # We'll enable this later separately when combining both this and GRPOConfig params
- # source = re.sub(
- # r"sampling_params\s*=\s*sampling_params",
- # r"sampling_params = getattr(self.args, 'vllm_sampling_params', sampling_params)",
- # source
- # )
- # Fix later versions of SamplingParams via grpo_update_SamplingParams
+ # Prefer Unsloth's sampling params and fall back to trl's; to be enabled once both these and
+ # GRPOConfig params are combined.
+ # Fix later versions of SamplingParams via grpo_update_SamplingParams.
source = source.replace(
"sampling_params = SamplingParams(**generation_kwargs)",
"sampling_params = SamplingParams("
@@ -4215,11 +3741,9 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
")",
)
- # Skip if no changes done
if source == original_source:
continue
- # Find all imports
imports += [x for x in all_imports if not x.startswith("_") and x in source]
changed[function] = (
@@ -4227,10 +3751,8 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
source,
)
- # Import all functions
imports = list(set(imports))
- # Patch all functions
for function in changed:
old, new = changed[function]
RLTrainer_source = RLTrainer_source.replace(old, new)
@@ -4242,7 +3764,6 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
def patch_trl_rl_trainers():
- # Patch all TRL modules if they have vLLM or PEFT
import trl.trainer
all_trainers = dir(trl.trainer)
@@ -4258,17 +3779,9 @@ def patch_trl_rl_trainers():
def patch_trl_disable_gradient_checkpointing():
- # TRL 1.0.0+ wraps generation in:
- # with torch.no_grad(), disable_gradient_checkpointing(self.model, ...):
- # The toggle only suppresses a cosmetic PyTorch warning; under no_grad it
- # has no functional effect. But on exit it calls
- # gradient_checkpointing_enable(), overwriting Unsloth's custom
- # "unsloth" wrapper -- for Gemma-4 this corrupts forward numerics and
- # blows GRPO KL divergence up to ~10^12 at step 1.
- #
- # Replacing the context manager with a no-op preserves Unsloth's wrapper.
- # trl < 1.0.0 (no disable_gradient_checkpointing): early return.
- # trl >= 1.0.0: noop is correct; only loss is the cosmetic warning.
+ # TRL 1.0.0+ wraps generation in disable_gradient_checkpointing(), which on exit calls
+ # gradient_checkpointing_enable() and overwrites Unsloth's "unsloth" wrapper: Gemma-4 forward
+ # numerics corrupt and GRPO KL hits ~1e12 at step 1. A no-op CM keeps the wrapper.
try:
import trl.models.utils as _tmu
except ImportError:
@@ -4290,10 +3803,8 @@ def patch_trl_disable_gradient_checkpointing():
_tmu.disable_gradient_checkpointing = _noop_disable_gradient_checkpointing
- # Also rebind any trl.* module that already imported the symbol by
- # reference (cached at import time). Walk sys.modules dynamically so this
- # catches every trainer doing
- # `from ...models.utils import disable_gradient_checkpointing`.
+ # Also rebind any trl.* module that imported the symbol by reference at import time, walking
+ # sys.modules so every `from ...models.utils import disable_gradient_checkpointing` is caught.
for _mod_name, _mod in list(sys.modules.items()):
if _mod is None or not _mod_name.startswith("trl."):
continue
@@ -4324,14 +3835,13 @@ def patch_trl_disable_gradient_checkpointing():
def patch_trl_openenv():
for function in RL_ADDITIONAL_FUNCTIONS["openenv"]:
logger.info(f"Unsloth: Patching trl openenv with function: {function.__name__}")
- function() # Call the function to apply the patch
+ function()
return
def patch_trl_vllm_generation():
- # trl moved vllm stuff to trl/generation/vllm_generation.py
- # We need to min_p patch it to not instantiate another vLLM instance if we already have one with fast_inference
- # Find the instance of self.llm = LLM(..) (multiline) and wrap it around an if clause
+ # trl moved vllm code to trl/generation/vllm_generation.py; patch it so it does not build a
+ # second vLLM instance when fast_inference has one: wrap the multiline `self.llm = LLM(..)`.
for function in RL_ADDITIONAL_FUNCTIONS["vllm_generation"]:
logger.info(f"Unsloth: Patching trl VLLMGeneration with function: {function.__name__}")
function()
@@ -4341,16 +3851,12 @@ def patch_trl_vllm_generation():
def PatchFastRL(algorithm = None, FastLanguageModel = None):
if FastLanguageModel is not None:
PatchRL(FastLanguageModel)
- # Under UNSLOTH_ALLOW_CPU=1 (CPU-only CI), skip TRL trainer rewriting so
- # downstream `inspect.getsource(trl.SFTTrainer)` drift detectors see the
- # pristine upstream class, not the compiled Unsloth* wrappers.
+ # Under UNSLOTH_ALLOW_CPU=1 (CPU-only CI), skip TRL trainer rewriting so downstream
+ # inspect.getsource(trl.SFTTrainer) drift detectors see the pristine upstream class.
if os.environ.get("UNSLOTH_ALLOW_CPU", "0") == "1":
return
- # Install the disable_gradient_checkpointing noop BEFORE
- # patch_trl_rl_trainers, which imports extra trl.* submodules; any module
- # imported after the sys.modules walk would keep the original broken
- # binding. Installing first ensures the canonical symbol is replaced before
- # those submodules bind it.
+ # Install the disable_gradient_checkpointing noop BEFORE patch_trl_rl_trainers, which imports
+ # more trl.* submodules: anything imported after the sys.modules walk keeps the old binding.
patch_trl_disable_gradient_checkpointing()
patch_trl_rl_trainers()
patch_trl_openenv()
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 11ebaaca1c4..3ec736431a4 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -65,7 +65,7 @@ UNSLOTH_GRPO_SEQ_PACKING_ON = os.environ.get("UNSLOTH_GRPO_SEQ_PACKING", "1").lo
"no",
"off",
)
-# Packing needs zoo#840's masked-column guard in grpo_compute_loss (installed zoo is fixed per-process).
+# Packing needs zoo#840's masked-column guard in grpo_compute_loss (the installed zoo is fixed per-process).
try:
UNSLOTH_ZOO_HAS_MASKED_COL_GUARD = "torch.where(_keep, new" in inspect.getsource(
RL_REPLACEMENTS["grpo_compute_loss"]
@@ -137,7 +137,6 @@ except Exception:
trl_version = Version("0.0.0")
-# Check untrained tokens
def sft_trainer_fix_untrained_tokens(call_args, extra_args):
if "model" in call_args and "train_dataset" in call_args:
fix_tokenizer = (
@@ -155,9 +154,8 @@ def sft_trainer_fix_untrained_tokens(call_args, extra_args):
RL_EXTRA_ARGS["sft_trainer"].append(sft_trainer_fix_untrained_tokens)
-# Fix top_k for GRPO vLLM.
-# https://github.com/huggingface/trl/pull/4695 with this change trl added top_k in GRPOConfig and defaults to 0
-# We don't want that since vllm's all include top_k is -1 and 0 returns an error on SamplingParams creation.
+# huggingface/trl#4695 added top_k to GRPOConfig defaulting to 0, but vLLM's include-all top_k
+# is -1 and 0 errors on SamplingParams creation.
def grpo_config_fix_vllm_top_k(old_RLTrainer_source, old_RLConfig_source):
return "if use_vllm and (top_k is None or top_k == 0): top_k = -1\n"
@@ -497,21 +495,17 @@ def _require_replace(
return function.replace(old, new, count)
-# The one line every unsloth_zoo revision of sft_prepare_dataset ends its worker
-# count on, as a regex so a renamed right-hand side still matches and the
-# indentation is carried into the replacement. `git log -S` on the Zoo says this
-# line last changed in Aug 2025 (#257) and has survived 24 later commits to
-# dataset_utils.py, while the block around it was rewritten three times in 2026
-# alone (#473, #553, #733) -- which is why it is the fallback rather than the
-# primary anchor.
+# The one line every unsloth_zoo sft_prepare_dataset ends its worker count on, as a regex so a
+# renamed right-hand side still matches and the indentation carries over. Unchanged since Aug
+# 2025 (#257) while the block around it was rewritten repeatedly, hence a fallback anchor.
_ZOO_MAP_NUM_PROC_ASSIGNMENT = re.compile(
r"^(?P<indent>[ \t]*)map_kwargs\[\"num_proc\"\][ \t]*=[ \t]*[^\n]+$",
flags = re.MULTILINE,
)
-# The Zoo seeds its truncation length from args.max_length and only falls through
-# to args.max_seq_length when that is 0. rl.py now hands TRL >= 1.0.0 a None
-# there, so normalise it or the fall-through is skipped and nothing truncates.
+# The Zoo seeds its truncation length from args.max_length and falls through to
+# args.max_seq_length only when that is 0; rl.py hands TRL >= 1.0.0 a None, so normalise it or
+# nothing truncates.
_ZOO_MAX_LENGTH_SEED = re.compile(
r"^(?P<indent>[ \t]*)max_seq_length[ \t]*=[ \t]*getattr\(args,[ \t]*[\"']max_length[\"'],[ \t]*0\)[ \t]*$",
flags = re.MULTILINE,
@@ -560,14 +554,10 @@ def _replace_or_fallback(
`consequence` says what that absence does, since the warning below speaks only
about the worker count.
"""
- # Already done upstream, and checked FIRST. An edit is only missing if its
- # RESULT is missing, and a Zoo that adopts the replacement itself is the
- # forward case this has to survive. Ordering matters twice over: `old` is a
- # prefix of `new` for the max_length seed, so the wide anchor below matches
- # the already-normalized line and appended a second `or 0` to it, while a
- # differently spelled upstream form matches no anchor at all and used to
- # raise under `required = True` -- failing every SFT trainer over behaviour
- # that is already present.
+ # Already done upstream, and checked FIRST: an edit is missing only if its RESULT is, and a Zoo
+ # that adopts the replacement is the forward case. Order matters: `old` is a prefix of `new`
+ # for the max_length seed, so the wide anchor matched the normalized line and appended a
+ # second `or 0`.
if _same_source(new) in _same_source(function):
return function
if old in function:
@@ -611,13 +601,10 @@ def sft_trainer_prepare_dataset(function_name, function):
flags = re.MULTILINE | re.DOTALL,
)
if matched:
- # Use fast version!
function = inspect.getsource(fast_sft_prepare_dataset)
- # why: anchor the wrapped-packing setup on the function signature -- a
- # structural anchor that always exists -- not the unsloth_zoo license comment,
- # which is only lower-bounded and a newer Zoo may move or drop. Anchoring there
- # let the setup silently no-op while edits below referenced its variables,
- # NameError-ing every SFT dataset prep. Fail loudly if the signature is missing.
+ # Anchor the wrapped-packing setup on the function signature, which always exists, not the
+ # unsloth_zoo license comment, which a newer Zoo may move: anchoring there let the setup
+ # silently no-op while later edits used its variables, NameError-ing every SFT dataset prep.
function, _n_setup = re.subn(
r"(def sft_prepare_dataset\s*\(.*?\)\s*(?:->[^:\n]*)?:[ \t]*\n)",
lambda match: match.group(1) + _WRAPPED_PACKING_SETUP,
@@ -637,19 +624,16 @@ def sft_trainer_prepare_dataset(function_name, function):
fallback_pattern = _ZOO_MAX_LENGTH_SEED,
fallback_new = r'\g<indent>max_seq_length = getattr(args, "max_length", 0) or 0',
where = "sft_prepare_dataset max_length seed",
- # Not optional, unlike the worker count this helper was written for.
- # The generated trainer clears `args.max_length` for padding-free, so
- # an unrewritten seed reads that `None` instead of the `0` that makes
- # the guard below fall through to `max_seq_length`: raw datasets stop
- # being truncated, or `None > 0` raises. Both anchors missing means
- # the neighbouring _require_replace edits have almost certainly gone
- # too, so this raises where they do rather than one call later.
+ # Not optional, unlike the worker count this helper was written for: the generated trainer
+ # clears args.max_length for padding-free, so an unrewritten seed reads None instead of the 0
+ # that falls through to max_seq_length. Both anchors missing means the neighbouring
+ # _require_replace edits have gone too.
required = True,
consequence = ", so `max_length` would not be enforced for raw "
"datasets under padding-free batching",
)
- # why: route each edit through _require_replace so a drifted anchor fails
- # loudly instead of leaving a dangling reference to the setup variables.
+ # Route each edit through _require_replace so a drifted anchor fails loudly instead of leaving
+ # a dangling reference to the setup variables.
function = _require_replace(
function,
"truncation = do_truncation,",
@@ -662,9 +646,9 @@ def sft_trainer_prepare_dataset(function_name, function):
"if do_truncation and not _unsloth_wrapped_packing and max_seq_length > 0:",
where = "sft_prepare_dataset truncation guard",
)
- # why: reuse the guarded _unsloth_pack_has_strategy from the setup instead of
- # re-calling _inspect.signature(pack_dataset) here -- the setup wraps that call
- # in try/except, so a non-introspectable pack_dataset must not crash here.
+ # Reuse the guarded _unsloth_pack_has_strategy from the setup rather than re-calling
+ # _inspect.signature(pack_dataset): the setup wraps that in try/except, so a
+ # non-introspectable pack_dataset must not crash here.
function = _require_replace(
function,
"""dataset = pack_dataset(
@@ -683,14 +667,10 @@ def sft_trainer_prepare_dataset(function_name, function):
)""",
where = "sft_prepare_dataset pack_dataset call",
)
- # why: the map() call site -- not the config -- is where the worker
- # count must be made safe. The Zoo copy asks stdlib `multiprocessing`
- # for a start method datasets takes from `multiprocess`, and its
- # low-memory branch yields 1, which still builds a Pool(1) on
- # datasets >= 4.1. The shared helper caps a config None instead of
- # the Zoo's uncapped `cpu_count + 4 -> 64`. Imported from the Zoo so
- # generated source never imports back into its generator;
- # unsloth.dataset_num_proc is only the fallback for an older Zoo.
+ # The map() call site, not the config, is where the worker count is made safe: the Zoo copy
+ # asks stdlib multiprocessing for a start method datasets takes from multiprocess, and its
+ # low-memory branch yields 1, still a Pool(1) on datasets >= 4.1. Imported from the Zoo so
+ # generated source never imports back into its generator.
function = _replace_or_fallback(
function,
"""if not isinstance(dataset, IterableDataset):
@@ -716,17 +696,10 @@ def sft_trainer_prepare_dataset(function_name, function):
map_kwargs["num_proc"] = _unsloth_get_dataset_num_proc(
getattr(args, "dataset_num_proc", None)
)""",
- # This block is the likeliest anchor in the file to drift -- the
- # Zoo rewrote it three times in 2026 -- but its absence is not
- # harmless, so it cannot simply be optional: the config layer
- # encodes serial as `1` for this site to turn back into None, and
- # an un-rewritten Zoo hands that `1` to Dataset.map. The fallback
- # keys on the assignment the block ends with, which has not
- # changed since Aug 2025, and reads args rather than the Zoo's
- # own local so it is the same computation the block anchor above
- # installs; the Zoo's now-dead sizing runs and is discarded.
- # Hard-failing instead would break every install on a newer Zoo,
- # which is a far larger blast radius than one worker.
+ # The likeliest anchor here to drift, but its absence is not harmless: the config layer encodes
+ # serial as 1 for this site to turn back into None, and an un-rewritten Zoo hands that 1 to
+ # Dataset.map. The fallback keys on the block's closing assignment, unchanged since Aug 2025.
+ # Hard-failing instead would break every install on a newer Zoo.
fallback_pattern = _ZOO_MAP_NUM_PROC_ASSIGNMENT,
fallback_new = r"""\g<indent>try:
\g<indent> from unsloth_zoo.dataset_num_proc import get_dataset_num_proc as _unsloth_get_dataset_num_proc
@@ -737,12 +710,9 @@ def sft_trainer_prepare_dataset(function_name, function):
\g<indent>)""",
where = "sft_prepare_dataset dataset_num_proc selection",
)
- # why: datasets never reads the child's exit status, so every worker
- # death -- OOM kill, segfault, real exception -- flattens into "One
- # of the subprocesses has abruptly died during map operation". Wrap
- # both tokenizing map() calls (count = 2: the prompt-completion and
- # plain-text branches) so the user gets the worker count, start
- # method and implied memory instead.
+ # datasets never reads the child's exit status, so every worker death flattens into "One of the
+ # subprocesses has abruptly died during map operation". Wrap both tokenizing map() calls so the
+ # user gets the worker count, start method and implied memory.
function = _require_replace(
function,
""" with _w.catch_warnings():
@@ -755,11 +725,9 @@ def sft_trainer_prepare_dataset(function_name, function):
_w.filterwarnings("ignore", message=".*couldn't be hashed properly.*")""",
count = 2,
where = "sft_prepare_dataset tokenizing map() calls",
- # required = False, like the selection above: this only improves the
- # message a dead worker produces, and a diagnostic must not be able
- # to fail a training run because a Zoo release moved the line it
- # anchors on. test_zoo_sft_prepare_dataset_anchor_has_not_drifted is
- # what notices, in CI rather than in someone's run.
+ # required = False, like the selection above: this only improves a dead worker's message, and a
+ # diagnostic must not fail a run because a Zoo release moved its anchor.
+ # test_zoo_sft_prepare_dataset_anchor_has_not_drifted notices in CI instead.
required = False,
)
function = function.split("\n")
@@ -800,7 +768,7 @@ def sft_trainer_prepare_dataset(function_name, function):
check_text = "\n".join(" " * 8 + x for x in check_text)
check_text = check_text.rstrip() + "\n"
- # .*? matches first match. .+? matches final match.
+ # .*? matches the first match, .+? the final one.
replacer = re.findall(
r"def " + function_name + r"\(.*?\).*?\:\n",
function,
@@ -823,8 +791,7 @@ def sft_trainer_prepare_dataset(function_name, function):
RL_FUNCTIONS["sft_trainer"].append(sft_trainer_prepare_dataset)
-# Ignore mean_token_accuracy since it needs logits
-# We override it directly with our version
+# Ignore mean_token_accuracy since it needs logits; it is overridden with our version.
def sft_trainer_compute_loss(function_name, function):
if function_name != "compute_loss":
return function
@@ -851,8 +818,8 @@ def sft_trainer_compute_loss(function_name, function):
RL_FUNCTIONS["sft_trainer"].append(sft_trainer_compute_loss)
-# Route ORPO/CPO row tokenization through the underlying text tokenizer when the
-# processing class is a multimodal processor; CPO reuses this code (#4952).
+# Route ORPO/CPO row tokenization through the underlying text tokenizer when the processing
+# class is a multimodal processor; CPO reuses this code (#4952).
def orpo_trainer_text_tokenizer(function_name, function):
if function_name == "build_tokenized_answer":
function = re.sub(
@@ -898,11 +865,9 @@ RL_FUNCTIONS["orpo_trainer"].append(orpo_trainer_text_tokenizer)
RL_FUNCTIONS["cpo_trainer"].append(orpo_trainer_text_tokenizer)
-# Resolve `processing_class.pad_token_id` through the underlying tokenizer when
-# a multimodal processor is supplied (processors lack `pad_token_id`). Without
-# this, ORPO/CPOTrainer.__init__ raises AttributeError on
-# `DPODataCollatorWithPadding(pad_token_id=processing_class.pad_token_id, ...)`
-# and on `self.padding_value = ... else processing_class.pad_token_id`.
+# Resolve processing_class.pad_token_id through the inner tokenizer when a multimodal processor
+# is supplied: processors lack pad_token_id, so ORPO/CPOTrainer.__init__ raises AttributeError
+# in the collator and padding_value.
_PAD_FALLBACK = (
"(getattr(processing_class, 'pad_token_id', None) "
"if getattr(processing_class, 'pad_token_id', None) is not None "
@@ -913,12 +878,9 @@ _PAD_FALLBACK = (
def orpo_trainer_processor_pad_token(function_name, function):
if function_name != "__init__":
return function
- # Multimodal processors (e.g. Gemma3/Gemma4 Processor) expose pad_token /
- # eos_token on `.tokenizer`, not on the processor itself. TRL 1.x CPO/ORPO
- # __init__ defaults `processing_class.pad_token` from `.eos_token` before
- # tokenizing, which AttributeErrors on such a processor. Route the default
- # through the inner tokenizer. Older TRL lacks this block, so the sub is a
- # no-op there and only the pad_token_id fallback below applies.
+ # Multimodal processors expose pad_token / eos_token on .tokenizer, not on themselves, and TRL
+ # 1.x CPO/ORPO __init__ defaults pad_token from eos_token before tokenizing. Older TRL lacks
+ # this block, so the sub is a no-op there.
function = re.sub(
r"(?m)^([ \t]*)if processing_class\.pad_token is None:\n"
r"\1[ \t]+processing_class\.pad_token\s*=\s*processing_class\.eos_token\n",
@@ -937,8 +899,8 @@ RL_FUNCTIONS["orpo_trainer"].append(orpo_trainer_processor_pad_token)
RL_FUNCTIONS["cpo_trainer"].append(orpo_trainer_processor_pad_token)
-# Fix bare pop("push_to_hub_token") in compiled SFT/IterativeSFT trainer __init__
-# On transformers 5.0+, to_dict() no longer includes push_to_hub_token, so bare pop KeyErrors
+# Fix the bare pop("push_to_hub_token") in the compiled SFT/IterativeSFT __init__: on
+# transformers 5.0+ to_dict() no longer includes it, so a bare pop KeyErrors.
def sft_trainer_push_to_hub_token(function_name, function):
if function_name != "__init__":
return function
@@ -950,7 +912,7 @@ def sft_trainer_push_to_hub_token(function_name, function):
RL_FUNCTIONS["sft_trainer"].append(sft_trainer_push_to_hub_token)
-# Autocast precision for GRPO
+# Autocast precision for GRPO.
def _unsloth_grpo_autocast(self):
"""Decide the GRPO autocast once and latch it on the trainer.
@@ -963,25 +925,25 @@ def _unsloth_grpo_autocast(self):
use_bf16 = getattr(args, "bf16", None)
use_fp16 = getattr(args, "fp16", None)
if not isinstance(precision, str):
- # transformers < 5 has no args.mixed_precision, but rl.py sets the
- # fp16 / bf16 flags on this same args for every branch it takes.
+ # transformers < 5 has no args.mixed_precision, but rl.py sets the fp16 / bf16 flags on this
+ # same args for every branch it takes.
if isinstance(use_bf16, bool) and isinstance(use_fp16, bool):
precision = "bf16" if use_bf16 else ("fp16" if use_fp16 else "no")
else:
precision = os.environ.get("ACCELERATE_MIXED_PRECISION", "fp16")
self._autocast_dtype = torch.float16 if precision == "fp16" else torch.bfloat16
- # "no" is a real value: full finetuning and an explicit float32 load
- # both set it, and reading it as bfloat16 raises on a T4 or V100.
+ # "no" is a real value: full finetuning and an explicit float32 load both set it, and reading
+ # it as bfloat16 raises on a T4 or V100.
self._autocast_enabled = precision != "no"
self._autocast_force_float32 = False
- # Stamped by from_pretrained: UNSLOTH_FORCE_FLOAT32 is process wide, so a
- # model loaded after this trainer was built would answer for it here.
+ # Stamped by from_pretrained: UNSLOTH_FORCE_FLOAT32 is process wide, so a model loaded after
+ # this trainer was built would answer for it here.
forced = getattr(getattr(self, "model", None), "_unsloth_forced_float32", None)
if forced is None:
forced = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"
if forced and precision != "bf16":
- # Gemma3 / gpt-oss set "no" but still want float16 autocast. A trainer
- # already on bf16 keeps it: float16 is what the forced list avoids.
+ # Gemma3 / gpt-oss set "no" but still want float16 autocast; a trainer already on bf16 keeps
+ # it, since float16 is what the forced list avoids.
self._autocast_dtype = torch.float16
self._autocast_enabled = True
self._autocast_force_float32 = True
@@ -995,8 +957,8 @@ def _unsloth_grpo_autocast_kwargs(self, device_type = "cuda"):
if not getattr(self, "_autocast_force_float32", False) and torch.is_autocast_enabled(
device_type
):
- # Already inside an autocast: inherit its dtype by omitting the key, since
- # autocast passes whatever it gets straight to set_autocast_dtype.
+ # Already inside an autocast: inherit its dtype by omitting the key, since autocast passes
+ # whatever it gets to set_autocast_dtype.
return {"enabled": enabled}
return {"enabled": enabled, "dtype": dtype}
@@ -1005,8 +967,7 @@ def grpo_trainer__prepare_inputs(function_name, function):
if function_name != "_prepare_inputs":
return function
- # Latched on the trainer, so a second trainer's __init__ cannot change this
- # trainer's autocast mid run.
+ # Latched on the trainer, so a second trainer's __init__ cannot change this trainer's autocast mid run.
function = function.replace(
"with torch.inference_mode():",
"with torch.inference_mode(), "
@@ -1022,8 +983,8 @@ def grpo_trainer__prepare_inputs(function_name, function):
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__prepare_inputs)
-# Guard reload_weights and sync_weights - skip when fast inference LoRA shares weights with vLLM
-# https://github.com/huggingface/trl/commit/7856d3b1f6518601732f489883b341bb6dd36434#diff-964e6fd373aa93037604064cb2b822d7f8e2735e33f791065acf2c4c3552d393R1168-R1169
+# Guard reload_weights and sync_weights: skip when fast inference LoRA shares weights with
+# vLLM (huggingface/trl commit 7856d3b).
def _guard_vllm_sync_reload_for_shared_weights(function):
# Guard reload_weights - only call when not sharing weights with vLLM
reload_weights_pattern = re.compile(
@@ -1065,22 +1026,10 @@ def grpo_trainer__generate_single_turn(function_name, function):
function = _guard_vllm_sync_reload_for_shared_weights(function)
- # TRL 0.24.0-0.25.1 truncation regression fix
- #
- # TRL 0.22.2-0.23.1 used smart truncation via truncate_with_protected_tokens():
- # - Tokenizes first without truncation
- # - Then truncates keeping the RIGHTMOST tokens (preserves assistant turn)
- # - Protects special tokens (image_token, vision_start/end) from removal
- #
- # TRL 0.24.0-0.25.1 removed this and passed kwargs directly to the tokenizer:
- # max_length=self.max_prompt_length, truncation=True, add_special_tokens=False
- # This causes issues because tokenizer truncation doesn't protect special tokens
- # and may not preserve the end of the prompt properly.
- #
- # TRL 0.26.2+ removed these kwargs entirely (no tokenizer-level truncation).
- #
- # Fix: Remove these kwargs so TRL 0.24.0-0.25.1 behaves like 0.26.2+ (no truncation).
- # This is a no-op for versions that don't have these kwargs (0.22.2-0.23.1, 0.26.2+).
+ # TRL 0.24.0-0.25.1 truncation regression: 0.22.2-0.23.1 used truncate_with_protected_tokens
+ # (tokenize, keep the RIGHTMOST tokens, protect vision tokens), 0.24.0-0.25.1 passed
+ # max_length/truncation to the tokenizer, which protects nothing, and 0.26.2+ removed those
+ # kwargs. Dropping them makes 0.24.0-0.25.1 behave like 0.26.2+; a no-op elsewhere.
for pattern in [
r'["\']?max_length["\']?\s*[:=]\s*self\.max_prompt_length\s*,\s*\n?',
r'["\']?truncation["\']?\s*[:=]\s*True\s*,\s*\n?',
@@ -1119,7 +1068,8 @@ def grpo_trainer__generate(function_name, function):
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__generate)
-# Fix incorrect special tokens handling and truncation in older TRL versions
+# Older TRL mishandles special tokens: 0.19.0 passed skip_special_tokens = True where it
+# should be False.
def grpo_trainer__generate_and_score_completions(function_name, function):
if function_name != "_generate_and_score_completions":
return function
@@ -1130,10 +1080,10 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
"prompt_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False",
)
- # Left pad prompt before calculation old and ref hidden states
+ # Left pad the prompt before calculating old and ref hidden states.
line_to_replace = 'batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size'
- # The new multi-line string that will replace the line above
+ # The new multi-line string that replaces the line above.
replacement_lines = """
max_left_pad = None
batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size
@@ -1165,7 +1115,6 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
if self.args.gradient_accumulation_steps % generate_every != 0 or (
self.use_vllm
):"""
- # Use re.sub() to perform the replacement
function, num_replacements = pattern_to_find.subn(replacement_text, function)
pattern_to_find = re.compile(
@@ -1176,8 +1125,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
re.DOTALL | re.MULTILINE,
)
- # sanitize_logprob is injected as a module-level function via RLTrainer_replacement
- # template in rl.py (from RL_REPLACEMENTS), so just reference it directly here.
+ # sanitize_logprob is injected as a module-level function by the RLTrainer_replacement template
+ # in rl.py, so reference it directly.
replacement_text = (
r"\1all_logprobs = [\n"
r"\1 [sanitize_logprob(next(iter(logprob.values()))) for logprob in output.logprobs]\n"
@@ -1188,7 +1137,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
function, num_replacements = pattern_to_find.subn(replacement_text, function)
- # Always between max_prompt_length and use_vllm
+ # Always between max_prompt_length and use_vllm.
found = re.findall(
r"\n(([ ]{8,})if self\.max_prompt_length is not None:.*?\2if self\.use_vllm:)",
function,
@@ -1226,9 +1175,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
if self.use_vllm:"""
function = function.replace(replace_part, new_replacement)
- # Important note: we disable TRL's importance sampling logic
- # It is disabled because the LLM path moves left padding to the right.
- # We must adjust the vLLM sampling_logprob tensor in Unsloth to account for this.
+ # TRL's importance sampling is disabled because the LLM path moves left padding to the right,
+ # so Unsloth adjusts the vLLM sampling_logprob tensor itself.
string_to_find = "if self.use_vllm and self.vllm_importance_sampling_correction:"
replacement_string = "if False and self.use_vllm and self.vllm_importance_sampling_correction:"
@@ -1250,8 +1198,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
function = function.replace(string_to_find, replacement_string)
- # TRL 0.24.0+ extracts prompts = [x["prompt"] for x in inputs], losing metadata
- # like reasoning_effort. Inject code to store per-sample chat_template_kwargs on self.
+ # TRL 0.24.0+ extracts prompts = [x["prompt"] for x in inputs], losing metadata like
+ # reasoning_effort, so inject code storing per-sample chat_template_kwargs on self.
_metadata_extraction = (
"\n"
" # Unsloth: Extract per-sample chat_template_kwargs before metadata is lost\n"
@@ -1267,7 +1215,6 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
" _kw_[_k_] = _inp_[_k_]\n"
" self._unsloth_batch_chat_kwargs.append(_kw_)\n"
)
- # Insert after: prompts = [x["prompt"] for x in inputs]
_target_line = 'prompts = [x["prompt"] for x in inputs]'
if _target_line in function:
function = function.replace(
@@ -1275,7 +1222,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
_target_line + _metadata_extraction,
)
- # This path is for TRL 0.24.0 images is a variable exclusive to this version
+ # This path is for TRL 0.24.0: `images` is a variable exclusive to that version.
string_to_find = """ if images is not None:
output["num_images"] = num_images"""
@@ -1292,7 +1239,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
function = function.replace(string_to_find, replacement_string)
if trl_version >= Version("0.24.0"):
- # We replace the call using 'completions' with one using 'completions_text'
+ # Replace the call using 'completions' with one using 'completions_text'.
string_to_find = " rewards_per_func = self._calculate_rewards(inputs, prompts, completions, completion_ids_list)"
replacement_string = (
" if images is not None:\n"
@@ -1315,14 +1262,13 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
)
if "wake_up()" not in function:
- # Sleep functionality has been added to trl in v0.23.0. We do not want to redo this.
- # https://github.com/huggingface/trl/commit/edbe8234bc7e528f72ac76607de9d3e4753e2709
+ # Sleep functionality was added to trl in v0.23.0 (commit edbe823), so do not redo it.
pattern = re.compile(r".*self\.llm\.generate\(.*\).*", re.MULTILINE)
matches = list(pattern.finditer(function))
patched = function
- # Generally there's only one match. But this is just to make sure we don't miss any.
+ # Generally there is only one match; the loop is to make sure none are missed.
for match in reversed(matches):
line = match.group(0)
indent_match = re.match(r"(\s*)", line)
@@ -1449,7 +1395,6 @@ def grpo_trainer_fix_maybe_apply_chat_template(function_name, function):
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_fix_maybe_apply_chat_template)
-# Remove _move_model_to_vllm
def grpo_trainer__move_model_to_vllm(function_name, function):
if function_name != "_move_model_to_vllm":
return function
@@ -1464,7 +1409,7 @@ def grpo_trainer__move_model_to_vllm(function_name, function):
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__move_model_to_vllm)
-# Edit _get_per_token_logps to handle mixed precision
+# Edit _get_per_token_logps to handle mixed precision.
def grpo_trainer__get_per_token_logps(function_name, function):
if function_name != "_get_per_token_logps":
return function
@@ -1479,7 +1424,6 @@ def grpo_trainer__get_per_token_logps(function_name, function):
):
if True: # os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0':
return None # Unsloth efficient GRPO
- # Otherwise, calculate normally:
_unsloth_grpo_autocast(self)
os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"
@@ -1488,30 +1432,15 @@ def grpo_trainer__get_per_token_logps(function_name, function):
dtype = self._autocast_dtype,
enabled = getattr(self, "_autocast_enabled", True),
):
- # We add 1 to `logits_to_keep` because the last logits of the sequence is later excluded
+ # logits_to_keep gets 1 added because the last logit of the sequence is excluded later.
logits = model(
input_ids = input_ids,
attention_mask = attention_mask,
logits_to_keep = logits_to_keep + 1,
).logits
- # logits = logits[:, :-1, :] # (B, L-1, V), exclude the last logit: it corresponds to the next token pred
return logits
- # input_ids = input_ids[:, -logits_to_keep:]
- # For transformers<=4.48, logits_to_keep argument isn't supported, so here we drop logits ourselves.
- # See https://github.com/huggingface/trl/issues/2770
- # logits = logits[:, -logits_to_keep:]
- # return logits
- # See https://huggingface.co/blog/the_n_implementation_details_of_rlhf_with_ppo#policy-training-implementation-details
- # logits = logits / self.temperature
- # logps = selective_log_softmax(logits, input_ids)
-
- # row_indices, col_indices = torch.where(logps < -20)
-
- # # Method 1: Check if tensors have elements
- # if len(row_indices) > 0 and len(col_indices) > 0:
- # breakpoint() # Breakpoint triggered here
- # print("Found high values!")
- # return logps # compute logprobs for the input tokens
+ # transformers <= 4.48 does not support logits_to_keep, so drop the logits here; see
+ # huggingface/trl#2770.
function = inspect.getsource(_get_per_token_logps)
return function
@@ -1524,7 +1453,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
if function_name != "_get_per_token_logps_and_entropies":
return function
- # Just copy over from _get_per_token_logps replacement function above. For now this returns None anyway
+ # Copied from the _get_per_token_logps replacement above; for now this returns None anyway.
def _get_per_token_logps_and_entropies(
self,
model,
@@ -1537,9 +1466,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
*args,
**kwargs,
):
- # All Unsloth code here in this function is licensed under AGPL3
- # if True: # os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0':
- # return None, None # logps, entropies Unsloth efficient GRPO
+ # All Unsloth code in this function is licensed under AGPL3.
if compute_efficient:
return None, None
else:
@@ -1556,7 +1483,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
kwargs.get("image_sizes", None),
)
num_images = kwargs.get("num_images", None)
- # Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models
+ # Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models.
token_type_ids = kwargs.get("token_type_ids", None)
mm_token_type_ids = kwargs.get("mm_token_type_ids", None)
if mm_token_type_ids is not None or image_grid_thw is not None:
@@ -1568,8 +1495,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
lm_head = self.model.get_output_embeddings().weight
- # Size on the dtype the forward actually runs in: with autocast off that
- # is the model's own dtype, float32 or bfloat16 depending on the load.
+ # Size on the dtype the forward actually runs in: with autocast off that is the model's own dtype.
forward_dtype = (
self._autocast_dtype if getattr(self, "_autocast_enabled", True) else lm_head.dtype
)
@@ -1624,8 +1550,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
rows_per_image = image_grid_thw.prod(dim = -1)
rows_per_sample = torch.split(rows_per_image, num_images)
rows_per_sample = torch.stack([s.sum() for s in rows_per_sample])
- # why: cum_rows is indexed via .item() inside the per-chunk loop;
- # keeping it on CPU avoids per-iteration GPU->CPU sync.
+ # cum_rows is indexed via .item() inside the per-chunk loop, so keeping it on CPU avoids a
+ # per-iteration GPU->CPU sync.
cum_rows = torch.cat(
[
torch.tensor([0], device = rows_per_sample.device),
@@ -1660,7 +1586,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
mm_token_type_ids_chunks = []
current_pixel_idx = 0
- # TRL 0.23.0 batching logic
+ # TRL 0.23.0 batching logic.
for start in range(0, total_samples, batch_size):
end = min(start + batch_size, total_samples)
@@ -1725,8 +1651,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
temperature = self.temperature
model_config = _unsloth_get_model_config(model)
if detect_logit_transforms is not None:
- # model_config, not model: under DDP/Accelerate `model` is a wrapper that
- # does not forward `.config`, so the helper would report zeros.
+ # model_config, not model: under DDP/Accelerate `model` is a wrapper that does not forward
+ # .config, so the helper would report zeros.
_transforms = detect_logit_transforms(model_config)
logit_softcapping = _transforms["logit_softcapping"]
logit_scale_multiply = _transforms["logit_scale_multiply"]
@@ -1752,27 +1678,24 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
)
os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"
+ # Sequence packing (default on; UNSLOTH_GRPO_SEQ_PACKING=0 disables): one varlen [1, sum L]
+ # forward replaces the padded [B, Lmax] loop and fixes the left-pad RoPE error. Self-verified
+ # against the per-row forward, re-checked as T grows, and falls back if a backend ignores
+ # packed_seq_lengths.
# ---- Sequence packing (default-on; disable with UNSLOTH_GRPO_SEQ_PACKING=0) ----
- # One varlen [1, sum L] forward replaces the padded [B, Lmax] loop (also fixes the
- # left-pad RoPE error). Self-verified against the per-row forward, re-checked as T
- # grows; falls back if a backend ignores packed_seq_lengths.
logprobs = None
- # ---- PrefixGrouper (GRPO shared-prompt dedup; default ON, exact + self-verified) ----
- # G completions per prompt share the prefix; the packed path forwards it G times,
- # PrefixGrouper stores it once (FlexAttention shared-prefix mask), cutting the trunk
- # forward from G*(P+R) to P+G*R tokens. Gated by UNSLOTH_GRPO_PREFIX_GROUPER (needs
- # seq-packing), tok_r auto-gate, and first-use self-verify vs the packed path
- # (mismatch => fall back + mark unsafe), so a mask/isolation regression cannot ship
- # silently. When off / ungrouped / unverified, the packed path below runs as before.
+ # PrefixGrouper (GRPO shared-prompt dedup, default ON): G completions share the prompt, so
+ # storing it once behind a FlexAttention shared-prefix mask cuts the trunk forward from
+ # G*(P+R) to P+G*R tokens. Gated by UNSLOTH_GRPO_PREFIX_GROUPER, a tok_r auto-gate and a
+ # first-use self-verify, so a mask/isolation regression cannot ship silently.
_pg_result = None
_pg_use = False
_pg_skip_pk = False # once a shape is PG-verified, skip the full-row forward
_pg_forward_fn = None # deferred PG forward (runs at the verify site below)
_pg_num_gen = getattr(self, "num_generations", None)
- # Env gate hoisted to module level (mirrored via RL_PRE_ITEMS). Skip PG under vLLM
- # (fast_inference=True): the rollout dominates the step, so PG saves little and its
- # first-use self-verify is net overhead.
+ # Env gate hoisted to module level (mirrored via RL_PRE_ITEMS). Skip PG under vLLM: the rollout
+ # dominates the step, so PG saves little and its self-verify is net overhead.
_pg_engage = (
UNSLOTH_GRPO_PREFIX_GROUPER_ON
and not getattr(self, "use_vllm", False)
@@ -1780,10 +1703,9 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
)
if _pg_engage:
try:
- # Skip softcap models (the flex kernel never applies attn_logit_softcapping)
- # and hybrid SSM / MoE models: only the threaded attention forwards get the
- # shared-prefix isolation, so a Mamba or MoE decoder that does not forward
- # prefix_seg_info would leak suffixes across completions. PG also rides on
+ # Skip softcap models (the flex kernel never applies attn_logit_softcapping) and hybrid SSM /
+ # MoE models: only the threaded attention forwards get shared-prefix isolation, so a decoder
+ # that does not forward prefix_seg_info leaks suffixes across completions. PG also rides on
# sequence packing, so it needs the same zoo masked-column guard.
_pg_cfg = getattr(unwrapped_model, "config", None)
_pg_engage = (
@@ -1795,8 +1717,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
and _pg_num_gen is not None
and _pg_num_gen >= 2
and not getattr(_pg_cfg, "attn_logit_softcapping", None)
- # normal backends apply config.attention_dropout in training; the flex
- # path is deterministic, so skip PG when it is set.
+ # Normal backends apply config.attention_dropout in training; the flex path is deterministic,
+ # so skip PG when it is set.
and not getattr(_pg_cfg, "attention_dropout", 0)
and not any(
getattr(_pg_cfg, _pg_a, None) is not None
@@ -1816,7 +1738,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
if _pg_engage:
try:
_pg_pad = self.processing_class.pad_token_id
- # cap the PG span (P+max(R)) at the sliding window, like the packed _pk_sw guard.
+ # Cap the PG span (P+max(R)) at the sliding window, like the packed _pk_sw guard.
_pg_sw = getattr(
getattr(unwrapped_model, "config", None), "sliding_window", None
)
@@ -1869,14 +1791,14 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
)
_pg_hidden = None # release before any verify forward
device_synchronize()
- # clip to the loss window [B, logits_to_keep+max_left_pad]
+ # Clip to the loss window [B, logits_to_keep+max_left_pad].
_pg_w = logits_to_keep + max_left_pad
if _pg_r.shape[1] > _pg_w:
_pg_r = _pg_r[:, -_pg_w:]
return _pg_r
- # trust only within the verified envelope: re-verify when T or the
- # longest segment grows, like the packed path
+ # Trust only within the verified envelope: re-verify when T or the longest segment grows, like
+ # the packed path.
_pg_T = int(_pg_layout.flat_ids.shape[1])
_pg_maxseg = int(_pg_layout.position_ids.max()) + 1
_pg_env = (
@@ -1885,25 +1807,23 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
if (not _pg_verify_on()) or (
_pg_env is not None and _pg_T <= _pg_env[0] and _pg_maxseg <= _pg_env[1]
):
- # trusted shape: run PG now and skip the full-row forward below
+ # Trusted shape: run PG now and skip the full-row forward below.
_pg_result = _pg_run_forward()
_pg_use = True
_pg_skip_pk = True
else:
- # unverified shape: defer the forward until the packed reference
- # exists (verify site below), so a declined packed path never wastes
- # a whole-batch PG forward
+ # Unverified shape: defer the forward until the packed reference exists, so a
+ # declined packed
+ # path never wastes a whole-batch PG forward.
_pg_forward_fn = _pg_run_forward
except Exception as _pg_err:
_pg_result = None
_pg_use = False
_pg_skip_pk = False
_pg_forward_fn = None
- # A FlexAttention/Triton compile failure or OOM here is GPU-wide, not
- # layout-specific, so retrying the same PG forward every step just re-pays
- # the failure. Persistently disable PG (mirrors the seq-packing handler
- # setting _unsloth_seq_packing_nograd_ok = False); the packed/padded path
- # below still produces the exact result.
+ # A FlexAttention/Triton compile failure or OOM here is GPU-wide, not layout-specific, so
+ # retrying every step just re-pays it. Disable PG persistently; the packed/padded path below
+ # still gives the exact result.
unwrapped_model._unsloth_prefix_grouper_nograd_disabled = True
if isinstance(_pg_err, torch.cuda.OutOfMemoryError):
torch.cuda.empty_cache()
@@ -1914,11 +1834,10 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
flush = True,
)
- # ---- Sequence packing (default-on; disable with UNSLOTH_GRPO_SEQ_PACKING=0) ----
- # One varlen [1, sum L] block-diagonal forward replaces the padded [B, Lmax] loop
- # (exact per-row result; also fixes the padded path's left-pad RoPE error).
- # Self-verified vs the per-row forward, re-checked as T grows; falls back if a
- # backend ignores packed_seq_lengths. lm_head runs on completion positions only.
+ # Sequence packing (default on; UNSLOTH_GRPO_SEQ_PACKING=0 disables): one varlen
+ # block-diagonal forward replaces the padded loop exactly and fixes its left-pad RoPE error.
+ # Self-verified, re-checked as T grows, falls back if a backend ignores packed_seq_lengths, and
+ # lm_head runs on completion positions only.
_pk_result = None
_pk_use = False
_pk_enabled = UNSLOTH_GRPO_SEQ_PACKING_ON
@@ -1944,22 +1863,22 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
_pk_L = input_ids.shape[1]
_pk_W = logits_to_keep + max_left_pad
_pk_maxseg = max(_pk_nz_cpu) if _pk_nz_cpu else 0
- # sliding-window models lose the per-sequence local window in a packed stream
+ # Sliding-window models lose the per-sequence local window in a packed stream.
_pk_sw = getattr(
getattr(unwrapped_model, "config", None), "sliding_window", None
)
_pk_sw_ok = not (isinstance(_pk_sw, int) and _pk_sw > 0 and _pk_maxseg > _pk_sw)
- # per-row completion mask (same as the loss); prompt-only rows count as inactive
+ # Per-row completion mask (same as the loss); prompt-only rows count as inactive.
_pk_cmask = create_completion_attention_mask(
input_ids[:, -_pk_W:], left_pad_tokens_per_prompt, max_left_pad, _pk_pad
)
_pk_active = int(_pk_cmask.any(dim = 1).sum())
- # skip the packed forward entirely at known-unsafe lengths (avoids a wasted pass / OOM)
+ # Skip the packed forward entirely at known-unsafe lengths, avoiding a wasted pass or OOM.
_pk_unsafe = getattr(
unwrapped_model, "_unsloth_seq_packing_nograd_unsafe_T", None
)
- # cap the flattened forward at one padded [batch_size, seq_len] mini-batch's
- # token budget; anything larger uses the chunked padded loop
+ # Cap the flattened forward at one padded [batch_size, seq_len] mini-batch's token budget;
+ # anything larger uses the chunked padded loop.
_pk_cap = batch_size * seq_len
if (
_pk_T >= 2
@@ -1976,7 +1895,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
as_tuple = False
) # [T, 2] = (row, col), row-major
_pk_within = _pk_nz_idx[1:, 0] == _pk_nz_idx[:-1, 0] # [T-1]
- # per-row completion start after left-packing (matches create_completion_attention_mask)
+ # Per-row completion start after left-packing, matching create_completion_attention_mask.
_pk_cstart = (_pk_L - logits_to_keep) - left_pad_tokens_per_prompt # [rows]
_pk_ctgt = (_pk_nz_idx[1:, 1] >= _pk_cstart[_pk_nz_idx[1:, 0]]) & _pk_within
with _get_inference_mode_context_manager(model):
@@ -1985,7 +1904,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
dtype = self._autocast_dtype,
enabled = getattr(self, "_autocast_enabled", True),
):
- # use_cache=False: a KV cache silently disables varlen packing
+ # use_cache=False: a KV cache silently disables varlen packing.
_pk_hidden = unwrapped_model(
input_ids = _pk_flat,
position_ids = _pk_pos,
@@ -1996,7 +1915,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
).logits
_pk_out = _pk_hidden[0, :-1, :][_pk_ctgt].unsqueeze(0)
_pk_ids = _pk_flat[0, 1:][_pk_ctgt].unsqueeze(0)
- # Guard: check if model returned hidden states or logits
+ # Hidden states or logits? Logits mean the forward already applied scaling/softcapping.
if _unsloth_grpo_returns_hidden_states(
unwrapped_model, _pk_out, lm_head
):
@@ -2011,16 +1930,17 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
temperature,
)[0]
else:
- # Model returned logits directly - scaling/softcapping already applied by model forward
+ # Model returned logits directly - scaling/softcapping already applied by
+ # model forward
_pk_sel = chunked_selective_log_softmax(
_pk_out,
_pk_ids,
temperature,
_pk_chunks,
)[0]
- # GPT-OSS offload race guard (matches the padded loop)
+ # GPT-OSS offload race guard, matching the padded loop.
device_synchronize()
- # scatter each logprob back to its (row, col) so [:, -_pk_W:] matches padded
+ # Scatter each logprob back to its (row, col) so [:, -_pk_W:] matches the padded path.
_pk_tgt = (_pk_nz_idx[1:, 0] * _pk_L + _pk_nz_idx[1:, 1])[_pk_ctgt]
_pk_result = (
torch.zeros(
@@ -2031,15 +1951,15 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
.index_put((_pk_tgt,), _pk_sel.to(torch.float32))
.view(total_rows, _pk_L)[:, -_pk_W:]
)
- # re-verify when T or the longest segment grows past what was verified
- # (a LongRoPE cache switch can change the result)
+ # Re-verify when T or the longest segment grows past the verified envelope; a LongRoPE cache
+ # switch can change the result.
_pk_vT = int(
getattr(unwrapped_model, "_unsloth_seq_packing_nograd_verified_T", 0)
)
_pk_vS = int(
getattr(unwrapped_model, "_unsloth_seq_packing_nograd_verified_seg", 0)
)
- # debug: hand-edit this condition to force re-verify every step
+ # Debug: hand-edit this condition to force re-verify every step.
if _pk_ok is True and _pk_T <= _pk_vT and _pk_maxseg <= _pk_vS:
_pk_use = True # already verified for this shape
else:
@@ -2066,7 +1986,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
use_cache = False,
).logits
_pk_rout = _pk_rh[:, :-1, :]
- # Guard: check if model returned hidden states or logits
+ # Hidden states or logits? Logits mean the forward already applied
+ # scaling/softcapping.
if _unsloth_grpo_returns_hidden_states(
unwrapped_model, _pk_rout, lm_head
):
@@ -2081,7 +2002,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
temperature,
)[0]
else:
- # Model returned logits directly - scaling/softcapping already applied by model forward
+ # Model returned logits directly - scaling/softcapping already
+ # applied by model forward
_pk_rsel = chunked_selective_log_softmax(
_pk_rout,
_pk_real[:, 1:],
@@ -2096,7 +2018,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
_pk_rkeep
].to(torch.float32)
device_synchronize()
- # compare over the loss-mask region only
+ # Compare over the loss-mask region only.
_pk_cm = _pk_cmask.float()
_pk_diff = float(((_pk_result - _pk_ref).abs() * _pk_cm).max())
if UNSLOTH_ENABLE_LOGGING:
@@ -2104,11 +2026,12 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
f"[Unsloth] GRPO seq-packing (no-grad) verify: T={_pk_T} maxseg={_pk_maxseg} packed-vs-perrow max|d|={_pk_diff:.4f}",
flush = True,
)
- # kernel-noise floor ~0.25; cross-sample contamination is >= 2.4
+ # Kernel-noise floor is ~0.25; cross-sample contamination is >= 2.4.
if _pk_diff < 7e-1:
unwrapped_model._unsloth_seq_packing_nograd_ok = True
- # widen the trusted shape only when >= 2 completion rows exercised
- # cross-sample packing; single-row passes prove nothing
+ # Widen the trusted shape only when at least 2 completion rows exercised cross-
+ # sample packing;
+ # a single row proves nothing.
if _pk_active >= 2:
unwrapped_model._unsloth_seq_packing_nograd_verified_T = max(
_pk_vT, _pk_T
@@ -2121,10 +2044,10 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
else:
_pk_use = False
if _pk_diff >= 1.5:
- # contamination (attention ignores the packed mask): disable packing
+ # Contamination (attention ignores the packed mask): disable packing.
unwrapped_model._unsloth_seq_packing_nograd_ok = False
else:
- # likely a length boundary (LongRoPE): mark unsafe, keep smaller shapes
+ # Likely a length boundary (LongRoPE): mark unsafe, keep smaller shapes.
unwrapped_model._unsloth_seq_packing_nograd_unsafe_T = (
_pk_T if _pk_unsafe is None else min(_pk_unsafe, _pk_T)
)
@@ -2134,7 +2057,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
flush = True,
)
except Exception as _pk_err:
- # any failure: drop intermediates, use the padded loop, do not retry
+ # Any failure: drop intermediates, use the padded loop, do not retry.
_pk_hidden = None
_pk_sel = None
_pk_result = None
@@ -2147,14 +2070,14 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
f"[Unsloth] GRPO sequence-packing (no-grad) disabled (fell back to padded): {_pk_err!r}",
flush = True,
)
+ # PrefixGrouper first-use self-verify (no-grad): compare the untrusted PG result to the packed
+ # result over the completion mask. Below tol_ok trust the structure, at or above TOL_KILL mark
+ # it unsafe forever, borderline falls back for this shape.
# ---- PrefixGrouper first-use self-verify (no-grad) ----
- # Compare the untrusted PG result to the full-row packed result (itself verified vs
- # per-row) over the completion mask: < tol_ok -> trust the structure; >= TOL_KILL ->
- # unsafe forever; borderline -> fall back this shape.
if _pg_forward_fn is not None and not _pg_use:
if _pk_use and _pk_result is not None:
try:
- # deferred PG forward, run only now that the packed reference exists
+ # Deferred PG forward, run only now that the packed reference exists.
_pg_result = _pg_forward_fn()
_pg_W2 = logits_to_keep + max_left_pad
_pg_cm = create_completion_attention_mask(
@@ -2208,7 +2131,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
f"[Unsloth] GRPO PrefixGrouper (no-grad) verify failed (fell back to packed): {_pg_err3!r}",
flush = True,
)
- # else: no packed reference (packing off/failed) -> cannot verify; fall back.
+ # No packed reference (packing off or failed) means this cannot be verified, so fall back.
if _pg_use and _pg_result is not None:
logprobs = _pg_result # PrefixGrouper verified/trusted -> skip the loop
@@ -2253,7 +2176,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
)
logits_chunk = outputs.logits
- del outputs # free hidden_states before chunked log-softmax
+ del outputs
completion_input_ids_chunk = input_ids_chunk[
:, -(logits_to_keep + max_left_pad) :
@@ -2262,7 +2185,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
:, -(logits_to_keep + max_left_pad + 1) :, :
]
logits_chunk = logits_chunk[:, :-1, :]
- # Guard: check if model returned hidden states or logits
+ # Hidden states or logits? Logits mean the forward already applied scaling/softcapping.
if _unsloth_grpo_returns_hidden_states(
unwrapped_model, logits_chunk, lm_head
):
@@ -2277,7 +2200,10 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
temperature = temperature,
)
else:
- # Model returned logits directly - scaling/softcapping already applied by model forward
+ # Model returned logits directly - scaling/softcapping already applied by model
+ # forward
+ # Model returned logits directly - scaling/softcapping already applied by model
+ # forward
logprobs_chunk = chunked_selective_log_softmax(
logits_chunk,
completion_input_ids_chunk,
@@ -2285,8 +2211,9 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
input_ids_chunk.shape[0] * multiplier,
)
else:
- # Essentially, for VLMs we do not go via the optimized path in models/,
- # so we don't encounter the Flash Attn left-padding issue.
+ # VLMs do not take the optimized path in models/, so they never hit the Flash Attn
+ # left-padding
+ # issue.
outputs = unwrapped_model(
input_ids = input_ids_chunk,
attention_mask = attention_mask_chunk,
@@ -2299,11 +2226,11 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
)
logits_chunk = outputs.logits
- del outputs # free hidden_states before chunked log-softmax
+ del outputs
logits_chunk = logits_chunk[:, :-1, :]
completion_input_ids_chunk = input_ids_chunk[:, -logits_to_keep:]
- # Guard: check if model returned hidden states or logits
+ # Hidden states or logits? Logits mean the forward already applied scaling/softcapping.
if _unsloth_grpo_returns_hidden_states(
unwrapped_model, logits_chunk, lm_head
):
@@ -2318,14 +2245,12 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
temperature = temperature,
)
else:
- # Model returned logits directly - scaling/softcapping already applied by model forward
logprobs_chunk = chunked_selective_log_softmax(
logits_chunk,
completion_input_ids_chunk,
temperature,
)
- # This is needed to avoid race conditions with GPT OSS offload_embbed=True
- # However, it seems that this line does not slow down or disrupt models.
+ # Avoids a race with GPT OSS offload_embbed=True; it does not appear to slow models down.
device_synchronize()
all_logprobs_list.append(logprobs_chunk)
if logprobs is None: # padded fallback when packing was not used
@@ -2334,35 +2259,18 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
entropies = None
os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0"
- # aux loss is unused: it is off by default (router_aux_loss_coef set to 0 in models/rl.py)
- # and explicit opt-in is rejected at trainer init, so this is always None (kept in the
- # return for TRL >= 1.7.0's 3-tuple contract).
+ # aux loss is unused: off by default (router_aux_loss_coef = 0 in models/rl.py) and explicit
+ # opt-in is rejected at trainer init, so it is always None. Kept for TRL >= 1.7.0's 3-tuple.
aux_loss = None
return logprobs.detach(), entropies, aux_loss # logps, entropies, aux_loss
- # input_ids = input_ids[:, -logits_to_keep:]
- # For transformers<=4.48, logits_to_keep argument isn't supported, so here we drop logits ourselves.
- # See https://github.com/huggingface/trl/issues/2770
- # logits = logits[:, -logits_to_keep:]
- # return logits
- # See https://huggingface.co/blog/the_n_implementation_details_of_rlhf_with_ppo#policy-training-implementation-details
- # logits = logits / self.temperature
- # logps = selective_log_softmax(logits, input_ids)
-
- # row_indices, col_indices = torch.where(logps < -20)
-
- # # Method 1: Check if tensors have elements
- # if len(row_indices) > 0 and len(col_indices) > 0:
- # breakpoint() # Breakpoint triggered here
- # print("Found high values!")
- # return logps # compute logprobs for the input tokens
+ # transformers <= 4.48 does not support logits_to_keep, so drop the logits here; see
+ # huggingface/trl#2770.
function = inspect.getsource(_get_per_token_logps_and_entropies)
if trl_version < Version("1.7.0"):
- # TRL < 1.7.0 unpacks (logps, entropies) at every call site; TRL >= 1.7.0
- # always unpacks (logps, entropies, aux_loss). Drop the aux_loss element so
- # the return arity matches the installed TRL. Regex tolerates comment /
- # whitespace drift on the return line; fail loud if the anchor ever stops
- # matching rather than silently shipping a 3-tuple to older TRL.
+ # TRL < 1.7.0 unpacks (logps, entropies) while TRL >= 1.7.0 unpacks (logps, entropies,
+ # aux_loss), so drop the third element to match. The regex tolerates comment/whitespace drift
+ # and fails loud rather than ship a 3-tuple to older TRL.
new_function, n = re.subn(
r"return (logprobs\.detach\(\), entropies), aux_loss[^\n]*",
r"return \1 # logps, entropies",
@@ -2489,7 +2397,7 @@ def _unsloth_grpo_hidden_states_signal(model):
getattr(candidate, "_unsloth_grpo_hidden_states_forward_wrapped", False)
for candidate in candidates
):
- # the wrapper honours the flag unless it recorded that this call could not
+ # The wrapper honours the flag unless it recorded that this call could not.
if any(
hasattr(candidate, "_unsloth_grpo_hidden_states_degraded") for candidate in candidates
):
@@ -2497,8 +2405,8 @@ def _unsloth_grpo_hidden_states_signal(model):
getattr(candidate, "_unsloth_grpo_hidden_states_degraded", False)
for candidate in candidates
)
- # an `unsloth/models/rl.py` predating the per-call attribute only ever set
- # the warn-once flag; it is the best signal such a wrapper offers
+ # An unsloth/models/rl.py predating the per-call attribute set only the warn-once flag; it is
+ # the best signal such a wrapper offers.
return not any(
getattr(candidate, "_unsloth_grpo_hidden_states_warning_issued", False)
for candidate in candidates
@@ -2513,10 +2421,9 @@ _GRPO_HIDDEN_STATES_WIDTH_DISPATCH = re.compile(
flags = re.MULTILINE,
)
-# Deliberately loose: any branch header that decides something off an ``lm_head``
-# dimension. Used only to count how many decisions the strict pattern above was
-# supposed to rewrite, so that a zoo which respells *some* of them is rejected
-# rather than half-patched.
+# Deliberately loose: any branch header deciding something off an lm_head dimension. Used only
+# to count how many the strict pattern should have rewritten, so a zoo respelling SOME of them
+# is rejected rather than half-patched.
_GRPO_HIDDEN_STATES_WIDTH_DISPATCH_CANDIDATE = re.compile(
r"^[ \t]*(?:el)?if[ \t]+[^\n]*\blm_head\.shape\[[^\]\n]+\][^\n]*:[ \t]*$",
flags = re.MULTILINE,
@@ -2599,7 +2506,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(grpo_compute_loss_slow)
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(grpo_update_SamplingParams))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_get_inference_mode_context_manager))
# inspect.getsource inlines function bodies but not module imports, so constants the inlined
-# grpo functions reference (e.g. UNSLOTH_ENABLE_LOGGING) must be redefined in the generated cache.
+# grpo functions use (UNSLOTH_ENABLE_LOGGING) must be redefined in the generated cache.
RL_PRE_ITEMS["grpo_trainer"].append(
"import os as _unsloth_os\n"
"UNSLOTH_ENABLE_LOGGING = _unsloth_os.environ.get('UNSLOTH_ENABLE_LOGGING', '0') in ('1', 'True', 'true')\n"
@@ -2626,8 +2533,8 @@ RL_PRE_ITEMS["grpo_trainer"].append(
" except Exception:\n"
" UNSLOTH_GRPO_PREFIX_GROUPER_ON = False\n"
)
-# getsource inlines the grpo function bodies but not this file's imports, so the
-# generated cache needs its own guarded import or detect_logit_transforms is a NameError.
+# getsource inlines the grpo bodies but not this file's imports, so the generated cache needs
+# its own guarded import or detect_logit_transforms is a NameError.
RL_PRE_ITEMS["grpo_trainer"].append(
"try:\n"
" from unsloth_zoo.device_map_planner import detect_logit_transforms\n"
@@ -2636,7 +2543,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(
)
-# Edit _get_per_token_logps to handle mixed precision
+# Edit _get_per_token_logps to handle mixed precision.
def grpo_trainer_compute_loss(function_name, function):
if function_name != "compute_loss":
return function
@@ -2650,7 +2557,6 @@ def grpo_trainer_compute_loss(function_name, function):
):
if return_outputs:
raise ValueError("The GRPOTrainer does not support returning outputs")
- # Compute the per-token log probabilities for the model
prompt_ids, prompt_mask = inputs["prompt_ids"], inputs["prompt_mask"]
completion_ids, completion_mask = (
@@ -2666,14 +2572,14 @@ def grpo_trainer_compute_loss(function_name, function):
inputs.get("image_sizes", None),
)
num_images = inputs.get("num_images", None)
- # Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models
+ # Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models.
token_type_ids = inputs.get("token_type_ids", None)
mm_token_type_ids = inputs.get("mm_token_type_ids", None)
num_items_in_batch = inputs.get("num_items_in_batch", None)
sampling_per_token_logps = inputs.get("sampling_per_token_logps", None)
tool_mask = inputs.get("tool_mask", None)
- # Missing when evaluate() runs standalone; eval does not accumulate, so
- # fall back to 1 to avoid underreporting eval_loss (#2464).
+ # Missing when evaluate() runs standalone; eval does not accumulate, so fall back to 1 rather
+ # than underreport eval_loss (#2464).
current_gradient_accumulation_steps = getattr(
self, "current_gradient_accumulation_steps", 1
)
@@ -2689,7 +2595,6 @@ def grpo_trainer_compute_loss(function_name, function):
mm_token_type_ids,
completion_ids = completion_ids,
)
- # attention_mask = None
logits_to_keep = completion_ids.size(
1
) # we only need to compute the logits for the completion tokens
@@ -2723,30 +2628,19 @@ def grpo_trainer_compute_loss(function_name, function):
per_token_logps = get_logps_func(
model, input_ids, attention_mask, logits_to_keep, compute_efficient = True
)
- # Compute the KL divergence between the model and the reference model
- # _prepare_inputs doesn't return reference log probs anymore. We need to calculate it ourselves.
- # https://github.com/huggingface/trl/blob/05bc43e960396581e458195b8388efe6b82cae1f/trl/trainer/grpo_trainer.py#L1328
- # if self.beta != 0.0:
- # with torch.inference_mode(), model.disable_adapter():
- # ref_per_token_logps = per_token_logps = get_logps_func(model, input_ids, attention_mask, logits_to_keep)
- # else:
- # ref_per_token_logps = None
+ # KL divergence between model and reference: _prepare_inputs no longer returns reference log
+ # probs. See trl grpo_trainer.py#L1328.
ref_logps = inputs.get("ref_per_token_logps", None)
- # per_token_kl = torch.exp(ref_per_token_logps - per_token_logps) - (ref_per_token_logps - per_token_logps) - 1
- # x - x.detach() allows for preserving gradients from x
+ # x - x.detach() preserves gradients from x.
advantages = inputs["advantages"]
- # per_token_loss = torch.exp(per_token_logps - per_token_logps.detach()) * advantages.unsqueeze(1)
- # per_token_loss = -(per_token_loss - self.beta * per_token_kl)
- # loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean()
old_logps = inputs.get("old_per_token_logps", None)
input_ids = input_ids[:, -logits_to_keep:]
- # Get logit softcapping and logit scale
model_config = _unsloth_get_model_config(model)
- # The old and reference logps come from _get_per_token_logps_and_entropies and the
- # gradient logps from here, so both must read the transforms the same way or the
- # importance ratio compares two different policies.
+ # The old and reference logps come from _get_per_token_logps_and_entropies and the gradient
+ # logps from here, so both must read the transforms alike or the importance ratio compares two
+ # different policies.
if detect_logit_transforms is not None:
# model_config, not model: see _get_per_token_logps_and_entropies.
_transforms = detect_logit_transforms(model_config)
@@ -2904,7 +2798,7 @@ def grpo_trainer_compute_loss(function_name, function):
**_grpo_accumulated_loss_kwargs,
)
else:
- # to ensure backwards compatibility with trl 0.15.2 and maybe even 0.17
+ # For backwards compatibility with trl 0.15.2 and maybe 0.17.
loss, completion_length, mean_kl, coef_1, completion_mask = grpo_accumulated_loss(
trainer = self,
input_ids = _input_ids,
@@ -3001,7 +2895,6 @@ def grpo_trainer_compute_loss(function_name, function):
advantages = advantages.unsqueeze(1)
if self.loss_type in ["grpo", "bnpo", "dr_grpo", "dapo"]:
- # Compute the clipped probability ratios
is_low_clipped = (coef_1 < 1 - self.epsilon_low) & (advantages < 0)
is_high_clipped = (coef_1 > 1 + self.epsilon_high) & (advantages > 0)
is_region_clipped = is_low_clipped | is_high_clipped
@@ -3037,15 +2930,13 @@ def grpo_trainer_compute_loss(function_name, function):
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_compute_loss)
-# Fix KTO shape mismatch when Unsloth model forward truncates input_ids
-# but labels aren't truncated. TRL 0.27.2+ _process_tokens only truncates
-# completions, not prompts -- so prompts exceeding max_seq_length cause the
-# model to produce shorter logits than the labels expect.
+# KTO shape mismatch: the Unsloth forward truncates input_ids while labels are untouched, and
+# TRL 0.27.2+ _process_tokens truncates only completions, so an over-length prompt makes the
+# model emit shorter logits than the labels expect.
def kto_trainer_get_batch_logps(function_name, function):
if function_name != "get_batch_logps":
return function
- # The raise is inside an if block inside the method, so we need
- # to preserve the exact indentation of the raise statement.
+ # The raise sits inside an if inside the method, so its exact indentation must be preserved.
old = 'raise ValueError("Logits (batch and sequence length dim) and labels must have the same shape.")'
new = (
"# Unsloth: auto-truncate to shorter sequence length (model may have truncated input_ids)\n"
@@ -3060,10 +2951,10 @@ def kto_trainer_get_batch_logps(function_name, function):
RL_FUNCTIONS["kto_trainer"].append(kto_trainer_get_batch_logps)
-# TRL 1.x dropped KTOTrainer.get_batch_logps and moved the log-prob math into
-# _compute_logps / compute_ref_log_probs / _compute_kl_logps, which call
-# selective_log_softmax on completion-only tokens. Same truncation hazard as
-# above, so clamp logits/ids/mask to the shorter seq length (no-op when equal).
+# TRL 1.x dropped KTOTrainer.get_batch_logps and moved the math into _compute_logps /
+# compute_ref_log_probs / _compute_kl_logps, which call selective_log_softmax on
+# completion-only tokens. Same truncation hazard, so clamp logits/ids/mask to the shorter
+# length (a no-op when equal).
_KTO_COMPLETION_RE = re.compile(
r"(?P<ws>[ \t]*)shift_logits = completion_logits\[:, :-1, :\]\.contiguous\(\)\n"
r"(?P=ws)per_token_logps = selective_log_softmax\(\s*shift_logits,\s*"
@@ -3117,11 +3008,10 @@ def kto_trainer_align_completion_logps(function_name, function):
RL_FUNCTIONS["kto_trainer"].append(kto_trainer_align_completion_logps)
-# https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_trainer.py#L356
-# TRL warns if batch size is not a multiple of num_generations -> fix this.
+# TRL warns if batch size is not a multiple of num_generations; see trl grpo_trainer.py#L356.
def grpo_trainer_fix_batch_size(RLTrainer_source, RLConfig_source):
if "divisible by the number of generations" not in RLTrainer_source:
- # in later trl versions this doesn't exist anymore
+ # In later trl versions this no longer exists.
return ""
if "num_generations" not in RLConfig_source:
return ""
@@ -3139,12 +3029,11 @@ def grpo_trainer_fix_batch_size(RLTrainer_source, RLConfig_source):
RL_CONFIG_CHANGES["grpo_trainer"].append(grpo_trainer_fix_batch_size)
-# Add other reward function names
def grpo_trainer_metrics(RLTrainer_source, RLConfig_source):
if "reward_funcs" not in RLTrainer_source:
return ""
- # For new TRL we have /mean and /std
+ # New TRL has /mean and /std.
use_mean = "rewards/{reward_func_name}/mean" in RLTrainer_source
use_std = "rewards/{reward_func_name}/std" in RLTrainer_source
if not use_mean:
@@ -3173,16 +3062,10 @@ RL_METRICS_CHANGES["grpo_trainer"].append(grpo_trainer_metrics)
def openenv_vllm_reload_weights():
- # This function patches the trl openenv generate_rollout_completions function to:
- # 1. Guard the reload_weights call (skip when sharing weights with vLLM)
- # 2. Fix wake_up call to be compatible with unsloth (remove tags to wake everything)
- #
- # The issue: TRL's wake_up(tags=["kv_cache"]) only wakes kv_cache, leaving is_sleeping=True
- # at the executor level. This causes unsloth's patched generate to try waking up again,
- # resulting in double create_and_map on already-mapped handles.
- #
- # The fix: Use wake_up() with no tags, which wakes everything. Unsloth's patched
- # CuMemAllocator.wake_up skips weights anyway, so this is safe.
+ # Patch trl's openenv generate_rollout_completions to guard reload_weights when sharing weights
+ # with vLLM, and to call wake_up() untagged: TRL's wake_up(tags=["kv_cache"]) leaves
+ # is_sleeping=True at the executor, so Unsloth's generate wakes again and double create_and_maps
+ # mapped handles. Unsloth's CuMemAllocator.wake_up skips weights anyway.
if importlib.util.find_spec("trl") is None:
return
if Version(importlib_version("trl")) < Version("0.26.0"):
@@ -3198,7 +3081,7 @@ def openenv_vllm_reload_weights():
)
return
- # trl 0.28 changed the function name yet again! Thanks trl :)
+ # trl 0.28 changed the function name again.
patch_target_name = "_generate_rollout_completions_colocate"
if hasattr(openenv_utils, patch_target_name):
patch_target = getattr(openenv_utils, patch_target_name)
@@ -3207,15 +3090,9 @@ def openenv_vllm_reload_weights():
patch_target_name = "generate_rollout_completions"
patch_target = getattr(openenv_utils, patch_target_name)
- # TRL 0.29.1+ ships some openenv helpers as compiled bytecode without
- # accessible source on disk; inspect.getsource raises OSError("could
- # not get source code") in that case. Skip the source-rewrite patch
- # rather than crash. The unmodified TRL openenv path will run, which
- # means the duplicate `collective_rpc("reload_weights")` is NOT
- # stripped (line 1800 below) and `wake_up(tags=["kv_cache"])` is NOT
- # retagged to `wake_up()` (line 1804). Users who do not use openenv
- # GRPO are unaffected; openenv GRPO users on this TRL build may see
- # redundant reload_weights calls or partial wake_up behavior.
+ # TRL 0.29.1+ ships some openenv helpers as bytecode with no source, so inspect.getsource
+ # raises OSError; skip the rewrite rather than crash. The unmodified path keeps the duplicate
+ # reload_weights and the tagged wake_up, so openenv GRPO users may see redundant reloads.
try:
src = inspect.getsource(patch_target)
except OSError as e:
@@ -3246,20 +3123,20 @@ def openenv_vllm_reload_weights():
src = reload_weights_pattern.sub(replace_reload_weights, src)
- # Change wake_up(tags=["kv_cache"]) to wake_up() - wake everything to set is_sleeping=False
- # This prevents double wake_up issues. Unsloth's allocator skips weights anyway.
+ # wake_up() with no tags wakes everything and sets is_sleeping=False, preventing a double
+ # wake_up; Unsloth's allocator skips weights anyway.
src = re.sub(r"\.wake_up\(tags=\[.*?\]\)", ".wake_up()", src)
if original_src == src:
logger.warning("Unsloth: Warning - regex did not match, patch may have failed")
return
- # Execute and explicitly assign to module
+ # Execute and explicitly assign to the module.
local_ns = {}
exec(compile(src, "<unsloth>", "exec"), openenv_utils.__dict__, local_ns)
patched_func = local_ns[patch_target_name]
- # Patch the target function in utils; if dispatcher was patched also update parent module alias.
+ # Patch the target function in utils; if the dispatcher was patched, also update the parent module alias.
setattr(openenv_utils, patch_target_name, patched_func)
if patch_target_name == "generate_rollout_completions":
openenv.generate_rollout_completions = patched_func
@@ -3270,11 +3147,8 @@ RL_ADDITIONAL_FUNCTIONS["openenv"].append(openenv_vllm_reload_weights)
def vllm_generation_init_patch():
- # trl moved vllm stuff to trl/generation/vllm_generation.py
- # We need to patch it to not instantiate another vLLM instance if we already have one with fast_inference
- # Edit the TRL source directly and install the patched function in the TRL module.
- # https://github.com/huggingface/trl/commit/0eb66d8f2fc63b3d00d8dbc18f99c3f48750bd16
- # This exists in trl versions 0.28.0 and above
+ # trl moved vllm code to trl/generation/vllm_generation.py (commit 0eb66d8, 0.28.0+), which
+ # must be patched so it does not build a second vLLM instance when fast_inference has one.
if importlib.util.find_spec("trl") is None:
return
@@ -3318,7 +3192,6 @@ def vllm_generation_init_patch():
setattr(vllm_generation.VLLMGeneration, method_name, local_ns[method_name])
return True
- # Patch init to remove vLLM.LLM instantiation
def patch_init_vllm(src):
pattern = re.compile(
r"(?P<llm_block>^(?P<indent>[ \t]*)self\.llm\s*=\s*LLM\s*\(\n(?:.*\n)*?^(?P=indent)\))",
@@ -3345,9 +3218,7 @@ def vllm_generation_init_patch():
)
return patched_src
- # has some sync_weights or reload rpc calls.
- # we patched the grpo_trainer to strip them for prev versions
- # Ref: grpo_trainer__generate_single_turn above around L270-280
+ # Newer versions have sync_weights or reload rpc calls; earlier ones are stripped in the patched grpo_trainer above.
def patch_sync_weights(src):
pattern = re.compile(
r"^(?P<def_line>def sync_weights\(self\):\n)(?P<body>(?:.*\n)*)",
@@ -3356,8 +3227,8 @@ def vllm_generation_init_patch():
def replace_sync_weights(match):
body = match.group("body")
- # Chain getattr so server mode (where self.llm is not set) does
- # not raise AttributeError before the default kicks in.
+ # Chain getattr so server mode, where self.llm is unset, does not raise AttributeError before
+ # the default kicks in.
guard = (
" if getattr(getattr(self, 'llm', None), 'shared_weights', False) or "
"getattr(self, 'unsloth_fast_inference_lora', False):\n"
@@ -3382,38 +3253,17 @@ def vllm_generation_init_patch():
)
return patched_src
- # `generate` is deliberately NOT source-patched.
- #
- # It used to be, with two regexes: one anchored on a
- # `self.llm.collective_rpc("reload_weights")` line, and one injecting
- # `lora_request=...` into `self.llm.generate(...)`. Both anchors live inside a TRL
- # method body, and both have already drifted:
- # * TRL >= 1.x deleted the `collective_rpc("reload_weights")` call from `generate`
- # entirely and calls `self.sync_weights()` instead
- # (https://github.com/vllm-project/vllm/issues/29341). The old anchor matched 0
- # times, raised, and - because the lora injection ran *after* it in the same
- # function - the adapter stopped being passed to vLLM at all. Since `_init_vllm`
- # and `sync_weights` had already been installed by then, weights were no longer
- # synced into vLLM either, so GRPO rollouts were silently sampled from the BASE
- # model with no error raised.
- # * `(self\.llm\.generate\([^\)]+)\)` is not paren-balanced, so it also mis-edits any
- # call whose arguments contain nested parentheses or span several lines.
- #
- # So intercept on the vLLM engine instead of on TRL's method body. `self.llm` is the
- # vLLM `LLM` object in colocate mode in every TRL release that has `VLLMGeneration`,
- # and `LLM.generate` / `LLM.chat` / `LLM.collective_rpc` are public, stable vLLM APIs.
- # Checked against every vLLM release from 0.11.0 to 0.27.1: all three exist on `LLM` at
- # each one, `generate` and `chat` both take `lora_request`, `LLM` has no `__slots__` and
- # no `__setattr__`/`__getattr__` hook (so the instance override below always takes), and
- # nothing here is decorated. TRL's method body is free to move around;
- # whatever shape it takes, it has to reach the engine through those calls. The
- # override is scoped to the dynamic extent of one `VLLMGeneration.generate` call and
- # is undone in a `finally`, so `model.fast_generate` and any other user of the same
- # engine are unaffected.
+ # `generate` is deliberately NOT source-patched. Its two anchors drifted: TRL >= 1.x deleted
+ # the collective_rpc("reload_weights") call for self.sync_weights(), so the anchor matched 0
+ # times, raised, and took the lora injection with it -- GRPO rollouts came from the BASE model.
+ # The generate regex is also not paren-balanced and mis-edits multi-line calls.
+ # Intercept on the vLLM engine instead of TRL's method body: self.llm is the vLLM LLM object in
+ # colocate mode in every release with VLLMGeneration, and generate / chat / collective_rpc are
+ # public stable APIs, checked across vLLM 0.11.0-0.27.1. The override is scoped to one
+ # VLLMGeneration.generate call and undone in a finally.
_UNSLOTH_GENERATE_WRAPPED = "_unsloth_vllm_generation_lora_wrapped"
- # Mirror the per-device naming in rl.py so two ranks on one node do not race on the
- # same adapter directory.
+ # Mirror the per-device naming in rl.py so two ranks on one node do not race on the same adapter directory.
lora_name = "vllm_gen_lora"
if "CUDA_VISIBLE_DEVICES" in os.environ:
lora_name += "_" + os.environ.get("CUDA_VISIBLE_DEVICES", "0").replace(",", "")
@@ -3432,7 +3282,7 @@ def vllm_generation_init_patch():
self, "unsloth_fast_inference_lora", False
)
if llm is None or not sharing:
- # Server mode, or a vLLM engine TRL created itself -> upstream behaviour.
+ # Server mode, or a vLLM engine TRL created itself: keep upstream behaviour.
return original_generate(self, *args, **kwargs)
load_lora = getattr(self, "_unsloth_load_lora", None)
@@ -3471,15 +3321,15 @@ def vllm_generation_init_patch():
try:
positional = inspect.signature(bound).bind_partial(*args).arguments
except (TypeError, ValueError):
- # Unintrospectable callable (C extension, odd wrapper): the keyword
- # check above is all we have, and injecting is the safe default.
+ # Unintrospectable callable (C extension, odd wrapper): the keyword check above is all we have,
+ # and injecting is the safe default.
return False
return "lora_request" in positional
def wrap_generation_call(bound):
def unsloth_generation_call(*args, **kwargs):
- # vLLM needs the adapter handed to it explicitly: the shared engine
- # holds the BASE weights, and sync_weights is a no-op when sharing.
+ # vLLM needs the adapter handed to it explicitly: the shared engine holds the BASE weights, and
+ # sync_weights is a no-op when sharing.
if load_lora is not None and not caller_already_bound_lora(bound, args, kwargs):
kwargs["lora_request"] = load_lora(lora_name, load_tensors = True)
return bound(*args, **kwargs)
@@ -3488,8 +3338,8 @@ def vllm_generation_init_patch():
def wrap_collective_rpc(bound):
def unsloth_collective_rpc(method, *args, **kwargs):
- # The engine already shares the live training weights, so
- # reload_weights would pull the ORIGINAL checkpoint back off disk.
+ # The engine already shares the live training weights, so reload_weights would pull the
+ # ORIGINAL checkpoint back off disk.
if method == "reload_weights":
return None
return bound(method, *args, **kwargs)
@@ -3516,17 +3366,16 @@ def vllm_generation_init_patch():
original_generate, "__qualname__", "VLLMGeneration.generate"
)
generate.__doc__ = getattr(original_generate, "__doc__", None)
- # inspect.getsource / inspect.signature unwrap this, so drift detectors and any
- # other source-reading patch still see TRL's own `generate`.
+ # inspect.getsource / inspect.signature unwrap this, so drift detectors and other source-reading
+ # patches still see TRL's own generate.
generate.__wrapped__ = original_generate
setattr(generate, _UNSLOTH_GENERATE_WRAPPED, True)
vllm_generation.VLLMGeneration.generate = generate
return True
- # Snapshot before patching: a HALF-patched VLLMGeneration is worse than an unpatched
- # one. `_init_vllm` + `sync_weights` without the generate-side adapter injection means
- # no weight sync AND no LoRA, i.e. rollouts from the base model with no error. If any
- # one of the three fails, put all three back.
+ # Snapshot before patching: a HALF-patched VLLMGeneration is worse than none, since _init_vllm
+ # plus sync_weights without the generate-side adapter injection means no weight sync AND no
+ # LoRA, i.e. base-model rollouts with no error. If one of the three fails, restore all three.
method_names = ("_init_vllm", "sync_weights", "generate")
originals = {name: getattr(vllm_generation.VLLMGeneration, name, None) for name in method_names}
try:
diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py
index 0a1bea85a27..7c1d4d16813 100644
--- a/unsloth/models/sentence_transformer.py
+++ b/unsloth/models/sentence_transformer.py
@@ -79,7 +79,6 @@ def _save_pretrained_torchao(
if hasattr(inner_model, "_orig_mod"):
inner_model = inner_model._orig_mod
- # merge LoRA first
if hasattr(inner_model, "merge_and_unload"):
inner_model = inner_model.merge_and_unload()
@@ -106,9 +105,9 @@ def _save_pretrained_torchao(
def patch_unsloth_save():
original_causal = transformers.AutoModelForCausalLM
original_rmtree = shutil.rmtree
- # unsloth_save_pretrained_torchao expects AutoModelForCausalLM
+ # unsloth_save_pretrained_torchao expects AutoModelForCausalLM.
transformers.AutoModelForCausalLM = transformers.AutoModel
- # prevent unsloth from deleting the unquantized model directory
+ # Prevent unsloth from deleting the unquantized model directory.
shutil.rmtree = lambda *args, **kwargs: None
try:
yield
@@ -126,7 +125,7 @@ def _save_pretrained_torchao(
token = token,
)
- # avoid `0_Transformer-torchao`, it was either this or fix modules.json
+ # Avoid `0_Transformer-torchao`; it was either this or fix modules.json.
torchao_dir = transformer_dir + "-torchao"
if os.path.exists(torchao_dir):
if not os.path.exists(transformer_dir):
@@ -157,7 +156,6 @@ def _save_pretrained_torchao(
pass
-# Thanks Etherl:
def _save_pretrained_gguf(
self,
save_directory,
@@ -177,10 +175,9 @@ def _save_pretrained_gguf(
Saves the SentenceTransformer model to GGUF format by saving the inner transformer model,
converting it, and placing the resulting GGUF files in the save directory.
"""
- # 1. Save standard SentenceTransformer structure (configs, modules.json, etc.)
self.save_pretrained(save_directory)
- # 2. Extract inner transformer model (PEFT merge handled by the gguf saver)
+ # Extract the inner transformer model; PEFT merge is handled by the gguf saver.
inner_model = self[0].auto_model
if hasattr(inner_model, "_orig_mod"):
inner_model = inner_model._orig_mod
@@ -199,36 +196,34 @@ def _save_pretrained_gguf(
except:
pass
- # Unsloth saves + converts here; absolute for later commonpath comparison
+ # Unsloth saves and converts here; absolute for the later commonpath comparison.
transformer_dir = os.path.join(save_directory, transformer_path)
transformer_dir = os.path.abspath(transformer_dir)
if tokenizer is None:
tokenizer = self.tokenizer
- # 4. Call Unsloth's GGUF saver on the inner model targeting the transformer subdirectory
- # No rmtree guard here: the merge cleanup that deletes save_directory is gated on
- # push_to_hub, which is forced False below.
+ # No rmtree guard here: the merge cleanup that deletes save_directory is gated on push_to_hub, which
+ # is forced False below.
result = unsloth_save_pretrained_gguf(
inner_model,
save_directory = transformer_dir,
tokenizer = tokenizer,
quantization_method = quantization_method,
first_conversion = first_conversion,
- push_to_hub = False, # Force local first to move files
+ push_to_hub = False,
token = token,
max_shard_size = max_shard_size,
temporary_location = temporary_location,
maximum_memory_usage = maximum_memory_usage,
- # transformer_dir is the ST's own 0_Transformer module (written in step 1, uploaded
- # in step 7), not a throwaway: reclaiming it would hand back a folder that no longer
- # loads as a SentenceTransformer, so a short disk fails loudly instead.
+ # transformer_dir is the ST's own 0_Transformer module, not a throwaway: reclaiming it would hand
+ # back a folder that no longer loads as a SentenceTransformer, so a short disk fails loudly.
merge_is_disposable = False,
imatrix_file = imatrix_file,
gguf_shard_size = gguf_shard_size,
)
- # 5. Move GGUF files from the subdirectory (0_Transformer) to the root save_directory
+ # Move GGUF files from the 0_Transformer subdirectory to the root save_directory.
gguf_files = result.get("gguf_files", [])
new_gguf_locations = []
@@ -243,26 +238,22 @@ def _save_pretrained_gguf(
try:
is_subpath = os.path.commonpath([abs_gguf_file, transformer_dir]) == transformer_dir
except ValueError:
- # Windows different-drive commonpath raises
+ # Windows different-drive commonpath raises.
is_subpath = False
if is_subpath:
- # Move the GGUF out of transformer_dir to root
shutil.move(gguf_file, dest_path)
new_gguf_locations.append(dest_path)
else:
- # Elsewhere: move to root if not already there
if os.path.abspath(dest_path) != abs_gguf_file:
shutil.move(gguf_file, dest_path)
new_gguf_locations.append(dest_path)
result["gguf_files"] = new_gguf_locations
- # 6. Add branding
try:
FastSentenceTransformer._add_unsloth_branding(save_directory)
- # Add GGUF details to README
readme_path = os.path.join(save_directory, "README.md")
if os.path.exists(readme_path):
with open(readme_path, "a", encoding = "utf-8") as f:
@@ -273,13 +264,12 @@ def _save_pretrained_gguf(
except:
pass
- # 7. Handle Push to Hub if requested
if push_to_hub:
if token is None:
token = get_token()
api = HfApi(token = token)
- repo_id = save_directory # Assuming save_directory is the repo name if pushing
+ repo_id = save_directory
print(f"Unsloth: Uploading to {repo_id}...")
try:
@@ -391,7 +381,6 @@ def _push_to_hub_gguf(
except Exception as e:
print(f"Unsloth Warning: Could not create repo: {e}")
- # Convert locally in a temp dir, then upload
with tempfile.TemporaryDirectory(prefix = "unsloth_st_gguf_") as temp_dir:
print(f"Unsloth: Converting SentenceTransformer to GGUF format...")
@@ -417,7 +406,6 @@ def _push_to_hub_gguf(
print(f"Unsloth: Uploading GGUF to https://huggingface.co/{full_repo_id}...")
- # Upload GGUF files
for file_location in gguf_files:
if os.path.exists(file_location):
filename = os.path.basename(file_location)
@@ -433,7 +421,6 @@ def _push_to_hub_gguf(
revision = revision,
)
- # Upload Modelfile if exists
if modelfile_location and os.path.exists(modelfile_location):
print(" Uploading Ollama Modelfile...")
api.upload_file(
@@ -446,7 +433,6 @@ def _push_to_hub_gguf(
revision = revision,
)
- # Upload config.json if exists
config_path = os.path.join(temp_dir, "config.json")
if os.path.exists(config_path):
print(" Uploading config.json...")
@@ -460,7 +446,6 @@ def _push_to_hub_gguf(
revision = revision,
)
- # Create and upload README
gguf_basenames = [os.path.basename(f) for f in gguf_files if os.path.exists(f)]
readme_content = f"""---
tags:
@@ -509,7 +494,6 @@ This sentence-transformers model was finetuned and converted to GGUF format usin
revision = revision,
)
- # Add tags
all_tags = ["gguf", "llama-cpp", "unsloth", "sentence-transformers"]
if is_vlm:
all_tags.append("vision-language-model")
@@ -593,7 +577,6 @@ class FastSentenceTransformer(FastModel):
if module.get("type", "") == "sentence_transformers.models.Pooling":
pooling_path = module.get("path", "")
if pooling_path:
- # find config.json for the pooling module
if os.path.exists(model_name) and os.path.exists(
os.path.join(model_name, pooling_path, "config.json")
):
@@ -613,8 +596,7 @@ class FastSentenceTransformer(FastModel):
if pooling_config_path:
with open(pooling_config_path, "r", encoding = "utf-8") as f:
pooling_config = json.load(f)
- # from here:
- # https://github.com/huggingface/sentence-transformers/blob/main/sentence_transformers/models/Pooling.py#L43
+ # From sentence-transformers models/Pooling.py#L43
pooling_map = {
"pooling_mode_cls_token": "cls",
"pooling_mode_mean_tokens": "mean",
@@ -635,7 +617,7 @@ class FastSentenceTransformer(FastModel):
)
return "mean"
- # should prolly be done upstream instead of this hackfest here
+ # Should probably be done upstream instead of here.
@staticmethod
def _patch_mpnet_v4():
"""Patch MPNetModel for gradient checkpointing (transformers 4)."""
@@ -655,8 +637,10 @@ class FastSentenceTransformer(FastModel):
modeling_mpnet.MPNetModel._set_gradient_checkpointing = _set_gradient_checkpointing
- # patch MPNetEncoder.forward for checkpointing; based on:
- # https://github.com/huggingface/transformers/blob/v4.57.3/src/transformers/models/mpnet/modeling_mpnet.py#L321
+ # Patch MPNetEncoder.forward for checkpointing; based on transformers v4.57.3
+ # models/mpnet/modeling_mpnet.py#L321
+ # based on: https://github.com/huggingface/transformers/blob/v4.57.3/src/transformers/models/distilbert/
+ # modeling_distilbert.py#L666
def forward(
self,
hidden_states: torch.Tensor,
@@ -689,7 +673,7 @@ class FastSentenceTransformer(FastModel):
attention_mask,
head_mask[i] if head_mask is not None else None,
position_bias,
- use_reentrant = True, # fix for torch 2.9
+ use_reentrant = True,
)
else:
layer_outputs = layer_module(
@@ -740,8 +724,8 @@ class FastSentenceTransformer(FastModel):
modeling_mpnet.MPNetModel._set_gradient_checkpointing = _set_gradient_checkpointing
- # patch MPNetEncoder.forward for checkpointing; based on:
- # https://github.com/huggingface/transformers/blob/v5.0.0rc1/src/transformers/models/mpnet/modeling_mpnet.py#L284
+ # Patch MPNetEncoder.forward for checkpointing; based on transformers v5.0.0rc1
+ # models/mpnet/modeling_mpnet.py#L284
def forward(
self,
hidden_states: torch.Tensor,
@@ -805,11 +789,10 @@ class FastSentenceTransformer(FastModel):
@staticmethod
def _patch_distilbert_v4():
- # change kwargs to positional args to be compatible with peft_utils
+ # Change kwargs to positional args to be compatible with peft_utils; based on transformers v4.57.3
+ # models/distilbert/modeling_distilbert.py#L666
"""Patch DistilBertModel.forward to use positional args (transformers 4)."""
- # based on:
- # https://github.com/huggingface/transformers/blob/v4.57.3/src/transformers/models/distilbert/modeling_distilbert.py#L666
def forward(
self,
input_ids: Optional[torch.Tensor] = None,
@@ -867,7 +850,7 @@ class FastSentenceTransformer(FastModel):
attention_mask = _prepare_4d_attention_mask_for_sdpa(
attention_mask, embeddings.dtype, tgt_len = input_shape[1]
)
- # patch here, change kwargs to positional args:
+ # patch here: unsloth gradient checkpointing hook needs positional arguments
return self.transformer(
embeddings,
attention_mask,
@@ -898,8 +881,7 @@ class FastSentenceTransformer(FastModel):
@staticmethod
def _patch_distilbert_v5():
"""Patch DistilBertModel.forward to use positional args (transformers 5)."""
- # based on:
- # https://github.com/huggingface/transformers/blob/v5.0.0rc1/src/transformers/models/distilbert/modeling_distilbert.py#L386
+ # Based on transformers v5.0.0rc1 models/distilbert/modeling_distilbert.py#L386
from transformers.masking_utils import create_bidirectional_mask
def forward(
@@ -921,7 +903,7 @@ class FastSentenceTransformer(FastModel):
attention_mask = attention_mask,
)
- # patch here: unsloth gradient checkpointing hook needs positional arguments
+ # Unsloth's gradient checkpointing hook needs positional arguments.
return self.transformer(
embeddings,
attention_mask,
@@ -962,11 +944,10 @@ class FastSentenceTransformer(FastModel):
with open(readme_path, "r", encoding = "utf-8") as f:
content = f.read()
- # add unsloth tag to frontmatter
if "---\ntags:\n" in content:
content = content.replace("---\ntags:\n", "---\ntags:\n- unsloth\n")
else:
- # tags exist but not at the start: append via regex
+ # Tags exist but not at the start: append via regex.
pattern = r"(^tags:\s*\n)"
if re.search(pattern, content, re.MULTILINE):
content = re.sub(pattern, r"\1- unsloth\n", content, count = 1, flags = re.MULTILINE)
@@ -1026,11 +1007,8 @@ class FastSentenceTransformer(FastModel):
"""Helper to create and configure a Transformer module."""
from sentence_transformers.models import Transformer
- # Prevents loading the model a second time and redirects AutoProcessor/
- # AutoTokenizer so Transformer.__init__ picks up our pre-fixed tokenizer.
- # On sentence-transformers >=5.4 `tokenizer` is a read-only @property
- # backed by `self.processor`, so a post-init assignment raises; the
- # constructor redirect sets self.processor correctly instead.
+ # Prevents loading the model a second time and redirects AutoProcessor/AutoTokenizer so
+ # Transformer.__init__ picks up our pre-fixed tokenizer.
from transformers import AutoProcessor, AutoTokenizer
def is_requested_model_name(args, kwargs):
@@ -1087,7 +1065,7 @@ class FastSentenceTransformer(FastModel):
return original_processor_from_pretrained(*args, **kwargs)
try:
- # Temporarily redirect Auto* loading to return our pre-loaded objects
+ # Temporarily redirect Auto* loading to return our pre-loaded objects.
AutoModel.from_pretrained = return_existing_model
AutoProcessor.from_pretrained = return_existing_processor
AutoTokenizer.from_pretrained = return_existing_tokenizer
@@ -1109,12 +1087,9 @@ class FastSentenceTransformer(FastModel):
elif "tokenizer_args" in transformer_init_params:
transformer_kwargs["tokenizer_args"] = trust_remote_code_kwargs.copy()
- # Build via Transformer.load so the saved modality_config is honored: plain
- # Transformer(...) makes ST 5.x infer a "message" modality for chat-template
- # models (e.g. Qwen3-Embedding), chat-wrapping inputs and degrading embeddings
- # (#6881). Only use .load when it resolves a Hub id (accepts the kwargs or
- # **kwargs); legacy ST 3.x/4.x load(input_path) is local-only with no modality
- # bug, so fall back to the constructor.
+ # Build via Transformer.load so the saved modality_config is honored: a plain Transformer(...)
+ # makes ST 5.x infer a "message" modality for chat-template models, chat-wrapping inputs and
+ # degrading embeddings (#6881). Only use .load for a Hub id; ST 3.x/4.x load() is local-only.
transformer_module = None
transformer_load = getattr(Transformer, "load", None)
has_modules_json = (
@@ -1139,8 +1114,9 @@ class FastSentenceTransformer(FastModel):
"trust_remote_code": trust_remote_code,
**transformer_kwargs,
}
- # Resolve config/tokenizer from the module's saved subfolder
- # (modules.json "path"), like stock ST; "" (root) is a no-op.
+ # Resolve config/tokenizer from the module's saved subfolder (modules.json "path"),
+ # like stock ST;
+ # "" (root) is a no-op.
if module_subfolder:
load_kwargs["subfolder"] = module_subfolder
if not accepts_var_kw:
@@ -1154,14 +1130,13 @@ class FastSentenceTransformer(FastModel):
AutoProcessor.from_pretrained = original_processor_from_pretrained
AutoTokenizer.from_pretrained = original_tokenizer_from_pretrained
- # On sentence-transformers >=5.4 `tokenizer` is a read-only property backed
- # by `self.processor` (already wired via the redirect above). On older
- # versions it's a regular attribute and the explicit assignment is required.
+ # On sentence-transformers >= 5.4 `tokenizer` is a read-only property backed by self.processor;
+ # on older versions it is a regular attribute and this assignment is required.
if not isinstance(getattr(type(transformer_module), "tokenizer", None), property):
transformer_module.tokenizer = tokenizer
transformer_module.do_lower_case = getattr(tokenizer, "do_lower_case", False)
- # sentence-transformers only passes along known keys to model.forward
+ # sentence-transformers only passes along known keys to model.forward.
preinit_model_forward_params = getattr(transformer_module, "model_forward_params", set())
model_forward_params = list(inspect.signature(model.forward).parameters)
transformer_module.model_forward_params = set(model_forward_params) | {
@@ -1172,12 +1147,11 @@ class FastSentenceTransformer(FastModel):
"return_dict",
}
if preinit_model_forward_params is None:
- # ST 6 uses None to allow arbitrary **kwargs
+ # ST 6 uses None to allow arbitrary **kwargs.
transformer_module.model_forward_params = None
else:
transformer_module.model_forward_params |= preinit_model_forward_params
- # determine max_seq_length if not provided
if max_seq_length is None:
if hasattr(model, "config") and hasattr(model.config, "max_position_embeddings"):
max_seq_length = model.config.max_position_embeddings
@@ -1272,7 +1246,6 @@ class FastSentenceTransformer(FastModel):
)
modules[name] = transformer_module
else:
- # load other modules (Pooling, Normalize, etc.)
module_path = module_config["path"]
if os.path.isdir(model_name):
load_path = os.path.join(model_name, module_path)
@@ -1298,7 +1271,7 @@ class FastSentenceTransformer(FastModel):
return modules, False
- # fallback if no modules.json (non sentence-transformers models)
+ # Fallback if there is no modules.json (non sentence-transformers models).
print(
"Unsloth: No modules.json found, falling back to [Transformer, Pooling, Normalize]. This may or may not work."
)
@@ -1327,7 +1300,7 @@ class FastSentenceTransformer(FastModel):
return modules, True
- # Encoder model types that benefit from native torch.compile instead of Unsloth patching
+ # Encoder model types that benefit from native torch.compile instead of Unsloth patching.
ENCODER_MODEL_TYPES = {
"mpnet",
"bert",
@@ -1372,9 +1345,8 @@ class FastSentenceTransformer(FastModel):
params_m = params / 1e6
- # Empirical formula based on benchmarks with batch_size=2, grad_accum=4
- # Small models: high fixed overhead, lower speedup
- # Large models: warmup scales but speedup is significant
+ # Empirical formula from benchmarks with batch_size=2, grad_accum=4: small models carry high fixed
+ # overhead and a lower speedup, large models scale warmup but gain significantly.
if params_m < 50:
estimated_warmup = 35 + params_m * 0.3
base_speedup = 1.35
@@ -1385,7 +1357,6 @@ class FastSentenceTransformer(FastModel):
estimated_warmup = 15 + params_m * 0.04
base_speedup = 1.60
- # Estimate time per step (ms) and time saved
naive_ms = 50 + params_m * 1.0
compiled_ms = naive_ms / base_speedup
time_saved_per_step_s = (naive_ms - compiled_ms) / 1000
@@ -1395,11 +1366,10 @@ class FastSentenceTransformer(FastModel):
else:
breakeven = float("inf")
- # Return threshold with 1.2x safety margin
threshold = breakeven * 1.2
- # Optional adjustment based on expected work per step.
- # This uses only pre-run information (batch size, grad accum, seq length).
+ # Optional adjustment based on expected work per step, using only pre-run information (batch size,
+ # grad accum, seq length).
generic_scale = 1.0
fast_scale = 1.0
if batch_size is not None or grad_accum is not None or max_seq_length is not None:
@@ -1461,11 +1431,8 @@ class FastSentenceTransformer(FastModel):
model[0].model = compiled
else:
model[0].auto_model = compiled
- # Fix for accelerate unwrap_model bug:
- # When SentenceTransformer contains a compiled inner model,
- # accelerate checks has_compiled_regions() which returns True,
- # then tries to access model.__dict__["_orig_mod"] which fails.
- # This workaround sets _orig_mod to satisfy accelerate.
+ # Works around an accelerate unwrap_model bug: with a compiled inner model accelerate sees
+ # has_compiled_regions() True and then fails on model.__dict__["_orig_mod"], so set it.
model.__dict__["_orig_mod"] = model
else:
model = torch.compile(model, mode = mode)
@@ -1508,17 +1475,15 @@ class FastSentenceTransformer(FastModel):
"Run `pip install sentence-transformers` to install it."
)
- # The other leaf loaders resolve the "unsloth" sentinel by planning; this one
- # declines. `st_device` below hands `device_map` to `SentenceTransformer(device=)`,
- # which ends in `self.to(device)`: the sentinel raises there, and that same `.to()`
- # would pull any split model back onto one card. The env-var opt-in is resolved too,
- # or `UNSLOTH_AUTO_DEVICE_MAP=1` asks for a plan without ever naming the sentinel.
+ # The other leaf loaders resolve the "unsloth" sentinel by planning; this one declines. st_device
+ # below hands device_map to SentenceTransformer(device=), which ends in self.to(device): the
+ # sentinel raises there, and that same .to() would pull a split model back onto one card. The
+ # env-var opt-in is resolved too, or UNSLOTH_AUTO_DEVICE_MAP=1 asks for a plan without naming the
+ # sentinel.
device_map = requested_device_map(device_map)
- # Always "sequential", never the asked-for name's own declined value: the `st_device`
- # blocks below normalise only dicts, "auto" and "sequential", so "balanced" would
- # reach `SentenceTransformer(device = "balanced")` and then `.to("balanced")`, which
- # is not a torch device. There is nothing to shard here in any case.
- # `isinstance` first: a caller's explicit dict is unhashable, so `in` alone raises.
+ # Always "sequential", never the asked-for name's own declined value: the st_device blocks
+ # normalise only dicts, "auto" and "sequential", so "balanced" would reach .to("balanced").
+ # isinstance first, since a caller's explicit dict is unhashable and `in` alone raises.
if isinstance(device_map, str) and device_map in _PLANNED_DEVICE_MAPS:
print(
"Unsloth: Not planning a device map; SentenceTransformer moves the assembled "
@@ -1526,8 +1491,8 @@ class FastSentenceTransformer(FastModel):
)
device_map = "sequential"
- # Validate the load modes BEFORE the prefetch so a bad config fails without downloading weights.
- # Guard on not for_inference: that branch below never used these flags.
+ # Validate the load modes BEFORE the prefetch so a bad config fails without downloading weights;
+ # guarded on not for_inference, since that branch never used these flags.
if not for_inference:
# sanity check, thanks Etherl:
if full_finetuning and (load_in_4bit or load_in_8bit):
@@ -1547,9 +1512,8 @@ class FastSentenceTransformer(FastModel):
"If you want 8bit finetuning, set both `load_in_16bit = False` and `load_in_8bit = True`"
)
- # Prefetch so the ST load below is a cache hit. weights_at_root stays False (ST component
- # weights live in per-module subfolders). Resolve the same cache the load uses: HF cache_dir,
- # else cache_folder, else SENTENCE_TRANSFORMERS_HOME, else default -- a wrong cache misses the warm.
+ # Prefetch so the ST load below is a cache hit. weights_at_root stays False, since ST component
+ # weights live in per-module subfolders.
_st_prefetched = maybe_prefetch_hf_snapshot(
model_name,
token = token,
@@ -1558,14 +1522,14 @@ class FastSentenceTransformer(FastModel):
or kwargs.get("cache_folder")
or os.environ.get("SENTENCE_TRANSFORMERS_HOME"),
local_files_only = kwargs.get("local_files_only", False),
- # Forward force_download so the refresh happens in the killable child, then clear it so the
- # in-process ST load reuses the warm cache instead of re-downloading over unguarded Xet.
+ # Forward force_download so the refresh happens in the killable child, then clear it so the in-
+ # process ST load reuses the warm cache instead of re-downloading over unguarded Xet.
force_download = kwargs.get("force_download", False),
)
if _st_prefetched and kwargs.get("force_download", False):
kwargs["force_download"] = False
- # if for_inference == True, skip Unsloth optimizations to avoid torch compile issues
+ # With for_inference == True, skip Unsloth optimizations to avoid torch compile issues.
if for_inference:
st_device = device_map
if isinstance(st_device, dict) or (
@@ -1573,7 +1537,7 @@ class FastSentenceTransformer(FastModel):
):
st_device = None
- # Propagate dtype to model_kwargs (else inference defaults to float32)
+ # Propagate dtype to model_kwargs, else inference defaults to float32.
model_kwargs = kwargs.get("model_kwargs", {})
model_kwargs["dtype"] = dtype if dtype is not None else "auto"
@@ -1595,8 +1559,8 @@ class FastSentenceTransformer(FastModel):
if k in kwargs:
st_kwargs[k] = kwargs[k]
- # ST takes cache_folder, not cache_dir: map cache_dir onto it so this load hits the warm
- # (None lets ST honor SENTENCE_TRANSFORMERS_HOME, matching the prefetch).
+ # ST takes cache_folder, not cache_dir: map cache_dir onto it so this load hits the warm (None lets
+ # ST honor SENTENCE_TRANSFORMERS_HOME, matching the prefetch).
_st_cache = kwargs.get("cache_dir") or kwargs.get("cache_folder")
if _st_cache is not None:
st_kwargs["cache_folder"] = _st_cache
@@ -1619,20 +1583,16 @@ class FastSentenceTransformer(FastModel):
except:
pass
- # Fast encoder path: Use native torch.compile for encoder models (6x speedup)
- # This bypasses Unsloth's auto-compiler which adds @torch.compiler.disable decorators
- # that interfere with torch.compile and cause runtime errors for encoder models.
- # NOTE: The old Unsloth path is BROKEN for encoder models with torch 2.9+ due to
- # conflicting @torch.compile and @torch.compiler.disable decorators.
- # Set UNSLOTH_COMPILE_DISABLE=1 to disable torch.compile and use the old path.
+ # Fast encoder path: native torch.compile for encoder models (6x speedup), bypassing Unsloth's
+ # auto-compiler, whose @torch.compiler.disable decorators error for encoders on torch 2.9+.
+ # Set UNSLOTH_COMPILE_DISABLE=1 for the old path.
is_encoder_model = model_type.lower() in FastSentenceTransformer.ENCODER_MODEL_TYPES
use_fast_encoder = os.environ.get("UNSLOTH_COMPILE_DISABLE", "0") != "1"
if use_fast_encoder and is_encoder_model:
- # torch.compile mode: "default" is safest for PEFT/LoRA training
- # Note: "reduce-overhead" uses CUDA Graphs which is incompatible with PEFT
+ # torch.compile mode "default" is safest for PEFT/LoRA training; "reduce-overhead" uses CUDA Graphs,
+ # which are incompatible with PEFT.
compile_mode = "default"
- # Determine dtype - handle float16 machines that don't support bfloat16
if dtype is None:
if load_in_16bit:
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
@@ -1642,14 +1602,12 @@ class FastSentenceTransformer(FastModel):
print("Unsloth: Device does not support bfloat16. Using float16 instead.")
dtype = torch.float16
- # Determine device
st_device = device_map
if isinstance(st_device, dict) or (
isinstance(st_device, str) and st_device in ["auto", "sequential"]
):
st_device = "cuda"
- # Build model_kwargs for SentenceTransformer
model_kwargs = {"torch_dtype": dtype}
encoder_attn_impl = resolve_encoder_attention_implementation(
@@ -1662,7 +1620,6 @@ class FastSentenceTransformer(FastModel):
if encoder_attn_impl is not None:
model_kwargs["attn_implementation"] = encoder_attn_impl
- # Print optimization status
sdpa_str = " + SDPA" if supports_sdpa else ""
if load_in_4bit:
print(
@@ -1673,7 +1630,7 @@ class FastSentenceTransformer(FastModel):
f"Unsloth: Using fast encoder path for {model_type} (torch.compile{sdpa_str})"
)
- # Handle 4-bit quantization via BitsAndBytesConfig
+ # Handle 4-bit quantization via BitsAndBytesConfig; when quantizing, device must be handled by accelerate.
if load_in_4bit:
from transformers import BitsAndBytesConfig
@@ -1687,12 +1644,12 @@ class FastSentenceTransformer(FastModel):
# When using quantization, device must be handled by accelerate
st_device = None
- # Handle gradient checkpointing - warn user it conflicts with torch.compile
+ # Gradient checkpointing conflicts with torch.compile, so warn the user.
_use_gc = use_gradient_checkpointing
if _use_gc and _use_gc != False:
print("Unsloth Warning: Gradient checkpointing is incompatible with torch.compile.")
print("Disabling torch.compile to enable gradient checkpointing.")
- compile_mode = None # Disable compilation
+ compile_mode = None
is_mpnet = "mpnet" == model_type.lower()
@@ -1701,8 +1658,8 @@ class FastSentenceTransformer(FastModel):
elif is_mpnet:
FastSentenceTransformer._patch_mpnet_v5()
- # ST takes cache_folder, not cache_dir: map cache_dir onto it so this load hits the warm
- # (None lets ST honor SENTENCE_TRANSFORMERS_HOME, matching the prefetch).
+ # ST takes cache_folder, not cache_dir: map cache_dir onto it so this load hits the warm (None lets
+ # ST honor SENTENCE_TRANSFORMERS_HOME, matching the prefetch).
st_model = SentenceTransformer(
model_name,
device = st_device,
@@ -1713,7 +1670,6 @@ class FastSentenceTransformer(FastModel):
cache_folder = kwargs.get("cache_dir") or kwargs.get("cache_folder"),
)
- # Store metadata for get_peft_model
st_model._unsloth_fast_encoder = True
st_model._compile_mode = compile_mode
st_model._dtype = dtype
@@ -1723,7 +1679,6 @@ class FastSentenceTransformer(FastModel):
st_model[0], getattr(st_model[0].auto_model, "config", None)
)
- # Add save methods
def _save_pretrained_merged(
self,
save_directory,
@@ -1731,15 +1686,13 @@ class FastSentenceTransformer(FastModel):
save_method = "merged_16bit",
**save_kwargs,
):
- # `tokenizer` and `save_method` are positional to match
- # FastLanguageModel.save_pretrained_merged(dir, tokenizer,
- # save_method = ...), which is how the docs and notebooks call
- # it. Keyword callers behave exactly as before.
+ # tokenizer and save_method are positional to match
+ # FastLanguageModel.save_pretrained_merged(dir, tokenizer, save_method = ...), which is how the
+ # docs and notebooks call it; keyword callers are unaffected.
save_method = _normalize_save_method(save_method)
if save_method not in ("merged_16bit", None):
- # Refused before anything is written: this path merges and
- # unloads unconditionally, so accepting "lora" would write
- # full weights for a request to write adapters.
+ # Refused before anything is written: this path merges and unloads unconditionally, so accepting
+ # "lora" would write full weights for a request to write adapters.
raise NotImplementedError(
f"Unsloth: save_method = {save_method!r} is not "
f"supported for this SentenceTransformer; only "
@@ -1752,7 +1705,6 @@ class FastSentenceTransformer(FastModel):
save_kwargs.pop("tokenizer", None)
if hasattr(self[0], "auto_model"):
inner = self[0].auto_model
- # Handle compiled model
if hasattr(inner, "_orig_mod"):
inner = inner._orig_mod
if hasattr(inner, "merge_and_unload"):
@@ -1800,7 +1752,7 @@ class FastSentenceTransformer(FastModel):
return st_model
- # Warn if using 4-bit with encoder (slow due to dequantization overhead)
+ # Warn if using 4-bit with an encoder: it is slow due to dequantization overhead.
if is_encoder_model and load_in_4bit:
print("Unsloth Warning: 4-bit quantization adds ~2.3x overhead for encoder models.")
print("Consider using load_in_16bit=True for better performance.")
@@ -1813,15 +1765,14 @@ class FastSentenceTransformer(FastModel):
if supported:
kwargs["add_pooling_layer"] = False
- # fp8 is not supported, force it off
+ # fp8 is not supported, so force it off.
fp8 = kwargs.pop("load_in_fp8", None)
if fp8:
logging.info("Unsloth: Disabling fp8 for model")
load_in_fp8 = False
- # this is a fix for Snowflake/snowflake-arctic-embed-l-v2.0
- # it has pooler weights which we don't care about for training,
- # however unsloth throws an exception if "UNSLOTH_WARN_UNINITIALIZED" == 1 and it sees unused weights
+ # Fix for Snowflake/snowflake-arctic-embed-l-v2.0: it has pooler weights irrelevant to training, but
+ # unsloth throws when UNSLOTH_WARN_UNINITIALIZED == 1 and it sees unused weights.
old_environ = os.environ.get("UNSLOTH_WARN_UNINITIALIZED", "1")
os.environ["UNSLOTH_WARN_UNINITIALIZED"] = "0"
@@ -1837,9 +1788,9 @@ class FastSentenceTransformer(FastModel):
elif is_mpnet:
FastSentenceTransformer._patch_mpnet_v5()
- # No modules.json -> force 16-bit: saving is custom for these models and
- # 4-bit would need dequant in save_pretrained_merged, not worth it.
- # Resolve the warmed cache: hf_hub_download ignores SENTENCE_TRANSFORMERS_HOME, so pass it as cache_dir.
+ # No modules.json means forcing 16-bit: saving is custom for these models and 4-bit would need
+ # dequant in save_pretrained_merged. Resolve the warmed cache: hf_hub_download ignores
+ # SENTENCE_TRANSFORMERS_HOME, so pass it as cache_dir.
has_modules_json = (
FastSentenceTransformer._module_path(
model_name,
@@ -1847,6 +1798,7 @@ class FastSentenceTransformer(FastModel):
cache_dir = kwargs.get("cache_dir")
or kwargs.get("cache_folder")
or os.environ.get("SENTENCE_TRANSFORMERS_HOME"),
+ # Same revision as the weight load so modules hit the warm (None = default branch).
revision = revision,
)
is not None
@@ -1860,20 +1812,15 @@ class FastSentenceTransformer(FastModel):
load_in_4bit = False
load_in_16bit = True
- # The fallback FastModel load reads HF cache_dir, not ST's cache_folder/SENTENCE_TRANSFORMERS_HOME.
- # Point it at the warmed cache, but only when no explicit cache_dir was passed (which wins).
+ # The fallback FastModel load reads HF cache_dir, not ST's cache_folder/SENTENCE_TRANSFORMERS_HOME,
+ # so point it at the warmed cache, but only when no explicit cache_dir was passed.
_st_cache_dir = kwargs.get("cache_folder") or os.environ.get("SENTENCE_TRANSFORMERS_HOME")
if _st_cache_dir is not None and "cache_dir" not in kwargs:
kwargs["cache_dir"] = _st_cache_dir
- # The decline above only spends the sentinel on our copy. Strip the marker before
- # the nested load, or FastModel reads it as "nobody chose this" and re-upgrades it
- # under UNSLOTH_AUTO_DEVICE_MAP, splitting a model `st_device` then pulls back onto
- # one card. Only the marker -- an explicit dict placement must arrive as a dict.
- #
- # A plain value rather than pinning the env var around the call: os.environ is
- # process-wide, so that pin reached unrelated loads on other threads, and two
- # overlapping sentence loads could restore it out of order.
+ # The decline above only spends the sentinel on our copy: strip the marker before the nested load,
+ # or FastModel reads it as "nobody chose this" and re-upgrades it under UNSLOTH_AUTO_DEVICE_MAP.
+ # A plain value rather than pinning the env var, since os.environ is process-wide.
device_map = unmarked_device_map(device_map)
try:
model, tokenizer = FastModel.from_pretrained(
@@ -1916,11 +1863,11 @@ class FastSentenceTransformer(FastModel):
max_seq_length,
pooling_mode,
trust_remote_code = trust_remote_code,
- # Same resolved cache as above so the fallback module loads hit the warm, not Xet.
+ # Same resolved cache as above so the fallback module loads hit the warm, not Xet, and the same
+ # revision as the weight load (None = default branch).
cache_dir = kwargs.get("cache_dir")
or kwargs.get("cache_folder")
or os.environ.get("SENTENCE_TRANSFORMERS_HOME"),
- # Same revision as the weight load so modules hit the warm (None = default branch).
revision = revision,
)
@@ -1940,15 +1887,12 @@ class FastSentenceTransformer(FastModel):
save_method = "merged_16bit",
**kwargs,
):
- # Positional to match FastLanguageModel.save_pretrained_merged;
- # see the note on the other definition above. This path forwards to
- # that merge, which understands every save_method.
+ # Positional to match FastLanguageModel.save_pretrained_merged; see the note on the other definition
+ # above. This path forwards to that merge, which understands every save_method.
save_method = _normalize_save_method(save_method)
if self.no_modules and save_method not in ("merged_16bit", None):
- # The no_modules branch below merges and unloads unconditionally
- # and drops save_method, so accepting "lora" here would return
- # full weights for a request to write adapters. Refuse before
- # writing, as the fast-encoder definition above does.
+ # The no_modules branch below merges and unloads unconditionally and drops save_method, so accepting
+ # "lora" here would return full weights for a request to write adapters.
raise NotImplementedError(
f"Unsloth: save_method = {save_method!r} is not supported "
f"for this SentenceTransformer: no modules.json was found, "
@@ -1957,16 +1901,16 @@ class FastSentenceTransformer(FastModel):
)
if save_method is not None:
kwargs.setdefault("save_method", save_method)
- # check which adapter files exist before save_pretrained
adapter_files = ["adapter_model.safetensors", "adapter_config.json"]
existing_before = {
f for f in adapter_files if os.path.exists(os.path.join(save_directory, f))
}
+ # sentence-transformers config and modules are only saved if save_pretrained is called.
# sentence-transformers config and modules only get saved if we call save_pretrained
self.save_pretrained(save_directory)
- # remove LoRA adapters only if they were created by save_pretrained (not pre-existing)
+ # Remove LoRA adapters only if they were created by save_pretrained, not pre-existing.
for file in adapter_files:
if file not in existing_before:
try:
@@ -1979,10 +1923,10 @@ class FastSentenceTransformer(FastModel):
else:
kwargs.pop("tokenizer", None)
if self.no_modules:
- # fallback for non-sentence-transformers models
+ # Fallback for non-sentence-transformers models.
print("Unsloth: No modules detected. Using standard merge_and_unload for saving...")
safe_kwargs = kwargs.copy()
- # filter out Unsloth-specific args that are not in huggingface's save_pretrained
+ # Filter out Unsloth-specific args that are not in huggingface's save_pretrained.
unsloth_args = [
"save_method",
"temporary_location",
@@ -2000,7 +1944,6 @@ class FastSentenceTransformer(FastModel):
save_directory, tokenizer = tokenizer, **kwargs
)
- # add Unsloth branding to the generated README
try:
FastSentenceTransformer._add_unsloth_branding(save_directory)
except Exception as e:
@@ -2084,7 +2027,7 @@ class FastSentenceTransformer(FastModel):
print("Setting task_type to FEATURE_EXTRACTION")
if isinstance(model, SentenceTransformer):
- # Check if this is a fast encoder model (uses torch.compile instead of Unsloth patching)
+ # Check whether this is a fast encoder model, which uses torch.compile instead of Unsloth patching.
is_fast_encoder = getattr(model, "_unsloth_fast_encoder", False)
if is_fast_encoder:
@@ -2092,17 +2035,14 @@ class FastSentenceTransformer(FastModel):
transformer_module = model[0]
inner_model = transformer_module.auto_model
- # Check if model is quantized (4-bit/8-bit)
is_quantized = (
getattr(inner_model, "is_quantized", False)
or getattr(inner_model.config, "quantization_config", None) is not None
)
- # Track if gradient checkpointing was actually enabled
gc_enabled = False
- # this is needed when from_pretrained was called without gradient
- # checkpointing but get_peft_model requests it
+ # Needed when from_pretrained was called without gradient checkpointing but get_peft_model requests it.
if use_gradient_checkpointing and use_gradient_checkpointing != False:
import transformers
from packaging.version import Version
@@ -2115,7 +2055,6 @@ class FastSentenceTransformer(FastModel):
elif model_type == "mpnet":
FastSentenceTransformer._patch_mpnet_v5()
- # Prepare for k-bit training if quantized
if is_quantized:
from ._utils import prepare_model_for_kbit_training
_gc_for_kbit = (
@@ -2130,7 +2069,7 @@ class FastSentenceTransformer(FastModel):
gc_enabled = bool(_gc_for_kbit)
except ValueError as e:
if "does not support gradient checkpointing" in str(e):
- # Model doesn't support gradient checkpointing, disable it
+ # The model does not support gradient checkpointing, so disable it.
print(
f"Unsloth Warning: {inner_model.__class__.__name__} does not support gradient checkpointing. Skipping."
)
@@ -2144,7 +2083,8 @@ class FastSentenceTransformer(FastModel):
else:
raise
- # Enable gradient checkpointing if requested (only for non-quantized, since prepare_model handles it)
+ # Enable gradient checkpointing if requested, only for non-quantized models, since prepare_model
+ # handles the rest.
elif use_gradient_checkpointing and use_gradient_checkpointing != False:
if hasattr(inner_model, "gradient_checkpointing_enable"):
try:
@@ -2157,7 +2097,6 @@ class FastSentenceTransformer(FastModel):
f"Unsloth Warning: {inner_model.__class__.__name__} does not support gradient checkpointing. Skipping."
)
- # Create LoRA config
lora_config = LoraConfig(
r = r,
lora_alpha = lora_alpha,
@@ -2168,16 +2107,15 @@ class FastSentenceTransformer(FastModel):
task_type = kwargs.get("task_type", "FEATURE_EXTRACTION"),
)
- # Apply PEFT directly (not through FastModel)
peft_model = peft_get_peft_model(inner_model, lora_config)
- # Apply QAT if specified
qat_scheme = kwargs.get("qat_scheme", None)
if qat_scheme is not None:
from ._utils import _prepare_model_for_qat
peft_model = _prepare_model_for_qat(peft_model, qat_scheme)
- # Determine compile mode (only if not using gradient checkpointing)
+ # Determine the compile mode, only if not using gradient checkpointing, and re-enable torch.compile
+ # when gradient checkpointing was requested but could not be enabled.
compile_mode = getattr(model, "_compile_mode", "default")
# Re-enable torch.compile if gradient checkpointing was requested but couldn't be enabled
if compile_mode is None and not gc_enabled:
@@ -2186,9 +2124,8 @@ class FastSentenceTransformer(FastModel):
"Unsloth: Re-enabling torch.compile since gradient checkpointing is not supported"
)
- # Re-assign the peft model back to the transformer module.
- # On sentence-transformers >=5.4 `auto_model` is a read-only property
- # backed by `self.model`, so write to the backing attribute there.
+ # Re-assign the peft model back to the transformer module; on sentence-transformers >= 5.4
+ # auto_model is a read-only property backed by self.model, so write to the backing attribute there.
if isinstance(getattr(type(transformer_module), "auto_model", None), property):
transformer_module.model = peft_model
else:
@@ -2197,14 +2134,12 @@ class FastSentenceTransformer(FastModel):
transformer_module, getattr(inner_model, "config", None)
)
- # Store compile info for auto-compile at trainer time
- # torch.compile is deferred until training starts so we can check max_steps
+ # torch.compile is deferred until training starts so max_steps can be checked.
if compile_mode is not None:
model._compile_mode = compile_mode
model._compile_threshold = FastSentenceTransformer._estimate_compile_threshold(
model
)
- # Flag to indicate compile has not been applied yet
model._compile_pending = True
print(
f"Unsloth: torch.compile will be applied automatically if max_steps > {model._compile_threshold}"
@@ -2216,7 +2151,6 @@ class FastSentenceTransformer(FastModel):
return model
- # Original path for non-fast-encoder models
transformer_module = model[0]
inner_model = transformer_module.auto_model
@@ -2240,9 +2174,8 @@ class FastSentenceTransformer(FastModel):
**kwargs,
)
- # re-assign the peft model back to the transformer module.
- # On sentence-transformers >=5.4 `auto_model` is a read-only property
- # backed by `self.model`, so write to the backing attribute there.
+ # Re-assign the peft model back to the transformer module; on sentence-transformers >= 5.4
+ # auto_model is a read-only property backed by self.model, so write to the backing attribute there.
if isinstance(getattr(type(transformer_module), "auto_model", None), property):
transformer_module.model = peft_model
else:
@@ -2294,7 +2227,6 @@ def _patch_sentence_transformer_trainer():
@wraps(_original_init)
def _patched_init(self, *args, **kwargs):
- # Extract model and training_args
model = kwargs.get("model") or (args[0] if args else None)
training_args = kwargs.get("args") or (args[1] if len(args) > 1 else None)
@@ -2307,7 +2239,7 @@ def _patch_sentence_transformer_trainer():
max_steps = getattr(training_args, "max_steps", -1)
compile_mode = getattr(model, "_compile_mode", "default")
- # Re-estimate threshold now that training args are available
+ # Re-estimate the threshold now that training args are available.
batch_size = getattr(training_args, "per_device_train_batch_size", None)
grad_accum = getattr(training_args, "gradient_accumulation_steps", None)
max_seq_length = getattr(model, "max_seq_length", None)
@@ -2340,7 +2272,6 @@ def _patch_sentence_transformer_trainer():
)
model._compile_pending = False
- # Call original __init__
_original_init(self, *args, **kwargs)
# Disable mixed precision when FORCE_FLOAT32 is active (matches rl.py behavior)
@@ -2362,6 +2293,8 @@ def _patch_sentence_transformer_trainer():
def _patch_st_trainer_load_from_checkpoint():
+ # Parameterless modules (Pooling, Normalize) make next(module.parameters()) raise StopIteration; route
+ # through the SentenceTransformer's device property instead.
try:
from sentence_transformers import SentenceTransformerTrainer
except ImportError:
@@ -2462,9 +2395,8 @@ def _patch_st_trainer_load_from_checkpoint():
fresh = module_cls.load(module_dir)
if not isinstance(fresh, module_cls):
raise RuntimeError(f"Unsloth: Module {idx} reload returned wrong type.")
- # Parameterless modules (Pooling, Normalize) make
- # next(module.parameters()) raise StopIteration; route through
- # the SentenceTransformer's device property instead.
+ # Parameterless modules (Pooling, Normalize) make next(module.parameters()) raise StopIteration, so
+ # route through the SentenceTransformer's device property instead.
try:
fresh.to(self.model.device)
except AttributeError:
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 2296e61f8bd..bbfd6d8cfc5 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -104,7 +104,7 @@ import contextlib
try:
from huggingface_hub.utils import get_token
except:
- # Old HF Hub versions <= 0.0.25
+ # Old HF Hub versions <= 0.0.25.
from huggingface_hub.utils._token import get_token
from ..device_type import (
get_device_stats,
@@ -170,16 +170,16 @@ def _repair_dispatch_hooks(model):
except ImportError:
return 0
- # dispatch passes the model's skip keys, and sets io_same_device on the ROOT
- # only; setting it on a submodule copies every result back and forth.
+ # dispatch sets io_same_device on the ROOT only; setting it on a submodule copies every result
+ # back and forth.
skip_keys = getattr(model, "_skip_keys_device_placement", None)
io_same_device = not hasattr(model, "_hf_hook")
targets = {name: device for name, device in device_map.items() if name}
- # A coarse map covers an endpoint through an ANCESTOR and dispatch hooks it
- # anyway, so iterating keys alone misses the rebuilt one. Endpoints only:
- # repairing every covered descendant would hook thousands that never lost one.
+ # A coarse map covers an endpoint through an ANCESTOR and dispatch hooks it anyway, so iterating
+ # keys alone misses the rebuilt one. Endpoints only: repairing every covered descendant would
+ # hook thousands that never lost one.
for get in ("get_input_embeddings", "get_output_embeddings"):
try:
replaced = getattr(model, get, lambda: None)()
@@ -212,7 +212,7 @@ def _repair_dispatch_hooks(model):
if isinstance(device, int) and not isinstance(device, bool)
else device
)
- # `.type`, not `==`: a map may carry `torch.device("cpu", 0)`.
+ # .type, not ==: a map may carry torch.device("cpu", 0).
device_kind = (
execution_device.type
if isinstance(execution_device, torch.device)
@@ -306,8 +306,8 @@ def _attach_bnb_multidevice_hooks(
"""
if fast_inference:
return
- # Before the bnb gate: the rebuild happens whatever the quantization. Not under
- # offload, where `_embedding_dispatch_device` READS this hook to place the ids.
+ # Before the bnb gate: the rebuild happens whatever the quantization. Not under offload, where
+ # _embedding_dispatch_device READS this hook to place the ids.
if not offload_embedding:
repaired = _repair_dispatch_hooks(model)
if repaired:
@@ -367,7 +367,7 @@ def _attach_bnb_multidevice_hooks(
_stripped.append((param, key, param.__dict__.pop(key)))
try:
- # CUDA -> int index, non-CUDA -> type string ("cpu", "meta").
+ # CUDA gets an int index, non-CUDA a type string ("cpu", "meta").
device_map_int = {
k: (v.index if v.type == "cuda" else v.type) if isinstance(v, torch.device) else v
for k, v in inferred_map.items()
@@ -412,7 +412,7 @@ NUM_LOGITS_TO_KEEP = dict()
def _unsloth_generate_accepts_kwarg(model, key):
- # True if the top level accepts this generate kwarg (some models expose it on an inner forward only).
+ # True if the top level accepts this generate kwarg; some models expose it on an inner forward only.
try:
model_args = set(inspect.signature(model.prepare_inputs_for_generation).parameters)
except (TypeError, ValueError, AttributeError):
@@ -427,8 +427,8 @@ def _unsloth_generate_accepts_kwarg(model, key):
def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_device):
# Lookup runs on the weight's current device (CPU when offloaded); the output returns to the
- # decoder device read live from output_embeddings (lm_head, untied here) so it tracks
- # model.to() moves. A meta (disk-offloaded) or missing lm_head falls back to return_device.
+ # decoder device read live from output_embeddings, so it tracks model.to() moves. A meta or
+ # missing lm_head falls back to return_device.
if embed_tokens is None:
return False
if getattr(embed_tokens, "_unsloth_offload_hooks_installed", False):
@@ -465,7 +465,7 @@ def _install_offload_embedding_hooks(embed_tokens, output_embeddings, return_dev
def _embeddings_are_tied(input_embeddings, output_embeddings):
- # Tied lm_head reuses this weight; offloading to CPU would strand the output projection.
+ # A tied lm_head reuses this weight, so offloading to CPU would strand the output projection.
if input_embeddings is None or output_embeddings is None:
return False
w_in = getattr(input_embeddings, "weight", None)
@@ -485,14 +485,14 @@ def _offload_embedding_unsupported_platform():
def _embedding_dispatch_device(input_embeddings):
- # accelerate's hook wraps forward and re-sends the ids to the device it recorded, after
- # our offload pre-hook already sent them to the CPU weight. Returns that device, or None.
+ # accelerate's hook re-sends the ids to the device it recorded, after our offload pre-hook
+ # already sent them to the CPU weight. Returns that device, or None.
hook = getattr(input_embeddings, "_hf_hook", None)
return None if hook is None else getattr(hook, "execution_device", None)
-# Big in absolute terms and big on this card, since every lookup then crosses PCIe. 2.5 GiB
-# is 16% of a 16 GB T4 and worth moving, 3% of an 80 GB card and not.
+# Big in absolute terms and big on this card, since every lookup then crosses PCIe: 2.5 GiB is
+# 16% of a 16 GB T4 and worth moving, 3% of an 80 GB card and not.
_OFFLOAD_EMBEDDING_MIN_BYTES = 2**30
_OFFLOAD_EMBEDDING_MIN_FRACTION = 0.05
@@ -555,9 +555,9 @@ def _resolve_offload_embedding(model, offload_embedding):
if _embedding_dispatch_device(in_embed) is not None:
return _decline("this model is dispatched across devices, which overrides the offload.")
if is_distributed():
- # The offload leaves embed_tokens on the CPU while the rest of the rank stays on
- # CUDA. Under full finetuning it is trainable, and DDP with device_ids refuses a
- # module whose trainable parameters span both, so the run dies before step one.
+ # The offload leaves embed_tokens on the CPU while the rest of the rank stays on CUDA; under
+ # full finetuning it is trainable, and DDP with device_ids refuses a module whose trainable
+ # parameters span both.
return _decline("a distributed launch cannot wrap a model split across CPU and GPU.")
return _embedding_is_worth_offloading(in_embed) if automatic else True
@@ -702,7 +702,7 @@ def _clear_generation_caches(model):
del module._flex_attention_cache
except:
pass
- # Solves AttributeError: 'SlidingWindowLayer' object has no attribute 'max_batch_size'
+ # Solves AttributeError: 'SlidingWindowLayer' object has no attribute 'max_batch_size'.
if hasattr(module, "_cache") and "cache_utils" in str(module._cache.__class__):
try:
del module._cache
@@ -720,7 +720,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
elif "input_features" in kwargs:
input_ids = kwargs["input_features"]
elif "inputs_embeds" in kwargs:
- # canonical HF name for embedding inputs (e.g. multimodal generate)
+ # Canonical HF name for embedding inputs (e.g. multimodal generate).
input_ids = kwargs["inputs_embeds"]
elif "input_embeds" in kwargs:
input_ids = kwargs["input_embeds"]
@@ -748,13 +748,12 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
is_vlm = is_vlm or hasattr(self.config, "vision_config")
arch = self.config.architectures[0]
- # Remove token_type_ids - WRONG for Gemma 3 since bidirectional attention
+ # Removing token_type_ids is WRONG for Gemma 3, which uses bidirectional attention.
if hasattr(self, "generate") and hasattr(self, "forward"):
# did not combine with below since self might not have model
keys = inspect.signature(self.forward).parameters.keys()
if "token_type_ids" not in keys:
kwargs.pop("token_type_ids", None)
- # kwargs.pop("token_type_ids", None)
# Vision processors emit mm_token_type_ids that generate() rejects (Qwen3-VL); unlike
# logits_to_keep it is an incoming kwarg, so drop it when generate does not accept it.
@@ -763,7 +762,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
):
kwargs.pop("mm_token_type_ids", None)
- # VLMs do not allow logits_to_keep
+ # VLMs do not allow logits_to_keep.
global NUM_LOGITS_TO_KEEP
if arch not in NUM_LOGITS_TO_KEEP:
m = self
@@ -790,7 +789,6 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)
- # Get pixel values for VLMs
try:
kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype)
except:
@@ -800,8 +798,8 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
except:
pass
- # Mixed precision autocast. Stamped by from_pretrained: UNSLOTH_FORCE_FLOAT32 is
- # process wide, so a later load would otherwise decide this model's rollouts.
+ # Mixed precision autocast. Stamped by from_pretrained: UNSLOTH_FORCE_FLOAT32 is process wide,
+ # so a later load would otherwise decide this model's rollouts.
forced_float32 = getattr(self, "_unsloth_forced_float32", None)
if forced_float32 is None:
forced_float32 = os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"
@@ -809,10 +807,9 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
autocaster = torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = torch.float16)
dtype = torch.float16
else:
- # CUDA autocast does not validate its dtype the way the CPU/XPU/MPS paths do,
- # so autocast(dtype = torch.float32) enters ENABLED rather than turning into a
- # no-op, and the compiled graph then returns inf/NaN logits. A float32 model has
- # nothing to autocast to; disable instead of asking for a dtype that is not one.
+ # CUDA autocast does not validate its dtype like the CPU/XPU/MPS paths, so
+ # autocast(dtype=torch.float32) enters ENABLED instead of becoming a no-op and the compiled
+ # graph returns inf/NaN logits. A float32 model has nothing to autocast to, so disable.
autocaster = torch.autocast(
device_type = DEVICE_TYPE_TORCH,
dtype = dtype,
@@ -821,13 +818,13 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
# Prepare LoRA
# state_dict = convert_lora_modules(self, dtype = dtype)
- # FlashAttention breaks on the forced static cache below (unfilled slots stay
- # unmasked while decoding), so delegate after normalization but before it.
+ # FlashAttention breaks on the forced static cache below (unfilled slots stay unmasked while
+ # decoding), so delegate after normalization but before it.
_clear_generation_caches(self)
if _uses_flash_attention_for_generation(self.config):
- # Pin the literal "dynamic": None is merged back to the model default, and a
- # static cache still arrives via kwargs / the caller's generation_config (TRL).
- # The kwarg wins (update runs last); skip it when the caller passed a cache.
+ # Pin the literal "dynamic": None is merged back to the model default, and a static cache still
+ # arrives via kwargs or the caller's generation_config (TRL). The kwarg wins, since update runs
+ # last; skip it when the caller passed a cache.
if kwargs.get("past_key_values") is None:
kwargs["cache_implementation"] = "dynamic"
try:
@@ -836,7 +833,6 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
finally:
_clear_generation_caches(self)
- # Set compile dynamic shapes
torch._dynamo.mark_static(input_ids, 0)
torch._dynamo.mark_dynamic(input_ids, 1)
if "attention_mask" in kwargs:
@@ -846,16 +842,15 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
torch._dynamo.mark_static(kwargs["token_type_ids"], 0)
torch._dynamo.mark_dynamic(kwargs["token_type_ids"], 1)
- # Fix generation_config
- # Use hybrid if sliding window seen, otherwise try static
+ # Fix generation_config: use hybrid if a sliding window was seen, otherwise try static.
cache_implementation = getattr(self.config, "cache_implementation", None)
if getattr(self, "_supports_static_cache", getattr(self, "_can_compile_fullgraph", True)):
if os.environ.get("UNSLOTH_DISABLE_STATIC_GENERATION", "0") == "0":
+ # Should work in latest transformers!
cache_implementation = "static"
elif Version(transformers_version) < Version("4.56.0.dev0"):
cache_implementation = None
else:
- # Should work in latest transformers!
cache_implementation = "static"
else:
cache_implementation = None
@@ -870,7 +865,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
cache_implementation = "hybrid"
else:
cache_implementation = "static"
- # [TODO] Unsure why static fails
+ # Static fails for reasons unknown.
if do_bfloat16_mixed_precision:
cache_implementation = None
@@ -889,11 +884,10 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
finally:
_clear_generation_caches(self)
- # FastBaseModel.for_training(self)
return output
-# Offline helpers live in loader_utils.py (shared canonical source).
+# Offline helpers live in loader_utils.py, the shared canonical source.
from .loader_utils import (
_get_effective_local_files_only,
_hub_repo_or_local_path,
@@ -947,7 +941,6 @@ def _construct_vlm_processor_fallback(
local_files_only = local_files_only,
revision = revision,
)
- # Load image processor
image_processor = AutoImageProcessor.from_pretrained(
load_path,
token = token,
@@ -956,8 +949,8 @@ def _construct_vlm_processor_fallback(
local_files_only = local_files_only,
revision = revision,
)
- # Load tokenizer via PreTrainedTokenizerFast (bypasses tokenizer_class check).
- # Resolve the cached snapshot first so transformers does not call model_info (#7481).
+ # Load the tokenizer via PreTrainedTokenizerFast, bypassing the tokenizer_class check and
+ # resolving the cached snapshot first so transformers does not call model_info (#7481).
tok = _load_pretrained_tokenizer_fast(
tokenizer_name,
padding_side = "left",
@@ -967,16 +960,16 @@ def _construct_vlm_processor_fallback(
local_files_only = local_files_only,
revision = revision,
)
- # Read tokenizer_config.json for special tokens: prefer the local file (offline
- # / local checkpoint dir), else hf_hub_download with local_files_only forwarded.
+ # Read tokenizer_config.json for special tokens: prefer the local file (offline / local
+ # checkpoint dir), else hf_hub_download with local_files_only forwarded.
try:
import json as _json
tok_config = None
_local_cfg = os.path.join(tokenizer_name, "tokenizer_config.json")
if os.path.isdir(tokenizer_name):
- # Local dir: read directly. A missing file raises a clear FileNotFoundError
- # rather than letting hf_hub_download treat the path as a repo id.
+ # Local dir: read directly, so a missing file raises a clear FileNotFoundError rather than
+ # letting hf_hub_download treat the path as a repo id.
if os.path.exists(_local_cfg):
with open(_local_cfg, "r", encoding = "utf-8") as f:
tok_config = _json.load(f)
@@ -996,7 +989,6 @@ def _construct_vlm_processor_fallback(
)
with open(config_path, "r", encoding = "utf-8") as f:
tok_config = _json.load(f)
- # Set model-specific special tokens and their IDs
for key in (
"image_token",
"image_start_token",
@@ -1013,11 +1005,10 @@ def _construct_vlm_processor_fallback(
except Exception as _e:
_fb_err = _e # remember (non-fatal here); surfaced only if no processor is built
- # Find the processor class - try model_type first, then top-level config model_type
+ # Find the processor class by model_type first: it may be a sub-model type ("lfm2" not
+ # "lfm2_vl"), so fall back to the top-level config.model_type, which often has the mapping.
proc_class_name = PROCESSOR_MAPPING_NAMES.get(model_type)
if proc_class_name is None:
- # model_type might be a sub-model type (e.g. "lfm2" instead of "lfm2_vl").
- # Try the top-level config.model_type which often has the processor mapping.
try:
config = AutoConfig.from_pretrained(
load_path,
@@ -1036,7 +1027,6 @@ def _construct_vlm_processor_fallback(
proc_class = getattr(transformers, proc_class_name, None)
if proc_class is not None:
processor = proc_class(image_processor = image_processor, tokenizer = tok)
- # Copy chat_template from tokenizer to processor if needed
if not getattr(processor, "chat_template", None) and getattr(
tok, "chat_template", None
):
@@ -1102,7 +1092,6 @@ class FastBaseModel:
auto_config = None,
offload_embedding = OFFLOAD_EMBEDDING_AUTO,
float32_mixed_precision = None, # Forces float32 mixed precision
- # vLLM parameters
fast_inference = False,
gpu_memory_utilization = 0.5,
float8_kv_cache = False,
@@ -1112,13 +1101,11 @@ class FastBaseModel:
unsloth_vllm_standby = False,
load_in_fp8 = False, # fp8 LoRA (True, False, 'block')
text_only = False,
- # True when the caller already swapped a multimodal config for its text sub-config,
- # so `auto_config` no longer describes the repo. Set by loader.py, and by the block
- # below on the direct-call path.
+ # True when the caller already swapped a multimodal config for its text sub-config, so
+ # auto_config no longer describes the repo. Set by loader.py and by the block below.
text_only_decoder = False,
- # True when `auto_config` came from the caller. It cannot be inferred here:
- # FastModel pops `config` out of kwargs before this sees them, so by now it looks
- # exactly like one we resolved ourselves.
+ # True when auto_config came from the caller. It cannot be inferred here: FastModel pops config
+ # out of kwargs before this sees them, so it looks exactly like one we resolved ourselves.
auto_config_from_caller = False,
**kwargs,
):
@@ -1126,8 +1113,8 @@ class FastBaseModel:
if auto_config is None and user_config is not None:
auto_config = user_config
- # Offline snapshot for the loads below; not popped, so the weight load still
- # reads local_files_only from **kwargs. See _get_effective_local_files_only.
+ # Offline snapshot for the loads below; not popped, so the weight load still reads
+ # local_files_only from **kwargs. See _get_effective_local_files_only.
local_files_only = _get_effective_local_files_only(kwargs)
if unsloth_vllm_standby and os.environ.get("UNSLOTH_VLLM_STANDBY", "0") != "1":
@@ -1142,13 +1129,12 @@ class FastBaseModel:
if os.environ.get("UNSLOTH_MODEL_NAME", "") == "":
os.environ["UNSLOTH_MODEL_NAME"] = model_name.lower()
- # Read from kwargs, not the signature: the weight load below forwards **kwargs, so
- # binding it would drop it there.
+ # Read from kwargs, not the signature: the weight load below forwards **kwargs, so binding it
+ # would drop it there.
_revision = kwargs.get("revision")
_tokenizer_revision_arg = kwargs.pop("tokenizer_revision", None)
if _revision is not None and fast_inference and is_vLLM_available():
- # vLLM fetches the default branch, so pinning only the config and tokenizer
- # would mix two refs in one model.
+ # vLLM fetches the default branch, so pinning only the config and tokenizer would mix two refs in one model.
logger.warning_once(
f"Unsloth: Ignoring revision = `{_revision}` since vLLM loads weights from "
"the default branch. Use `fast_inference = False` to load a pinned revision."
@@ -1158,8 +1144,8 @@ class FastBaseModel:
kwargs.pop("revision", None)
_revision_repo = model_name # The repo the pin names, captured before any remap.
- # Resolve text-only before the is_vlm / vLLM checks so is_vlm stays consistent;
- # skip the vision tower only for families with their own text decoder (Gemma 3). #5816
+ # Resolve text-only before the is_vlm / vLLM checks so is_vlm stays consistent; skip the vision
+ # tower only for families with their own text decoder (Gemma 3) (#5816).
if text_only and auto_config is None:
auto_config = AutoConfig.from_pretrained(
model_name,
@@ -1187,12 +1173,10 @@ class FastBaseModel:
# Pure text model requested text-only with a VLM auto class.
auto_model = AutoModelForCausalLM
is_vlm = auto_model in [AutoModelForVision2Seq, AutoModelForImageTextToText]
- # A repo-code VLM may register only AutoModel / AutoModelForCausalLM (e.g.
- # DeepSeek-OCR, Nemotron-VL), so auto_model is not a VLM class even though the
- # config is a vision model. Keep is_vlm (auto-class derived) for processor
- # selection below -- these repos ship no AutoProcessor -- but treat the model as a
- # VLM on the vLLM path so a vision_config model is never silently loaded/converted
- # as text-only. text_only resolves the tower away first, so honour that here.
+ # A repo-code VLM may register only AutoModel / AutoModelForCausalLM (DeepSeek-OCR,
+ # Nemotron-VL), so auto_model is not a VLM class though the config is a vision model. Keep
+ # is_vlm for processor selection, but treat it as a VLM on the vLLM path so a vision_config
+ # model is never silently loaded as text-only.
is_vlm_config = is_vlm or (not text_only and hasattr(auto_config, "vision_config"))
is_whisper = whisper_language is not None and whisper_task is not None
auto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer
@@ -1213,10 +1197,8 @@ class FastBaseModel:
)
if any(arch in VLLM_NON_LORA_VLM for arch in model_types):
- # mllama is still only in vllm v0 https://arc.net/l/quote/llwkfgmu
- # https://docs.vllm.ai/en/stable/models/supported_models.html#text-generation_1
- # vLLM V0 does not support LoRA on multi modal models.
- # TODO: Update this once vLLM V1 supports Llama 3.2 aka mllama
+ # mllama is still only in vllm v0, and vLLM V0 does not support LoRA on multimodal
+ # models. TODO: revisit once vLLM V1 supports Llama 3.2 (mllama).
vllm_enable_lora = False
os.environ["UNSLOTH_USE_NEW_MODEL"] = "1"
@@ -1249,7 +1231,6 @@ class FastBaseModel:
print(statistics)
- # Warn about fast transfers
if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ:
old_hf_transfer = os.environ["HF_HUB_ENABLE_HF_TRANSFER"]
if old_hf_transfer in ("False", "false"):
@@ -1265,14 +1246,14 @@ class FastBaseModel:
if old_hf_transfer != "0":
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
- # For debugging - we use a download counter to see if environments are not breaking or if HF is down
+ # A download counter, to see whether environments are breaking or HF is down.
get_statistics(kwargs.get("local_files_only", False))
- # The base + tokenizer prefetch runs AFTER the load-mode validation below, so an invalid
+ # The base plus tokenizer prefetch runs AFTER the load-mode validation below, so an invalid
# load_in_* combination fails without first downloading a snapshot.
- # Whether float32 was asked for rather than arrived at by upcasting. Only an
- # explicit request may suppress the V100/T4 float16 autocast (see #4082).
+ # Whether float32 was asked for rather than arrived at by upcasting: only an explicit request
+ # may suppress the V100/T4 float16 autocast (#4082).
user_float32 = _requested_float32(dtype)
if dtype is None:
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
@@ -1296,15 +1277,15 @@ class FastBaseModel:
# Check for custom data-types
custom_datatype = None
correct_dtype = None
- # `custom_datatype` has to stay None when unset: the consumer below tests
- # `is not None` and would walk every module of every model to `exec("")`.
+ # custom_datatype has to stay None when unset: the consumer below tests `is not None` and would
+ # walk every module of every model to exec("").
_raw_custom_dtype, _code_is_trusted = trusted_custom_dtype()
if _raw_custom_dtype != "":
assert _raw_custom_dtype.count(";") >= 4
checker, _dtype, _bnb_compute_dtype, _custom_datatype, execute_code = (
_raw_custom_dtype.split(";", 4)
)
- # Allow custom dtypes on all runs
+ # Allow custom dtypes on all runs, or only on float16 datatypes.
allow_all_runs = checker == "all"
# Allow only on float16 datatypes
allow_float16_runs = (checker == "float16" or checker == "torch.float16") and (
@@ -1319,11 +1300,10 @@ class FastBaseModel:
if _resolved_bnb_compute_dtype is not None:
bnb_compute_dtype = _resolved_bnb_compute_dtype
correct_dtype = bnb_compute_dtype
- # The last two fields are code, run only when this process set the
- # variable; an inherited environment picks dtypes but injects nothing.
+ # The last two fields are code, run only when this process set the variable; an inherited
+ # environment picks dtypes but injects nothing.
if _code_is_trusted:
custom_datatype = _custom_datatype
- # Execute code as well
if len(execute_code.strip()) != 0:
exec(execute_code)
else:
@@ -1345,15 +1325,14 @@ class FastBaseModel:
revision = _revision,
)
model_class = resolve_model_class(auto_model, auto_config)
- # Check if using forced float32 - we load it in bfloat16, then cast to float16!
- # Resolved here rather than at the load, because attention resolution below and the
- # device-map planner both have to size the same dtype the load will really use.
+ # Forced float32 loads in bfloat16 then casts to float16. Resolved here, not at the load,
+ # because attention resolution and the device-map planner both size the same dtype.
torch_dtype = dtype
if do_forced_float32:
torch_dtype = torch.bfloat16
- # What attention actually runs in, not the checkpoint load dtype: the
- # UNSLOTH_FORCE_CUSTOM_DTYPE families (csm, falcon_h1, nemotron_h) load float32 to keep
- # Mamba precision, then cast projections back to correct_dtype (float16), so flash stays.
+ # What attention actually runs in, not the load dtype: the UNSLOTH_FORCE_CUSTOM_DTYPE families
+ # (csm, falcon_h1, nemotron_h) load float32 for Mamba precision then cast projections back to
+ # correct_dtype, so flash stays.
attn_dtype = correct_dtype if correct_dtype is not None else torch_dtype
attn_impl = resolve_attention_implementation(
model_class,
@@ -1363,15 +1342,15 @@ class FastBaseModel:
dtype = attn_dtype,
)
- # Handle FP8 models: get_model_name has already redirected this to BF16 sibling if the model ships with
- # FP8 weights. We just need to update it here for sanity.
+ # FP8 models: get_model_name already redirected to the BF16 sibling if the model ships FP8
+ # weights, so just update it here for sanity.
auto_config.model_name = model_name
kwargs["attn_implementation"] = attn_impl
bnb_config = None
user_quantization_config = kwargs.get("quantization_config", None)
- # Check if model already has a non-bitsandbytes quantization config (e.g. compressed-tensors/NVFP4)
+ # Check whether the model already has a non-bitsandbytes quantization config (compressed-tensors/NVFP4).
from .loader_utils import (
check_and_disable_bitsandbytes_loading,
sync_unsloth_model_name_bnb_flags,
@@ -1380,8 +1359,8 @@ class FastBaseModel:
load_in_4bit, load_in_8bit, _ = check_and_disable_bitsandbytes_loading(
auto_config, load_in_4bit = load_in_4bit, load_in_8bit = load_in_8bit
)
- # Correct UNSLOTH_MODEL_NAME's bnb tokens now that the effective bnb state is known
- # (the per-load env was built before remap/disable). gpt-oss only; no-op otherwise.
+ # Correct UNSLOTH_MODEL_NAME's bnb tokens now the effective bnb state is known (the per-load
+ # env was built before remap/disable). gpt-oss only.
sync_unsloth_model_name_bnb_flags(load_in_4bit, load_in_8bit)
if full_finetuning and (load_in_4bit or load_in_8bit):
@@ -1392,26 +1371,25 @@ class FastBaseModel:
load_in_8bit = False
load_in_16bit = False
- # text_only loads the decoder alone, but the planner gets only `model_name` and
- # rebuilds the whole VLM: it budgets a vision tower this load never creates and
- # names `model.language_model.layers.0` where the standalone decoder has
- # `model.layers.0`, so the first decoder weight ends up with no device set.
+ # text_only loads the decoder alone, but the planner gets only model_name and rebuilds the whole
+ # VLM: it budgets a vision tower this load never creates and names
+ # model.language_model.layers.0 where the standalone decoder has model.layers.0.
_planner_skip_reason = (
"text_only loads a decoder the repo config does not describe"
if text_only_decoder
else None
)
- # Same failure from the other direction: `num_labels` (or an explicit `auto_model`)
- # loads a task head whose `score` replaces the planned `lm_head`, and dispatch
- # refuses a map with no `score.weight`.
+ # Same failure from the other direction: num_labels (or an explicit auto_model) loads a task
+ # head whose `score` replaces the planned lm_head, and dispatch refuses a map with no
+ # score.weight.
if _planner_skip_reason is None:
_planner_skip_reason = planner_class_mismatch_reason(
model_class,
planner_model_class(auto_config, trust_remote_code = trust_remote_code),
)
- # The comparison above returns None for a concrete `PreTrainedModel` subclass:
- # `resolve_model_class` reads `auto_model._model_mapping`, which only Auto classes
- # have. Unknown is not compatible, so decline rather than plan the repo's class.
+ # The comparison above returns None for a concrete PreTrainedModel subclass, since
+ # resolve_model_class reads auto_model._model_mapping, which only Auto classes have. Unknown is
+ # not compatible, so decline.
if (
_planner_skip_reason is None
and model_class is None
@@ -1421,17 +1399,16 @@ class FastBaseModel:
"an explicit model class has no auto mapping, so the plan cannot be "
"checked against the class the load builds"
)
- # The weights load against the caller's config; the planner rebuilds the repo's from
- # a name. Same class, another `num_hidden_layers` or `vocab_size`, and the map omits
- # blocks or under-budgets weights. Only the caller knows, so do not guess.
+ # The weights load against the caller's config while the planner rebuilds the repo's from a
+ # name, so the same class with another num_hidden_layers or vocab_size gives a map that omits
+ # blocks or under-budgets weights.
if _planner_skip_reason is None and (user_config is not None or auto_config_from_caller):
_planner_skip_reason = (
"a caller-supplied config may not describe the repo the planner rebuilds"
)
- # A no-op unless the caller asked for "unsloth" (or set UNSLOTH_AUTO_DEVICE_MAP);
- # an already-planned map comes back unchanged, so a direct FastBaseModel call
- # behaves the same as going through FastModel.
+ # A no-op unless the caller asked for "unsloth" (or set UNSLOTH_AUTO_DEVICE_MAP); an
+ # already-planned map comes back unchanged.
device_map = resolve_unsloth_device_map(
requested_device_map(device_map),
model_name,
@@ -1443,14 +1420,14 @@ class FastBaseModel:
token = token,
trust_remote_code = trust_remote_code,
**planner_hub_kwargs(kwargs),
- # The pin the config and weights below use; the default branch would size a
- # different checkpoint than the one being loaded.
+ # The pin the config and weights below use; the default branch would size a different
+ # checkpoint than the one being loaded.
revision = _revision,
- # The dtype the load below is given: `from_pretrained` overrides config.json,
- # so planning the checkpoint's own mis-sizes by 2x whenever it changed.
+ # The dtype the load below is given: from_pretrained overrides config.json, so planning the
+ # checkpoint's own mis-sizes by 2x whenever it changed.
**add_dtype_kwargs(torch_dtype),
- # A caller-supplied config overrides the flags: loader.py clears them when it
- # forwards one, so the flags alone would size a 4bit load at full precision.
+ # A caller-supplied config overrides the flags: loader.py clears them when it forwards one, so
+ # the flags alone would size a 4bit load at full precision.
**planner_quantization_kwargs(
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
@@ -1467,9 +1444,9 @@ class FastBaseModel:
"Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!"
)
- # Prefetch the repo (killable child) so the in-process load below is a cache hit. vLLM owns the
- # weight download only when actually available; if fast_inference was requested but vLLM is
- # missing, the load falls through in-process, so weights must still be warmed here.
+ # Prefetch the repo (killable child) so the in-process load is a cache hit. vLLM owns the
+ # download only when actually available; if it was requested but is missing, the load falls
+ # through in-process, so warm the weights here.
_vllm_owns_weights = fast_inference and is_vLLM_available()
_prefetched = maybe_prefetch_hf_snapshot(
model_name,
@@ -1483,19 +1460,19 @@ class FastBaseModel:
use_safetensors = kwargs.get("use_safetensors"),
from_tf = kwargs.get("from_tf", False),
from_flax = kwargs.get("from_flax", False),
- # Bare load reads only ROOT weights; skip subdir weights. Ignored when a subfolder is set.
+ # A bare load reads only ROOT weights, so skip subdir weights; ignored when a subfolder is set.
weights_at_root = True,
variant = kwargs.get("variant"), # forward so the warm keeps the variant .bin
gguf_file = kwargs.get(
"gguf_file"
), # forward so the warm fetches the GGUF (else ignored)
)
- # Child did the forced download; clear the flag so the load reuses the warm cache.
+ # The child did the forced download; clear the flag so the load reuses the warm cache.
if _prefetched and kwargs.get("force_download", False):
kwargs["force_download"] = False
- # Warm a SEPARATE tokenizer repo only (model_name is covered above). Not model_name here: this
- # runs before fast_inference_setup may remap the repo, so it would warm the wrong one.
+ # Warm a SEPARATE tokenizer repo only. Not model_name: this runs before fast_inference_setup
+ # may remap the repo, so it would warm the wrong one.
_tokenizer_repo = (
tokenizer_name if (isinstance(tokenizer_name, str) and tokenizer_name) else model_name
)
@@ -1515,9 +1492,8 @@ class FastBaseModel:
)
_skip_modules = SKIP_QUANTIZATION_MODULES.copy()
- # Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers.
- # Mamba fused kernels pass out_proj.weight directly to F.linear,
- # which fails with quantized Params4bit. Skip out_proj from quantization.
+ # Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers, whose fused kernels pass
+ # out_proj.weight straight to F.linear and fail with quantized Params4bit, so skip out_proj.
if any(mt == "nemotron_h" for mt in (model_types or [])):
_skip_modules.append("out_proj")
@@ -1563,13 +1539,13 @@ class FastBaseModel:
else:
os.environ["UNSLOTH_ENABLE_FULL_FINETUNING"] = "0"
- # Fix AttributeError: 'BitsAndBytesConfig' object has no attribute 'get_loading_attributes'
+ # Fix AttributeError: 'BitsAndBytesConfig' object has no attribute 'get_loading_attributes'.
if bnb_config is not None and not hasattr(bnb_config, "get_loading_attributes"):
bnb_config.get_loading_attributes = lambda *args, **kwargs: {}
- # Cannot be None, since HF now checks for the config
+ # Cannot be None, since HF now checks for the config.
if load_in_4bit or load_in_8bit:
- # Ignore load_in_4bit / load_in_8bit for MXFP4 - best to get config file
+ # Ignore load_in_4bit / load_in_8bit for MXFP4; better to get the config file.
if "gpt-oss-20b" in model_name.lower() or "gpt-oss-120b" in model_name.lower():
pass
else:
@@ -1591,7 +1567,7 @@ class FastBaseModel:
quantization_config = auto_config.quantization_config
quant_method = quantization_config["quant_method"]
- # Sometimes bitsandbytes_4bit + bitsandbytes_8bit is provided
+ # Sometimes bitsandbytes_4bit + bitsandbytes_8bit is provided.
if (
quant_method == "bitsandbytes"
and "bitsandbytes" not in AUTO_QUANTIZATION_CONFIG_MAPPING
@@ -1605,14 +1581,13 @@ class FastBaseModel:
quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]
quantizer_kwargs = {}
if quant_method == "compressed-tensors":
- # Ignore these
pass
else:
- # We cannot dequantize since gpt-oss-20b MXFP4 will now be gpt-oss-20b-BF16
+ # Cannot dequantize, since gpt-oss-20b MXFP4 would become gpt-oss-20b-BF16.
if load_in_16bit and "dequantize" in inspect.signature(quantizer).parameters:
quantizer_kwargs["dequantize"] = True
try:
- # Sometimes this fails so we wrap it in a try except
+ # Sometimes this fails, so wrap it in try/except.
quantization_config = quantizer.from_dict(
quantization_config, **quantizer_kwargs
)
@@ -1650,10 +1625,10 @@ class FastBaseModel:
)
offload_embedding = False
if not fast_inference:
- # Prevent load_in_fp8 from being forwarded into HF internal model loading
+ # Prevent load_in_fp8 from being forwarded into HF internal model loading.
load_in_fp8 = kwargs.pop("load_in_fp8", None)
- # Transformers 5.x @strict config classes reject unexpected kwargs.
- # Move config-level attributes onto the config object directly.
+ # Transformers 5.x @strict config classes reject unexpected kwargs, so move config-level
+ # attributes onto the config object directly.
_num_labels = kwargs.pop("num_labels", None)
if _num_labels is not None:
set_task_config_attr(model_config, "num_labels", _num_labels)
@@ -1668,15 +1643,11 @@ class FastBaseModel:
model_name,
config = model_config,
device_map = device_map,
- # torch_dtype = torch_dtype, # Transformers removed torch_dtype
- # quantization_config = bnb_config,
token = token,
trust_remote_code = trust_remote_code,
- # attn_implementation = attn_implementation,
**kwargs,
)
- # Must precede _attach_bnb_multidevice_hooks: it returns early
- # while offload_embedding is True.
+ # Must precede _attach_bnb_multidevice_hooks: it returns early while offload_embedding is True.
offload_embedding = _resolve_offload_embedding(
model,
offload_embedding,
@@ -1720,7 +1691,7 @@ class FastBaseModel:
# Device-safe embedding offload.
_install_offload_embedding_hooks(embed_tokens, out_embed, _embed_device)
- # Must free GPU memory otherwise will not free!
+ # GPU memory must be freed explicitly or it will not be freed.
clean_gpu_cache()
gc.collect()
else:
@@ -1778,10 +1749,8 @@ class FastBaseModel:
if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs:
load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]
- # Load vLLM first
llm = load_vllm(**load_vllm_kwargs)
- # Convert to HF format
_, quant_state_dict = get_vllm_state_dict(
llm,
config = model_config,
@@ -1802,7 +1771,6 @@ class FastBaseModel:
finally:
raise_handler.remove()
- # Return old flag
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer
# Check float32 norm weights
@@ -1814,26 +1782,24 @@ class FastBaseModel:
or "layer_norm" in name
) and hasattr(module, "weight"):
module._pre_set_compute_dtype = torch.float32
- # Edit data-types
if custom_datatype is not None:
with torch.no_grad():
for jj, (name, module) in enumerate(model.named_modules()):
exec(custom_datatype)
- # Clear deleted GPU items
for _ in range(3):
gc.collect()
clean_gpu_cache()
- # Counteract saved tokenizers
+ # Counteract saved tokenizers.
tokenizer_name = model_name if tokenizer_name is None else tokenizer_name
- # The loader resolves this (caller's adapter repo vs resolved base model); the
- # fallback below covers a direct call.
+ # The loader resolves this (caller's adapter repo vs resolved base model); the fallback below
+ # covers a direct call.
_tokenizer_revision = _tokenizer_revision_arg
if _tokenizer_revision is None and tokenizer_name == _revision_repo:
_tokenizer_revision = _revision
- # On the vLLM path the tokenizer warm was deferred (fast_inference_setup may remap model_name).
- # Warm the now-final tokenizer repo so the load below hits the cache (a cached/local repo is a no-op).
+ # On the vLLM path the tokenizer warm was deferred, since fast_inference_setup may remap
+ # model_name, so warm the now-final tokenizer repo (cached or local is a no-op).
if _vllm_owns_weights and isinstance(tokenizer_name, str) and tokenizer_name:
maybe_prefetch_hf_snapshot(
tokenizer_name,
@@ -1845,7 +1811,7 @@ class FastBaseModel:
tokenizer_only = True,
)
- # Fix _Unsloth_Patched_ prefix in local config files from old saves (issue #4085)
+ # Fix the _Unsloth_Patched_ prefix in local config files from old saves (#4085).
if os.path.isdir(tokenizer_name):
import json as _json
for _cfg_name in (
@@ -1867,8 +1833,8 @@ class FastBaseModel:
except Exception:
pass
- # Functional load chain (AutoProcessor -> get_auto_processor -> manual VLM
- # fallback); offline is already forced upstream. Surfaces the error for the retry.
+ # Functional load chain (AutoProcessor -> get_auto_processor -> manual VLM fallback); offline is
+ # already forced upstream, and the error is surfaced for the retry.
def _acquire_processor(lfo):
_err = None # underlying load failure (used by the entry-point retry)
if (whisper_language and whisper_task) or auto_model.__name__.endswith(
@@ -1916,8 +1882,8 @@ class FastBaseModel:
# Swallow so the manual fallback / entry-point retry can run.
_tok = None
- # Build the processor manually if it failed to load or silently degraded to
- # a text-only tokenizer (no image_processor) for a VLM (issue #4085).
+ # Build the processor manually if it failed to load or silently degraded to a text-only
+ # tokenizer (no image_processor) for a VLM (#4085).
_processor_is_degraded = (
is_vlm and _tok is not None and not hasattr(_tok, "image_processor")
)
@@ -1937,26 +1903,25 @@ class FastBaseModel:
if _fallback is not None:
_tok = _fallback
elif _err is None or (_fb_err is not None and _is_offline_related_error(_fb_err)):
- # Prefer a network fallback error over a permanent primary one so the
- # offline retry still fires.
+ # Prefer a network fallback error over a permanent primary one so the offline retry still fires.
_err = _fb_err
return _tok, _err
def _is_degraded_vlm(_t):
- # VLM that loaded only a text-only tokenizer (no image_processor).
+ # A VLM that loaded only a text-only tokenizer (no image_processor).
return is_vlm and _t is not None and not hasattr(_t, "image_processor")
tokenizer, _primary_err = _acquire_processor(local_files_only)
- # Online network failure/degrade: raise so @_offline_aware_load retries from cache.
- # Permanent / missing-file errors propagate; when already offline keep what we got.
+ # Online network failure or degrade: raise so @_offline_aware_load retries from cache.
+ # Permanent / missing-file errors propagate, and when already offline keep what we got.
if (
(tokenizer is None or _is_degraded_vlm(tokenizer))
and not local_files_only
and _is_offline_related_error(_primary_err)
):
raise _primary_err
- # Missing torchvision silently degrades a VLM processor to text-only; surface the
- # real cause instead of a later collator error (#4202), incl. on a silent degrade.
+ # Missing torchvision silently degrades a VLM processor to text-only; surface the real cause
+ # instead of a later collator error (#4202).
if is_vlm and (tokenizer is None or not hasattr(tokenizer, "image_processor")):
if _missing_torchvision_error(_primary_err):
raise ImportError(
@@ -1970,8 +1935,8 @@ class FastBaseModel:
f"Unsloth: Warning - VLM processor fallback returned None for model_type={model_type_arch}",
file = sys.stderr,
)
- # Backwards compat: if processor has no chat_template (e.g. old saves without
- # chat_template.jinja) but the inner tokenizer does, copy it to the processor.
+ # Backwards compat: if the processor has no chat_template (old saves without chat_template.jinja)
+ # but the inner tokenizer does, copy it to the processor.
if (
hasattr(tokenizer, "tokenizer")
and getattr(tokenizer, "chat_template", None) is None
@@ -1981,9 +1946,7 @@ class FastBaseModel:
if hasattr(tokenizer, "tokenizer"):
__tokenizer = tokenizer.tokenizer
- # Add padding side as well
__tokenizer.padding_side = "left"
- # Check bos, eos, pad tokens
if hasattr(__tokenizer, "bos_token"):
tokenizer.bos_token = __tokenizer.bos_token
tokenizer.bos_token_id = __tokenizer.bos_token_id
@@ -2006,7 +1969,7 @@ class FastBaseModel:
try:
model, tokenizer = patch_tokenizer(model, tokenizer)
except Exception as _patch_err:
- # Some VLM processors (e.g. ERNIE VL) fail patching; fall back to AutoTokenizer.
+ # Some VLM processors (ERNIE VL) fail patching; fall back to AutoTokenizer.
try:
from transformers import AutoTokenizer as _AutoTokenizer
@@ -2020,24 +1983,24 @@ class FastBaseModel:
revision = _tokenizer_revision,
)
model, _fallback_tok = patch_tokenizer(model, _fallback_tok)
- # Re-attach as processor wrapper if original was a processor
+ # Re-attach as a processor wrapper if the original was a processor.
if hasattr(tokenizer, "image_processor"):
tokenizer.tokenizer = _fallback_tok
else:
tokenizer = _fallback_tok
except Exception as _fb_err:
- # Online network failure: propagate for the offline retry; else raise the patch error.
+ # Online network failure: propagate for the offline retry, else raise the patch error.
if not local_files_only and _is_offline_related_error(_fb_err):
raise
raise _patch_err
model = post_patch_loss_function(model)
- # Log Unsloth version for future fastpaths for inference
+ # Log the Unsloth version for future inference fastpaths.
if hasattr(model, "config"):
model.config.update({"unsloth_version": __version__})
patch_saving_functions(model, vision = True)
if tokenizer is None:
- # Last resort: AutoTokenizer, then PreTrainedTokenizerFast (raise on network failure to retry).
+ # Last resort: AutoTokenizer, then PreTrainedTokenizerFast, raising on network failure to retry.
def _last_resort_tokenizer(lfo):
from transformers import AutoTokenizer as _AutoTokenizer
load_path = _hub_repo_or_local_path(
@@ -2087,19 +2050,18 @@ class FastBaseModel:
) from _last_resort_err
patch_saving_functions(tokenizer, vision = True)
- # Fix gradient accumulation. See issue #4982.
+ # Fix gradient accumulation; see #4982.
from transformers.trainer import Trainer
apply_accepts_loss_kwargs_fix(model)
patch_gradient_accumulation_fix(Trainer)
- # Save tokenizer for inference purposes
- tokenizer.padding_side = "left" # Force inference
+ tokenizer.padding_side = "left"
if hasattr(tokenizer, "tokenizer"):
- tokenizer.tokenizer.padding_side = "left" # Force inference
- # Audio feature extractors must stay right padded: left (a text setting,
- # forwarded by from_pretrained) shifts Whisper mels and desyncs Gemma 4
- # audio token counts (crash on transformers < 5.10).
+ tokenizer.tokenizer.padding_side = "left"
+ # Audio feature extractors must stay right padded: left (a text setting, forwarded by
+ # from_pretrained) shifts Whisper mels and desyncs Gemma 4 audio token counts, crashing on
+ # transformers < 5.10.
feature_extractor = getattr(tokenizer, "feature_extractor", None)
if (
feature_extractor is not None
@@ -2112,14 +2074,12 @@ class FastBaseModel:
m._saved_temp_tokenizer = tokenizer
m = m.model
m.max_seq_length = max_seq_length
- # Save to modules as well
for module in model.modules():
module.max_seq_length = max_seq_length
m._saved_temp_tokenizer = tokenizer
# Prevent Transformers Trainer from auto-wrapping Unsloth LoRA models in DP.
_mark_unsloth_disable_data_parallel(model, disable = not full_finetuning)
- # Patch generate
if os.environ.get("UNSLOTH_DISABLE_FAST_GENERATION", "0") == "0" and hasattr(
model, "generate"
):
@@ -2128,7 +2088,6 @@ class FastBaseModel:
unsloth_base_fast_generate.__doc__ = model._old_generate.__doc__
model.generate = types.MethodType(unsloth_base_fast_generate, model)
model._unsloth_trust_remote_code = trust_remote_code
- # Post patches
model = FastBaseModel.post_patch_model(
model,
use_gradient_checkpointing = use_gradient_checkpointing,
@@ -2137,22 +2096,18 @@ class FastBaseModel:
tokenizer = tokenizer,
float32_mixed_precision = float32_mixed_precision,
)
- # Clear deleted GPU items
for _ in range(3):
gc.collect()
clean_gpu_cache()
- # Saving restores sentencepiece assets from the repo name alone, which carries no
- # branch. Stamped here, not per processor branch, so a fallback cannot lose it.
+ # Saving restores sentencepiece assets from the repo name alone, which carries no branch.
+ # Stamped here, not per processor branch, so a fallback cannot lose it.
_mark_loaded_revision(tokenizer, _tokenizer_revision)
model = _mark_forced_float32(model, do_forced_float32)
model = _mark_full_finetuning(model, full_finetuning)
- # LAST, for the reason the llama loader repairs last: the attach above
- # runs during the load, and `patch_model_and_tokenizer` further down
- # REPLACES the embedding and lm_head with freshly built modules, which
- # carry the weights over but not the `_hf_hook`. A model this loader
- # split across cards would otherwise still meet the original
- # cross-device `index_select`. Idempotent, so the earlier attach stands.
+ # LAST, like the llama loader: patch_model_and_tokenizer below REPLACES the embedding and
+ # lm_head with fresh modules carrying the weights but not the _hf_hook, so a model split across
+ # cards would still meet the original cross-device index_select. Idempotent.
if not fast_inference and not offload_embedding:
try:
_repaired = _repair_dispatch_hooks(model)
@@ -2200,8 +2155,8 @@ class FastBaseModel:
):
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
print("Unsloth: Full finetuning is enabled, so .get_peft_model has no effect")
- # Full finetuning still compiles, so a stray pre-train forward can poison the
- # cache; install the detector here too (it is idempotent).
+ # Full finetuning still compiles, so a stray pre-train forward can poison the cache; install the
+ # detector here too (idempotent).
_unsloth_install_pretrain_detector(model)
return model
transformers_set_seed(random_state)
@@ -2214,9 +2169,8 @@ class FastBaseModel:
if isinstance(model, PeftModelForCausalLM):
raise RuntimeError("Unsloth: You already added LoRA adapters to your model!")
- # Remember whether the CALLER explicitly opted into audio. "all-linear" turns
- # the flag on implicitly below, but an old unsloth_zoo that cannot do audio
- # must not make a plain all-linear (text/vision) run fail.
+ # Remember whether the CALLER explicitly opted into audio: "all-linear" turns the flag on
+ # implicitly below, but an old unsloth_zoo without audio must not fail a plain all-linear run.
_audio_explicitly_requested = bool(finetune_audio_layers)
if target_modules == "all-linear":
finetune_vision_layers = True
@@ -2224,11 +2178,9 @@ class FastBaseModel:
finetune_attention_modules = True
finetune_mlp_modules = True
finetune_audio_layers = True
- # Older unsloth_zoo (before get_peft_regex gained finetune_audio_layers) does
- # not accept the kwarg. Pass it only when supported; if the caller EXPLICITLY
- # asked for audio but it is unsupported, fail loudly rather than silently
- # training a language-only adapter. (all-linear's implicit opt-in degrades
- # gracefully instead of raising.)
+ # Older unsloth_zoo (before get_peft_regex gained finetune_audio_layers) rejects the kwarg, so
+ # pass it only when supported; an EXPLICIT audio request fails loudly rather than silently
+ # training a language-only adapter, while all-linear's implicit opt-in degrades gracefully.
if "finetune_audio_layers" in inspect.signature(get_peft_regex).parameters:
_audio_kwargs = {"finetune_audio_layers": finetune_audio_layers}
elif _audio_explicitly_requested:
@@ -2238,19 +2190,13 @@ class FastBaseModel:
)
else:
_audio_kwargs = {}
- # Remember the caller's ORIGINAL explicit leaf list for MoE expert
- # detection. When an explicit list is routed through get_peft_regex for
- # family scoping below, the generated regex carries get_peft_regex's full
- # "mlp|feed_forward|ffn|dense" component block even when the caller named
- # only attention leaves (q/k/v/o_proj). Keying expert detection on that
- # regex would train the experts for an attention-only request. The
- # original list carries the true leaf intent, so use it for MoE detection;
- # only the auto (None / "all-linear") path relies on the regex, whose mlp
- # block is the sole remaining MLP-intent signal on fused-expert models.
+ # Remember the caller's ORIGINAL explicit leaf list for MoE expert detection: routing it through
+ # get_peft_regex adds the full "mlp|feed_forward|ffn|dense" block even for attention-only leaves,
+ # so keying on that regex would train the experts. Only the auto path relies on the regex.
_moe_detect_target = target_modules if type(target_modules) in (list, tuple) else None
- # get_peft_regex drops these (no attention/MLP ancestor) and LoRA on them never
- # trains, so redirect before scoping, matching FastLanguageModel.
+ # get_peft_regex drops these (no attention/MLP ancestor) and LoRA on them never trains, so
+ # redirect before scoping, matching FastLanguageModel.
target_modules, modules_to_save, _moved = _redirect_embedding_targets(
target_modules,
modules_to_save,
@@ -2287,11 +2233,9 @@ class FastBaseModel:
)
else:
assert type(target_modules) in (list, tuple, str)
- # Route an explicit list through get_peft_regex when the caller scoped a
- # layer family (one of the finetune_* below is off) OR opted into audio (so
- # the new audio/embedder branches are considered). finetune_audio_layers is
- # a POSITIVE term here: using `not finetune_audio_layers` would -- since it
- # defaults False -- force every explicit list through the filter.
+ # Route an explicit list through get_peft_regex when the caller scoped a layer family or opted
+ # into audio, so the new audio/embedder branches are considered. finetune_audio_layers is a
+ # POSITIVE term: negating it would force every explicit list through the filter.
_scoping = (
not finetune_vision_layers
or not finetune_language_layers
@@ -2321,42 +2265,33 @@ class FastBaseModel:
and hasattr(model.vllm_engine.llm_engine, "vllm_config")
and getattr(model.vllm_engine.llm_engine.vllm_config, "lora_config", None) is None
):
- # If vLLM is being used but lora is not enabled, throw an error
- # Ref https://github.com/vllm-project/vllm/blob/51ba839555a5d122eadd91e9c16463ac288f5fa1/vllm/v1/engine/processor.py#L148-L151
+ # If vLLM is used but lora is not enabled, throw an error; see vllm v1/engine/processor.py#L148-L151.
raise RuntimeError("Unsloth: LoRA is not enabled for this model!")
if finetune_vision_layers:
- # vLLM does not support LoRA on vision layers
- # https://github.com/vllm-project/vllm/blob/main/vllm/lora/models.py#L471-L477
- # TODO: Update this once vLLM V1 supports LoRA on vision layers (possibly not happening)
+ # vLLM does not support LoRA on vision layers; see vllm lora/models.py#L471-L477.
raise RuntimeError(
"Unsloth: Finetuning vision layers is not supported for fast_inference. Only text layers are supported!"
)
if model.config.model_type in VLLM_NON_LORA_VLM:
- # mllama is still only in vllm v0 https://arc.net/l/quote/llwkfgmu
- # https://docs.vllm.ai/en/stable/models/supported_models.html#text-generation_1
- # vLLM V0 does not support LoRA on multi modal models.
- # TODO: Update this once vLLM V1 supports Llama 3.2 aka mllama
+ # mllama is still only in vllm v0, and vLLM V0 does not support LoRA on multimodal
+ # models. TODO: revisit once vLLM V1 supports Llama 3.2 (mllama).
raise RuntimeError(
"Unsloth: LoRA finetuning for Llama 3.2 aka mllama models is not supported with fast_inference!"
)
- # Clear deleted GPU items
for _ in range(3):
gc.collect()
clean_gpu_cache()
max_seq_length = model.max_seq_length
- # If we pass loftq_config = None we will get an error
+ # Passing loftq_config = None gives an error.
loftq_config = validate_loftq_config(
loftq_config, lora_dropout, bias, init_lora_weights, model
)
- # Auto-detect MoE models and populate target_parameters for expert layers.
- # Prefer the caller's ORIGINAL explicit leaf list over the scoped regex so an
- # attention-only request does not train experts via get_peft_regex's mlp block,
- # but only when MLP and language families are both still in scope. If the caller
- # scoped MLP or language OFF (finetune_mlp_modules / finetune_language_layers
- # False), the scoped regex already dropped the experts, so honor it instead of
- # re-introducing the original list's gate/up/down leaves.
+ # Auto-detect MoE models and populate target_parameters for expert layers. Prefer the caller's
+ # ORIGINAL explicit leaf list over the scoped regex so an attention-only request does not train
+ # experts, but only while MLP and language families are both in scope: with finetune_mlp_modules
+ # or finetune_language_layers False the scoped regex already dropped the experts.
if target_parameters is None:
_moe_targets = _select_moe_detection_targets(
_moe_detect_target,
@@ -2366,9 +2301,8 @@ class FastBaseModel:
)
target_parameters = get_moe_target_parameters(model, _moe_targets)
- # Per-expert Linear expert layouts (e.g. gpt-oss bnb-4bit) target experts via
- # target_modules, not fused Parameters. Extend either form PEFT accepts: a leaf
- # list (explicit) or a regex string (auto / all-linear / scoped). No-op otherwise.
+ # Per-expert Linear layouts (gpt-oss bnb-4bit) target experts via target_modules, not fused
+ # Parameters. Extend either form PEFT accepts: a leaf list, or a regex string.
_moe_module_detect = _select_moe_detection_targets(
_moe_detect_target,
target_modules,
@@ -2401,7 +2335,6 @@ class FastBaseModel:
n = max(1, min(int(finetune_last_n_layers), _total_layers))
layers_to_transform = list(range(_total_layers - n, _total_layers))
- # Get only allowed parameters for LoraConfig
local_variables = {
**locals(),
**kwargs,
@@ -2415,14 +2348,15 @@ class FastBaseModel:
model,
use_gradient_checkpointing = use_gradient_checkpointing,
)
- # Gemma4 ClippableLinear wraps nn.Linear -- PEFT can't inject LoRA on it directly.
- # Monkey-patch PEFT to target the inner .linear child instead.
+ # Gemma4 ClippableLinear wraps nn.Linear and PEFT cannot inject LoRA on it directly, so patch
+ # PEFT to target the inner .linear child.
_clippable_linear_cls = None
try:
from transformers.models.gemma4.modeling_gemma4 import (
Gemma4ClippableLinear as _clippable_linear_cls,
)
except ImportError:
+ # Respect whatever it was set before
pass
if _clippable_linear_cls is not None:
from peft.tuners.lora.model import LoraModel as _LoraModel
@@ -2465,8 +2399,8 @@ class FastBaseModel:
model = _get_peft_model(model, lora_config)
- # PEFT may have wrapped an endpoint this load repaired; the hook stays on
- # `base_layer` and the adapter branch reads the caller's tensor.
+ # PEFT may have wrapped an endpoint this load repaired; the hook stays on base_layer and the
+ # adapter branch reads the caller's tensor.
try:
_lifted = _lift_endpoint_hooks_onto_adapters(model)
if _lifted:
@@ -2477,16 +2411,12 @@ class FastBaseModel:
except Exception as _exc:
logger.warning_once(f"Unsloth: could not lift adapter hooks: {_exc}")
- # Restore original PEFT method
if _clippable_linear_cls is not None:
_LoraModel._create_and_replace = _original_car
- # Apply QAT + LoRA if specified
if qat_scheme is not None:
print("Unsloth: Applying QAT to mitigate quantization degradation")
model = _prepare_model_for_qat(model, qat_scheme)
- # Fix LoraConfig.auto_mapping is None
fix_lora_auto_mapping(model)
- # Enable gradients on modules which are trainable
requires_grad_for_gradient_checkpointing(model)
trust_remote_code = getattr(model, "_unsloth_trust_remote_code", False)
model = FastBaseModel.post_patch_model(
@@ -2495,17 +2425,14 @@ class FastBaseModel:
trust_remote_code = trust_remote_code,
)
model.max_seq_length = max_seq_length
- # Save to modules as well
for module in model.modules():
module.max_seq_length = max_seq_length
- # Clear deleted GPU items
for _ in range(3):
gc.collect()
clean_gpu_cache()
patch_saving_functions(model, vision = True)
patch_peft_fast_inference(model)
- # Add for_inference and for_training
model.for_training = functools.partial(FastBaseModel.for_training, model)
model.for_inference = functools.partial(FastBaseModel.for_inference, model)
m = model
@@ -2513,8 +2440,8 @@ class FastBaseModel:
m.for_training = functools.partial(FastBaseModel.for_training, m)
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
- # Detect a stray pre-train forward so train() can drop the torch.compile
- # graph cache it would otherwise poison (see prepare_for_training_mode).
+ # Detect a stray pre-train forward so train() can drop the torch.compile graph cache it would
+ # otherwise poison (see prepare_for_training_mode).
_unsloth_install_pretrain_detector(model)
model = _exclude_rope_inv_freq_from_ddp(model)
return model
@@ -2531,7 +2458,7 @@ class FastBaseModel:
full_finetuning = os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1"
if type(float32_mixed_precision) is bool:
- # Respect whatever it was set before
+ # Respect whatever it was set to before.
pass
else:
float32_mixed_precision = True
@@ -2539,8 +2466,8 @@ class FastBaseModel:
# Use bfloat16 precision for full finetuning
float32_mixed_precision = False
- # VLMs can hit DDP "marked ready twice" with re-entrant checkpointing.
- # See: https://github.com/unslothai/unsloth/issues/3713.
+ # VLMs can hit DDP "marked ready twice" with re-entrant checkpointing (#3713), so under DDP skip
+ # the offloaded/re-entrant checkpoint patch and default native checkpoint to non-reentrant.
use_reentrant = not is_distributed()
if not use_reentrant:
# Under DDP, avoid the offloaded/re-entrant checkpoint patch.
@@ -2567,18 +2494,14 @@ class FastBaseModel:
float32_mixed_precision = float32_mixed_precision,
patch_modules_to_save = True,
)
- # Persist the configured GC mode so the trainer restores it verbatim.
- # for_inference() clears the module flags (GRPO does this every generation
- # step), and a plain TrainingArguments defaults gradient_checkpointing=False,
- # which would otherwise silently disable this setting at train time (#4735).
+ # Persist the configured GC mode so the trainer restores it verbatim: for_inference() clears the
+ # module flags every GRPO generation step, and TrainingArguments defaults
+ # gradient_checkpointing=False, which would silently disable it at train time (#4735).
model._unsloth_gradient_checkpointing = use_gradient_checkpointing
- # Gemma3N audio conformer processes variable-length audio tensors
- # that cause stride mismatches in AOT autograd compiled backward
- # when non-reentrant checkpointing is used. The notebook or TRL
- # may override gradient_checkpointing_kwargs with use_reentrant=False
- # after this point, so we intercept gradient_checkpointing_enable
- # to always force use_reentrant=True for Gemma3N.
+ # The Gemma3N audio conformer's variable-length tensors cause stride mismatches in the AOT
+ # autograd compiled backward under non-reentrant checkpointing, and TRL may override
+ # gradient_checkpointing_kwargs later, so intercept the enable and force use_reentrant=True.
_model_type = getattr(getattr(model, "config", None), "model_type", "") or ""
if "gemma3n" in _model_type.lower() or "gemma4" in _model_type.lower():
_original_gc_enable = model.gradient_checkpointing_enable
@@ -2600,7 +2523,6 @@ class FastBaseModel:
raise RuntimeError("Unsloth: Unsuccessfully patched inner_training_loop")
patch_saving_functions(model, vision = True)
- # Patch tokenizer to pad to the left
m = model
while hasattr(m, "model"):
if hasattr(m, "_saved_temp_tokenizer"):
@@ -2613,11 +2535,9 @@ class FastBaseModel:
# Prevent Transformers Trainer from auto-wrapping Unsloth LoRA models in DP.
_mark_unsloth_disable_data_parallel(model, disable = not full_finetuning)
- # Clear deleted GPU items
for _ in range(3):
gc.collect()
clean_gpu_cache()
- # Add for_inference and for_training
model.for_training = functools.partial(FastBaseModel.for_training, model)
model.for_inference = functools.partial(FastBaseModel.for_inference, model)
m = model
@@ -2625,10 +2545,9 @@ class FastBaseModel:
m.for_training = functools.partial(FastBaseModel.for_training, m)
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
- # Set weight[padding_idx] = 0 for embeddings that are NOT tied with the
- # lm_head. When weights are tied, zeroing the padding row also zeros
- # the corresponding lm_head row, forcing logit = 0 for the pad token.
- # Only do this if tokenizer is defined since eos_token == pad_token sometimes!
+ # Set weight[padding_idx] = 0 only for embeddings NOT tied with the lm_head: when tied, zeroing
+ # the padding row zeros the lm_head row and forces logit = 0 for the pad token. Only when the
+ # tokenizer is defined, since eos_token == pad_token sometimes.
pad_token_id = getattr(tokenizer, "pad_token_id", None)
lm_head = getattr(model, "lm_head", None)
lm_head_weight = getattr(lm_head, "weight", None) if lm_head is not None else None
@@ -2644,7 +2563,6 @@ class FastBaseModel:
module.padding_idx == pad_token_id
and module.padding_idx < module.weight.shape[0]
):
- # Skip if tied to lm_head
if (
lm_head_weight is not None
and module.weight.data_ptr() == lm_head_weight.data_ptr()
@@ -2665,10 +2583,8 @@ class FastBaseModel:
m.gradient_checkpointing = False
if hasattr(m, "training"):
m.training = False
- # Pad tokenizer to the left
if hasattr(m, "_saved_temp_tokenizer"):
m._saved_temp_tokenizer.padding_side = "left"
- # Set a flag for generation!
m._flag_for_generation = True
m = model
@@ -2678,12 +2594,12 @@ class FastBaseModel:
_for_inference(m)
model.eval() # to turn off training on modules deeper in
+ # Since transformers 4.53, this must be turned off explicitly.
# Since transformers 4.53, must turn off explicitly
for module in model.modules():
if hasattr(module, "gradient_checkpointing"):
module.gradient_checkpointing = False
- # Also disable training for embeddings for NEFTune
if hasattr(model, "get_input_embeddings"):
embeddings = model.get_input_embeddings()
if hasattr(embeddings, "training"):
@@ -2692,19 +2608,20 @@ class FastBaseModel:
embeddings = model.get_output_embeddings()
if hasattr(embeddings, "training"):
embeddings.training = False
- # Restore use_cache values that prepare_model_for_training disabled
- # for gradient checkpointing (older unsloth_zoo has no restore helper)
+ # Restore use_cache values that prepare_model_for_training disabled for gradient checkpointing
+ # (older unsloth_zoo has no restore helper).
+ # Restore use_cache values that prepare_model_for_training disabled for gradient checkpointing (older
+ # unsloth_zoo has no restore helper)
try:
from unsloth_zoo.training_utils import restore_use_cache
restore_use_cache(model)
except ImportError:
pass
- # Must disable returning hidden states in the case for GRPO
+ # Returning hidden states must be disabled for GRPO, and returning logits enabled.
os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0"
- # Must enable returning logits
os.environ["UNSLOTH_RETURN_LOGITS"] = "1"
- # Turn off skip guards and set stance to default
+ # Turn off skip guards and set stance to default.
if torch_compiler_set_stance is not None:
torch_compiler_set_stance(stance = "default", skip_guard_eval_unsafe = False)
return model
@@ -2716,7 +2633,6 @@ class FastBaseModel:
"Unsloth: I think you're passing a tokenizer, not the model to for_training!"
)
- # Delete all fast inference loras
for param in model.parameters():
if hasattr(param, "_fast_lora"):
del param._fast_lora
@@ -2726,13 +2642,11 @@ class FastBaseModel:
m.gradient_checkpointing = use_gradient_checkpointing
if hasattr(m, "training"):
m.training = True
- # Pad tokenizer to the left
if hasattr(m, "_saved_temp_tokenizer"):
m._saved_temp_tokenizer.padding_side = "right"
- # Set a flag for generation!
if hasattr(m, "_flag_for_generation"):
try:
- # A PEFT wrapper delegates the read but owns nothing to delete
+ # A PEFT wrapper delegates the read but owns nothing to delete.
del m._flag_for_generation
except AttributeError:
pass
@@ -2744,12 +2658,12 @@ class FastBaseModel:
_for_training(m)
model.train() # to turn on training on modules deeper in
+ # Since transformers 4.53, this must be turned on explicitly.
# Since transformers 4.53, must turn on explicitly
for module in model.modules():
if hasattr(module, "gradient_checkpointing"):
module.gradient_checkpointing = use_gradient_checkpointing
- # Also re-enable training for embeddings for NEFTune
if hasattr(model, "get_input_embeddings"):
embeddings = model.get_input_embeddings()
if hasattr(embeddings, "training"):
@@ -2758,21 +2672,24 @@ class FastBaseModel:
embeddings = model.get_output_embeddings()
if hasattr(embeddings, "training"):
embeddings.training = True
- # Re-disable use_cache if prepare_model_for_training had disabled it
- # and for_inference restored it (record only exists after a disable)
+ # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored
+ # it; the record only exists after a disable.
+ # Re-disable use_cache if prepare_model_for_training had disabled it and for_inference restored it
+ # (record only exists after a disable)
if (
use_gradient_checkpointing
and getattr(model, "_unsloth_use_cache_originals", None) is not None
):
+ # Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts); see llama.py.
try:
from unsloth_zoo.training_utils import disable_use_cache
disable_use_cache(model)
except ImportError:
pass
- # Can re-enable not returning logits
+ # Not returning logits can be re-enabled.
os.environ["UNSLOTH_RETURN_LOGITS"] = "0"
- # Turn off skip guards and set stance to default
+ # Turn off skip guards and set stance to default.
if torch_compiler_set_stance is not None:
torch_compiler_set_stance(stance = "default", skip_guard_eval_unsafe = False)
return model
@@ -2799,7 +2716,7 @@ def _iter_message_lists(example, column):
def _local_path_from_video_value(video_path):
- # data: URIs are inline payloads, not files, and contain no "://"
+ # data: URIs are inline payloads, not files, and contain no "://".
if video_path.startswith("data:"):
return None
if "://" not in video_path:
@@ -2810,7 +2727,7 @@ def _local_path_from_video_value(video_path):
from urllib.request import url2pathname
parsed = urlparse(video_path)
- # RFC 8089: only an empty authority or "localhost" is the local machine
+ # RFC 8089: only an empty authority or "localhost" is the local machine.
if parsed.netloc and parsed.netloc != "localhost":
return None
path = url2pathname(parsed.path)
@@ -2855,8 +2772,8 @@ def check_dataset_for_missing_videos(
pass
missing = []
- # Report each missing path once; only confirmed-existing paths enter
- # `checked`, so retries after an error re-check previously missing files.
+ # Report each missing path once; only confirmed-existing paths enter `checked`, so retries after
+ # an error re-check previously missing files.
seen_missing = set()
if checked is None:
checked = set()
@@ -2900,7 +2817,7 @@ def check_dataset_for_missing_videos(
return missing
-# Auto-enable grouped-GEMM MoE (transformers<5 ModuleList experts); see llama.py.
+# Auto-enable grouped-GEMM MoE (transformers < 5 ModuleList experts); see llama.py.
try:
from unsloth_zoo.temporary_patches.moe_grouped_modulelist import wrap_loader_for_grouped_moe
FastBaseModel.from_pretrained = staticmethod(
diff --git a/unsloth/ollama_template_mappers.py b/unsloth/ollama_template_mappers.py
index 065165d5d79..f457caeaff9 100644
--- a/unsloth/ollama_template_mappers.py
+++ b/unsloth/ollama_template_mappers.py
@@ -20,6 +20,7 @@ __all__ = [
OLLAMA_TEMPLATES = {}
+
# =========================================== Unsloth
unsloth_ollama = '''
@@ -36,6 +37,7 @@ SYSTEM """You are a helpful assistant to the user"""
OLLAMA_TEMPLATES["unsloth"] = unsloth_ollama
+
# =========================================== Zephyr
zephyr_ollama = '''
@@ -72,10 +74,8 @@ PARAMETER min_p 0.1
OLLAMA_TEMPLATES["chatml"] = chatml_ollama
+# Ollama templates from ollama.com/library/mistral (v0.1 blob 22e1b2e8dc2f, v0.2 blob e6836092461f).
# =========================================== Mistral-1
-# Ollama from https://www.ollama.com/library/mistral
-# Mistral v0.1 https://ollama.com/library/mistral:v0.1/blobs/22e1b2e8dc2f
-# Mistral v0.2 https://ollama.com/library/mistral:v0.2/blobs/e6836092461f
mistral_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """[INST] {{ if .System }}{{ .System }} {{ end }}{{ .Prompt }} [/INST]"""
@@ -83,8 +83,7 @@ PARAMETER stop "[INST]"
PARAMETER stop "[/INST]"
'''
-# mistral:v0.3 https://ollama.com/library/mistral:v0.3/blobs/1ff5b64b61b9
-# mistral-large https://ollama.com/library/mistral-large:latest/blobs/96adabcf2c08
+# mistral:v0.3 blob 1ff5b64b61b9; mistral-large blob 96adabcf2c08.
mistral_v03_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{- if .Messages }}
@@ -439,8 +438,8 @@ OLLAMA_TEMPLATES["magistral"] = magistral_ollama
OLLAMA_TEMPLATES["codestral"] = codestral_ollama
-# =========================================== Llama-2
# Ollama from https://www.ollama.com/library/llama3
+# =========================================== Llama-2
llama_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """[INST] <<SYS>>{{ .System }}<</SYS>>
@@ -453,8 +452,8 @@ PARAMETER min_p 0.1
OLLAMA_TEMPLATES["llama"] = llama_ollama
-# =========================================== Vicuna
# Ollama from https://www.ollama.com/library/vicuna
+# =========================================== Vicuna
vicuna_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{ if .System }}{{ .System }} {{ end }}{{ if .Prompt }}USER: {{ .Prompt }} {{ end }}ASSISTANT: {{ .Response }} {__EOS_TOKEN__}"""
@@ -501,8 +500,8 @@ SYSTEM """Below are some instructions that describe some tasks. Write responses
OLLAMA_TEMPLATES["alpaca"] = alpaca_ollama
-# =========================================== Gemma
# Ollama from https://www.ollama.com/library/gemma
+# =========================================== Gemma
gemma_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """<start_of_turn>user
@@ -540,9 +539,8 @@ PARAMETER min_p 0.1
OLLAMA_TEMPLATES["gemma_chatml"] = gemma_chatml_ollama
+# Same as Gemma 1, but with sliding window attention; ollama.com/library/gemma2 blob 6522ca797f47.
# =========================================== Gemma 2
-# Same as Gemma 1, but with sliding window attention!
-# https://ollama.com/library/gemma2/blobs/6522ca797f47
gemma2_ollama = gemma_ollama + "PARAMETER num_ctx 4096\n"
OLLAMA_TEMPLATES["gemma2"] = gemma2_ollama
@@ -550,8 +548,8 @@ OLLAMA_TEMPLATES["gemma2"] = gemma2_ollama
gemma2_chatml_ollama = gemma_chatml_ollama + "PARAMETER num_ctx 4096\n"
OLLAMA_TEMPLATES["gemma2_chatml"] = gemma2_chatml_ollama
-# =========================================== Llama-3
# Ollama from https://www.ollama.com/library/llama3
+# =========================================== Llama-3
llama3_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>
@@ -573,8 +571,8 @@ OLLAMA_TEMPLATES["llama-3"] = llama3_ollama
OLLAMA_TEMPLATES["llama3"] = llama3_ollama
-# =========================================== Phi-3
# Ollama from https://www.ollama.com/library/phi3
+# =========================================== Phi-3
phi3_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{ if .System }}<|system|>
@@ -595,7 +593,6 @@ OLLAMA_TEMPLATES["phi-3"] = phi3_ollama
OLLAMA_TEMPLATES["phi-35"] = OLLAMA_TEMPLATES["phi-3"]
OLLAMA_TEMPLATES["phi-3.5"] = OLLAMA_TEMPLATES["phi-3"]
-# =========================================== Llama-3.1
"""
No trimming in Llama 3.1 Instruct!
Also an extra newline for Cutting Knowledge Date
@@ -794,8 +791,8 @@ OLLAMA_TEMPLATES["llama-32-vision"] = llama_32_vision_ollama
for version in ("llama-3.2", "llama-3.3", "llama-32", "llama-33"):
OLLAMA_TEMPLATES[version] = OLLAMA_TEMPLATES["llama-3.1"]
+# tinyllama-chat from ollama.com/library/tinyllama blob af0ddbdaaa26.
# =========================================== tinyllama
-# tinyllama-chat https://ollama.com/library/tinyllama:latest/blobs/af0ddbdaaa26
tinyllama_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """<|system|>
@@ -813,9 +810,8 @@ SYSTEM """You are a helpful AI assistant."""
OLLAMA_TEMPLATES["tinyllama"] = tinyllama_ollama
+# Qwen2 blob 77c91b422cc9; Qwen2.5 from ollama.com/library/qwen2.5 blob eb4402837c78.
# =========================================== Qwen 2/2.5
-# Qwen2 https://ollama.com/library/qwen2:latest/blobs/77c91b422cc9
-# Qwen2.5 from https://ollama.com/library/qwen2.5/blobs/eb4402837c78
qwen25_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{- if .Messages }}
@@ -987,7 +983,7 @@ _phi4_ollama_template = (
"<|im_start|><|assistant|><|im_sep|>{{ .Response }}<|im_end|>"
)
-# Ollama from https://www.ollama.com/library/phi4 is different
+# The ollama.com/library/phi4 template is different.
phi_4_ollama = f'''
FROM {{__FILE_LOCATION__}}
TEMPLATE """{_phi4_ollama_template}"""
@@ -1057,8 +1053,8 @@ OLLAMA_TEMPLATES["phi-4-mini"] = phi_4_mini_ollama
OLLAMA_TEMPLATES["phi-4-mini-reasoning"] = phi_4_mini_reasoning_ollama
-# =========================================== Gemma-3
# Ollama from https://ollama.com/library/gemma3/blobs/e0a42594d802
+# =========================================== Gemma-3
gemma3_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{- range $i, $_ := .Messages }}
@@ -1111,7 +1107,6 @@ OLLAMA_TEMPLATES["gemma3"] = gemma3_ollama
OLLAMA_TEMPLATES["gemma3-270m"] = gemma3_270m_ollama
-# =========================================== Qwen-3
# Ollama template for Qwen-3 (see https://ollama.com/library/qwen3/blobs/eb4402837c78)
qwen3_ollama = '''
FROM {__FILE_LOCATION__}
@@ -1179,8 +1174,8 @@ OLLAMA_TEMPLATES["qwen-3"] = qwen3_ollama
OLLAMA_TEMPLATES["qwen3"] = qwen3_ollama
-# =========================================== Gemma-3n
# Ollama from https://ollama.com/library/gemma3n/blobs/e0a42594d802
+# =========================================== Gemma-3n
gemma3n_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{- range $i, $_ := .Messages }}
@@ -1214,9 +1209,10 @@ TEMPLATE """{{- range $i, $_ := .Messages }}
OLLAMA_TEMPLATES["gemma-4"] = gemma4_ollama
OLLAMA_TEMPLATES["gemma4"] = gemma4_ollama
-# =========================================== GPT-OSS
# Ollama from https://ollama.com/library/gpt-oss:latest/blobs/fa6710a93d78
+# =========================================== GPT-OSS
+
gptoss_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
@@ -1401,7 +1397,6 @@ OLLAMA_TEMPLATES["gpt-oss"] = gptoss_ollama
OLLAMA_TEMPLATES["gptoss"] = gptoss_ollama
-# =========================================== Qwen3
# Ollama from https://ollama.com/library/qwen3/blobs/53e4ea15e8f5
qwen3_ollama = '''
@@ -1464,10 +1459,11 @@ OLLAMA_TEMPLATES["qwen3-instruct"] = qwen3_ollama
OLLAMA_TEMPLATES["qwen3-thinking"] = qwen3_ollama
-# =========================================== Starling-LM
# Ollama from https://ollama.com/library/starling-lm:7b/blobs/4b21bfc435b4
+# =========================================== Starling-LM
+
starling_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{ if .System }}GPT4 Correct System: {{ .System }}<|end_of_turn|>
@@ -1484,10 +1480,11 @@ PARAMETER min_p 0.1
OLLAMA_TEMPLATES["starling"] = starling_ollama
-# =========================================== Yi-chat
# Ollama from https://ollama.com/library/yi:34b-chat/blobs/62fbfd9ed093
+# =========================================== Yi-chat
+
yi_chat_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{ if .System }}<|im_start|>system
@@ -1500,9 +1497,10 @@ TEMPLATE """{{ if .System }}<|im_start|>system
OLLAMA_TEMPLATES["yi-chat"] = yi_chat_ollama
-# =========================================== Granite
# Ollama from https://ollama.com/library/granite3.2:latest/blobs/3e7ca51acd6e
+# =========================================== Granite
+
granite_32_ollama = '''
FROM {__FILE_LOCATION__}
TEMPLATE """{{- /*
@@ -2219,7 +2217,6 @@ for key, values in OLLAMA_TEMPLATE_TO_MODEL_MAPPER.items():
for value in values:
MODEL_TO_OLLAMA_TEMPLATE_MAPPER[value] = key
- # Get lowercased
lowered_key = key.lower()
for value in values:
MODEL_TO_OLLAMA_TEMPLATE_MAPPER[value.lower()] = lowered_key
diff --git a/unsloth/optimizers/q_galore_adamw.py b/unsloth/optimizers/q_galore_adamw.py
index 9acd597f614..d3083aff8e9 100644
--- a/unsloth/optimizers/q_galore_adamw.py
+++ b/unsloth/optimizers/q_galore_adamw.py
@@ -133,7 +133,6 @@ class QGaLoreAdamW8bit(Optimizer2State):
has_weight_quant = self._has_weight_quant(p, group)
- # --- Dequantize weight if INT8 ---
if has_weight_quant:
if p._q_scales is not None:
float_weight = _dequantize(
@@ -143,9 +142,7 @@ class QGaLoreAdamW8bit(Optimizer2State):
p._q_shape,
)
p.data = float_weight
- # else: first step, weights are still float — skip dequantize
- # --- GaLore projection ---
if "rank" in group:
if "projector" not in state:
state["projector"] = GaLoreProjector(
@@ -161,8 +158,7 @@ class QGaLoreAdamW8bit(Optimizer2State):
queue_size = group.get("queue_size", 5),
)
- # Temporarily disable weight decay for GaLore params
- # (we apply it manually after project-back)
+ # Temporarily disable weight decay for GaLore params; it is applied manually after project-back.
if "weight_decay" in group and group["weight_decay"] > 0:
group["_wd_saved"] = group["weight_decay"]
group["weight_decay"] = 0
@@ -174,19 +170,17 @@ class QGaLoreAdamW8bit(Optimizer2State):
p.data = torch.zeros_like(grad, dtype = p.data.dtype, device = p.data.device)
p.grad = grad
- # --- 8-bit Adam update ---
if "state1" not in state:
self.init_state(group, p, gindex, pindex)
self.prefetch_state(p)
self.update_step(group, p, gindex, pindex)
- # --- GaLore project-back ---
if "rank" in group:
- # p.data now holds the weight update in low-rank space
+ # p.data now holds the weight update in low-rank space.
p.data = p._saved_data.add_(state["projector"].project_back(p.data))
- # Re-apply decoupled weight decay using pre-update weights
+ # Re-apply decoupled weight decay using pre-update weights.
if "_wd_saved" in group:
p.data.add_(
p.data,
@@ -197,7 +191,6 @@ class QGaLoreAdamW8bit(Optimizer2State):
del p._saved_data
- # --- Re-quantize weight to INT8 ---
if has_weight_quant:
float_data = p.data
stochastic = group.get("stochastic_round", True)
@@ -208,9 +201,8 @@ class QGaLoreAdamW8bit(Optimizer2State):
p._q_scales = scales
p._q_zeros = zeros
p._q_shape = shape
- # Scalar placeholder to free float memory; the forward
- # pre-hook (install_weight_quant_hooks) dequantizes before
- # the next forward pass.
+ # Scalar placeholder to free float memory; the forward pre-hook (install_weight_quant_hooks)
+ # dequantizes before the next forward pass.
p.data = torch.empty(1, dtype = p.data.dtype, device = p.data.device)
state["step"] += 1
@@ -251,9 +243,8 @@ class QGaLoreAdamW8bit(Optimizer2State):
for name, p in model.named_parameters():
if id(p) in weight_quant_params:
- # Store metadata without converting weights to uint8; the first
- # step() quantizes after the update. Dummy scales/zeros keep
- # _has_weight_quant() True on the first step.
+ # Store metadata without converting weights to uint8; the first step() quantizes after the update,
+ # and dummy scales/zeros keep _has_weight_quant() True on that first step.
p._q_scales = None
p._q_zeros = None
p._q_shape = p.data.shape
@@ -355,8 +346,8 @@ def make_q_galore_param_groups(
if not param.requires_grad:
continue
- # Match target module names; exclude 1-D params (biases, norms) since
- # GaLoreProjector.project requires 2-D gradients.
+ # Match target module names and exclude 1-D params (biases, norms), since GaLoreProjector.project
+ # requires 2-D gradients.
name_parts = name.split(".")
is_galore = param.dim() >= 2 and any(t in name_parts for t in targets)
diff --git a/unsloth/optimizers/q_galore_projector.py b/unsloth/optimizers/q_galore_projector.py
index 3fba75e5b39..cc1ef9ea9ea 100644
--- a/unsloth/optimizers/q_galore_projector.py
+++ b/unsloth/optimizers/q_galore_projector.py
@@ -111,8 +111,6 @@ class GaLoreProjector:
self.ortho_matrix_shape = None
# ------------------------------------------------------------------
- # Public API
- # ------------------------------------------------------------------
def project(self, full_rank_grad: torch.Tensor, step: int) -> torch.Tensor:
"""Project a full-rank gradient into the low-rank subspace.
@@ -131,7 +129,8 @@ class GaLoreProjector:
assert self.proj_type == "std", "Only proj_type='std' is supported."
if full_rank_grad.shape[0] >= full_rank_grad.shape[1]:
- # "tall" matrix → right projection (grad @ Q^T)
+ # A "tall" matrix takes the right projection (grad @ Q^T), a "wide" one the left (Q^T @ grad).
+ # "wide" matrix → left projection (Q^T @ grad)
if self.ortho_matrix is None or step % self.update_proj_gap == 0:
float_ortho = self._compute_orthogonal(
full_rank_grad,
@@ -144,7 +143,6 @@ class GaLoreProjector:
self._ortho_float_cache = self._load_ortho()
low_rank_grad = torch.matmul(full_rank_grad, self._ortho_float_cache.t())
else:
- # "wide" matrix → left projection (Q^T @ grad)
if self.ortho_matrix is None or step % self.update_proj_gap == 0:
float_ortho = self._compute_orthogonal(
full_rank_grad,
@@ -181,8 +179,6 @@ class GaLoreProjector:
return full_rank_grad * self.scale
# ------------------------------------------------------------------
- # SVD
- # ------------------------------------------------------------------
@staticmethod
def _compute_orthogonal(weights: torch.Tensor, rank: int, side: str) -> torch.Tensor:
@@ -209,8 +205,8 @@ class GaLoreProjector:
U, s, Vh = torch.linalg.svd(matrix, full_matrices = False)
result = Vh[:rank, :] if side == "right" else U[:, :rank]
else:
- # Oversampling p=10 per Halko et al. 2009 (arXiv:0909.4061)
- # recommendation of p=5..10 for large low-rank matrices.
+ # Oversampling p=10, per Halko et al. 2009 (arXiv:0909.4061), which recommends p=5..10 for large
+ # low-rank matrices.
q = min(rank + 10, min(m, n))
U, s, V = torch.svd_lowrank(matrix, q = q, niter = 2)
result = V[:, :rank].t() if side == "right" else U[:, :rank]
@@ -220,8 +216,6 @@ class GaLoreProjector:
return result
# ------------------------------------------------------------------
- # Adaptive scheduling
- # ------------------------------------------------------------------
def _update_adaptive_schedule(self, float_ortho: torch.Tensor, side: str) -> None:
"""Track subspace stability and increase ``update_proj_gap`` if stable."""
@@ -246,8 +240,6 @@ class GaLoreProjector:
self.past_ortho_vector = current_vector.clone()
# ------------------------------------------------------------------
- # Quantized projection matrix storage
- # ------------------------------------------------------------------
def _store_ortho(self, float_ortho: torch.Tensor) -> None:
"""Store the orthogonal matrix, optionally quantized."""
@@ -276,11 +268,6 @@ class GaLoreProjector:
return self.ortho_matrix
-# ======================================================================
-# Quantization utilities (shared with the optimizer)
-# ======================================================================
-
-
@torch.no_grad()
def _quantize(
w: torch.Tensor,
@@ -318,7 +305,7 @@ def _dequantize(
w: torch.Tensor, scales: torch.Tensor, zeros: torch.Tensor, original_shape: tuple
) -> torch.Tensor:
"""Dequantize from uint8 back to float."""
- # Infer group size: scales has shape (n_groups, 1), so n_groups = scales.shape[0]
+ # Infer the group size: scales has shape (n_groups, 1), so n_groups = scales.shape[0].
total = w.numel()
n_groups = scales.shape[0] if scales.dim() > 1 else scales.numel()
group_size = total // n_groups if n_groups > 0 else total
diff --git a/unsloth/registry/_deepseek.py b/unsloth/registry/_deepseek.py
index 618453fb82f..ff3fc9607b9 100644
--- a/unsloth/registry/_deepseek.py
+++ b/unsloth/registry/_deepseek.py
@@ -24,7 +24,6 @@ class DeepseekR1ModelInfo(ModelInfo):
return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key)
-# Deepseek V3 Model Meta
DeepseekV3Meta = ModelMeta(
org = "deepseek-ai",
base_name = "DeepSeek",
@@ -80,7 +79,6 @@ DeepseekR1DistillLlamaMeta = ModelMeta(
quant_types = {"8": [QuantType.UNSLOTH, QuantType.GGUF], "70": [QuantType.GGUF]},
)
-# Deepseek R1 Distill Qwen Model Meta
DeepseekR1DistillQwenMeta = ModelMeta(
org = "deepseek-ai",
base_name = "DeepSeek-R1-Distill",
@@ -164,7 +162,7 @@ def _list_deepseek_r1_distill_models():
for model in models:
model_id = model.id
model_name = model_id.split("/")[-1]
- # parse out only the version
+ # Parse out only the version.
version = model_name.removeprefix("DeepSeek-R1-Distill-")
distill_models.append(version)
@@ -184,7 +182,3 @@ if __name__ == "__main__":
print(f"\u2718 {model_id}")
else:
print(f"\u2713 {model_id}")
- # distill_models = _list_deepseek_r1_distill_models()
- # for model in sorted(distill_models):
- # if "qwen" in model.lower():
- # print(model)
diff --git a/unsloth/registry/_gemma.py b/unsloth/registry/_gemma.py
index 8b43dc35665..08b40dbd0e6 100644
--- a/unsloth/registry/_gemma.py
+++ b/unsloth/registry/_gemma.py
@@ -11,7 +11,6 @@ class GemmaModelInfo(ModelInfo):
return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key)
-# Gemma3 Base Model Meta
GemmaMeta3Base = ModelMeta(
org = "google",
base_name = "gemma",
@@ -23,7 +22,6 @@ GemmaMeta3Base = ModelMeta(
quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH],
)
-# Gemma3 Instruct Model Meta
GemmaMeta3Instruct = ModelMeta(
org = "google",
base_name = "gemma",
diff --git a/unsloth/registry/_phi.py b/unsloth/registry/_phi.py
index 17d8f40a903..da33d64fcf5 100644
--- a/unsloth/registry/_phi.py
+++ b/unsloth/registry/_phi.py
@@ -11,7 +11,6 @@ class PhiModelInfo(ModelInfo):
return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key)
-# Phi Model Meta
PhiMeta4 = ModelMeta(
org = "microsoft",
base_name = "phi",
@@ -23,7 +22,6 @@ PhiMeta4 = ModelMeta(
quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH],
)
-# Phi Instruct Model Meta
PhiInstructMeta4 = ModelMeta(
org = "microsoft",
base_name = "phi",
diff --git a/unsloth/registry/_qwen.py b/unsloth/registry/_qwen.py
index 8041ff21030..d754c633a96 100644
--- a/unsloth/registry/_qwen.py
+++ b/unsloth/registry/_qwen.py
@@ -33,7 +33,6 @@ class QwenQVQPreviewModelInfo(ModelInfo):
return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key)
-# Qwen2.5 Model Meta
Qwen_2_5_Meta = ModelMeta(
org = "Qwen",
base_name = "Qwen",
@@ -45,7 +44,6 @@ Qwen_2_5_Meta = ModelMeta(
quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH],
)
-# Qwen2.5 VL Model Meta
Qwen_2_5_VLMeta = ModelMeta(
org = "Qwen",
base_name = "Qwen",
@@ -57,7 +55,6 @@ Qwen_2_5_VLMeta = ModelMeta(
quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH],
)
-# Qwen QwQ Model Meta
QwenQwQMeta = ModelMeta(
org = "Qwen",
base_name = "QwQ",
@@ -69,10 +66,8 @@ QwenQwQMeta = ModelMeta(
quant_types = [QuantType.NONE, QuantType.BNB, QuantType.UNSLOTH, QuantType.GGUF],
)
-# Qwen QVQ Preview Model Meta
-# No QuantType.NONE: the unquantized mirror unsloth/QVQ-72B-Preview was removed
-# from the Hub (only unsloth/QVQ-72B-Preview-bnb-4bit remains). The upstream
-# Qwen/QVQ-72B-Preview is still registered via include_original_model.
+# No QuantType.NONE: the unquantized mirror unsloth/QVQ-72B-Preview was removed from the Hub,
+# leaving only the bnb-4bit build; upstream Qwen/QVQ-72B-Preview is still registered.
QwenQVQPreviewMeta = ModelMeta(
org = "Qwen",
base_name = "QVQ",
diff --git a/unsloth/registry/registry.py b/unsloth/registry/registry.py
index 220dde99447..e853daf939c 100644
--- a/unsloth/registry/registry.py
+++ b/unsloth/registry/registry.py
@@ -16,7 +16,6 @@ class QuantType(Enum):
BF16 = "bf16" # only for Deepseek V3
-# Tags for Hugging Face model paths
BNB_QUANTIZED_TAG = "bnb-4bit"
UNSLOTH_DYNAMIC_QUANT_TAG = "unsloth" + "-" + BNB_QUANTIZED_TAG
GGUF_TAG = "GGUF"
@@ -31,7 +30,7 @@ QUANT_TAG_MAP = {
}
-# NOTE: models registered with org="unsloth" and QUANT_TYPE.NONE are aliases of QUANT_TYPE.UNSLOTH
+# Models registered with org="unsloth" and QUANT_TYPE.NONE are aliases of QUANT_TYPE.UNSLOTH.
@dataclass
class ModelInfo:
org: str
@@ -164,13 +163,13 @@ def _register_models(model_meta: ModelMeta, include_original_model: bool = False
for size in model_sizes:
for instruct_tag in instruct_tags:
- # Handle quant types per model size
if isinstance(quant_types, dict):
_quant_types = quant_types[size]
else:
_quant_types = quant_types
for quant_type in _quant_types:
- # NOTE: models registered with org="unsloth" and QUANT_TYPE.NONE are aliases of QUANT_TYPE.UNSLOTH
+ # NOTE: models registered with org="unsloth" and QUANT_TYPE.NONE are aliases of
+ # QUANT_TYPE.UNSLOTH
_org = "unsloth" # quantized versions of the original model
register_model(
model_info_cls = model_info_cls,
diff --git a/unsloth/save.py b/unsloth/save.py
index 2c401d6b0ef..d3da157a764 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Defensive imports: added in unsloth-zoo PR #526 and may not exist on older versions.
from unsloth_zoo.utils import Version
from importlib.metadata import version as importlib_version
from unsloth_zoo.hf_utils import dtype_from_config, HAS_TORCH_DTYPE
@@ -25,16 +26,14 @@ from unsloth_zoo.llama_cpp import (
_download_convert_hf_to_gguf,
)
-# H4: Defensive imports -- these were added in unsloth-zoo PR #526
-# and may not exist on older versions
try:
from unsloth_zoo.llama_cpp import LLAMA_CPP_DEFAULT_DIR, IS_WINDOWS
except ImportError:
import sys
IS_WINDOWS = sys.platform == "win32"
LLAMA_CPP_DEFAULT_DIR = "llama.cpp"
-# Without bnb, peft stops exporting its 4bit LoRA layer too. Both names only feed
-# isinstance checks, so placeholders nothing can match are exact stand-ins.
+# Without bnb, peft stops exporting its 4bit LoRA layer too. Both names only feed isinstance checks, so
+# placeholders nothing can match are exact stand-ins.
try:
from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit
from peft.tuners.lora import Linear4bit as Peft_Linear4bit
@@ -84,7 +83,7 @@ except:
try:
from huggingface_hub.utils import get_token
except:
- # For older versions of huggingface_hub
+ # For older versions of huggingface_hub.
from huggingface_hub.utils._token import get_token
from pathlib import Path
from peft import PeftModelForCausalLM, PeftModel
@@ -97,18 +96,16 @@ __all__ = [
"create_huggingface_repo",
]
-# llama.cpp specific targets - all takes 90s. Below takes 60s
+# llama.cpp specific targets: all takes 90s, the below 60s.
LLAMA_CPP_TARGETS = [
"llama-quantize",
"llama-cli",
"llama-server",
]
-# Check environments. `is_kaggle_environment` needs a real Kaggle kernel, not
-# merely a KAGGLE_* variable: KAGGLE_USERNAME / KAGGLE_KEY / KAGGLE_CONFIG_DIR
-# are what the Kaggle CLI reads on an ordinary laptop, and every one of the
-# Kaggle branches below (rewriting save paths to /tmp, deleting the cached base
-# model) is wrong to take there.
+# is_kaggle_environment needs a real kernel, not a KAGGLE_* variable: the Kaggle CLI reads those on an
+# ordinary laptop, and the branches below (/tmp save paths, deleting the cached base model) are all wrong
+# there.
from .disk_utils import (
KAGGLE_TMP,
is_colab_environment,
@@ -123,7 +120,6 @@ from .disk_utils import (
IS_COLAB_ENVIRONMENT = is_colab_environment()
IS_KAGGLE_ENVIRONMENT = is_kaggle_environment()
-# Weights
LLAMA_WEIGHTS = (
"self_attn.q_proj",
"self_attn.k_proj",
@@ -142,8 +138,7 @@ LLAMA_LAYERNORMS = (
"self_attn.k_norm",
)
-# https://github.com/ggerganov/llama.cpp/blob/master/examples/quantize/quantize.cpp#L19
-# From https://mlabonne.github.io/blog/posts/Quantize_Llama_2_models_using_ggml.html
+# See llama.cpp examples/quantize/quantize.cpp#L19 and mlabonne.github.io Quantize_Llama_2_models_using_ggml.
ALLOWED_QUANTS = {
"not_quantized": "Recommended. Fast conversion. Slow inference, big files.",
"fast_quantized": "Recommended. Fast conversion. OK inference, OK file size.",
@@ -171,8 +166,8 @@ ALLOWED_QUANTS = {
"q3_k_xs": "3-bit extra small quantization",
}
-# IQ (importance-matrix) quants. llama.cpp refuses these without an imatrix, so they are only
-# accepted when imatrix_file=... is supplied to save_pretrained_gguf / push_to_hub_gguf.
+# IQ quants need an importance matrix; llama.cpp refuses them without one, so they are only accepted when
+# imatrix_file=... is supplied.
IMATRIX_QUANTS = {
"iq1_s": "1.56 bpw. Smallest, lowest quality. Needs an imatrix.",
"iq1_m": "1.75 bpw. Very small. Needs an imatrix.",
@@ -207,14 +202,10 @@ def has_curl():
CURL_FLAG = "-DLLAMA_CURL=ON" if has_curl() else "-DLLAMA_CURL=OFF"
-# FP8/FP4 compressed export via llm-compressor (for vLLM).
-# save_method alias -> (llm-compressor scheme, needs_calibration, output dir suffix).
-# alias -> (llm-compressor scheme, needs_calibration, output-dir suffix). needs_calibration is
-# True only for schemes with static activation scales (FP8 static, NVFP4); everything else is
-# weight-only or dynamic-activation and runs data-free. Unsupported schemes in the installed
-# compressed-tensors (e.g. MXFP8 on older stacks) are gated by _scheme_is_available at runtime.
+# llm-compressor FP8/FP4 export for vLLM: alias -> (scheme, needs_calibration, dir suffix). needs_calibration
+# only for static activation scales (FP8 static, NVFP4); the rest run data-free. Schemes absent from the
+# installed compressed-tensors are gated at runtime.
COMPRESSED_EXPORT_SCHEMES = {
- # FP8
"fp8": ("FP8_DYNAMIC", False, "fp8"),
"fp8_dynamic": ("FP8_DYNAMIC", False, "fp8"),
"dynamic_fp8": ("FP8_DYNAMIC", False, "fp8"),
@@ -223,7 +214,6 @@ COMPRESSED_EXPORT_SCHEMES = {
"static_fp8": ("FP8", True, "fp8-static"),
"fp8_block": ("FP8_BLOCK", False, "fp8-block"),
"block_fp8": ("FP8_BLOCK", False, "fp8-block"),
- # INT8 / INT-weight
"int8": ("INT8", False, "int8"),
"w8a8": ("W8A8", False, "w8a8"),
"w8a8_int8": ("W8A8", False, "w8a8"),
@@ -235,14 +225,12 @@ COMPRESSED_EXPORT_SCHEMES = {
"w4a16_asym": ("W4A16_ASYM", False, "w4a16-asym"),
"w4a8": ("W4A8", False, "w4a8"),
"w4afp8": ("W4AFP8", False, "w4afp8"),
- # MXFP (microscaling)
"mxfp8": ("MXFP8", False, "mxfp8"),
"w8a8_mxfp8": ("MXFP8", False, "mxfp8"),
"mxfp4": ("MXFP4", False, "mxfp4"),
"w4a4_mxfp4": ("MXFP4", False, "mxfp4"),
"mxfp4a16": ("MXFP4A16", False, "mxfp4a16"),
"w4a16_mxfp4": ("MXFP4A16", False, "mxfp4a16"),
- # NVFP4
"nvfp4": ("NVFP4", True, "nvfp4"),
"w4a4_nvfp4": ("NVFP4", True, "nvfp4"),
"nvfp4a16": ("NVFP4A16", False, "nvfp4a16"),
@@ -250,8 +238,8 @@ COMPRESSED_EXPORT_SCHEMES = {
}
-# torchao "portable" quant export: device-agnostic FP8 / INT8, no NVIDIA GPU needed.
-# alias -> (kind, sibling suffix). FP8 saves to safetensors, INT8 to .bin; both load in vLLM.
+# torchao portable export: device-agnostic FP8 / INT8, no NVIDIA GPU needed. alias -> (kind, sibling suffix);
+# FP8 writes safetensors, INT8 .bin, both load in vLLM.
TORCHAO_EXPORT_SCHEMES = {
"torchao_fp8": ("fp8", "torchao-fp8"),
"torchao_int8": ("int8", "torchao-int8"),
@@ -287,8 +275,8 @@ def _loaded_via_remote_code(obj):
if node is None or id(node) in seen:
continue
seen.add(id(node))
- # __module__ can be None/absent on some dynamically created or C-extension classes;
- # treat anything non-string as "not remote code" rather than crashing the export.
+ # __module__ can be None on dynamically created or C-extension classes; treat a non-string as "not
+ # remote code" rather than crashing the export.
module = getattr(type(node), "__module__", None)
if isinstance(module, str) and module.startswith("transformers_modules"):
return True
@@ -297,8 +285,8 @@ def _loaded_via_remote_code(obj):
queue.append(node.get_base_model())
except Exception:
pass
- # PEFT / trainer wrappers hold the real model in base_model / model; a built-in
- # ProcessorMixin holds its (possibly custom-code) components as attributes.
+ # PEFT / trainer wrappers hold the real model in base_model / model; a ProcessorMixin holds its
+ # (possibly custom-code) components as attributes.
for attr in (
"base_model",
"model",
@@ -344,7 +332,7 @@ def _is_cmake_only_llama_cpp(llama_cpp_dir: str = "llama.cpp") -> bool:
"""
makefile_path = os.path.join(llama_cpp_dir, "Makefile")
if not os.path.exists(makefile_path):
- # No Makefile: only CMake-only if a real CMake project is present
+ # No Makefile: only CMake-only if a real CMake project is present.
return os.path.exists(os.path.join(llama_cpp_dir, "CMakeLists.txt"))
try:
with open(makefile_path, "r", encoding = "utf-8", errors = "ignore") as f:
@@ -382,8 +370,8 @@ def _quantize_q2_k_l(
print_output: bool = True,
imatrix = None,
):
- # "Q2_K_L" is an Unsloth preset, not a native llama.cpp ftype: q2_k with
- # output/token-embedding tensors kept at q8_0 for higher precision.
+ # "Q2_K_L" is an Unsloth preset, not a native llama.cpp ftype: q2_k with output and token embeddings kept
+ # at q8_0.
command = [
str(quantizer_location),
*(["--imatrix", str(imatrix)] if imatrix else []),
@@ -496,7 +484,7 @@ def _has_tokenizer_model(tokenizer, token = None):
if cache_key in _TOKENIZER_MODEL_CACHE:
return _TOKENIZER_MODEL_CACHE[cache_key]
- # Hub repo id: probe local cache before model_info (issue #7481).
+ # Hub repo id: probe the local cache before model_info (#7481).
cache_dir = _tokenizer_cache_dir(tokenizer) or os.environ.get("HF_HUB_CACHE")
if not cache_dir:
hf_home = os.environ.get("HF_HOME")
@@ -617,8 +605,7 @@ def _free_cached_model(model):
from huggingface_hub import scan_cache_dir
cached_repos = list(scan_cache_dir().repos)
- # Go through every cached repo, and delete the one that matches the model we want to save.
- # Can save 4GB of disk space - useful for Kaggle systems.
+ # Delete the cached repo matching the model being saved: saves 4GB of disk, useful on Kaggle.
for cached_repo in cached_repos:
if cached_repo.repo_id == model.config._name_or_path:
remove_cache_commit = list(cached_repo.revisions)[0].commit_hash
@@ -637,7 +624,6 @@ def _free_cached_model(model):
def _merge_lora(layer, name):
bias = getattr(layer, "bias", None)
if isinstance(layer, (Bnb_Linear4bit, Peft_Linear4bit, Peft_Linear)):
- # Is LoRA so we need to merge!
W, quant_state, A, B, s, bias = get_lora_parameters_bias(layer)
if quant_state is not None:
dtype = quant_state.dtype if type(quant_state) is not list else quant_state[2]
@@ -645,14 +631,9 @@ def _merge_lora(layer, name):
else:
dtype = W.dtype
W = W.to(torch.float32).t()
- # W = W.t()
if A is not None:
- # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32)))
- # W += sAB
W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s)
- # W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s)
- # if not torch.isfinite(W).all():
maximum_element = torch.max(W.min().abs(), W.max())
if not torch.isfinite(maximum_element).item():
raise ValueError(f"Unsloth: Merge failed.\n{name} has some elements = infinity.")
@@ -663,14 +644,11 @@ def _merge_lora(layer, name):
def fast_save_pickle(shard, name):
- # Use this if # CPUs is <= 2
print(f"Unsloth: Saving {name}...")
torch.save(
shard,
name,
- # HIGHEST_PROTOCOL seems to not work with Pytorch!
- # pickle_module = pickle,
- # pickle_protocol = pickle.HIGHEST_PROTOCOL,
+ # HIGHEST_PROTOCOL seems not to work with Pytorch.
)
return
@@ -851,7 +829,6 @@ def unsloth_save_model(
safe_serialization: bool = True,
variant: Optional[str] = None,
save_peft_format: bool = True,
- # Push to hub
use_temp_dir: Optional[bool] = None,
commit_message: Optional[str] = "Trained with Unsloth",
private: Optional[bool] = None,
@@ -859,7 +836,6 @@ def unsloth_save_model(
revision: str = None,
commit_description: str = "Upload model trained with Unsloth 2x faster",
tags: List[str] = None,
- # Our functions
temporary_location: str = "_unsloth_temporary_saved_buffers",
maximum_memory_usage: float = 0.9,
datasets: Optional[List[str]] = None,
@@ -902,7 +878,6 @@ def unsloth_save_model(
):
del save_pretrained_settings[deletion]
- # First check for a token!
if push_to_hub:
from huggingface_hub import whoami
try:
@@ -914,7 +889,6 @@ def unsloth_save_model(
assert maximum_memory_usage > 0 and maximum_memory_usage <= 0.95
- # Clean memory up first
for _ in range(3):
torch.cuda.empty_cache()
gc.collect()
@@ -932,7 +906,6 @@ def unsloth_save_model(
print("Unsloth: Merging 4bit and LoRA weights to 4bit...")
print("This might take 5 minutes...")
- # Counteract no LoRA adapters!
if hasattr(model, "merge_and_unload"):
model = model.merge_and_unload()
print("Done.")
@@ -960,7 +933,6 @@ def unsloth_save_model(
elif save_method == "merged_4bit":
print("Unsloth: Saving 4bit Bitsandbytes model. Please wait...")
- # Update model tag
_ = upload_to_huggingface(
model,
save_directory,
@@ -987,7 +959,6 @@ def unsloth_save_model(
tags = tags,
)
if tokenizer is not None:
- # Set padding side to left for inference
_tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizer
old_padding_side = _tokenizer.padding_side
_tokenizer.padding_side = "left"
@@ -1006,14 +977,12 @@ def unsloth_save_model(
tags = tags,
)
- # Revert back padding side
_tokenizer.padding_side = old_padding_side
if hasattr(model, "config"):
print(f"Saved {save_method} model to https://huggingface.co/" + save_directory)
return save_directory, None
- # Tokenizer has different saving arguments
tokenizer_save_settings = {
"save_directory": save_pretrained_settings["save_directory"],
"legacy_format": None,
@@ -1023,7 +992,6 @@ def unsloth_save_model(
"token": save_pretrained_settings["token"],
}
- # Check if PEFT Model or not - if yes, 3 levels. If not 2 levels.
from peft import PeftModelForCausalLM
if isinstance(model, PeftModelForCausalLM):
@@ -1031,16 +999,13 @@ def unsloth_save_model(
else:
internal_model = model
- # Cannot be converted properly!
if (
(save_method == "merged_4bit")
or (save_method == "lora")
or (not hasattr(model, "model") or not hasattr(internal_model.model, "layers"))
):
- # Do general saving
- # Edit save_pretrained_settings
- # [TODO] _create_repo has errors due to **kwargs getting accepted
- # commit_description does not seem to work?
+ # _create_repo has errors due to **kwargs getting accepted, and commit_description does not seem to
+ # work.
what_to_delete = (
(
"use_temp_dir",
@@ -1068,7 +1033,6 @@ def unsloth_save_model(
]
)
- # Update model tag
if push_to_hub:
_ = upload_to_huggingface(
model,
@@ -1085,14 +1049,12 @@ def unsloth_save_model(
if tokenizer is not None:
print("Unsloth: Saving tokenizer...", end = "")
- # Set padding side to left for inference
_tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizer
old_padding_side = _tokenizer.padding_side
_tokenizer.padding_side = "left"
tokenizer.save_pretrained(**tokenizer_save_settings)
- # Revert back padding side
_tokenizer.padding_side = old_padding_side
print(" Done.")
@@ -1103,7 +1065,7 @@ def unsloth_save_model(
if save_method != "lora":
print(" This might take 10 minutes for Llama-7b...", end = "")
- # [TODO] Is this correct?
+ # Correctness unverified.
if save_method == "lora":
save_pretrained_settings["selected_adapters"] = None
@@ -1115,10 +1077,9 @@ def unsloth_save_model(
print(" Done.")
return save_directory, None
- # If push_to_hub, we must remove the .../ part of a repo
+ # With push_to_hub the ".../" part of a repo must be removed; the +1 solves absolute path issues.
username = None
if push_to_hub and "/" in save_directory:
- # +1 solves absolute path issues
new_save_directory = save_directory
username = new_save_directory[: new_save_directory.find("/")]
new_save_directory = new_save_directory[new_save_directory.find("/") + 1 :]
@@ -1142,7 +1103,6 @@ def unsloth_save_model(
print("Unsloth: Merging 4bit and LoRA weights to 16bit...")
- # Determine max RAM usage minus sharding
max_ram = psutil.virtual_memory().available
sharded_ram_usage = 5 * 1024 * 1024 * 1024
if type(max_shard_size) is str:
@@ -1155,7 +1115,6 @@ def unsloth_save_model(
elif type(max_shard_size) is int:
sharded_ram_usage = max_shard_size
- # Switch to our fast saving modules if it's a slow PC!
n_cpus = psutil.cpu_count(logical = False)
if n_cpus is None:
n_cpus = psutil.cpu_count()
@@ -1177,7 +1136,6 @@ def unsloth_save_model(
save_pretrained_settings["safe_serialization"] = safe_serialization
save_pretrained_settings["save_function"] = save_function
- # Only safe_serialization uses more RAM
if safe_serialization:
max_ram -= sharded_ram_usage
else:
@@ -1190,24 +1148,20 @@ def unsloth_save_model(
f"{round(psutil.virtual_memory().total / 1024 / 1024 / 1024, 2)} RAM for saving."
)
- # Move temporary_location to /tmp in Kaggle
if IS_KAGGLE_ENVIRONMENT:
temporary_location = os.path.join(KAGGLE_TMP, temporary_location)
- # Max directory for disk saving
if not os.path.exists(temporary_location):
os.makedirs(temporary_location)
- # Check if Kaggle or Colab, since only 20GB of Disk space allowed.
+ # Kaggle and Colab allow only 20GB of disk, so free up 4GB of space.
if IS_KAGGLE_ENVIRONMENT or IS_COLAB_ENVIRONMENT:
- # We free up 4GB of space
logger.warning_once(
"Unsloth: Kaggle/Colab has limited disk space. We need to delete the downloaded\n"
"model which will save 4-16GB of disk space, allowing you to save on Kaggle/Colab."
)
_free_cached_model(internal_model)
- # HF also uses a OrderedDict
from collections import OrderedDict
state_dict = OrderedDict()
@@ -1219,13 +1173,12 @@ def unsloth_save_model(
elif torch_dtype == "bfloat16":
torch_dtype = torch.bfloat16
- # Check modules to save float32 dtype
state_dict["model.embed_tokens.weight"] = internal_model.model.embed_tokens.weight.data.to(
torch_dtype
)
- # A merged tensor lives on the GPU of its source layer, so budget against W's own
- # device, not GPU0, else a sharded model OOMs GPU1+ while only GPU0 is checked.
+ # A merged tensor lives on the GPU of its source layer, so budget against W's own device: a GPU0-only
+ # check OOMs GPU1+ on a sharded model.
_max_vram_by_device = {}
def _device_vram_budget(dev):
@@ -1248,7 +1201,6 @@ def unsloth_save_model(
name = f"model.layers.{j}.{item}.weight"
W, bias = _merge_lora(proj, name)
- # Bias term
if bias is not None:
state_dict[f"model.layers.{j}.{item}.bias"] = bias
@@ -1262,14 +1214,8 @@ def unsloth_save_model(
elif W.device.type != "cuda":
# Already off-GPU: keeping it costs no VRAM
state_dict[name] = W
- # [TODO] Saving to RAM seems to leak memory???
- # elif (max_ram - W.nbytes) > 0:
- # # Save to CPU memory
- # logger.warning_once(f"We will save to RAM and not VRAM now.")
- # state_dict[name] = W.to("cpu", non_blocking = True, copy = True)
- # max_ram = max(max_ram - W.nbytes, 0)
+ # Saving to RAM seems to leak memory.
else:
- # Save to Disk
logger.warning_once("\nWe will save to Disk and not RAM now.")
filename = os.path.join(temporary_location, f"{name}.pt")
torch.save(
@@ -1278,39 +1224,33 @@ def unsloth_save_model(
pickle_module = pickle,
pickle_protocol = pickle.HIGHEST_PROTOCOL,
)
- # weights_only = True weirdly fails?
+ # weights_only = True weirdly fails.
state_dict[name] = torch.load(
filename, map_location = "cpu", mmap = True, weights_only = False
)
for item in LLAMA_LAYERNORMS:
try:
- # Skip for Gemma 2
state_dict[f"model.layers.{j}.{item}.weight"] = eval(f"layer.{item}.weight.data")
except:
continue
state_dict["model.norm.weight"] = internal_model.model.norm.weight.data
- # Check for modules_to_save float32 dtype
- # Check for tied weights
if (
internal_model.model.embed_tokens.weight.data_ptr()
!= internal_model.lm_head.weight.data_ptr()
):
state_dict["lm_head.weight"] = internal_model.lm_head.weight.data.to(torch_dtype)
- # All tensors MUST be type torch.Tensor and not torch.nn.parameter.Parameter
+ # All tensors MUST be torch.Tensor and not torch.nn.parameter.Parameter.
for key, value in state_dict.items():
if hasattr(value, "data"):
state_dict[key] = value = value.data
if type(value) is not torch.Tensor:
logger.warning_once(f"Unsloth: {key} is not a Tensor but a {type(value)}.")
- # Edit save_pretrained_settings
- # [TODO] _create_repo has errors due to **kwargs getting accepted
save_pretrained_settings["state_dict"] = state_dict
- # commit_description does not seem to work?
what_to_delete = (
(
"use_temp_dir",
@@ -1338,7 +1278,6 @@ def unsloth_save_model(
]
)
- # Update model tag
if push_to_hub:
_ = upload_to_huggingface(
model,
@@ -1352,7 +1291,6 @@ def unsloth_save_model(
datasets = datasets,
)
- # First check if we're pushing to an organization!
save_directory = save_pretrained_settings["save_directory"]
if save_pretrained_settings["push_to_hub"]:
@@ -1364,18 +1302,14 @@ def unsloth_save_model(
else:
actual_username = username
- # Check if pushing to an organization
if save_pretrained_settings["push_to_hub"] and (username != actual_username):
print(f"Unsloth: Saving to organization with address {new_save_directory}")
- # We upload everything at the end!
tokenizer_save_settings["push_to_hub"] = False
tokenizer_save_settings["save_directory"] = new_save_directory
- # Save tokenizer
if tokenizer is not None:
print("Unsloth: Saving tokenizer...", end = "")
- # Set padding side to left for inference
_tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizer
old_padding_side = _tokenizer.padding_side
_tokenizer.padding_side = "left"
@@ -1387,14 +1321,12 @@ def unsloth_save_model(
filename_prefix = tokenizer_save_settings.get("filename_prefix"),
)
- # Revert back padding side
_tokenizer.padding_side = old_padding_side
print(" Done.")
else:
print()
- # Since merged, edit quantization_config
old_config = model.config
new_config = model.config.to_dict()
if "quantization_config" in new_config:
@@ -1406,20 +1338,14 @@ def unsloth_save_model(
original_model.config = new_config
model.config = new_config
- # Save!
- # [TODO] --> is this correct?
- # save_pretrained_settings["selected_adapters"] = None
-
- # Check if pushing to an organization
if save_pretrained_settings["push_to_hub"] and (username != actual_username):
print(f"Unsloth: Saving to organization with address {new_save_directory}")
- # Pushing to organization: .save_pretrained doesn't work, so save
- # locally first then upload manually.
+ # Pushing to an organization: .save_pretrained does not work, so save locally first and upload
+ # manually.
save_pretrained_settings["save_directory"] = new_save_directory
save_pretrained_settings["push_to_hub"] = False
internal_model.save_pretrained(**save_pretrained_settings)
- # Now manually go through each file and upload them manually!
filenames = os.listdir(new_save_directory)
hf_api = HfApi(token = save_pretrained_settings["token"])
@@ -1436,7 +1362,6 @@ def unsloth_save_model(
else:
internal_model.save_pretrained(**save_pretrained_settings)
- # Revert config back
original_model = model
while hasattr(original_model, "model"):
original_model = original_model.model
@@ -1461,8 +1386,6 @@ def unsloth_save_model(
torch.cuda.empty_cache()
gc.collect()
- # Remove temporary location
-
shutil.rmtree(temporary_location, ignore_errors = True)
for _ in range(3):
@@ -1485,15 +1408,12 @@ def install_llama_cpp_clone_non_blocking():
def install_llama_cpp_make_non_blocking():
- # https://github.com/ggerganov/llama.cpp/issues/7062
- # Weirdly GPU conversion for GGUF breaks??
- # env = { **os.environ, "LLAMA_CUDA": "1", }
+ # GPU conversion for GGUF weirdly breaks; see ggerganov/llama.cpp#7062.
- # Skip the make-clean probe on CMake-only checkouts (its error output is misleading)
+ # Skip the make-clean probe on CMake-only checkouts, whose error output is misleading.
IS_CMAKE = _is_cmake_only_llama_cpp("llama.cpp")
if not IS_CMAKE:
- # Confirm make still works, silently
try:
result = subprocess.run(
["make", "clean", "-C", "llama.cpp"],
@@ -1502,16 +1422,13 @@ def install_llama_cpp_make_non_blocking():
)
IS_CMAKE = result.returncode != 0
except FileNotFoundError:
- # No make executable; use CMake
IS_CMAKE = True
if not IS_CMAKE:
- # Uses old MAKE
n_jobs = max(int((psutil.cpu_count() or 1) * 1.5), 1)
full_command = ["make", "all", "-j" + str(n_jobs), "-C", "llama.cpp"]
else:
- # Uses new CMAKE
- n_jobs = max(int(psutil.cpu_count() or 1), 1) # Use less CPUs since 1.5x faster
+ n_jobs = max(int(psutil.cpu_count() or 1), 1)
check = os.system(
f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}"
)
@@ -1520,7 +1437,6 @@ def install_llama_cpp_make_non_blocking():
raise RuntimeError(
f"*** Unsloth: Failed compiling llama.cpp using os.system(...) with error {check}. Please report this ASAP!"
)
- # f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",
full_command = [
"cmake",
"--build",
@@ -1531,9 +1447,6 @@ def install_llama_cpp_make_non_blocking():
"--clean-first",
"--target",
] + LLAMA_CPP_TARGETS
- # https://github.com/ggerganov/llama.cpp/issues/7062
- # Weirdly GPU conversion for GGUF breaks??
- # run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
run_installer = subprocess.Popen(
full_command, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT
)
@@ -1542,24 +1455,20 @@ def install_llama_cpp_make_non_blocking():
def install_python_non_blocking(packages = []):
full_command = ["pip", "install"] + packages
+ # GPU conversion for GGUF weirdly breaks; see ggerganov/llama.cpp#7062.
run_installer = subprocess.Popen(
full_command, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT
)
return run_installer
-# Bound the first-use auto-install so no unvetted release is pulled: not an inflated "0.999.0", nor
-# a crafted higher in-range patch like "0.12.999" from a mirror. Cap to the exact vetted patch and
-# bump deliberately. Floor 0.6.0 keeps torch>=2.4 resolvable (0.7+ need torch>=2.7; torch pinned below).
+# Cap the first-use auto-install at the exact vetted patch, so no inflated "0.999.0" or in-range "0.12.999"
+# from a mirror is pulled. Floor 0.6.0 keeps torch>=2.4 resolvable (0.7+ need torch>=2.7).
_LLM_COMPRESSOR_SPEC = "llmcompressor>=0.6.0,<=0.12.0"
-# Highest transformers release llm-compressor 0.10.x/0.12.x can run against (its metadata pins
-# transformers<=4.57.6). Models that require a newer-transformers sidecar (e.g. Qwen3.5 needs
-# transformers 5.3.0) cannot be quantized by llm-compressor at all: it imports
-# transformers.modeling_utils.TORCH_INIT_FUNCTIONS, which was removed in transformers 5.x, so the
-# compressed-export subprocess dies with a cryptic ImportError AFTER the expensive 16bit merge.
-# Detect that up front and fail fast with an actionable message. Bump this in lockstep with a
-# llm-compressor release that supports newer transformers.
+# Highest transformers llm-compressor 0.10.x/0.12.x runs against (it pins <=4.57.6). It imports
+# TORCH_INIT_FUNCTIONS, removed in transformers 5.x, so a newer-transformers model dies with a cryptic
+# ImportError AFTER the expensive merge. Bump with llm-compressor.
_LLM_COMPRESSOR_MAX_TRANSFORMERS = "4.57.6"
@@ -1587,11 +1496,8 @@ def _transformers_exceeds_llm_compressor_ceiling(transformers_version = None):
return False, str(transformers_version)
-# A caller (e.g. Unsloth Studio) can enable FP8/FP4 export of newer-transformers models (Qwen3.5,
-# Gemma-4, ...) by provisioning a dedicated llm-compressor-main "shadow" (transformers>=5.9 layered
-# over the existing torch) and pointing us at its sys.path entry via this env var. When set, the
-# quantization subprocess uses it instead of the workspace llm-compressor and the ceiling fail-fast
-# is bypassed.
+# A caller (Unsloth Studio) can point this at an llm-compressor-main shadow (transformers>=5.9 over the
+# existing torch); the subprocess then uses it and the ceiling check is bypassed.
_COMPRESSED_QUANTIZE_PYTHONPATH_ENV = "UNSLOTH_COMPRESSED_QUANTIZE_PYTHONPATH"
@@ -1650,8 +1556,8 @@ def install_llm_compressor():
except Exception:
pass
- # Prefer pip, but fall back to uv when this interpreter has no pip seeded (common in
- # uv-created / relocatable venvs), so the export does not hard-fail with "No module named pip".
+ # Prefer pip, falling back to uv when the interpreter has no pip seeded (uv-created or relocatable venvs),
+ # so the export does not die on "No module named pip".
import importlib.util
if importlib.util.find_spec("pip") is not None:
@@ -1732,8 +1638,7 @@ def try_execute(commands, force_complete = False):
def install_llama_cpp_old(version = -10):
- # Download the 10th latest release since the latest might be broken!
- # FALLBACK mechanism
+ # Download the 10th latest release since the latest might be broken; this is the fallback mechanism.
releases = subprocess.check_output(
["git", "ls-remote", "--tags", "https://github.com/ggerganov/llama.cpp.git"]
)
@@ -1745,7 +1650,6 @@ def install_llama_cpp_old(version = -10):
latest = releases[-1]
version = releases[version].split(" ")[0]
- # Check if the llama.cpp exists
if os.path.exists("llama.cpp"):
print(
"**[WARNING]** You have a llama.cpp directory which is broken.\n"
@@ -1760,19 +1664,15 @@ def install_llama_cpp_old(version = -10):
shutil.rmtree("llama.cpp", ignore_errors = True)
- # Clone a specific commit
- # Also don't use the GPU!
commands = [
"git clone --recursive https://github.com/ggerganov/llama.cpp",
f"cd llama.cpp && git reset --hard {version} && git clean -df",
]
try_execute(commands)
- # Detect CMake-only build system before trying make
use_cmake = _is_cmake_only_llama_cpp("llama.cpp")
if not use_cmake:
- # Try using MAKE
commands = [
"make clean -C llama.cpp",
f"make all -j{(psutil.cpu_count() or 1) * 2} -C llama.cpp",
@@ -1780,7 +1680,6 @@ def install_llama_cpp_old(version = -10):
use_cmake = try_execute(commands) == "CMAKE"
if use_cmake:
- # Use CMAKE
commands = [
f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}",
f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1) * 2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",
@@ -1789,7 +1688,6 @@ def install_llama_cpp_old(version = -10):
]
try_execute(commands)
- # Check if successful
if not (
os.path.exists("llama.cpp/llama-quantize.exe")
or os.path.exists("llama.cpp/llama-quantize")
@@ -1806,9 +1704,7 @@ def install_llama_cpp_old(version = -10):
def install_llama_cpp_blocking(use_cuda = False):
- # https://github.com/ggerganov/llama.cpp/issues/7062
- # Weirdly GPU conversion for GGUF breaks??
- # use_cuda = "LLAMA_CUDA=1" if use_cuda else ""
+ # GPU conversion for GGUF weirdly breaks; see ggerganov/llama.cpp#7062.
commands = [
"git clone --recursive https://github.com/ggerganov/llama.cpp",
@@ -1818,21 +1714,16 @@ def install_llama_cpp_blocking(use_cuda = False):
return
try_execute(commands)
- # Detect CMake-only build system before trying make
use_cmake = _is_cmake_only_llama_cpp("llama.cpp")
if not use_cmake:
commands = [
"make clean -C llama.cpp",
- # https://github.com/ggerganov/llama.cpp/issues/7062
- # Weirdly GPU conversion for GGUF breaks??
- # f"{use_cuda} make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp",
f"make all -j{(psutil.cpu_count() or 1) * 2} -C llama.cpp",
]
use_cmake = try_execute(commands) == "CMAKE"
if use_cmake:
- # Use CMAKE
commands = [
f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}",
f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1) * 2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",
@@ -1843,13 +1734,11 @@ def install_llama_cpp_blocking(use_cuda = False):
def get_executable(executables):
- # Get system locations (System Path).split(system separator)
system_directories = os.environ.get("PATH").split(os.pathsep)
for directory in system_directories:
for executable in executables:
path = os.path.join(directory, executable)
- # Check if the executable exists and is executable
if os.path.exists(path) and os.access(path, os.X_OK):
return path
return None
@@ -2105,17 +1994,15 @@ def save_to_gguf(
`gguf_shard_size` applies to final f32, f16 and bf16 outputs. Quantized outputs
remain single-file. None preserves the historical 50GB converter limit.
"""
- # print_output True only if UNSLOTH_ENABLE_LOGGING=1
+ # print_output is True only if UNSLOTH_ENABLE_LOGGING=1.
if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1":
print_output = True
else:
print_output = False
- # Validate model dtype
assert model_dtype == "float16" or model_dtype == "bfloat16"
model_dtype = "f16" if model_dtype == "float16" else "bf16"
- # Convert quantization_method to list
if isinstance(quantization_method, list):
pass
elif isinstance(quantization_method, str):
@@ -2127,7 +2014,6 @@ def save_to_gguf(
else:
raise TypeError("Unsloth: quantization_method can only be a string or a list of strings")
- # Check if bfloat16 is supported
if model_dtype == "bf16" and not torch.cuda.is_bf16_supported():
logger.warning(
"Unsloth: Cannot convert to bf16 GGUF since your computer doesn't support it.\n"
@@ -2137,8 +2023,8 @@ def save_to_gguf(
has_imatrix = imatrix is not None and str(imatrix) != ""
if has_imatrix:
- # quantize_gguf gained the imatrix kwarg in a recent unsloth_zoo; fail fast (before the
- # expensive conversion) if the installed version cannot apply it, rather than dropping it.
+ # quantize_gguf gained the imatrix kwarg in a recent unsloth_zoo; fail fast before the expensive
+ # conversion rather than silently dropping it.
import inspect
if "imatrix" not in inspect.signature(quantize_gguf).parameters:
raise RuntimeError(
@@ -2146,7 +2032,6 @@ def save_to_gguf(
"Please upgrade it: uv pip install --upgrade unsloth_zoo"
)
- # Map quant methods
new_quantization_methods = []
for quant_method in quantization_method:
if quant_method == "not_quantized":
@@ -2177,11 +2062,10 @@ def save_to_gguf(
new_quantization_methods.append(quant_method)
quantization_method = new_quantization_methods
- # Determine optimal first_conversion
if is_gpt_oss:
print("Unsloth: GPT-OSS model detected - using special conversion settings")
- first_conversion = "None" # No quantization for GPT-OSS
- # Only keep one conversion method since GPT-OSS doesn't quantize
+ first_conversion = "None"
+ # GPT-OSS does not quantize, so keep only one conversion method.
quantization_method = ["None"]
elif first_conversion is None:
first_conversion = _choose_first_conversion(
@@ -2190,7 +2074,6 @@ def save_to_gguf(
has_imatrix = has_imatrix,
)
- # Check bfloat16 support again for first_conversion
if first_conversion == "bf16" and not torch.cuda.is_bf16_supported():
logger.warning("Unsloth: Switching bf16 to f16 due to hardware limitations")
first_conversion = "f16"
@@ -2203,7 +2086,6 @@ def save_to_gguf(
quantization_method,
is_vlm,
)
- # Print conversion info
needs_quantize_pass = any(m != first_conversion for m in quantization_method)
if needs_quantize_pass:
second_step = f"[2] Converting GGUF {first_conversion_dtype} to {quantization_method} might take 10 minutes each."
@@ -2220,31 +2102,28 @@ def save_to_gguf(
)
print(print_info)
- # Step 1: Ensure llama.cpp is installed
try:
quantizer_location, converter_location = check_llama_cpp()
print("Unsloth: llama.cpp found in the system. Skipping installation.")
except:
print("Unsloth: Installing llama.cpp. This might take 3 minutes...")
if IS_KAGGLE_ENVIRONMENT:
- # Kaggle: no CUDA support due to environment limitations
quantizer_location, converter_location = install_llama_cpp(
gpu_support = False, print_output = print_output
)
else:
+ # Kaggle: no CUDA support due to environment limitations.
quantizer_location, converter_location = install_llama_cpp(
- gpu_support = False, # GGUF conversion doesn't need CUDA
+ gpu_support = False,
print_output = print_output,
)
- # Step 2: Download and patch converter script
print("Unsloth: Preparing converter script...")
with use_local_gguf():
converter_path, supported_text_archs, supported_vision_archs = (
_download_convert_hf_to_gguf()
)
- # Step 3: Initial GGUF conversion
print(f"Unsloth: [1] Converting model into {first_conversion_dtype} GGUF format.")
print(f"This might take 3 minutes...")
@@ -2261,14 +2140,11 @@ def save_to_gguf(
max_shard_size = converter_shard_size,
print_output = print_output,
)
- # update is_vlm switch
is_vlm = is_vlm_update
- # Check conversion success
for file in initial_files:
if not os.path.exists(file):
- # Gated like the outer handler: the disk advice is only right when
- # the disk is actually the problem, and a converter that has no
- # llama.cpp support fails here with plenty of space free.
+ # Gated like the outer handler: disk advice is only right when disk is the problem, and a
+ # converter with no llama.cpp support fails here with space to spare.
if IS_KAGGLE_ENVIRONMENT and _gguf_failure_looks_like_disk(
RuntimeError(f"Conversion produced no output at {file}"),
os.path.dirname(file) or None,
@@ -2284,7 +2160,6 @@ def save_to_gguf(
f"Unsloth: Conversion failed for {file}\nPlease check disk space and try again."
)
- # Move initial GGUF files into a dedicated _gguf directory
if gguf_directory is None:
gguf_directory = f"{model_directory}_gguf"
else:
@@ -2310,10 +2185,8 @@ def save_to_gguf(
print(f"Unsloth: Initial conversion completed! Files: {initial_files}")
- # Step 4: Additional quantizations using llama-quantize
all_saved_locations = initial_files.copy()
- # Get CPU count for quantization
n_cpus = psutil.cpu_count()
if n_cpus is None:
n_cpus = 1
@@ -2322,20 +2195,14 @@ def save_to_gguf(
if not is_gpt_oss:
base_gguf = initial_files[0]
- # Deduplicate while keeping order; methods equal to the base conversion already
- # exist on disk and need no quantize pass.
+ # Deduplicate while keeping order; a method equal to the base conversion is already on disk.
methods_to_quantize = [
m for m in dict.fromkeys(quantization_method) if m != first_conversion
]
- # The merge is not read again -- llama-quantize reads the GGUF, already
- # written and moved -- and on a disk holding merge, base GGUF and quants
- # at once those bytes are what runs it out. Only fires when the room is
- # not there, so an export that would have worked keeps the merge.
- # Nemotron-3-Nano-30B-A3B on a 132GB disk: 63GB merge + 60GB BF16 GGUF +
- # ~18GB Q4_K_M = 141GB, and llama-quantize died at tensor 88 of 401 with
- # `basic_ios::clear: iostream error`, its own `fout.exceptions(failbit)`
- # firing on a write with nowhere to go.
+ # The merge is not read again, and on a tight disk its bytes are what run the export out.
+ # Nemotron-3-Nano-30B-A3B on 132GB: 63GB merge + 60GB GGUF + 18GB Q4_K_M = 141GB, and llama-quantize
+ # died mid-write. Only fires when the room is not there.
_free_merge_if_disk_is_tight(
model_directory,
gguf_directory,
@@ -2361,9 +2228,8 @@ def save_to_gguf(
imatrix = imatrix,
)
else:
- # Use unsloth-zoo's standard quantization for all other methods. Only pass
- # imatrix when set so older unsloth_zoo (no imatrix kwarg) still works for
- # plain quants; an imatrix that cannot be applied was rejected above.
+ # Standard unsloth-zoo quantization for everything else. Pass imatrix only when set, so an
+ # older zoo still handles plain quants; an inapplicable imatrix was rejected above.
quant_kwargs = dict(
input_gguf = base_gguf,
output_gguf = output_location,
@@ -2377,13 +2243,8 @@ def save_to_gguf(
quant_kwargs["n_threads"] = n_threads
return quantize_gguf(**quant_kwargs)
except Exception as e:
- # What this pass was going to write, so "no room" is judged
- # against the output rather than a constant. Priced as a lower
- # bound, not the reclamation estimate: that one is deliberately
- # generous, and here a size that comes out high calls a disk full
- # that had the room and hides the rebuild advice below. Best
- # effort -- an unreadable base, or a type with no known width,
- # just falls back to the fixed floor.
+ # Judge "no room" against what this pass writes, not a constant. Priced as a lower bound, not
+ # the generous reclamation estimate: a high guess calls a disk full that had room.
try:
_ratio = _gguf_output_size_ratio(
quant_method,
@@ -2404,9 +2265,8 @@ def save_to_gguf(
)
except OSError:
_needed = None
- # Same gate as above: a broken quantizer with 19GB free is not
- # a disk problem, and the outer handler cannot undo an
- # explanation already baked into this message.
+ # Same gate as above: a broken quantizer with 19GB free is not a disk problem, and the outer
+ # handler cannot undo an explanation already baked into this message.
if IS_KAGGLE_ENVIRONMENT and _gguf_failure_looks_like_disk(
e,
gguf_directory,
@@ -2430,10 +2290,8 @@ def save_to_gguf(
needed_bytes = _needed,
partial_output = output_location,
):
- # Kaggle is not the only place a disk fills. The rebuild
- # advice below is only correct when the quantizer is the
- # problem; on a full disk it is a long compile that fixes
- # nothing, so say what actually happened instead.
+ # Kaggle is not the only place a disk fills, and the rebuild advice below only fits a broken
+ # quantizer; on a full disk it is a long compile that fixes nothing.
try:
_free_gb = shutil.disk_usage(gguf_directory).free / 1024**3
_where = f" ({_free_gb:.1f}GB free at {gguf_directory})"
@@ -2477,18 +2335,15 @@ def save_to_gguf(
for m in methods_to_quantize
if os.path.exists(os.path.join(gguf_directory, f"{model_name}.{m.upper()}.gguf"))
}
- # Each llama-quantize pass loads the whole base GGUF into RAM, so only run two at
- # once when the host has headroom for two copies, else a multi-quant export that
- # fit sequentially could OOM.
+ # Each llama-quantize pass loads the whole base GGUF into RAM, so run two at once only with headroom
+ # for two copies, else a multi-quant export that fit sequentially OOMs.
try:
base_bytes = sum(os.path.getsize(f) for f in initial_files if not _is_gguf_companion(f))
mem_ok = psutil.virtual_memory().available >= int(2.5 * base_bytes)
except Exception:
mem_ok = False
- # Independent llama-quantize runs on the same base GGUF can overlap. Kept at 2
- # workers; run sequentially when streaming logs (UNSLOTH_ENABLE_LOGGING), on
- # Kaggle/Colab, when RAM is tight, or when the kill switch (0/false/no/off/empty)
- # is set.
+ # Independent llama-quantize runs can overlap, capped at 2 workers. Sequential when streaming logs, on
+ # Kaggle/Colab, when RAM is tight, or when the kill switch is set.
_parallel_flag = os.environ.get("UNSLOTH_PARALLEL_GGUF_QUANTS", "1").strip().lower()
parallel_quants = (
len(methods_to_quantize) > 1
@@ -2515,14 +2370,13 @@ def save_to_gguf(
for i, method in enumerate(methods_to_quantize)
}
done, pending = wait(future_to_idx, return_when = FIRST_EXCEPTION)
- # Do not start queued passes after a failure (avoid filling the disk).
+ # Do not start queued passes after a failure, to avoid filling the disk.
for fut in pending:
fut.cancel()
first_exc = next((f.exception() for f in done if f.exception() is not None), None)
if first_exc is not None:
- # Remove only outputs this run newly created; a file that pre-dated the
- # run (or a canceled pass that never wrote) is left intact, so a rerun
- # never deletes a prior artifact. Base kept for retry.
+ # Remove only outputs this run created: a pre-existing file, or a canceled pass that never
+ # wrote, stays, so a rerun never deletes a prior artifact. The base is kept for retry.
wait(future_to_idx)
for method in methods_to_quantize:
if method in preexisting_outputs:
@@ -2554,11 +2408,11 @@ def save_to_gguf(
all_saved_locations.remove(f)
Path(f).unlink(missing_ok = True)
- # flip the list to get [text_model, mmproj] order. for text models stays the same.
+ # Flip the list to get [text_model, mmproj] order; text models stay the same.
all_saved_locations.reverse()
- # When the base format is preserved, move base files (incl. shards) away from
- # list boundaries so example commands ([0]=model, [-1]=mmproj) stay correct.
+ # When the base format is preserved, move base files (and shards) off the list boundaries so
+ # example commands ([0]=model, [-1]=mmproj) stay correct.
if want_full_precision and len(all_saved_locations) > len(base_files) + 1:
for f in base_files:
if f in all_saved_locations:
@@ -2626,10 +2480,9 @@ def unsloth_save_pretrained_merged(
"You can do it separately via `tokenizer.save_pretrained(...)`"
)
- # Kaggle's working directory is ~20GB while the overlay at /tmp on the same
- # kernel is measured in terabytes. Relative paths under /kaggle/working that
- # do not fit move there; absolute paths, hub pushes and every non-Kaggle
- # machine are untouched.
+ # Kaggle's working directory is ~20GB while /tmp on the same kernel is terabytes, so relative paths under
+ # /kaggle/working that do not fit move there. Absolute paths, hub pushes and non-Kaggle machines are
+ # untouched.
_forwards_state_dict, _writes_model_verbatim = _merge_writer_disposition(self, save_method)
save_directory = _preflight_merge_disk(
self,
@@ -2639,14 +2492,11 @@ def unsloth_save_pretrained_merged(
state_dict = state_dict,
forwards_state_dict = _forwards_state_dict,
writes_model_verbatim = _writes_model_verbatim,
- # No `writer_runs_merge_guard`: this entrypoint's writer for a plain
- # merge is `unsloth_save_model`, which merges and writes the shards
- # itself and never reaches `merge_and_overwrite_lora`. A compressed
- # export from here does go through `unsloth_generic_save`, and the
- # preflight recognises that from the method alone.
+ # No writer_runs_merge_guard: a plain merge here is written by unsloth_save_model, which never reaches
+ # merge_and_overwrite_lora. A compressed export does go through unsloth_generic_save, which the
+ # preflight recognises from the method alone.
)
- # FP8 / FP4 compressed-tensors export (llm-compressor) -> handled separately.
_compressed = _normalize_compressed_method(save_method)
if _compressed is not None:
scheme, needs_calibration, suffix = _compressed
@@ -2663,7 +2513,6 @@ def unsloth_save_pretrained_merged(
calibration_dataset = calibration_dataset,
num_calibration_samples = num_calibration_samples,
max_seq_length = max_seq_length,
- # Forward standard save kwargs to the 16bit merge.
state_dict = state_dict,
save_function = save_function,
max_shard_size = max_shard_size,
@@ -2679,7 +2528,6 @@ def unsloth_save_pretrained_merged(
gc.collect()
return
- # torchao portable FP8/INT8 export (no NVIDIA GPU) -> separate path.
_torchao = _normalize_torchao_method(save_method)
if _torchao is not None:
kind, suffix = _torchao
@@ -2692,7 +2540,6 @@ def unsloth_save_pretrained_merged(
push_to_hub = push_to_hub,
token = token,
is_main_process = is_main_process,
- # Forward standard save kwargs to the 16bit merge.
state_dict = state_dict,
save_function = save_function,
max_shard_size = max_shard_size,
@@ -2761,7 +2608,6 @@ def unsloth_push_to_hub_merged(
"You can do it separately via `tokenizer.push_to_hub(...)`"
)
- # FP8 / FP4 compressed-tensors export (llm-compressor) -> handled separately.
_compressed = _normalize_compressed_method(save_method)
if _compressed is not None:
scheme, needs_calibration, suffix = _compressed
@@ -2782,7 +2628,6 @@ def unsloth_push_to_hub_merged(
calibration_dataset = calibration_dataset,
num_calibration_samples = num_calibration_samples,
max_seq_length = max_seq_length,
- # Forward standard save kwargs to the 16bit merge.
use_temp_dir = use_temp_dir,
max_shard_size = max_shard_size,
safe_serialization = safe_serialization,
@@ -2795,7 +2640,6 @@ def unsloth_push_to_hub_merged(
gc.collect()
return
- # torchao portable FP8/INT8 export (no NVIDIA GPU) -> separate path.
_torchao = _normalize_torchao_method(save_method)
if _torchao is not None:
kind, suffix = _torchao
@@ -2813,7 +2657,6 @@ def unsloth_push_to_hub_merged(
commit_description = commit_description,
create_pr = create_pr,
revision = revision,
- # Forward standard save kwargs to the 16bit merge.
use_temp_dir = use_temp_dir,
max_shard_size = max_shard_size,
safe_serialization = safe_serialization,
@@ -2906,7 +2749,6 @@ def create_huggingface_repo(
private = private,
)
- # Create model card
from huggingface_hub import ModelCard
content = MODEL_CARD.format(
@@ -2921,7 +2763,7 @@ def create_huggingface_repo(
card.data.datasets = datasets
card.push_to_hub(save_directory, token = token)
except:
- # Repo already exists — update datasets metadata separately
+ # Repo already exists, so update datasets metadata separately.
if datasets:
try:
from huggingface_hub import metadata_update
@@ -2959,7 +2801,6 @@ def upload_to_huggingface(
private = private,
)
- # Create model card
from huggingface_hub import ModelCard
content = MODEL_CARD.format(
@@ -2974,7 +2815,6 @@ def upload_to_huggingface(
card.data.datasets = datasets
card.push_to_hub(save_directory, token = token)
except:
- # Repo already exists — update datasets metadata separately
if datasets:
try:
from huggingface_hub import metadata_update
@@ -2985,7 +2825,6 @@ def upload_to_huggingface(
)
if file_location is not None:
- # Now upload file
hf_api = HfApi(token = token)
if "/" in file_location:
@@ -2993,7 +2832,6 @@ def upload_to_huggingface(
else:
uploaded_location = file_location
- # find ftevent file from tensorboard and upload it
import glob
ftevent_files = glob.glob("*out.tfevents*", recursive = True)
@@ -3019,7 +2857,6 @@ def upload_to_huggingface(
commit_message = "(Trained with Unsloth)",
)
- # We also upload a config.json file
if create_config:
import json
@@ -3037,7 +2874,6 @@ def upload_to_huggingface(
def fix_tokenizer_bos_token(tokenizer):
- # Check if BOS added already, then warn
fix_bos_token = False
chat_template = getattr(tokenizer, "chat_template", None)
@@ -3053,11 +2889,9 @@ def fix_tokenizer_bos_token(tokenizer):
"Unsloth: ##### Your chat template has a BOS token. We shall remove it temporarily."
)
- # Remove {{bos_token}}
new_chat_template = re.sub(
r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\}[\s]{0,}\}", "", chat_template
)
- # Remove {{bos_token +
new_chat_template = re.sub(
r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\+[\s]{0,}",
"",
@@ -3086,7 +2920,7 @@ def create_ollama_modelfile(tokenizer, base_model_name, model_location):
f"Unsloth: No Ollama template mapping found for model '{base_model_name}'. Skipping Ollama Modelfile"
)
return None
- tokenizer._ollama_modelfile = ollama_modelfile # This comes from the unpacking above
+ tokenizer._ollama_modelfile = ollama_modelfile
modelfile = ollama_modelfile
FILE_LOCATION_REPLACER = "⚫@✅#🦥__FILE_LOCATION__⚡@🦥#⛵"
@@ -3094,8 +2928,7 @@ def create_ollama_modelfile(tokenizer, base_model_name, model_location):
LEFT_BRACKET_REPLACER = "⚫@✅#🦥"
RIGHT_BRACKET_REPLACER = "⚡@🦥#⛵"
- # Fixes https://github.com/unslothai/unsloth/issues/1087
- # We must convert all {'s and }'s but keep {__FILE_LOCATION__} intact
+ # Convert all {'s and }'s but keep {__FILE_LOCATION__} intact, reverted below (#1087).
modelfile = (
modelfile.replace("{__FILE_LOCATION__}", FILE_LOCATION_REPLACER)
.replace("{__EOS_TOKEN__}", EOS_TOKEN_REPLACER)
@@ -3103,7 +2936,6 @@ def create_ollama_modelfile(tokenizer, base_model_name, model_location):
.replace("}", RIGHT_BRACKET_REPLACER)
)
- # Revert {__FILE_LOCATION__} back
modelfile = modelfile.replace(FILE_LOCATION_REPLACER, "{__FILE_LOCATION__}").replace(
EOS_TOKEN_REPLACER, "{__EOS_TOKEN__}"
)
@@ -3252,12 +3084,9 @@ def _hub_cache_prewarm_disabled(disable):
os.environ[key] = previous
-# unsloth_zoo's own merge guard compares the save against `int(free * 0.95)`
-# (`saving_utils.merge_and_overwrite_lora`), and `model_16bit_bytes` counts
-# tensors only: config.json, the tokenizer files and the safetensors index are
-# written on top of them. Ask the redirect for that same effective figure, or a
-# working directory that measures "just big enough" keeps the export and the
-# merge then refuses it outright instead of using /tmp.
+# The zoo's merge guard compares against int(free * 0.95), and model_16bit_bytes counts tensors only, with
+# config, tokenizer and index on top. Ask the redirect for that same effective figure, else a "just big
+# enough" directory is kept and the merge then refuses.
_MERGE_FREE_SPACE_RESERVE = 0.95
# torchao weight-only fp8 / int8: one byte per quantized weight.
@@ -3429,11 +3258,9 @@ def _full_model_checkpoint_bytes(model, state_dict = None):
which is how the 20GB Kaggle working directory fills instead.
"""
try:
- # `is not None`, matching `unsloth_generic_save`, which forwards the
- # dict on exactly that test. An explicitly empty one therefore reaches
- # `save_pretrained` and writes no tensors at all, so sizing the resident
- # model there would price tens of gigabytes for a save that writes none
- # and move it off persistent Kaggle storage for nothing.
+ # `is not None`, matching unsloth_generic_save, which forwards the dict on that test. An empty dict
+ # reaches save_pretrained and writes no tensors, so sizing the resident model would price tens of
+ # gigabytes for a save that writes none.
if state_dict is not None:
total = 0
for tensor in state_dict.values():
@@ -3586,10 +3413,8 @@ def _warn_if_torchao_staging_filesystem_is_short(destination, staging_bytes):
if _same_filesystem(staging_directory, destination):
return
free = free_bytes(staging_directory)
- # The staging merge is written by `merge_and_overwrite_lora`, which
- # refuses it unless `free * 0.95` covers it. Comparing against the
- # bare size leaves the 5% band where the merge predictably dies and
- # this says nothing, which is the one thing it exists to prevent.
+ # The staging merge is written by merge_and_overwrite_lora, which refuses it unless free * 0.95 covers
+ # it, so the bare size leaves a 5% band where the merge dies silently.
needed = math.ceil(staging_bytes / _MERGE_FREE_SPACE_RESERVE)
if free is None or free >= needed:
return
@@ -3673,12 +3498,12 @@ def _merge_writer_disposition(model, save_method):
return True, False
if method != "merged_16bit":
return False, False
- # A PeftModel in the generic fallback writes ADAPTERS and not a checkpoint,
- # so it keeps the sizing it has rather than being priced a full model.
+ # A PeftModel in the generic fallback writes ADAPTERS, not a checkpoint, so it keeps its own sizing rather
+ # than being priced a full model.
if isinstance(model, (PeftModel, PeftModelForCausalLM)):
return False, False
try:
- # The same predicate `unsloth_save_model` dispatches on, and no other.
+ # The same predicate unsloth_save_model dispatches on, and no other.
takes_generic_fallback = not hasattr(model, "model") or not hasattr(
getattr(model, "model", None), "layers"
)
@@ -3734,56 +3559,37 @@ def _preflight_merge_disk(
"""
if push_to_hub:
return save_directory
- # `unsloth_save_model` normalizes spaces before it dispatches, so
- # "merged 16bit" is the same full merge as "merged_16bit" and has to be
- # measured as one. `_normalize_compressed_method` normalizes the same way.
+ # unsloth_save_model normalizes spaces before dispatching, so "merged 16bit" is the same full merge as
+ # "merged_16bit" and must be measured as one.
method = str(save_method).lower().strip().replace("-", "_").replace(" ", "_")
try:
- # Every compressed export keeps the 16-bit merge at `save_directory`
- # and writes a quantized sibling next to it, so all of them belong
- # here, not just mxfp4.
+ # Every compressed export keeps the 16-bit merge at save_directory and writes a quantized sibling
+ # beside it, so all of them belong here, not just mxfp4.
compressed = _normalize_compressed_method(method)
except Exception:
# An unsupported near-miss name raises later, where the message is.
return save_directory
- # The torchao portable exports are the same shape as the compressed ones
- # from here: a quantized sibling at `save_directory + "-<suffix>"`.
+ # The torchao portable exports have the same shape: a quantized sibling at save_directory + "-<suffix>".
torchao = _normalize_torchao_method(method)
- # `lora` on a model that has no adapter writes the WHOLE model instead:
- # both `unsloth_generic_save` and `unsloth_save_model` fall back to
- # `save_pretrained` there, so a full fine-tune asked for "lora" fills
- # /kaggle/working exactly like a merge. A real PeftModel writes adapters
- # only and is still skipped.
+ # "lora" on a model with no adapter writes the WHOLE model: both save paths fall back to save_pretrained,
+ # so a full fine-tune asked for "lora" fills /kaggle/working like a merge. A real PeftModel writes
+ # adapters only and is still skipped.
is_peft = isinstance(model, (PeftModel, PeftModelForCausalLM))
full_model_lora = method == "lora" and not is_peft
- # `unsloth_generic_save` only reaches for `model.state_dict()` when it was
- # given none, and a model with no adapter is saved from whatever dictionary
- # it ends up holding. A PeftModel goes to `merge_and_overwrite_lora`, which
- # takes no state dict at all, so only the non-PEFT case follows it.
+ # unsloth_generic_save reaches for model.state_dict() only when given none, so a model with no adapter is
+ # saved from whatever dict it holds; a PeftModel goes to merge_and_overwrite_lora, which takes no state dict
+ # at all.
supplied_dict = state_dict if (forwards_state_dict and not is_peft) else None
- # Same reason the dict is only followed for a non-PEFT model: a PeftModel
- # never reaches the fallback that would copy one verbatim.
+ # Same reason the dict is only followed for a non-PEFT model: a PeftModel never reaches the fallback that
+ # would copy one verbatim.
verbatim = bool(writes_model_verbatim) and not is_peft
if compressed is None and torchao is None and method != "merged_16bit" and not full_model_lora:
return save_directory
try:
- # A merge writes 2 bytes per logical parameter and no GGUF at all, so
- # this is `model_16bit_bytes` and not the GGUF export estimate: that
- # one always prices an intermediate GGUF conversion this never does.
- # The full-model fallback casts nothing, so it is sized from the
- # tensors' own dtype instead.
- #
- # A 16-bit save of a model with no adapter writes the dictionary it was
- # handed, cast, and not the resident parameters: an empty or partial one
- # writes correspondingly less, and one carrying more than the model does
- # writes more. Sizing the model there redirects a nearly empty save off
- # persistent Kaggle storage, or leaves a larger one to fill the working
- # filesystem.
- #
- # A compressed or torchao export writes that same 16-bit checkpoint
- # through the same `unsloth_generic_save`, as its staging (torchao) or
- # kept (compressed) merge, so the dictionary decides its size too and
- # not only a literal `merged_16bit` request.
+ # A merge writes 2 bytes per parameter and no GGUF, so this is model_16bit_bytes, not the GGUF
+ # estimate, which always prices an intermediate conversion. The no-adapter save writes the dictionary
+ # it was handed, cast, so an empty one writes almost nothing and sizing the model would relocate it
+ # off persistent Kaggle storage.
if full_model_lora or verbatim:
need = _full_model_checkpoint_bytes(model, state_dict)
elif supplied_dict is not None:
@@ -3792,45 +3598,26 @@ def _preflight_merge_disk(
need = model_16bit_bytes(model)
if need <= 0:
return save_directory
- # What the torchao staging directory costs on whatever filesystem
- # `tempfile` resolves to. Zero for every other export, which stages
- # nothing.
+ # What the torchao staging directory costs on whatever filesystem tempfile resolves to; zero for every
+ # other export, which stages nothing.
staging = 0
- # The lexical suffix the quantized sibling is written under, so the
- # filesystem holding it can be measured separately below.
+ # The lexical suffix the quantized sibling is written under, so its filesystem can be measured
+ # separately below.
sibling_suffix = ""
sibling_bytes = 0
- # What `merge_and_overwrite_lora` will be asked to write HERE, which is
- # the only part its `free * 0.95` guard measures. Split out from `need`
- # because the reserve belongs on this and on nothing else: charging it
- # around the whole estimate relocates an export that fits, and on
- # Kaggle that means moving it to a /tmp the kernel does not keep.
- # The guard belongs to `merge_and_overwrite_lora` and to nothing else,
- # and only `unsloth_generic_save`'s PEFT branch calls it. Every other
- # writer that lands a 16-bit checkpoint here is a bare
- # `save_pretrained`: the generic architecture fallback, the full-model
- # `lora` fallback, `unsloth_save_model`'s own merge, and
- # `unsloth_generic_save`'s no-adapter branch, which casts the dict and
- # writes it directly. None of them reserves anything, so charging them
- # 1/0.95 refuses a filesystem their writer would have accepted.
- #
- # Sizing is decided separately, above: a compressed export really is
- # cast to two bytes by `unsloth_generic_save`, so it keeps that sizing
- # whether or not there is an adapter to merge.
+ # What merge_and_overwrite_lora writes HERE, the only part its free * 0.95 guard measures. Split out
+ # from `need` because the reserve belongs on this alone: charging it around the whole estimate
+ # relocates an export that fits. Every other writer here is a bare save_pretrained and reserves
+ # nothing.
guard_runs_here = is_peft and (compressed is not None or bool(writer_runs_merge_guard))
merge_here = need if guard_runs_here else 0
if torchao is not None:
- # `_unsloth_save_torchao` merges into a `tempfile.mkdtemp` staging
- # directory rather than `save_directory`, so the only thing landing
- # here is the 8-bit sibling, and no merge guard runs against this
- # filesystem at all.
+ # _unsloth_save_torchao merges into a tempfile.mkdtemp staging dir, so only the 8-bit sibling lands
+ # here and no merge guard runs against this filesystem.
staging = need
merge_here = 0
- # No ignore list: `_unsloth_save_torchao` quantizes with a bare
- # `Float8WeightOnlyConfig()` / `Int8WeightOnlyConfig()`. Charging it
- # the compressed recipe's exclusions would over-count, and
- # over-counting relocates an export that fits into a /tmp Kaggle
- # does not keep as notebook output.
+ # No ignore list: _unsloth_save_torchao quantizes with a bare Float8/Int8WeightOnlyConfig(), and
+ # charging it the compressed recipe's exclusions would over-count and relocate an export that fits.
need = _quantized_sibling_bytes(model, need, _TORCHAO_SIBLING_WEIGHT_BITS)
sibling_suffix = torchao[1]
sibling_bytes = need
@@ -3839,16 +3626,15 @@ def _preflight_merge_disk(
model,
need,
_compressed_scheme_weight_bits(compressed[0]),
- # Everything the recipe refuses to quantize is copied across at
- # 16 bits: the vision tower, linear attention, MTP, MoE gates.
+ # Everything the recipe refuses to quantize is copied at 16 bits: vision tower, linear
+ # attention, MTP, MoE gates.
_compressed_ignore_patterns(model),
)
sibling_suffix = compressed[2]
need += sibling_bytes
- # The reserve raises the ask for the merge alone, and the rest is added
- # at face value. `max` rather than a sum of the two halves: the sibling
- # coexists with the merge, so the peak is the whole estimate, and it
- # must also clear the merge's own reserved figure.
+ # The reserve raises the ask for the merge alone; the rest is added at face value. max, not a sum: the
+ # sibling coexists with the merge, so the peak is the whole estimate and it must also clear the
+ # merge's reserved figure.
need = max(need, math.ceil(merge_here / _MERGE_FREE_SPACE_RESERVE))
new_directory, message = kaggle_tmp_redirect(
save_directory,
@@ -4173,9 +3959,8 @@ def _gguf_source_dtype(model):
if not torch.cuda.is_bf16_supported():
return "f16"
except Exception:
- # `save_to_gguf` calls this unguarded, so a raise here means no
- # export happens at all and the figure never gets used. "f16" is
- # the same width either way; only the name has to match.
+ # save_to_gguf calls this unguarded, so a raise means no export at all and the figure is never used.
+ # "f16" is the same width either way; only the name has to match.
return "f16"
return dtype
@@ -4233,27 +4018,17 @@ def _preflight_gguf_disk(
first_conversion = _choose_first_conversion(
methods, model_dtype, has_imatrix = has_imatrix
)
- # `save_to_gguf` drops a bf16 initial conversion to f16 on hardware
- # with no bf16, AFTER it has resolved one, so the drop has to be
- # applied after both branches above and not only inside the resolver.
- # `_gguf_source_dtype` covers the dtype this is TOLD; it cannot cover
- # a `first_conversion` the caller passed, nor the single direct-convert
- # method `_choose_first_conversion` returns unchanged.
- #
- # It is the requested-output rule that makes this cost a checkpoint:
- # the estimate omits an output EQUAL to the initial conversion, so
- # `["bf16"]` at first_conversion "bf16" is priced as one 16-bit file,
- # while a T4 writes an f16 intermediate AND a separate bf16 output.
- # 15.3GB unaccounted for on Qwen3-8B, the same figure and the same
- # mechanism as the dtype case.
+ # save_to_gguf drops a bf16 initial conversion to f16 on hardware without bf16 AFTER resolving one, so
+ # the drop must be applied after both branches, not only in the resolver. The estimate omits an output
+ # equal to the initial conversion, so ["bf16"] at first_conversion "bf16" was priced as one file while
+ # a T4 writes f16 plus bf16: 15.3GB missing on Qwen3-8B.
if first_conversion == "bf16":
try:
if not torch.cuda.is_bf16_supported():
first_conversion = "f16"
except Exception:
- # `save_to_gguf` asks unguarded, so a raise here means no
- # export runs and this figure is never used. f16 is the wider
- # of the two readings, and never the narrower.
+ # save_to_gguf asks unguarded, so a raise means no export runs and this is never used. f16 is
+ # the wider of the two readings, never the narrower.
first_conversion = "f16"
need = estimate_gguf_export_bytes(
model = model,
@@ -4261,11 +4036,9 @@ def _preflight_gguf_disk(
first_conversion = first_conversion,
needs_merge = needs_merge,
)
- # The pre-warm only happens on the merge path, and only when it has
- # not already been switched off. Kaggle and Colab return before it
- # ever runs (`_prewarm_base_model_hub_cache`), so pricing a cache copy
- # there would send an export that fits in /kaggle/working to /tmp,
- # which is not kept as notebook output.
+ # The pre-warm runs only on the merge path and only while enabled. Kaggle and Colab return before it,
+ # so pricing a cache copy there sends an export that fits in /kaggle/working to a /tmp that is not
+ # kept as notebook output.
prewarm_possible = (
needs_merge
and not IS_KAGGLE_ENVIRONMENT
@@ -4284,19 +4057,15 @@ def _preflight_gguf_disk(
if prewarm_possible
else need
)
- # The estimate prices the checkpoint at two bytes per parameter. The
- # non-PEFT fallback writes it at the model's own dtype, so an fp32
- # model needs the difference on top. Zero for every 16-bit model.
+ # The estimate prices the checkpoint at 2 bytes per parameter, but the non-PEFT fallback writes the
+ # model's own dtype, so an fp32 model needs the difference. Zero for a 16-bit model.
if need > 0 and needs_merge:
extra = _fallback_checkpoint_extra_bytes(model)
need += extra
need_with_cache += extra
- # The same estimate without the checkpoint: what lands in the `_gguf`
- # sibling directory, which is the intermediate conversion plus every
- # quant. Only used when that sibling turns out to be on a different,
- # smaller filesystem than `save_directory`. Its own try, so an
- # estimator that cannot answer this leaves the main guard standing
- # rather than turning the whole preflight off.
+ # The same estimate without the checkpoint: the `_gguf` sibling's intermediate plus every quant. Used
+ # only when that sibling sits on a smaller filesystem. Its own try, so an estimator that cannot answer
+ # leaves the main guard standing.
try:
need_sibling = estimate_gguf_export_bytes(
model = model,
@@ -4306,12 +4075,8 @@ def _preflight_gguf_disk(
)
except Exception:
need_sibling = 0
- # The first of the two phases a disposable merge really peaks at: the
- # merge and the intermediate GGUF on disk together, before any quant
- # is written. `quantization_methods = ()` still prices the
- # intermediate, which is what the conversion has to write before it
- # can be read. Its own try for the same reason as the sibling: an
- # estimator that cannot answer leaves the aggregate standing.
+ # The first phase a disposable merge peaks at: merge plus intermediate GGUF, before any quant.
+ # quantization_methods = () still prices the intermediate, which must exist to be read.
try:
need_merge_phase = estimate_gguf_export_bytes(
model = model,
@@ -4321,12 +4086,10 @@ def _preflight_gguf_disk(
) + (extra if need > 0 and needs_merge else 0)
except Exception:
need_merge_phase = 0
- # Reclamation only helps when a quantize pass actually follows it, and
- # `save_to_gguf` skips a method equal to the initial conversion because
- # that file is already on disk. Same rule, same list.
+ # Reclamation only helps when a quantize pass follows, and save_to_gguf skips a method equal to the
+ # initial conversion, whose file is already on disk.
has_quantize_pass = bool([m for m in dict.fromkeys(methods) if m != first_conversion])
- # The intermediate conversion on its own, for the filesystem it is
- # written to before the move. Same shape of `try` as the two above.
+ # The intermediate conversion on its own, for the filesystem it is written to before the move.
try:
need_conversion = estimate_gguf_export_bytes(
model = model,
@@ -4337,20 +4100,29 @@ def _preflight_gguf_disk(
except Exception:
need_conversion = 0
except Exception:
- # Sizing is best effort. A failure here must not stop an export that
- # would otherwise have worked.
+ # Sizing is best effort: a failure here must not stop an export that would otherwise have worked.
return save_directory, True
if need <= 0:
return save_directory, True
- # The redirect has to be asked for the same figure the refusal below reads.
- # `need_with_cache` is the aggregate, and a disposable merge is reclaimed
- # before the quants are written, so an export whose phased peak fits in
- # /kaggle/working was relocated to a /tmp Kaggle does not keep as notebook
- # output. Same predicates as the branch that lowers `need_here`, read
- # against the directory as it stands BEFORE any move, and it can only ever
- # lower the ask: nothing that used to stay put is relocated now.
+ # Ask the redirect for the same figure the refusal reads. need_with_cache is the aggregate, but a
+ # disposable merge is reclaimed before the quants, so a phased peak that fits in /kaggle/working was
+ # relocated. Read before any move, and it can only lower the ask.
phased_need = 0
+ # The intermediate conversion is written to the process CWD, not into `save_directory` and not into the
+ # `_gguf` sibling, and only afterwards moved. When that CWD is on its own filesystem, nothing above has
+ # measured the disk the largest staging artefact actually lands on: on Kaggle it is the 20GB working
+ # directory the redirect just moved the export away from.
+ # The intermediate conversion is written to the working directory and only moved to the sibling afterwards,
+ # so when that working directory is on THIS filesystem the checkpoint and the conversion are on it together.
+ # Charging each of them alone lets a 60GB checkpoint and a 60GB conversion both pass on 100GB, and then it
+ # fills. Only the split branch needs this: on one filesystem the aggregate already counts them both, and a
+ # conversion sharing the sibling's disk is counted in `need_sibling`. `max` and not `+`, because the two
+ # figures are two PHASES and not two artefacts. The merge runs first and its guard wants the checkpoint over
+ # 0.95 with nothing else written yet; the conversion is written after, against the unreserved checkpoint.
+ # Summing them charges the reserve on top of a conversion that does not exist while the guard runs: a 60GB
+ # merge and a 60GB conversion on 122GB free clear both phases (63.2GB, then 120GB) and the sum asks 123.2GB
+ # and refuses.
if (
merge_is_disposable
and needs_merge
@@ -4372,18 +4144,10 @@ def _preflight_gguf_disk(
need_bytes = redirect_need,
what = "GGUF export",
)
- # Asked the aggregate because THIS directory already holds weights the
- # reclamation may not touch, and declined. The relocation writes the merge
- # into a directory of the export's own, where it can be reclaimed, so the
- # move is worth a second ask at the phased peak: 63GB merge, 60GB
- # intermediate, 18GB Q4_K_M asks 141GB, a 130GB /tmp declines it, and the
- # 123GB peak it would really have peaked at there fits.
- #
- # Gated on this directory already being too small for the aggregate the
- # refusal below will read, which is what makes the lower ask safe. A lower
- # ask can otherwise CANCEL a move: /kaggle/working with room for the peak
- # but not the aggregate keeps an export that has no reclamation here, and
- # then the refusal reads the aggregate anyway.
+ # Asked the aggregate and declined, because this directory already holds weights reclamation may not
+ # touch. After the move the merge is reclaimable, so re-ask at the phased peak: 141GB aggregate declines a
+ # 130GB /tmp, but the real 123GB peak fits. Gated on this directory being too small for the aggregate,
+ # else a lower ask could cancel the move for nothing.
if message is None and 0 < phased_need < redirect_need:
free_before_move = free_bytes(save_directory)
if free_before_move is not None and free_before_move < need:
@@ -4398,17 +4162,9 @@ def _preflight_gguf_disk(
free = free_bytes(save_directory)
- # The quants and the intermediate GGUF are written to a SIBLING of
- # `save_directory`, so they live on the parent's filesystem. That is the
- # same disk `free` just measured unless `save_directory` is itself a mount
- # point or a symlink onto another one. When it is, each filesystem is
- # charged only for what is written to it: the checkpoint here, the
- # conversion and the quants there. Charging the aggregate to either one
- # refuses a split export whose every artefact has room.
- #
- # One predicate, computed once, so the two halves cannot disagree about
- # whether the storage is split, and an unmeasurable path leaves every
- # decision below exactly as it was.
+ # The quants and intermediate GGUF go to a SIBLING of save_directory, on the parent's filesystem, which is
+ # the disk just measured unless save_directory is a mount or symlink. When split, charge each filesystem
+ # only what it holds. One predicate, computed once, so the two halves cannot disagree.
gguf_directory = _gguf_output_directory(save_directory)
gguf_free = free_bytes(gguf_directory)
separate_storage = (
@@ -4416,34 +4172,24 @@ def _preflight_gguf_disk(
and gguf_free is not None
and _on_separate_filesystems(save_directory, gguf_directory)
)
- # Resolved before the split is priced, because where the conversion lands
- # decides which filesystem it is charged to.
+ # Resolved before the split is priced, because where the conversion lands decides which filesystem it is
+ # charged to.
conversion_directory = _gguf_conversion_directory(
_gguf_model_input_directory(model, save_directory)
)
- # Cleared where the base-model cache turns out to share a filesystem that
- # has room for what the export writes there but not for a cached base as
- # well: dropping the optional half beats failing the export. The message
- # travels with the flag because more than one filesystem can now set it,
- # and each has to name the one it measured.
+ # Cleared when the cache shares a filesystem with room for the export but not a cached base too: dropping
+ # the optional half beats failing. The message travels with the flag, since more than one filesystem can
+ # set it.
sibling_prewarm_ok = True
prewarm_drop_message = None
- # Resolved once, above the split, because the cache is not necessarily on
- # either of the two filesystems the split is about: the conversion's own
- # filesystem below asks the same question of the same path. Zero whenever
- # the pre-warm cannot run, which is what leaves every branch below inert.
+ # Resolved once, above the split, since the cache need not be on either filesystem. Zero whenever the pre-
+ # warm cannot run, which leaves every branch below inert.
cache_extra = max(0, need_with_cache - need)
cache_directory = _hub_cache_directory() if cache_extra > 0 else None
- # The cache copy lands wherever the cache is, which is not necessarily the
- # filesystem holding `save_directory` - `HF_HOME` on a data volume is the
- # ordinary layout on a machine with more than one disk. Charging it here
- # anyway drops the pre-warm on a disk that had room for everything written
- # to it, and the next export downloads the whole base again, which is the
- # exact re-download the pre-warm exists to stop. It only ever LOWERS the
- # figure, and only the pre-warm decision reads it, so no export that fit
- # before is refused now. Unresolvable leaves it charged here, where it was
- # charged before. The split branch asks the same of the sibling below.
+ # The cache lands wherever HF_HOME points, not necessarily save_directory's disk. Charging it here drops
+ # the pre-warm on a disk that had room, and the next export re-downloads the base. It only LOWERS the
+ # figure and only the pre-warm reads it.
cache_here = cache_extra
if (
cache_extra > 0
@@ -4454,33 +4200,15 @@ def _preflight_gguf_disk(
need_here = need
need_here_with_cache = need + cache_here
if separate_storage:
- # `need_sibling` is the same estimate without the checkpoint, so the
- # difference is the checkpoint and nothing else. An estimator that
- # could not size the sibling leaves it 0, and then this is `need`:
- # the previous behaviour, not a relaxed one.
- #
- # Nothing else, though, includes the slack: both terms carry the
- # estimator's flat `DISK_SLACK_BYTES` and the subtraction cancels it,
- # leaving two bytes per parameter exactly. `merge_and_overwrite_lora`
- # refuses that merge unless `free * 0.95` covers it, so the same
- # reserve `_preflight_merge_disk` already applies has to be applied
- # here too, or this passes an export the merge kills seconds later.
- # The aggregate branch needs no such thing: it charges the quants as
- # well, which is already more than the checkpoint plus its reserve.
+ # need_sibling is this estimate without the checkpoint, so the difference is the checkpoint alone; an
+ # unsizable sibling leaves it 0, giving the previous behaviour. Both terms carry DISK_SLACK_BYTES and
+ # it cancels, so merge_and_overwrite_lora's free * 0.95 reserve must be re-applied or this passes an
+ # export the merge kills seconds later.
checkpoint_here = max(0, need - need_sibling)
need_here = checkpoint_here
- # The cache copy is not part of the merge and is not what the zoo
- # guard measures, so it rides on top of the checkpoint rather than
- # being reserved itself.
- #
- # It also does not necessarily land HERE. `save_directory` being a
- # mount onto another disk is the whole premise of this branch, and
- # then `~/.cache` and the lexical `_gguf` sibling both stay on the
- # parent disk. Charging the checkpoint's filesystem for bytes written
- # to the sibling's drops a pre-warm that had room and, worse, lets the
- # sibling check accept `need_sibling` alone on a filesystem the base
- # model is about to be downloaded onto. `cache_here` above has already
- # answered the first half; the sibling is the half only this branch has.
+ # The cache copy is not part of the merge and not what the zoo guard measures, so it rides on top of
+ # the checkpoint. It need not land HERE either: with save_directory mounted elsewhere, ~/.cache and
+ # the _gguf sibling stay on the parent disk.
cache_sibling = 0
if (
cache_extra > 0
@@ -4491,45 +4219,20 @@ def _preflight_gguf_disk(
need_here_with_cache = checkpoint_here + cache_here
writes_a_lora_merge = isinstance(model, (PeftModel, PeftModelForCausalLM))
if writes_a_lora_merge and needs_merge and need_sibling > 0 and checkpoint_here > 0:
- # Four conditions, each removing a way to charge for a guard that
- # will not run. Only a PEFT model reaches `merge_and_overwrite_lora`
- # and its `free * 0.95`: a non-PEFT model with no reusable local
- # checkpoint still needs `needs_merge`, because the GGUF path has
- # to write one, but it is written by a bare `self.save_pretrained`
- # that reserves nothing, so charging the reserve there refuses a
- # filesystem with room for the checkpoint the writer would have
- # accepted. `needs_merge = False` writes no merge at all. A sibling
- # the estimator could not size leaves `checkpoint_here` equal to
- # the aggregate, which is a fallback figure and not a checkpoint
- # to reserve against. And `min(need, ...)` keeps the promise the
- # split made when it was introduced: it may cancel a redirect,
- # never cause a refusal the aggregate would have allowed.
+ # Four conditions, each dropping a charge for a guard that will not run: only a PEFT model reaches
+ # merge_and_overwrite_lora's free * 0.95, a bare self.save_pretrained reserves nothing,
+ # needs_merge = False writes no merge, and an unsizable sibling leaves a fallback figure.
+ # min(need, ...) keeps the split's promise: cancel a redirect, never cause a refusal.
reserved = min(need, math.ceil(checkpoint_here / _MERGE_FREE_SPACE_RESERVE))
need_here = max(need_here, reserved)
- # The cache copy is written BEFORE the merge and is still there when
- # the guard runs, so it comes off the free space the guard measures.
- # It rides on top of the reserved figure rather than beside it: a
- # 16GB checkpoint with a 14GB cache on 30.5GB free passes a `max`
- # asking 30GB, and then the merge sees 16.5GB and refuses 16GB under
- # its own 5%. Added instead, this band drops the pre-warm, which is
- # the optional half, rather than failing the export.
+ # The cache copy is written BEFORE the merge and is still there when the guard runs, so it comes
+ # off the free space the guard sees. Riding on top of the reserved figure rather than beside it:
+ # 16GB checkpoint + 14GB cache on 30.5GB passes a max of 30GB and then the merge refuses under its
+ # own 5%. Added instead, this drops the pre-warm, not the export.
need_here_with_cache = max(need_here_with_cache, reserved + cache_here)
- # The intermediate conversion is written to the working directory and
- # only moved to the sibling afterwards, so when that working directory
- # is on THIS filesystem the checkpoint and the conversion are on it
- # together. Charging each of them alone lets a 60GB checkpoint and a
- # 60GB conversion both pass on 100GB, and then it fills. Only the
- # split branch needs this: on one filesystem the aggregate already
- # counts them both, and a conversion sharing the sibling's disk is
- # counted in `need_sibling`.
- #
- # `max` and not `+`, because the two figures are two PHASES and not two
- # artefacts. The merge runs first and its guard wants the checkpoint
- # over 0.95 with nothing else written yet; the conversion is written
- # after, against the unreserved checkpoint. Summing them charges the
- # reserve on top of a conversion that does not exist while the guard
- # runs: a 60GB merge and a 60GB conversion on 122GB free clear both
- # phases (63.2GB, then 120GB) and the sum asks 123.2GB and refuses.
+ # The intermediate conversion lands in the working directory before the move, so on THIS filesystem
+ # the checkpoint and conversion coexist and charging each alone lets 60GB + 60GB pass on 100GB. max,
+ # not +: they are two phases, and summing them refuses 122GB free that clears both.
if (
need_conversion > 0
and conversion_directory is not None
@@ -4539,11 +4242,9 @@ def _preflight_gguf_disk(
need_here_with_cache = max(
need_here_with_cache, checkpoint_here + need_conversion + cache_here
)
- # Not gated on the sibling being the TIGHTER of the two any more.
- # Now that the checkpoint is charged only its own portion, a sibling
- # with more free space than `save_directory` and still less than
- # `need_sibling` has to be refused here, because the aggregate
- # comparison that used to catch it no longer exists.
+ # No longer gated on the sibling being the TIGHTER of the two: now that the checkpoint is charged only
+ # its own portion, a sibling roomier than save_directory but under need_sibling must be refused here,
+ # since the aggregate comparison is gone.
if gguf_free < need_sibling:
raise RuntimeError(
f"Unsloth: Not enough disk space to convert to GGUF.\n"
@@ -4572,49 +4273,30 @@ def _preflight_gguf_disk(
and need_sibling > 0
and _merge_reclamation_is_possible(save_directory)
):
- # `_free_merge_if_disk_is_tight` deletes this export's own merge once
- # the intermediate GGUF exists and before the first quantize pass, so
- # the three artefacts never coexist: the peak is the larger of "merge
- # plus intermediate" and "intermediate plus every quant", not their
- # sum. Nemotron-3-Nano-30B-A3B is the case in front of it - 63GB merge,
- # 60GB BF16 intermediate, 18GB Q4_K_M - which peaks at 123GB on a 132GB
- # disk and which the aggregate 141GB refuses.
- #
- # Only in the single-filesystem branch, because the reclamation itself
- # declines when the merge and the GGUF output are on different devices:
- # freeing bytes on one filesystem does nothing for a quantize pass
- # writing to another.
+ # _free_merge_if_disk_is_tight deletes this export's merge once the intermediate exists, so the three
+ # artefacts never coexist and the peak is the larger of the two phases, not their sum:
+ # Nemotron-3-Nano-30B-A3B peaks at 123GB where the 141GB aggregate refuses a 132GB disk. Single
+ # filesystem only, since reclamation declines across devices.
peak = max(need_merge_phase, need_sibling)
if peak < need:
- # Can only ever lower the figure, so no input that head allowed is
- # refused here. The cache copy is not what gets reclaimed, so it
- # rides on top of the peak exactly as it rode on top of the sum.
+ # Can only lower the figure, so nothing the head allowed is refused here. The cache copy is not
+ # reclaimed, so it rides on top of the peak as it did on the sum.
need_here = peak
need_here_with_cache = peak + cache_here
- # The intermediate conversion is written to the process CWD, not into
- # `save_directory` and not into the `_gguf` sibling, and only afterwards
- # moved. When that CWD is on its own filesystem, nothing above has measured
- # the disk the largest staging artefact actually lands on: on Kaggle it is
- # the 20GB working directory the redirect just moved the export away from.
+ # The intermediate conversion is written to the process CWD and only then moved, so when that CWD is its
+ # own filesystem nothing above has measured the disk the largest staging artefact lands on -- on Kaggle,
+ # the 20GB working directory the redirect just left.
if (
need_conversion > 0
and conversion_directory is not None
and _on_separate_filesystems(conversion_directory, gguf_directory)
):
conversion_free = free_bytes(conversion_directory)
- # The pre-warm runs BEFORE the merge and leaves the base model in the
- # Hugging Face cache for the rest of the export, so when the cache
- # shares this filesystem the cached base and the intermediate are on
- # it together. Nothing above has charged that: `cache_here` and
- # `cache_sibling` only ever place the cache on the checkpoint's
- # filesystem or the sibling's, and this is neither.
- #
- # The pre-warmer's own gate does not cover it either. It asks for two
- # base copies free on the cache filesystem, and an `f32` conversion is
- # two base copies on its own: 38.1GB free clears the pre-warm's 30.5GB
- # threshold and this 30.5GB conversion check, then holds a 15.3GB
- # cached base while writing 30.5GB, and runs out 7.6GB short.
+ # The pre-warm leaves the base model cached for the rest of the export, so on a shared filesystem the
+ # cached base and the intermediate coexist, which cache_here and cache_sibling never charge. The pre-
+ # warmer's own gate asks for two base copies, and an f32 conversion is two copies on its own: 38.1GB
+ # free clears both checks and runs out 7.6GB short.
cache_with_conversion = (
cache_extra
if cache_directory is not None
@@ -4641,9 +4323,8 @@ def _preflight_gguf_disk(
and cache_with_conversion > 0
and conversion_free < need_conversion + cache_with_conversion
):
- # Only reached once the conversion alone has cleared the raise
- # above, so this can never turn a refusal into a pass: it drops
- # the optional half of an export that otherwise fits.
+ # Only reached once the conversion alone cleared the raise above, so this cannot turn a refusal into
+ # a pass: it drops the optional half of an export that otherwise fits.
sibling_prewarm_ok = False
prewarm_drop_message = (
f"Unsloth: Skipping the Hugging Face cache pre-warm - the Hugging Face "
@@ -4833,7 +4514,7 @@ def unsloth_save_pretrained_gguf(
save_directory = os.path.normpath(os.fspath(save_directory))
gguf_shard_size = _resolve_gguf_shard_size(gguf_shard_size)
- # save_method="lora" exports the adapter itself as a GGUF LoRA (not a merged model).
+ # save_method="lora" exports the adapter itself as a GGUF LoRA, not a merged model.
if save_method is not None and str(save_method).lower() == "lora":
if not is_main_process:
return None
@@ -4857,8 +4538,8 @@ def unsloth_save_pretrained_gguf(
_outtype = "f16"
return _unsloth_save_lora_gguf(self, tokenizer, save_directory, outtype = _outtype)
- # base_model_name keeps the full id for create_ollama_modelfile's mapper lookup;
- # only the filename stem is trimmed.
+ # base_model_name keeps the full id for create_ollama_modelfile's mapper lookup; only the filename stem is
+ # trimmed.
base_model_name = getattr(getattr(self, "config", None), "_name_or_path", None)
try:
base_model_name = get_model_name(base_model_name, load_in_4bit = False)
@@ -4866,23 +4547,20 @@ def unsloth_save_pretrained_gguf(
pass
model_name = _model_basename(base_model_name)
- # Check if push_to_hub is requested
if push_to_hub:
raise ValueError(
"Unsloth: Please use .push_to_hub_gguf() instead of .save_pretrained_gguf() with push_to_hub=True"
)
- # Step 1: Check if this is a VLM (Vision-Language Model) and check if gpt-oss
is_vlm = _is_vlm(self)
is_processor = is_vlm and isinstance(tokenizer, ProcessorMixin)
is_gpt_oss = _is_gpt_oss(self)
- # Step 1b: Will this fit? Ask before the merge, not after. Runs here rather
- # than lower down because `arguments` below snapshots locals(), so a
- # redirected `save_directory` has to be in place first. gpt-oss takes the
- # mxfp4 route instead of the merge/convert/quantize one this sizes.
+ # Will this fit? Ask before the merge. Runs here because `arguments` below snapshots locals(), so a
+ # redirected save_directory must be in place first. gpt-oss takes the mxfp4 route, not the
+ # merge/convert/quantize one this sizes.
_gguf_prewarm_ok = True
if not is_gpt_oss:
save_directory, _gguf_prewarm_ok = _preflight_gguf_disk(
@@ -4890,24 +4568,20 @@ def unsloth_save_pretrained_gguf(
save_directory = save_directory,
quantization_method = quantization_method,
first_conversion = first_conversion,
- # Resolved here rather than left at the default, because the
- # default says "f16" and the export asks the config.
+ # Resolved rather than left at the default, which says "f16" while the export asks the config.
model_dtype = _gguf_source_dtype(self),
has_imatrix = _imatrix_is_enabled(imatrix_file),
needs_merge = _gguf_writes_16bit_checkpoint(self),
- # The same flag `save_to_gguf` reclaims on. Where a non-PEFT model
- # reuses its own checkpoint the flag is cleared below, and there
- # `_gguf_writes_16bit_checkpoint` is already False on the same
- # condition, so the two cannot disagree.
+ # The same flag save_to_gguf reclaims on. Where a non-PEFT model reuses its own checkpoint the
+ # flag is cleared below on the same condition, so the two cannot disagree.
merge_is_disposable = merge_is_disposable,
)
- # Step 2: Prepare arguments for model saving
arguments = dict(locals())
arguments["model"] = self
arguments["tokenizer"] = tokenizer
arguments["push_to_hub"] = False # We handle upload ourselves
- # GPT-OSS needs mxfp4 save method
+ # GPT-OSS needs the mxfp4 save method.
if is_gpt_oss:
if quantization_method is not None:
_qm = (
@@ -4939,30 +4613,22 @@ def unsloth_save_pretrained_gguf(
del arguments["merge_is_disposable"] # decides reclamation, not how the merge is written
del arguments["gguf_shard_size"] # only used by the gguf converter
- # Preserve the requested output before reusing a non-PEFT checkpoint as input.
- # Same definition the preflight sized, so the disk it measured is the one
- # these files land on.
+ # Preserve the requested output before reusing a non-PEFT checkpoint as input. Same definition the
+ # preflight sized, so it measured the disk these files land on.
gguf_directory = _gguf_output_directory(save_directory)
- # Step 3: Fix tokenizer BOS token if needed
if is_processor:
fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer.tokenizer)
else:
fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer)
- # Resolve the importance matrix (download upstream / validate path / rename *.gguf_file) up
- # front, so a bad path or an unavailable upstream imatrix fails before the expensive 16-bit
- # merge, and a failed auto-resolution never reaches the IQ-quant gate.
+ # Resolve the imatrix (download, validate, rename *.gguf_file) up front, so a bad path or an unavailable
+ # upstream fails before the expensive merge and never reaches the IQ-quant gate.
imatrix_path = _resolve_imatrix_file(self, imatrix_file, token, save_directory)
- # Who owns what, settled before a byte is written. Reclamation may only take
- # files this export produced, and that is not a question the directory can
- # answer afterwards: a caller reusing an output directory can already hold a
- # finished sharded save, index and every shard it names, self-consistent and
- # entirely present. transformers removes neither (its stale sweep does not
- # match `model.safetensors.index`, and it only prunes shards under the stem it
- # is writing), so after the merge that set is indistinguishable from one this
- # run wrote. Recorded here instead, where the difference still exists.
+ # Settle ownership before a byte is written: reclamation may only take files this export made, and
+ # afterwards the directory cannot say, since a reused output directory can already hold a finished sharded
+ # save transformers neither removes nor distinguishes from ours.
try:
preexisting_weights = frozenset(os.listdir(save_directory))
except FileNotFoundError:
@@ -4972,16 +4638,13 @@ def unsloth_save_pretrained_gguf(
# Provenance unreadable. Proving nothing, the reclamation takes nothing.
preexisting_weights = None
- # Step 4: Save/merge model to 16-bit format
is_peft_model = isinstance(self, PeftModelForCausalLM) or isinstance(self, PeftModel)
- # The flag holds for both branches that write the weights themselves; the middle
- # branch reuses the user's own checkpoint instead and clears it, whatever the
- # caller asked for.
+ # The flag holds for both branches that write the weights; the middle branch reuses the user's own
+ # checkpoint and clears it, whatever the caller asked for.
if is_peft_model:
print(f"Unsloth: Merging model weights to {'mxfp4' if is_gpt_oss else '16-bit'} format...")
try:
- # Call unsloth_generic_save directly (it's in the same file)
with _hub_cache_prewarm_disabled(not _gguf_prewarm_ok):
unsloth_generic_save(**arguments)
@@ -4991,26 +4654,28 @@ def unsloth_save_pretrained_gguf(
f"{_offloaded_parameter_hint(self)}"
) from e
else:
- # Non-PEFT model: checkpoint files already exist; point save_to_gguf
- # at the original path instead of re-saving to a temp subdir.
+ # Non-PEFT model: the checkpoint already exists, so point save_to_gguf at the original path instead of
+ # re-saving into a temp subdir.
original_path = getattr(self.config, "_name_or_path", None)
if original_path and os.path.isdir(original_path):
print(
f"Unsloth: Model is not a PEFT model. Using existing checkpoint at {original_path}"
)
save_directory = original_path
- # The user's own checkpoint, not an intermediate: without this an
- # ordinary save_pretrained_gguf on a tight disk would delete the
- # model it was handed.
+ # The user's own checkpoint, not an intermediate: without this a save_pretrained_gguf on a tight
+ # disk would delete the model it was handed.
merge_is_disposable = False
- # Persist tokenizer fixes (e.g. BOS token stripping) to disk
- # so the GGUF converter picks up the corrected chat template.
+ # Persist tokenizer fixes (BOS stripping) so the GGUF converter reads the corrected template.
+ # Persist tokenizer fixes (e.g. BOS token stripping) to disk so the GGUF converter picks up the
+ # corrected chat template.
if tokenizer is not None:
tokenizer.save_pretrained(save_directory)
else:
- # Fallback: save the in-memory model to save_directory
print("Unsloth: Model is not a PEFT model. Saving directly without LoRA merge...")
os.makedirs(save_directory, exist_ok = True)
+ # `gguf_directory` can point anywhere, and freeing bytes on one filesystem does nothing for a
+ # quantize pass writing to another: without this the merge could be deleted for a destination it
+ # cannot help, data gone and the export still out of space.
try:
self.save_pretrained(save_directory)
if tokenizer is not None:
@@ -5024,18 +4689,15 @@ def unsloth_save_pretrained_gguf(
if is_processor:
tokenizer = tokenizer.tokenizer
- # Use old chat template if the bos is removed
if fix_bos_token:
tokenizer.chat_template = old_chat_template
- # Step 6: Clean up memory
for _ in range(3):
import gc
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
- # Step 7: Get model dtype and type
try:
model_dtype = dtype_from_config(self.config)
model_type = self.config.model_type
@@ -5048,18 +4710,13 @@ def unsloth_save_pretrained_gguf(
else:
raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")
except Exception as e:
- # Fallback if dtype_from_config fails
print(f"Unsloth: Could not determine dtype ({e}), defaulting to float16")
model_dtype = "float16"
- # Step 8: Convert to GGUF format
print("Unsloth: Converting to GGUF format...")
- # Convert quantization_method to list if string
- # Use old style quantization_method
quantization_methods = []
if quantization_method is not None:
- # Convert quantization_method to list
if isinstance(quantization_method, list):
pass
elif isinstance(quantization_method, str):
@@ -5129,7 +4786,6 @@ def unsloth_save_pretrained_gguf(
else:
raise RuntimeError(f"Unsloth: GGUF conversion failed: {_describe_exception(e)}") from e
- # Step 9: Create Ollama modelfile
modelfile_location = None
ollama_success = False
if all_file_locations:
@@ -5150,7 +4806,6 @@ def unsloth_save_pretrained_gguf(
except Exception as e:
print(f"Warning: Could not create Ollama modelfile: {e}")
- # Step 10: Show BOS token warning if applicable
if fix_bos_token:
logger.warning(
"Unsloth: ##### The current model auto adds a BOS token.\n"
@@ -5181,7 +4836,6 @@ def unsloth_save_pretrained_gguf(
f"Unsloth: convert model to ollama format by running - ollama create model_name -f {modelfile_location}"
)
- # Return a dict with all needed info for push_to_hub
return {
"save_directory": save_directory,
"gguf_directory": gguf_directory,
@@ -5203,13 +4857,12 @@ _DISK_FULL_PATTERNS = (
"write failed: no space",
)
-# Kaggle allows 20GB. Below this headroom a failed conversion is plausibly
-# about space; above it, blaming disk sends the user nowhere useful.
+# Kaggle allows 20GB. Below this headroom a failed conversion is plausibly about space; above it, blaming disk
+# sends the user nowhere useful.
_DISK_HEADROOM_BYTES = 2 * 1024**3
-# A SIGKILLed child is 128 + 9 to a shell, so llama-quantize (run through
-# `shell = True`) surfaces as "returned non-zero exit status 137" with no
-# signal named anywhere in the text.
+# A SIGKILLed child is 128 + 9 to a shell, so llama-quantize under shell = True surfaces as "returned non-zero
+# exit status 137" with no signal named.
_OOM_KILL_PATTERNS = (
"sigkill",
"exit status 137",
@@ -5300,10 +4953,9 @@ def _gguf_failure_looks_like_disk(
return True
if getattr(exc, "errno", None) == 28:
return True
- # The output directory holds the file, so the first path that answers decides
- # and the working directory is only a fallback. A roomy output disk called
- # full because some unrelated filesystem is short would blame the disk for a
- # quantizer failure and hide the advice that would have fixed it.
+ # The output directory holds the file, so it decides and the working directory is only a fallback: calling
+ # a roomy output disk full because an unrelated filesystem is short hides the advice that would have fixed
+ # the quantizer failure.
threshold = needed_bytes if needed_bytes and needed_bytes > 0 else _DISK_HEADROOM_BYTES
written, written_device = 0, None
if partial_output:
@@ -5322,8 +4974,8 @@ def _gguf_failure_looks_like_disk(
# Never let the diagnostic be the thing that raises.
continue
if written:
- # Only on the filesystem that actually holds the partial file --
- # bytes on one device are not room on another.
+ # Only on the filesystem that actually holds the partial file: bytes on one device are not room on
+ # another.
try:
if os.stat(path).st_dev == written_device:
free += written
@@ -5333,14 +4985,13 @@ def _gguf_failure_looks_like_disk(
return False
-# The full-precision output types. Everything else llama-quantize writes is
-# quantized and carries its nominal width as the leading digit of the name
-# (`q4_k_m`, `iq3_xxs`).
+# The full-precision output types; everything else llama-quantize writes is quantized and carries its nominal
+# width as the leading digit of the name.
_GGUF_BITS_PER_WEIGHT = {"f32": 32.0, "f16": 16.0, "bf16": 16.0}
-# k- and i-quants store block scales and mins beside the weights, so a type is
-# wider than its name. llama.cpp's published 7B sizes keep that overhead under a
-# bit a weight (Q4_K_M near 4.5, Q6_K near 6.6), so 1.5 bounds every type here.
+# k- and i-quants store block scales and mins beside the weights, so a type is wider than its
+# name; llama.cpp's 7B sizes keep that under a bit per weight (Q4_K_M near 4.5, Q6_K near 6.6),
+# so 1.5 bounds every type.
_QUANT_OVERHEAD_BITS = 1.5
_QUANT_NOMINAL_BITS = re.compile(r"^i?q(\d+)")
@@ -5385,38 +5036,25 @@ def _gguf_output_size_ratio(
base = _gguf_type_bits(first_conversion) or 16.0
target = _gguf_type_bits(quant_method)
if target is None:
- # Unrecognised: charge a whole copy of the base, as every method used to
- # get -- or, for a diagnosis, admit the size is unknown.
+ # Unrecognised: charge a whole copy of the base, as every method used to, or for a diagnosis admit the
+ # size is unknown.
return 1.0 if upper_bound else None
if upper_bound and str(quant_method).lower() not in _GGUF_BITS_PER_WEIGHT:
target += _QUANT_OVERHEAD_BITS
return target / base
-# The names a disposable merge is written under, and only those. Both writers of
-# one produce safetensors: the PEFT branch goes through unsloth_zoo's
-# `merge_and_overwrite_lora`, which rewrites the base safetensors shards (and the
-# `consolidated.safetensors` some repos ship), and the non-PEFT fallback calls
-# `save_pretrained` with no arguments, so transformers' `SAFE_WEIGHTS_NAME`.
-#
-# Neither the stem nor the extension is decoration. transformers clears stale
-# shards only when both hold (`modeling_utils`:
-# `filename.startswith(weights_no_suffix)` and `r"(.*?)-\d{5}-of-\d{5}"`), so a
-# shard set under another stem, and a `pytorch_model.bin` from an earlier save in
-# the other serialization, are files `save_pretrained` neither writes nor removes.
-# This helper deletes permanently, so it leaves both alone. A merge under a name
-# this misses is still reclaimed whenever it wrote an index, because the index
-# names its own shards.
+# The names a disposable merge is written under, and only those: both writers emit safetensors. Neither stem
+# nor extension is decoration, since transformers clears stale shards only when both match, so another stem or
+# an older .bin is a file save_pretrained never writes or removes -- and this helper deletes permanently.
_MERGE_WEIGHT_NAME = re.compile(r"^(model|consolidated)(-\d{5}-of-\d{5})?\.safetensors$")
-# The index `save_pretrained` writes for a sharded save. Safetensors only, for
-# the same reason the matcher is: a `pytorch_model.bin.index.json` belongs to an
-# earlier save in the other serialization, which transformers leaves in place,
-# and reading it would hand its shards to the deletion.
+# The index save_pretrained writes for a sharded save. Safetensors only, like the matcher: a
+# pytorch_model.bin.index.json belongs to an earlier save transformers leaves in place, and reading it would
+# hand its shards to the deletion.
_WEIGHT_INDEX_NAMES = ("model.safetensors.index.json",)
-# One shard of a sharded save, under any stem. Only ever applied to names an
-# index already listed -- on its own this is far too wide, and dropping it from
-# the matcher above is exactly what stopped a user's `backup-00001-of-00002`
-# being read as the merge.
+# One shard of a sharded save, under any stem. Applied only to names an index already listed: alone it is far
+# too wide, and dropping it from the matcher is what stopped a user's backup-00001-of-00002 being read as the
+# merge.
_INDEX_SHARD_NAME = re.compile(r"^(?P<stem>.+)-(?P<part>\d{5})-of-(?P<total>\d{5})\.safetensors$")
@@ -5444,14 +5082,9 @@ def _merge_weight_files(model_directory, names):
continue
if _is_one_whole_shard_set(listed, names):
indexed.update(listed)
- # The index goes with the shards it named. `names` is already
- # filtered to what this export wrote, so reaching here means this
- # export wrote this index too, and leaving it behind is not neutral:
- # it points at files that are about to be deleted, and on the next
- # export into the same directory the provenance snapshot calls that
- # leftover preexisting and filters it out, taking with it the only
- # way a shard set under a stem `_MERGE_WEIGHT_NAME` misses is ever
- # found again.
+ # The index goes with the shards it named, and `names` is already filtered to this export's files.
+ # Leaving it behind points at deleted files, and the next export's snapshot calls it preexisting,
+ # losing the only way a shard set under a missed stem is found again.
spent_indexes.add(index_name)
return sorted(
n for n in names if n in indexed or n in spent_indexes or _MERGE_WEIGHT_NAME.match(n)
@@ -5487,8 +5120,7 @@ def _is_one_whole_shard_set(listed, names):
totals.add(shard.group("total"))
if len(stems) != 1 or len(totals) != 1:
return False
- # `of-000NN` states the count outright, so a listing missing shards -- or
- # naming extra ones -- is not the set it claims to be.
+ # "of-000NN" states the count, so a listing missing or adding shards is not the set it claims.
if len(listed) != int(totals.pop()):
return False
return listed <= set(names)
@@ -5519,10 +5151,8 @@ def _free_merge_if_disk_is_tight(
"""
if not merge_is_disposable:
return 0
- # The other half of the safety story. `merge_is_disposable` says the export
- # wrote this directory; this says which files in it the export wrote. A
- # caller that cannot say gets no reclamation rather than a guess, because
- # the guess is permanent.
+ # merge_is_disposable says the export wrote this directory; this says which files it wrote. A caller that
+ # cannot say gets no reclamation rather than a guess, since the guess is permanent.
if preexisting_weights is None:
return 0
if not model_directory or not os.path.isdir(model_directory):
@@ -5531,8 +5161,8 @@ def _free_merge_if_disk_is_tight(
if not quant_methods:
return 0
try:
- # llama-quantize copies companions rather than quantizing them, so they
- # are excluded from the output and memory estimates.
+ # llama-quantize copies companions rather than quantizing them, so they are excluded from the output
+ # and memory estimates.
base_bytes = sum(
os.path.getsize(f)
for f in initial_files
@@ -5544,18 +5174,16 @@ def _free_merge_if_disk_is_tight(
return 0
target_directory = gguf_directory or model_directory
- # `gguf_directory` can point anywhere, and freeing bytes on one filesystem
- # does nothing for a quantize pass writing to another: without this the merge
- # could be deleted for a destination it cannot help, data gone and the export
- # still out of space.
+ # gguf_directory can point anywhere, and freeing bytes on one filesystem does nothing for a pass writing
+ # to another: the merge would be deleted for a destination it cannot help.
try:
if os.stat(model_directory).st_dev != os.stat(target_directory).st_dev:
return 0
except OSError:
return 0
- # Every output stays on disk, so the passes add up. Overestimating costs a
- # deletion that was not strictly required; underestimating costs the export.
+ # Every output stays on disk, so the passes add up. Overestimating costs an unnecessary deletion;
+ # underestimating costs the export.
needed = (
base_bytes * sum(_gguf_output_size_ratio(m, first_conversion) for m in quant_methods)
+ _DISK_HEADROOM_BYTES
@@ -5573,12 +5201,9 @@ def _free_merge_if_disk_is_tight(
except OSError:
# Unreadable is no reason to fail an export that no longer needs it.
return 0
- # Anything that was already here is the caller's, whatever it is named. This
- # is what stops a reused output directory losing a finished sharded save, or
- # a `consolidated.safetensors` the merge never wrote: both are answers the
- # names alone cannot give, since they are exactly the names a merge uses.
- # It also drops a stale index out of the reading below, so the shards it
- # names are never inherited.
+ # Anything already here is the caller's, whatever it is named: that is what stops a reused directory
+ # losing a finished sharded save or a consolidated.safetensors. It also drops a stale index from the
+ # reading below, so its shards are never inherited.
names = [name for name in names if name not in preexisting_weights]
for name in _merge_weight_files(model_directory, names):
path = os.path.join(model_directory, name)
@@ -5661,7 +5286,7 @@ def unsloth_push_to_hub_gguf(
if not is_main_process:
return None
- # save_method="lora" exports the adapter itself as a GGUF LoRA (not a merged model).
+ # save_method="lora" exports the adapter itself as a GGUF LoRA, not a merged model.
if save_method is not None and str(save_method).lower() == "lora":
_qm = quantization_method
if isinstance(_qm, (list, tuple)) and len(_qm) == 1:
@@ -5690,7 +5315,6 @@ def unsloth_push_to_hub_gguf(
revision = revision,
)
- # Step 1: Determine save directory
model_name = repo_id.split("/")[-1] if "/" in repo_id else repo_id
if use_temp_dir or use_temp_dir is None:
@@ -5700,14 +5324,12 @@ def unsloth_push_to_hub_gguf(
save_directory = temp_dir
cleanup_temp = True
else:
- save_directory = model_name # Use model name, not repo_id
+ save_directory = model_name
cleanup_temp = False
- # Step 2: Call save_pretrained_gguf to do the conversion
print(f"Unsloth: Converting model to GGUF format...")
try:
- # Call save_pretrained_gguf - it returns all the info we need
result = unsloth_save_pretrained_gguf(
self = self,
save_directory = save_directory,
@@ -5725,7 +5347,6 @@ def unsloth_push_to_hub_gguf(
gguf_shard_size = gguf_shard_size,
)
- # Extract results
all_file_locations = result["gguf_files"]
modelfile_location = result["modelfile_location"]
want_full_precision = result["want_full_precision"]
@@ -5742,7 +5363,6 @@ def unsloth_push_to_hub_gguf(
pass
raise RuntimeError(f"Failed to convert model to GGUF: {_describe_exception(e)}") from e
- # Step 3: Upload to HuggingFace Hub
print("Unsloth: Uploading GGUF to Huggingface Hub...")
try:
@@ -5750,14 +5370,12 @@ def unsloth_push_to_hub_gguf(
api = HfApi(token = token)
- # Get full repo id
if "/" not in repo_id:
username = api.whoami()["name"]
full_repo_id = f"{username}/{repo_id}"
else:
full_repo_id = repo_id
- # Create repo
api.create_repo(
repo_id = full_repo_id,
repo_type = "model",
@@ -5765,12 +5383,9 @@ def unsloth_push_to_hub_gguf(
exist_ok = True,
)
- # Upload GGUF files
for file_location in all_file_locations:
original_name = os.path.basename(file_location)
- # Replace temp directory name with proper model name
if cleanup_temp and "unsloth_gguf_" in original_name:
- # Extract the quantization part (e.g., ".Q8_0.gguf" or ".Q8_0-mmproj.gguf")
quant_suffix = (
original_name.split(".", 1)[1] if "." in original_name else original_name
)
@@ -5791,7 +5406,6 @@ def unsloth_push_to_hub_gguf(
revision = revision,
)
- # Upload config.json if exists
config_path = os.path.join(actual_save_directory, "config.json")
if os.path.exists(config_path):
print("Uploading config.json...")
@@ -5805,7 +5419,6 @@ def unsloth_push_to_hub_gguf(
revision = revision,
)
- # Upload Modelfile if exists
if modelfile_location and os.path.exists(modelfile_location):
print("Uploading Ollama Modelfile...")
api.upload_file(
@@ -5818,7 +5431,6 @@ def unsloth_push_to_hub_gguf(
revision = revision,
)
- # Create and upload README
readme_content = f"""---
tags:
- gguf
@@ -5838,7 +5450,6 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
## Available Model files:
"""
for file in all_file_locations:
- # Fix filename in README too
original_name = os.path.basename(file)
if cleanup_temp and "unsloth_gguf_" in original_name:
quant_suffix = (
@@ -5849,7 +5460,6 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
proper_name = original_name.replace(os.path.basename(save_directory), model_name)
readme_content += f"- `{proper_name}`\n"
- # Special note for VLM with Modelfile
if is_vlm and modelfile_location:
readme_content += "\n## ⚠️ Ollama Note for Vision Models\n"
readme_content += "**Important:** Ollama currently does not support separate mmproj files for vision models.\n\n"
@@ -5889,7 +5499,6 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
print(f"Unsloth: Successfully uploaded GGUF to https://huggingface.co/{full_repo_id}")
- # Add tags
if tags is None:
tags = []
tags.extend(["gguf", "llama-cpp", "unsloth"])
@@ -5918,7 +5527,6 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
raise RuntimeError(f"Failed to upload to Hugging Face Hub: {_describe_exception(e)}") from e
finally:
- # Clean up temporary directory
if cleanup_temp:
print("Unsloth: Cleaning up temporary files...")
for d in [save_directory, f"{save_directory}_gguf"]:
@@ -5931,12 +5539,9 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
return full_repo_id
-# Corrected function to save LoRA to a custom directory
def save_lora_to_custom_dir(model, tokenizer, save_directory):
- # Create the custom directory if it doesn't exist
os.makedirs(save_directory, exist_ok = True)
- # Call the unsloth_save_model function with the custom directory
unsloth_save_model(
model,
tokenizer,
@@ -5973,8 +5578,8 @@ def _lora_base_model_id(model):
return os.fspath(base) if base else ""
-# Upstream Unsloth GGUF repos ship a calibration imatrix under one of these names; the GGUF-format
-# one is suffixed .gguf_file so the Hub does not list it as a model GGUF (renamed to .gguf locally).
+# Upstream Unsloth GGUF repos ship a calibration imatrix under one of these names; the GGUF one is suffixed
+# .gguf_file so the Hub does not list it as a model, and renamed locally.
_IMATRIX_UPSTREAM_NAMES = ("imatrix_unsloth.dat", "imatrix_unsloth.gguf_file")
@@ -5997,8 +5602,8 @@ def _gguf_repo_candidates(model):
pass
if not name:
continue
- # The upstream imatrix lives in unsloth/<base>-GGUF, so map any org (e.g. meta-llama/...)
- # onto the unsloth org; keep an already-formed -GGUF id as-is.
+ # The upstream imatrix lives in unsloth/<base>-GGUF, so map any org onto unsloth; an already-formed
+ # -GGUF id is kept as-is.
repo = name if name.endswith("-GGUF") else f"unsloth/{name.split('/')[-1]}-GGUF"
if repo not in candidates:
candidates.append(repo)
@@ -6037,8 +5642,8 @@ def _resolve_imatrix_file(model, imatrix_file, token, dest_dir):
f"(got {type(imatrix_file).__name__})."
)
- # imatrix_file=True: auto-resolve from the upstream Unsloth GGUF repo. HfApi is the module-level
- # import (save.py top); hf_hub_download is imported here as it is not needed elsewhere.
+ # imatrix_file=True auto-resolves from the upstream Unsloth GGUF repo; hf_hub_download is imported here
+ # since nothing else needs it.
from huggingface_hub import hf_hub_download
if token is None:
@@ -6096,7 +5701,7 @@ def _unsloth_save_lora_gguf(
if token is None:
token = get_token()
- # Resolve the dequantized base id (the adapter usually references a 4bit repo).
+ # Resolve the dequantized base id, since the adapter usually references a 4bit repo.
base_model_id = _lora_base_model_id(model)
if not base_model_id:
raise RuntimeError(
@@ -6109,6 +5714,8 @@ def _unsloth_save_lora_gguf(
pass
model_name = _model_basename(base_model_id)
+ # Save the adapter: an isolated temp dir for a hub push, else save_directory itself, wrapped so it is always
+ # cleaned up.
# Save the adapter; for a hub push use an isolated temp dir, else save_directory itself.
if push_to_hub:
lora_dir = tempfile.mkdtemp(prefix = "unsloth-lora-gguf-")
@@ -6120,12 +5727,13 @@ def _unsloth_save_lora_gguf(
try:
save_lora_to_custom_dir(model, tokenizer, lora_dir)
- # Ensure a full llama.cpp checkout (ships convert_lora_to_gguf.py) and locate the converter.
+ # Ensure a full llama.cpp checkout, which ships convert_lora_to_gguf.py: a prebuilt install or a
+ # reused CWD copy carries binaries but not the script.
install_llama_cpp(just_clone_repo = True)
converter = os.path.join(LLAMA_CPP_DEFAULT_DIR, "convert_lora_to_gguf.py")
if not os.path.exists(converter):
- # A prebuilt llama.cpp install (or a reused CWD copy) carries binaries but not the
- # converter script, so force a dedicated source checkout that ships it.
+ # A prebuilt llama.cpp install (or a reused CWD copy) carries binaries but not the converter script,
+ # so force a dedicated source checkout that ships it.
source_dir = os.path.join(
os.path.dirname(os.path.normpath(LLAMA_CPP_DEFAULT_DIR)), "llama.cpp-source"
)
@@ -6144,8 +5752,8 @@ def _unsloth_save_lora_gguf(
cmd += ["--base", base_model_id]
else:
cmd += ["--base-model-id", base_model_id]
- # Only pass --trust-remote-code when the loaded model actually came from custom code (the
- # approved load decision), not merely because its config carries an auto_map entry.
+ # Pass --trust-remote-code only when the model really came from custom code (the approved load
+ # decision), not merely because its config carries an auto_map entry.
if _loaded_via_remote_code(model):
cmd.append("--trust-remote-code")
@@ -6156,6 +5764,8 @@ def _unsloth_save_lora_gguf(
env["HUGGING_FACE_HUB_TOKEN"] = token
print(f"Unsloth: Converting LoRA adapter at '{lora_dir}' to GGUF -> '{out_gguf}'")
+ # Resolve the cache from the live env like the merge, not huggingface_hub's frozen constants: a runtime
+ # cache redirect (read-only default, Unsloth) would else miss (#6890).
try:
with subprocess.Popen(
cmd,
@@ -6257,9 +5867,8 @@ from .models.loader_utils import (
_tokenizer_wants_local_only,
)
-# Imported lazily at the two call sites below: a zoo older than the one that made
-# its own bitsandbytes import optional would otherwise break `import unsloth` on a
-# host without bnb, which is the whole point of the guards above.
+# Imported lazily at the two call sites: an older zoo, before its bitsandbytes import became optional, would
+# otherwise break `import unsloth` on a host without bnb.
from unsloth_zoo.llama_cpp import (
install_llama_cpp,
convert_to_gguf as _convert_to_gguf,
@@ -6320,8 +5929,8 @@ def _prewarm_base_model_hub_cache(
)
if not model_name or is_local_path:
return
- # Mirror the merge: an FP8 base with a 16bit sibling merges onto the sibling, so
- # pre-warm the sibling (what the merge downloads), not the FP8 repo (#6890).
+ # Mirror the merge: an FP8 base with a 16bit sibling merges onto the sibling, so pre-warm the sibling
+ # rather than the FP8 repo (#6890).
if base_is_quantized and quant_type == "fp8" and save_method == "merged_16bit":
try:
from unsloth_zoo.saving_utils import _resolve_fp8_16bit_sibling
@@ -6339,8 +5948,8 @@ def _prewarm_base_model_hub_cache(
from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download
- # Resolve the cache from the live env like the merge, not huggingface_hub's frozen
- # constants: a runtime cache redirect (read-only default, Unsloth) would else miss (#6890).
+ # Resolve the cache from the live env like the merge, not huggingface_hub's frozen constants, or a
+ # runtime cache redirect misses (#6890).
try:
from unsloth_zoo.hf_cache import _active_caches
_hub_cache = _active_caches()[1]
@@ -6348,8 +5957,8 @@ def _prewarm_base_model_hub_cache(
except Exception:
hub_cache_dir = None
- # Mirror the zoo's shard listing (drop consolidated.safetensors when proper
- # shards coexist) so the cached set is a superset of what the merge looks up.
+ # Mirror the zoo's shard listing (drop consolidated.safetensors when real shards coexist) so the
+ # cached set is a superset of what the merge looks up.
shard_names = []
total_size_in_bytes = 0
for x in HfFileSystem(token = token).ls(model_name, detail = True):
@@ -6373,9 +5982,8 @@ def _prewarm_base_model_hub_cache(
except Exception:
pass
- # Mirror the merge's index filter (download path only): some repos ship leftover shards
- # the index omits; keep only indexed ones, else the disk gate over-counts and we fetch
- # unused shards.
+ # Mirror the merge's index filter on the download path: some repos ship shards the index omits, so
+ # keep only indexed ones or the disk gate over-counts.
if len(shard_names) > 1:
try:
import json as _json
@@ -6398,7 +6006,7 @@ def _prewarm_base_model_hub_cache(
pass
total_size_in_bytes = sum(size for _, size in shard_names)
- # The cache copy is extra disk on top of the merge working copy; need room for both.
+ # The cache copy is extra disk on top of the merge working copy; there must be room for both.
from huggingface_hub import constants as _hf_constants
# abspath so a relative HF_HUB_CACHE walks up to an existing root, not "".
@@ -6460,10 +6068,8 @@ def save_to_gguf_generic(
if not os.path.exists(os.path.join("llama.cpp", "unsloth_convert_hf_to_gguf.py")):
install_llama_cpp(just_clone_repo = True)
- # Use old style quantization_method
new_quantization_methods = []
if quantization_method is not None:
- # Convert quantization_method to list
if isinstance(quantization_method, list):
pass
elif isinstance(quantization_method, str):
@@ -6490,7 +6096,6 @@ def save_to_gguf_generic(
new_quantization_methods.append(quant_method.lower())
else:
new_quantization_methods.append(quantization_type.lower())
- # Check if wrong method
for quant_method in new_quantization_methods:
if quant_method not in ALLOWED_QUANTS.keys():
error = f"Unsloth: Quant method = [{quant_method}] not supported. Choose from below:\n"
@@ -6498,8 +6103,6 @@ def save_to_gguf_generic(
error += f"[{key}] => {value}\n"
raise RuntimeError(error)
- # Go through all types and save individually - somewhat inefficient
- # since we save F16 / BF16 multiple times
for quantization_type in new_quantization_methods:
metadata = _convert_to_gguf(
save_directory,
@@ -6546,7 +6149,6 @@ def unsloth_generic_save(
safe_serialization: bool = True,
variant: Optional[str] = None,
save_peft_format: bool = True,
- # Push to hub
use_temp_dir: Optional[bool] = None,
commit_message: Optional[str] = "Trained with Unsloth",
private: Optional[bool] = None,
@@ -6554,7 +6156,6 @@ def unsloth_generic_save(
revision: str = None,
commit_description: str = "Upload model trained with Unsloth 2x faster",
tags: List[str] = None,
- # Our functions
temporary_location: str = "_unsloth_temporary_saved_buffers",
maximum_memory_usage: float = 0.9,
datasets: Optional[List[str]] = None,
@@ -6575,8 +6176,8 @@ def unsloth_generic_save(
elif save_method == "merged_4bit_forced":
save_method = "merged_4bit"
- # Full-finetuned models (no LoRA) have no adapters to merge, so fall back
- # to save_pretrained, mirroring the torchao and GGUF save paths.
+ # Full-finetuned models have no adapters to merge, so fall back to save_pretrained, mirroring the torchao
+ # and GGUF save paths.
_is_peft = isinstance(model, PeftModel)
if not _is_peft:
if not is_main_process:
@@ -6709,25 +6310,23 @@ def unsloth_generic_save_pretrained_merged(
"You can do it separately via `tokenizer.save_pretrained(...)`"
)
- # Kaggle's working directory is ~20GB while the overlay at /tmp on the same
- # kernel is measured in terabytes. Relative paths under /kaggle/working that
- # do not fit move there; absolute paths, hub pushes and every non-Kaggle
- # machine are untouched.
+ # Kaggle's working directory is ~20GB while /tmp on the same kernel is terabytes, so relative paths under
+ # /kaggle/working that do not fit move there. Absolute paths, hub pushes and non-Kaggle machines are
+ # untouched.
save_directory = _preflight_merge_disk(
self,
save_directory,
save_method,
push_to_hub = push_to_hub,
state_dict = state_dict,
- # `unsloth_generic_save` writes a supplied dictionary rather than the
- # resident model when there is no adapter to merge.
+ # unsloth_generic_save writes a supplied dictionary rather than the resident model when there is no
+ # adapter, and it alone runs merge_and_overwrite_lora when there is one.
forwards_state_dict = True,
- # And it is the writer that runs `merge_and_overwrite_lora` when there
- # IS one, which no other entrypoint here does.
+ # And it is the writer that runs `merge_and_overwrite_lora` when there IS one, which no other entrypoint
+ # here does.
writer_runs_merge_guard = True,
)
- # FP8 / FP4 compressed-tensors export (llm-compressor) -> handled separately.
_compressed = _normalize_compressed_method(save_method)
if _compressed is not None:
scheme, needs_calibration, suffix = _compressed
@@ -6744,7 +6343,6 @@ def unsloth_generic_save_pretrained_merged(
calibration_dataset = calibration_dataset,
num_calibration_samples = num_calibration_samples,
max_seq_length = max_seq_length,
- # Forward standard save kwargs to the 16bit merge.
state_dict = state_dict,
save_function = save_function,
max_shard_size = max_shard_size,
@@ -6760,7 +6358,6 @@ def unsloth_generic_save_pretrained_merged(
gc.collect()
return
- # torchao portable FP8/INT8 export (no NVIDIA GPU) -> separate path.
_torchao = _normalize_torchao_method(save_method)
if _torchao is not None:
kind, suffix = _torchao
@@ -6773,7 +6370,6 @@ def unsloth_generic_save_pretrained_merged(
push_to_hub = push_to_hub,
token = token,
is_main_process = is_main_process,
- # Forward standard save kwargs to the 16bit merge.
state_dict = state_dict,
save_function = save_function,
max_shard_size = max_shard_size,
@@ -6840,7 +6436,6 @@ def unsloth_generic_push_to_hub_merged(
"You can do it separately via `tokenizer.push_to_hub(...)`"
)
- # FP8 / FP4 compressed-tensors export (llm-compressor) -> handled separately.
_compressed = _normalize_compressed_method(save_method)
if _compressed is not None:
scheme, needs_calibration, suffix = _compressed
@@ -6861,7 +6456,6 @@ def unsloth_generic_push_to_hub_merged(
calibration_dataset = calibration_dataset,
num_calibration_samples = num_calibration_samples,
max_seq_length = max_seq_length,
- # Forward standard save kwargs to the 16bit merge.
use_temp_dir = use_temp_dir,
max_shard_size = max_shard_size,
safe_serialization = safe_serialization,
@@ -6874,7 +6468,6 @@ def unsloth_generic_push_to_hub_merged(
gc.collect()
return
- # torchao portable FP8/INT8 export (no NVIDIA GPU) -> separate path.
_torchao = _normalize_torchao_method(save_method)
if _torchao is not None:
kind, suffix = _torchao
@@ -6892,7 +6485,6 @@ def unsloth_generic_push_to_hub_merged(
commit_description = commit_description,
create_pr = create_pr,
revision = revision,
- # Forward standard save kwargs to the 16bit merge.
use_temp_dir = use_temp_dir,
max_shard_size = max_shard_size,
safe_serialization = safe_serialization,
@@ -6929,9 +6521,7 @@ def _unsloth_save_torchao_with_attached_config(
token: Optional[Union[str, bool]] = None,
):
"""Save a QAT-trained model by converting fake-quantized weights to real quantized weights."""
- # Convert QAT fake-quantized weights to real quantized weights
_convert_torchao_model(model)
- # PEFT models also might come here, so parse it
if isinstance(model, PeftModelForCausalLM):
_unsloth_save_torchao_with_given_config(
model = model,
@@ -6943,7 +6533,7 @@ def _unsloth_save_torchao_with_attached_config(
)
return
- # TorchAO does not support safe_serialization reliably
+ # TorchAO does not support safe_serialization reliably.
safe_serialization = False
if push_to_hub:
@@ -6977,7 +6567,6 @@ def _unsloth_save_torchao_with_given_config(
torchao_config is not None
), "Unsloth: Please specify a torchao_config for post-training quantization!"
- # first merge the lora weights
arguments = dict(locals())
arguments["push_to_hub"] = False # We save ourselves
arguments["save_method"] = "merged_16bit" # Must be 16bit
@@ -7006,7 +6595,6 @@ def _unsloth_save_torchao_with_given_config(
else:
quantization_config = TorchAoConfig(quant_type = torchao_config)
- # Determine if this is a VLM
is_vlm = _is_vlm(model)
auto_model = AutoModelForImageTextToText if is_vlm else AutoModelForCausalLM
auto_processor = AutoProcessor if is_vlm else AutoTokenizer
@@ -7015,14 +6603,13 @@ def _unsloth_save_torchao_with_given_config(
if isinstance(tokenizer, (PreTrainedTokenizerBase, ProcessorMixin)):
tokenizer = patch_saving_functions(tokenizer)
- # TorchAO must only use bfloat16 for loading (float16 fails)
+ # TorchAO must only use bfloat16 for loading; float16 fails.
if HAS_TORCH_DTYPE:
kwargs = {"torch_dtype": torch.bfloat16}
else:
kwargs = {"dtype": torch.bfloat16}
- # Else the original stays resident on every GPU while device_map="auto" below
- # loads a second copy.
+ # Else the original stays resident on every GPU while device_map="auto" below loads a second copy.
model_restore = _offload_model_for_quantize_subprocess(model)
for _ in range(3):
gc.collect()
@@ -7031,10 +6618,11 @@ def _unsloth_save_torchao_with_given_config(
if hasattr(torch, "xpu") and torch.xpu.is_available():
torch.xpu.empty_cache()
- # The original stays offloaded until the quantized copy is saved AND released,
- # else both are resident at once and the restore OOMs.
+ # The original stays offloaded until the quantized copy is saved AND released, else both are resident and
+ # the restore OOMs.
+ # The original stays offloaded until the quantized copy is saved AND released, else both are resident at
+ # once and the restore OOMs.
try:
- # Reload with quantization applied
quantized_model = auto_model.from_pretrained(
save_directory,
device_map = "auto",
@@ -7044,7 +6632,7 @@ def _unsloth_save_torchao_with_given_config(
torchao_save_directory = save_directory + "-torchao"
- # TorchAO does not support safe_serialization right now 0.14.0 seems broken!
+ # TorchAO does not support safe_serialization right now; 0.14.0 seems broken.
safe_serialization = Version(importlib_version("torchao")) > Version("0.14.0")
safe_serialization = False
@@ -7060,12 +6648,12 @@ def _unsloth_save_torchao_with_given_config(
tokenizer.save_pretrained(torchao_save_directory, token = token)
finally:
- # del here, not at the end of the try: if save_pretrained raises, the copy
- # would otherwise still be resident while the original is restored.
+ # del here, not at the end of the try: if save_pretrained raises, the copy would still be resident
+ # while the original is restored.
quantized_model = None
del quantized_model
- # A failed save leaves a live traceback whose frames still hold the copy, so
- # dropping the local alone does not free its VRAM.
+ # A failed save leaves a live traceback whose frames hold the copy, so dropping the local alone does
+ # not free its VRAM.
_exc = sys.exc_info()[1]
if _exc is not None:
traceback.clear_frames(_exc.__traceback__)
@@ -7077,7 +6665,6 @@ def _unsloth_save_torchao_with_given_config(
torch.xpu.empty_cache()
_restore_model_after_quantize_subprocess(model, model_restore)
- # Clean up the intermediate unquantized model
if os.path.exists(save_directory):
try:
shutil.rmtree(save_directory)
@@ -7167,18 +6754,16 @@ def _snapshot_dispatch_state(root):
for name, mod in root.named_modules()
if "_hf_hook" in mod.__dict__
]
- # remove_duplicate=False: the default hides one half of every tied pair, exactly
- # the half that needs re-tying below.
+ # remove_duplicate=False: the default hides one half of every tied pair, exactly the half that needs re-
+ # tying below.
named = list(root.named_parameters(remove_duplicate = False)) + list(
root.named_buffers(remove_duplicate = False)
)
places = {name: tensor.device for name, tensor in named}
- # Tied weights share one storage, but the CPU round trip repoints every tensor and
- # tied_params_map is keyed on the old pointer, so replaying the hooks alone gives
- # independent copies: double VRAM, and updates to one no longer reach the other.
- # Skip meta tensors: offloaded parameters all sit on meta with pointer 0, which
- # would collapse into one fake "tied" group of differently shaped tensors, and
- # each side of a tie is already its own meta placeholder so nothing is lost.
+ # Tied weights share one storage, but the CPU round trip repoints every tensor while tied_params_map is
+ # keyed on the old pointer, so replaying hooks alone yields independent copies: double VRAM and divergent
+ # updates. Skip meta tensors, since offloaded parameters all have pointer 0 and would collapse into one
+ # fake tied group.
groups = {}
for name, tensor in named:
if tensor.device.type == "meta":
@@ -7187,18 +6772,17 @@ def _snapshot_dispatch_state(root):
if ptr:
groups.setdefault(ptr, []).append(name)
ties = [names for names in groups.values() if len(names) > 1]
- # Removing a hook restores `forward = _old_forward`, captured before unsloth patched
- # the module, so a remove/re-add permanently drops every fused kernel installed after
- # the dispatch (measured: apply_lora_mlp_swiglu on all 28 MLPs). It also deletes the
- # `to`/`cuda`/... move guards, so record those too.
+ # Removing a hook restores forward = _old_forward, captured before unsloth patched the module, so a
+ # remove/re-add permanently drops every fused kernel installed after the dispatch (measured on all 28
+ # MLPs) and the to/cuda move guards. Record those too.
attrs = ("forward", "_old_forward") + tuple(_ACCELERATE_MOVE_GUARDS)
saved_attrs = {
name: {a: mod.__dict__[a] for a in attrs if a in mod.__dict__}
for name, mod in root.named_modules()
if any(a in mod.__dict__ for a in attrs)
}
- # Re-adding a hook runs init_hook -> set_module_tensor_to_device, which builds a fresh
- # Parameter and so drops .grad. Snapshot the gradients and reattach them on restore.
+ # Re-adding a hook runs init_hook -> set_module_tensor_to_device, building a fresh Parameter and dropping
+ # .grad, so snapshot the gradients and reattach on restore.
grads = {
name: getattr(tensor, "grad", None)
for name, tensor in root.named_parameters(remove_duplicate = False)
@@ -7271,16 +6855,16 @@ def _restore_dispatch_state(root, snapshot) -> None:
for name, hook in hooks:
add_hook_to_module(root.get_submodule(name) if name else root, hook)
- # Re-adding a hook rewraps whatever `_old_forward` now holds, so put the exact
- # callables back, `_old_forward` first.
+ # Re-adding a hook rewraps whatever _old_forward now holds, so put the exact callables back, _old_forward
+ # first.
for name, values in saved_attrs.items():
mod = root.get_submodule(name) if name else root
for attr in ("_old_forward", "forward", *_ACCELERATE_MOVE_GUARDS):
if attr in values:
mod.__dict__[attr] = values[attr]
- # init_hook only re-places tensors the hooked module owns, so anything added after
- # the dispatch (the LoRA adapters) is still on CPU.
+ # init_hook only re-places tensors the hooked module owns, so anything added after the dispatch (the LoRA
+ # adapters) is still on CPU.
for mod_name, mod in root.named_modules():
for attr in ("_parameters", "_buffers"):
store = getattr(mod, attr, None)
@@ -7312,8 +6896,8 @@ def _restore_dispatch_state(root, snapshot) -> None:
continue
for follower in names[1:]:
_share_tensor(root, follower, leader)
- # init_hook refilled tied_params_map with the pre-retie tensors, now unreferenced
- # by the model but still pinned by the map.
+ # init_hook refilled tied_params_map with the pre-retie tensors, now unreferenced by the model but still
+ # pinned by the map.
if ties:
_drop_accelerator_tied_param_cache(snapshot)
@@ -7344,9 +6928,8 @@ def _offload_model_for_quantize_subprocess(model):
device_map = getattr(model, "hf_device_map", None)
if device_map:
targets = {str(v).lower() for v in device_map.values()}
- # A cpu spill is fine to move, it is already in host RAM. disk/meta is not:
- # those parameters are off the model, so .to("cpu") would materialize the
- # whole checkpoint into RAM.
+ # A cpu spill is safe to move, being in host RAM; disk/meta is not, since those parameters are off
+ # the model and .to("cpu") would materialize the whole checkpoint.
if not all(t.isdigit() or t.startswith(("cuda", "xpu")) or t == "cpu" for t in targets):
return None
if not any(t.isdigit() or t.startswith(("cuda", "xpu")) for t in targets):
@@ -7358,6 +6941,8 @@ def _offload_model_for_quantize_subprocess(model):
try:
setattr(root, _DISPATCH_SNAPSHOT_ATTR, _snapshot_dispatch_state(root))
except Exception as snap_exc:
+ # A silent `return None` is indistinguishable from "nothing to move", which hides a real bug
+ # behind a merely slower export.
# Restore will fall back to re-deriving from the device_map.
logger.warning_once(
f"Unsloth: could not snapshot the accelerate dispatch "
@@ -7367,8 +6952,8 @@ def _offload_model_for_quantize_subprocess(model):
try:
model.to("cpu")
except Exception:
- # The move failed after the hooks came off; re-dispatch so the model is
- # left usable rather than hookless and half-moved across CPU/GPUs.
+ # The move failed after the hooks came off, so re-dispatch and leave the model usable rather
+ # than hookless and half-moved across CPU/GPUs.
_restore_model_after_quantize_subprocess(model, ("dispatch", dict(device_map)))
return None
snapshot = getattr(root, _DISPATCH_SNAPSHOT_ATTR, None)
@@ -7385,8 +6970,8 @@ def _offload_model_for_quantize_subprocess(model):
return None
return ("device", device)
except Exception as exc:
- # A silent `return None` is indistinguishable from "nothing to move", which
- # hides a real bug behind a merely slower export.
+ # A silent `return None` is indistinguishable from "nothing to move", hiding a real bug behind a merely
+ # slower export.
logger.warning_once(
f"Unsloth: could not free the model's accelerator memory before the quantized "
f"export ({type(exc).__name__}: {exc}); continuing with the model resident."
@@ -7409,15 +6994,15 @@ def _restore_model_after_quantize_subprocess(model, restore_token) -> None:
else:
from accelerate import dispatch_model
- # skip_keys matters: without it accelerate moves every forward kwarg
- # to the executing device, wrong for device-invariant cache tensors.
+ # skip_keys matters: without it accelerate moves every forward kwarg to the executing device,
+ # wrong for device-invariant cache tensors.
dispatch_model(
root,
device_map = value,
skip_keys = getattr(root, "_skip_keys_device_placement", None),
)
else:
- model.to(value) # restore the model to its original device
+ model.to(value)
except Exception:
logger.warning_once(
"Unsloth: could not restore the model to its original device(s) after the "
@@ -7456,21 +7041,19 @@ def _unsloth_save_compressed_tensors(
if token is None:
token = get_token()
- # Only the main process installs deps, merges, quantizes, and uploads (mirrors the non-PEFT
- # save path); other ranks return at once so they neither race on dirs nor run pip installs.
+ # Only the main process installs deps, merges, quantizes and uploads, mirroring the non-PEFT save path;
+ # other ranks return at once rather than race on dirs or run pip installs.
if not is_main_process:
return None
- # 1) Prepare the quantization runtime BEFORE merging, so an unusable config fails fast instead of
- # writing a full 16bit checkpoint first. With the llm-compressor-main shadow the subprocess
- # validates everything itself, so skip the workspace install / ceiling / scheme checks; without
- # it, install the workspace llm-compressor and fail fast past its transformers ceiling.
+ # Prepare the quantization runtime BEFORE merging, so an unusable config fails fast instead of writing a
+ # full 16bit checkpoint first. Under the llm-compressor-main shadow the subprocess validates itself, so
+ # the workspace install / ceiling / scheme checks are skipped.
_shadow_pythonpath = _compressed_quantize_pythonpath()
if _shadow_pythonpath is None:
install_llm_compressor()
- # llm-compressor cannot run under a newer transformers than its ceiling: the quantization
- # subprocess would die with a cryptic ImportError (TORCH_INIT_FUNCTIONS) only AFTER the costly
- # 16bit merge. Detect and fail fast with an actionable message instead.
+ # llm-compressor cannot run under a newer transformers than its ceiling: the subprocess dies on a
+ # cryptic TORCH_INIT_FUNCTIONS ImportError only AFTER the costly merge.
_exceeds, _tf_ver = _transformers_exceeds_llm_compressor_ceiling()
if _exceeds:
raise RuntimeError(
@@ -7492,8 +7075,8 @@ def _unsloth_save_compressed_tensors(
"Use save_method in {fp8, mxfp4, nvfp4}, or upgrade transformers + llm-compressor."
)
- # 2) Pick the local working dir. For a hub push, save_directory is a repo id, so merge and
- # quantize inside an isolated temp dir instead of writing ./<repo_id> into the cwd.
+ # Pick the local working dir: on a hub push save_directory is a repo id, so merge and quantize inside a
+ # temp dir rather than writing ./<repo_id> into the cwd.
repo_id, work_tmp, calib_tmp, model_restore = None, None, None, None
if push_to_hub:
repo_id = os.fspath(save_directory)
@@ -7504,12 +7087,14 @@ def _unsloth_save_compressed_tensors(
local_dir = os.fspath(save_directory)
local_dir = local_dir.rstrip("/\\") or local_dir
- # Wrap the body so the isolated temp dirs are always cleaned up, even when the merge,
- # quantization, validation, or hub upload raises.
+ # Wrap the body so the isolated temp dirs are always cleaned up, even when the merge, quantization,
+ # validation, or hub upload raises.
api = None
try:
- # Validate Hub access up front (a bad token / denied repo should fail before the expensive
- # merge and quantization, matching the normal push path). create_repo is idempotent.
+ # Validate Hub access up front, so a bad token or denied repo fails before the expensive merge and
+ # quantization; create_repo is idempotent.
+ # Validate Hub access up front (a bad token / denied repo should fail before the expensive merge and
+ # quantization, matching the normal push path). create_repo is idempotent.
if push_to_hub:
from huggingface_hub import HfApi
api = HfApi(token = token)
@@ -7520,12 +7105,10 @@ def _unsloth_save_compressed_tensors(
exist_ok = True,
)
- # 3) Merge to 16bit at local_dir (kept for local saves) via unsloth_generic_save, so LoRA
- # adapters are merged and full-finetuned models written in 16bit consistently. Extra
- # save kwargs (state_dict, max_shard_size, ...) flow through merge_kwargs.
- # The intermediate 16bit checkpoint is internal staging that the converter subprocess
- # reloads with default weight filenames, so never write variant-named shards here; the
- # user's variant (if any) is applied to the final compressed checkpoint in the subprocess.
+ # Merge to 16bit at local_dir via unsloth_generic_save, so adapters are merged and full-finetuned
+ # models written in 16bit alike. The subprocess reloads this staging checkpoint with default weight
+ # filenames, so never write variant-named shards here; the user's variant goes on the final compressed
+ # checkpoint.
variant = merge_kwargs.pop("variant", None)
print(f"Unsloth: Merging to 16bit before {scheme} quantization...")
merge_args = dict(merge_kwargs)
@@ -7542,10 +7125,9 @@ def _unsloth_save_compressed_tensors(
)
unsloth_generic_save(**merge_args)
- # 4) Detect VLM from the in-memory model config. A vision/multimodal model exposes a
- # vision_config or an explicitly vision-named architecture; a bare
- # *ForConditionalGeneration also matches text seq2seq models (T5/BART/Whisper), so it
- # is not treated as a VLM on its own.
+ # Detect a VLM from the in-memory config: a vision_config or a vision-named architecture. A bare
+ # *ForConditionalGeneration also matches text seq2seq (T5/BART/Whisper), so it does not count on its
+ # own.
is_vlm = False
if hasattr(model, "config"):
archs = getattr(model.config, "architectures", None) or []
@@ -7557,26 +7139,26 @@ def _unsloth_save_compressed_tensors(
"Unsloth: FP8/FP4 compressed export for vision / multimodal models is "
"experimental; vision-tower layers may be affected."
)
- # trust_remote_code must reflect the approved load decision (whether the model / tokenizer
- # was actually loaded from custom code), not the config's static auto_map, so a
- # built-in-loadable model carrying auto_map cannot run unvetted code in the subprocess.
- # Model and tokenizer trust stay separate, like the torchao path: an approved custom
- # tokenizer must not enable an unapproved model's code in the subprocess (or vice versa).
+ # trust_remote_code must reflect the approved load decision, not the config's static auto_map, so a
+ # built-in-loadable model carrying auto_map cannot run unvetted code in the subprocess. Model and
+ # tokenizer trust stay separate, as on the torchao path.
model_trust = _loaded_via_remote_code(model)
tok_trust = _loaded_via_remote_code(tokenizer)
- # 5) Marshal the calibration dataset for the subprocess: None -> ultrachat default; a
- # str/PathLike is a local save_to_disk dir if it exists else a Hub id; Dataset -> temp.
+ # Marshal the calibration dataset for the subprocess: None means the ultrachat default, a str/PathLike
+ # is a local save_to_disk dir if it exists else a Hub id, a Dataset goes to temp.
calib_kind, calib_value = "none", ""
if needs_calibration and calibration_dataset is not None:
if isinstance(calibration_dataset, (str, os.PathLike)):
calib_value = os.fspath(calibration_dataset)
calib_kind = "disk" if os.path.isdir(calib_value) else "hfid"
elif hasattr(calibration_dataset, "save_to_disk"):
- # Only persist the samples we need, so multi-GB training sets are not fully copied.
+ # Only persist the samples needed, so multi-GB training sets are not fully copied.
ds_to_save = calibration_dataset
- # A DatasetDict's len() is the split count, not rows; pick one split first so the
- # row subsample below applies and we do not save every split to the temp dir.
+ # A DatasetDict's len() is the split count, not rows, so pick one split first or every split
+ # is saved to the temp dir.
+ # A DatasetDict's len() is the split count, not rows; pick one split first so the row subsample
+ # below applies and we do not save every split to the temp dir.
try:
from datasets import DatasetDict
if isinstance(ds_to_save, DatasetDict):
@@ -7611,9 +7193,9 @@ def _unsloth_save_compressed_tensors(
f"Unsloth: scheme '{scheme}' is data-free; ignoring calibration_dataset."
)
- # 6) Quantize in a separate process: importing Unsloth patches transformers attention,
- # which breaks the forward llm-compressor runs for calibration. Run the converter by
- # file path (not `-m`) so the subprocess stays unpatched, like GGUF -> llama.cpp.
+ # Quantize in a separate process: importing Unsloth patches transformers attention, which breaks the
+ # forward llm-compressor runs for calibration. Invoke the converter by file path, not `-m`, so the
+ # subprocess stays unpatched.
out_dir = local_dir + "-" + suffix
runner = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_compressed_quantize.py")
cmd = [
@@ -7645,8 +7227,8 @@ def _unsloth_save_compressed_tensors(
if variant:
cmd += ["--variant", variant]
- # Free the in-memory model's CUDA memory before the subprocess loads its own copy from
- # disk, so the GPUs need not hold both at once. Best-effort, restored in finally.
+ # Free the in-memory model's CUDA memory before the subprocess loads its own copy, so the GPUs need
+ # not hold both. Best-effort, restored in finally.
model_restore = _offload_model_for_quantize_subprocess(model)
for _ in range(3):
gc.collect()
@@ -7659,9 +7241,9 @@ def _unsloth_save_compressed_tensors(
env["HF_TOKEN"] = token
env["HUGGING_FACE_HUB_TOKEN"] = token
- # Clean PYTHONPATH = shadow only. torch still comes from the interpreter's site-packages;
- # transformers 5.x + llm-compressor main come from the shadow. Dropping the inherited
- # PYTHONPATH removes any parent transformers sidecar so the shadow's is authoritative.
+ # Clean PYTHONPATH means shadow only: torch still comes from site-packages while transformers 5.x and
+ # llm-compressor main come from the shadow, and dropping the inherited PYTHONPATH removes any parent
+ # transformers sidecar.
if _shadow_pythonpath is not None:
env["PYTHONPATH"] = _shadow_pythonpath
@@ -7678,7 +7260,6 @@ def _unsloth_save_compressed_tensors(
f"{e.returncode}). See the output above for details."
)
- # 7) Validate the artifact.
cfg_path = os.path.join(out_dir, "config.json")
cfg = {}
if os.path.exists(cfg_path):
@@ -7689,8 +7270,10 @@ def _unsloth_save_compressed_tensors(
f"Unsloth: {scheme} export failed - no quantization_config written to {cfg_path}"
)
- # 8) Optional hub upload of the compressed artifact (not the intermediate 16bit one).
- # The repo was already created/validated up front, so just upload here.
+ # Optional hub upload of the compressed artifact, not the intermediate 16bit one; the repo was created
+ # and validated up front.
+ # 8) Optional hub upload of the compressed artifact (not the intermediate 16bit one). The repo was
+ # already created/validated up front, so just upload here.
if push_to_hub:
print(f"Unsloth: Uploading {scheme} checkpoint to '{repo_id}' ...")
api.upload_folder(
@@ -7713,7 +7296,6 @@ def _unsloth_save_compressed_tensors(
f"Unsloth: could not update datasets metadata for {repo_id}: {meta_err}"
)
- # 9) Inference hardware note.
result = repo_id if push_to_hub else out_dir
_print_compressed_hw_note(scheme, result)
return result
@@ -7753,7 +7335,7 @@ def _unsloth_save_torchao(
if token is None:
token = get_token()
- # Only the main process merges, quantizes, and uploads; other ranks return at once.
+ # Only the main process merges, quantizes and uploads; other ranks return at once.
if not is_main_process:
return None
@@ -7774,9 +7356,9 @@ def _unsloth_save_torchao(
else:
raise RuntimeError(f"Unsloth: unknown torchao export kind '{kind}' (expected fp8/int8).")
- # Always merge into an isolated temp staging dir (never save_directory itself), so a co-selected
- # 16-bit export written to save_directory is not overwritten or deleted; the torchao output is
- # the sibling "<save_directory>-<suffix>" (or the repo id on a hub push).
+ # Always merge into an isolated temp staging dir, never save_directory, so a co-selected 16-bit export
+ # there is not overwritten. The torchao output is the sibling "<save_directory>-<suffix>", or the repo id
+ # on a hub push.
repo_id, work_tmp, model_restore = None, None, None
work_tmp = tempfile.mkdtemp(prefix = "unsloth-torchao-")
if push_to_hub:
@@ -7800,8 +7382,8 @@ def _unsloth_save_torchao(
exist_ok = True,
)
- # 1) Merge to 16bit at a staging dir (LoRA and base alike). The reload reads default
- # weight filenames, so never write variant-named shards here.
+ # Merge to 16bit at a staging dir, LoRA and base alike. The reload reads default weight filenames, so
+ # never write variant-named shards here.
merge_kwargs.pop("variant", None)
print(f"Unsloth: Merging to 16bit before torchao {kind} quantization...")
merge_args = dict(merge_kwargs)
@@ -7818,23 +7400,21 @@ def _unsloth_save_torchao(
)
unsloth_generic_save(**merge_args)
- # 2) Detect VLM + reload class. A bare *ForConditionalGeneration also matches text seq2seq
- # (T5/BART/Whisper), so key off vision_config / a vision-named architecture only.
+ # Detect VLM and reload class: a bare *ForConditionalGeneration also matches text seq2seq, so key off
+ # vision_config or a vision-named architecture only.
is_vlm = False
if hasattr(model, "config"):
archs = getattr(model.config, "architectures", None) or []
is_vlm = hasattr(model.config, "vision_config") or any(
x.endswith("ForVisionText2Text") for x in archs
)
- # trust_remote_code must reflect the approved load decision - whether the in-memory model /
- # tokenizer was itself loaded from custom code - not the staged config's auto_map, which an
- # attacker can set on a built-in-loadable model to run unvetted code past the consent gate.
+ # trust_remote_code must reflect the approved load decision, not the staged config's auto_map, which
+ # an attacker can set on a built-in-loadable model to run unvetted code past the gate.
model_trust = _loaded_via_remote_code(model)
tok_trust = _loaded_via_remote_code(tokenizer)
- # Reload with the class that matches the checkpoint: an image-text VLM class (with a
- # fallback for older Transformers that lack AutoModelForImageTextToText); the model's own
- # architecture class for encoder-decoder seq2seq (T5/BART/Whisper are not causal LMs, and
- # AutoModelForCausalLM would fail to load them); otherwise causal-LM.
+ # Reload with the class matching the checkpoint: an image-text VLM class (with a fallback for
+ # Transformers lacking AutoModelForImageTextToText), the model's own class for encoder-decoder
+ # seq2seq, otherwise causal-LM.
if is_vlm:
try:
from transformers import AutoModelForImageTextToText as _reload_model
@@ -7855,10 +7435,9 @@ def _unsloth_save_torchao(
auto_model = AutoModelForCausalLM
auto_processor = AutoProcessor if is_vlm else AutoTokenizer
- # 3) Free the in-memory model's accelerator memory before reloading a fresh copy from
- # disk, else it sits resident alongside the copy and OOMs a device that fit the
- # model once. Covers CUDA and XPU (torchao runs on Intel GPUs too) plus multi-GPU
- # dispatched shards, which a plain .to("cpu") cannot move.
+ # Free the in-memory model's accelerator memory before reloading from disk, else it sits beside the
+ # copy and OOMs a device that fit the model once. Covers CUDA, XPU and multi-GPU dispatched shards,
+ # which a plain .to("cpu") cannot move.
_has_xpu = hasattr(torch, "xpu") and torch.xpu.is_available()
model_restore = _offload_model_for_quantize_subprocess(model)
for _ in range(3):
@@ -7868,8 +7447,8 @@ def _unsloth_save_torchao(
if _has_xpu:
torch.xpu.empty_cache()
- # 4) Reload the staged 16bit checkpoint with torchao applied. bfloat16 is required;
- # device_map="auto" falls back to CPU, so this works on any hardware.
+ # Reload the staged 16bit checkpoint with torchao applied: bfloat16 is required, and device_map="auto"
+ # falls back to CPU, so this works on any hardware.
print(f"Unsloth: Quantizing the merged model to torchao {kind}...")
dtype_kw = {"torch_dtype": torch.bfloat16} if HAS_TORCH_DTYPE else {"dtype": torch.bfloat16}
quantized_model = auto_model.from_pretrained(
@@ -7889,7 +7468,6 @@ def _unsloth_save_torchao(
if torch.cuda.is_available():
torch.cuda.empty_cache()
- # 5) Validate the artifact.
cfg_path = os.path.join(out_dir, "config.json")
cfg = {}
if os.path.exists(cfg_path):
@@ -7901,7 +7479,7 @@ def _unsloth_save_torchao(
f"{cfg_path}"
)
- # 6) Optional hub upload of the quantized artifact (the temp staging is cleaned in finally).
+ # Optional hub upload of the quantized artifact; the temp staging is cleaned in finally.
if push_to_hub:
print(f"Unsloth: Uploading torchao {kind} checkpoint to '{repo_id}' ...")
api.upload_folder(
@@ -7931,8 +7509,7 @@ def _unsloth_save_torchao(
)
return result
finally:
- # A raise pins the copy in the local and the live traceback, so free both or the
- # restore below OOMs.
+ # A raise pins the copy in the local and the live traceback, so free both or the restore below OOMs.
quantized_model = None
del quantized_model
_exc = sys.exc_info()[1]
@@ -7991,7 +7568,7 @@ def unsloth_save_pretrained_torchao(
has_qat_config = hasattr(self, "_torchao_config") and self._torchao_config is not None
if torchao_config is not None:
- # PTQ path: user provided a config, model must NOT have QAT config unless PEFT
+ # PTQ path: the user provided a config, so the model must NOT have a QAT config unless PEFT.
assert not has_qat_config, (
"Unsloth: You passed `torchao_config` but this model was trained with `qat_scheme`. "
"For QAT models, do not pass `torchao_config` - the quantization config is already "
@@ -8006,7 +7583,7 @@ def unsloth_save_pretrained_torchao(
token = token,
)
else:
- # QAT path: no config provided, model must have QAT config
+ # QAT path: no config provided, so the model must have a QAT config.
assert has_qat_config, (
"Unsloth: No `torchao_config` provided and model was not trained with `qat_scheme`. "
"Either train with `qat_scheme` parameter, or provide a `torchao_config` for "
@@ -8033,7 +7610,6 @@ def patch_saving_functions(model, vision = False):
import types
from typing import Callable, Optional, Union, List
- # And now re add our saving methods!
if model.push_to_hub.__name__ == "unsloth_push_to_hub":
original_push_to_hub = model.original_push_to_hub
else:
@@ -8142,7 +7718,6 @@ def patch_saving_functions(model, vision = False):
original_model = model
while True:
- # Check if push_to_hub exists before accessing its __name__
if (
hasattr(original_model, "push_to_hub")
and original_model.push_to_hub.__name__ != "unsloth_push_to_hub"
@@ -8161,10 +7736,8 @@ def patch_saving_functions(model, vision = False):
else:
break
- # Add saving methods to top level model
if not vision:
if hasattr(model, "config"):
- # Counteract tokenizers
model.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model)
model.save_pretrained_merged = types.MethodType(
unsloth_generic_save_pretrained_merged, model
@@ -8179,7 +7752,6 @@ def patch_saving_functions(model, vision = False):
unsloth_convert_lora_to_ggml_and_save_locally, model
)
else:
- # Vision only 1 option
model.push_to_hub_merged = types.MethodType(unsloth_generic_push_to_hub_merged, model)
model.save_pretrained_merged = types.MethodType(
unsloth_generic_save_pretrained_merged, model
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 1bc5fe30138..d073bf54bcb 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -58,7 +58,7 @@ IGNORED_TOKENIZER_CHECKING = frozenset(
IGNORED_TOKENIZER_NAMES = [
- # Qwen Coder did not train on tool calling. Math did!
+ # Qwen Coder did not train on tool calling; Math did.
"unsloth/Qwen2.5-Coder-1.5B-Instruct",
"unsloth/Qwen2.5-Coder-7B-Instruct",
]
@@ -68,10 +68,8 @@ IGNORED_TOKENIZER_NAMES = frozenset(
)
os.environ["UNSLOTH_IGNORED_TOKENIZER_NAMES"] = "\n".join(IGNORED_TOKENIZER_NAMES)
-# Check environments
-# A KAGGLE_* variable is not a Kaggle kernel - the Kaggle CLI reads
-# KAGGLE_USERNAME / KAGGLE_KEY from the environment on ordinary machines, and
-# redirecting their tokenizer cache to /tmp because of it was wrong.
+# A KAGGLE_* variable is not a Kaggle kernel: the Kaggle CLI reads KAGGLE_USERNAME / KAGGLE_KEY on
+# ordinary machines, and redirecting their tokenizer cache to /tmp because of it was wrong.
from .disk_utils import (
KAGGLE_TMP,
is_colab_environment,
@@ -90,13 +88,12 @@ def try_fix_tokenizer(tokenizer, prepend = True):
tokenizer_string = converted_tokenizer.to_str()
- # Llama does _apple. Sometimes this is wrong!!
+ # Llama does _apple; sometimes this is wrong.
prepend_text = '{"type":"Prepend","prepend":"▁"},'
if not prepend and prepend_text in tokenizer_string:
tokenizer_string = tokenizer_string.replace(prepend_text, "", 1)
dir_names = dir(tokenizer)
- # Get eos_token, bos_token etc
token_names = [x for x in dir_names if x.endswith("_token") and x.count("_") == 1]
for token_name in token_names:
@@ -107,7 +104,7 @@ def try_fix_tokenizer(tokenizer, prepend = True):
if token_id is None:
continue
- # Locate the token's id mapping in the string
+ # Locate the token's id mapping in the string.
find_text = f'"id":{token_id},"content":"'
find_pos = tokenizer_string.find(find_text)
if find_pos == -1:
@@ -118,13 +115,12 @@ def try_fix_tokenizer(tokenizer, prepend = True):
continue
bad_token = tokenizer_string[start:end]
- # Check if token is the actual same one - if not, edit it
+ # Check whether the token is actually the same one; if not, edit it and replace the vocab section.
if bad_token != token:
bad_text = f'{find_text}{bad_token}",'
good_text = f'{find_text}{token}",'
tokenizer_string = tokenizer_string.replace(bad_text, good_text, 1)
- # And replace vocab section
bad_text = f'"{bad_token}":{token_id},'
good_text = f'"{token}":{token_id},'
tokenizer_string = tokenizer_string.replace(bad_text, good_text, 1)
@@ -162,13 +158,11 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_sen
except:
FastTokenizer = PreTrainedTokenizerFast
- # Get all arguments (bos_token, etc)
docs = FastTokenizer.__doc__
docs = docs[docs.find("Args:") :]
args = re.findall(r"\n[\s]+([^\s]{1,}) \(", docs, flags = re.MULTILINE)
args = [x for x in args if not x.endswith("_file")]
- # Also some missing maybe!
docs = PreTrainedTokenizerFast.__doc__
docs = docs[docs.find("Args:") :]
args2 = re.findall(r"\n[\s]+([^\s]{1,}) \(", docs, flags = re.MULTILINE)
@@ -181,27 +175,24 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_sen
kwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = True)
fast_tokenizer = FastTokenizer(**kwargs)
- # Check if they're similar!
sorted_slow_tokenizer = get_sorted_dict(slow_tokenizer.get_vocab())
sorted_fast_tokenizer = get_sorted_dict(fast_tokenizer.get_vocab())
check_vocab = sorted_slow_tokenizer == sorted_fast_tokenizer
check_special = slow_tokenizer.all_special_tokens == fast_tokenizer.all_special_tokens
- # Failure so return slow_tokenizer
+ # Failure, so return the slow_tokenizer.
if not check_vocab or not check_special:
return slow_tokenizer
- # Now confirm if they match
if not assert_same_tokenization(slow_tokenizer, fast_tokenizer):
- # Maybe remove prepending of __apple?
+ # Maybe remove the prepending of __apple.
kwargs["tokenizer_object"] = try_fix_tokenizer(slow_tokenizer, prepend = False)
fast_tokenizer = FastTokenizer(**kwargs)
if not assert_same_tokenization(slow_tokenizer, fast_tokenizer):
- # Failure :(
return slow_tokenizer
- # Also tokenizer.model is missing!
+ # tokenizer.model is missing too.
name = slow_tokenizer.name_or_path.replace("/", "_")
if not os.path.exists(temporary_location):
os.makedirs(temporary_location)
@@ -209,7 +200,6 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_sen
slow_tokenizer.save_pretrained(new_location)
fast_tokenizer.save_pretrained(new_location)
- # Now load it!
fast_tokenizer = AutoTokenizer.from_pretrained(new_location)
if assert_same_tokenization(slow_tokenizer, fast_tokenizer):
return fast_tokenizer
@@ -266,7 +256,6 @@ llama_template = (
def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
- # Get eos_token, bos_token etc
if not hasattr(slow_tokenizer, "all_special_tokens"):
return True
dir_names = dir(slow_tokenizer)
@@ -286,7 +275,6 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
replacement_char = b"\xc3\xaf\xc2\xbf\xc2\xbd".decode("utf-8")
all_special_tokens = [x for x in all_special_tokens if x != replacement_char]
- # Check if chat template is enabled!
check_chat_template1 = True
check_chat_template2 = True
check_chat_template3 = True
@@ -330,7 +318,6 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
"""
check_chat_template = check_chat_template1 and check_chat_template2 and check_chat_template3
- # Try special tokens
try:
string = (
"\n".join(all_special_tokens)
@@ -341,9 +328,8 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer):
return check_chat_template and check_special_tokens
except:
- # For eg see https://github.com/unslothai/unsloth/issues/292
- # Sometimes tokenizer has weird tokens, causing a combined tokenization to fail.
- # [TODO] We temporarily disable this for CodeLlama tokenizers
+ # Sometimes a tokenizer has weird tokens, causing a combined tokenization to fail; temporarily
+ # disabled for CodeLlama tokenizers (#292).
if slow_tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING:
return check_chat_template
else:
@@ -356,8 +342,8 @@ def fix_sentencepiece_tokenizer(
token_mapping,
temporary_location = "_unsloth_sentencepiece_temp",
):
- # From https://github.com/google/sentencepiece/issues/121
- # We need to manually edit the sentencepiece tokenizer!
+ # The sentencepiece tokenizer must be edited manually; see google/sentencepiece#121. Only works for
+ # SentencePiece <= 3.20.3.
try:
from transformers.convert_slow_tokenizer import import_protobuf
sentencepiece_model_pb2 = import_protobuf()
@@ -379,11 +365,10 @@ def fix_sentencepiece_tokenizer(
if not os.path.exists(temporary_location):
os.makedirs(temporary_location)
- # Fresh per-call subdir so concurrent/repeated calls can't clobber each other's
- # tokenizer.model or leak stale files, without deleting anything the caller owns.
+ # Fresh per-call subdir so concurrent or repeated calls cannot clobber each other's tokenizer.model
+ # or leak stale files, without deleting anything the caller owns.
temporary_location = tempfile.mkdtemp(prefix = "tokenizer_", dir = temporary_location)
- # First save the old tokenizer
old_tokenizer.save_pretrained(temporary_location)
# Only sentencepiece tokenizers write tokenizer.model, so check after the save.
@@ -395,10 +380,9 @@ def fix_sentencepiece_tokenizer(
tokenizer_file = sentencepiece_model_pb2.ModelProto()
tokenizer_file.ParseFromString(open(f"{temporary_location}/tokenizer.model", "rb").read())
- # Now save the new tokenizer
new_tokenizer.save_pretrained(temporary_location)
- # Now correct the old tokenizer's .model file
+ # Correct the old tokenizer's .model file.
for old_token, new_token in token_mapping.items():
ids = old_tokenizer([old_token], add_special_tokens = False).input_ids
ids = ids[0]
@@ -409,7 +393,7 @@ def fix_sentencepiece_tokenizer(
)
continue
ids = ids[0]
- # [TODO] Hack for Starling - try except
+ # Hack for Starling: try/except.
try:
tokenizer_piece = tokenizer_file.pieces[ids]
except:
@@ -417,11 +401,9 @@ def fix_sentencepiece_tokenizer(
assert tokenizer_piece.piece == old_token
tokenizer_piece.piece = new_token
- # And now write it
with open(f"{temporary_location}/tokenizer.model", "wb") as file:
file.write(tokenizer_file.SerializeToString())
- # And load it!
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(
@@ -429,8 +411,8 @@ def fix_sentencepiece_tokenizer(
eos_token = new_tokenizer.eos_token,
pad_token = new_tokenizer.pad_token,
)
- # vocab_file points here, so the dir must outlive the tokenizer (a later
- # save_pretrained copies the patched tokenizer.model from it); reclaim it on GC.
+ # vocab_file points here, so the dir must outlive the tokenizer (a later save_pretrained copies the
+ # patched tokenizer.model from it); reclaim it on GC.
weakref.finalize(tokenizer, shutil.rmtree, temporary_location, ignore_errors = True)
return tokenizer
@@ -468,16 +450,14 @@ def fix_sentencepiece_gguf(saved_location):
UNUSED = 5
BYTE = 6
- # Load tokenizer.model
tokenizer_file = sentencepiece_model_pb2.ModelProto()
if not os.path.isfile(f"{saved_location}/tokenizer.model"):
return
tokenizer_file.ParseFromString(open(f"{saved_location}/tokenizer.model", "rb").read())
sentence_piece_size = len(tokenizer_file.pieces)
- # Build a set of token IDs that are marked as special in tokenizer.json.
- # These tokens should use CONTROL type in the sentencepiece model so that
- # llama.cpp writes them as CONTROL (type=3) in the GGUF token_type array.
+ # Build a set of token IDs marked special in tokenizer.json: these must use CONTROL type in the
+ # sentencepiece model so llama.cpp writes them as CONTROL (type=3) in the GGUF token_type array.
special_token_ids = set()
if os.path.isfile(f"{saved_location}/tokenizer.json"):
with open(f"{saved_location}/tokenizer.json", "r", encoding = "utf-8") as f:
@@ -487,8 +467,8 @@ def fix_sentencepiece_gguf(saved_location):
if entry.get("special", False) and isinstance(token_id, int):
special_token_ids.add(token_id)
- # Fix existing sentencepiece tokens that are marked as special in tokenizer.json
- # but have the wrong type (NORMAL instead of CONTROL) in the sentencepiece model.
+ # Fix existing sentencepiece tokens marked special in tokenizer.json but typed NORMAL instead of
+ # CONTROL in the sentencepiece model.
patched = 0
for token_id in special_token_ids:
if 0 <= token_id < sentence_piece_size:
@@ -502,7 +482,6 @@ def fix_sentencepiece_gguf(saved_location):
f"from NORMAL to CONTROL type so llama.cpp / GGUF chat inference works correctly."
)
- # Load added_tokens_json
if not os.path.isfile(f"{saved_location}/added_tokens.json"):
if patched > 0:
with open(f"{saved_location}/tokenizer.model", "wb") as file:
@@ -519,7 +498,7 @@ def fix_sentencepiece_gguf(saved_location):
added_tokens_json = dict(sorted(added_tokens_json.items(), key = lambda item: item[1]))
new_size = sentence_piece_size + len(added_tokens_json)
- # Confirm added_tokens_json is correct
+ # Confirm added_tokens_json is correct.
added_tokens_ids = np.array(list(added_tokens_json.values()))
_real_added_tokens_ids = added_tokens_ids
if len(added_tokens_ids) < 2:
@@ -537,7 +516,7 @@ def fix_sentencepiece_gguf(saved_location):
file.write(tokenizer_file.SerializeToString())
return
- # Edit sentence piece tokens with added_tokens_json
+ # Edit sentencepiece tokens with added_tokens_json.
logger.warning(
f"Unsloth: Extending {saved_location}/tokenizer.model with added_tokens.json.\n"
f"Originally tokenizer.model is of size ({sentence_piece_size}).\n"
@@ -548,8 +527,7 @@ def fix_sentencepiece_gguf(saved_location):
added_token_id = added_tokens_json[added_token_str]
new_token.piece = added_token_str.encode("utf-8")
new_token.score = -1000.0
- # Use CONTROL type for tokens marked as special in tokenizer.json,
- # otherwise fall back to USER_DEFINED.
+ # Use CONTROL type for tokens marked special in tokenizer.json, otherwise fall back to USER_DEFINED.
if added_token_id in special_token_ids:
new_token.type = SentencePieceTokenTypes.CONTROL
else:
@@ -560,9 +538,6 @@ def fix_sentencepiece_gguf(saved_location):
with open(f"{saved_location}/tokenizer.model", "wb") as file:
file.write(tokenizer_file.SerializeToString())
- # Add padding tokens
- # actual_vocab_size = model.config.vocab_size
- # padding = actual_vocab_size - len(tokenizer_file.pieces)
return
@@ -579,17 +554,15 @@ def _load_correct_tokenizer(
if IS_COLAB_ENVIRONMENT:
cache_dir = cache_dir
elif IS_KAGGLE_ENVIRONMENT:
- # /tmp of Kaggle seems has a 80GB limit!
- # Let's utilize them
+ # Kaggle's /tmp seems to have an 80GB limit, so use it.
cache_dir = os.path.join(KAGGLE_TMP, cache_dir)
elif cache_dir == "huggingface_tokenizers_cache":
- # This default name is Colab/Kaggle-only; elsewhere use the HF default cache.
+ # This default name is Colab/Kaggle-only; elsewhere use the HF default cache, and keep a
+ # caller-supplied cache_dir so the tokenizer loads from the prefetch-warmed dir.
cache_dir = None
- # else: keep a caller-supplied cache_dir so the tokenizer loads from the prefetch-warmed dir instead
- # of risking an in-process Hub/Xet transfer.
- # Try loading the slow tokenizer. If it fails, then try Fast only
- # Mainly to solve Deepseek models with no tokenizer.model file
+ # Try the slow tokenizer first and fall back to fast only, mainly to solve Deepseek models with no
+ # tokenizer.model file.
slow_tokenizer = None
try:
slow_tokenizer = AutoTokenizer.from_pretrained(
@@ -598,7 +571,7 @@ def _load_correct_tokenizer(
padding_side = padding_side,
token = token,
trust_remote_code = trust_remote_code,
- # Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373
+ # use_fast = False alone is not enough; see twitter.com/danielhanchen/status/1789659394302718373
use_fast = False,
legacy = False,
from_slow = True,
@@ -607,11 +580,7 @@ def _load_correct_tokenizer(
)
except:
slow_tokenizer = None
- # print(
- # f"Unsloth: {tokenizer_name} has no tokenizer.model file.\n"\
- # "Just informing you about this - this is not a critical error."
- # )
- # Unsure why this occurs!
+ # Reason unknown.
if type(slow_tokenizer) is bool:
slow_tokenizer = None
@@ -639,7 +608,7 @@ def _load_correct_tokenizer(
if hasattr(fast_tokenizer, "add_eos_token") and hasattr(slow_tokenizer, "add_eos_token"):
fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token
- # Confirm if slow and fast are equivalent!
+ # Confirm whether slow and fast are equivalent.
if assert_same_tokenization(slow_tokenizer, fast_tokenizer):
return fast_tokenizer
else:
@@ -694,10 +663,10 @@ def load_correct_tokenizer(
if fix_tokenizer:
_fix_pad_token(tokenizer)
- ### 1. Fixup tokenizer's chat_template
+ # Fixup the tokenizer's chat_template.
old_chat_template = getattr(tokenizer, "chat_template", None)
- # Ignore mistral type models since they don't have an add_generation_prompt
+ # Ignore mistral type models, which have no add_generation_prompt, and check Llama-2 old style models too.
if any(
s in str(getattr(tokenizer, "name_or_path", "")).lower() for s in ["mistral", "qwen3guard"]
):
@@ -722,16 +691,15 @@ def load_correct_tokenizer(
pass
tokenizer.chat_template = chat_template
- # Saving restores sentencepiece assets from the repo name alone, which carries no
- # branch, so stamp it for the save path. Imported late to break a circular import.
+ # Saving restores sentencepiece assets from the repo name alone, which carries no branch, so stamp
+ # it for the save path. Imported late to break a circular import.
from .models.loader_utils import _mark_loaded_revision
_mark_loaded_revision(tokenizer, revision)
return tokenizer
-# All four Jinja whitespace-control variants of endfor/endif:
-# {% endfor %} {%- endfor %} {% endfor -%} {%- endfor -%}
+# All four Jinja whitespace-control variants of endfor/endif.
_RE_ENDFOR = re.compile(r"\{%(-?)\s*endfor\s*(-?)%\}")
_RE_ENDIF = re.compile(r"\{%(-?)\s*endif\s*(-?)%\}")
_RE_JINJA_COMMENT = re.compile(r"\{#.*?#\}", flags = re.DOTALL)
@@ -832,8 +800,8 @@ def _has_add_generation_prompt_block(chat_template):
return "if add_generation_prompt" in chat_template and "%}" in chat_template
for if_node in ast.find_all(jinja2.nodes.If):
test = if_node.test
- # Reject negated gates: `{% if not add_generation_prompt %}` fires
- # when agp=False, so it's not a generation block even if it emits.
+ # Reject negated gates: `{% if not add_generation_prompt %}` fires when agp=False, so it is not a
+ # generation block even if it emits.
if isinstance(test, jinja2.nodes.Not):
continue
# find_all skips the test root, so check bare Name tests explicitly.
@@ -850,9 +818,8 @@ def _has_add_generation_prompt_block(chat_template):
return False
-# Sentinels for _derive_assistant_prefix_by_render. Diverge at char 0 so
-# commonprefix can't absorb them; long random tail makes collision with real
-# template literals negligible (see T18).
+# Sentinels for _derive_assistant_prefix_by_render: they diverge at char 0 so commonprefix cannot
+# absorb them, and the long random tail makes collision with real template literals negligible.
_RENDER_DIFF_SENTINEL_A = "AAAA_0123456789_UNSLOTH_RENDER_DIFF_SENTINEL"
_RENDER_DIFF_SENTINEL_B = "BBBB_0123456789_UNSLOTH_RENDER_DIFF_SENTINEL"
_RENDER_DIFF_SENTINEL_C = "CCCC_0123456789_UNSLOTH_RENDER_DIFF_SENTINEL"
@@ -889,9 +856,8 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):
sent_b_msgs = base_msgs + [{"role": "assistant", "content": _RENDER_DIFF_SENTINEL_B}]
sent_c_msgs = base_msgs + [{"role": "user", "content": _RENDER_DIFF_SENTINEL_C}]
- # Strip trailing whitespace/comments after the last endfor/endif: they
- # appear after the message loop and would break Guard A. The splice in
- # `_fix_chat_template` drops them too.
+ # Strip trailing whitespace/comments after the last endfor/endif: they appear after the message loop
+ # and would break Guard A, and the splice in _fix_chat_template drops them too.
probe_template = chat_template
end = _find_end_position(chat_template)
if end is not None:
@@ -899,8 +865,8 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):
if _RE_JINJA_COMMENT.sub("", after).strip() == "":
probe_template = chat_template[: end["end"]]
- # Sandboxed: probe renders at load time, before user calls
- # apply_chat_template. SandboxedEnvironment blocks attribute-chain exploits.
+ # Sandboxed: probe renders run at load time, before the user calls apply_chat_template, and
+ # SandboxedEnvironment blocks attribute-chain exploits.
try:
env = SandboxedEnvironment(
autoescape = False,
@@ -913,9 +879,8 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):
except Exception:
return None
- # Best-effort: alternation-enforcing templates (e.g. Gemma's
- # raise_exception) fail on [user, user]; that's a positive signal
- # for Guard C, not a probe failure.
+ # Best-effort: alternation-enforcing templates (Gemma's raise_exception) fail on [user, user], which
+ # is a positive signal for Guard C rather than a probe failure.
out_user_c = None
try:
out_user_c = tmpl.render(messages = sent_c_msgs, add_generation_prompt = False)
@@ -940,8 +905,7 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):
):
return None
- # Guard C: reject if a [user, user] render also emits the same prefix
- # (role-insensitive template, e.g. `{% set greeting='Hi' %}...`).
+ # Guard C: reject if a [user, user] render also emits the same prefix, i.e. a role-insensitive template.
if out_user_c is not None and out_user_c.startswith(out_base):
tail_c = out_user_c[len(out_base) :]
if tail_c.startswith(prefix) and prefix != "":
@@ -954,10 +918,8 @@ def _derive_assistant_prefix_by_render(chat_template, is_sharegpt = False):
def _fix_chat_template(chat_template, is_sharegpt = False):
- # Fast path: already has an {% if add_generation_prompt %} block, nothing
- # to do. This catches cases the old string-based check would miss (e.g.
- # templates that use {%- if add_generation_prompt -%} with both-side dash,
- # or that sneak the block into a nested If/For).
+ # Fast path: an {% if add_generation_prompt %} block already exists. This catches cases the old
+ # string-based check would miss, such as both-side dashes or a block nested inside an If/For.
if _has_add_generation_prompt_block(chat_template):
return chat_template
@@ -970,9 +932,8 @@ def _fix_chat_template(chat_template, is_sharegpt = False):
dash_r = "-" if end["dash_right"] else ""
open_tag = lambda body: "{%" + dash_l + " " + body + " " + dash_r + "%}"
- # Case 1 (pre-existing base case): template ends with a single trailing
- # {{ expr }} that is the generation prefix. Wrap it in an
- # {% if add_generation_prompt %} ... {% endif %}.
+ # Case 1: the template ends with a single trailing {{ expr }} that is the generation prefix, so wrap
+ # it in an {% if add_generation_prompt %} block.
if (
"{%" + dash_l + " if" not in after_endfor
and "{%" + dash_l + " set " not in after_endfor
@@ -984,18 +945,15 @@ def _fix_chat_template(chat_template, is_sharegpt = False):
wrapped = open_tag("if add_generation_prompt") + after_endfor + open_tag("endif")
return chat_template[: end["end"]] + wrapped
- # Case 2 (GH#4150): template ends at {% endfor %} with only whitespace
- # or comments left. Inject an {% if add_generation_prompt %} block with
- # the assistant prefix derived by render-diff. The top-level-For gate
- # keeps us out of outer-If wrappers (e.g. Qwen3-Guard).
+ # Case 2 (#4150): the template ends at {% endfor %} with only whitespace or comments left, so inject
+ # an {% if add_generation_prompt %} block with the assistant prefix derived by render-diff.
if _RE_JINJA_COMMENT.sub("", after_endfor).strip() == "" and _template_ends_with_toplevel_for(
chat_template
):
- # No redundant "agp not in scrubbed" check: the fast path already
- # confirmed no *positive* block, and a mere reference (header
- # guard) should still get repaired.
+ # No redundant "agp not in scrubbed" check: the fast path already confirmed no POSITIVE block, and a
+ # mere reference (header guard) should still get repaired.
assistant_prefix = _derive_assistant_prefix_by_render(chat_template, is_sharegpt)
- # Dual-probe: dict/list callers don't know the shape up front.
+ # Dual-probe: dict/list callers do not know the shape up front.
if assistant_prefix is None and not is_sharegpt:
assistant_prefix = _derive_assistant_prefix_by_render(chat_template, is_sharegpt = True)
if assistant_prefix is None:
@@ -1104,7 +1062,7 @@ def _validate_patched_template(tokenizer, patched_template, is_sharegpt):
try:
tokenizer.chat_template = patched_template
except Exception:
- return False # read-only tokenizer, skip validation
+ return False
try:
yes = tokenizer.apply_chat_template(
msgs,
@@ -1123,9 +1081,8 @@ def _validate_patched_template(tokenizer, patched_template, is_sharegpt):
tokenizer.chat_template = original
except Exception:
pass # best-effort restore
- # Contract after a successful repair: the two renders differ, and the
- # "yes" render is a strict extension of the "no" render (we only
- # appended content inside the new add_generation_prompt block).
+ # Contract after a successful repair: the two renders differ, and the "yes" render is a strict
+ # extension of the "no" render, since content was only appended in the new block.
return yes != no and yes.startswith(no)
@@ -1135,10 +1092,8 @@ def _repair_string_template(tokenizer, chat_template, is_sharegpt):
candidate = _fix_chat_template(chat_template, is_sharegpt = is_sharegpt)
if not _has_add_generation_prompt_block(candidate):
return None
- # Validate with the caller's is_sharegpt first. If that fails, the
- # dual-probe in _fix_chat_template may have fallen back to the other
- # schema internally -- try validating with the opposite schema before
- # giving up.
+ # Validate with the caller's is_sharegpt first; if that fails, the dual-probe in
+ # _fix_chat_template may have fallen back to the other schema, so try the opposite one.
if _validate_patched_template(tokenizer, candidate, is_sharegpt):
return candidate
if _validate_patched_template(tokenizer, candidate, not is_sharegpt):
@@ -1199,13 +1154,14 @@ def _fix_chat_template_for_tokenizer(tokenizer, chat_template):
return chat_template
if no != yes:
- # Template already responds to the flag; leave as is.
+ # The template already responds to the flag; leave it as is.
return chat_template
- # no == yes: template ignores add_generation_prompt. Try to repair.
+ # no == yes: the template ignores add_generation_prompt, so try to repair. It may have the block
+ # while the output does not change, the "wasn't provided correctly" case.
if _has_add_generation_prompt_block(chat_template):
- # Template has the block but it does not change output. This is the
- # "wasn't provided correctly" case from the pre-warn code path.
+ # Template has the block but it does not change output. This is the "wasn't provided correctly" case
+ # from the pre-warn code path.
strict = _is_strict_chat_template_mode()
msg = _format_chat_template_message(
name,
@@ -1311,8 +1267,8 @@ def fix_chat_template(tokenizer):
if chat_template is None:
return None
- # Multi-variant dict (e.g. Hermes-3 {default, tool_use}): route each
- # variant through the full repair contract via _VariantTokenizerProxy.
+ # Multi-variant dict (Hermes-3 {default, tool_use}): route each variant through the full repair
+ # contract via _VariantTokenizerProxy.
if isinstance(chat_template, dict):
fixed = {}
for key, tmpl in chat_template.items():
@@ -1323,7 +1279,7 @@ def fix_chat_template(tokenizer):
fixed[key] = _fix_chat_template_for_tokenizer(proxy, tmpl)
return fixed
- # List-of-dicts form (older HF multi-template style).
+ # List-of-dicts form, the older HF multi-template style.
if isinstance(chat_template, list):
fixed = []
for item in chat_template:
@@ -1356,13 +1312,10 @@ def check_tokenizer(
_reload = True,
cache_dir = None,
):
- # Checks tokenizer for out of bounds ids.
- # Mainly a fix for https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha
- # where <sep> had token id=32002.
- # See https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha/discussions/25
- # Seems like the Fast tokenizer in Rust breaks things!
+ # Check the tokenizer for out of bounds ids, mainly for berkeley-nest/Starling-LM-7B-alpha where
+ # <sep> had token id=32002 (its discussion 25); the Rust fast tokenizer seems to break things.
- # We ignore some of them!
+ # Some of them are ignored.
if tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING:
return tokenizer
@@ -1377,7 +1330,6 @@ def check_tokenizer(
bad_indices = list(added_tokens_fast.keys())[j:]
bad_tokens = list(added_tokens_fast.values())[j:]
if not _reload:
- # Try removing the token
added_tokens = [str(x) for x in tokenizer.added_tokens_decoder.values()]
special_tokens = tokenizer.special_tokens_map
import itertools
@@ -1412,14 +1364,13 @@ def check_tokenizer(
try_removal.append(token)
try_mapper.append(name_token)
- # Recheck!
can_be_removed = len(try_removal) == len(bad_tokens)
if can_be_removed:
remove_generic = True
can_be_removed1 = bad_tokens
if can_be_removed:
- # Yes it can be fixed!
+ # Yes, it can be fixed.
for j, bad_token in enumerate(can_be_removed1):
remove_id = tokenizer._added_tokens_encoder[bad_token]
del tokenizer._added_tokens_decoder[remove_id]
@@ -1429,7 +1380,7 @@ def check_tokenizer(
# Remove sep token for example
setattr(tokenizer, try_mapper[j], None)
setattr(tokenizer, try_mapper[j] + "_id", None)
- # Confirm 1 more time!
+ # Confirm one more time.
if max(tokenizer.added_tokens_decoder.keys()) < max_embedding_size:
logger.warning_once(
f"Unsloth loaded a broken tokenizer `{model_name}`, but managed to repair it!\n"
@@ -1438,19 +1389,20 @@ def check_tokenizer(
)
return convert_to_fast_tokenizer(tokenizer)
- # :( Failure
+ # Failure.
raise RuntimeError(
f"Unsloth tried to load `{model_name}`, but cannot succeed.\n"
f"Tokens {bad_tokens} with ids {bad_indices} exceeds the max vocab size of {max_embedding_size}.\n"
f"Fix your tokenizer since it'll perform out of bounds memory accesses."
)
- # Reuse a caller-supplied cache_dir (warmed cache) for the repair reload; else the
- # Colab/Kaggle sentinel (HF default elsewhere), as load_correct_tokenizer does.
+ # Reuse a caller-supplied cache_dir (the warmed cache) for the repair reload; else the Colab/Kaggle
+ # sentinel, as load_correct_tokenizer does.
reload_cache_dir = cache_dir
if reload_cache_dir is None and (IS_COLAB_ENVIRONMENT or IS_KAGGLE_ENVIRONMENT):
reload_cache_dir = "huggingface_tokenizers_cache"
+ # Sometimes the slow tokenizer does not work, e.g. Deepseek, but it can also fix things.
# Sometimes slow tokenizer does not work like Deepseek
try:
# Try slow tokenizer which can fix things!
@@ -1459,7 +1411,7 @@ def check_tokenizer(
model_max_length = model_max_length,
padding_side = padding_side,
token = token,
- # Cannot just use use_fast = False as per https://twitter.com/danielhanchen/status/1789659394302718373
+ # use_fast = False alone is not enough; see twitter.com/danielhanchen/status/1789659394302718373
use_fast = False,
legacy = False,
from_slow = True,
@@ -1477,8 +1429,7 @@ def check_tokenizer(
)
break
except:
- # Tokenizer has out of bounds issues and we can't
- # load the slow tokenizer version :(
+ # The tokenizer has out of bounds issues and the slow version cannot be loaded.
logger.warning_once(
"Unsloth: Tokenizer is most likely buggy, and Unsloth failed to repair it.\n"
"It will still work, but beware of out of bounds memory accesses.\n"
@@ -1584,18 +1535,14 @@ def patch_sft_trainer_tokenizer():
except:
return
all_imports = dir(trl.trainer.sft_trainer)
- # Make typing names available to the exec'd source bodies. TRL >= 1.x
- # type-hints _prepare_dataset / _prepare_non_packed_dataloader with
- # `Union[...]` and friends; without these imports in the exec namespace
- # those become NameErrors at exec time. Mirrors the pattern used in
- # unsloth/models/_utils.py:patch_linear_scaling.
+ # Make typing names available to the exec'd source bodies: TRL >= 1.x type-hints _prepare_dataset
+ # with Union[...] and friends, which become NameErrors at exec time without these imports.
from typing import Union, Optional, List, Any, Callable, Tuple, Dict, Iterator # noqa: F401
for (
function_name,
replacer,
) in (
- # ("_prepare_non_packed_dataloader", "def tokenize(element):",),
(
"_prepare_non_packed_dataloader",
None,
@@ -1604,7 +1551,6 @@ def patch_sft_trainer_tokenizer():
"_prepare_dataset",
None,
),
- # ("_prepare_packed_dataloader", "if dataset_text_field is not None",),
):
if not hasattr(sft_trainer, function_name):
continue
@@ -1645,7 +1591,7 @@ def patch_sft_trainer_tokenizer():
check_text = check_text.rstrip() + "\n"
if replacer is None:
- # .*? matches first match. .+? matches final match.
+ # .*? matches the first match, .+? the final one.
replacer = re.findall(
f"def {function_name}" + r"\(.*?\).*?\:\n",
function,
@@ -1673,7 +1619,6 @@ def patch_sft_trainer_tokenizer():
globals(),
)
- # Patch train with fix_untrained_tokens
for path_to_trainer in (
"sft_trainer.SFTTrainer",
"dpo_trainer.DPOTrainer",
@@ -1720,7 +1665,7 @@ def patch_sft_trainer_tokenizer():
"fix_zero_training_loss(self.model, tokenizer, self.train_dataset)\n\n"
)
- # Warn on gradient accumulation steps if it's used
+ # Also DPO weirdly tokenizes non numeric columns? Delete them!
check_text += (
"\n"
"try:\n"
@@ -1736,7 +1681,7 @@ def patch_sft_trainer_tokenizer():
"\n\n"
)
- # Add NEFTune since it doesn't seem to work?? We need to manually inject it
+ # NEFTune does not seem to work, so inject it manually.
check_text += (
"\n"
"if hasattr(self, 'neftune_hook_handle'):\n"
@@ -1750,7 +1695,7 @@ def patch_sft_trainer_tokenizer():
"\n"
)
- # Also DPO weirdly tokenizes non numeric columns? Delete them!
+ # DPO weirdly tokenizes non numeric columns, so delete them.
check_text += (
"\n"
"if hasattr(self.train_dataset, 'column_names'):\n"
@@ -1776,5 +1721,4 @@ def patch_sft_trainer_tokenizer():
)
-# Finally patch TRL tokenizer things -> moved to RL
-# patch_sft_trainer_tokenizer()
+# Finally patch TRL tokenizer things, moved to RL.
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 824c3e6b46e..7b47d027c16 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -26,7 +26,7 @@ import trl
import inspect
from trl import SFTTrainer
-# why: bypass partially-initialised unsloth ns during _gpu_init load
+# Bypass the partially-initialised unsloth namespace during the _gpu_init load.
from .models._utils import is_bfloat16_supported
from unsloth.utils import (
configure_padding_free,
@@ -87,9 +87,7 @@ class UnslothVisionDataCollator(_UnslothVisionDataCollatorBase):
if formatting_func is None:
return super().__call__(examples)
- # why: base __call__ would reapply formatting_func; applied above. A
- # per-call shallow view shares every other attribute by reference, so a
- # concurrent caller can never observe formatting_func blanked on self.
+ # The base __call__ would reapply formatting_func, which was applied above.
view = copy.copy(self)
view.formatting_func = None
return super(UnslothVisionDataCollator, view).__call__(examples)
@@ -103,10 +101,9 @@ PADDING_FREE_BLOCKLIST = {
"gemma2", # - gemma2: Uses slow_attention_softcapping which has torch.compile issues
"gpt_oss", # - gpt_oss: Uses Flex Attention which doesn't handle padding_free correctly
}
-# Hybrid linear-attention / state-space models (Qwen3.5, Qwen3-Next, ...) carry a
-# recurrent gated-delta state plus a causal conv1d that leak across sequence
-# boundaries once packing flattens the batch. Detected structurally by
-# _is_hybrid_linear_attention_model, not by model name.
+# Hybrid linear-attention / state-space models (Qwen3.5, Qwen3-Next) carry a recurrent gated-delta
+# state plus a causal conv1d that leak across sequence boundaries once packing flattens the batch.
+# Detected structurally by _is_hybrid_linear_attention_model, not by model name.
def _should_pack(config) -> bool:
@@ -196,7 +193,7 @@ def _cap_is_enforceable_without_padding_free(config, train, evals) -> bool:
except Exception:
return True # nothing to check against; do not invent a failure
packing = bool(getattr(config, "packing", False))
- # TRL's own default: `eval_packing = None` means "whatever `packing` is".
+ # TRL's own default: eval_packing = None means "whatever packing is".
eval_packing = getattr(config, "eval_packing", None)
eval_packing = packing if eval_packing is None else bool(eval_packing)
if not packing and not pretokenized_within_cap(train, cap):
@@ -345,9 +342,9 @@ def _resolve_string_model_config(model_name, config_arg):
from transformers import AutoConfig
init_kwargs = getattr(config_arg, "model_init_kwargs", None) or {}
- # why: forward auth + cache args too. Dropping token/use_auth_token made a
- # private hybrid fail to load (resolve as None) -> treated as non-hybrid ->
- # packing enabled without the shim even though TRL later loads it with the token.
+ # Forward auth and cache args too: dropping token/use_auth_token made a private hybrid fail to
+ # load (resolving as None), so it was treated as non-hybrid and packing was enabled without the
+ # shim.
forward = {
key: init_kwargs[key]
for key in (
@@ -361,10 +358,9 @@ def _resolve_string_model_config(model_name, config_arg):
)
if key in init_kwargs
}
- # why: TRL merges top-level args.trust_remote_code into the load via setdefault
- # before create_model_from_path, so honor it here (model_init_kwargs wins), else
- # a remote-code hybrid with SFTConfig(trust_remote_code=True) resolves as None
- # and skips the guard.
+ # TRL merges top-level args.trust_remote_code into the load via setdefault before
+ # create_model_from_path, so honor it here (model_init_kwargs wins), else a remote-code hybrid
+ # resolves as None and skips the guard.
top_level_trust_remote_code = getattr(config_arg, "trust_remote_code", None)
if top_level_trust_remote_code is not None:
forward.setdefault("trust_remote_code", top_level_trust_remote_code)
@@ -388,7 +384,7 @@ def _chunked_loss_bypasses_forward(config) -> bool:
return getattr(config, "loss_type", None) in (None, "chunked_nll")
-# Unsloth gradient accumulation fix:
+# Unsloth gradient accumulation fix.
from transformers import __version__ as transformers_version, ProcessorMixin
if Version(transformers_version) > Version("4.45.2"):
@@ -501,13 +497,11 @@ def _create_unsloth_optimizer(
class UnslothTrainer(SFTTrainer):
def create_optimizer(self):
- # --- Q-GaLore optimizer ---
q_galore_config = getattr(self.args, "q_galore_config", None)
if q_galore_config is not None and self.optimizer is None:
embedding_lr = getattr(self.args, "embedding_learning_rate", None)
return self._create_q_galore_optimizer(q_galore_config, embedding_lr)
- # --- Embedding-LR optimizer ---
embedding_learning_rate = getattr(self.args, "embedding_learning_rate", None)
if embedding_learning_rate is None:
return super().create_optimizer()
@@ -558,16 +552,16 @@ class UnslothTrainer(SFTTrainer):
# --- Split embedding params with custom LR (Fix #2) ---
if embedding_lr is not None:
- # Fast param->name lookup (O(N) instead of O(N*M))
+ # Fast param -> name lookup, O(N) instead of O(N*M).
param_to_name = {id(p): name for name, p in self.model.named_parameters()}
new_groups = []
for group in param_groups:
if "rank" in group:
- # GaLore group: keep as-is (no embeddings here)
+ # GaLore group: keep as-is, since no embeddings are here.
new_groups.append(group)
continue
- # Non-GaLore group: split out embedding params
+ # Non-GaLore group: split out embedding params.
embed_params = []
other_params = []
for p in group["params"]:
@@ -608,8 +602,8 @@ class UnslothTrainer(SFTTrainer):
group_size = config.weight_group_size,
stochastic = config.stochastic_round,
)
- # Pre-hooks dequantize INT8 weights to float before each forward,
- # letting the optimizer free float weight memory between steps.
+ # Pre-hooks dequantize INT8 weights to float before each forward, letting the optimizer free float
+ # weight memory between steps.
install_weight_quant_hooks(self.model)
n_galore = sum(len(g["params"]) for g in param_groups if "rank" in g)
@@ -623,8 +617,7 @@ class UnslothTrainer(SFTTrainer):
return self.optimizer
-# From `trl>=0.13.0`, they changed how to pass several params to the trainer
-# We need to patch to make the transition smooth
+# From trl >= 0.13.0 several params are passed to the trainer differently; patch to make the transition smooth.
def _resolve_trainer_params(trainer_class, init_fn):
"""Resolve the real named parameters for a trainer __init__.
@@ -641,7 +634,7 @@ def _resolve_trainer_params(trainer_class, init_fn):
if named:
return set(params.keys())
- # Thin wrapper detected - walk MRO for real signature
+ # Thin wrapper detected: walk the MRO for the real signature.
for cls in trainer_class.__mro__[1:]:
if cls is object:
continue
@@ -699,15 +692,13 @@ def _ensure_warnings_issued(model):
if existing is None:
model.warnings_issued = {}
else:
- # Preserve a non-dict value rather than discard it; trl only ever
- # writes one boolean key.
+ # Preserve a non-dict value rather than discard it; trl only ever writes one boolean key.
try:
model.warnings_issued = dict(existing)
except Exception:
model.warnings_issued = {}
except Exception:
- # A model refusing the assignment is trl's to report, not ours to turn
- # into a different traceback.
+ # A model refusing the assignment is trl's to report, not ours to turn into a different traceback.
pass
@@ -793,7 +784,7 @@ def _backwards_compatible_trainer(trainer_class, config_class):
@wraps(original_init)
def new_init(self, *args, **kwargs):
- # tokenizer is now processing_class
+ # tokenizer is now processing_class.
trainer_params = _resolve_trainer_params(trainer_class, original_init)
if "processing_class" in trainer_params and "tokenizer" in kwargs:
@@ -807,7 +798,7 @@ def _backwards_compatible_trainer(trainer_class, config_class):
trainer_params.discard("self")
trainer_params.discard("args")
- # Fields that should be passed to Config init
+ # Fields that should be passed to Config init.
config_fields = {
field.name: field for field in dataclasses.fields(config_class) if field.init
}
@@ -818,14 +809,14 @@ def _backwards_compatible_trainer(trainer_class, config_class):
if hasattr(training_args, name)
}
- # Params in Config but not in TrainingArguments
+ # Params in Config but not in TrainingArguments.
from transformers import TrainingArguments
moved_params = set(inspect.signature(config_class).parameters.keys()) - set(
inspect.signature(TrainingArguments).parameters.keys()
)
- # Separate kwargs into trainer kwargs and config kwargs
+ # Separate kwargs into trainer kwargs and config kwargs.
trainer_kwargs = {}
additional_config_kwargs = {}
unknown_kwargs = {}
@@ -849,22 +840,20 @@ def _backwards_compatible_trainer(trainer_class, config_class):
config_dict.update(additional_config_kwargs)
- # Only build the config if the previous init wasn't TrainingArguments:
- # reinitialising it would re-trigger mutually-exclusive param checks.
- # See https://github.com/huggingface/trl/blob/main/trl/trainer/grpo_config.py#L499-L502
+ # Only build the config if the previous init was not TrainingArguments: reinitialising it would re-
+ # trigger the mutually-exclusive param checks (trl grpo_config.py#L499-L502).
if not isinstance(training_args, TrainingArguments):
config = config_class(**config_dict)
else:
- # Every trl config subclasses TrainingArguments, so this is the
- # branch real calls take and config_dict was going nowhere. Set
- # the moved values on the caller's config rather than rebuild it.
+ # Every trl config subclasses TrainingArguments, so this is the branch real calls take and
+ # config_dict was going nowhere: set the moved values on the caller's config rather than rebuild it.
config = training_args
for key, value in additional_config_kwargs.items():
# `migrated` keys are already known to be accepted here.
if key in config_fields or key in moved_params or key in migrated:
setattr(config, key, value)
- # Reconstruct kwargs for Trainer
+ # Reconstruct kwargs for Trainer.
kwargs = trainer_kwargs
kwargs["args"] = config
_ensure_warnings_issued(args[0] if args else kwargs.get("model"))
@@ -912,11 +901,8 @@ def _patch_sft_trainer_auto_packing(trl_module):
else model
)
is_hybrid = _is_hybrid_linear_attention_model(hybrid_target)
- # Hybrid models corrupt packed batches unless the gated-delta conv + scan
- # reset at sequence boundaries. Enable the experimental varlen shim (flag +
- # kernels) so packing stays correct, else keep them blocked. A string model
- # (patched only after init) and TRL's chunked-loss forward bypass both leave
- # the shim off, so hybrid packing falls back to the padded path.
+ # Hybrid models corrupt packed batches unless the gated-delta conv and scan reset at sequence
+ # boundaries, so enable the experimental varlen shim (flag plus kernels) or keep them blocked.
if (
is_hybrid
and not isinstance(model, str)
@@ -953,9 +939,7 @@ def _patch_sft_trainer_auto_packing(trl_module):
or is_unsupported_model
or is_encoder_decoder
or (is_hybrid and not hybrid_varlen_active)
- or (
- os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1"
- ) # Disable padding free on forced logits
+ or (os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1")
)
requested_pack = bool(getattr(config_arg, "packing", False))
if blocked:
@@ -979,8 +963,8 @@ def _patch_sft_trainer_auto_packing(trl_module):
elif is_unsupported_model:
reason = f"unsupported model type(s): {', '.join(model_types)}"
elif data_collator is None:
- # compute_metrics, preprocess_logits_for_metrics, for_inference() and the
- # user can all set it, so name the flag and not a setter.
+ # compute_metrics, preprocess_logits_for_metrics, for_inference() and the user can all set it, so
+ # name the flag and not a setter.
reason = "UNSLOTH_RETURN_LOGITS=1"
logger.warning(f"Unsloth: packing=True ignored ({reason}).")
@@ -990,7 +974,7 @@ def _patch_sft_trainer_auto_packing(trl_module):
packing_active = True
logger.info("Unsloth: Sample packing enabled for SFTTrainer instance.")
- # Resolve padding_free: None (default) = auto-enable unless env-disabled or packing
+ # Resolve padding_free: None (default) means auto-enable unless env-disabled or packing.
auto_padding_free_active = False
padding_free_requested = getattr(config_arg, "padding_free", None) is True
if not blocked:
@@ -1047,11 +1031,9 @@ def _patch_sft_trainer_auto_packing(trl_module):
)
print(message)
- # get_peft_model installs a pre-train forward detector for plain LoRA/vision models,
- # but only RL trainers run the reset via prepare_for_training_mode. Wire it into the
- # SFT train() path too, else a grad-enabled probe before train() leaves the poisoned
- # Dynamo cache in place and the detector hook installed on every training forward.
- # (For UnslothSFTTrainer the later prepare_for_training_mode assignment supersedes this.)
+ # get_peft_model installs a pre-train forward detector for plain LoRA/vision models, but only RL
+ # trainers run the reset via prepare_for_training_mode, so wire it into the SFT train() path too,
+ # else a grad-enabled probe before train() leaves the poisoned Dynamo cache in place.
if not getattr(self, "_unsloth_train_reset_wrapped", False):
try:
from unsloth.models._utils import _unsloth_reset_stray_compile_cache
@@ -1090,10 +1072,8 @@ def _patch_trl_trainer():
trl_configs = set(x[: -len("Config")] for x in trl_classes if x.endswith("Config"))
trl_classes = list(trl_trainers & trl_configs)
- # Auto-packing wraps first so it lands INSIDE the backwards-compatible one: a
- # moved `packing` kwarg must reach the config before packing is decided, else
- # the block is undone right after. Guarded so a failure here still leaves the
- # pre-0.13 compatibility wrappers installed.
+ # Auto-packing wraps first so it lands INSIDE the backwards-compatible wrapper: a moved `packing`
+ # kwarg must reach the config before packing is decided, else the block is undone right after.
try:
_patch_sft_trainer_auto_packing(trl)
except Exception as exc:
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index cd3bbd678ad..4607092f181 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -42,13 +42,13 @@ HAS_XFORMERS = xformers is not None
def _xformers_runs_on_device() -> bool:
"""One tiny attention forward; True iff the xformers kernel actually runs here."""
try:
- # Pre-Ampere GPUs (sm < 80: Turing/Volta) have no bfloat16 attention kernel
- # but run xformers fine in float16, so pick the dtype the device supports.
+ # Pre-Ampere GPUs (sm < 80: Turing/Volta) have no bfloat16 attention kernel but run xformers fine in
+ # float16, so pick the dtype the device supports.
dtype = torch.bfloat16 if SUPPORTS_BFLOAT16 else torch.float16
q = torch.zeros((1, 8, 1, 64), device = "cuda", dtype = dtype)
attn_bias = xformers.attn_bias.BlockDiagonalCausalMask.from_seqlens([8])
xformers_attention(q, q, q, attn_bias = attn_bias)
- # Launches are async; synchronize so a deferred kernel failure fails the probe here.
+ # Launches are async, so synchronize or a deferred kernel failure escapes this probe.
torch.cuda.synchronize()
return True
except Exception:
@@ -56,30 +56,29 @@ def _xformers_runs_on_device() -> bool:
def _xformers_disabled_for_capability(capability, probe = _xformers_runs_on_device) -> bool:
- # At sm_120 (RTX 50-series) xformers' cutlass op is capability-rejected (caps at
- # sm_90) and its flash-2 op runs only if the build ships an sm_120 kernel, so run
- # one real forward to decide. Below sm_120 xformers always works; skip the probe.
+ # At sm_120 (RTX 50-series) xformers' cutlass op is capability-rejected (it caps at sm_90) and its
+ # flash-2 op runs only if the build ships an sm_120 kernel, so run one real forward to decide;
+ # below sm_120 xformers always works.
if capability[0] < 12:
return False
return not probe()
-# FlashAttention always wins in select_attention_backend and nothing downgrades
-# flash -> xformers, so when it's installed xformers is never selected: skip the probe.
+# FlashAttention always wins in select_attention_backend and nothing downgrades flash to xformers,
+# so when it is installed xformers is never selected: skip the probe.
if HAS_XFORMERS and not HAS_FLASH_ATTENTION and torch.cuda.is_available():
if _xformers_disabled_for_capability(torch.cuda.get_device_capability()):
HAS_XFORMERS = False
-# On sm_100+ (B200, sm_120) xformers' fp32-capable cutlass op is capability-rejected and
-# only its fp16/bf16 flash-2 op runs, so fp32 Q/K/V (DoRA, #1013) must be downcast there;
-# below sm_100 cutlass handles fp32 natively. Read once from device 0, like the probe gate.
+# On sm_100+ (B200, sm_120) xformers' fp32-capable cutlass op is capability-rejected and only its
+# fp16/bf16 flash-2 op runs, so fp32 Q/K/V (DoRA, #1013) must be downcast there; below sm_100
+# cutlass handles fp32 natively. Read once from device 0.
_XFORMERS_FP32_UNSUPPORTED = (
torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10
)
SDPA_HAS_GQA = "enable_gqa" in (scaled_dot_product_attention.__doc__ or "")
-# PrefixGrouper kernel, resolved once when the env gate is on so PG-off users never load
-# torch flex_attention.
+# PrefixGrouper kernel, resolved once when the env gate is on so PG-off users never load torch flex_attention.
_flex_shared_prefix_attention = None
if os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER", "1").lower() not in ("0", "false", "no", "off"):
try:
@@ -98,32 +97,11 @@ SDPA = "sdpa"
XFORMERS_BLOCK_DIAG_CLS = xformers.attn_bias.BlockDiagonalCausalMask if HAS_XFORMERS else None
+# flash-attn 2 varlen backward int32 overflow guard: the varlen BACKWARD kernel allocates
+# dq_accum = zeros(total_q + 128 * n_seqs, n_heads, round_up(head_dim, 32)) and indexes it with
+# int32, so at 2**31 elements the kernel faults with an illegal memory access and poisons the CUDA
+# context. Forward-only never allocates dq_accum, so the guard requires a backward to be possible.
# ---- flash-attn 2 varlen backward int32 overflow guard -------------------------------------
-# The varlen BACKWARD kernel (used by flash-attn 2 directly and by the flash-2 op xFormers
-# dispatches a BlockDiagonal* bias to) allocates
-#
-# dq_accum = zeros(total_q + 128 * n_seqs, n_heads, round_up(head_dim, 32), dtype = fp32)
-#
-# and indexes it with int32, so once that element count reaches 2**31 the kernel faults with
-# "CUDA error: an illegal memory access was encountered". That poisons the CUDA context, so the
-# run dies hours in, far from the cause.
-#
-# Measured on a B200 (bf16, one flattened row, forward + backward) by bisecting the document
-# count in a packed row: the last working count and the count at which
-# (total_q + 128 * n_seqs) * n_heads * round_up(head_dim, 32) crosses 2**31 agree to within one
-# document across every shape tried:
-#
-# heads head_dim doc_len predicted last OK
-# 16 128 1 8128 8129
-# 16 96 1 10837 10838
-# 16 64 1 16257 16257
-# 8 128 1 16257 >= 16257, fails by 16400
-# 16 128 2 8065 8060 OK, 8200 fails
-# 16 128 4 7944 7944
-# 4 128 1 32518 no failure up to 20000 (below the bound)
-#
-# Forward-only never allocates dq_accum and never faults (20000 documents ran clean), so the
-# guard requires a backward to be possible. Plain SDPA is unaffected.
_INT32_ELEMENTS = 2**31
_VARLEN_INT32_GUARD_DISABLED = os.environ.get(
"UNSLOTH_DISABLE_VARLEN_INT32_GUARD", "0"
@@ -223,8 +201,8 @@ class AttentionContext:
attention_mask: Optional[Tensor]
causal_mask: Optional[Any]
sliding_window: Optional[int] = None
- # PrefixGrouper: non-None routes Q/K/V through the FlexAttention shared-prefix kernel;
- # None leaves every existing construction/behavior unchanged.
+ # PrefixGrouper: non-None routes Q/K/V through the FlexAttention shared-prefix kernel; None leaves
+ # every existing construction and behavior unchanged.
prefix_seg_info: Optional[Any] = None
@@ -268,10 +246,9 @@ def resolve_prefix_seg_info(kwargs, past_key_value, attention_mask):
return seg
-# One dense window mask per (device, shape, window), reused across layers. Every layer of a
+# One dense window mask per (device, shape, window), reused across layers: every layer of a
# Mistral-style model asks for the identical mask, and at 32K that tensor is 1 GiB with two more
-# alive while it is built, so rebuilding it 32 times is how this SDPA fallback OOMs a run that
-# xFormers or flash would have carried. Same single-entry-per-device shape as _SDPA_MASK_CACHE.
+# alive while it is built, so rebuilding it 32 times OOMs the run.
_WINDOW_MASK_CACHE: dict = {}
@@ -281,8 +258,8 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -
entry = _WINDOW_MASK_CACHE.get(device)
if entry is not None and entry["params"] == params:
return entry["mask"]
- # Drop the outgoing mask first. It is dead either way, and holding it while the replacement
- # and its temporaries are allocated would make a shape change peak a whole mask higher.
+ # Drop the outgoing mask first: it is dead either way, and holding it while the replacement and its
+ # temporaries are allocated would make a shape change peak a whole mask higher.
_WINDOW_MASK_CACHE.pop(device, None)
entry = None
q_pos = torch.arange(k_len - q_len, k_len, device = device)
@@ -307,14 +284,12 @@ def run_attention(
and SDPA handle packing via a block-diagonal mask.
"""
- # PrefixGrouper shared-prefix attention (GRPO dedup). Q/K/V here are [bsz, H, T, D];
- # the kernel takes/returns [1, T, H, D], matching the other backends. The field is
- # only set when the env gate is on and grouping succeeded; None keeps every backend
- # byte-identical.
+ # PrefixGrouper shared-prefix attention (GRPO dedup): Q/K/V here are [bsz, H, T, D] while the kernel
+ # takes and returns [1, T, H, D], matching the other backends.
if context.prefix_seg_info is not None:
flex_shared_prefix_attention = _flex_shared_prefix_attention
if flex_shared_prefix_attention is None:
- # gate flipped on after import (or one-time load failed): resolve lazily.
+ # Gate flipped on after import, or a one-time load failed: resolve lazily.
from ..utils.prefix_grouper_kernel import flex_shared_prefix_attention
scale = None
@@ -333,8 +308,8 @@ def run_attention(
if backend == FLASH_VARLEN and context.seq_info is None:
backend = FLASH_DENSE if HAS_FLASH_ATTENTION else SDPA
- # [TODO] Flash/xFormers don't support arbitrary attn masks; with a padding
- # mask present (e.g. left-padded generation), fall back to SDPA.
+ # Flash/xFormers do not support arbitrary attn masks, so with a padding mask present (left-padded
+ # generation) fall back to SDPA.
if context.attention_mask is not None and backend in (
FLASH_DENSE,
FLASH_VARLEN,
@@ -342,15 +317,13 @@ def run_attention(
):
backend = SDPA
- # Both varlen-capable backends land in the same flash-attn 2 backward kernel, so guard both
- # before the int32 overflow aborts the process. Integer arithmetic only, no device sync.
+ # Both varlen-capable backends land in the same flash-attn 2 backward kernel, so guard both before
+ # the int32 overflow aborts the process. Integer arithmetic only, no device sync.
if backend in (FLASH_VARLEN, XFORMERS) and not _VARLEN_INT32_GUARD_DISABLED:
- # Both terms are needed. Q/K/V: a frozen hidden state feeding trainable LoRA q/k/v
- # still yields a Q that requires grad, which context.requires_grad alone would miss.
- # context.requires_grad: gradient checkpointing runs its FIRST forward under no_grad
- # and recomputes with grad on, so keying only on the tensors would pick a different
- # backend per pass and the backward would see activations the loss never came from.
- # The hidden state keeps requires_grad = True across both passes; inference has it False.
+ # Both terms are needed. Q/K/V: a frozen hidden state feeding trainable LoRA q/k/v still yields a
+ # Q that requires grad. context.requires_grad: gradient checkpointing runs its FIRST forward under
+ # no_grad and recomputes with grad on, so keying only on the tensors would pick a different
+ # backend per pass.
will_backward = context.requires_grad or (
torch.is_grad_enabled() and (Q.requires_grad or K.requires_grad or V.requires_grad)
)
@@ -360,9 +333,8 @@ def run_attention(
n_seqs = seq_info[0].numel() if seq_info is not None else context.bsz
total_q = context.bsz * context.q_len
if _varlen_backward_overflows_int32(n_seqs, total_q, context.n_heads, context.head_dim):
- # SDPA cannot apply logit softcapping, so rerouting a softcapped model (Gemma 2)
- # would keep it training on wrong logits and gradients -- worse than the fault
- # this guard avoids. Stop and name the ways out instead.
+ # SDPA cannot apply logit softcapping, so rerouting a softcapped model (Gemma 2) would keep it
+ # training on wrong logits and gradients, worse than the fault this guard avoids.
softcap = _configured_softcap(config)
if softcap:
raise RuntimeError(
@@ -399,15 +371,13 @@ def run_attention(
kv_seq_len = context.kv_seq_len
requires_grad = context.requires_grad
sliding_window = context.sliding_window
- # A non-positive window means "no local attention", not "a window of nothing": a config
- # spelling it 0 would otherwise put the mask's lower bound above its causal upper bound
- # and hide every position from every other.
+ # A non-positive window means "no local attention", not "a window of nothing": a config spelling
+ # it 0 would put the mask's lower bound above its causal upper bound.
if sliding_window is not None and sliding_window <= 0:
sliding_window = None
- # DoRA promotes q/k/v_proj outputs to fp32, which FlashAttention rejects (and so does
- # the xformers flash-2 op on sm_100+, see _XFORMERS_FP32_UNSUPPORTED), so downcast any
- # fp32 Q/K/V to a supported dtype (#1013).
+ # DoRA promotes q/k/v_proj outputs to fp32, which FlashAttention rejects (as does the xformers
+ # flash-2 op on sm_100+), so downcast any fp32 Q/K/V to a supported dtype (#1013).
if (
backend in (FLASH_DENSE, FLASH_VARLEN)
or (backend == XFORMERS and _XFORMERS_FP32_UNSUPPORTED)
@@ -530,7 +500,8 @@ def run_attention(
local_mask = local_mask.to(device = Q.device)
if local_mask.dim() == 2:
- # key padding keep mask: (bsz, k_len), 1/True = real token
+ # Key padding keep mask (bsz, k_len), where 1/True is a real token; the tokenizer attention_mask is
+ # typically int 0/1.
if local_mask.dtype == torch.bool:
key_keep = local_mask
else:
@@ -545,7 +516,7 @@ def run_attention(
if sliding_window is not None:
causal_keep &= k_pos[None, :] >= (q_pos[:, None] - (sliding_window - 1))
- # (bsz, 1, q_len, k_len) boolean keep mask
+ # (bsz, 1, q_len, k_len) boolean keep mask.
local_mask = causal_keep[None, None, :, :] & key_keep[:, None, None, :]
elif local_mask.dim() == 3:
@@ -559,16 +530,14 @@ def run_attention(
else:
raise ValueError(f"Unsupported SDPA attention_mask rank: {local_mask.dim()}")
- # Avoid NaNs from fully-masked rows (common with left padding).
+ # Avoid NaNs from fully-masked rows, common with left padding.
if local_mask.dtype == torch.bool:
no_allowed = ~local_mask.any(dim = -1, keepdim = True) # (bsz,1,q_len,1)
local_mask = local_mask | no_allowed
if local_mask is None and sliding_window is not None and k_len_local > sliding_window:
- # SDPA's is_causal is FULL causal; it has no window. With no padding mask to
- # hang the window off, a model whose config declares one attended its whole
- # history whenever neither the xformers bias nor flash's window_size was the
- # thing running.
+ # SDPA's is_causal is FULL causal and has no window, so with no padding mask to hang the window
+ # off, a model whose config declares one attended its whole history.
local_mask = _windowed_causal_mask(
q_len_local, k_len_local, sliding_window, Q.device
)
@@ -587,8 +556,8 @@ def run_attention(
and local_mask.dim() >= 3
and local_mask.shape[0] > 1
):
- # Batched masked inference has shown row-coupled drift with SDPA GQA.
- # Fall back to explicit KV expansion for deterministic row-wise behavior.
+ # Batched masked inference has shown row-coupled drift with SDPA GQA, so fall back to explicit KV
+ # expansion for deterministic row-wise behavior.
use_sdpa_gqa = False
if use_sdpa_gqa:
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index 3262589a6d8..1e460d7791f 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -74,10 +74,9 @@ def move_xformers_attention_bias(attn_bias: Any, device: torch.device):
):
return attn_bias
- # Move the device-bearing metadata instead of the top-level mask. Older
- # xFormers versions demote causal masks in their inherited ``to`` method.
- # Copies also keep later model shards from rewriting masks retained for
- # backward by earlier shards.
+ # Move the device-bearing metadata instead of the top-level mask: older xFormers versions demote
+ # causal masks in their inherited `to` method, and copies also keep later model shards from
+ # rewriting masks retained for backward.
moved_bias = copy.copy(attn_bias)
moved_seqinfos = {}
for name, seqinfo in seqinfos:
@@ -214,21 +213,19 @@ def enable_sample_packing(
lengths = example.get(sequence_lengths_key)
if isinstance(lengths, Iterable):
seq_lengths.extend(int(length) for length in lengths)
- # Fallback: infer lengths from tokenized inputs when metadata is absent
+ # Fallback: infer lengths from tokenized inputs when metadata is absent.
if not seq_lengths:
for example in examples:
ids = example.get("input_ids")
if isinstance(ids, Iterable):
seq_lengths.append(len(ids))
if seq_lengths:
- # Boundary labels are NOT masked here. unsloth_zoo's
- # _unsloth_get_batch_samples counts num_items_in_batch off this batch and
- # discounts the N-1 boundary targets itself, idempotently: it zeroes those
- # slots rather than subtracting a constant, so the count is unaffected by
- # upstream masking (TRL >= 0.24's labels[position_ids == 0] = -100,
- # completion-only masking, assistant_masks). Masking here would be harmless
- # to the count; labels are left alone because the guard that needs these
- # positions runs in the forward, off packed_seq_lengths.
+ # Boundary labels are NOT masked here: unsloth_zoo's _unsloth_get_batch_samples counts
+ # num_items_in_batch off this batch and discounts the N-1 boundary targets itself, idempotently,
+ # zeroing those slots rather than subtracting a constant, so the count is unaffected by upstream
+ # masking (TRL >= 0.24's labels[position_ids == 0] = -100, completion-only masking,
+ # assistant_masks). Labels are left alone because the guard that needs these positions runs in the
+ # forward, off packed_seq_lengths.
batch["packed_seq_lengths"] = torch.tensor(seq_lengths, dtype = torch.int32)
if "attention_mask" in batch:
batch.pop("attention_mask")
@@ -271,9 +268,8 @@ def enable_padding_free_metadata(model, trainer):
batch = original_torch_call(examples)
if seq_lengths:
- # Labels left alone for the same reason as enable_sample_packing:
- # num_items_in_batch is counted off this batch, and the zoo's discount of the
- # boundary targets is idempotent, so masked slots do not change the count.
+ # Labels left alone for the same reason as enable_sample_packing: num_items_in_batch is counted off
+ # this batch and the zoo's discount of the boundary targets is idempotent.
batch["packed_seq_lengths"] = torch.tensor(
seq_lengths,
dtype = torch.int32,
@@ -284,17 +280,12 @@ def enable_padding_free_metadata(model, trainer):
collator._unsloth_padding_free_lengths_wrapped = True
-# --- Experimental: correct packing / padding-free for hybrid linear-attention ---
-# Qwen3.5 / Qwen3-Next mix a gated-delta recurrence with a causal conv1d. Packing
-# flattens the batch, and both ops leak state across sequence boundaries unless we
-# pass seq_idx (conv) and cu_seqlens (scan). Only the accelerated kernels accept
-# these, so we fail closed on the pure-torch fallbacks. Gated behind an env flag.
-#
-# Overrides only the per-module prefill kernels (causal_conv1d_fn /
-# chunk_gated_delta_rule), leaving decode untouched so generation is unaffected.
-# Recompute-safe under gradient checkpointing; never fires for cached forwards.
-# Feature-detect (never version-detect), fail closed, idempotent, one deduped
-# diagnostic when it declines to activate.
+# Experimental correct packing / padding-free for hybrid linear-attention: Qwen3.5 / Qwen3-Next
+# mix a gated-delta recurrence with a causal conv1d, and packing flattens the batch so both leak
+# state across sequence boundaries unless seq_idx (conv) and cu_seqlens (scan) are passed. Only
+# the accelerated kernels accept these, so it fails closed on the pure-torch fallbacks, behind an
+# env flag, and overrides only the per-module prefill kernels (causal_conv1d_fn /
+# chunk_gated_delta_rule), leaving decode untouched.
_HYBRID_PACKING_ENV_VAR = "UNSLOTH_EXPERIMENTAL_HYBRID_PACKING"
_HYBRID_LOGGER = logging.getLogger("unsloth.hybrid_packing")
_HYBRID_WARNED: set = set()
@@ -447,7 +438,7 @@ def _hybrid_varlen_metadata(kwargs):
if total is None:
return None
psl = kwargs.get("packed_seq_lengths")
- if psl is not None and getattr(psl, "numel", lambda: 1)() > 0: # skip empty (no max())
+ if psl is not None and getattr(psl, "numel", lambda: 1)() > 0:
info = get_packed_info_from_kwargs(kwargs, device)
if info is not None:
_, cu_seqlens, _ = info
@@ -520,9 +511,8 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
module._unsloth_varlen = None
module._unsloth_varlen_wrapped = True
- # Refresh the boundary stash on the outermost forward (once per step, outside
- # gradient-checkpoint recompute, so it stays valid for recomputed inner
- # forwards). Read from both positional and keyword args via the bound signature.
+ # Refresh the boundary stash on the outermost forward, once per step and outside gradient-checkpoint
+ # recompute so it stays valid for recomputed inner forwards.
if not getattr(model, "_unsloth_varlen_forward_wrapped", False):
forward_orig = model.forward
try:
@@ -550,12 +540,8 @@ def patch_hybrid_linear_attention_varlen(model) -> bool:
module._unsloth_varlen_conv_hit = False
module._unsloth_varlen_scan_hit = False
out = forward_orig(*args, **kwargs)
- # Runtime dispatch handshake: on the first packed forward, confirm BOTH
- # boundary kernels ran for EVERY module. seq_idx (conv) and cu_seqlens
- # (scan) are both load-bearing, so a partial/absent dispatch (a future
- # version no longer routing through self.<kernel>) leaves cross-sequence
- # contamination. The batch is already flattened with no padded recovery,
- # so abort before loss/backward rather than train on corrupted data.
+ # Runtime dispatch handshake: on the first packed forward, confirm BOTH boundary kernels ran for
+ # EVERY module.
if first_pack:
model._unsloth_varlen_handshake_done = True
missing = [
diff --git a/unsloth/utils/prefix_grouper.py b/unsloth/utils/prefix_grouper.py
index 4e6ff9672ce..ad89d907821 100644
--- a/unsloth/utils/prefix_grouper.py
+++ b/unsloth/utils/prefix_grouper.py
@@ -57,14 +57,12 @@ import torch
from .prefix_grouper_kernel import build_seg_info_multigroup, PrefixSegInfo
-# ---------------------------------------------------------------------------
-# Env helpers
# ---------------------------------------------------------------------------
def env_on(name: str, default: str = "0") -> bool:
return os.environ.get(name, default).lower() not in ("0", "false", "no", "off")
-# One-time env reads; the helpers stay callable since unsloth_zoo imports and calls them.
+# One-time env reads; the helpers stay callable, since unsloth_zoo imports and calls them.
_ENABLED = env_on("UNSLOTH_GRPO_SEQ_PACKING", "1") and env_on("UNSLOTH_GRPO_PREFIX_GROUPER", "1")
_VERIFY_ON = env_on("UNSLOTH_GRPO_PREFIX_GROUPER_VERIFY", "1")
_TOKR_THRESHOLD = float(os.environ.get("UNSLOTH_GRPO_PREFIX_GROUPER_TOKR", "1.3"))
@@ -88,8 +86,8 @@ def tol_ok() -> float:
return _TOL_OK
-# diff >= TOL_KILL = broken mask/isolation -> structure permanently unsafe; between
-# tol_ok and TOL_KILL -> fall back for this shape but keep trying others.
+# A diff at or above TOL_KILL means a broken mask/isolation, so the structure is permanently unsafe;
+# between tol_ok and TOL_KILL, fall back for this shape but keep trying others.
TOL_KILL = 1.5
@@ -100,7 +98,7 @@ class GroupLayout:
flat_ids: torch.Tensor # [1, T] (T == seg.T)
position_ids: torch.Tensor # [1, T]
prefix_seg_info: PrefixSegInfo
- # per completion target token, aligned 1:1:
+ # Per completion target token, aligned 1:1.
tgt_rows: torch.Tensor # [N] original row index
tgt_cols: torch.Tensor # [N] original padded column in that row
tgt_pred: torch.Tensor # [N] flat predicting index (into the T stream)
@@ -124,8 +122,8 @@ class GroupLayout:
) -> torch.Tensor:
"""hidden: [1, T, Hdim] (pre-lm_head hidden states, UNSLOTH_RETURN_HIDDEN_STATES=1).
Returns [total_rows, W] float32, byte-compatible with the packed path result."""
- # In a sharded model hidden may live on the lm-head device; move the small index
- # maps to hidden.device before indexing.
+ # In a sharded model hidden may live on the lm-head device, so move the small index maps to
+ # hidden.device before indexing.
device = hidden.device
pred_h = hidden[0, self.tgt_pred.to(device), :].unsqueeze(0) # [1, N, Hdim]
tgt_ids = self.flat_ids[0, self.tgt_flat].to(device).unsqueeze(0) # [1, N]
@@ -179,7 +177,7 @@ def _build_groups(ids_cpu, real_cols_cpu, cstart_cpu, num_generations, total_row
comp_cols_per_row.append(c_cols)
if any(len(p) == 0 for p in prompt_toks_per_row):
return None
- # require BYTE-IDENTICAL prompts across the group (shared-prefix precondition).
+ # Require BYTE-IDENTICAL prompts across the group, the shared-prefix precondition.
P = len(prompt_toks_per_row[0])
if any(len(prompt_toks_per_row[k]) != P for k in range(1, G)):
return None
@@ -236,12 +234,12 @@ def build_group_layout(
device = input_ids.device
total_rows, L = input_ids.shape
keep = input_ids != pad_id
- # completion start column per row (matches create_completion_attention_mask / _pk_cstart).
+ # Completion start column per row, matching create_completion_attention_mask / _pk_cstart.
cstart = ((L - logits_to_keep) - left_pad_tokens_per_prompt).to(torch.long)
cstart_cpu = cstart.tolist()
ids_cpu = input_ids.tolist()
- # per-row real (non-pad) columns. GRPO rows are one contiguous real run, so derive
- # [first, first+n) on GPU; the O(B*L) scan is only a non-contiguous fallback.
+ # Per-row real (non-pad) columns: GRPO rows are one contiguous real run, so derive [first, first+n)
+ # on GPU; the O(B*L) scan is only a non-contiguous fallback.
n_real = keep.sum(dim = 1)
first = torch.argmax(keep.to(torch.int8), dim = 1)
ar = torch.arange(L, device = device)
@@ -258,7 +256,7 @@ def build_group_layout(
if groups is None:
return None
- # sliding-window guard: a group's PG span is P + max(R); fall back if it exceeds the window.
+ # Sliding-window guard: a group's PG span is P + max(R), so fall back if it exceeds the window.
if max_segment_cap is not None:
for gm in groups:
if gm["P"] + max(gm["R_list"]) > max_segment_cap:
@@ -286,12 +284,12 @@ def build_group_layout(
r0 = gm["prefix_row"]
prefix_cols = gm["prefix_cols"] # ORIGINAL real prompt columns (len P) of row0
plast = meta["prefix_last_index"] # base + P - 1
- # gather the shared prefix once, from row0.
+ # Gather the shared prefix once, from row0.
flat_src_rows.extend([r0] * P)
flat_src_cols.extend(prefix_cols)
pos_list.extend(range(P))
- # suffixes: every suffix token is a completion-region target (scattered like the
- # packed path; completion_mask hides prompt-tail positions).
+ # Suffixes: every suffix token is a completion-region target, scattered like the packed path, with
+ # completion_mask hiding prompt-tail positions.
for i, r in enumerate(rows):
cols = gm["suf_cols"][i]
r_i = len(cols)
@@ -300,7 +298,7 @@ def build_group_layout(
flat_src_cols.extend(cols)
pos_list.extend(range(P, P + r_i))
for j in range(r_i):
- # pos 0 is predicted from the prefix's last token; j>=1 from the previous suffix token.
+ # Position 0 is predicted from the prefix's last token; j >= 1 from the previous suffix token.
pred = plast if j == 0 else (s + j - 1)
tgt_rows.append(r)
tgt_cols.append(cols[j]) # ORIGINAL padded column in row r
@@ -317,9 +315,8 @@ def build_group_layout(
max_left_pad = int(left_pad_tokens_per_prompt.max().item()) if total_rows else 0
W = logits_to_keep + max_left_pad
- # self-verify cache key: the mask/index-map/scatter logic is structural, so key on
- # (num_groups, group_sizes), not exact lengths -- GRPO lengths change every step and
- # keying on T would re-verify forever ("verify once, then trust", like the packed path).
+ # Self-verify cache key: the mask/index-map/scatter logic is structural, so key on (num_groups,
+ # group_sizes), not exact lengths.
grp_sizes = tuple(sorted(len(gm["R_list"]) for gm in groups))
sig = (len(groups), grp_sizes)
diff --git a/unsloth/utils/prefix_grouper_kernel.py b/unsloth/utils/prefix_grouper_kernel.py
index 9a9719b0157..34457a6d74d 100644
--- a/unsloth/utils/prefix_grouper_kernel.py
+++ b/unsloth/utils/prefix_grouper_kernel.py
@@ -60,23 +60,22 @@ from torch.nn.attention.flex_attention import (
flex_attention,
)
-# GRPO feeds many distinct segment lengths; at dynamo's default recompile_limit (8) the
-# compiled kernel silently reuses a mismatched specialisation (wrong results). Raise it.
+# GRPO feeds many distinct segment lengths, and at dynamo's default recompile_limit (8) the compiled
+# kernel silently reuses a mismatched specialisation, giving wrong results.
torch._dynamo.config.recompile_limit = max(getattr(torch._dynamo.config, "recompile_limit", 8), 256)
torch._dynamo.config.accumulated_recompile_limit = max(
getattr(torch._dynamo.config, "accumulated_recompile_limit", 256), 2048
)
-# Compiled kernels: torch.compile fuses the sparse mask into one kernel. dynamic=True is
-# required: T changes almost every GRPO batch and dynamic=False recompiles per T (~14s
-# each). T is still padded to a multiple of 128 (_pad_len) for the backward kernel.
+# Compiled kernels: torch.compile fuses the sparse mask into one kernel, and dynamic=True is
+# required, since T changes almost every GRPO batch and dynamic=False recompiles per T (~14s each).
_flex_attention_compiled = torch.compile(flex_attention, dynamic = True)
_create_block_mask_compiled = torch.compile(create_block_mask, dynamic = True)
-# Flash block sizes by Q dtype (env-overridable). The two disjoint key runs (prefix +
-# own-suffix) stress online-softmax accumulation: fp32 needs 32/32 for a ~1e-6 floor;
-# bf16 passes parity at 128/64 and is ~5x faster (128/128 OOMs Triton on B200).
+# Flash block sizes by Q dtype (env-overridable): the two disjoint key runs stress online-softmax
+# accumulation, so fp32 needs 32/32 for a ~1e-6 floor, while bf16 passes parity at 128/64 and is
+# ~5x faster (128/128 OOMs Triton on B200).
_FP32_BLOCK_M = int(os.environ.get("PG_FLEX_BLOCK_M", "32"))
_FP32_BLOCK_N = int(os.environ.get("PG_FLEX_BLOCK_N", "32"))
_BF16_BLOCK_M = int(os.environ.get("PG_FLEX_BF16_BLOCK_M", "128"))
@@ -93,9 +92,9 @@ def _kernel_options_for_dtype(dtype):
# Backward-compat constant (fp32 default).
_FLEX_KERNEL_OPTIONS = {"BLOCK_M": _FP32_BLOCK_M, "BLOCK_N": _FP32_BLOCK_N}
-# The compiled backward trips an Inductor assertion when T is not a multiple of 128, so
-# pad the flat sequence. Pad tokens form a group that attends to / is attended by nothing
-# (all-masked rows return 0, not NaN) and are sliced off the output.
+# The compiled backward trips an Inductor assertion when T is not a multiple of 128, so pad the
+# flat sequence: pad tokens attend to and are attended by nothing (all-masked rows return 0) and
+# are sliced off the output.
_PAD_MULTIPLE = 128
_PAD_GROUP = -99 # sentinel group id / suffix id for pad tokens
@@ -105,8 +104,6 @@ def _pad_len(T: int) -> int:
# ---------------------------------------------------------------------------
-# Segment metadata
-# ---------------------------------------------------------------------------
@dataclass
@@ -213,7 +210,7 @@ def build_seg_info_multigroup(
suffix_counter = 0
sig_parts = []
for gid, (P, R_list) in enumerate(group_specs):
- # prefix
+ # Prefix.
group_of_list.append(torch.full((P,), gid, dtype = torch.long, device = device))
is_prefix_list.append(torch.ones(P, dtype = torch.bool, device = device))
suffix_of_list.append(torch.full((P,), -1, dtype = torch.long, device = device))
@@ -256,9 +253,10 @@ def build_seg_info_multigroup(
return seg, group_meta
+# Block-mask builder and cache, keyed on (signature, device): the mask depends only on the per-token
+# labels and T, so it is reused across layers and steps.
+
# ---------------------------------------------------------------------------
-# Block-mask builder + cache, keyed on (signature, device): the mask depends only on the
-# per-token labels and T, so it is reused across layers and steps.
_BLOCK_MASK_CACHE: Dict[Tuple, BlockMask] = {}
@@ -309,12 +307,9 @@ def get_block_mask(
return bm
# Move labels to the consumer (Q) device: with a sharded model the seg tensors live on
- # input_ids.device and would index cross-device. Copies once per (signature, device).
- # These copies must also run with inference mode DISABLED (same reason as the mask build):
- # when this entry is first built under the no-grad old/ref forward's inference_mode and
- # device != seg.device, a .to(device) copy would be an inference tensor that mask_mod
- # captures, which then cannot be saved for backward when the grad training forward reuses
- # the cached mask.
+ # input_ids.device and would index cross-device. These copies must also run with inference mode
+ # DISABLED, like the mask build: built under the no-grad forward's inference_mode, a .to(device)
+ # copy would be an inference tensor that mask_mod captures and could not be saved for backward.
builder = _create_block_mask_compiled if compile_mask else create_block_mask
with torch.inference_mode(False):
mask_mod = _make_mask_mod(
@@ -364,7 +359,7 @@ def _run_flex(q, k, v, block_mask, enable_gqa, scale, compiled, T, T_pad):
kernel_options = _kernel_options_for_dtype(qp.dtype),
)
else:
- # eager path (fp64 parity): dense scores, no kernel_options.
+ # Eager path (fp64 parity): dense scores, no kernel_options.
out = flex_attention(
qp,
kp,
@@ -377,8 +372,6 @@ def _run_flex(q, k, v, block_mask, enable_gqa, scale, compiled, T, T_pad):
# ---------------------------------------------------------------------------
-# The kernel entry point
-# ---------------------------------------------------------------------------
def flex_shared_prefix_attention(
@@ -422,7 +415,7 @@ def flex_shared_prefix_attention(
block_mask = get_block_mask(prefix_seg_info, device, compile_mask = compiled)
out = _run_flex(q, k, v, block_mask, enable_gqa, scale, compiled, T, T_pad)
- # back to [1, T, n_heads, D]
+ # Back to [1, T, n_heads, D].
return out.transpose(1, 2).contiguous()