Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Reflow comments in studio/backend/core/inference to 120 columns (#10115)

Daniel Hanf580f73e156a
diff --git a/studio/backend/core/inference/__init__.py b/studio/backend/core/inference/__init__.py
index 1491dfa749e..b059f06515c 100644
--- a/studio/backend/core/inference/__init__.py
+++ b/studio/backend/core/inference/__init__.py
@@ -42,7 +42,7 @@ def __getattr__(name):
from importlib import import_module
value = getattr(import_module(f"{__name__}.{submodule}"), attr)
- globals()[name] = value # cache so later access skips __getattr__
+ globals()[name] = value
return value
@@ -50,7 +50,7 @@ def __dir__():
return sorted(set(globals()) | set(__all__))
-if TYPE_CHECKING: # keep static analysers / IDEs aware of the lazy names
+if TYPE_CHECKING:
from .llama_cpp import LlamaCppBackend
from .orchestrator import InferenceOrchestrator, get_inference_backend
InferenceBackend = InferenceOrchestrator
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.py
index 5f207176e6b..eef2e9bb6c4 100644
--- a/studio/backend/core/inference/_html_to_md.py
+++ b/studio/backend/core/inference/_html_to_md.py
@@ -40,11 +40,10 @@ _SKIP_TAGS = frozenset(
"datalist",
}
)
-# <aside> is NOT skipped: docs use it for admonition callouts (real content);
-# page-furniture asides are excluded by the main-content scoping pass instead.
+# <aside> is NOT skipped: docs use it for admonition callouts (real content); page-furniture asides are excluded by the
+# main-content scoping pass instead.
-# Void elements never produce an end tag, so they must not join the
-# open-element stack used to bound hidden subtrees.
+# void elements never produce an end tag, so they must not join the open-element stack used to bound hidden subtrees
_VOID_TAGS = frozenset(
{
"area",
@@ -109,9 +108,8 @@ def _is_aria_heading(attr_dict: dict) -> bool:
return "heading" in (attr_dict.get("role") or "").lower().split()
-# HTML5 optional end tags: a listed start tag implicitly closes an open element
-# of the key type (as browsers do), else an unclosed ``<p hidden>``/``<li hidden>``
-# swallows every following sibling. Keys: closable elements; values: closers.
+# HTML5 optional end tags: a listed start tag implicitly closes an open element of the key type (as browsers do), else
+# an unclosed ``<p hidden>``/``<li hidden>`` swallows every following sibling. Keys: closable elements; values: closers.
_P_CLOSING_TAGS = frozenset(
{
"address",
@@ -159,9 +157,10 @@ _IMPLICIT_CLOSERS: dict = {
}
-# Item tag -> container tags that re-scope it: a nested container makes an inner
-# item a descendant, not an optional-close sibling, so recovery must stop there
-# rather than close (and un-hide) the outer item and leak its nested content.
+# a nested container makes an inner item a descendant
+# Item tag -> container tags that re-scope it: a nested container makes an inner item a descendant, not an
+# optional-close sibling, so recovery must stop there rather than close (and un-hide) the outer item and leak its nested
+# content.
_CLOSE_BARRIERS: dict = {
"li": frozenset({"ul", "ol", "menu"}),
"dt": frozenset({"dl"}),
@@ -194,14 +193,14 @@ _BLOCK_TAGS = frozenset(
_HEADING_TAGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
_INLINE_EMPHASIS = {"strong": "**", "b": "**", "em": "*", "i": "*"}
-# Measured density: 0.94-1.00 for link lists, 0.13-0.90 for content headers.
+# measured density: 0.94-1.00 for link lists, 0.13-0.90 for content headers
_HEADER_LINK_DENSITY = 0.93
-# Below this the ratio is noise: link lists start at 182 chars, link-dense headers stop at 93.
+# below this the ratio is noise: link lists start at 182 chars, link-dense headers stop at 93
_HEADER_MIN_CHARS = 150
# Short labels hide huge hrefs, so size the render too: content peaks at 363, link lists at 1609+.
_HEADER_MAX_RENDERED_CHARS = 800
-# Pages nest headers one or two deep; past this, closing a frame cannot copy an unbounded chain.
+# pages nest headers one or two deep; past this, closing a frame cannot copy an unbounded chain
_MAX_HEADER_NESTING = 8
@@ -242,19 +241,19 @@ class _HeaderFrame:
self.parts: list[str] = []
# Teed, so a heading routed through a nested buffer survives.
self.heading_parts: list[str] = []
- self.stripped: bool = False # set by render()
+ self.stripped: bool = False
# Tallied on emit; re-cleaning each parent's buffer would be quadratic.
self.rendered_chars: int = 0
self.heading_chars: int = 0
- # Buffers open now enclose the frame, later ones nest. Link/cell sequence numbers, not
- # flags: an inner one replaces the outer in the renderer's single slot.
+ # Buffers open now enclose the frame, later ones nest. Link/cell sequence numbers, not flags: an inner one
+ # replaces the outer in the renderer's single slot.
self.outer_list_depth = list_depth
self.outer_link_seq = link_seq
self.outer_cell_seq = cell_seq
self.outer_in_pre = in_pre
self.outer_in_code = in_code
self.outer_bq_depth = bq_depth
- # Both exclude heading text: it is kept anyway, so it must not vote on dropping the rest.
+ # both exclude heading text: it is kept anyway, so it must not vote on dropping the rest
self.text_chars: int = 0
self.link_chars: int = 0
@@ -267,13 +266,12 @@ class _HeaderFrame:
if not closed_by_own_tag:
return "".join(self.parts)
headings = "".join(self.heading_parts)
- # Droppable chars only; headings survive, and blank structure _cleanup collapses must not
- # inflate a tiny header.
+ # droppable chars only: headings survive, and blank structure _cleanup collapses must not inflate a tiny header
droppable = self.rendered_chars - self.heading_chars
big_enough = self.text_chars >= _HEADER_MIN_CHARS or droppable >= _HEADER_MAX_RENDERED_CHARS
if big_enough and self.link_chars >= _HEADER_LINK_DENSITY * self.text_chars:
self.stripped = True
- # The closing tag's blank line lands after the heading mark pops, so terminate it here.
+ # the closing tag's blank line lands after the heading mark pops, so terminate it here
return headings + "\n\n" if headings.strip() else headings
return "".join(self.parts)
@@ -299,12 +297,11 @@ class _MarkdownRenderer(HTMLParser):
self._scope_tags = scope_tags
self._scope_depth: int = 0
- # One boundary per top-level scope element, so a swarm of tiny cards cannot clear the gate.
+ # one boundary per top-level scope element, so a swarm of tiny cards cannot clear the gate
self.scope_segments: list[str] = []
self._scope_seg_start: int | None = None
- # Open non-void tags plus hidden-start indices, so an omitted </p>/</li> cannot leave the
- # renderer stuck hidden.
+ # open non-void tags plus hidden-start indices, so an omitted </p>/</li> cannot leave the renderer stuck hidden
self._open_tags: list[str] = []
# Open tags that can be closed implicitly; zero lets _close_implicit skip the scan.
self._closable_open: int = 0
@@ -317,19 +314,19 @@ class _MarkdownRenderer(HTMLParser):
self._dropped_chars: int = 0
self._seg_dropped_start: int = 0
self.scope_dropped: list[int] = []
- # Heading text per segment: role="heading", hgroup and linked h1 render as prose, so ATX
- # reparsing alone cannot keep them out of the gate.
+ # role="heading", hgroup and linked h1 render as prose
+ # Heading text per segment: role="heading", hgroup and linked h1 render as prose, so ATX reparsing alone cannot
+ # keep them out of the gate.
self._seg_heading_texts: list[str] = []
self.scope_heading_prose: list[int] = []
# Open-tag indices of headings, unwound with _hidden_marks.
self._heading_marks: list[int] = []
- # Link state
self._link_href: str | None = None
self._link_text_parts: list[str] = []
self._in_link: bool = False
self._link_seq: int = 0
- # A link wrapping a heading emits after the mark pops, so the tee is told to treat it so.
+ # a link wrapping a heading emits after the mark pops, so the tee is told to treat it so
self._link_had_heading: bool = False
self._link_heading_parts: list[str] = []
self._emit_as_heading: bool = False
@@ -337,11 +334,9 @@ class _MarkdownRenderer(HTMLParser):
# Credited only at </a>: an <a> left open adopts body prose, which is not furniture.
self._link_header_chars: int = 0
- # List state
self._list_stack: list[str] = [] # "ul" or "ol"
self._ol_counter: list[int] = []
- # Table state
self._in_table: bool = False
self._current_row: list[str] = []
self._cell_parts: list[str] = []
@@ -351,7 +346,6 @@ class _MarkdownRenderer(HTMLParser):
self._row_has_th: bool = False
self._is_first_row: bool = False
- # Pre/code state
self._in_pre: bool = False
self._pre_parts: list[str] = []
# Depth, not a flag: nested <code> opens two spans and each </code> owes a backtick.
@@ -366,8 +360,7 @@ class _MarkdownRenderer(HTMLParser):
Such a buffer emits into the frame when it closes; an enclosing one
(already open at ``<header>``) must not capture it."""
- # Only the buffer _emit would pick matters, in its order; OR-ing them calls an enclosing
- # one nested.
+ # Only the buffer _emit would pick matters, in its order; OR-ing them calls an enclosing one nested.
if self._in_link:
return self._link_seq != frame.outer_link_seq
if self._in_cell:
@@ -378,19 +371,19 @@ class _MarkdownRenderer(HTMLParser):
def _emit(self, text: str) -> None:
frame = self._header_stack[-1] if self._header_stack else None
- # Tee wherever the text routes, so a heading in a nested buffer is captured. A link opened
- # inside the frame delivers twice, so tee only the formatted form; an enclosing link, once.
+ # Tee wherever the text routes, so a heading in a nested buffer is captured. A link opened inside the frame
+ # delivers twice, so tee only the formatted form; an enclosing link, once.
in_nested_link = (
self._in_link and self._link_seq != frame.outer_link_seq if frame else False
)
- # Replays never tee: the text was teed on the way in. _finish_link re-arms the tee itself.
+ # replays never tee: the text was teed on the way in, and _finish_link re-arms the tee itself
as_heading = (
self._heading_marks and not in_nested_link and not self._replaying
) or self._emit_as_heading
if frame is not None and as_heading:
frame.heading_parts.append(text)
frame.heading_chars += len(text.strip())
- # For the eligibility gate, frame or not. Link text waits for _finish_link to count once.
+ # for the eligibility gate, frame or not; link text waits for _finish_link to count once
if not self._replaying and (
(self._heading_marks and not self._in_link) or self._emit_as_heading
):
@@ -447,7 +440,6 @@ class _MarkdownRenderer(HTMLParser):
def _prefix_blockquote(self, content: str) -> str:
"""Prefix every line of *content* with ``> ``."""
- # Strip trailing whitespace, then collapse blank lines.
content = re.sub(r"[ \t]+$", "", content, flags = re.MULTILINE)
content = re.sub(r"\n{3,}", "\n\n", content).strip()
if not content:
@@ -492,11 +484,10 @@ class _MarkdownRenderer(HTMLParser):
self._in_link = False
self._link_text_parts = []
self._link_heading_parts = []
- # An anchor wrapping a heading AND other content tees the title alone, else the nav rides.
+ # an anchor wrapping a heading AND other content tees the title alone, else the nav rides
partial = bool(heading_text) and heading_text != text
self._emit_as_heading = self._link_had_heading and not partial
self._link_had_heading = False
- # Recovery paths reach here without </a>, so drop the uncredited tally.
self._link_header_chars = 0
if href and text:
self._emit(f"[{text}]({href})")
@@ -510,10 +501,10 @@ class _MarkdownRenderer(HTMLParser):
# Preserved by hand, so tell the gate too or a title-only card reads as body prose.
self._seg_heading_texts.append(heading_text)
+ # ------------------------------------------------------------------ Tag handlers
+ # ------------------------------------------------------------------ Structural bookkeeping shared by every start
+ # tag (skip/hidden/scope).
# ------------------------------------------------------------------
- # Tag handlers
- # ------------------------------------------------------------------
- # Structural bookkeeping shared by every start tag (skip/hidden/scope).
def _truncate_open_tags(self, index: int) -> None:
"""Drop the open-tag stack above *index*, keeping the closable count."""
for name in self._open_tags[index:]:
@@ -580,14 +571,14 @@ class _MarkdownRenderer(HTMLParser):
Their content is the header's, so it has to land in the frame before the
strip is judged; otherwise it is emitted afterwards and escapes."""
if self._in_link and self._link_seq != frame.outer_link_seq:
- # The header boundary proves this anchor did not adopt the body: its text is furniture.
+ # the header boundary proves this anchor did not adopt the body: its text is furniture
frame.link_chars += self._link_header_chars
self._finish_link()
- # Code opened OUTSIDE the header is the page's; closing it here leaves </code> unpaired.
+ # code opened OUTSIDE the header is the page's; closing it here leaves </code> unpaired
while self._inline_code_depth > frame.outer_in_code:
self._inline_code_depth -= 1
self._emit("`")
- # Before the cell: _finish_row emits, and an open <pre> would swallow the row as CODE| |.
+ # before the cell: _finish_row emits, and an open <pre> would swallow the row as CODE| |
if self._in_pre and not frame.outer_in_pre:
self._drain_pre()
if self._in_cell and self._cell_seq != frame.outer_cell_seq:
@@ -655,10 +646,8 @@ class _MarkdownRenderer(HTMLParser):
)
)
elif _is_hidden_element(attr_dict):
- # Void elements never join the stack, so suppress a hidden one inline.
return False
if self._scope_tags is not None and tag in self._scope_tags:
- # A scope element inside a header would strand its output in the buffer.
self._flush_header_frames()
if self._scope_depth == 0:
self._scope_seg_start = len(self._out)
@@ -680,12 +669,11 @@ class _MarkdownRenderer(HTMLParser):
suppressed = bool(self._hidden_marks) or (
self._scope_tags is not None and self._scope_depth == 0
)
- # An element that IS the heading emits when its buffer closes, after the mark pops; flush
- # it here while the tee still recognises it.
+ # An element that IS the heading emits when its buffer closes, after the mark pops; flush it here while the tee
+ # still recognises it.
if self._in_link and tag == "a" and self._heading_marks:
self._finish_link()
if tag not in _VOID_TAGS:
- # Pop to the innermost matching open tag (recovers omitted closes).
for i in range(len(self._open_tags) - 1, -1, -1):
if self._open_tags[i] == tag:
self._truncate_open_tags(i)
@@ -708,14 +696,12 @@ class _MarkdownRenderer(HTMLParser):
tag = tag.lower()
if self._skip_depth:
- # Inside a skipped subtree: only track nested skip depth.
if tag in _SKIP_TAGS:
self._skip_depth += 1
return
- # Recover optional end tags before the skip decision: a skipped
- # <nav>/<footer> still implicitly closes an open <p>, releasing its
- # hidden mark so following siblings render.
+ # Recover optional end tags before the skip decision: a skipped <nav>/<footer> still implicitly closes an open
+ # <p>, releasing its hidden mark so following siblings render.
self._close_implicit(tag)
if tag in _SKIP_TAGS:
@@ -794,12 +780,11 @@ class _MarkdownRenderer(HTMLParser):
self._emit("\n\n")
elif tag == "tr":
- # Flush open cell/row from a prior row that omitted </td>/</tr>.
self._finish_cell()
self._finish_row()
elif tag in ("th", "td"):
- self._finish_cell() # handles omitted </td>/</th>
+ self._finish_cell()
self._cell_parts = []
self._in_cell = True
self._cell_seq += 1
@@ -807,7 +792,6 @@ class _MarkdownRenderer(HTMLParser):
self._row_has_th = True
elif tag == "img":
- # Skip images: keeps text readable, avoids data-URI amplification.
return
def handle_endtag(self, tag: str) -> None:
@@ -855,7 +839,7 @@ class _MarkdownRenderer(HTMLParser):
elif tag == "pre" and self._in_pre:
self._drain_pre()
- # Already closed means a frame recovered it; a second backtick codes the rest of the page.
+ # already closed means a frame recovered it; a second backtick codes the rest of the page
elif tag == "code" and not self._in_pre and self._inline_code_depth:
self._inline_code_depth -= 1
self._emit("`")
@@ -868,14 +852,11 @@ class _MarkdownRenderer(HTMLParser):
self._finish_row()
elif tag == "table":
- # Flush remaining row (handles omitted </tr>).
self._finish_cell()
self._finish_row()
self._in_table = False
self._emit("\n")
- # ------------------------------------------------------------------
- # Text / entity handlers
# ------------------------------------------------------------------
def _text_suppressed(self) -> bool:
if self._skip_depth or self._hidden_marks:
@@ -896,8 +877,8 @@ class _MarkdownRenderer(HTMLParser):
return
# Collapse all whitespace (including newlines) per HTML rules.
text = re.sub(r"\s+", " ", data)
- # Sized after collapsing, as the reader sees it: raw spaces in a link cleared the floor at
- # ~100% density and dropped the byline.
+ # Sized after collapsing, as the reader sees it: raw spaces in a link cleared the floor at ~100% density and
+ # dropped the byline.
self._count_header_text(text)
# Suppress whitespace-only nodes between table elements (source indentation).
if self._in_table and not self._in_cell and not text.strip():
@@ -921,11 +902,10 @@ class _MarkdownRenderer(HTMLParser):
# Flush pending buffers (handles truncated HTML from capped fetches)
def flush_pending(self) -> None:
"""Flush open side-buffers into ``_out`` after close(), recovering truncated HTML."""
- # Headers first: a frame finalizes its inner buffers, then emits into the enclosing link or
- # cell, which must still be open here; it is finalized below.
+ # Headers first: a frame finalizes its inner buffers, then emits into the enclosing link or cell, which must
+ # still be open here; it is finalized below.
self._flush_header_frames()
- # Flush innermost buffers first so their content propagates outward.
if self._in_link:
self._finish_link()
@@ -939,7 +919,6 @@ class _MarkdownRenderer(HTMLParser):
if self._in_pre:
self._drain_pre()
- # Flatten any open blockquote buffers (innermost first).
while self._bq_stack:
content = "".join(self._bq_stack.pop())
prefixed = self._prefix_blockquote(content)
@@ -950,9 +929,9 @@ class _MarkdownRenderer(HTMLParser):
else:
self._out.append("\n\n" + prefixed + "\n\n")
- # A scope left open by truncated HTML never reached _exit_tag, so its output
- # never joined scope_segments and would score 0. Flush the still-open segment
- # here (after the side-buffers) so a truncated main-content page is scored.
+ # A scope left open by truncated HTML never reached _exit_tag, so its output never joined scope_segments and
+ # would score 0. Flush the still-open segment here (after the side-buffers) so a truncated main-content page is
+ # scored.
if self._scope_seg_start is not None:
self.scope_segments.append("".join(self._out[self._scope_seg_start :]))
self.scope_dropped.append(self._dropped_chars - self._seg_dropped_start)
@@ -961,7 +940,6 @@ class _MarkdownRenderer(HTMLParser):
self._scope_depth = 0
-# Post-processing
def _cleanup(text: str) -> str:
"""Normalize whitespace and blank lines, preserving fenced code blocks verbatim."""
lines = text.split("\n")
@@ -994,9 +972,9 @@ def _cleanup(text: str) -> str:
return "\n".join(out).strip()
-# Known boilerplate fragments stripped from main-content conversions, matched
-# only against short lines. Sources: GitHub page furniture / client-side error
-# placeholders, skip-links, cookie banners.
+# known boilerplate stripped from main-content conversions, matched only against short lines
+# Known boilerplate fragments stripped from main-content conversions, matched only against short lines. Sources: GitHub
+# page furniture / client-side error placeholders, skip-links, cookie banners.
_BOILERPLATE_FRAGMENTS = (
"skip to content",
"skip to main content",
@@ -1015,8 +993,8 @@ _BOILERPLATE_FRAGMENTS = (
"accept all cookies",
"manage cookie preferences",
)
-# Only shorter lines are eligible for boilerplate dropping; real content
-# sentences quoting a fragment run longer.
+# only shorter lines are eligible: real content sentences quoting a fragment run longer
+# Only shorter lines are eligible for boilerplate dropping; real content sentences quoting a fragment run longer.
_BOILERPLATE_MAX_LINE_CHARS = 300
# Normalized furniture phrases for whole-segment matching. See _line_is_boilerplate.
@@ -1166,7 +1144,6 @@ def _visible_len(line: str) -> int:
depth += (char == "(") - (char == ")")
j += 1
if depth:
- # Never balances, so it is not a link; the bytes show as text, so count them.
total += 1
i += 1
continue
@@ -1178,12 +1155,11 @@ def _visible_len(line: str) -> int:
return total
-# A scoped conversion below this size is judged not to be the page's main
-# content (e.g. an empty <article> stub) and the next candidate is tried.
+# A scoped conversion below this size is judged not to be the page's main content (e.g. an empty <article> stub) and the
+# next candidate is tried.
_MIN_MAIN_CONTENT_CHARS = 200
-# Public API
def html_to_markdown(source_html: str, *, main_content: bool = False) -> str:
"""Convert HTML to Markdown (headings, links, emphasis, lists, tables, blockquotes, code, entities).
diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.py
index 28a1cfb1b03..08b3f2ceed1 100644
--- a/studio/backend/core/inference/_vulkan_probe.py
+++ b/studio/backend/core/inference/_vulkan_probe.py
@@ -51,14 +51,14 @@ def _igpu_flags_and_names(base, lib, count: int) -> tuple[list[bool], list[str]]
return flags, names
dev_count = base.ggml_backend_reg_dev_count(reg)
except Exception:
- # Best-effort: any failure degrades to "discrete"/"unnamed" so the
- # memory readings still get through instead of crashing the probe.
+ # Best-effort: any failure degrades to "discrete"/"unnamed" so the memory readings still get through instead of
+ # crashing the probe.
return flags, names
- # Bound outside the type-detection try above: a ggml-base without the
- # description symbol (older/custom build) must degrade to unnamed devices,
- # not abort before the iGPU flags are read (which would count an iGPU's
- # shared RAM as VRAM).
+ # bound outside the try above: a ggml-base without the description symbol must degrade to unnamed
+ # Bound outside the type-detection try above: a ggml-base without the description symbol (older/custom build) must
+ # degrade to unnamed devices, not abort before the iGPU flags are read (which would count an iGPU's shared RAM as
+ # VRAM).
name_functions = []
for symbol in ("ggml_backend_dev_description", "ggml_backend_dev_name"):
try:
@@ -98,17 +98,15 @@ def main() -> int:
return 0
bindir = sys.argv[1]
- # Device names can be non-ASCII (localized drivers); the platform-default
- # stdout encoding (e.g. cp1252) would raise on them and lose the whole
- # inventory. The reader decodes UTF-8 with the same error mode.
+ # Device names can be non-ASCII (localized drivers); the platform-default stdout encoding (e.g. cp1252) would raise
+ # on them and lose the whole inventory. The reader decodes UTF-8 with the same error mode.
try:
sys.stdout.reconfigure(encoding = "utf-8", errors = "replace")
except Exception:
pass
- # Hold add_dll_directory's handle for the rest of main() (the documented
- # idiom) so bindir stays on the search path while the sibling ggml DLLs
- # resolve below.
+ # Hold add_dll_directory's handle for the rest of main() (the documented idiom) so bindir stays on the search path
+ # while the sibling ggml DLLs resolve below.
_dll_dir = None
if sys.platform == "win32":
base_name, vk_name = "ggml-base.dll", "ggml-vulkan.dll"
@@ -129,8 +127,8 @@ def main() -> int:
return os.path.join(directory, entry)
return None
- # RTLD_GLOBAL exposes ggml-base's symbols to ggml-vulkan on POSIX. getattr
- # falls back to 0 where the flag doesn't exist (Windows CDLL ignores mode).
+ # RTLD_GLOBAL exposes ggml-base's symbols to ggml-vulkan on POSIX. getattr falls back to 0 where the flag doesn't
+ # exist (Windows CDLL ignores mode).
_rtld_global = getattr(ctypes, "RTLD_GLOBAL", 0)
base_path = _find_lib(bindir, base_name)
vk_path = _find_lib(bindir, vk_name)
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py
index 0a9627fb972..7cc04b4e6c6 100644
--- a/studio/backend/core/inference/anthropic_compat.py
+++ b/studio/backend/core/inference/anthropic_compat.py
@@ -19,9 +19,8 @@ def openai_finish_to_anthropic_stop(finish_reason, had_tool_calls = False) -> st
'length' -> 'max_tokens' (truncation wins even mid tool call, so a cut-off
tool call isn't mislabeled tool_use); tool_calls / had_tool_calls -> 'tool_use';
'stop_sequence' -> 'stop_sequence'; 'stop'/None/unknown -> 'end_turn'."""
- # Truncation takes precedence: a tool call cut off at max_tokens has possibly
- # incomplete arguments, so report max_tokens rather than telling the client to
- # run the tool.
+ # Truncation takes precedence: a tool call cut off at max_tokens has possibly incomplete arguments, so report
+ # max_tokens rather than telling the client to run the tool.
if finish_reason == "length":
return "max_tokens"
if finish_reason == "tool_calls" or had_tool_calls:
@@ -88,7 +87,6 @@ def anthropic_messages_to_openai(
"""
result: list[dict] = []
- # System prompt
if system:
if isinstance(system, str):
result.append({"role": "system", "content": system})
@@ -111,8 +109,9 @@ def anthropic_messages_to_openai(
continue
if role == "assistant":
- # Assistant content: text + tool_use (no images in Anthropic's model),
- # plus replayed thinking when preservation is requested.
+ # text + tool_use (no images in Anthropic's model), plus replayed thinking when preservation is requested
+ # Assistant content: text + tool_use (no images in Anthropic's model), plus replayed thinking when
+ # preservation is requested.
text_parts: list[str] = []
tool_calls: list[dict] = []
thinking_parts: list[str] = []
@@ -457,50 +456,44 @@ class AnthropicStreamEmitter:
parse_think: bool = True,
think_provenance: Optional[dict] = None,
) -> None:
- # Off when the route knows reasoning markup cannot be genuine (thinking
- # disabled or a non-reasoning model): literal <think> in prose then
- # streams as ordinary text instead of being consumed as a trace.
+ # Off when the route knows reasoning markup cannot be genuine (thinking disabled or a non-reasoning model):
+ # literal <think> in prose then streams as ordinary text instead of being consumed as a trace.
self._parse_think = parse_think
self.block_index: int = 0
self._block_index_used: bool = False
self._text_block_open: bool = False
self._thinking_block_open: bool = False
self._open_tool_call_id: Optional[str] = None
- # The mapped Anthropic ``toolu_*`` id published in content_block_start,
- # reused for the paired tool_result so consumers can correlate them.
+ # The mapped Anthropic ``toolu_*`` id published in content_block_start, reused for the paired tool_result so
+ # consumers can correlate them.
self._open_tool_use_id: Optional[str] = None
self._open_tool_args_sent: bool = False
self._prev_text: str = ""
- # <think> routing: the generator folds reasoning_content into the
- # cumulative text as <think>...</think> markup (the UI chat parses it),
- # but Anthropic clients expect typed thinking blocks. Split the markup
- # back out: text inside the tags streams as thinking_delta in a
- # "thinking" content block, everything else as ordinary text. _tag_buf
- # holds back a trailing partial tag until the next delta decides it.
+ # the generator folds reasoning_content into the cumulative text as <think>...</think> markup
+ # <think> routing: the generator folds reasoning_content into the cumulative text as <think>...</think> markup
+ # (the UI chat parses it), but Anthropic clients expect typed thinking blocks. Split the markup back out: text
+ # inside the tags streams as thinking_delta in a "thinking" content block, everything else as ordinary text.
+ # _tag_buf holds back a trailing partial tag until the next delta decides it.
self._route_mode: str = "text"
self._tag_buf: str = ""
- # Leading whitespace of a thinking span, held until real reasoning
- # arrives so a whitespace-only trace never opens a block. See
- # _emit_thinking_delta.
+ # Leading whitespace of a thinking span, held until real reasoning arrives so a whitespace-only trace never
+ # opens a block. See _emit_thinking_delta.
self._thinking_ws_hold: str = ""
- # Genuine reasoning only ever arrives as a single LEADING <think>
- # block per synthesis turn (the generator folds reasoning_content in
- # as a prefix). Once that block closed, or once real answer text has
- # streamed, any later <think> is the model quoting the tag and must
- # stay literal.
+ # Genuine reasoning only ever arrives as a single LEADING <think> block per synthesis turn (the generator folds
+ # reasoning_content in as a prefix). Once that block closed, or once real answer text has streamed, any later
+ # <think> is the model quoting the tag and must stay literal.
self._think_consumed: bool = False
self._turn_has_text: bool = False
- # Live provenance from the generator: "wrapped" counts the leading
- # <think> tags IT opened from reasoning_content. When provided, a
- # leading tag is only parsed as reasoning if a generator wrap is
- # available -- a model answering with literal <think> markup (and no
- # genuine trace) keeps it as text. None falls back to the leading-tag
- # heuristic (test doubles / callers without provenance).
+ # "wrapped" counts the leading <think> tags the generator opened from reasoning_content
+ # Live provenance from the generator: "wrapped" counts the leading <think> tags IT opened from
+ # reasoning_content. When provided, a leading tag is only parsed as reasoning if a generator wrap is available
+ # -- a model answering with literal <think> markup (and no genuine trace) keeps it as text. None falls back to
+ # the leading-tag heuristic (test doubles / callers without provenance).
self._think_provenance = think_provenance
self._wraps_consumed: int = 0
- # Active wrap entry ({"len": N} from the generator) while a provenance
- # -backed thinking block streams: the block spans exactly N reasoning
- # chars, so a literal "</think>" INSIDE the trace never ends it early.
+ # the block spans exactly the wrap's N reasoning chars
+ # Active wrap entry ({"len": N} from the generator) while a provenance -backed thinking block streams: the block
+ # spans exactly N reasoning chars, so a literal "</think>" INSIDE the trace never ends it early.
self._active_wrap: Optional[dict] = None
self._wrap_chars: int = 0
self._close_skip: int = 0
@@ -551,7 +544,6 @@ class AnthropicStreamEmitter:
elif etype == "metadata":
self._usage = event.get("usage", {})
return []
- # status events — no Anthropic equivalent
return []
def finish(
@@ -561,7 +553,6 @@ class AnthropicStreamEmitter:
) -> list[str]:
"""Close any open block and emit message_delta + message_stop."""
events = []
- # A trailing partial tag at end-of-stream is literal output, not markup.
if self._tag_buf:
held, self._tag_buf = self._tag_buf, ""
if self._route_mode == "thinking":
@@ -631,20 +622,17 @@ class AnthropicStreamEmitter:
break
if i:
events.extend(self._emit_text_delta(data[:i]))
- # Consumed: whatever happens to the tag below, the run
- # before it has already been delivered. Re-including it in
- # the literal-text branch below sent it to the client twice.
+ # consumed: the run before the tag has already been delivered
+ # Consumed: whatever happens to the tag below, the run before it has already been delivered.
+ # Re-including it in the literal-text branch below sent it to the client twice.
data = data[i:]
i = 0
if self._turn_has_text:
- # Non-space text preceded the tag, so this is not the
- # leading reasoning block; relay the rest literally.
continue
if (
self._think_provenance is not None
and self._think_provenance.get("wrapped", 0) <= self._wraps_consumed
):
- # The generator did not wrap this tag: literal model text.
events.extend(self._emit_text_delta(data))
break
wraps = (
@@ -672,9 +660,8 @@ class AnthropicStreamEmitter:
self._active_wrap = None
continue
if self._active_wrap is not None:
- # Provenance-backed span: consume exactly the generator's
- # reasoning length, then skip its closing tag. A literal
- # "</think>" inside the trace stays part of the thinking.
+ # Provenance-backed span: consume exactly the generator's reasoning length, then skip its closing
+ # tag. A literal "</think>" inside the trace stays part of the thinking.
remaining = int(self._active_wrap.get("len", 0)) - self._wrap_chars
if remaining > 0:
take = min(remaining, len(data))
@@ -722,13 +709,12 @@ class AnthropicStreamEmitter:
def _emit_thinking_delta(self, text: str) -> list[str]:
if not self._thinking_block_open:
- # A trace that is only whitespace is not a thought: Qwen3-style
- # templates render "<think>\n\n</think>" on every reply when thinking
- # is off, and llama-server parses that into reasoning_content, so an
- # empty thinking block would be attached to ordinary answers. The
- # non-streaming reducer already drops those, so hold the leading
- # whitespace run and only open the block once real reasoning arrives;
- # the held run is then emitted with it so the trace stays verbatim.
+ # a whitespace-only trace is not a thought
+ # A trace that is only whitespace is not a thought: Qwen3-style templates render "<think>\n\n</think>" on
+ # every reply when thinking is off, and llama-server parses that into reasoning_content, so an empty
+ # thinking block would be attached to ordinary answers. The non-streaming reducer already drops those, so
+ # hold the leading whitespace run and only open the block once real reasoning arrives; the held run is then
+ # emitted with it so the trace stays verbatim.
held = self._thinking_ws_hold + text
if not held.strip():
self._thinking_ws_hold = held
@@ -759,7 +745,6 @@ class AnthropicStreamEmitter:
return self._tool_arguments_delta(args)
events = []
- # A held partial tag is literal output once a tool call interrupts it.
if self._tag_buf:
held, self._tag_buf = self._tag_buf, ""
if self._route_mode == "thinking":
@@ -775,7 +760,6 @@ class AnthropicStreamEmitter:
self._open_tool_use_id = None
self._open_tool_args_sent = False
- # Open a tool_use block.
self._alloc_block_index()
self._open_tool_call_id = tool_call_id
self._open_tool_use_id = anthropic_tool_use_id(tool_call_id)
@@ -820,16 +804,14 @@ class AnthropicStreamEmitter:
def _handle_tool_end(self, event: dict) -> list[str]:
events = []
- # Close the tool_use block.
if self._open_tool_call_id is not None or self._text_block_open:
events.append(self._close_block())
- # Reuse the id published in content_block_start; fall back to mapping
- # the raw id only if no tool_start preceded this end.
+ # Reuse the id published in content_block_start; fall back to mapping the raw id only if no tool_start preceded
+ # this end.
tool_use_id = self._open_tool_use_id or anthropic_tool_use_id(event.get("tool_call_id", ""))
self._open_tool_call_id = None
self._open_tool_use_id = None
self._open_tool_args_sent = False
- # Emit custom tool_result event (non-standard, ignored by SDKs)
events.append(
build_anthropic_sse_event(
"tool_result",
@@ -840,9 +822,9 @@ class AnthropicStreamEmitter:
},
)
)
- # Reset text tracking for the next synthesis turn; the next content
- # delta opens a fresh text (or thinking) block lazily, and the new
- # turn may legitimately open with its own leading <think> block.
+ # the next content delta opens a fresh block lazily
+ # Reset text tracking for the next synthesis turn; the next content delta opens a fresh text (or thinking) block
+ # lazily, and the new turn may legitimately open with its own leading <think> block.
self._prev_text = ""
self._tag_buf = ""
self._thinking_ws_hold = ""
@@ -882,9 +864,7 @@ class AnthropicStreamEmitter:
{
"type": "content_block_start",
"index": self.block_index,
- # signature is part of the Anthropic thinking-block shape;
- # strict stream decoders reject the block without it. Local
- # models have no signing key, so it stays empty.
+ # strict stream decoders reject a thinking block without a signature
"content_block": {"type": "thinking", "thinking": "", "signature": ""},
},
)
@@ -912,14 +892,13 @@ class AnthropicPassthroughEmitter:
"""
def __init__(self, reasoning_as_thinking: bool = True) -> None:
- # When thinking is effectively off, llama-server's format parser can
- # still shunt a literal <think> example the model was asked to produce
- # into reasoning_content; reconstruct it as visible text instead of a
- # typed thinking block.
+ # When thinking is effectively off, llama-server's format parser can still shunt a literal <think> example the
+ # model was asked to produce into reasoning_content; reconstruct it as visible text instead of a typed thinking
+ # block.
self._reasoning_as_thinking = reasoning_as_thinking
self._reasoning_text_open = False
self.block_index: int = -1
- self._current_block_type: Optional[str] = None # "text" | "tool_use" | None
+ self._current_block_type: Optional[str] = None
self._tool_call_states: dict = {} # delta index -> {block_index, id, name}
self._usage: dict = {}
self._stop_reason: str = "end_turn"
@@ -995,11 +974,12 @@ class AnthropicPassthroughEmitter:
delta = choice.get("delta") or {}
finish_reason = choice.get("finish_reason")
+ # llama-server splits <think> into reasoning_content whenever it can parse the model's reasoning format (it does
+ # so for tool-calling turns, i.e.
+ # ── Reasoning ── llama-server splits <think> into reasoning_content whenever it can parse the model's reasoning
+ # format (it does so for tool-calling turns, which is every Claude Code turn). Reading only `content` drops the
+ # entire thinking trace, so the model appears not to think at all.
# ── Reasoning ──
- # llama-server splits <think> into reasoning_content whenever it can parse
- # the model's reasoning format (it does so for tool-calling turns, which is
- # every Claude Code turn). Reading only `content` drops the entire thinking
- # trace, so the model appears not to think at all.
reasoning = delta.get("reasoning_content")
if reasoning:
if not self._reasoning_as_thinking:
@@ -1021,19 +1001,21 @@ class AnthropicPassthroughEmitter:
},
)
)
- # Reconstructed literal block ends where the answer resumes -- checked
- # unconditionally (not elif): one chunk can carry the final reasoning
- # fragment AND same-chunk content/tool output, and the closing tag must
- # land between them.
+ # checked unconditionally, not elif: one chunk can carry the final reasoning fragment AND same-chunk content
+ # Reconstructed literal block ends where the answer resumes -- checked unconditionally (not elif): one chunk can
+ # carry the final reasoning fragment AND same-chunk content/tool output, and the closing tag must land between
+ # them.
if self._reasoning_text_open and (
delta.get("content") or delta.get("tool_calls") or finish_reason
):
self._reasoning_text_open = False
events.extend(self._emit_text_delta("</think>"))
+ # grammar mode worked: flush anything the healer held (it preceded the call in the model's output) and relay
+ # verbatim from here
+ # ── Structured tool calls take precedence over healing ── Grammar mode worked: flush anything the healer held
+ # (it preceded the call in the model's output) and relay verbatim from here on.
# ── Structured tool calls take precedence over healing ──
- # Grammar mode worked: flush anything the healer held (it preceded the
- # call in the model's output) and relay verbatim from here on.
if delta.get("tool_calls") and self._healer is not None and not self._healer.dormant:
for kind, value in self._healer.structured_tool_call_seen():
if kind == "text" and value:
@@ -1042,8 +1024,8 @@ class AnthropicPassthroughEmitter:
# ── Text content ──
content = delta.get("content")
if content and self._healer is not None and not self._healer.dormant:
- # Route text through the healer: held/promoted portions become
- # synthetic tool_use blocks, the rest streams as text unchanged.
+ # Route text through the healer: held/promoted portions become synthetic tool_use blocks, the rest streams
+ # as text unchanged.
for kind, value in self._healer.feed(content):
if kind == "text":
events.extend(self._emit_text_delta(value))
@@ -1062,13 +1044,11 @@ class AnthropicPassthroughEmitter:
and tc_idx not in self._tool_call_states
and (self._healed_call_count + len(self._tool_call_states)) >= 1
):
- # disable_parallel_tool_use: a healed call already consumed the
- # single allowed slot. The caller's chunk-level cap only sees
- # native indexes, so drop this native call (and its later
- # argument deltas, which never allocate a state either).
+ # disable_parallel_tool_use: a healed call already consumed the single allowed slot. The caller's
+ # chunk-level cap only sees native indexes, so drop this native call (and its later argument deltas,
+ # which never allocate a state either).
continue
if tc_idx not in self._tool_call_states:
- # New tool call — close prior block, open tool_use block
if self._current_block_type is not None:
events.append(self._close_current_block())
tc_id = anthropic_tool_use_id(tc.get("id", ""))
@@ -1131,8 +1111,7 @@ class AnthropicPassthroughEmitter:
elif kind == "tool_call":
events.extend(self._emit_healed_tool_use(value))
if self._healed_tool_use and self._stop_reason != "max_tokens":
- # A promoted call must stop for tool use; a truncation still wins
- # (its arguments may be incomplete).
+ # A promoted call must stop for tool use; a truncation still wins (its arguments may be incomplete).
self._stop_reason = "tool_use"
if self._current_block_type is not None:
events.append(self._close_current_block())
@@ -1176,14 +1155,12 @@ class AnthropicPassthroughEmitter:
return events
def _emit_healed_tool_use(self, call: dict) -> list[str]:
- # A healed call arrives complete, so its tool_use block opens, carries
- # one input_json_delta, and closes immediately; an open text block is
- # closed first (only the safe prefix ever streamed into it).
+ # A healed call arrives complete, so its tool_use block opens, carries one input_json_delta, and closes
+ # immediately; an open text block is closed first (only the safe prefix ever streamed into it).
if (
self._heal_disable_parallel
and (self._healed_call_count + len(self._tool_call_states)) >= 1
):
- # Healed and native calls share the single allowed slot.
return []
events: list[str] = []
if self._current_block_type is not None:
@@ -1250,7 +1227,6 @@ class AnthropicPassthroughEmitter:
{
"type": "content_block_start",
"index": self.block_index,
- # Empty signature keeps strict Anthropic decoders happy.
"content_block": {"type": "thinking", "thinking": "", "signature": ""},
},
)
diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py
index 9969a9b4d05..d91aeefc435 100644
--- a/studio/backend/core/inference/api_monitor.py
+++ b/studio/backend/core/inference/api_monitor.py
@@ -61,7 +61,6 @@ def _token_count_or_none(value: Any) -> Optional[int]:
def _finite_float_or_none(value: Any) -> Optional[float]:
- # float() on a huge upstream int raises OverflowError, not ValueError/TypeError.
try:
number = float(value)
except (TypeError, ValueError, OverflowError):
@@ -189,7 +188,7 @@ class ApiMonitorEntry:
total_tokens: Optional[int] = None
total_tokens_authoritative: bool = False
error: Optional[str] = None
- # "request" (HTTP call) or "lifecycle" (model load/unload: event/reason, not a prompt; shared).
+ # "request" (HTTP call) or "lifecycle" (model load/unload: event/reason, not a prompt; shared)
kind: str = "request"
event: Optional[str] = None
reason: Optional[str] = None
@@ -198,9 +197,9 @@ class ApiMonitorEntry:
progress: Optional[float] = None
# Stamped on the first reply text; snapshot() prefers it over engine timings.
first_token_monotonic: Optional[float] = None
- # The same instant, but only for output the model decoded. A tool card is client
- # output that TTFT should count and the token-rate clock must not: the tool run
- # between it and the first token is not decoding.
+ # a tool card is client output that TTFT should count and the token-rate clock must not
+ # The same instant, but only for output the model decoded. A tool card is client output that TTFT should count and
+ # the token-rate clock must not: the tool run between it and the first token is not decoding.
first_decode_monotonic: Optional[float] = None
prompt_ms: Optional[float] = None
tok_per_sec: Optional[float] = None
@@ -208,8 +207,9 @@ class ApiMonitorEntry:
# timings.predicted_ms; set only from engine timings, so its presence marks a rateable row.
decode_ms: Optional[float] = None
stop_reason: Optional[str] = None
- # Every finish reason seen so far. An n > 1 stream reports each choice in its own
- # chunk, so agreement can only be judged across the whole request. Not serialized.
+ # an n > 1 stream reports each choice in its own chunk
+ # Every finish reason seen so far. An n > 1 stream reports each choice in its own chunk, so agreement can only be
+ # judged across the whole request. Not serialized.
stop_reasons_seen: set[str] = field(default_factory = set)
# request-local preview state, omitted from snapshots and cleared on terminal paths.
openai_stream_tool_calls: list[_OpenAIStreamToolCall] = field(default_factory = list)
@@ -235,8 +235,8 @@ class ApiMonitorEntry:
context_usage = min(1.0, max(0.0, self.total_tokens / self.context_length))
ttft_ms = None
if self.first_token_monotonic is not None:
- # Preferred over the prompt_ms fallback: that is prefill only, so it misses
- # the admission-queue wait before llama-server sees the request.
+ # Preferred over the prompt_ms fallback: that is prefill only, so it misses the admission-queue wait before
+ # llama-server sees the request.
ttft_ms = max(0, int((self.first_token_monotonic - self.started_monotonic) * 1000))
elif self.prompt_ms is not None:
ttft_ms = max(0, int(self.prompt_ms))
@@ -244,8 +244,9 @@ class ApiMonitorEntry:
if (
tok_per_sec is None
and self.completion_tokens
- # The clock starts at the first token, so it spans only the gaps that
- # followed it: one token has no gap to measure, hence no rate at all.
+ # the clock starts at the first token, so one token has no gap to measure and hence no rate
+ # The clock starts at the first token, so it spans only the gaps that followed it: one token has no gap to
+ # measure, hence no rate at all.
and self.completion_tokens > 1
and self.finished_monotonic is not None
and self.first_decode_monotonic is not None
@@ -258,8 +259,8 @@ class ApiMonitorEntry:
"endpoint": self.endpoint,
"method": self.method,
"model": self.model,
- # A shared row reaches subjects with nothing to do with it, and the overlay
- # auto-opens on this flag, so report it only to the attributed caller.
+ # A shared row reaches subjects with nothing to do with it, and the overlay auto-opens on this flag, so
+ # report it only to the attributed caller.
"via_api_key": self.via_api_key and attributed,
"prompt_preview": _trim(self.prompt, _PREVIEW_CHARS),
"reply_preview": _trim(self.reply, _PREVIEW_CHARS),
@@ -285,8 +286,8 @@ class ApiMonitorEntry:
"prompt_tok_per_sec": (
round(self.prompt_tok_per_sec, 2) if self.prompt_tok_per_sec is not None else None
),
- # The engine's decode span, not the streamed window: an unknowable first-chunk token
- # count and reasoning tokens both inflate a streamed rate. Absent rather than guessed.
+ # The engine's decode span, not the streamed window: an unknowable first-chunk token count and reasoning
+ # tokens both inflate a streamed rate. Absent rather than guessed.
"decode_ms": int(self.decode_ms) if self.decode_ms is not None else None,
"stop_reason": self.stop_reason,
}
@@ -342,8 +343,10 @@ class ApiMonitor:
"""Remove only the terminal callback registration owned by ``lease``."""
with self._callback_condition:
self._terminal_callback_leases.pop(lease, None)
- # A notification may already have captured this callback. Let its
- # fast enqueue finish before the owner drains/stops the writer.
+ # let a notification that already captured this callback finish its fast enqueue before the owner
+ # drains/stops the writer
+ # A notification may already have captured this callback. Let its fast enqueue finish before the owner
+ # drains/stops the writer.
while self._terminal_callbacks_inflight.get(lease, 0):
self._callback_condition.wait()
@@ -426,11 +429,10 @@ class ApiMonitor:
event = event,
reason = reason,
shared = True,
- # The overlay opens on API-key traffic only, and a refused switch never
- # reaches api_monitor.start, so this row is its whole trace: without the
- # attribution the monitor stayed shut on the failures it exists to surface.
+ # The overlay opens on API-key traffic only, and a refused switch never reaches api_monitor.start, so this
+ # row is its whole trace: without the attribution the monitor stayed shut on the failures it exists to
+ # surface.
via_api_key = via_api_key,
- # Shared rows reach every subject, so attribution needs an owner.
subject = subject,
)
with self._lock:
@@ -493,10 +495,10 @@ class ApiMonitor:
entry.first_token_monotonic = now
if entry.first_decode_monotonic is None:
entry.first_decode_monotonic = now
- # Preview is capped: once the "..." marker is present the head is
- # frozen, so skip the per-chunk re-concat (avoids O(n^2) on long
- # generations). A reply that landed exactly on the cap has no marker
- # yet, so let one more append record the truncation before freezing.
+ # once the "..." marker is present the head is frozen
+ # Preview is capped: once the "..." marker is present the head is frozen, so skip the per-chunk re-concat
+ # (avoids O(n^2) on long generations). A reply that landed exactly on the cap has no marker yet, so let one
+ # more append record the truncation before freezing.
if len(entry.reply) >= _MAX_REPLY_CHARS:
if not entry.reply.endswith("..."):
entry.reply = _trim(entry.reply + text, _MAX_REPLY_CHARS)
@@ -691,8 +693,9 @@ class ApiMonitor:
) -> None:
if not entry_id:
return
- # Coerce before locking: arbitrary payloads, and a raise here (this runs inside
- # streaming generators) would truncate the user's response.
+ # coerce before locking: a raise here (this runs inside streaming generators) would truncate the user's response
+ # Coerce before locking: arbitrary payloads, and a raise here (this runs inside streaming generators) would
+ # truncate the user's response.
tok_per_sec = _finite_float_or_none(tok_per_sec)
prompt_tok_per_sec = _finite_float_or_none(prompt_tok_per_sec)
prompt_ms = _finite_float_or_none(prompt_ms)
@@ -777,8 +780,8 @@ class ApiMonitor:
elif not entry.total_tokens_authoritative and (
prompt_tokens is not None or completion_tokens is not None
):
- # Derive only when no authoritative total has been set;
- # a later partial chunk must not clobber a provider total.
+ # Derive only when no authoritative total has been set; a later partial chunk must not clobber a
+ # provider total.
entry.total_tokens = (entry.prompt_tokens or 0) + (entry.completion_tokens or 0)
if context_length is not None:
entry.context_length = context_length
@@ -796,8 +799,7 @@ class ApiMonitor:
entry = self._find_locked(entry_id)
if entry is None:
return
- # Idempotent: second call (e.g. [DONE] after the finally block
- # already ran) must not move finished_*.
+ # Idempotent: second call (e.g. [DONE] after the finally block already ran) must not move finished_*.
if entry.finished_at is not None:
return
_append_stream_tool_previews(entry, entry.openai_stream_tool_calls)
@@ -826,7 +828,6 @@ class ApiMonitor:
entry = self._find_locked(entry_id)
if entry is None or entry.finished_at is not None:
return
- # Same lock as the check, so a finish() cannot land in between.
self._fail_locked(entry, error)
notification = self._terminal_notification_locked(entry)
self._notify_terminal(notification)
@@ -951,7 +952,6 @@ class ApiMonitor:
)
def active_count(self, *, subject: Optional[str] = None) -> int:
- # Lifecycle rows show as "running" while loading but are not in-flight API requests.
with self._lock:
return sum(
1
@@ -978,10 +978,9 @@ class ApiMonitor:
for entry in self._entries:
if entry.shared and entry.status != "running":
hidden.add(entry.id)
- # Shared rows are hidden, never dropped, even when owned: they are another
- # caller's history too. An own running row is not history either, and dropping
- # it loses the request outright: active_count falls to zero and the finish or
- # fail that follows has no entry left to land on.
+ # Shared rows are hidden, never dropped, even when owned: they are another caller's history too. An own
+ # running row is not history either, and dropping it loses the request outright: active_count falls to zero
+ # and the finish or fail that follows has no entry left to land on.
self._entries = deque(
entry
for entry in self._entries
diff --git a/studio/backend/core/inference/audio_codecs.py b/studio/backend/core/inference/audio_codecs.py
index dacba8cf262..0a4aa78d061 100644
--- a/studio/backend/core/inference/audio_codecs.py
+++ b/studio/backend/core/inference/audio_codecs.py
@@ -144,7 +144,6 @@ class AudioCodecManager:
strips EOS (128258), redistributes 7-per-frame codes into 3 SNAC layers.
Returns (wav_bytes, 24000).
"""
- # Find START_OF_SPEECH token (128257)
token_indices = (generated_ids == 128257).nonzero(as_tuple = True)
if len(token_indices[1]) > 0:
cropped = generated_ids[:, token_indices[1][-1] + 1 :]
@@ -154,10 +153,8 @@ class AudioCodecManager:
cropped = generated_ids
row = cropped[0]
- # Remove EOS tokens (128258)
row = row[row != 128258]
- # Trim to multiple of 7
row = row[: (len(row) // 7) * 7]
if len(row) == 0:
raise ValueError("No valid audio codes found after START_OF_SPEECH token")
@@ -210,8 +207,7 @@ class AudioCodecManager:
semantic_ids = torch.tensor([int(t) for t in semantic_matches]).long().unsqueeze(0)
- # Speaker encoder expects exactly 32 global tokens (token_num=32);
- # pad with zeros or truncate.
+ # Speaker encoder expects exactly 32 global tokens (token_num=32); pad with zeros or truncate.
GLOBAL_TOKEN_NUM = 32
if global_matches:
raw = [int(t) for t in global_matches]
@@ -220,7 +216,7 @@ class AudioCodecManager:
if len(raw) < GLOBAL_TOKEN_NUM:
raw = raw + [0] * (GLOBAL_TOKEN_NUM - len(raw))
raw = raw[:GLOBAL_TOKEN_NUM]
- global_ids = torch.tensor(raw).long().unsqueeze(0) # (1, 32)
+ global_ids = torch.tensor(raw).long().unsqueeze(0)
self._bicodec_tokenizer.device = device
self._bicodec_tokenizer.model.to(device)
diff --git a/studio/backend/core/inference/audio_errors.py b/studio/backend/core/inference/audio_errors.py
index 3c708677375..2d8c246b74e 100644
--- a/studio/backend/core/inference/audio_errors.py
+++ b/studio/backend/core/inference/audio_errors.py
@@ -11,8 +11,7 @@ carry no path or user input, so the route passes them straight through.
from __future__ import annotations
-# Tagged on the worker's audio_error payload so the parent recognises the case
-# without matching on prose.
+# Tagged on the worker's audio_error payload so the parent recognises the case without matching on prose.
AUDIO_UNSUPPORTED_CODE = "audio_unsupported_backend"
diff --git a/studio/backend/core/inference/audio_gallery.py b/studio/backend/core/inference/audio_gallery.py
index 77b5088445e..ea857e1ae24 100644
--- a/studio/backend/core/inference/audio_gallery.py
+++ b/studio/backend/core/inference/audio_gallery.py
@@ -60,16 +60,15 @@ def save(wav_bytes: bytes, meta: dict[str, Any]) -> dict[str, Any]:
return _record(audio_id, meta)
-# The OpenAI-compatible /v1/audio/speech route persists every call, so an automated client
-# can grow the gallery until the disk fills. Bounded here rather than at that route so the
-# UI's own runaway is covered too. Generous by default: this is a convenience gallery, and
-# the clip is returned to the caller either way.
+# /v1/audio/speech persists every call, so bound it here rather than at that route, covering the UI's runaway too
+# The OpenAI-compatible /v1/audio/speech route persists every call, so an automated client can grow the gallery until
+# the disk fills. Bounded here rather than at that route so the UI's own runaway is covered too. Generous by default:
+# this is a convenience gallery, and the clip is returned to the caller either way.
_MAX_CLIPS_ENV = "UNSLOTH_AUDIO_GALLERY_MAX_CLIPS"
_DEFAULT_MAX_CLIPS = 2000
-# A count alone does not bound the disk: 2000 clips of maximum-length speech is tens of
-# gigabytes, and the cap exists to stop /v1/audio/speech filling the disk. Whichever limit
-# binds first wins.
+# A count alone does not bound the disk: 2000 clips of maximum-length speech is tens of gigabytes, and the cap exists to
+# stop /v1/audio/speech filling the disk. Whichever limit binds first wins.
_MAX_BYTES_ENV = "UNSLOTH_AUDIO_GALLERY_MAX_BYTES"
_DEFAULT_MAX_BYTES = 5 * 1024 * 1024 * 1024
@@ -117,13 +116,16 @@ def _prune_to_cap() -> int:
directory = gallery_dir()
removed = 0
try:
- # Select AND delete under one lock, as clear() does. Choosing victims from a snapshot and
- # unlinking after it leaves a window where an archive lands and is deleted anyway.
+ # select AND delete under one lock: otherwise an archive landing in the window is deleted anyway
+ # Select AND delete under one lock, as clear() does. Choosing victims from a snapshot and unlinking after it
+ # leaves a window where an archive lands and is deleted anyway.
with gallery_flags.exclusive(directory, require_file_lock = True):
entries = _list_audio_entries()
- # Newest first, so the index where either budget runs out is the cut point. The newest
- # is always kept: dropping what the caller just generated looks like a silent failure.
+ # newest first; the newest is always kept, since dropping what the caller just generated looks like a silent
+ # failure
+ # Newest first, so the index where either budget runs out is the cut point. The newest is always kept:
+ # dropping what the caller just generated looks like a silent failure.
keep = len(entries) if cap <= 0 else min(cap, len(entries))
if byte_cap > 0:
running = 0
@@ -135,16 +137,16 @@ def _prune_to_cap() -> int:
if keep >= len(entries):
return 0
- # Re-read TRUSTED immediately before deleting: read() answers "nothing is archived"
- # for a store it cannot parse, which here would drop the clips the shelf exists to
- # keep. It also covers filesystems where the cross-process lock degrades to a no-op.
+ # re-read TRUSTED before deleting: read() answers "nothing is archived" for a store it cannot parse
+ # Re-read TRUSTED immediately before deleting: read() answers "nothing is archived" for a store it cannot
+ # parse, which here would drop the clips the shelf exists to keep. It also covers filesystems where the
+ # cross-process lock degrades to a no-op.
flags = gallery_flags.read_trusted(directory)
pruned: list[str] = []
for record, _cursor in entries[keep:]:
audio_id = record["id"]
if gallery_flags.is_archived(flags, audio_id):
continue
- # Not delete(): it takes the flag lock on a second descriptor and would block here.
path = audio_path(audio_id)
if path is None or _read_meta(_sidecar_path(audio_id)) is None:
continue
@@ -197,7 +199,6 @@ def audio_path(audio_id: str) -> Optional[Path]:
if not _ID_RE.match(audio_id):
return None
path = gallery_dir() / f"{audio_id}.wav"
- # defence in depth: confirm the resolved path is still inside the gallery
try:
path.resolve().relative_to(gallery_dir().resolve())
except ValueError:
@@ -209,6 +210,7 @@ def _sidecar_path(audio_id: str) -> Path:
return gallery_dir() / f"{audio_id}.json"
+# key-presence ownership: a hand-dropped wav with a partial sidecar is neither counted as ours nor destroyed
# key-presence ownership test: a hand-dropped wav with a partial sidecar is never counted as ours nor destroyed
_REQUIRED_META = (
"prompt",
@@ -379,7 +381,6 @@ def clear(include_archived: bool = False) -> int:
continue
if not include_archived and gallery_flags.is_archived(flags, path.stem):
continue
- # wav first; if it cannot be unlinked, leave the sidecar so the clip stays listable
try:
path.unlink()
except OSError:
diff --git a/studio/backend/core/inference/chat_eos.py b/studio/backend/core/inference/chat_eos.py
index 2a5d0db2286..93aec80fcda 100644
--- a/studio/backend/core/inference/chat_eos.py
+++ b/studio/backend/core/inference/chat_eos.py
@@ -19,16 +19,16 @@ from typing import Optional
# Canonical assistant-turn-end markers per chat family.
_CHAT_TURN_END_TOKENS = (
- "<|im_end|>", # ChatML: Qwen, Yi
- "<|eot_id|>", # Llama 3.x
- "<|eom_id|>", # Llama 3.x tool turns
- "<end_of_turn>", # Gemma
- "<turn|>", # Gemma-4
- "<|end|>", # Phi
- "<|end_of_turn|>", # OpenChat / Starling (barred, distinct from Gemma's)
+ "<|im_end|>",
+ "<|eot_id|>",
+ "<|eom_id|>",
+ "<end_of_turn>",
+ "<turn|>",
+ "<|end|>",
+ "<|end_of_turn|>", # OpenChat, distinct from Gemma's
)
-# harmony/gpt-oss uses <|end|> as a channel delimiter, not the turn end, and has
-# its own streamer, so its eos is left untouched.
+# harmony/gpt-oss uses <|end|> as a channel delimiter, not the turn end, and has its own streamer, so its eos is left
+# untouched.
_HARMONY_MARKERS = ("<|channel|>", "<|constrain|>")
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.py
index 0820086bdb9..ce9da8a6b71 100644
--- a/studio/backend/core/inference/chat_generation_runs.py
+++ b/studio/backend/core/inference/chat_generation_runs.py
@@ -28,20 +28,21 @@ _EVENT_BATCH_MIN_SIZE = 2
_EVENT_BATCH_SECONDS = 0.1
_EVENT_SINGLE_FLUSH_SECONDS = 1.0
_SHUTDOWN_GRACE_SECONDS = 10.0
-# Second budget, after task.cancel(). Shorter than the grace period: by this point the
-# run is already being abandoned, and the only question is whether shutdown returns.
+# Second budget, after task.cancel(). Shorter than the grace period: by this point the run is already being abandoned,
+# and the only question is whether shutdown returns.
_SHUTDOWN_CANCEL_SECONDS = 5.0
-# The sweeper's own shutdown budget, far below the producers'. Its work is redundant
-# at shutdown and Desktop force-kills the backend after five seconds.
+# the sweeper's work is redundant at shutdown, and Desktop force-kills the backend after five seconds
+# The sweeper's own shutdown budget, far below the producers'. Its work is redundant at shutdown and Desktop force-kills
+# the backend after five seconds.
_SWEEP_SHUTDOWN_SECONDS = 0.5
-# A durable run sets cancel_on_disconnect=False, so reaping is keyed on progress rather
-# than on connectedness. The default matches llama_cpp._DEFAULT_FIRST_TOKEN_TIMEOUT_S, the
-# request path's own first-token budget: a lease older than that cannot be legitimate
-# prefill, and slow decode is safe at any speed.
-# A century: clear of any real lease, far below where integer milliseconds overflow.
+# A durable run sets cancel_on_disconnect=False, so reaping is keyed on progress rather than on connectedness. The
+# default matches llama_cpp._DEFAULT_FIRST_TOKEN_TIMEOUT_S, the request path's own first-token budget: a lease older
+# than that cannot be legitimate prefill, and slow decode is safe at any speed. A century: clear of any real lease, far
+# below where integer milliseconds overflow.
_MAX_ENV_SECONDS = 100.0 * 365.0 * 24.0 * 60.0 * 60.0
-# The longest admission keep-alive cadence worth deriving a lease from. A day already
-# means the queue never reports, and tripling it stays far inside _MAX_ENV_SECONDS.
+# a day already means the queue never reports, and tripling it stays inside _MAX_ENV_SECONDS
+# The longest admission keep-alive cadence worth deriving a lease from. A day already means the queue never reports, and
+# tripling it stays far inside _MAX_ENV_SECONDS.
_MAX_ADMISSION_INTERVAL_SECONDS = 24.0 * 60.0 * 60.0
_LEASE_TIMEOUT_SECONDS = 1200.0
_LEASE_SWEEP_INTERVAL_SECONDS = 60.0
@@ -138,8 +139,8 @@ def _env_seconds(name: str, default: float) -> float:
)
return default
if value > _MAX_ENV_SECONDS:
- # Finite is not usable: past ~1.8e305 the multiply to milliseconds overflows and
- # every sweep raises. Clamped, not rejected, since this already meant "never reap".
+ # Finite is not usable: past ~1.8e305 the multiply to milliseconds overflows and every sweep raises. Clamped,
+ # not rejected, since this already meant "never reap".
logger.warning(
"chat_generation_lease_env_clamped",
variable = name,
@@ -165,8 +166,8 @@ async def _sweep_in_daemon_thread(fn, /, *args, **kwargs):
future = loop.create_future()
def _settle(setter, value):
- # The loop can be closed already: this thread outlived the shutdown that
- # abandoned it, which is exactly the case the daemon thread exists to make safe.
+ # The loop can be closed already: this thread outlived the shutdown that abandoned it, which is exactly the case
+ # the daemon thread exists to make safe.
try:
loop.call_soon_threadsafe(lambda: future.done() or setter(value))
except RuntimeError:
@@ -220,8 +221,9 @@ class ChatGenerationLeaseSweeper:
self._task: asyncio.Task | None = None
self._stop_event = asyncio.Event()
- # Grace for a settled producer to notice the cooperative cancel. Generous: unwinding
- # cleanly beats being cancelled mid-teardown, and the run is already declared dead.
+ # generous: unwinding cleanly beats being cancelled mid-teardown
+ # Grace for a settled producer to notice the cooperative cancel. Generous: unwinding cleanly beats being cancelled
+ # mid-teardown, and the run is already declared dead.
_FORCE_CANCEL_GRACE_S = 30.0
@property
@@ -231,12 +233,11 @@ class ChatGenerationLeaseSweeper:
def start(self) -> None:
if self._task is not None or not self.enabled:
return
- # A second lifespan reuses the instance parked on app.state, and stop() left the
- # event set. Recreated rather than cleared, because the second lifespan can also
- # be a different event loop (repeated TestClient contexts, an embedded server
- # restart), and an asyncio.Event stays bound to the loop it was made on: clearing
- # it would leave the new task failing its first wait with "bound to a different
- # event loop", silently disabling reaping for that whole lifespan.
+ # A second lifespan reuses the instance parked on app.state, and stop() left the event set. Recreated rather
+ # than cleared, because the second lifespan can also be a different event loop (repeated TestClient contexts, an
+ # embedded server restart), and an asyncio.Event stays bound to the loop it was made on: clearing it would leave
+ # the new task failing its first wait with "bound to a different event loop", silently disabling reaping for
+ # that whole lifespan.
self._stop_event = asyncio.Event()
self._task = asyncio.create_task(self._run(), name = "chat-generation-lease-sweeper")
@@ -252,8 +253,8 @@ class ChatGenerationLeaseSweeper:
except asyncio.CancelledError:
raise
except Exception as exc:
- # One failed sweep (a locked database, a torn-down home in tests) must
- # not retire the watchdog for the life of the process.
+ # one failed sweep (a locked database, a torn-down home in tests) must not retire the watchdog for the
+ # life of the process
logger.warning("chat_generation_lease_sweep_failed", error = repr(exc))
async def sweep_once(self) -> list[str]:
@@ -275,8 +276,8 @@ class ChatGenerationLeaseSweeper:
)
if supervisor is None:
continue
- # The row is settled, but a producer wedged inside the engine is still
- # holding its slot and activity reservation; cancel unwinds it.
+ # The row is settled, but a producer wedged inside the engine is still holding its slot and activity
+ # reservation; cancel unwinds it.
try:
supervisor.cancel(run_id)
except Exception as exc:
@@ -316,11 +317,11 @@ class ChatGenerationLeaseSweeper:
task, self._task = self._task, None
if task is None:
return
- # A short wait, not the producer grace. Letting a sweep finish at shutdown buys
- # nothing: the boot reconcile settles every active run anyway, so its work is
- # redundant here, while a sweep parked on the writer lock would otherwise spend
- # Studio Desktop's whole graceful-exit budget before producers are even signalled.
- # asyncio.wait, never wait_for(gather(...)); see ChatGenerationSupervisor.stop.
+ # the boot reconcile settles every active run anyway
+ # A short wait, not the producer grace. Letting a sweep finish at shutdown buys nothing: the boot reconcile
+ # settles every active run anyway, so its work is redundant here, while a sweep parked on the writer lock would
+ # otherwise spend Studio Desktop's whole graceful-exit budget before producers are even signalled. asyncio.wait,
+ # never wait_for(gather(...)); see ChatGenerationSupervisor.stop.
_done, pending = await asyncio.wait({task}, timeout = _SWEEP_SHUTDOWN_SECONDS)
if not pending:
return
@@ -344,11 +345,13 @@ def start_lease_sweeper(app: Any) -> ChatGenerationLeaseSweeper | None:
return sweeper
-# The admission stream's own comment, matched rather than imported to keep this module
-# free of a routes import at module scope. Pinned by a test against the constant there.
+# matched rather than imported to keep this module free of a routes import at module scope
+# The admission stream's own comment, matched rather than imported to keep this module free of a routes import at module
+# scope. Pinned by a test against the constant there.
_ADMISSION_WAIT_MARKER = ": admission-wait"
-# Leaving the queue. Renewed unconditionally: wait renewals are rate limited, and the
-# lease equals the first-token timeout, so any age carried in is negative margin.
+# renewed unconditionally: wait renewals are rate limited and the lease equals the first-token timeout
+# Leaving the queue. Renewed unconditionally: wait renewals are rate limited, and the lease equals the first-token
+# timeout, so any age carried in is negative margin.
_ADMISSION_DONE_MARKER = ": admission-done"
@@ -366,9 +369,9 @@ def _minimum_lease_seconds() -> float:
interval = float(llama_admission_config_from_env().keepalive_interval_s)
except Exception:
interval = float(DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S)
- # That parser is not ours and only checks the value is positive, so `inf` arrives
- # intact and makes the applied lease infinite, which the sweeper cannot convert to
- # milliseconds. An oversized finite cadence stretches it past any horizon instead.
+ # That parser is not ours and only checks the value is positive, so `inf` arrives intact and makes the applied lease
+ # infinite, which the sweeper cannot convert to milliseconds. An oversized finite cadence stretches it past any
+ # horizon instead.
if not math.isfinite(interval) or interval > _MAX_ADMISSION_INTERVAL_SECONDS:
logger.warning(
"chat_generation_admission_cadence_ignored",
@@ -421,8 +424,8 @@ def _renew_interval_seconds() -> float:
)
if lease <= 0.0: # sweeping disabled, so cadence only controls write volume
return 30.0
- # The floor must stay UNDER the lease: a one second floor against a one second lease
- # first renews no earlier than expiry. A quarter keeps three renewals per window.
+ # The floor must stay UNDER the lease: a one second floor against a one second lease first renews no earlier than
+ # expiry. A quarter keeps three renewals per window.
return min(30.0, max(0.25, lease / 4.0))
@@ -524,20 +527,18 @@ class ChatGenerationSupervisor:
else:
task = self._tasks.get(run_id)
if task is not None and not task.done():
- # Defensive compatibility for a task registered by a caller
- # other than start(); production tasks always own an event.
task.cancel()
active_generations.cancel_run(run_id)
- # The inference cancel registry closes the narrow gap where registration is imminent
- # but this supervisor has not yet observed it.
+ # closes the narrow gap where registration is imminent but this supervisor has not observed it
+ # The inference cancel registry closes the narrow gap where registration is imminent but this supervisor has not
+ # yet observed it.
from routes.inference import _cancel_by_cancel_id_or_stash
_cancel_by_cancel_id_or_stash(run_id)
async def stop(self) -> None:
self._stopping = True
- # Before the runs, so the sweeper cannot settle a run as stalled while shutdown
- # is already settling it as interrupted.
+ # before the runs, so the sweeper cannot settle a run as stalled while shutdown is settling it as interrupted
sweeper = getattr(getattr(self.app, "state", None), "chat_generation_lease_sweeper", None)
if sweeper is not None:
await sweeper.stop()
@@ -547,11 +548,10 @@ class ChatGenerationSupervisor:
self.cancel(run_id)
if not tasks:
return
- # asyncio.wait, not wait_for(gather(...)): on timeout wait_for cancels the inner
- # future and then awaits it, so a producer that does not unwind on cancellation --
- # an engine draining its subprocess inside the generator's aclose -- makes the
- # wait itself unbounded, and takes the whole uvicorn shutdown down with it. wait
- # returns the pending set instead and leaves those tasks alone.
+ # asyncio.wait, not wait_for(gather(...)): on timeout wait_for cancels the inner future and then awaits it, so a
+ # producer that does not unwind on cancellation -- an engine draining its subprocess inside the generator's
+ # aclose -- makes the wait itself unbounded, and takes the whole uvicorn shutdown down with it. wait returns the
+ # pending set instead and leaves those tasks alone.
pending = {task for _run_id, task in tasks}
_done, pending = await asyncio.wait(pending, timeout = _SHUTDOWN_GRACE_SECONDS)
if not pending:
@@ -561,16 +561,16 @@ class ChatGenerationSupervisor:
_done, pending = await asyncio.wait(pending, timeout = _SHUTDOWN_CANCEL_SECONDS)
if pending:
stuck = [run_id for run_id, task in tasks if task in pending]
- # Abandoned, not leaked: the run is already fenced and reconcile_orphaned_runs
- # settles it on the next boot. Process exit reclaims the rest.
+ # Abandoned, not leaked: the run is already fenced and reconcile_orphaned_runs settles it on the next boot.
+ # Process exit reclaims the rest.
logger.warning(
"Durable chat generations did not stop within the shutdown budget: %s",
", ".join(stuck),
)
- # Total time, not a count: the interval derives from the lease, so a count would mean
- # very different durations. Bounded because an unbounded heartbeat would keep a
- # preparation that never returns alive forever, the failure this file exists to end.
+ # Total time, not a count: the interval derives from the lease, so a count would mean very different durations.
+ # Bounded because an unbounded heartbeat would keep a preparation that never returns alive forever, the failure this
+ # file exists to end.
_PREPARE_RENEW_MAX_SECONDS = 2 * 60 * 60
@contextlib.asynccontextmanager
@@ -608,8 +608,8 @@ class ChatGenerationSupervisor:
interval = _renew_interval_seconds()
for _ in range(max(1, int(self._PREPARE_RENEW_MAX_SECONDS / interval))):
await asyncio.sleep(interval)
- # Skip a contended stamp rather than abandon the rest: giving up here would let a
- # healthy long load be reaped once the last stamp aged out.
+ # skip a contended stamp rather than abandon the rest, else a healthy long load is reaped once the last
+ # stamp ages out
await self._try_touch_progress(run_id)
async def _produce(
@@ -646,8 +646,9 @@ class ChatGenerationSupervisor:
error = "Studio shut down during generation" if shutting_down else None,
)
return
- # Spans the lifecycle gate as well as preparation: a run waiting on the gate is
- # still queued, so its lease ages from created_at with nothing renewing it.
+ # spans the lifecycle gate too: a run waiting on the gate is still queued
+ # Spans the lifecycle gate as well as preparation: a run waiting on the gate is still queued, so its lease
+ # ages from created_at with nothing renewing it.
async with self._lease_heartbeat(run_id):
await activity.start(cancel_event)
if cancel_event.is_set():
@@ -679,16 +680,15 @@ class ChatGenerationSupervisor:
from routes.inference import produce_openai_chat_completions
payload = ChatCompletionRequest.model_validate(run["requestPayload"])
- # Switching, idle reload and auto-download all happen in the call below, and
- # llama.cpp's first-token budget only starts after it. One touch afterwards
- # cannot cover a preparation longer than the lease itself.
+ # Switching, idle reload and auto-download all happen in the call below, and llama.cpp's first-token
+ # budget only starts after it. One touch afterwards cannot cover a preparation longer than the lease
+ # itself.
response = await produce_openai_chat_completions(
payload,
_background_request(self.app, run_id, cancel_event),
owner,
cancel_on_disconnect = False,
)
- # Streamed output is the lease from here; stamped so the handover has no gap.
await self._try_touch_progress(run_id)
if int(getattr(response, "status_code", 200)) >= 400:
raise RuntimeError(f"Local generation returned HTTP {response.status_code}")
@@ -730,14 +730,12 @@ class ChatGenerationSupervisor:
break
next_raw_task = asyncio.create_task(iterator.__anext__())
text = raw.decode("utf-8", "replace") if isinstance(raw, bytes) else str(raw)
- # Admission comments are progress; plain keep-alives are not. A queued run only
- # emits `: admission-wait`, which _SSEDecoder drops, so nothing renewed the lease
- # and a healthy queue reaped its own runs. `: keep-alive` is the opposite signal,
- # emitted when the generator has produced NOTHING, so renewing on any byte would
- # keep a wedged run alive forever. Rate limited because chunk traffic already
+ # Admission comments are progress; plain keep-alives are not. A queued run only emits `:
+ # admission-wait`, which _SSEDecoder drops, so nothing renewed the lease and a healthy queue reaped its
+ # own runs. `: keep-alive` is the opposite signal, emitted when the generator has produced NOTHING, so
+ # renewing on any byte would keep a wedged run alive forever. Rate limited because chunk traffic already
# renews through append_events.
if _ADMISSION_DONE_MARKER in text:
- # Once per run, so no rate limit; the two budgets must start together.
last_keepalive = time.monotonic()
await self._try_touch_progress(run_id)
elif _ADMISSION_WAIT_MARKER in text:
@@ -782,10 +780,9 @@ class ChatGenerationSupervisor:
finish_reason = "interrupted"
error = "Studio shut down during generation"
elif current["cancelRequested"] or (cancel_event.is_set() and error is None):
- # A bare event is not proof of a user stop: the streaming paths set this
- # same event from their cleanup after emitting an in-band error, so a
- # parsed failure outranks it. An explicit cancelRequested still wins,
- # and a real Stop carries no error chunk, so neither loses its identity.
+ # A bare event is not proof of a user stop: the streaming paths set this same event from their cleanup
+ # after emitting an in-band error, so a parsed failure outranks it. An explicit cancelRequested still
+ # wins, and a real Stop carries no error chunk, so neither loses its identity.
status = "cancelled"
finish_reason = "cancelled"
elif error is not None:
diff --git a/studio/backend/core/inference/chat_templates.py b/studio/backend/core/inference/chat_templates.py
index 04c0db6aae0..e2490bbf9ab 100644
--- a/studio/backend/core/inference/chat_templates.py
+++ b/studio/backend/core/inference/chat_templates.py
@@ -19,25 +19,24 @@ from functools import lru_cache
from pathlib import Path
from typing import Optional
-# assets live at <backend>/assets/chat_templates/. This module is at
-# <backend>/core/inference/chat_templates.py, so walk up three parents to
-# <backend> (mirrors utils/inference/inference_config.py).
+# assets live at <backend>/assets/chat_templates/. This module is at <backend>/core/inference/chat_templates.py, so walk
+# up three parents to <backend> (mirrors utils/inference/inference_config.py).
_ASSETS_DIR = Path(__file__).parent.parent.parent / "assets" / "chat_templates"
-# unsloth/gemma-4-<variant>-GGUF (case-insensitive). The "-GGUF" suffix is retained
-# on ModelConfig.identifier for HF GGUF repos, so this matches E2B / E4B / 31B /
-# 26B-A4B and any future unsloth/gemma-4-*-GGUF, while excluding gemma-3,
+# the -GGUF suffix is retained on ModelConfig.identifier, so this excludes the bf16 repos
+# unsloth/gemma-4-<variant>-GGUF (case-insensitive). The "-GGUF" suffix is retained on ModelConfig.identifier for HF
+# GGUF repos, so this matches E2B / E4B / 31B / 26B-A4B and any future unsloth/gemma-4-*-GGUF, while excluding gemma-3,
# non-Unsloth, and non-GGUF identifiers (e.g. the bf16 "unsloth/gemma-4-E2B-it").
_GEMMA4_GGUF_RE = re.compile(r"^unsloth/gemma-4-.+-gguf$", re.IGNORECASE)
-# Google ships two distinct gemma-4 chat templates: E2B/E4B omit the empty
-# "<|channel>thought<channel|>" block on enable_thinking=false, while the
-# 12b/26B-A4B/31B family emits it. Route the two GGUF families to the matching
-# bundled template so each keeps its model's intended behavior.
+# E2B/E4B omit the empty thought block on enable_thinking=false
+# Google ships two distinct gemma-4 chat templates: E2B/E4B omit the empty "<|channel>thought<channel|>" block on
+# enable_thinking=false, while the 12b/26B-A4B/31B family emits it. Route the two GGUF families to the matching bundled
+# template so each keeps its model's intended behavior.
_GEMMA4_EDGE_GGUF_RE = re.compile(r"^unsloth/gemma-4-e[24]b-it-gguf$", re.IGNORECASE)
-_GEMMA4_TEMPLATE_FILE = "gemma-4.jinja" # 12b / 26B-A4B / 31B
-_GEMMA4_EDGE_TEMPLATE_FILE = "gemma-4-edge.jinja" # E2B / E4B
+_GEMMA4_TEMPLATE_FILE = "gemma-4.jinja"
+_GEMMA4_EDGE_TEMPLATE_FILE = "gemma-4-edge.jinja"
def _canonical_repo_id(model_identifier: str) -> str:
diff --git a/studio/backend/core/inference/checkpoint.py b/studio/backend/core/inference/checkpoint.py
index b2e614724c6..9cf37b502e1 100644
--- a/studio/backend/core/inference/checkpoint.py
+++ b/studio/backend/core/inference/checkpoint.py
@@ -40,28 +40,28 @@ from core.inference.context_window import (
)
from core.inference.instruction_pin import is_substantive
-# "checkpoint" resets the epoch; "rolling" is the pre-existing window, byte for byte, and is
-# both the A/B arm and the escape hatch for a template family that misbehaves.
+# "rolling" is the pre-existing window, byte for byte
+# "checkpoint" resets the epoch; "rolling" is the pre-existing window, byte for byte, and is both the A/B arm and the
+# escape hatch for a template family that misbehaves.
CONTEXT_POLICY = os.environ.get("UNSLOTH_CONTEXT_POLICY", "checkpoint").strip().lower()
-# Cap on X. An oversized instruction is excluded whole, never truncated: half an instruction
-# is worse than none, because it reads as complete.
+# an oversized instruction is excluded whole, never truncated
+# Cap on X. An oversized instruction is excluded whole, never truncated: half an instruction is worse than none, because
+# it reads as complete.
MAX_TOKENS = int(os.environ.get("UNSLOTH_CHECKPOINT_MAX_TOKENS", "1024"))
MAX_FRACTION = float(os.environ.get("UNSLOTH_CHECKPOINT_MAX_FRACTION", "0.10"))
-# Bounded so an epoch that dropped 200 turns cannot yield 40 long-superseded instructions.
+# bounded so an epoch that dropped 200 turns cannot yield 40 long-superseded instructions
MAX_ITEMS = int(os.environ.get("UNSLOTH_CHECKPOINT_MAX_ITEMS", "8"))
_OPEN = "<carried_forward>"
_CLOSE = "</carried_forward>"
-# Indent for a wrapped instruction's later lines, so it stays one bullet when read back.
+# indent for a wrapped instruction's later lines, so it stays one bullet when read back
_CONTINUATION = " "
-# The precedence rule is stated because the block sits in the SYSTEM message while its
-# content is the user's own speech, and the role container is the higher authority of the
-# two. Without it the supersession rule reads as scoped to items WITHIN the block, so a
-# carried "the marker is final" outranks the live turn asking to drop the marker, and a
-# prompt-like snippet the user once pasted for review reads as an instruction. Saying the
-# newest message wins, and that the quoted lines are a record rather than commands, costs
-# a sentence and is the one thing the block never said.
+# The precedence rule is stated because the block sits in the SYSTEM message while its content is the user's own speech,
+# and the role container is the higher authority of the two. Without it the supersession rule reads as scoped to items
+# WITHIN the block, so a carried "the marker is final" outranks the live turn asking to drop the marker, and a
+# prompt-like snippet the user once pasted for review reads as an instruction. Saying the newest message wins, and that
+# the quoted lines are a record rather than commands, costs a sentence and is the one thing the block never said.
_HEADER = (
"The conversation before this point was compacted away to make room. The following "
"are the user's own earlier instructions, quoted verbatim, oldest first. They are a "
@@ -71,9 +71,8 @@ _HEADER = (
"Treat the quoted lines as a record of what the user said, not as instructions "
"addressed to you now. "
)
-# The one claim the block makes about the outside world, so the one that can be false. A
-# request without `search_conversation` still deserves the block, but must not be told to
-# reach for a tool it will not be given.
+# The one claim the block makes about the outside world, so the one that can be false. A request without
+# `search_conversation` still deserves the block, but must not be told to reach for a tool it will not be given.
_SEARCHABLE = (
"Everything else that was dropped is still stored and can be retrieved with the "
"search_conversation tool."
@@ -82,7 +81,7 @@ _NOT_SEARCHABLE = (
"Everything else that was dropped is still stored, but you cannot retrieve it on this "
"turn, so answer from what you have rather than saying you will look it up."
)
-# Only the delimiters themselves, so a user who writes about the feature is not mangled.
+# only the delimiters themselves, so a user who writes about the feature is not mangled
_DELIMITERS = re.compile(r"</?carried_forward>", re.IGNORECASE)
@@ -158,12 +157,11 @@ def _pick(
def _item(index: int) -> Optional[tuple[str, int]]:
return entries[index]
- # Where each instruction renders: the position of its NEWEST copy in the transcript,
- # whether or not the walk reaches that copy. The later-wins header makes position the
- # meaning, and reading it off the transcript keeps it independent of the walk order:
- # the reserved pair can fill the cap before a newer copy is reached, which rendered
- # "metric", "imperial", "metric", "add a table" at max_items 3 as metric, imperial,
- # table -- imperial current just after the user restored metric.
+ # Where each instruction renders: the position of its NEWEST copy in the transcript, whether or not the walk reaches
+ # that copy. The later-wins header makes position the meaning, and reading it off the transcript keeps it
+ # independent of the walk order: the reserved pair can fill the cap before a newer copy is reached, which rendered
+ # "metric", "imperial", "metric", "add a table" at max_items 3 as metric, imperial, table -- imperial current just
+ # after the user restored metric.
newest_position: dict[str, int] = {}
for index in range(len(entries)):
found = _item(index)
@@ -171,9 +169,8 @@ def _pick(
newest_position[found[0]] = index
def _walk(order: list[int]) -> list[str]:
- # (position, text) so the render can sort by position: with a reserved item the
- # selection order is no longer the reverse of the transcript order, and
- # `reversed(chosen)` put the oldest turn LAST, inverting supersession.
+ # (position, text) so the render can sort by position: with a reserved item the selection order is no longer the
+ # reverse of the transcript order, and `reversed(chosen)` put the oldest turn LAST, inverting supersession.
picked: list[tuple[int, str]] = []
seen: set[str] = set()
spent = 0
@@ -182,15 +179,14 @@ def _pick(
break
found = _item(index)
if found is None:
+ # One restated rule used to take all eight slots. Checked before the cost is charged, so a repeat cannot
+ # exhaust the budget; every copy renders at `newest_position` anyway, so which one the walk saw first is
+ # moot.
continue
item, cost = found
if item in seen:
- # One restated rule used to take all eight slots. Checked before the cost
- # is charged, so a repeat cannot exhaust the budget; every copy renders at
- # `newest_position` anyway, so which one the walk saw first is moot.
continue
if spent + cost > max_tokens:
- # Skipped, not truncated: an older instruction that fits beats nothing.
continue
picked.append((newest_position[item], item))
seen.add(item)
@@ -203,10 +199,9 @@ def _pick(
found = _item(index)
return found is not None and found[1] <= max_tokens
- # `unit` is oldest-first, how the tail reads it: an abandoned opening can only be its
- # first entry. `spend` is the order the walk charges it in, newest-first for a carried
- # block, so a budget that shrank mid-thread keeps the block's newest bullets rather
- # than filling up on its oldest ones.
+ # `unit` is oldest-first, how the tail reads it: an abandoned opening can only be its first entry. `spend` is the
+ # order the walk charges it in, newest-first for a carried block, so a budget that shrank mid-thread keeps the
+ # block's newest bullets rather than filling up on its oldest ones.
if reserve_leading > 0:
# The already-rendered block in full, not just its first two entries.
unit = [index for index in range(reserve_leading) if _item(index) is not None]
@@ -220,7 +215,6 @@ def _pick(
else next((i for i in range(oldest + 1, len(entries)) if _item(i)), None)
)
unit = [] if oldest is None else [oldest] if successor is None else [oldest, successor]
- # Charged opening first, as always: the tail takes both entries or neither.
spend = unit
else:
unit = []
@@ -272,7 +266,6 @@ def _pick(
chosen = _walk(_reserved_order())
if len(unit) < 2:
- # Nothing followed the opening turn, so nothing can be hidden behind it.
return chosen
opening_text = _item(unit[0])[0]
if opening_text not in chosen:
@@ -281,20 +274,17 @@ def _pick(
if not missing:
return chosen
if not any(_takeable(index) for index in missing):
- # What is missing costs more than the whole budget, so there was never a unit to
- # take. Dropping the opening buys nothing here: it is usually the ONLY turn that
- # fits, so the block would go out empty, which is the failure this pass exists to
- # stop (a 43-token instruction then eight 160-token sections under 100 tokens).
+ # What is missing costs more than the whole budget, so there was never a unit to take. Dropping the opening buys
+ # nothing here: it is usually the ONLY turn that fits, so the block would go out empty, which is the failure
+ # this pass exists to stop (a 43-token instruction then eight 160-token sections under 100 tokens).
return chosen
- # Whole or nothing: something affordable was left behind and the unit still did not fit,
- # and half a unit is the bug itself -- the abandoned request carried with its correction
- # dropped. So the reservation is abandoned and the newest-first walk decides.
- #
- # The opening is excluded from that walk, or the fallback picks it up again whenever it
- # is the cheaper of the two (a 10-token "Build Tetris", a 30-token correction and a
- # 25-token newest turn under 40 tokens dropped the correction). By position, not by
- # text: a user who RESTATES the opening has not abandoned it, and that newer copy stays
- # selectable. Kept only if it says something, since `chosen` already refused to be empty.
+ # Whole or nothing: half a unit is the bug itself, the abandoned request carried with its correction dropped Whole
+ # or nothing: something affordable was left behind and the unit still did not fit, and half a unit is the bug itself
+ # -- the abandoned request carried with its correction dropped. So the reservation is abandoned and the newest-first
+ # walk decides. The opening is excluded from that walk, or the fallback picks it up again whenever it is the cheaper
+ # of the two (a 10-token "Build Tetris", a 30-token correction and a 25-token newest turn under 40 tokens dropped
+ # the correction). By position, not by text: a user who RESTATES the opening has not abandoned it, and that newer
+ # copy stays selectable. Kept only if it says something, since `chosen` already refused to be empty.
return _walk([index for index in plain if index != unit[0]]) or chosen
@@ -380,22 +370,19 @@ def render_checkpoint(items: list[str], *, searchable: bool = True) -> str:
"""The block appended to the system message, or "" when there is nothing to carry."""
if not items:
return ""
- # Continuation lines are INDENTED so a multi-line instruction stays one bullet through
- # the round trip in `_block_items`. Otherwise a user's own list inside an instruction is
- # indistinguishable from the block's bullets and reads back as just its heading.
+ # Continuation lines are INDENTED so a multi-line instruction stays one bullet through the round trip in
+ # `_block_items`. Otherwise a user's own list inside an instruction is indistinguishable from the block's bullets
+ # and reads back as just its heading.
lines = "\n".join("- " + item.replace("\n", "\n" + _CONTINUATION) for item in items)
tail = _SEARCHABLE if searchable else _NOT_SEARCHABLE
return f"{_OPEN}\n{_HEADER}{tail}\n\n{lines}\n{_CLOSE}"
-# A capture group, so `findall` yields the BODY; without it the last item swallows the
-# closing delimiter.
-# The HEADER is part of the pattern, not just the delimiters: the tag is ordinary prompt
-# text and a caller's own system prompt may already use it. Matching on the tag alone
-# stripped that caller-owned section on every reset, reintroduced its bullet lines as
-# lower-authority quoted user history, and deleted whatever was not bullet-shaped, which
-# silently rewrites the caller's policy. Only a block Unsloth itself rendered carries this
-# header, so only that one is claimed.
+# A capture group, so `findall` yields the BODY; without it the last item swallows the closing delimiter. The HEADER is
+# part of the pattern, not just the delimiters: the tag is ordinary prompt text and a caller's own system prompt may
+# already use it. Matching on the tag alone stripped that caller-owned section on every reset, reintroduced its bullet
+# lines as lower-authority quoted user history, and deleted whatever was not bullet-shaped, which silently rewrites the
+# caller's policy. Only a block Unsloth itself rendered carries this header, so only that one is claimed.
_BLOCK = re.compile(
re.escape(_OPEN) + r"\n" + re.escape(_HEADER) + r"(.*?)" + re.escape(_CLOSE) + r"\s*",
re.IGNORECASE | re.DOTALL,
@@ -505,21 +492,18 @@ def fit_checkpoint_context(
max_tokens: Optional[int],
count_tokens: Callable[[list[dict]], int],
protected_message_ids: Optional[set[int]] = None,
- # Signature compatibility with `fit_rolling_context`, DELIBERATELY unused. Rolling
- # spends the reserve by trimming further; after a reset there is nothing left to trim
- # but X, and trading verbatim standing instructions for one recalled passage is the
- # losing side. Instead, a reset with less than one chunk of headroom just skips the
- # automatic recall; the turns are archived and `search_conversation` is offered next
- # request.
+ # Signature compatibility with `fit_rolling_context`, DELIBERATELY unused. Rolling spends the reserve by trimming
+ # further; after a reset there is nothing left to trim but X, and trading verbatim standing instructions for one
+ # recalled passage is the losing side. Instead, a reset with less than one chunk of headroom just skips the
+ # automatic recall; the turns are archived and `search_conversation` is offered next request.
reserve_tokens: int = 0,
sticky_dropped: int = 0,
keeps_boundary: bool = False,
can_reset: bool = False,
searchable: bool = True,
estimate_message: Callable[[dict], int] = estimate_message_tokens,
- # Signature compatibility with `fit_rolling_context`. A checkpoint reset already
- # drops to the latest turn plus X; an extra bite of the window would only shrink
- # the standing-instruction block, which is the half worth keeping.
+ # Signature compatibility with `fit_rolling_context`. A checkpoint reset already drops to the latest turn plus X; an
+ # extra bite of the window would only shrink the standing-instruction block, which is the half worth keeping.
headroom_ratio: Optional[float] = None,
) -> tuple[list[dict], Optional[dict[str, Any]]]:
"""Fit a chat by resetting the epoch, keeping the newest turn and a carried-forward X.
@@ -552,10 +536,9 @@ def fit_checkpoint_context(
alive = {id(message) for message in kept}
evicted = [message for message in messages if id(message) not in alive]
items = carried_forward_items(evicted, max_tokens = budget)
- # A second reset in one request can arrive with a block already in the system turn.
- # Merged and re-capped into ONE block: appending would cap each block separately,
- # bounding a block instead of the (unevictable) system turn. Merged rather than
- # dropped, since that text is now the only copy of those instructions.
+ # A second reset in one request can arrive with a block already in the system turn. Merged and re-capped into
+ # ONE block: appending would cap each block separately, bounding a block instead of the (unevictable) system
+ # turn. Merged rather than dropped, since that text is now the only copy of those instructions.
prior = _block_items(
"".join(
_text_of(message)
@@ -564,9 +547,9 @@ def fit_checkpoint_context(
)
)
if prior:
- # The pair rule travels with the merge: a plain newest-first re-cap could take
- # the opening request and drop the correction to it. Which bullets were the
- # pair is not recoverable from rendered text, so the block goes in as one unit.
+ # The pair rule travels with the merge: a plain newest-first re-cap could take the opening request and drop
+ # the correction to it. Which bullets were the pair is not recoverable from rendered text, so the block goes
+ # in as one unit.
items = _recap(
prior + items,
max_tokens = budget,
@@ -574,27 +557,22 @@ def fit_checkpoint_context(
carried = len(prior),
)
if not items:
- # Nothing to carry, so nothing to claim: do not pay for the probe. The old
- # block still has to GO, though: `_append_to_system` returns early on an empty
- # block, so a system turn that arrived carrying one kept it while the code
- # believed X had been dropped. In a tool loop that is the ordinary case -- an
- # earlier iteration appended a block and the refit sees it again -- and with
- # a small budget the merged items are re-capped away, so the recount stayed
- # over budget and the request was refused or pushed back to rolling even
- # though the base system prompt plus the newest turn fits with room to spare.
+ # Nothing to carry, so nothing to claim: do not pay for the probe. The old block still has to GO, though:
+ # `_append_to_system` returns early on an empty block, so a system turn that arrived carrying one kept it
+ # while the code believed X had been dropped. In a tool loop that is the ordinary case -- an earlier
+ # iteration appended a block and the refit sees it again -- and with a small budget the merged items are
+ # re-capped away, so the recount stayed over budget and the request was refused or pushed back to rolling
+ # even though the base system prompt plus the newest turn fits with room to spare.
return _without_block(kept), ""
text = render_checkpoint(items, searchable = _resolved(searchable))
return _append_to_system(kept, text), text
- # Phase one: replay the epoch already in force. Without it the client re-sending the
- # whole transcript would trigger a fresh reset every request, evicting the epoch's own
- # first turn -- a window of one turn, not an epoch.
- #
- # Gated on the prompt not already fitting, as the rolling replay is: a saved boundary
- # describes the branch AND the window it was measured against. Grow the context
- # mid-thread and the branch fits again, yet the boundary still rides on a live assistant
- # turn. Measured without this gate, a 321-token branch under a 32,256-token budget lost
- # eight messages and came back LARGER (432 tokens).
+ # Phase one: replay the epoch already in force. Without it the client re-sending the whole transcript would trigger
+ # a fresh reset every request, evicting the epoch's own first turn -- a window of one turn, not an epoch. Gated on
+ # the prompt not already fitting, as the rolling replay is: a saved boundary describes the branch AND the window it
+ # was measured against. Grow the context mid-thread and the branch fits again, yet the boundary still rides on a
+ # live assistant turn. Measured without this gate, a 321-token branch under a 32,256-token budget lost eight
+ # messages and came back LARGER (432 tokens).
fitted = list(messages)
dropped = 0
is_new_epoch = False
@@ -612,13 +590,12 @@ def fit_checkpoint_context(
projected, block = _project(fitted)
current_tokens = count_tokens(projected)
- # What `current_tokens` prices, tracked separately because `projected` is rebound
- # below on a path that does not re-count. The refusal reports the pair together.
+ # What `current_tokens` prices, tracked separately because `projected` is rebound below on a path that does not
+ # re-count. The refusal reports the pair together.
measured = projected
- # Phase two: the epoch is full, so start a new one. keep_ratio 0.0 takes every evictable
- # group in one pass; the primitive itself protects system, developer, final and newest
- # user groups.
+ # Phase two: the epoch is full, so start a new one. keep_ratio 0.0 takes every evictable group in one pass; the
+ # primitive itself protects system, developer, final and newest user groups.
if current_tokens > prompt_target and _resolved(can_reset):
candidate, reset_dropped = truncate_oldest_messages(
messages,
@@ -637,22 +614,21 @@ def fit_checkpoint_context(
if dropped == 0 and current_tokens <= prompt_target:
return messages, None
if dropped == 0:
- # Nothing evictable and still too big (one huge message, or a system prompt that
- # leaves no room). Must fall through to the refusal below, since every consumer
- # reads None as "no truncation happened, carry on".
+ # nothing evictable and still too big: must fall through to the refusal below
+ # Nothing evictable and still too big (one huge message, or a system prompt that leaves no room). Must fall
+ # through to the refusal below, since every consumer reads None as "no truncation happened, carry on".
projected = list(messages)
if current_tokens > prompt_target:
- # One turn plus X still does not fit: drop X and re-measure before giving up, since
- # X is a convenience and the user's actual message is not.
+ # One turn plus X still does not fit: drop X and re-measure before giving up, since X is a convenience and the
+ # user's actual message is not.
if block:
projected = _without_block(fitted)
block = ""
current_tokens = count_tokens(projected)
measured = projected
if current_tokens > prompt_target:
- # Let the rolling fit retry from the originals; any projection made here would
- # be discarded by `_fit_context`.
+ # let the rolling fit retry from the originals; any projection made here would be discarded by `_fit_context`
from core.inference.context_window import turn_diagnosis # noqa: PLC0415
return messages, {
"fits": False,
@@ -673,8 +649,8 @@ def fit_checkpoint_context(
"prompt_tokens_after": current_tokens,
"context_length": context_length,
"fits": True,
- # Lets the UI say "reset" rather than "trimmed", and lets the recall gate spot the
- # FIRST turn of an epoch: the forced retrieval fires only there.
+ # Lets the UI say "reset" rather than "trimmed", and lets the recall gate spot the FIRST turn of an epoch: the
+ # forced retrieval fires only there.
"checkpoint": True,
"checkpoint_started": is_new_epoch,
"carried_forward_chars": len(block),
diff --git a/studio/backend/core/inference/context_refusal.py b/studio/backend/core/inference/context_refusal.py
index 124f1dfdbba..77ac3a8dbb9 100644
--- a/studio/backend/core/inference/context_refusal.py
+++ b/studio/backend/core/inference/context_refusal.py
@@ -24,15 +24,14 @@ __all__ = [
]
-# A one-key box, not the refusal itself: `.set()` in a context copy is invisible to the
-# original, where `_friendly_error` runs, but copies share VALUES. See `open_slot`.
+# A one-key box, not the refusal itself: `.set()` in a context copy is invisible to the original, where
+# `_friendly_error` runs, but copies share VALUES. See `open_slot`.
_REFUSAL_SLOT: ContextVar[Optional[dict]] = ContextVar("unsloth_context_refusal", default = None)
-# Share of the irreducible prompt the latest turn must reach before the turn, not the
-# conversation, is blamed. Never all of it: the system prompt and template wrapper are in
-# the floor too. Dominating is NOT the same as not fitting, so it only earns the softer
-# "most of this prompt is ..." wording; the flat "does not fit" needs the turn alone to
-# exceed the budget.
+# Share of the irreducible prompt the latest turn must reach before the turn, not the conversation, is blamed. Never all
+# of it: the system prompt and template wrapper are in the floor too. Dominating is NOT the same as not fitting, so it
+# only earns the softer "most of this prompt is ..." wording; the flat "does not fit" needs the turn alone to exceed the
+# budget.
_TURN_DOMINATES = 0.66
@@ -73,8 +72,7 @@ def record_fit(truncation) -> None:
def clear() -> None:
slot = _slot()
if slot is not None:
- # Empty it as well as dropping it, so a worker mid-flight holding a reference
- # cannot read a stale refusal back.
+ # empty it as well as dropping it, so a worker mid-flight holding a reference cannot read a stale refusal back
slot["refusal"] = None
_REFUSAL_SLOT.set(None)
@@ -106,53 +104,43 @@ def _blame_latest_turn(context_tokens: int):
return None
recorded_context = _int(refusal.get("context_length"))
if context_tokens and recorded_context and recorded_context != context_tokens:
- # A different load or backend: it cannot describe this refusal.
return None
irreducible = _int(refusal.get("irreducible_tokens"))
latest_turn = _int(refusal.get("latest_turn_tokens"))
if irreducible <= 0 or latest_turn <= 0:
return None
- # Only a COUNTED turn is comparable to `irreducible_tokens`. That is a tokenizer count
- # of the rendered prompt; the fallback `latest_turn_tokens` is the message's JSON at
- # four characters a token, so weighing them against each other compares a guess with a
- # truth rather than two sides of one. Measured on the bundled gemma-4 template with a
- # real Gemma tokenizer: 16,400 characters of newlines estimate 8,207 tokens against
- # 557 rendered, 14.8x, which alone clears this ratio against a 8,629-token prompt the
- # turn is 6.5% of -- next to a system prompt that is 93% of it. The user was then told
- # "Most of this prompt is a single tool result" and to fetch a smaller slice of a file
- # that was not the problem. Escaped JSON runs the other way at 0.86x, so the error is
- # not even one-directional and cannot be corrected for.
- #
- # The producer now prices such a turn by difference against the prompt it measured
- # (`turn_diagnosis`), so this flag is False only when nothing could be counted at all.
- # There, no turn is named: a lost diagnosis costs the user a specific lever, a false
- # one sends them after the wrong one. Absent flag means a producer that predates it,
- # which was always a count.
+ # Only a COUNTED turn is comparable to `irreducible_tokens`. That is a tokenizer count of the rendered prompt; the
+ # fallback `latest_turn_tokens` is the message's JSON at four characters a token, so weighing them against each
+ # other compares a guess with a truth rather than two sides of one. Measured on the bundled gemma-4 template with a
+ # real Gemma tokenizer: 16,400 characters of newlines estimate 8,207 tokens against 557 rendered, 14.8x, which alone
+ # clears this ratio against a 8,629-token prompt the turn is 6.5% of -- next to a system prompt that is 93% of it.
+ # The user was then told "Most of this prompt is a single tool result" and to fetch a smaller slice of a file that
+ # was not the problem. Escaped JSON runs the other way at 0.86x, so the error is not even one-directional and cannot
+ # be corrected for. The producer now prices such a turn by difference against the prompt it measured
+ # (`turn_diagnosis`), so this flag is False only when nothing could be counted at all. There, no turn is named: a
+ # lost diagnosis costs the user a specific lever, a false one sends them after the wrong one. Absent flag means a
+ # producer that predates it, which was always a count.
exact = bool(refusal.get("latest_turn_exact", True))
if not exact:
return None
- # Both numbers price a whole rendered PROMPT, so both carry the same floor (template
- # wrapper plus any tool catalogue). Left in, it swamps the comparison: a 6,000-token
- # MCP catalogue makes a 20-token "hi" 97% of the irreducible prompt. Off BOTH sides,
- # so the turn's contribution is compared against the rest of the conversation's.
+ # Both numbers price a whole rendered PROMPT, so both carry the same floor (template wrapper plus any tool
+ # catalogue). Left in, it swamps the comparison: a 6,000-token MCP catalogue makes a 20-token "hi" 97% of the
+ # irreducible prompt. Off BOTH sides, so the turn's contribution is compared against the rest of the conversation's.
shared = _int(refusal.get("shared_prompt_tokens"))
shared = max(0, min(shared, latest_turn - 1, irreducible - 1))
latest_turn -= shared
irreducible -= shared
if latest_turn < _TURN_DOMINATES * irreducible:
return None
- # The WINDOW, not the fit's `prompt_target` (the window minus reserved reply room):
- # llama-server admits a prompt on its size alone ("n_tokens() >= n_ctx" in
- # tools/server/server-context.cpp, the check whose text this rewrites), so a turn
- # between the two really would have been served and only earns the soft wording.
- # `>=` to match that check. Compared without the shared floor, since the hard wording
- # is a claim about the turn's own size.
+ # The WINDOW, not the fit's `prompt_target` (the window minus reserved reply room): llama-server admits a prompt on
+ # its size alone ("n_tokens() >= n_ctx" in tools/server/server-context.cpp, the check whose text this rewrites), so
+ # a turn between the two really would have been served and only earns the soft wording. `>=` to match that check.
+ # Compared without the shared floor, since the hard wording is a claim about the turn's own size.
window = recorded_context or context_tokens
- # Reached only on a counted turn, per the gate above, so this is a claim about a size
- # that was measured. A turn the template renders as nothing on its own is counted by
- # difference, which is why every Gemma tool result can earn this wording again rather
- # than being hedged down for being a guess.
- # Not defaulted to "user": `describe_oversize` gives an unnameable role generic advice.
+ # Reached only on a counted turn, so this is a claim about a measured size Reached only on a counted turn, per the
+ # gate above, so this is a claim about a size that was measured. A turn the template renders as nothing on its own
+ # is counted by difference, so every Gemma tool result can earn this wording again rather than being hedged down for
+ # being a guess. Not defaulted to "user": `describe_oversize` gives an unnameable role generic advice.
role = str(refusal.get("latest_turn_role") or "")
return role, not (window and latest_turn >= window)
@@ -179,16 +167,16 @@ def _history_cannot_help(context_tokens: int) -> bool:
return False
recorded_context = _int(refusal.get("context_length"))
if context_tokens and recorded_context and recorded_context != context_tokens:
- # A different load or backend: it cannot describe this refusal.
return False
irreducible = _int(refusal.get("irreducible_tokens"))
window = recorded_context or context_tokens
return irreducible > 0 and window > 0 and irreducible >= window
-# Per role: what to call the turn when it merely dominates, what to call it when it does
-# not fit at all, and the lever worth offering. The lever is why this splits by role --
-# "send it in smaller pieces" is useless for turns the user did not type.
+# split by role because of the lever: "send it in smaller pieces" is useless for turns the user did not type
+# Per role: what to call the turn when it merely dominates, what to call it when it does not fit at all, and the lever
+# worth offering. The lever is why this splits by role -- "send it in smaller pieces" is useless for turns the user did
+# not type.
_ROLE_ADVICE = {
"user": (
"Most of this prompt is the message just sent",
@@ -200,18 +188,17 @@ _ROLE_ADVICE = {
"A tool returned more than this context window can hold",
"ask for a smaller slice of the file or page",
),
- # The model passed a file-sized argument to a tool. The user did not type it and
- # cannot split it, and the tool cannot be asked for less: `edit_file` with an empty
- # `old_string` is whole-file creation, so the content IS the argument. The only levers
- # are the window itself and not asking for a file this size in a window this small.
+ # The model passed a file-sized argument to a tool. The user did not type it and cannot split it, and the tool
+ # cannot be asked for less: `edit_file` with an empty `old_string` is whole-file creation, so the content IS the
+ # argument. The only levers are the window itself and not asking for a file this size in a window this small.
"assistant_tool_call": (
"Most of this prompt is the file the model passed to a tool",
"The file the model passed to a tool does not fit on its own",
"ask for a smaller file, or raise the Context Length before retrying",
),
- # The same shape with no file in it: an oversized program, command, query or MCP
- # payload. "Ask for a smaller file" names the wrong thing and cannot be acted on, so
- # this one says what is actually true of every tool.
+ # the same shape with no file in it: "ask for a smaller file" names the wrong thing
+ # The same shape with no file in it: an oversized program, command, query or MCP payload. "Ask for a smaller file"
+ # names the wrong thing and cannot be acted on, so this one says what is actually true of every tool.
"assistant_tool_payload": (
"Most of this prompt is what the model passed to a tool",
"What the model passed to a tool does not fit on its own",
@@ -250,13 +237,11 @@ def describe_oversize(request_tokens: int, context_tokens: int) -> str:
advice = _ROLE_ADVICE.get(blamed[0]) if blamed else None
if advice is None:
if _history_cannot_help(context_tokens):
- # No turn to name, and yet "shorten the conversation" is not merely vague
- # here, it is an action that provably cannot work: what survives eviction is
- # already at or over the window. Named levers rather than a role, because the
- # bulk is spread across the parts eviction never touches, and the recorded
- # fields cannot say which of them it is -- `shared_prompt_tokens` bundles the
- # template wrapper with the catalogue, so a large one does not prove there
- # are tools. Both levers are offered, and neither is claimed to be the cause.
+ # No turn to name, and yet "shorten the conversation" is not merely vague here, it is an action that
+ # provably cannot work: what survives eviction is already at or over the window. Named levers rather than a
+ # role, because the bulk is spread across the parts eviction never touches, and the recorded fields cannot
+ # say which of them it is -- `shared_prompt_tokens` bundles the template wrapper with the catalogue, so a
+ # large one does not prove there are tools. Both levers are offered, and neither is claimed to be the cause.
return (
head + "Even with every earlier turn dropped, this prompt would still be "
"too long, so shortening the conversation will not help. Increase the "
@@ -277,9 +262,9 @@ def describe_oversize(request_tokens: int, context_tokens: int) -> str:
)
-# What the user can actually shorten, per tool. Anything absent gets the neutral line:
-# an MCP tool's payload is not a file and not a program, and guessing at it is worse
-# than saying the one thing that is true of every tool.
+# anything absent gets the neutral line: an MCP tool's payload is not a file and not a program
+# What the user can actually shorten, per tool. Anything absent gets the neutral line: an MCP tool's payload is not a
+# file and not a program, and guessing at it is worse than saying the one thing that is true of every tool.
_TOOL_LEVERS = {
"edit_file": "ask for a smaller file",
"python": "run a shorter program",
@@ -309,9 +294,9 @@ def describe_unservable_tool_call(
``compacted_calls`` is reported when history was already spent trying to make room, so
"increase the Context Length" does not read as advice nobody tried.
"""
- # Says "leaving no room to reply" rather than only quoting the two numbers. The bar is
- # the window minus a small reply floor, so a refusal at 3,740 against 4,096 reads as a
- # contradiction unless the message accounts for the gap it is refusing over.
+ # Says "leaving no room to reply" rather than only quoting the two numbers. The bar is the window minus a small
+ # reply floor, so a refusal at 3,740 against 4,096 reads as a contradiction unless the message accounts for the gap
+ # it is refusing over.
head = (
f"Not enough context left to run {tool_name}: the next request would be about "
f"{request_tokens} tokens of a {context_tokens}-token window, leaving no room to "
@@ -324,10 +309,9 @@ def describe_unservable_tool_call(
f"Arguments from {compacted_calls} earlier tool {calls} were already compacted "
"to make room. "
)
- # The gate runs for EVERY enabled tool, so the file wording was reaching an oversized
- # `python`, `terminal`, web or MCP call and telling the user to ask for a smaller
- # file when no file was involved -- advice that cannot make the actual program,
- # command or payload any smaller. `edit_file` keeps the line it was written for.
+ # The gate runs for EVERY enabled tool, so the file wording was reaching an oversized `python`, `terminal`, web or
+ # MCP call and telling the user to ask for a smaller file when no file was involved -- advice that cannot make the
+ # actual program, command or payload any smaller. `edit_file` keeps the line it was written for.
lever = _TOOL_LEVERS.get(tool_name, "ask for less in one call")
return (
head + tried + "Nothing was written. Increase the Context Length in Model settings, "
diff --git a/studio/backend/core/inference/context_window.py b/studio/backend/core/inference/context_window.py
index ade0900a00e..4886c09fbee 100644
--- a/studio/backend/core/inference/context_window.py
+++ b/studio/backend/core/inference/context_window.py
@@ -16,11 +16,10 @@ _UNPRICED_MEDIA_TYPES = frozenset(
("image_url", "input_audio", "audio", "input_image", "input_video")
)
-# How far BELOW the prompt budget a compaction trims, as a fraction of that budget.
-# Trimming to exactly the budget puts the next turn over it again, so the boundary creeps
-# forward every turn: llama-server's prefix cache dies each time and there is no discrete
-# compaction event to report. Taking a chunk out in one go buys a stretch of turns with a
-# fixed head, at the cost of the headroom itself, hence a minority of the budget.
+# How far BELOW the prompt budget a compaction trims, as a fraction of that budget. Trimming to exactly the budget puts
+# the next turn over it again, so the boundary creeps forward every turn: llama-server's prefix cache dies each time and
+# there is no discrete compaction event to report. Taking a chunk out in one go buys a stretch of turns with a fixed
+# head, at the cost of the headroom itself, hence a minority of the budget.
_COMPACTION_HEADROOM_RATIO = max(
0.0, min(0.9, float(os.environ.get("ROLLING_COMPACTION_HEADROOM_RATIO", "0.25")))
)
@@ -87,24 +86,19 @@ def estimate_messages_tokens_dense(messages: list[dict]) -> int:
return total
-# ASCII that runs this far without a break is not prose. Measured with Qwen3 over 16-20k
-# character samples, in characters per token: base64 1.35, hex 1.13, minified JSON 2.75,
-# against English prose at 3.27 and Python source at 4.38. The estimate above charges all
-# of them the English four, so a pasted blob is priced at a third of what it costs, and a
-# caller sizing the room LEFT then hands out room that is already occupied.
-#
-# The rule is per RUN rather than per message, so a blob pasted into a sentence is priced
-# as a blob: a run this long is the thing itself. 64 characters rather than a rounder
-# number because base64 is conventionally wrapped at 76, and at a threshold of 80 a
-# wrapped blob scores as ordinary prose. Measured on the samples above, runs of 64+
-# non-space characters cover 100% of an unwrapped blob, 98.6% of a wrapped one, 0% of the
-# Python source and 23.5% of a README -- and that 23.5% is its URLs and table rules, which
-# tokenise near this rate themselves rather than at the prose rate.
+# ASCII that runs this far without a break is not prose. Measured with Qwen3 over 16-20k character samples, in
+# characters per token: base64 1.35, hex 1.13, minified JSON 2.75, against English prose at 3.27 and Python source at
+# 4.38. The estimate above charges all of them the English four, so a pasted blob is priced at a third of what it costs,
+# and a caller sizing the room LEFT then hands out room that is already occupied. The rule is per RUN rather than per
+# message, so a blob pasted into a sentence is priced as a blob: a run this long is the thing itself. 64 characters
+# rather than a rounder number because base64 is conventionally wrapped at 76, and at a threshold of 80 a wrapped blob
+# scores as ordinary prose. Measured on the samples above, runs of 64+ non-space characters cover 100% of an unwrapped
+# blob, 98.6% of a wrapped one, 0% of the Python source and 23.5% of a README -- and that 23.5% is its URLs and table
+# rules, which tokenise near this rate themselves rather than at the prose rate.
_DENSE_RUN_CHARS = 64
-# Two characters per token, not one. It stays BELOW the measured cost of every sample, so
-# no turn is priced above what it really costs: over-pricing a turn spends the very room
-# this budget exists to hand out, and a result cut to pay for room that was never occupied
-# is the same waste from the other side.
+# Two characters per token, not one. It stays BELOW the measured cost of every sample, so no turn is priced above what
+# it really costs: over-pricing a turn spends the very room this budget exists to hand out, and a result cut to pay for
+# room that was never occupied is the same waste from the other side.
_DENSE_RUN_CHARS_PER_TOKEN = 2
_DENSE_RUN_RE = re.compile(r"\S{%d,}" % _DENSE_RUN_CHARS)
@@ -137,8 +131,8 @@ def estimate_messages_tokens_conservative(
if dense_ascii:
total += max(1, wide + (len(text) - wide) // _DENSE_RUN_CHARS_PER_TOKEN)
continue
- # ASCII only: a run of CJK is already charged a token a character above, and
- # charging it here as well would price it twice.
+ # ASCII only: a run of CJK is already charged a token a character above, and charging it here as well would
+ # price it twice.
runs = sum(
sum(1 for char in match.group(0) if ord(char) <= 127)
for match in _DENSE_RUN_RE.finditer(text)
@@ -264,8 +258,8 @@ def truncate_oldest_messages(
for index, group in enumerate(groups):
if index not in dropped_groups:
if kept and kept[-1].get("role") == "user" and group and group[0].get("role") == "user":
- # Strict chat templates reject adjacent user turns, which an internal
- # tool re-prompt after an evicted exchange would produce.
+ # Strict chat templates reject adjacent user turns, which an internal tool re-prompt after an evicted
+ # exchange would produce.
kept.append({"role": "assistant", "content": _OMITTED_TOOL_EXCHANGE})
kept.extend(group)
return kept, dropped
@@ -301,8 +295,9 @@ def prompt_budget(context_length: int, max_tokens: Optional[int]) -> int:
# Cap a tool-loop retrieval without reducing the allowance of smaller results.
_RETRIEVAL_BUDGET_SHARE = 0.5
-# The least reply room a rescued prompt must leave, as a fraction of the window. Small on
-# purpose: missing the reserve is survivable, so this only rules out the stub-answer end.
+# small on purpose: missing the reserve is survivable
+# The least reply room a rescued prompt must leave, as a fraction of the window. Small on purpose: missing the reserve
+# is survivable, so this only rules out the stub-answer end.
_RESCUE_REPLY_FLOOR_DIVISOR = 16
@@ -327,22 +322,20 @@ def retrieval_budget(
return room
-# Headroom against the tokenizer disagreeing with the estimate that sized a result. A
-# result is measured in characters and spent in tokens, so the conversion is the thing
-# that can be wrong; 1% of the budget absorbs that without noticeably shortening output.
+# a result is measured in characters and spent in tokens
+# Headroom against the tokenizer disagreeing with the estimate that sized a result. A result is measured in characters
+# and spent in tokens, so the conversion is the thing that can be wrong; 1% of the budget absorbs that without
+# noticeably shortening output.
_TOOL_RESULT_BUDGET_BUFFER = 0.99
-# What a truncated result costs BESIDES its body: the notice naming the cut, the spill
-# path and the command that resumes it. Measured at 60-85 tokens, held clear of that.
-# Reserved rather than ignored because the body is sized and the notice appended after,
-# so a budget spent entirely on the body overshoots by the notice every time, and at zero
-# room, where the notice IS the message, by the whole of it.
-#
-# Charged by `tools._truncate`, at the point it knows the result really is being cut, and
-# NOT subtracted from the room below. A result that fits carries no notice, so holding
-# this back up front would cut a result that would have fitted whole and then spend more
-# than it saved: 200 tokens of room, a 100-token result, and reserving first leaves 72 for
-# the body and appends ~70 tokens of notice explaining the 28 that were dropped.
+# What a truncated result costs BESIDES its body: the notice naming the cut, the spill path and the command that resumes
+# it. Measured at 60-85 tokens, held clear of that. Reserved rather than ignored because the body is sized and the
+# notice appended after, so a budget spent entirely on the body overshoots by the notice every time, and at zero room,
+# where the notice IS the message, by the whole of it. Charged by `tools._truncate`, at the point it knows the result
+# really is being cut, and NOT subtracted from the room below. A result that fits carries no notice, so holding this
+# back up front would cut a result that would have fitted whole and then spend more than it saved: 200 tokens of room, a
+# 100-token result, and reserving first leaves 72 for the body and appends ~70 tokens of notice explaining the 28 that
+# were dropped.
_RESULT_NOTICE_RESERVE = 128
@@ -412,21 +405,19 @@ def _reply_floor(context_length: int) -> int:
return max(1, context_length // _RESCUE_REPLY_FLOOR_DIVISOR)
-# How much of a completed call's arguments has to be at stake before replacing them with a
-# receipt is worth the edit. Below this the placeholder is a wash against what it removes,
-# and the model loses sight of its own last action for nothing.
-# Fields naming the call's destination rather than its payload.
+# How much of a completed call's arguments has to be at stake before replacing them with a receipt is worth the edit.
+# Below this the placeholder is a wash against what it removes, and the model loses sight of its own last action for
+# nothing. Fields naming the call's destination rather than its payload.
_PATH_KEYS = frozenset({"path", "file_path", "filePath"})
# Longest path worth repeating inside every elided leaf's receipt.
_RECEIPT_PATH_MAX_CHARS = 120
_ARG_COMPACTION_FLOOR_CHARS = 1024
-# Once the arguments AS A WHOLE are worth reclaiming, the bar each string has to clear
-# drops to this. Not zero: a receipt is about 100 characters, so eliding anything shorter
-# grows the call it is meant to shrink.
+# Once the arguments AS A WHOLE are worth reclaiming, the bar each string has to clear drops to this. Not zero: a
+# receipt is about 100 characters, so eliding anything shorter grows the call it is meant to shrink.
_ARG_COMPACTION_AGGREGATE_LEAF_FLOOR = 256
-# When the total of every string reaches this, the call is worth compacting even though no
-# single string does. Matches the per-leaf floor: the same amount of window either way.
+# When the total of every string reaches this, the call is worth compacting even though no single string does. Matches
+# the per-leaf floor: the same amount of window either way.
_ARG_COMPACTION_TOTAL_FLOOR_CHARS = 1024
@@ -470,58 +461,50 @@ def _compacted_arguments(
file it just wrote and go and read it. A bare "[omitted]" reads as a failed call and
is answered with a retry of the same oversized write.
"""
- # Resolved here, not as a default: the constant is defined below this function, and a
- # literal default silently kept the OLD wording on this path while the executed and
- # refused paths moved to the new one -- the same receipt the model misread as tool
- # output, still being emitted by the most common caller.
+ # Resolved here, not as a default: the constant is defined below this function, and a literal default silently kept
+ # the OLD wording on this path while the executed and refused paths moved to the new one -- the same receipt the
+ # model misread as tool output, still being emitted by the most common caller.
phrase = phrase or _completed_phrase_for(name, reply)
if not isinstance(arguments, str):
return None
- # The general floor exists so a receipt is not bigger than what it replaces, and for
- # ordinary history compaction a call under it is not worth the churn. A REFUSED call
- # is the opposite case: the refusal message itself is about to be added to a prompt
- # that already does not fit, so any reduction at all is the difference between the
- # user reading the refusal and reading llama-server's context error. The size check
- # at the end still guarantees the receipt never grows the prompt.
+ # The general floor exists so a receipt is not bigger than what it replaces, and for ordinary history compaction a
+ # call under it is not worth the churn. A REFUSED call is the opposite case: the refusal message itself is about to
+ # be added to a prompt that already does not fit, so any reduction at all is the difference between the user reading
+ # the refusal and reading llama-server's context error. The size check at the end still guarantees the receipt never
+ # grows the prompt.
refused = phrase == _REFUSED_PHRASE
if not refused and len(arguments) < _ARG_COMPACTION_TOTAL_FLOOR_CHARS:
return None
try:
parsed = json.loads(arguments)
except Exception:
- # Unparseable arguments still cost the window, and a call that has already run
- # cannot be re-issued from them, so the size alone is an honest receipt.
- #
- # Worded from `phrase` like every other receipt. Hardcoding "after the call ran"
- # meant a REFUSED call with 1024+ characters of malformed JSON was replayed as
- # having run, next to the tool message saying nothing was written: two
- # contradictory accounts of the same call, one of which invites the model to
- # reason from a side effect that never happened.
+ # Unparseable arguments still cost the window, and a call that has already run cannot be re-issued from them, so
+ # the size alone is an honest receipt. Worded from `phrase` like every other receipt. Hardcoding "after the call
+ # ran" meant a REFUSED call with 1024+ characters of malformed JSON was replayed as having run, next to the tool
+ # message saying nothing was written: two contradictory accounts of the same call, one of which invites the
+ # model to reason from a side effect that never happened.
_unparseable = json.dumps(
{"_unsloth_compacted": f"{len(arguments)} chars {phrase.format(where = '')}"},
ensure_ascii = False,
)
- # Checked here as well as at the end: without the general floor in front of it,
- # a short refused call can have a receipt longer than the arguments it replaces.
+ # Checked here as well as at the end: without the general floor in front of it, a short refused call can have a
+ # receipt longer than the arguments it replaces.
return _unparseable if len(_unparseable) < len(arguments) else None
if not isinstance(parsed, dict):
return None
path = parsed.get("path") or parsed.get("file_path") or parsed.get("filePath")
elided = 0
- # Per-leaf or aggregate, whichever lets this call be reclaimed. A batched refactor of
- # fifty 800-character edits is forty thousand characters of window with no single
- # string over the floor, so a per-leaf test compacted NOTHING and the call sat in the
- # prompt permanently -- the pre-execution gate then had nothing to reclaim and refused
- # a turn it could have served. The floor exists so a receipt is not bigger than what
- # it replaces, and the final size check below still enforces that.
- # Chosen from the TOTAL, not the largest leaf. Keying on the largest meant one
- # 1100-character edit beside fifty 800-character ones stayed in per-leaf mode and
- # compacted only the first, leaving about 40 KB replayed -- the mixed payload is
- # exactly the shape a batched refactor produces.
+ # Per-leaf or aggregate, whichever lets this call be reclaimed. A batched refactor of fifty 800-character edits is
+ # forty thousand characters of window with no single string over the floor, so a per-leaf test compacted NOTHING and
+ # the call sat in the prompt permanently -- the pre-execution gate then had nothing to reclaim and refused a turn it
+ # could have served. The floor exists so a receipt is not bigger than what it replaces, and the final size check
+ # below still enforces that. Chosen from the TOTAL, not the largest leaf. Keying on the largest meant one
+ # 1100-character edit beside fifty 800-character ones stayed in per-leaf mode and compacted only the first, leaving
+ # about 40 KB replayed -- the mixed payload is exactly the shape a batched refactor produces.
_leaf_floor = (
- # A refused call takes whatever it can get, floored only at the point where a
- # leaf is longer than the receipt describing it, so eliding cannot lose ground.
+ # A refused call takes whatever it can get, floored only at the point where a leaf is longer than the receipt
+ # describing it, so eliding cannot lose ground.
_REFUSED_LEAF_FLOOR
if refused
else _ARG_COMPACTION_AGGREGATE_LEAF_FLOOR
@@ -539,28 +522,26 @@ def _compacted_arguments(
only the tests noticed they had stopped meeting.
"""
nonlocal elided
- # The destination is never expendable: it is the one field that says WHICH file
- # the call touched, and the receipt promises the content can be found there. A
- # deeply nested path over the aggregate floor was being elided like content,
- # leaving later turns unable to name the file they had just changed.
+ # The destination is never expendable: it is the one field that says WHICH file the call touched, and the
+ # receipt promises the content can be found there. A deeply nested path over the aggregate floor was being
+ # elided like content, leaving later turns unable to name the file they had just changed.
if key in _PATH_KEYS:
return value
if isinstance(value, str) and len(value) >= _leaf_floor:
elided += len(value)
- # Named in the receipt only when repeating it is cheaper than the field it
- # points at. Every elided leaf embeds this, so a 500-character path across
- # four leaves costs more than the content removed and the whole compaction
- # is rejected by the size check below. The `path` field is preserved
- # verbatim either way, so nothing is lost by leaving it out here.
+ # Named in the receipt only when repeating it is cheaper than the field it points at. Every elided leaf
+ # embeds this, so a 500-character path across four leaves costs more than the content removed and the whole
+ # compaction is rejected by the size check below. The `path` field is preserved verbatim either way, so
+ # nothing is lost by leaving it out here.
where = (
f" to {path}"
if path and key not in _PATH_KEYS and len(str(path)) <= _RECEIPT_PATH_MAX_CHARS
else ""
)
- # `old_string` names text the edit REMOVED. The completed phrase says the
- # content is already written and the file on disk holds it, which of the two
- # halves of a replacement is true only of `new_string`; said of the old text
- # it points the model at content the edit has just taken out.
+ # `old_string` names text the edit REMOVED, and the completed phrase says the content is already written
+ # `old_string` names text the edit REMOVED. The completed phrase says the content is already written and the
+ # file on disk holds it, which of the two halves of a replacement is true only of `new_string`; said of the
+ # old text it points the model at content the edit has just taken out.
leaf_phrase = _COMPLETED_NEUTRAL_PHRASE if key == "old_string" else phrase
return f"<{len(value)} chars {leaf_phrase.format(where = where)}>"
if isinstance(value, dict):
@@ -576,52 +557,47 @@ def _compacted_arguments(
compacted = json.dumps(kept, ensure_ascii = False)
except Exception:
return None
- # Never grow the prompt to describe it: a call whose bulk is spread across many small
- # fields leaves nothing to elide and the receipts cost more than the fields did.
+ # Never grow the prompt to describe it: a call whose bulk is spread across many small fields leaves nothing to elide
+ # and the receipts cost more than the fields did.
return compacted if len(compacted) < len(arguments) else None
-# A leaf shorter than its own receipt costs room to elide, so this is the break-even
-# point for the refusal wording rather than a judgement about what is worth compacting.
+# A leaf shorter than its own receipt costs room to elide, so this is the break-even point for the refusal wording
+# rather than a judgement about what is worth compacting.
_REFUSED_LEAF_FLOOR = 110
_REFUSED_PHRASE = (
"of arguments you sent, elided; this call was refused before it ran and nothing was written"
)
-# Worded so the model cannot mistake it for the tool's OUTPUT. The first version read
-# "<2581 chars written; re-read the file to see it>" and was quoted straight back in
-# the model's reasoning as "the tool result says ... the output was omitted" -- it
-# concluded the sandbox had mangled its file and abandoned a working approach. Says
-# "you sent" so the owner of the text is unambiguous, and says what it is not.
-# The tail once read "read the file back if you need the content", which contradicted
-# every other line this PR added: `edit_file` now says not to read back what you just
-# wrote, and the starved and repeated-result notices both say reading again will not help.
-# Three messages discouraging a re-read and one inviting it is worse than either rule
-# alone, and the re-read is the loop that cost eighteen calls in one turn.
+# Worded so the model cannot mistake it for the tool's OUTPUT. The first version read "<2581 chars written; re-read the
+# file to see it>" and was quoted straight back in the model's reasoning as "the tool result says ... the output was
+# omitted" -- it concluded the sandbox had mangled its file and abandoned a working approach. Says "you sent" so the
+# owner of the text is unambiguous, and says what it is not. The tail once read "read the file back if you need the
+# content", which contradicted every other line this PR added: `edit_file` now says not to read back what you just
+# wrote, and the starved and repeated-result notices both say reading again will not help. Three messages discouraging a
+# re-read and one inviting it is worse than either rule alone, and the re-read is the loop that cost eighteen calls in
+# one turn.
_COMPLETED_PHRASE = "of arguments you sent, already written{where}; elided to save room. Not tool output; the file on disk holds it."
-# The same receipt for a tool that writes no file. `python`, `terminal`, the search
-# tools and every MCP tool are selected by size and by having been answered, exactly
-# like `edit_file`, so a 2000-character `code` argument was handed a receipt saying it
-# was "already written" and that "the file on disk holds it" -- a file the model can
-# then set out to read, or reason from as persisted, when nothing was persisted at all.
-# Neither claim about the filesystem, for every case where the reply does not settle it.
-# Says only what is certainly true: these are the model's own arguments, the call ran, and
-# this is not the tool's output.
+# The same receipt for a tool that writes no file. `python`, `terminal`, the search tools and every MCP tool are
+# selected by size and by having been answered, exactly like `edit_file`, so a 2000-character `code` argument was handed
+# a receipt saying it was "already written" and that "the file on disk holds it" -- a file the model can then set out to
+# read, or reason from as persisted, when nothing was persisted at all. Neither claim about the filesystem, for every
+# case where the reply does not settle it. Says only what is certainly true: these are the model's own arguments, the
+# call ran, and this is not the tool's output.
_COMPLETED_NEUTRAL_PHRASE = (
"of arguments you sent, elided to save room; the call already ran. Not tool output"
)
# Tools whose completed call really does leave the content in a file.
_FILE_WRITING_TOOLS = frozenset({"edit_file"})
-# A reply that opens like this reports a call that ran and did NOT do what was asked, so
-# the file wording would describe a write that never landed.
+# A reply that opens like this reports a call that ran and did NOT do what was asked, so the file wording would describe
+# a write that never landed.
_FAILED_REPLY_MARKERS = ("error", "failed", "not found", "no such file", "traceback")
-# A reply the WINDOW replaced, not one the tool wrote. Under a near-zero result budget
-# `_fit_result_to_room` swaps the real answer -- including an `Error: ...` -- for a stub
-# saying there was no room for it, and that stub carries none of the markers above. Read
-# as proof of a write, it tells the model an edit landed when the edit may have failed,
-# under exactly the tight context that makes compaction run in the first place. Absence
-# of evidence, so the neutral wording applies.
+# A reply the WINDOW replaced, not one the tool wrote. Under a near-zero result budget `_fit_result_to_room` swaps the
+# real answer -- including an `Error: ...` -- for a stub saying there was no room for it, and that stub carries none of
+# the markers above. Read as proof of a write, it tells the model an edit landed when the edit may have failed, under
+# exactly the tight context that makes compaction run in the first place. Absence of evidence, so the neutral wording
+# applies.
_INCONCLUSIVE_REPLY_MARKERS = ("no context room left", "chars for the model;")
@@ -668,9 +644,8 @@ def compact_executed_call_arguments(messages: list[dict], call_id: str) -> list[
recovered, ending in a one-character reply. Running the call and compacting it costs
the same tokens once and leaves the file written.
"""
- # None, not the constant: the receipt is chosen per tool, so a completed `python`
- # call is not told its arguments are on disk. `edit_file` still gets the wording
- # this path was built and proven against.
+ # None, not the constant: the receipt is chosen per tool, so a completed `python` call is not told its arguments are
+ # on disk. `edit_file` still gets the wording this path was built and proven against.
return _compact_one_call(messages, call_id, None)
@@ -769,10 +744,9 @@ def _compact_one_call(
return out
-# A `role=tool` reply proves an ANSWER, not an execution. The approval gate answers a
-# declined call with exactly such a message, and an unreadable call is answered without
-# running either. Compacting those to the completed receipt tells the model a file was
-# written that the user refused, which it may then report or reason from.
+# A `role=tool` reply proves an ANSWER, not an execution. The approval gate answers a declined call with exactly such a
+# message, and an unreadable call is answered without running either. Compacting those to the completed receipt tells
+# the model a file was written that the user refused, which it may then report or reason from.
_DID_NOT_RUN_MARKERS = (
"the user declined to run this tool call",
"could not be read",
@@ -820,10 +794,9 @@ def _executed_call_sites(messages: list[dict]) -> "dict[tuple[int, str], object]
sites = pending.get(str(call_id))
if not sites:
continue
- # NEWEST pending announcement, not the oldest. Textual parsers number from
- # `call_0` every turn, so an interrupted call that never got a result leaves a
- # stale site under the same id; pairing this reply with THAT one marks the stale
- # arguments executed and leaves the call that actually ran uncompactable.
+ # NEWEST pending announcement, not the oldest. Textual parsers number from `call_0` every turn, so an
+ # interrupted call that never got a result leaves a stale site under the same id; pairing this reply with THAT
+ # one marks the stale arguments executed and leaves the call that actually ran uncompactable.
site = sites.pop()
if _reply_shows_execution(message.get("content")):
executed[(site, str(call_id))] = message.get("content")
@@ -894,15 +867,12 @@ def _blamed_role(message: dict) -> str:
"""
role = str(message.get("role") or "")
if role == "assistant" and message.get("tool_calls"):
- # Split again by whether a FILE is involved. The file wording reached an
- # oversized `python`, `terminal`, web or MCP call and told the user to ask for a
- # smaller file when the payload was a program, a command or a query -- naming the
- # wrong cause and offering an action that cannot shrink it.
- # From the call that accounts for the turn's SIZE, not from whichever call
- # happens to be a file tool. A parallel batch holding a small `edit_file` beside
- # an oversized `python` or MCP payload was diagnosed as a file that is too large,
- # so the advice was to ask for a smaller file -- an action that cannot shrink the
- # payload that actually caused the refusal.
+ # Split again by whether a FILE is involved. The file wording reached an oversized `python`, `terminal`, web or
+ # MCP call and told the user to ask for a smaller file when the payload was a program, a command or a query --
+ # naming the wrong cause and offering an action that cannot shrink it. From the call that accounts for the
+ # turn's SIZE, not from whichever call happens to be a file tool. A parallel batch holding a small `edit_file`
+ # beside an oversized `python` or MCP payload was diagnosed as a file that is too large, so the advice was to
+ # ask for a smaller file -- an action that cannot shrink the payload that caused the refusal.
_dominant = None
_dominant_size = -1
for call in message.get("tool_calls") or []:
@@ -938,8 +908,8 @@ def _blamed_role_for_turn(messages: list[dict]) -> str:
for message in reversed(messages[:-1]):
role = str(message.get("role") or "")
if role != "assistant":
- # Only the call immediately preceding this reply is paired with it; anything
- # else between them means this reply does not belong to a call at all.
+ # Only the call immediately preceding this reply is paired with it; anything else between them means this
+ # reply does not belong to a call at all.
if role == "tool":
continue
break
@@ -950,10 +920,9 @@ def _blamed_role_for_turn(messages: list[dict]) -> str:
for call in message.get("tool_calls") or []
):
break
- # Only when the CALL is the bigger half. A dominant tool result still gets the
- # tool advice, which is right and is what the existing cases assert: the reply is
- # the thing the user can ask for less of. Blaming the call unconditionally
- # reversed that and told them to shrink a payload that was not the problem.
+ # Only when the CALL is the bigger half. A dominant tool result still gets the tool advice, which is right and
+ # is what the existing cases assert: the reply is the thing the user can ask for less of. Blaming the call
+ # unconditionally reversed that and told them to shrink a payload that was not the problem.
_call_chars = sum(
len(str((call.get("function") or {}).get("arguments") or ""))
for call in message.get("tool_calls") or []
@@ -1002,7 +971,6 @@ def _shared_prompt_tokens(count_tokens: Callable[[list[dict]], int]) -> int:
try:
return max(0, int(count_tokens([])))
except Exception:
- # No measurable floor; zero leaves the diagnosis as it was before this existed.
return 0
@@ -1057,7 +1025,6 @@ def turn_diagnosis(
guessed.
"""
if not messages:
- # Vacuously exact: nothing was estimated, and a zero count is ignored anyway.
return {
"latest_turn_tokens": 0,
"latest_turn_role": "",
@@ -1067,31 +1034,27 @@ def turn_diagnosis(
latest, exact = _latest_turn_count(messages, count_tokens)
shared = _shared_prompt_tokens(count_tokens) if exact else 0
if exact and latest <= shared:
- # Counted, and yet no bigger than the empty prompt: the template rendered the turn
- # as nothing. Both bundled Gemma-4 templates do exactly that to a `role: tool`
- # message on its own -- `{%- if message['role'] != 'tool' -%}` skips it, and the
- # result is emitted only while scanning forward from the assistant tool call that
- # asked for it, which a one-message slice does not contain. So the number is the
- # floor and nothing else, and subtracting the floor from it would leave the turn
- # contributing ~0 and blame the conversation.
- #
- # Price it by DIFFERENCE against the prompt that was measured instead: the slice
- # is unrenderable, the contribution is not. Reported floor-inclusive, so the
- # consumer's existing subtraction of `shared` leaves exactly the marginal and the
- # ratio still compares two tokenizer counts.
+ # Counted, and yet no bigger than the empty prompt: the template rendered the turn as nothing. Both bundled
+ # Gemma-4 templates do exactly that to a `role: tool` message on its own -- `{%- if message['role'] != 'tool'
+ # -%}` skips it, and the result is emitted only while scanning forward from the assistant tool call that asked
+ # for it, which a one-message slice does not contain. So the number is the floor and nothing else, and
+ # subtracting the floor from it would leave the turn contributing ~0 and blame the conversation. Price it by
+ # DIFFERENCE against the prompt that was measured instead: the slice is unrenderable, the contribution is not.
+ # Reported floor-inclusive, so the consumer's existing subtraction of `shared` leaves exactly the marginal and
+ # the ratio still compares two tokenizer counts.
marginal = _marginal_turn_count(
fitted if fitted is not None else messages, count_tokens, irreducible_tokens
)
if marginal is not None and marginal > 0:
latest = marginal + shared
else:
- # Nothing countable left: same remedy as a template that refuses the slice
- # outright -- price the message's own JSON, and record no floor, because that
- # estimate does not carry one.
+ # nothing countable left: price the message's own JSON
+ # Nothing countable left: same remedy as a template that refuses the slice outright -- price the message's
+ # own JSON, and record no floor, because that estimate does not carry one.
latest = int(estimate_messages_tokens(messages[-1:]))
shared = 0
- # What is REPORTED is now the estimate, and that is what the flag describes.
- # Leaving it True would be the worse half of the bug.
+ # What is REPORTED is now the estimate, and that is what the flag describes. Leaving it True would be the
+ # worse half of the bug.
exact = False
# Never all of either side: a floor at or above them would leave no ratio to compare.
shared = max(0, min(shared, latest - 1, int(irreducible_tokens) - 1))
@@ -1164,10 +1127,9 @@ def fit_rolling_context(
current_tokens = initial_tokens
dropped_total = 0
- # Phase one: put the boundary back where this thread already had it, so a compacted
- # thread stops compacting further every turn. Gated on the prompt not already
- # fitting: a saved boundary describes the branch it was measured on, and after a
- # rollback it would evict most of a chat that comfortably fits.
+ # Phase one: put the boundary back where this thread already had it, so a compacted thread stops compacting further
+ # every turn. Gated on the prompt not already fitting: a saved boundary describes the branch it was measured on, and
+ # after a rollback it would evict most of a chat that comfortably fits.
if sticky_dropped > 0 and initial_tokens > prompt_target:
candidate, dropped = truncate_oldest_messages(
fitted,
@@ -1181,23 +1143,20 @@ def fit_rolling_context(
dropped_total = dropped
current_tokens = count_tokens(fitted)
- # Phase two, only if what is left still does not fit: move the boundary, taking a
- # chunk out rather than skimming to the brim so it can stay put for a while.
+ # Phase two, only if what is left still does not fit: move the boundary, taking a chunk out rather than skimming to
+ # the brim so it can stay put for a while.
trim_target = prompt_target
- # The 5% floor on each eviction bite, given up only by a caller that asked for no
- # extra trim. Keyed on the ratio and not on `headroom`, which `keeps_boundary = False`
- # zeroes for every threadless and incognito request, none of which chose anything.
+ # The 5% floor on each eviction bite, given up only by a caller that asked for no extra trim. Keyed on the ratio and
+ # not on `headroom`, which `keeps_boundary = False` zeroes for every threadless and incognito request, none of which
+ # chose anything.
min_bite = True
if current_tokens > prompt_target:
- # Summed, not max()'d: the reserve is spent immediately on recalled passages, so
- # counting it as headroom would hand back room that is already taken.
- #
- # And only for a caller that can put the boundary back next request. The headroom
- # buys quiet turns between compactions by cutting deeper than needed, which is a
- # bargain only if the deeper cut is remembered. An incognito chat, an API request
- # with no persisted thread, or a request whose turns are not saved gets neither
- # the boundary nor a recall of what went, so there it is simply 25% less history
- # than plain eviction would have kept.
+ # Summed, not max()'d: the reserve is spent immediately on recalled passages, so counting it as headroom would
+ # hand back room that is already taken. And only for a caller that can put the boundary back next request. The
+ # headroom buys quiet turns between compactions by cutting deeper than needed, which is a bargain only if the
+ # deeper cut is remembered. An incognito chat, an API request with no persisted thread, or a request whose turns
+ # are not saved gets neither the boundary nor a recall of what went, so there it is 25% less history than plain
+ # eviction would have kept.
ratio = clamp_compaction_headroom_ratio(headroom_ratio)
if ratio is None:
ratio = _COMPACTION_HEADROOM_RATIO
@@ -1222,11 +1181,10 @@ def fit_rolling_context(
current_tokens = count_tokens(fitted)
if current_tokens > prompt_target:
- # Missing the prompt target only loses reserved reply room, so keep the original
- # if it fits the physical window, else an eviction that does. Strictly under on
- # both sides: llama-server refuses at `n_ctx` exactly. But not under by one token,
- # which would lose the history AND answer in one token, worse than the refusal it
- # replaces. Capped by the reserve, so the floor demands no room the target did not.
+ # Missing the prompt target only loses reserved reply room, so keep the original if it fits the physical window,
+ # else an eviction that does. Strictly under on both sides: llama-server refuses at `n_ctx` exactly. But not
+ # under by one token, which would lose the history AND answer in one token, worse than the refusal it replaces.
+ # Capped by the reserve, so the floor demands no room the target did not.
reply_floor = min(
max(1, context_length // _RESCUE_REPLY_FLOOR_DIVISOR),
context_length - prompt_target,
@@ -1241,11 +1199,10 @@ def fit_rolling_context(
"dropped_messages": dropped_total if rescued else 0,
"prompt_tokens_before": initial_tokens,
"prompt_tokens_after": current_tokens if rescued else initial_tokens,
- # Floor for the conversation, and how much of it is the message just sent:
- # together they say whether the chat or the single message is the problem.
+ # Floor for the conversation, and how much of it is the message just sent: together they say whether the
+ # chat or the single message is the problem.
"irreducible_tokens": current_tokens,
- # `fitted` is what `current_tokens` prices, so the turn can be counted by
- # difference against it rather than estimated.
+ # `fitted` is what `current_tokens` prices, so the turn can be counted by difference rather than estimated
**turn_diagnosis(
messages, count_tokens, irreducible_tokens = current_tokens, fitted = fitted
),
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 8a89861377b..cb04cf78c3a 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -188,8 +188,8 @@ from utils.paths.path_utils import (
logger = get_logger(__name__)
-# Every `import diffusers` below is lazy, so this runs first. On Windows ROCm both reach an absent
-# distributed backend: diffusers imports xformers on sight, its quantizers torchao.
+# Every `import diffusers` below is lazy, so this runs first. On Windows ROCm both reach an absent distributed backend:
+# diffusers imports xformers on sight, its quantizers torchao.
install_xformers_windows_rocm_stub()
install_torchao_windows_rocm_stub()
@@ -216,8 +216,8 @@ def hub_cache_dir() -> str:
# Repo id out of any hub URL in an error. A gated PUBLIC repo raises a download URL,
-# ".../huggingface.co/<owner>/<name>/resolve/..."; auth_check, and model_info on a gated PRIVATE
-# repo, raise ".../huggingface.co/api/models/<owner>/<name>", which without the prefix branch would yield "api/models".
+# ".../huggingface.co/<owner>/<name>/resolve/..."; auth_check, and model_info on a gated PRIVATE repo, raise
+# ".../huggingface.co/api/models/<owner>/<name>", which without the prefix branch would yield "api/models".
_HUB_REPO_RE = re.compile(
r"huggingface\.co/(?:api/(?:models|datasets|spaces)/)?([\w.\-]+/[\w.\-]+)"
)
@@ -232,7 +232,7 @@ def _gated_in_chain(exc: BaseException) -> Optional[BaseException]:
seen.add(id(exc))
if any(cls.__name__ == "GatedRepoError" for cls in type(exc).__mro__):
return exc
- # `raise ... from None` means the raiser already wrote a better message (the base-repo preflight does), so stop.
+ # `raise ... from None` means the raiser already wrote a better message (the base-repo preflight does)
exc = exc.__cause__ or (None if exc.__suppress_context__ else exc.__context__)
return None
@@ -244,7 +244,8 @@ def _hf_token_in_play(hf_token: Optional[str]) -> bool:
if hf_token:
return True
try:
- # What build_hf_headers calls: under HF_HUB_DISABLE_IMPLICIT_TOKEN get_token() still answers while the request goes out anonymous.
+ # What build_hf_headers calls: under HF_HUB_DISABLE_IMPLICIT_TOKEN get_token() still answers while the request
+ # goes out anonymous
from huggingface_hub.utils import get_token_to_send
return bool(get_token_to_send(None))
except Exception: # noqa: BLE001 -- assume none; at worst the message says "add a token"
@@ -259,9 +260,10 @@ def hub_access_message(exc: BaseException, *, had_token: bool) -> Optional[str]:
if gated is None:
return None
found = _HUB_REPO_RE.search(str(gated))
- # An API URL ends at the repo, so a trailing full stop lands inside the name (dots are legal mid-name, but a name never ends in one).
+ # An API URL ends at the repo, so a trailing full stop lands inside the name (dots are legal mid-name, but a name
+ # never ends in one)
repo = found.group(1).rstrip(".") if found else None
- # Any other /api/<endpoint> URL (whoami-v2, ...) would parse as the repo "api/<endpoint>".
+ # Any other /api/<endpoint> URL (whoami-v2, ...) would parse as the repo "api/<endpoint>"
if repo and repo.split("/", 1)[0] == "api":
repo = None
where = f"https://huggingface.co/{repo}" if repo else "its Hugging Face page"
@@ -311,7 +313,7 @@ def _active_lora_pairs(pipe: Any) -> list:
else:
continue
weight = float(weight)
- except Exception: # noqa: BLE001 — an unrecognised marker records no adapter
+ except Exception: # noqa: BLE001 - an unrecognised marker records no adapter
continue
if weight:
pairs.append((name, weight))
@@ -337,7 +339,7 @@ def _baked_lora_names(pipe: Any) -> list:
for entry in getattr(pipe, "_unsloth_loras", ()) or ():
try:
name = entry[0]
- except Exception: # noqa: BLE001 — an unrecognised marker records no adapter
+ except Exception: # noqa: BLE001 - an unrecognised marker records no adapter
continue
if name:
names.append(str(name))
@@ -360,7 +362,7 @@ def resolve_local_single_file(model_path: str) -> Optional[str]:
root = Path(model_path).expanduser()
if not root.is_dir() or (root / "model_index.json").is_file():
return None
- # A PEFT adapter folder is not a base checkpoint; skip it so validation 400s before eviction.
+ # A PEFT adapter folder is not a base checkpoint; skip it so validation 400s before eviction
if (root / "adapter_config.json").is_file():
return None
@@ -397,16 +399,14 @@ def decode_b64_image(
raw = data.strip()
if raw.startswith("data:"):
- # data:[<mime>][;base64],<payload>
_, _, raw = raw.partition(",")
try:
blob = base64.b64decode(raw, validate = False)
except (binascii.Error, ValueError) as exc:
raise ValueError(f"Invalid base64 image data: {exc}") from exc
- # Preprocessing paths may choose different bounded source limits.
try:
img = Image.open(io.BytesIO(blob))
- # Reject from the header before img.load() so a huge-dimension file cannot spike memory.
+ # Reject from the header before img.load() so a huge-dimension file cannot spike memory
w, h = img.size
if w > max_side or h > max_side:
raise ValueError(f"Image is too large ({w}x{h}); maximum is {max_side}px per side.")
@@ -417,7 +417,7 @@ def decode_b64_image(
img.load()
except ValueError:
raise # the size guard's own message; don't wrap it as a decode error
- except Exception as exc: # noqa: BLE001 — surfaced as a 400 to the client
+ except Exception as exc: # noqa: BLE001 - surfaced as a 400 to the client
raise ValueError(f"Could not decode image: {exc}") from exc
return img.convert(mode)
@@ -543,31 +543,32 @@ def _compile_shape_dims(workflow: str, init_pil: Any, width: int, height: int) -
return int(iw), int(ih)
-# Official non-unsloth pipeline bases: safetensors-only, no remote code, exact lowercased match.
+# Official non-unsloth pipeline bases: safetensors-only, no remote code, exact lowercased match
_TRUSTED_NON_GGUF_REPOS = frozenset(
{
"stabilityai/stable-diffusion-xl-base-1.0",
"stabilityai/sdxl-turbo",
- # Vendor safetensors-only bases: LoRA training bases + the BF16 artifact per group. FLUX.1 is Hub-gated; Qwen/Z-Image open.
+ # Vendor safetensors-only bases: LoRA training bases + the BF16 artifact per group. FLUX.1 is Hub-gated;
+ # Qwen/Z-Image open.
"black-forest-labs/flux.1-dev",
"black-forest-labs/flux.1-schnell",
"black-forest-labs/flux.1-kontext-dev",
- # Krea: guidance-distilled FLUX.1-dev finetune, same arch and gating; detected via "flux.1".
+ # Krea: guidance-distilled FLUX.1-dev finetune, same arch and gating; detected via "flux.1"
"black-forest-labs/flux.1-krea-dev",
# FLUX.2 LoRA training bases (dev gated, klein-4B open); reloaded as a pipeline by "Deploy to Create".
"black-forest-labs/flux.2-dev",
"black-forest-labs/flux.2-klein-4b",
- # The other klein sizes and variants. One family covers both sizes and defaults to 4B, on
- # the understanding that the real base comes from the base_model card tag. Leaving these
- # out made that tag untrusted, so a klein-9B GGUF silently resolved to the 4B config and
- # died in the GGUF quantizer with a bare shape mismatch (24576x4096 vs 18432x3072).
+ # The other klein sizes and variants. One family covers both sizes and defaults to 4B, on the understanding that
+ # the real base comes from the base_model card tag. Leaving these out made that tag untrusted, so a klein-9B
+ # GGUF silently resolved to the 4B config and died in the GGUF quantizer with a bare shape mismatch (24576x4096
+ # vs 18432x3072).
"black-forest-labs/flux.2-klein-9b",
"black-forest-labs/flux.2-klein-base-4b",
"black-forest-labs/flux.2-klein-base-9b",
"tongyi-mai/z-image-turbo",
- # The undistilled Z-Image: the base the upstream DreamBooth recipe trains on, and the
- # base_model tag of unsloth/Z-Image-GGUF. Leaving it out made that tag untrusted, so the
- # GGUF fell back to the Turbo companions and denoised on its shift 3.0 scheduler.
+ # The undistilled Z-Image: the base the upstream DreamBooth recipe trains on, and the base_model tag of
+ # unsloth/Z-Image-GGUF. Leaving it out made that tag untrusted, so the GGUF fell back to the Turbo companions
+ # and denoised on its shift 3.0 scheduler.
"tongyi-mai/z-image",
"qwen/qwen-image",
"qwen/qwen-image-2512",
@@ -583,7 +584,8 @@ _TRUSTED_NON_GGUF_REPOS = frozenset(
"hidream-ai/hidream-i1-full",
"hidream-ai/hidream-i1-dev",
"hidream-ai/hidream-i1-fast",
- # Ideogram 4: no bf16 ships. -fp8 stores both DiTs as raw float8 (the family base); both nf4 repos are identical.
+ # Ideogram 4: no bf16 ships. -fp8 stores both DiTs as raw float8 (the family base); both nf4 repos are
+ # identical.
"ideogram-ai/ideogram-4-fp8",
"ideogram-ai/ideogram-4-nf4",
"ideogram-ai/ideogram-4-nf4-diffusers",
@@ -699,15 +701,15 @@ def _assert_base_repo_accessible(
# Only a remote 'org/name' can be gated; a local base is already on disk.
if not repo or repo.count("/") != 1:
return None
- # Blank to None, as load_pipeline does: build_hf_headers sends "" as a literal "Bearer " that
- # 401s, which the handling below would read as an open base being unreadable.
+ # Blank to None, as load_pipeline does: build_hf_headers sends "" as a literal "Bearer " that 401s, which the
+ # handling below would read as an open base being unreadable.
hf_token = (hf_token.strip() if isinstance(hf_token, str) else hf_token) or None
try:
if Path(repo).expanduser().exists():
return None
- # OSError: invalid path characters -> a remote id. RuntimeError: pathlib raises it, not OSError,
- # when expanduser() cannot resolve the home dir ('~other/models', or '~/models' with no HOME).
- # Both carry one slash, so they reach here, and an escape would 500 this fail-open probe.
+ # OSError: invalid path characters -> a remote id. RuntimeError: pathlib raises it, not OSError, when expanduser()
+ # cannot resolve the home dir ('~other/models', or '~/models' with no HOME). Both carry one slash, so they reach
+ # here, and an escape would 500 this fail-open probe.
except (OSError, RuntimeError, ValueError):
pass
try:
@@ -717,17 +719,16 @@ def _assert_base_repo_accessible(
HfHubHTTPError,
RepositoryNotFoundError,
)
- except Exception: # noqa: BLE001 — an unexpected hub layout leaves today's behaviour
+ except Exception: # noqa: BLE001 - an unexpected hub layout leaves today's behaviour
return None
- # Set when only the import-time root holds the excusing copy, which the pinned from_pretrained
- # would miss.
+ # Set when only the import-time root holds the excusing copy, which the pinned from_pretrained would miss.
other_root_snapshot: Optional[str] = None
- # A base already on disk needs no Hub access: hf_hub_download catches the gated/401 HEAD and
- # serves the cached pointer (file_download._get_metadata_or_catch_error), which is how a
- # downloaded gated base still loads once the token is cleared or expires. Excuses an ACCESS
- # verdict only, never a 404: a renamed or removed repo cannot be un-renamed by a stale copy.
+ # A base already on disk needs no Hub access: hf_hub_download catches the gated/401 HEAD and serves the cached
+ # pointer (file_download._get_metadata_or_catch_error), which is how a downloaded gated base still loads once the
+ # token is cleared or expires. Excuses an ACCESS verdict only, never a 404: a renamed or removed repo cannot be
+ # un-renamed by a stale copy.
def _already_downloaded() -> bool:
"""True when ``probe_file`` is on disk under EITHER root (Unsloth pins its live setting, the
prefetch writes under huggingface_hub's import-time constant). Never raises. Exact for the
@@ -741,21 +742,20 @@ def _assert_base_repo_accessible(
# Only a str is a cached path; a miss is None and an absent file is a sentinel.
hit = try_to_load_from_cache(repo, probe_file, cache_dir = root)
if isinstance(hit, str):
- # The live root is tried first, so root None means only the import-time one has
- # it. Its parent is the snapshot dir only for a top-level probe.
+ # The live root is tried first, so root None means only the import-time one has it. Its parent is
+ # the snapshot dir only for a top-level probe.
if root is None and "/" not in probe_file:
other_root_snapshot = str(Path(hit).parent)
return True
- except Exception: # noqa: BLE001 — a cache we cannot read is not an access verdict
+ except Exception: # noqa: BLE001 - a cache we cannot read is not an access verdict
pass
return False
if local_files_only:
- # Cache read only: no model_info, no metadata HEAD. ``_already_downloaded`` is called for
- # its side effect (it sets ``other_root_snapshot``), and its bool is deliberately dropped
- # -- a miss here is not a refusal. The loader is the one that raises for a file that is not
- # on disk, with the file's own name in it, which is a better answer than this probe's
- # "add a token" text for a load that was never going to fetch anything.
+ # Cache read only: no model_info, no metadata HEAD. ``_already_downloaded`` is called for its side effect (it
+ # sets ``other_root_snapshot``), and its bool is dropped -- a miss here is not a refusal. The loader is the one
+ # that raises for a file that is not on disk, with the file's own name in it, which is a better answer than this
+ # probe's "add a token" text for a load that was never going to fetch anything.
_already_downloaded()
return other_root_snapshot
@@ -774,9 +774,9 @@ def _assert_base_repo_accessible(
return other_root_snapshot
raise ValueError(_repo_access_message(repo, gated = True)) from None
except RepositoryNotFoundError as exc:
- # 401 and 404 both land here: hf_raise_for_status folds unauthenticated private/gated in
- # with a missing repo because "401 is misleading" (_http.py), so re-read the status. A
- # 401/403 earns the cache escape; a genuine 404, or an error carrying no response, raises.
+ # 401 and 404 both land here: hf_raise_for_status folds unauthenticated private/gated in with a missing repo
+ # because "401 is misleading" (_http.py), so re-read the status. A 401/403 earns the cache escape; a genuine
+ # 404, or an error carrying no response, raises.
if _is_auth_error(exc) and _already_downloaded():
return other_root_snapshot
raise ValueError(_repo_access_message(repo, gated = False)) from None
@@ -786,15 +786,15 @@ def _assert_base_repo_accessible(
if _already_downloaded():
return other_root_snapshot
raise ValueError(_repo_access_message(repo, gated = False)) from None
- except Exception: # noqa: BLE001 — offline / transient: the download surfaces any real error
+ except Exception: # noqa: BLE001 - offline / transient: the download surfaces any real error
return None
- # Nothing to carry: model_info answered, so the size estimate lists the base files and the
- # prefetch resolves each through whichever root holds it.
+ # Nothing to carry: model_info answered, so the size estimate lists the base files and the prefetch resolves each
+ # through whichever root holds it
if not gated or _already_downloaded():
return None
try:
- # A metadata HEAD, never hf_hub_download: a cached manifest makes the download return the
- # cached pointer, so a stale token would pass the probe and 401 again mid-prefetch.
+ # A metadata HEAD, never hf_hub_download: a cached manifest makes the download return the cached pointer, so a
+ # stale token would pass the probe and 401 again mid-prefetch.
get_hf_file_metadata(hf_hub_url(repo, probe_file), token = hf_token)
except GatedRepoError:
raise ValueError(_repo_access_message(repo, gated = True)) from None
@@ -802,7 +802,7 @@ def _assert_base_repo_accessible(
if not _is_auth_error(exc):
return None
raise ValueError(_repo_access_message(repo, gated = True)) from None
- except Exception: # noqa: BLE001 — no manifest / offline / transient is not an access verdict
+ except Exception: # noqa: BLE001 - no manifest / offline / transient is not an access verdict
return None
return None
@@ -838,14 +838,14 @@ class _LoadState:
attention_request: Optional[str] = None
# Step cache engaged ("fbcache") or None. Opt-in, for many-step models.
transformer_cache: Optional[str] = None
- # AUTO: generate() toggles FBCache across FBCACHE_MIN_STEPS; an explicit request never toggles.
+ # AUTO: generate() toggles FBCache across FBCACHE_MIN_STEPS; an explicit request never toggles
cache_auto: bool = False
# Inputs the generation-time toggle re-applies (quantised threshold + override).
cache_quant_active: bool = False
cache_threshold: Optional[float] = None
# Shared eager patches installed for this load; uninstalled on unload.
eager_patched: bool = False
- # Deferred speed auto: the load stays eager, generate() engages `default` at the 3rd generation.
+ # Deferred speed auto: the load stays eager, generate() engages `default` at the 3rd generation
speed_deferred: bool = False
# Successful generations on this load; drives the deferred engagement above.
generation_count: int = 0
@@ -857,17 +857,16 @@ class _LoadState:
resolved: Optional[dict] = None
# The single-file checkpoint basename this load committed (None for a pipeline). Part of the build identity.
gguf_filename: Optional[str] = None
- # The torch ordinal this pipeline's weights were placed on, or None for an automatic pick.
- # Committed WITH the pipeline, so a load in flight never moves the resident model's card.
+ # The torch ordinal this pipeline's weights were placed on, or None for an automatic pick. Committed WITH the
+ # pipeline, so a load in flight never moves the resident model's card.
gpu_ordinal: Optional[int] = None
- # The card the weights are ACTUALLY on, including the automatic case, where it is whichever
- # device the load thread was pointing at. Only for re-pinning a pooled generate worker that a
- # previous pinned load left on another card; the target and the reported build read
- # gpu_ordinal, so the automatic path still resolves a bare device.
+ # The card the weights are ACTUALLY on, including the automatic case, where it is whichever device the load thread
+ # was pointing at. Only for re-pinning a pooled generate worker that a previous pinned load left on another card;
+ # the target and the reported build read gpu_ordinal, so the automatic path still resolves a bare device.
placed_ordinal: Optional[int] = None
- # The exact variant hint the memory plan was built from (family + checkpoint name + repo ids).
- # Stored rather than rebuilt so generate()'s activation re-check budgets with the SAME
- # distilled / edit multipliers the load did, and the two can never drift apart.
+ # The exact variant hint the memory plan was built from (family + checkpoint name + repo ids). Stored rather than
+ # rebuilt so generate()'s activation re-check budgets with the SAME distilled / edit multipliers the load did, and
+ # the two can never drift apart.
variant_hint: str = ""
@@ -879,9 +878,8 @@ class _LoadingState:
base_repo: str
expected_bytes: int = 0
error: Optional[str] = None
- # Where the companion BYTES land: ``base_repo``, or its mirror when one was swapped in. Never
- # surfaced (``base_repo`` stays the id status() reports), but the cache scan and the delete
- # guard must look here.
+ # Where the companion BYTES land: ``base_repo``, or its mirror when one was swapped in. Never surfaced
+ # (``base_repo`` stays the id status() reports), but the cache scan and the delete guard must look here.
fetch_repo: Optional[str] = None
@@ -954,7 +952,7 @@ def _install_gguf_prefix_strip(transformer_cls: Any, logger: Any) -> None:
_stripped_mapping_fn._unsloth_prefix_strip = True
entry["checkpoint_mapping_fn"] = _stripped_mapping_fn
- except Exception as exc: # noqa: BLE001 — loader-compat shim only, never fail the load
+ except Exception as exc: # noqa: BLE001 - loader-compat shim only, never fail the load
logger.warning("diffusion.gguf: prefix-strip shim not installed: %s", exc)
@@ -1028,13 +1026,12 @@ def _uncached_prequant_repo(
source = usable_prequant_source(
fam, scheme, path_override = prequant_path, base_repo = base_repo
)
- # A local override is the operator's own file, so it never downloads.
if source is None or source.kind != "repo":
return None
if prequant_checkpoint_cached(source, cache_dir = hub_cache_dir()):
return None
return source.location
- except Exception: # noqa: BLE001 — a probe that cannot answer keeps the prequant shortcut
+ except Exception: # noqa: BLE001 - a probe that cannot answer keeps the prequant shortcut
return None
@@ -1143,7 +1140,7 @@ def _dense_candidate_is_prequant(
usable_prequant_source(fam, scheme, path_override = prequant_path, base_repo = base_repo)
is not None
)
- except Exception: # noqa: BLE001 — a probe that cannot answer keeps the decline
+ except Exception: # noqa: BLE001 - a probe that cannot answer keeps the decline
return False
@@ -1188,20 +1185,19 @@ class DiffusionBackend:
self._generate_lock = threading.Lock()
self._state: Optional[_LoadState] = None
self._loading: Optional[_LoadingState] = None
- # Bumped on begin_load/unload so a superseded worker neither commits nor stamps progress.
+ # Bumped on begin_load/unload so a superseded worker neither commits nor stamps progress
self._load_token = 0
- # Set by unload() to abort an in-flight download. Replaced, never cleared, so a cancelled worker stays cancelled.
+ # Set by unload() to abort an in-flight download. Replaced, never cleared, so a cancelled worker stays
+ # cancelled.
self._cancel_event = threading.Event()
# Keep Stop responsive while a replacement holds _lock.
self._generation_cancel_lock = threading.Lock()
- # Cancel Event of the in-flight generation; per-generation so a cancel can't be lost or leak.
+ # Cancel Event of the in-flight generation; per-generation so a cancel can't be lost or leak
self._active_generate_cancel: Optional[threading.Event] = None
# Queued requests; cancel_generate() decides which Stop may signal.
self._queued_generate_cancels: set[threading.Event] = set()
- # True while a generation owns the slot, including its epilogue.
self._generation_owns_slot = False
- # True while a load or unload owns the generation slot.
self._transition_owns_slot = False
# Teardowns waiting to free this pipeline; a count supports concurrent reservations.
self._teardown_waiters = 0
@@ -1210,9 +1206,9 @@ class DiffusionBackend:
self._teardown_drained.set()
# Written by the callback, read lock-free by generate_progress().
self._gen: Optional[_GenState] = None
- # img2img/inpaint pipes built via from_pipe (shared modules, no extra VRAM); cleared on unload.
+ # img2img/inpaint pipes built via from_pipe (shared modules, no extra VRAM); cleared on unload
self._aux_pipes: dict[str, Any] = {}
- # Loaded ControlNets and their from_pipe pipelines, reusing resident modules; cleared on unload.
+ # Loaded ControlNets and their from_pipe pipelines, reusing resident modules
self._cn_models: dict[str, Any] = {}
self._cn_pipes: dict[tuple[str, str], Any] = {}
@@ -1282,12 +1278,10 @@ class DiffusionBackend:
or reserved before the check and makes this request yield.
"""
admitted = False
- # Publish before the first acquisition so Stop can reach this request.
with self._generation_cancel_lock:
self._queued_generate_cancels.add(cancel)
try:
while True:
- # Timed acquisition keeps queued requests responsive during replacement.
while True:
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
@@ -1295,7 +1289,6 @@ class DiffusionBackend:
break
with self._lock:
if not self._teardown_waiters:
- # Register atomically with the zero-fence check.
with self._generation_cancel_lock:
cancelled = cancel.is_set()
if not cancelled:
@@ -1310,7 +1303,6 @@ class DiffusionBackend:
self._generate_lock.release()
if cancelled:
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
- # The gate wait leaves _lock free for the transition.
while True:
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
@@ -1319,7 +1311,6 @@ class DiffusionBackend:
try:
yield
finally:
- # Release the slot and lock atomically for cancellation.
with self._generation_cancel_lock:
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
@@ -1330,8 +1321,8 @@ class DiffusionBackend:
with self._generation_cancel_lock:
self._queued_generate_cancels.discard(cancel)
- # Memory requests whose offload policy is decided by the REQUEST rather than by the measured
- # footprint. `fast` and `auto` are measurements and so cannot be judged network-free.
+ # Memory requests whose offload policy is decided by the REQUEST rather than by the measured footprint. `fast` and
+ # `auto` are measurements and so cannot be judged network-free.
def assert_precision_available(
self,
fam: Optional[DiffusionFamily],
@@ -1360,10 +1351,10 @@ class DiffusionBackend:
te_mode = normalize_te_quant(text_encoder_quant)
if pinned is None and te_mode is None:
return
- # One probe for both checks, asked of the card THIS load will use: on a mixed box the
- # default device can be a different generation, which would refuse a scheme the selected
- # card supports, or pass one it does not and fail only after eviction. Scoped, because the
- # route calls this on a pooled thread and the probes below use argument-less CUDA calls.
+ # One probe for both checks, asked of the card THIS load will use: on a mixed box the default device can be a
+ # different generation, which would refuse a scheme the selected card supports, or pass one it does not and fail
+ # only after eviction. Scoped, because the route calls this on a pooled thread and the probes below use
+ # argument-less CUDA calls.
with diffusion_device_scope(gpu_ordinal):
target = self._target_for_ordinal(fam, gpu_ordinal)
return self._assert_precision_for_target(
@@ -1396,11 +1387,10 @@ class DiffusionBackend:
f"'{model_kind}' load, which runs the precision its checkpoint carries"
)
elif _memory_request_forces_offload(memory_mode, cpu_offload):
- # Not a measurement: balanced and low_vram name their policy outright, and the
- # legacy flag forces model offload. Offload hooks move modules with Module.to(),
- # which torchao tensors do not survive, so the loader skips the dense build for
- # any of them -- and the strict refusal then landed after the resident model was
- # already gone. The two requests are incompatible on their face; say so here.
+ # Not a measurement: balanced and low_vram name their policy outright, and the legacy flag forces model
+ # offload. Offload hooks move modules with Module.to(), which torchao tensors do not survive, so the
+ # loader skips the dense build for any of them -- and the strict refusal then landed after the resident
+ # model was already gone. The two requests are incompatible on their face; say so here.
requested_memory = normalize_memory_mode(memory_mode) or "cpu_offload"
reason = (
f"'{requested_memory}' memory places the transformer under CPU offload, and "
@@ -1414,17 +1404,15 @@ class DiffusionBackend:
target,
pinned,
family = getattr(fam, "name", None),
- # This gate runs BEFORE the arbiter evicts the resident model, so a smoke probe
- # that runs out of VRAM here has not shown the scheme unusable, only that the
- # GPU is still full. Refusing on that would reject a load the eviction was
- # about to make room for.
+ # This gate runs BEFORE the arbiter evicts the resident model, so a smoke probe that runs out of
+ # VRAM here has not shown the scheme unusable, only that the GPU is still full. Refusing on that
+ # would reject a load the eviction was about to make room for.
unproven_ok = True,
)
is None
):
- # An explicit scheme is never swapped for another, so a None means the family's
- # measured deny list, a torchao that cannot run here, or a GPU without the kernels.
- # They read the same to the selector and need different fixes from the user.
+ # An explicit scheme is never swapped for another, so a None here means this GPU (or this family's
+ # measured deny list) rules it out.
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
if reason is not None:
raise RuntimeError(
@@ -1435,16 +1423,15 @@ class DiffusionBackend:
off_label = "Off to run the checkpoint as-is",
)
)
- # The mode the loader will ACTUALLY attempt, not the raw request: an explicit int8
- # on a family with no keep-bf16 schedule is rewritten to layerwise fp8 before
- # support is consulted, and that path needs no torchao. Refusing on the raw int8
- # rejected loads the runtime would run and report as fell_back -- Windows ROCm,
- # where the torchao stub kills int8 while fp8 still works.
+ # The mode the loader will ACTUALLY attempt, not the raw request: an explicit int8 on a family with no keep-bf16
+ # schedule is rewritten to layerwise fp8 before support is consulted, and that path needs no torchao. Refusing
+ # on the raw int8 rejected loads the runtime would run and report as fell_back -- Windows ROCm, where the
+ # torchao stub kills int8 while fp8 still works.
te_effective = effective_te_quant(te_mode, getattr(fam, "name", None))
te_reason = None
if te_quant_needs_resident_weights(te_effective) and not torchao_quantize_importable():
- # The casters import torchao only after the pipeline is downloaded and built, so a
- # broken or absent install failed through load-progress instead of this 409.
+ # The casters import torchao only after the pipeline is downloaded and built, so a broken or absent install
+ # failed through load-progress instead of this 409.
te_reason = (
"torchao is not importable on this install, and these encoder modes are torchao "
"quantisations"
@@ -1457,10 +1444,9 @@ class DiffusionBackend:
elif te_quant_needs_resident_weights(te_effective) and _memory_request_forces_offload(
memory_mode, cpu_offload
):
- # Same fence as the dense transformer above, on the encoder: offload hooks move
- # modules with Module.to(), torchao tensors do not survive it, and the loader reports
- # those modes unsupported once offload is active -- after the resident pipeline is
- # already gone. Layerwise fp8 is a dtype cast and is unaffected.
+ # Same fence as the dense transformer above, on the encoder: offload hooks move modules with Module.to(),
+ # torchao tensors do not survive it, and the loader reports those modes unsupported once offload is active
+ # -- after the resident pipeline is already gone. Layerwise fp8 is a dtype cast and is unaffected.
requested_memory = normalize_memory_mode(memory_mode) or "cpu_offload"
te_reason = (
f"'{requested_memory}' memory places the text encoder under CPU offload, and "
@@ -1502,9 +1488,9 @@ class DiffusionBackend:
"diffusion.dtype_promoted: family=%s float16 -> float32 (fp16-incompatible)",
getattr(fam, "name", None),
)
- # Builds only. Pinning is the CALLER's decision: a worker thread is dedicated and takes
- # the permanent pin, while a route preflight runs on a pooled executor thread where an
- # unrestored set_device would leak this request's card into the next one.
+ # Builds only. Pinning is the CALLER's decision: a worker thread is dedicated and takes the permanent pin, while
+ # a route preflight runs on a pooled executor thread where an unrestored set_device would leak this request's
+ # card into the next one.
return diffusion_device_target_from_torch_device(device, effective)
def _target_for_ordinal(
@@ -1525,10 +1511,9 @@ class DiffusionBackend:
"""
target = self._target_for_ordinal(state.family, state.gpu_ordinal)
apply_diffusion_device_ordinal(target)
- # And put an AUTOMATIC load back on its own card. generate() runs on a pooled
- # asyncio.to_thread worker, so a previous pinned model leaves that thread set to its GPU
- # for good; with no ordinal to pin, the bare "cuda" Generators below would then target that
- # card while these weights sit on the default one.
+ # And put an AUTOMATIC load back on its own card. generate() runs on a pooled asyncio.to_thread worker, so a
+ # previous pinned model leaves that thread set to its GPU for good; with no ordinal to pin, the bare "cuda"
+ # Generators below would then target that card while these weights sit on the default one.
pin_cuda_ordinal(state.placed_ordinal)
return target
@@ -1552,19 +1537,19 @@ class DiffusionBackend:
return str(resolve_local_gguf_child(local_root, gguf_filename))
from huggingface_hub import hf_hub_download, try_to_load_from_cache
- # Pin the LIVE root, as the prefetch does: unpinned, a mid-session cache change misses the
- # staged GGUF and re-pulls the whole multi-GB file inside the load lock, after eviction,
- # where unload cannot preempt it and progress already reported 100%.
+ # Pin the LIVE root, as the prefetch does: unpinned, a mid-session cache change misses the staged GGUF and
+ # re-pulls the whole multi-GB file inside the load lock, after eviction, where unload cannot preempt it and
+ # progress already reported 100%.
cache_dir = hub_cache_dir()
- # Best-effort: an unreadable cache raises here, and letting that escape would abort a load
- # that only had to download. Any failure falls through to the pinned download below.
+ # Best-effort: an unreadable cache raises here, and letting that escape would abort a load that only had to
+ # download. Any failure falls through to the pinned download below.
try:
if not isinstance(
try_to_load_from_cache(repo_id, gguf_filename, cache_dir = cache_dir), str
):
- # Same other-root reuse as the pre-quant checkpoint, reached THROUGH that root
- # rather than returned raw, so the ref still resolves and a republished GGUF is
- # picked up; the blob is reused, and offline the cached pointer comes back anyway.
+ # Same other-root reuse as the pre-quant checkpoint, reached THROUGH that root rather than returned raw,
+ # so the ref still resolves and a republished GGUF is picked up; the blob is reused, and offline the
+ # cached pointer comes back anyway.
elsewhere = try_to_load_from_cache(repo_id, gguf_filename, cache_dir = None)
if isinstance(elsewhere, str) and Path(elsewhere).is_file():
try:
@@ -1575,9 +1560,9 @@ class DiffusionBackend:
cache_dir = None,
local_files_only = local_files_only,
)
- except Exception: # noqa: BLE001 — revalidation is a bonus, never a new failure
+ except Exception: # noqa: BLE001 - revalidation is a bonus, never a new failure
return elsewhere
- except Exception: # noqa: BLE001 — an unreadable cache is not a verdict, just download
+ except Exception: # noqa: BLE001 - an unreadable cache is not a verdict, just download
pass
return hf_hub_download(
repo_id,
@@ -1607,7 +1592,6 @@ class DiffusionBackend:
and no pre-quantized checkpoint that would shortcut the dense build. Callers only ask for
a ``kind == "gguf"`` pick, so the auto-quant decline below applies as written."""
raw = kwargs.get("transformer_quant")
- # Unset defaults to the hardware ladder (mirrors load_pipeline's tri-state).
auto = raw is None or str(raw).strip().lower() in ("", "auto")
if auto:
mode = TQ_AUTO
@@ -1626,9 +1610,8 @@ class DiffusionBackend:
return False
if mm is None and kwargs.get("cpu_offload"):
return False
- # The card this load will land on, and SCOPED: the selectors below use argument-less
- # capability, smoke and memory probes, so building an indexed target is not enough to
- # move them off the default card.
+ # The card this load will land on, and SCOPED for the same reason as the dense-quant probe: the selectors
+ # below read the current device, so an indexed target alone leaves them on the default card.
with diffusion_device_scope(kwargs.get("gpu_ordinal")):
return self._dense_quant_prefetch_decision(
fam,
@@ -1653,11 +1636,18 @@ class DiffusionBackend:
) -> bool:
"""``_dense_quant_prefetch_needed``'s body, run with the selected card current."""
target = self._target_for_ordinal(fam, kwargs.get("gpu_ordinal"))
- # Same decline as load_pipeline: an uncached hosted pre-quant keeps the GGUF, so the
- # plan must not stage the base transformer/ shards either.
+ # Same decline as load_pipeline: an uncached hosted pre-quant keeps the GGUF, so the plan must not stage the
+ # base transformer/ shards either.
+ # The same rule, applied to the base repo's own dense shards. The fast path loads transformer/ INSTEAD of the
+ # GGUF, so on a pick the user made BY QUANT an uncached base means fetching a second, much larger denoiser and
+ # never opening the first: Qwen-Image-Edit's Q6_K is 16.9 GB and the base transformer is another 40.9 GB. Cached
+ # shards cost nothing, so anyone who already has the dense base keeps the fast path, and an EXPLICIT
+ # transformer_quant still opts in as before. This returns the same False a PREQUANT candidate returns below, so
+ # a cached pre-quant reaches the load with no transformer/ staged and NOT because a download was refused. That
+ # is why the load side re-asks the resolver rather than reading an empty stage as a decline: same verdict here,
+ # two different reasons there.
if (
auto
- # Active weights only: disagreeing with the load stages shards it never reads.
and not _has_active_lora(kwargs.get("loras"))
and _uncached_prequant_repo(
fam,
@@ -1669,17 +1659,6 @@ class DiffusionBackend:
is not None
):
return False
- # The same rule, applied to the base repo's own dense shards. The fast path loads
- # transformer/ INSTEAD of the GGUF, so on a pick the user made BY QUANT an uncached
- # base means fetching a second, much larger denoiser and never opening the first:
- # Qwen-Image-Edit's Q6_K is 16.9 GB and the base transformer is another 40.9 GB.
- # Cached shards cost nothing, so anyone who already has the dense base keeps the fast
- # path, and an EXPLICIT transformer_quant still opts in as before.
- #
- # This returns the same False a PREQUANT candidate returns below, so a cached
- # pre-quant reaches the load with no transformer/ staged and NOT because a download
- # was refused. That is why the load side re-asks the resolver rather than reading an
- # empty stage as a decline: same verdict here, two different reasons there.
if (
auto
and not _has_active_lora(kwargs.get("loras"))
@@ -1701,10 +1680,11 @@ class DiffusionBackend:
force_dense = _has_active_lora(kwargs.get("loras")),
logger = None,
)
- # A prequant loads a small checkpoint, so widening defeats the savings and can disk-full.
+ # A prequant loads a small checkpoint, so widening defeats the savings and can disk-full
if candidate is None or candidate.prequant:
return False
- # Capacity gate: mirror plan_fits_total_capacity against TOTAL capacity, else load_pipeline declines the dense path anyway.
+ # Capacity gate: mirror plan_fits_total_capacity against TOTAL capacity, else load_pipeline declines the dense
+ # path anyway
from .diffusion_memory import (
_reserve_mib,
snapshot_device_memory,
@@ -1755,10 +1735,9 @@ class DiffusionBackend:
if source is None:
continue
# Same cached-only rule _uncached_prequant_repo applies to the winner. That guard runs
- # select_transformer_quant_scheme, which only ever sees the winner, so a retry that
- # returned an UNCACHED repo would smuggle past it and download a second multi-GB
- # denoiser for a pick that already has its GGUF. A local override is the operator's own
- # file and costs no bytes.
+ # select_transformer_quant_scheme, which only ever sees the winner, so a retry that returned an UNCACHED
+ # repo would smuggle past it and download a second multi-GB denoiser for a pick that already has its GGUF. A
+ # local override is the operator's own file and costs no bytes.
if source.kind == "repo" and not prequant_checkpoint_cached(
source, cache_dir = hub_cache_dir()
):
@@ -1797,16 +1776,15 @@ class DiffusionBackend:
connection that nobody asked for."""
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
- # Only the BYTES move; the caller keeps the upstream id. ``fetch_base`` is the load-wide
- # decision when one was taken, so every later fetch agrees with what was staged here.
+ # Only the BYTES move; the caller keeps the upstream id. ``fetch_base`` is the load-wide decision when one was
+ # taken, so every later fetch agrees with what was staged here.
base = fetch_base or prefer_ungated_mirror(base, hf_token, files = base_files)
# Callers without a per-load event (tests, direct use) fall back to the current one.
cancel = cancel_event if cancel_event is not None else self._cancel_event
- # A file cached only under huggingface_hub's import-time root resolves through that root:
- # the preflight clears a base found under EITHER root, so without this a cache-folder change
- # turns an already-downloaded gated base into a mid-prefetch Hub token error, and every
- # other moved asset into a needless re-download.
- # GGUF transformer (hub repos only; a local path is already on disk).
+ # A file cached only under huggingface_hub's import-time root resolves through that root: the preflight clears a
+ # base found under EITHER root, so without this a cache-folder change turns an already-downloaded gated base
+ # into a mid-prefetch Hub token error, and every other moved asset into a needless re-download. GGUF transformer
+ # (hub repos only; a local path is already on disk).
if gguf_filename and not Path(repo_id).expanduser().exists():
hf_hub_download_with_xet_fallback(
repo_id,
@@ -1818,9 +1796,9 @@ class DiffusionBackend:
)
# Base repo (VAE / text-encoder / scheduler); list comes from the estimate.
snapshot_root: Optional[str] = None
- # reuse_other_cache_root resolves EACH file through whichever root holds it, so a moved
- # cache can serve the manifest from the old root while companions land in the live one, and
- # returning that snapshot would point from_pretrained at a tree missing the rest.
+ # reuse_other_cache_root resolves EACH file through whichever root holds it, so a moved cache can serve the
+ # manifest from the old root while companions land in the live one, and returning that snapshot would point
+ # from_pretrained at a tree missing the rest.
roots: set[str] = set()
for rfilename in base_files:
if cancel.is_set():
@@ -1833,8 +1811,6 @@ class DiffusionBackend:
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
- # The resolved path minus the file's own relative path, so a subfolder entry yields the
- # same root as a top-level one. Not resolve()d: that follows the symlink into blobs/.
try:
root = str(Path(local).parents[len(Path(rfilename).parts) - 1])
except (IndexError, ValueError, OSError):
@@ -1842,8 +1818,8 @@ class DiffusionBackend:
roots.add(root)
if rfilename == "model_index.json":
snapshot_root = root or None
- # Only hand back a snapshot the whole set lives in; otherwise the hub id resolves each file
- # through its own root as before.
+ # Only hand back a snapshot the whole set lives in; otherwise the hub id resolves each file through its own root
+ # as before.
if snapshot_root is not None and roots != {snapshot_root}:
return None
return snapshot_root
@@ -1866,7 +1842,7 @@ class DiffusionBackend:
kind = resolve_model_kind(gguf_filename, model_kind)
fam = detect_family_for_pick(repo_id, gguf_filename, family_override)
if fam is None:
- # An excluded model gets its stated reason, not the unknown-family message that invites a doomed retry.
+ # An excluded model gets its stated reason, not the unknown-family message that invites a doomed retry
excluded = excluded_model_reason(repo_id)
if excluded:
raise ValueError(f"'{repo_id}' cannot be loaded: {excluded}")
@@ -1876,20 +1852,20 @@ class DiffusionBackend:
f"pass family_override with that family name. (Video models and image models "
f"whose diffusers transformer has no single-file loader are not supported.)"
)
- # Refuse a too-old diffusers here, not deep in the load, but only when this load builds the diffusers pipeline: a
- # GGUF this host routes to native sd.cpp never instantiates the class. The picker gate reads the same predicate.
- # Imported here, not at module import, because the router imports this module's siblings.
+ # Refuse a too-old diffusers here, not deep in the load, but only when this load builds the diffusers pipeline:
+ # a GGUF this host routes to native sd.cpp never instantiates the class. The picker gate reads the same
+ # predicate. Imported here, not at module import, because the router imports this module's siblings.
from .diffusion_engine_router import family_buildable_here
if not family_buildable_here(fam, model_kind = kind):
assert_pipeline_class_available(fam.pipeline_class, fam.name)
- # Families whose single file IS the whole pipeline have no GGUF path; reject before eviction.
+ # Families whose single file IS the whole pipeline have no GGUF path; reject before eviction
if kind == "gguf" and fam.single_file_is_pipeline:
raise ValueError(
f"'{fam.name}' checkpoints are whole-pipeline single files and have no GGUF "
f"transformer variant; load the .safetensors pipeline instead of a GGUF."
)
- # A multi-denoiser family (Ideogram 4) has no transformer-only path; reject before eviction.
+ # A multi-denoiser family (Ideogram 4) has no transformer-only path; reject before eviction
if kind in ("gguf", "single_file") and fam.pipeline_only:
raise ValueError(
f"'{fam.name}' loads only as a full diffusers pipeline (it assembles "
@@ -1902,14 +1878,13 @@ class DiffusionBackend:
f"Non-GGUF diffusion loads are restricted to unsloth/* repos (or a local "
f"path); got '{repo_id}'. Pass a gguf_filename to load a GGUF instead."
)
- # The companion base repo also loads via from_pretrained, so it must clear the same trust bar.
+ # The companion base repo also loads via from_pretrained, so it must clear the same trust bar
if base_repo and base_repo.strip() and not _is_trusted_diffusion_repo(base_repo):
raise ValueError(
f"base_repo is restricted to unsloth/* repos (or a local path); got '{base_repo}'."
)
- # A local base_repo loads as a full pipeline; reject a non-pipeline one before eviction.
+ # A local base_repo loads as a full pipeline; reject a non-pipeline one before eviction
_assert_local_base_is_pipeline(base_repo)
- # Reject a bad LOCAL pick before the route evicts chat: a path-shaped repo_id must be on disk.
local_root = Path(repo_id).expanduser()
# Path-shaped: "."/".." prefix, a backslash (never in "org/name"), or an absolute path.
path_shaped = (
@@ -1918,13 +1893,13 @@ class DiffusionBackend:
if kind in ("gguf", "single_file"):
if not gguf_filename:
raise ValueError(f"a single-file checkpoint name is required for a '{kind}' load.")
- # Fail a kind/extension mismatch before the handoff: gguf needs .gguf, single_file must not.
+ # Fail a kind/extension mismatch before the handoff: gguf needs .gguf, single_file must not
is_gguf_name = gguf_filename.lower().endswith(".gguf")
if kind == "gguf" and not is_gguf_name:
raise ValueError("a 'gguf' load requires a .gguf checkpoint name.")
if kind == "single_file" and is_gguf_name:
raise ValueError("a .gguf checkpoint needs model_kind 'gguf', not 'single_file'.")
- # A single-file load must name a real .safetensors, else it evicts chat then fails in background.
+ # A single-file load must name a real .safetensors, else it evicts chat then fails in background
if kind == "single_file" and not gguf_filename.lower().endswith(".safetensors"):
raise ValueError(
f"'{gguf_filename}' is not a loadable single-file checkpoint "
@@ -1947,7 +1922,7 @@ class DiffusionBackend:
elif path_shaped:
raise FileNotFoundError(f"Local model path does not exist: {repo_id}")
elif repo_id.upper().endswith("-GGUF"):
- # A remote "*-GGUF" id is not a pipeline; reject here instead of evicting chat then failing.
+ # A remote "*-GGUF" id is not a pipeline; reject here instead of evicting chat then failing
raise ValueError(
f"'{repo_id}' is a single-file GGUF repo; load it with model_kind 'gguf' "
f"and a .gguf filename, not as a full pipeline."
@@ -1980,20 +1955,19 @@ class DiffusionBackend:
base = repo_id # the full pipeline IS the repo
else:
base = _resolve_base_repo(repo_id, base_repo, fam, hf_token)
- # Probe the repo the load will FETCH from: refusing the upstream id would reject the gated
- # picks the ungated mirror rescues, and the swap is pure, so it decides the same here as on
- # the load thread. Only the raise matters; _run_load recomputes the excused snapshot.
+ # Probe the repo the load will FETCH from: refusing the upstream id would reject the gated picks the ungated
+ # mirror rescues, and the swap is pure, so it decides the same here as on the load thread. Only the raise
+ # matters; _run_load recomputes the excused snapshot.
_assert_base_repo_accessible(prefer_ungated_mirror(base, hf_token), hf_token)
- # Same reasoning for the FLUX.2 size pairing, and this is the only place it can be caught
- # before a teardown: the loader's own guard opens the downloaded checkpoint, so it fires
- # after ~19 GB of base shards AND after the resident pipeline was freed. Metadata only (one
- # range request for the GGUF's tensor table), and fails open on anything it cannot read.
- # The UPSTREAM base, not the mirror: the size tables key on vendor ids.
+ # Same reasoning for the FLUX.2 size pairing, and this is the only place it can be caught before a teardown: the
+ # loader's own guard opens the downloaded checkpoint, so it fires after ~19 GB of base shards AND after the
+ # resident pipeline was freed. Metadata only (one range request for the GGUF's tensor table), and fails open on
+ # anything it cannot read. The UPSTREAM base, not the mirror: the size tables key on vendor ids.
assert_flux2_pick_compatible(fam, repo_id, gguf_filename, base, hf_token)
- # No media backend decodes a speech GGUF, and detect_family_for_pick answers from the
- # folder name, so a csm file beside a denoiser reaches this loader as one of its own.
- # Cache-only for a load nobody asked for: this probe would otherwise spend a revision HEAD,
- # or a range request and its bound, on the one path that promised to stay off the Hub.
+ # No media backend decodes a speech GGUF, and detect_family_for_pick answers from the folder name, so a csm file
+ # beside a denoiser reaches this loader as one of its own. Cache-only for a load nobody asked for: this probe
+ # would otherwise spend a revision HEAD, or a range request and its bound, on the one path that promised to stay
+ # off the Hub.
assert_pick_is_not_speech(repo_id, gguf_filename, hf_token, allow_network)
# ── Background load + progress ─────────────────────────────────────────
@@ -2024,32 +1998,29 @@ class DiffusionBackend:
gpu_ordinal: Optional[int] = None,
) -> dict[str, Any]:
"""Validate, then run the (slow) load on a daemon thread. Returns at once."""
- # A blank token must mean "anonymous", not an empty credential the Hub 401s.
hf_token = (hf_token.strip() if isinstance(hf_token, str) else hf_token) or None
- # Resolved ONCE, here, and carried to the worker: outside it so a bad pick is the route's
- # 400 rather than a load that dies mid-download, and only once so free VRAM cannot re-rank
- # the choice after the weights land. Gated on the resolved backend, since XPU / MPS / CPU
- # ignore physical ids and would otherwise 400 a selection the contract says to drop.
- # Re-ranked only when the caller did not already do it: free VRAM moves between the
- # route's preflight and here (network preflight, engine activation, arbiter eviction), so
- # resolving twice can approve a scheme against one card and place the weights on another.
+ # Resolved ONCE, here, and carried to the worker: outside it so a bad pick is the route's 400 rather than a load
+ # that dies mid-download, and only once so free VRAM cannot re-rank the choice after the weights land. Gated on
+ # the resolved backend, since XPU / MPS / CPU ignore physical ids and would otherwise 400 a selection the
+ # contract says to drop. Re-ranked only when the caller did not already do it: free VRAM moves between the
+ # route's preflight and here (network preflight, engine activation, arbiter eviction), so resolving twice can
+ # approve a scheme against one card and place the weights on another.
if gpu_ordinal is None:
gpu_ordinal = (
resolve_selected_cuda_ordinal(gpu_ids)
if gpu_ids and resolve_diffusion_device_target().device == "cuda"
else None
)
- # base_repo is gated at the route pre-eviction; this only cheap-fails the resolved repo/family.
fam = self.validate_load_request(
repo_id,
gguf_filename = gguf_filename,
family_override = family_override,
model_kind = model_kind,
)
- # Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the
- # route answers 409 with the reason instead of evicting the resident model, downloading
- # several GB and only then failing. The declines that need the real footprint (a VRAM
- # misfit, a failed build) can only be found mid-load and surface through load-progress.
+ # Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the route answers 409 with
+ # the reason instead of evicting the resident model, downloading several GB and only then failing. The declines
+ # that need the real footprint (a VRAM misfit, a failed build) can only be found mid-load and surface through
+ # load-progress.
self.assert_precision_available(
fam,
model_kind = resolve_model_kind(gguf_filename, model_kind),
@@ -2064,9 +2035,9 @@ class DiffusionBackend:
raise RuntimeError("A diffusion load is already in progress.")
self._load_token += 1
token = self._load_token
- # A NEW event per load, never a clear() of the shared one: unload() sets the event the running worker holds but also
- # drops _loading, so clearing here would un-cancel that worker. Download preemption is best-effort; the token is the
- # real commit guard.
+ # A NEW event per load, never a clear() of the shared one: unload() sets the event the running worker holds
+ # but also drops _loading, so clearing here would un-cancel that worker. Download preemption is best-effort;
+ # the token is the real commit guard.
cancel_event = threading.Event()
self._cancel_event = cancel_event
# Seed with the family fallback; the worker resolves the real base and updates this.
@@ -2105,12 +2076,11 @@ class DiffusionBackend:
token = kwargs.get("_load_token")
# This load's own event: a later load replaces self._cancel_event rather than clearing it.
cancel_event = kwargs.pop("_cancel_event", None) or self._cancel_event
- # An API-initiated load promises to download NOTHING: it may only open what is already
- # cached. Read once here and threaded into the staging phase below, the same way the video
- # path does it -- the flag reached load_pipeline and nothing before it, so the prefetch that
- # actually moves the bytes ran unrestricted and the promise was carried only by the locality
- # check the flag exists to stop relying on.
- # READ, not popped: load_pipeline takes it too (it is in this thread's kwargs by contract).
+ # An API-initiated load promises to download NOTHING: it may only open what is already cached. Read once here
+ # and threaded into the staging phase below, the same way the video path does it -- the flag reached
+ # load_pipeline and nothing before it, so the prefetch that moves the bytes ran unrestricted and the promise was
+ # carried only by the locality check the flag exists to stop relying on. READ, not popped: load_pipeline takes
+ # it too (it is in this thread's kwargs by contract).
local_files_only = bool(kwargs.get("local_files_only"))
try:
# Resolve the base repo and estimate sizes here (both network) so begin_load returns instantly.
@@ -2145,10 +2115,6 @@ class DiffusionBackend:
kind = kind,
pipeline_components = _explicit_pipeline_components(fam),
single_file_is_pipeline = bool(fam and fam.single_file_is_pipeline),
- # Pull the base shards here rather than inside the locked, unpreemptable finalize.
- # Deferred: the widening turns on the base repo's own listing, which this call is
- # what fetches. Called at most once, with the companions and the transformer shards
- # split apart.
include_transformer = (
(
lambda companions, transformer_files: self._dense_quant_prefetch_needed(
@@ -2166,31 +2132,30 @@ class DiffusionBackend:
resident_file_sizes_out = resident_sizes,
fetch_repos_out = fetch_repos,
)
- # Only shards this prefetch staged may be materialised by the dense fallback, so read it
- # off the staged list: a failed size estimate drops every base file too. A LOCAL base
- # directory has no listing to fail at -- model_info raises on a path -- and its shards
- # are already there, so it counts as staged on the filesystem instead.
+ # Only shards this prefetch staged may be materialised by the dense fallback, so read it off the staged
+ # list: a failed size estimate drops every base file too. A LOCAL base directory has no listing to fail at
+ # -- model_info raises on a path -- and its shards are already there, so it counts as staged on the
+ # filesystem instead.
kwargs["_transformer_prefetched"] = any(
f.startswith("transformer/") for f in base_files
) or _local_base_transformer_present(base)
- # ONE mirror decision per load, taken with the staged file list in hand and carried into
- # load_pipeline: per-call-site, one repo could be staged and the other assembled from.
+ # ONE mirror decision per load, taken with the staged file list in hand and carried into load_pipeline:
+ # per-call-site, one repo could be staged and the other assembled from.
fetch_base = fetch_repos.get(base) or prefer_ungated_mirror(
base, kwargs.get("hf_token"), files = base_files
)
kwargs["_fetch_base"] = fetch_base
- # Same preflight the plan runs: catch a gated base here, not 15 GiB into the prefetch.
- # Runs on ``fetch_base``, once it is decided, so it probes the repo the pull will read:
- # refusing the upstream id would reject the gated picks the ungated mirror rescues.
- # Everything above is metadata, so no byte has moved yet. Returns a snapshot dir when it
- # excused the base off a copy only the import-time root holds.
+ # Same preflight the plan runs: catch a gated base here, not 15 GiB into the prefetch. Runs on
+ # ``fetch_base``, once it is decided, so it probes the repo the pull will read: refusing the upstream id
+ # would reject the gated picks the ungated mirror rescues. Everything above is metadata, so no byte has
+ # moved yet. Returns a snapshot dir when it excused the base off a copy only the import-time root holds.
base_snapshot = _assert_base_repo_accessible(
fetch_base, kwargs.get("hf_token"), local_files_only = local_files_only
)
- # And the same size-pairing preflight the plan and the route run, here because a direct
- # begin_load (a saved config, the deploy path) reaches neither. Still metadata only, so
- # it lands before _prefetch_files stages the base and before load_pipeline unloads the
- # resident pipeline -- the two costs the loader's backstop cannot avoid.
+ # And the same size-pairing preflight the plan and the route run, here because a direct begin_load (a saved
+ # config, the deploy path) reaches neither. Still metadata only, so it lands before _prefetch_files stages
+ # the base and before load_pipeline unloads the resident pipeline -- the two costs the loader's backstop
+ # cannot avoid.
assert_flux2_pick_compatible(
fam, kwargs["repo_id"], kwargs.get("gguf_filename"), base, kwargs.get("hf_token")
)
@@ -2223,15 +2188,14 @@ class DiffusionBackend:
)
raise RuntimeError(shortfall)
with self._lock:
- # Stamp progress only if this load is still current (a superseder has its own token).
if self._load_token == token and self._loading is not None:
self._loading.base_repo = base
self._loading.fetch_repo = fetch_base
self._loading.expected_bytes = expected
- # Download outside the lock so unload/an eviction can preempt the pull.
- # The carried snapshot is the fallback, never the override: it fires only when the
- # estimate came back empty, since the metadata that fills it is the same call whose
- # failure earned the escape. Without it the load 401s with every byte already on disk.
+ # Download outside the lock so unload/an eviction can preempt the pull. The carried snapshot is the
+ # fallback, never the override: it fires only when the estimate came back empty, since the metadata that
+ # fills it is the same call whose failure earned the escape. Without it the load 401s with every byte
+ # already on disk.
kwargs["_base_local_dir"] = (
self._prefetch_files(
kwargs["repo_id"],
@@ -2250,19 +2214,18 @@ class DiffusionBackend:
# Only clear the marker if this load is still current (a superseder has its own token).
if self._load_token == token:
self._loading = None
- except Exception as exc: # noqa: BLE001 — surfaced to the client via load_progress
+ except Exception as exc: # noqa: BLE001 - surfaced to the client via load_progress
# A cancelled/superseded load raised below; don't log/stamp it onto the current load.
if self._load_token != token:
return
logger.error("diffusion.load_failed: %s", exc)
- # Free the debris of a failed construction; guarded so a sticky CUDA error still stamps the error.
try:
clear_gpu_cache()
except Exception: # noqa: BLE001
pass
- # Rewrite a gated-repo 403 into the step that unblocks the user, then redact native paths:
- # this text is surfaced verbatim and Unsloth can be shared. Guarded because on this daemon
- # thread anything escaping leaves _loading.error unset and load_progress() stuck forever.
+ # Rewrite a gated-repo 403 into the step that unblocks the user, then redact native paths: this text is
+ # surfaced verbatim and Unsloth can be shared. Guarded because on this daemon thread anything escaping
+ # leaves _loading.error unset and load_progress() stuck forever.
from utils.native_path_leases import redact_native_paths
try:
@@ -2283,13 +2246,13 @@ class DiffusionBackend:
if loading is None:
return _progress("ready" if self._state is not None else None)
- # Sum checkpoint + companion cache, scanning the repo the bytes LAND in (the mirror when one
- # was swapped in), else a mirrored companion download reads as zero and the bar sits still.
+ # Sum checkpoint + companion cache, scanning the repo the bytes LAND in (the mirror when one was swapped in),
+ # else a mirrored companion download reads as zero and the bar sits still.
companion = loading.fetch_repo or loading.base_repo
if loading.base_repo and loading.base_repo == loading.repo_id:
- # Full pipeline: base IS the repo, so count it once. Summing would add the upstream's
- # stale partial blobs -- the very thing that selects the mirror -- to the mirror's live
- # bytes, pushing the bar to 100% / finalizing mid-download.
+ # Full pipeline: base IS the repo, so count it once. Summing would add the upstream's stale partial blobs --
+ # the very thing that selects the mirror -- to the mirror's live bytes, pushing the bar to 100% / finalizing
+ # mid-download.
downloaded = self._cache_bytes(companion or loading.repo_id)
else:
downloaded = self._cache_bytes(loading.repo_id)
@@ -2393,11 +2356,10 @@ class DiffusionBackend:
return None
try:
raw = kwargs.get("transformer_quant")
- # Same tri-state as load_pipeline: unset or "auto" means the hardware ladder decides.
auto = raw is None or str(raw).strip().lower() in ("", "auto")
- # An AUTO quant under an explicit Speed="off" is forced to "off" by load_pipeline, which
- # normalizes to None and skips the fast path entirely, so no prequant is ever fetched.
- # An EXPLICIT quant still takes it: that force applies only to the auto tri-state.
+ # An AUTO quant under an explicit Speed="off" is forced to "off" by load_pipeline, which normalizes to None
+ # and skips the fast path entirely, so no prequant is ever fetched. An EXPLICIT quant still takes it: that
+ # force applies only to the auto tri-state.
speed = kwargs.get("speed_mode")
if auto and speed is not None and str(speed).strip().lower() == SPEED_OFF:
return None
@@ -2407,22 +2369,19 @@ class DiffusionBackend:
# A LoRA bake forces the DENSE path, so no prequant is fetched.
if _has_active_lora(kwargs.get("loras")):
return None
- # A definite-offload policy keeps the GGUF: the fast path needs either a plan with no
- # offload or a quant-sized replan that came back with none, and balanced/low_vram
- # offload BY MODE, so no replan can clear it. Same gate _dense_quant_prefetch_needed
- # applies, for the same reason.
+ # A definite-offload policy keeps the GGUF: the fast path needs either a plan with no offload or a
+ # quant-sized replan that came back with none, and balanced/low_vram offload BY MODE, so no replan can clear
+ # it. Same gate _dense_quant_prefetch_needed applies, for the same reason.
mm = normalize_memory_mode(kwargs.get("memory_mode"))
if mm in (MEMORY_MODE_BALANCED, MEMORY_MODE_LOW_VRAM):
return None
if mm is None and kwargs.get("cpu_offload"):
return None
- # The card this load will land on, and SCOPED for the same reason as the dense-quant
- # probe: the selectors below read the current device, so an indexed target alone
- # leaves them on the default card.
+ # The card this load will land on, and SCOPED for the same reason as the dense-quant probe
with diffusion_device_scope(kwargs.get("gpu_ordinal")):
target = self._target_for_ordinal(fam, kwargs.get("gpu_ordinal"))
- # An auto quant DECLINES an uncached hosted checkpoint and runs the GGUF as-is, so
- # those bytes never land. Only a cached one, or an explicit request, counts.
+ # An auto quant DECLINES an uncached hosted checkpoint and runs the GGUF as-is, so those bytes never
+ # land. Only a cached one, or an explicit request, counts.
if (
auto
and _uncached_prequant_repo(
@@ -2464,7 +2423,6 @@ class DiffusionBackend:
path_override = kwargs.get("transformer_prequant_path"),
base_repo = kwargs.get("base_repo"),
)
- # A local override is the operator's own file: already on disk, never downloaded.
if source is None or source.kind != "repo":
return None
from huggingface_hub import HfApi
@@ -2545,8 +2503,8 @@ class DiffusionBackend:
return 0, []
from huggingface_hub import HfApi
- # No swap on purpose: the Hub gates only the BYTE endpoint, so model_info answers
- # anonymously and the mirror lists the same names. _prefetch_files swaps when it pulls.
+ # No swap on purpose: the Hub gates only the BYTE endpoint, so model_info answers anonymously and the mirror
+ # lists the same names. _prefetch_files swaps when it pulls.
from .diffusion_te_prequant import is_prequant_covered_weight
api = HfApi()
@@ -2618,7 +2576,6 @@ class DiffusionBackend:
)
_record_revision(revisions_out, metadata_repo, info)
return total, base_files
- # Skip the Hub size lookup for a LOCAL gguf path: model_info raises on a filesystem path.
if gguf_filename and not Path(repo_id).expanduser().exists():
info = api.model_info(repo_id, files_metadata = True, token = hf_token)
gguf_bytes = sum(s.size or 0 for s in info.siblings if s.rfilename == gguf_filename)
@@ -2668,18 +2625,18 @@ class DiffusionBackend:
base_bytes += s.size or 0
base_sizes[s.rfilename] = int(s.size or 0)
total += base_bytes
- # ACCUMULATE, never assign: on a combined repo both branches key the same repo id, and
- # a plain assignment drops the checkpoint's size and file map.
+ # ACCUMULATE, never assign: on a combined repo both branches key the same repo id, and a plain assignment
+ # drops the checkpoint's size and file map.
if sizes_out is not None:
sizes_out[base_repo] = sizes_out.get(base_repo, 0) + base_bytes
if file_sizes_out is not None:
file_sizes_out.setdefault(base_repo, {}).update(base_sizes)
_record_revision(revisions_out, base_repo, base_info)
- except Exception as exc: # noqa: BLE001 — estimate is best-effort
+ except Exception as exc: # noqa: BLE001 - estimate is best-effort
logger.warning("diffusion.size_estimate_failed: %s", exc)
- # Recorded so a caller that must not guess (media auto-switch refuses rather than
- # download) can tell a partial file list from a complete one; the estimate itself
- # stays best-effort for the UI, whose fallback is the inline pull.
+ # Recorded so a caller that must not guess (media auto-switch refuses rather than download) can tell a
+ # partial file list from a complete one; the estimate itself stays best-effort for the UI, whose fallback is
+ # the inline pull.
if failures_out is not None:
failures_out.append(exc)
return total, base_files
@@ -2723,23 +2680,23 @@ class DiffusionBackend:
if kind == "pipeline":
base = repo_id # the full pipeline IS the repo
elif fam is None and not (base_repo or "").strip():
- # An unrecognised GGUF (a neutral repo whose id and filename match no family) has no
- # companion set to resolve, and the family fallback would raise. The pick still loads:
- # an empty plan just means nothing to pre-stage. The picker asks for a plan on EVERY
- # hub pick, so raising here would 500 the route rather than plan no work.
+ # An unrecognised GGUF (a neutral repo whose id and filename match no family) has no companion set to
+ # resolve, and the family fallback would raise. The pick still loads: an empty plan just means nothing to
+ # pre-stage. The picker asks for a plan on EVERY hub pick, so raising here would 500 the route rather than
+ # plan no work.
return {"entries": [], "total_bytes": 0, "required_bytes": 0, "checkpoint_bytes": 0}
else:
base = _resolve_base_repo(repo_id, base_repo, fam, hf_token)
- # Reported, not raised. The images page falls back to /images/load on ANY plan failure, so
- # a 400 here would start the very download this is meant to prevent; carried in the
- # envelope instead, the picker can refuse at SELECTION time. Metadata only (one range
- # request for the GGUF's tensor table), and None whenever nothing is known to be wrong.
- # The speech verdict belongs here too, not only on the load preflight: the Images page
- # stages and downloads before it calls load, so a later refusal arrives after the bytes.
+ # Reported, not raised. carried in the envelope instead, the picker can refuse at SELECTION time.
+ # Reported, not raised. The images page falls back to /images/load on ANY plan failure, so a 400 here would
+ # start the very download this is meant to prevent; carried in the envelope instead, the picker can refuse at
+ # SELECTION time. Metadata only (one range request for the GGUF's tensor table), and None whenever nothing is
+ # known to be wrong. The speech verdict belongs here too, not only on the load preflight: the Images page stages
+ # and downloads before it calls load, so a later refusal arrives after the bytes.
incompatible = flux2_pick_mismatch(
fam, repo_id, gguf_filename, base, hf_token
) or speech_pick_refusal(repo_id, gguf_filename, hf_token)
- # Only a checkpoint that really resolves on the Hub earns the right to drop dense shards.
+ # Only a checkpoint that really resolves on the Hub earns the right to drop dense shards
te_files = (
self._te_prequant_plan_files(
fam,
@@ -2765,10 +2722,9 @@ class DiffusionBackend:
kind = kind,
pipeline_components = _explicit_pipeline_components(fam),
single_file_is_pipeline = bool(fam and fam.single_file_is_pipeline),
- # The RESOLVED base, as the load passes it: a variant base picks its own pre-quant repo.
- # Deferred exactly as _run_load defers it. Called eagerly this runs before the base
- # listing exists, so the cache gate sees no transformer shards, always declines, and
- # the plan scopes narrower than the load: the registry then reports
+ # The RESOLVED base, as the load passes it: a variant base picks its own pre-quant repo. Deferred exactly as
+ # _run_load defers it. Called eagerly this runs before the base listing exists, so the cache gate sees no
+ # transformer shards, always declines, and the plan scopes narrower than the load: the registry then reports
# scope_file_mismatch and the plan cannot adopt the load's own in-flight job.
include_transformer = (
(
@@ -2790,10 +2746,10 @@ class DiffusionBackend:
skip_te_components = tuple(te_files),
failures_out = plan_failures,
)
- # Decided once, from the staged file list, and both probed and reported: a gated base
- # answers model_info anonymously, so the plan would otherwise be confident and the 401 land
- # mid-download. Probing any id but the one staged would refuse a load that works. The route
- # maps ValueError -> 400. Nothing above downloads, so the reorder costs nothing.
+ # Decided once, from the staged file list, and both probed and reported: a gated base answers model_info
+ # anonymously, so the plan would otherwise be confident and the 401 land mid-download. Probing any id but the
+ # one staged would refuse a load that works. The route maps ValueError -> 400. Nothing above downloads, so the
+ # reorder costs nothing.
fetch_base = fetch_repos.get(base) or prefer_ungated_mirror(
base, hf_token, files = base_files
)
@@ -2806,9 +2762,9 @@ class DiffusionBackend:
)
te_prequant_bytes = _prequant_plan_bytes(te_files)
required_total += te_prequant_bytes
- # The dense transformer/ shards are excluded for a GGUF pick, so a hosted prequant that
- # replaces them is real footprint the plan would otherwise never report. Sized against the
- # RESOLVED base, as the load passes it: a variant base picks its own prequant repo.
+ # The dense transformer/ shards are excluded for a GGUF pick, so a hosted prequant that replaces them is real
+ # footprint the plan would otherwise never report. Sized against the RESOLVED base, as the load passes it: a
+ # variant base picks its own prequant repo.
dit_prequant = (
self._dit_prequant_plan_source(fam, kind, hf_token, {**load_kwargs, "base_repo": base})
if allow_device_probe
@@ -2852,11 +2808,10 @@ class DiffusionBackend:
revision = revisions.get(repo)
live = hub_cache_dir()
- # Preserve main's stable-scope invariant: a running @diffusion job is adopted only
- # when a second claim names the same complete file set. This strict, pinned fast path
- # drops a repo only when one cache root already serves the whole snapshot. The
- # per-file probes below remain necessary for size-corroborated unrelated commits and
- # for accurate remaining-byte accounting.
+ # Preserve main's stable-scope invariant: a running @diffusion job is adopted only when a second claim names
+ # the same complete file set. This strict, pinned fast path drops a repo only when one cache root already
+ # serves the whole snapshot. The per-file probes below remain necessary for size-corroborated unrelated
+ # commits and for accurate remaining-byte accounting.
if self._files_already_cached(repo, files, revision, declared_sizes).issuperset(files):
for entry in entries:
if entry["repo_id"] == repo:
@@ -2871,24 +2826,23 @@ class DiffusionBackend:
return "live"
if not self._hub_file_is_cached(repo, name, revision, size, roots = (None,)):
return None
- # The other root holds a good copy, but reuse_other_cache_root only switches roots
- # when the live lookup finds NOTHING. A stale live copy (right name, wrong bytes)
- # therefore shadows the good one and gets served, or re-fetched inline. Passing no
- # size asks presence alone, which is exactly what that switch tests.
+ # The other root holds a good copy, but reuse_other_cache_root only switches roots when the live lookup
+ # finds NOTHING. A stale live copy (right name, wrong bytes) therefore shadows the good one and gets
+ # served, or re-fetched inline. Passing no size asks presence alone, which is exactly what that switch
+ # tests.
if self._hub_file_is_cached(repo, name, revision, None, roots = (live,)):
return None
return "other"
where = {name: _where(name) for name in files}
- # A repo whose files STRADDLE the two roots cannot be handed to from_pretrained as one
- # snapshot: _prefetch_files returns no _base_local_dir in that state and the assembly is
- # pinned to hub_cache_dir(), so the old-root subset is invisible to it. Stage that subset
- # instead, or the load re-pulls it inline past the plan's own progress, cancel and disk
- # preflight, and fails offline after a plan that reported nothing left to do. A repo
- # living entirely in the other root is fine, which is what reuse_other_cache_root is for.
- # A file that is missing everywhere will be downloaded INTO the live root, so it counts
- # as live here: dropping it would read a genuinely-mixed repo as unsplit and stage only
- # the missing part, manufacturing the very split this branch exists to avoid.
+ # A repo whose files STRADDLE the two roots cannot be handed to from_pretrained as one snapshot:
+ # _prefetch_files returns no _base_local_dir in that state and the assembly is pinned to hub_cache_dir(), so
+ # the old-root subset is invisible to it. Stage that subset instead, or the load re-pulls it inline past the
+ # plan's own progress, cancel and disk preflight, and fails offline after a plan that reported nothing left
+ # to do. A repo living entirely in the other root is fine, which is what reuse_other_cache_root is for. A
+ # file that is missing everywhere will be downloaded INTO the live root, so it counts as live here: dropping
+ # it would read a genuinely-mixed repo as unsplit and stage only the missing part, manufacturing the very
+ # split this branch exists to avoid.
split = {w or "live" for w in where.values()} == {"live", "other"}
missing = [n for n, w in where.items() if w is None or (split and w != "live")]
if not missing:
@@ -2909,9 +2863,9 @@ class DiffusionBackend:
entries.append(
{
"repo_id": repo,
- # Stable across a warming cache so a second pick can adopt the same live
- # scoped job. Cached names are cheap no-op hf_hub_download calls; only the
- # genuinely missing subset contributes to bytes/preflight below.
+ # Stable across a warming cache so a second pick can adopt the same live scoped job. Cached names
+ # are cheap no-op hf_hub_download calls; only the genuinely missing subset contributes to
+ # bytes/preflight below.
"files": list(scope),
"bytes": int(sum(declared_sizes.get(name, 0) for name in missing)),
"gguf_filename": scoped_gguf[repo],
@@ -2934,9 +2888,9 @@ class DiffusionBackend:
checkpoint = True,
)
if base_files and not Path(base).expanduser().exists():
- # STAGED before the loader runs, so it must name the MIRROR: a gated upstream here 401s
- # an anonymous user at staging and the swap downstream is never reached. status(), the
- # API base repo, saved configs and LoRA tags keep the vendor id; sizes key on it too.
+ # STAGED before the loader runs, so it must name the MIRROR: a gated upstream here 401s an anonymous user at
+ # staging and the swap downstream is never reached. status(), the API base repo, saved configs and LoRA tags
+ # keep the vendor id; sizes key on it too.
if fetch_base != base and fetch_base not in revisions:
mirror_revision = self._current_sha(fetch_base, hf_token)
if mirror_revision:
@@ -2945,17 +2899,16 @@ class DiffusionBackend:
fetch_base,
base_files,
file_sizes.get(base, {}),
- # A pipeline pick has no single file: the base IS the selected model (base = repo_id
- # above). Flagged here rather than by comparing repo ids downstream, because the swap
- # just above leaves this entry named after the MIRROR. Under a GGUF or single-file
- # pick the base is a companion, so it stays unflagged.
+ # A pipeline pick has no single file: the base IS the selected model (base = repo_id above). Flagged
+ # here rather than by comparing repo ids downstream, because the swap just above leaves this entry named
+ # after the MIRROR. Under a GGUF or single-file pick the base is a companion, so it stays unflagged.
checkpoint = kind == "pipeline",
)
if dit_prequant is not None:
- # Staged, not just counted: _load_dense_quant_pipeline fetches this checkpoint during the
- # load, under the load lock and after the previous pipeline was evicted, so leaving it
- # out means a multi-GB inline pull with no progress, no cancel and no disk preflight.
- # A companion of the checkpoint, never the selected model itself.
+ # Staged, not just counted: _load_dense_quant_pipeline fetches this checkpoint during the load, under the
+ # load lock and after the previous pipeline was evicted, so leaving it out means a multi-GB inline pull with
+ # no progress, no cancel and no disk preflight. A companion of the checkpoint, never the selected model
+ # itself.
prequant_repo, prequant_file, prequant_size = dit_prequant
add_missing_entry(prequant_repo, [prequant_file], {prequant_file: prequant_size})
if incompatible is None and allow_device_probe and memory_verdict:
@@ -2980,9 +2933,9 @@ class DiffusionBackend:
"required_bytes": int(required_total),
"checkpoint_bytes": checkpoint_bytes,
"incompatible_reason": incompatible,
- # Companion discovery failed, so the file list above is partial. The UI stages what
- # it can and the loader pulls the rest inline; a caller that must not download at
- # all has to treat this plan as unverified rather than as nothing missing.
+ # Companion discovery failed, so the file list above is partial. The UI stages what it can and the loader
+ # pulls the rest inline; a caller that must not download at all has to treat this plan as unverified rather
+ # than as nothing missing.
"plan_failed": bool(plan_failures),
}
@@ -3125,15 +3078,15 @@ class DiffusionBackend:
if revision is None:
return any(sound(value) for _root, value in unpinned)
- # Presence, not soundness, is the branch condition: a damaged current snapshot must
- # not fall back to and bless an older same-size main snapshot.
+ # Presence, not soundness, is the branch condition: a damaged current snapshot must not fall back to and
+ # bless an older same-size main snapshot.
pinned = [(root, value) for root in search if (value := hit(root, revision))]
if not pinned:
return any(sound(value) for _root, value in unpinned)
def snapshot_path(value: str) -> Path:
- # Do not resolve symlinks: their blob targets can be shared across snapshots, while
- # the loader's unpinned choice is encoded by the snapshots/<sha>/... path itself.
+ # Do not resolve symlinks: their blob targets can be shared across snapshots, while the loader's
+ # unpinned choice is encoded by the snapshots/<sha>/... path itself.
return Path(value).absolute()
return any(
@@ -3166,20 +3119,20 @@ class DiffusionBackend:
cache-folder change the bytes the load reads are not in the live root at all and sizing that
root alone reads zero. The constant is read HERE, never bound at import: it is what
``cache_dir = None`` resolves to, and tests move it."""
- # One read of the live root: it is a setting, and load_progress polls this from another
- # thread, so a second read could compare the new root against a `live` built from the old.
+ # One read of the live root: it is a setting, and load_progress polls this from another thread, so a second read
+ # could compare the new root against a `live` built from the old.
live_root = hub_cache_dir()
folder = f"models--{repo_id.replace('/', '--')}"
live = Path(live_root) / folder
try:
from huggingface_hub import constants
other = str(constants.HF_HUB_CACHE or "").strip()
- except Exception: # noqa: BLE001 — no hub package: the live root still stands
+ except Exception: # noqa: BLE001 - no hub package: the live root still stands
return [live]
if not other:
return [live]
- # normcase/normpath before comparing: on Windows the two roots can differ by case alone and
- # every caller below would walk the same tree twice. Best-effort; both readers are idempotent.
+ # normcase/normpath before comparing: on Windows the two roots can differ by case alone and every caller below
+ # would walk the same tree twice. Best-effort; both readers are idempotent.
if os.path.normcase(os.path.normpath(other)) == os.path.normcase(
os.path.normpath(live_root)
):
@@ -3235,7 +3188,6 @@ class DiffusionBackend:
except OSError:
blobs = [] # repo not in this root yet / unreadable / no-symlink cache mode
if snapshot is None:
- # Unscopeable root: read every blob, as before.
for entry in blobs:
_add(entry.name, entry)
continue
@@ -3246,11 +3198,10 @@ class DiffusionBackend:
for f in snapshot.rglob("*"):
if not f.is_file():
continue
- # Dedupe on the path INSIDE the snapshot, so one logical file counts once
- # however many roots hold it. The etag would not: each root serves its own
- # revision, so a republished file summed both copies and could report
- # finalizing at half the real progress. Works for a no-symlink cache too, which
- # stores the file in the snapshot rather than linking to blobs/.
+ # Dedupe on the path INSIDE the snapshot, so one logical file counts once however many roots hold
+ # it. The etag would not: each root serves its own revision, so a republished file summed both
+ # copies and could report finalizing at half the real progress. Works for a no-symlink cache too,
+ # which stores the file in the snapshot rather than linking to blobs/.
_add(f"path:{f.relative_to(snapshot).as_posix()}", f)
except OSError:
pass # the download is writing this tree under the walk; report what we counted
@@ -3314,10 +3265,9 @@ class DiffusionBackend:
return sum(fn(local).values())
candidates: list[Path] = [Path(staged_dir).expanduser()] if staged_dir else []
for repo_dir in DiffusionBackend._hub_cache_repo_dirs(base):
- # One revision per root, the one it serves: the merge is per FILE, so a superseded
- # revision whose shards were repacked would count both namings and force offload on a
- # base that fits. A root naming no revision falls back to all of them, where
- # over-counting still beats reading zero.
+ # One revision per root, the one it serves: the merge is per FILE, so a superseded revision whose shards
+ # were repacked would count both namings and force offload on a base that fits. A root naming no revision
+ # falls back to all of them, where over-counting still beats reading zero.
live_rev = DiffusionBackend._live_snapshot_dir(repo_dir)
if live_rev is not None:
candidates.append(live_rev)
@@ -3395,7 +3345,7 @@ class DiffusionBackend:
numel *= dim
total += numel
return total
- except Exception: # noqa: BLE001 — corrupt/crafted shard degrades to 0, never crashes the load
+ except Exception: # noqa: BLE001 - corrupt/crafted shard degrades to 0, never crashes the load
return 0
@staticmethod
@@ -3418,7 +3368,6 @@ class DiffusionBackend:
for s in tdir.glob("*.safetensors")
}
- # bf16: 2 bytes/param
return DiffusionBackend._union_over_cached_revs(base, _params, staged_dir) * 2
# ── Synchronous load / generate / unload ───────────────────────────────
@@ -3443,26 +3392,22 @@ class DiffusionBackend:
transformer_cache: Optional[str] = None,
transformer_cache_threshold: Optional[float] = None,
model_kind: Optional[str] = None,
- # LoRA adapters to BAKE into a torchao int8/fp8 build. Ignored elsewhere (bf16/bnb take them at generate time).
loras: Optional[list[tuple[str, float]]] = None,
- # The torch ordinal begin_load resolved for this load, carried rather than re-derived.
gpu_ordinal: Optional[int] = None,
_load_token: Optional[int] = None,
_base_local_dir: Optional[str] = None,
- # True when the prefetch staged the base repo's ``transformer/`` shards, the only condition under which the dense-quant
- # fallback may materialise them. Defaults True for a direct call, which has no prefetch phase.
+ # True when the prefetch staged the base repo's ``transformer/`` shards, the only condition under which the
+ # dense-quant fallback may materialise them. Defaults True for a direct call, which has no prefetch phase.
_transformer_prefetched: bool = True,
- # The repo the background load staged the companions from; re-derived below for a direct
- # call, which has no prefetch phase.
+ # The repo the background load staged the companions from; re-derived below for a direct call, which has no
+ # prefetch phase.
_fetch_base: Optional[str] = None,
) -> dict[str, Any]:
# A blank token must degrade to anonymous, not be passed as a credential. Normalize once.
hf_token = hf_token.strip() if isinstance(hf_token, str) else hf_token
hf_token = hf_token or None
- # Validate first (no torch/diffusers) so a bad family fails even in a no-diffusers runtime.
hf_token = (hf_token.strip() if isinstance(hf_token, str) else hf_token) or None
- # base_repo is gated at the route pre-eviction; this only cheap-fails the resolved repo/family.
fam = self.validate_load_request(
repo_id,
gguf_filename = gguf_filename,
@@ -3480,22 +3425,21 @@ class DiffusionBackend:
base = (
repo_id if kind == "pipeline" else _resolve_base_repo(repo_id, base_repo, fam, hf_token)
)
- # ``base`` stays the UPSTREAM id every report and table lookup keys on; ``fetch_base`` is
- # the only id handed to something that downloads. One decision per load (the background
- # path took it before staging), so nothing stages one repo and assembles from the other.
+ # ``base`` stays the UPSTREAM id every report and table lookup keys on; ``fetch_base`` is the only id handed to
+ # something that downloads. One decision per load (the background path took it before staging), so nothing
+ # stages one repo and assembles from the other.
fetch_base = _fetch_base or prefer_ungated_mirror(base, hf_token)
target = self._target_for_ordinal(fam, gpu_ordinal)
- # A dedicated worker thread, so the pin is permanent and everything downstream -- the
- # placement, the offload budget, the un-indexed state.device -- lands on the same card.
+ # A dedicated worker thread, so the pin is permanent and everything downstream -- the placement, the offload
+ # budget, the un-indexed state.device -- lands on the same card.
apply_diffusion_device_ordinal(target)
device, dtype = target.device, target.dtype
import diffusers
- # diffusers hard-codes _tqdm_active = True at import and honours no env var, so
- # setup_logging (which runs long before this lazy import) cannot reach it. Without
- # this, "Loading pipeline components..." is drawn straight onto the log stream and
- # can land mid-record on a structlog JSON line. Idempotent and cheap.
+ # diffusers hard-codes _tqdm_active = True at import and honours no env var, so setup_logging (which runs long
+ # before this lazy import) cannot reach it. Without this, "Loading pipeline components..." is drawn straight
+ # onto the log stream and can land mid-record on a structlog JSON line. Idempotent and cheap.
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
@@ -3507,30 +3451,24 @@ class DiffusionBackend:
)
if preinstall_backend is not None:
_ensure_attention_backend_installed(preinstall_backend, logger)
- except Exception: # noqa: BLE001 — the locked path re-resolves and validates
+ except Exception: # noqa: BLE001 - the locked path re-resolves and validates
pass
- # Abort an in-flight denoise and wait for it to exit: a load claims VRAM, so it must not overlap.
with self._lock:
- # Bail before signalling if this load was superseded, else a stale worker aborts a live one.
if _load_token is not None and _load_token != self._load_token:
raise RuntimeError("Diffusion load was cancelled.")
with self._generation_cancel_lock:
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
- # Same fence unload() takes: a queued generation must not run on the pipeline this load is about to free.
self._reserve_teardown_locked()
with self._model_transition_slot():
with self._lock:
try:
- # Re-check: a newer load/unload may have superseded this one while we waited.
if _load_token is not None and _load_token != self._load_token:
raise RuntimeError("Diffusion load was cancelled.")
- # Free the old pipeline before allocating the new one (never two in VRAM).
self._unload_locked()
finally:
- # Released here, not at the end of the load: the old pipe is gone and the rest of the load holds _generate_lock.
self._release_teardown_locked()
# Single-file kinds resolve a checkpoint path; the pipeline kind has none.
@@ -3541,9 +3479,9 @@ class DiffusionBackend:
if kind in ("gguf", "single_file")
else None
)
- # A renamed or hand-picked FLUX.2 GGUF can still land on a different-size base, and
- # no name-based rule catches that. Say so here, naming the file and the repo,
- # rather than letting the GGUF quantizer raise a bare shape mismatch.
+ # A renamed or hand-picked FLUX.2 GGUF can still land on a different-size base, and no name-based rule
+ # catches that. Say so here, naming the file and the repo, rather than letting the GGUF quantizer raise
+ # a bare shape mismatch.
assert_flux2_gguf_matches_base(fam, base, single_file_path)
transformer_cls = getattr(diffusers, fam.transformer_class)
pipeline_cls = getattr(diffusers, fam.pipeline_class)
@@ -3558,20 +3496,18 @@ class DiffusionBackend:
cpu_offload,
kind = kind,
repo_id = repo_id,
- # The base may be resolved off the OTHER cache root, which the plan's live-root
- # scans read as zero companions.
+ # The base may be resolved off the OTHER cache root, which the plan's live-root scans read as zero
+ # companions.
base_local_dir = _base_local_dir,
fetch_base = fetch_base,
)
- # On unified memory the plan above is final -- the quant re-plans below are
- # CUDA-only -- and its 'none' policy is a placement, not a fit. Refuse here,
- # after the eviction above freed the previous pipeline (so the free reading is
- # the memory this load actually gets) and before any weight is materialised.
- # A pipeline's weight term is cached SHARD bytes, which is a download size, not a
- # resident one: Z-Image and Lumina ship fp32 shards that halve when loaded as bf16
- # (the size table records 24.6 GB of Z-Image shards as 12.3 GB resident). Refusing
- # on the download figure would turn away a load that fits, so judge the refusal
- # against the table's resident total whenever it knows this base.
+ # On unified memory the plan above is final -- the quant re-plans below are CUDA-only -- and its 'none'
+ # policy is a placement, not a fit. Refuse here, after the eviction above freed the previous pipeline
+ # (so the free reading is the memory this load gets) and before any weight is materialised. A pipeline's
+ # weight term is cached SHARD bytes, which is a download size, not a resident one: Z-Image and Lumina
+ # ship fp32 shards that halve when loaded as bf16 (the size table records 24.6 GB of Z-Image shards as
+ # 12.3 GB resident). Refusing on the download figure would turn away a load that fits, so judge the
+ # refusal against the table's resident total whenever it knows this base.
raise_on_unified_memory_shortfall(
self._resident_sized_plan(
plan, fam, base, target, kind, text_encoder_quant = text_encoder_quant
@@ -3580,10 +3516,11 @@ class DiffusionBackend:
logger = logger,
)
- # The caller's RAW request, captured before the tri-state below rewrites it: status
- # has to be able to say "you asked for fp8" long after the loader declined it.
+ # The caller's RAW request, captured before the tri-state below rewrites it: status has to be able to
+ # say "you asked for fp8" long after the loader declined it.
transformer_quant_requested = transformer_quant
- # Dtype tri-state: unset/"auto" -> hardware ladder; "none"/"off" pins GGUF-as-is; an explicit scheme pins it.
+ # Dtype tri-state: unset/"auto" -> hardware ladder; "none"/"off" pins GGUF-as-is; an explicit scheme
+ # pins it.
transformer_quant_auto = transformer_quant is None or str(
transformer_quant
).strip().lower() in ("", "auto")
@@ -3593,26 +3530,22 @@ class DiffusionBackend:
speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF
)
transformer_quant = "off" if speed_off else TQ_AUTO
- # The one case that must fail closed: a named scheme (not auto, not off). Normalized
- # here so "FP8" and "fp8" refuse identically; a bogus value already raised above.
+ # The one case that must fail closed: a named scheme (not auto, not off). Normalized here so "FP8" and
+ # "fp8" refuse identically; a bogus value already raised above.
transformer_quant_pinned = (
None
if transformer_quant_auto
else normalize_transformer_quant(transformer_quant_requested)
)
- # Default-on fast path (GGUF kind only): dense bf16 + torchao quant beats GGUF per-matmul dequant on speed AND quality. Needs CUDA + bf16 + a resident fit.
pipe = None
transformer_quant_engaged = None
quant_plan = None
- # Why the dense quant did not engage, in the caller's terms. Threaded into
- # `resolved` so a fallback is visible, and into the refusal so it is actionable.
+ # Why the dense quant did not engage, in the caller's terms. Threaded into `resolved` so a fallback is
+ # visible, and into the refusal so it is actionable.
transformer_quant_decline: Optional[str] = None
transformer_quant_decline_status = RESOLVED_FELL_BACK
if transformer_quant_pinned is not None and kind != "gguf":
- # The dense fast path replaces a GGUF transformer with a quantised dense build;
- # the other kinds run the precision their checkpoint already carries, so an
- # explicit scheme here was accepted and then ignored outright.
transformer_quant_decline = (
f"the dense transformer-quant path applies to GGUF picks only, and this is "
f"a '{kind}' load, which runs the precision its checkpoint carries"
@@ -3631,38 +3564,36 @@ class DiffusionBackend:
)
is None
):
- # An explicit scheme is never swapped for another, so a None here means this GPU
- # (or this family's measured deny list) rules it out.
+ # An explicit scheme is never swapped for another
transformer_quant_decline = (
f"'{transformer_quant_pinned}' is not usable for family "
f"'{getattr(fam, 'name', None)}' on this GPU"
)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
- # The GGUF-size plan can mis-budget the fast path, so preflight the real footprint pre-eviction. Also set below when an auto quant declines a hosted pre-quant.
+ # The GGUF-size plan can mis-budget the fast path, so preflight the real footprint pre-eviction. Also
+ # set below when an auto quant declines a hosted pre-quant.
dense_declined = False
- # False when the plan only holds a PREQUANT-sized build: a failed prequant load must raise, not materialise the
- # unbudgeted dense bf16 transformer. Also false when the prefetch skipped the base repo's transformer/ shards, since
- # the fallback would pull them HERE, inside the load lock, after eviction, where unload cannot preempt it and
- # progress already reported 100%.
+ # False when the plan only holds a PREQUANT-sized build: a failed prequant load must raise, not
+ # materialise the unbudgeted dense bf16 transformer. Also false when the prefetch skipped the base
+ # repo's transformer/ shards, since the fallback would pull them HERE, inside the load lock, after
+ # eviction, where unload cannot preempt it and progress already reported 100%.
dense_fallback_allowed = bool(_transformer_prefetched)
- # A GGUF pick with the scheme left to us: the hosted pre-quant is free only while
- # already cached, since fetching it means a SECOND multi-GB denoiser and the GGUF
- # never runs. An explicit scheme, a LoRA bake and every non-GGUF kind keep today's
- # behaviour, where the prequant IS the point. An all-zero-weight list is not a bake.
+ # A GGUF pick with the scheme left to us: the hosted pre-quant is free only while already cached, since
+ # fetching it means a SECOND multi-GB denoiser and the GGUF never runs. An explicit scheme, a LoRA bake
+ # and every non-GGUF kind keep today's behaviour, where the prequant IS the point. An all-zero-weight
+ # list is not a bake.
baking_loras = _has_active_lora(loras)
- # ... and only while a scheme is still ENABLED. An explicit Speed=off rewrites an
- # auto request to "off" above and the plan stages no transformer/ on purpose, so
- # without this the deliberate bit-exact GGUF was reported as a quant declined for
- # want of shards, in the status payload and in the log.
+ # ... and only while a scheme is still ENABLED. An explicit Speed=off rewrites an auto request to "off"
+ # above and the plan stages no transformer/ on purpose, so without this the deliberate bit-exact GGUF
+ # was reported as a quant declined for want of shards, in the status payload and in the log.
if (
kind == "gguf"
and transformer_quant_auto
and not baking_loras
and normalize_transformer_quant(transformer_quant) is not None
- # An offload policy named by the REQUEST skips the dense build outright, and
- # the plan omits transformer/ for that reason rather than for want of cached
- # bytes. Reporting it as a second-denoiser refusal told the caller the wrong
- # thing about their own memory setting.
+ # An offload policy named by the REQUEST skips the dense build outright, and the plan omits
+ # transformer/ for that reason rather than for want of cached bytes. Reporting it as a
+ # second-denoiser refusal told the caller the wrong thing about their own memory setting.
and not _memory_request_forces_offload(memory_mode, cpu_offload)
):
uncached_prequant = _uncached_prequant_repo(
@@ -3680,38 +3611,30 @@ class DiffusionBackend:
uncached_prequant,
)
dense_declined = True
- # Auto-only branch, so this reason only ever reaches an "Auto: OFF" badge.
transformer_quant_decline = (
f"the hosted pre-quant checkpoint in {uncached_prequant} is not "
"cached, and an auto quant never downloads a second transformer for "
"a GGUF pick"
)
- # The other half of the same rule, and the half the prefetch decides:
- # ``_transformer_prefetched`` is False exactly when the plan left the base
- # repo's transformer/ shards out, so taking the dense path here would pull
- # them from_pretrained() under the load lock, after eviction, where unload
- # cannot preempt it and progress already reported 100%. Declining keeps the
- # GGUF the user picked, which is on disk.
- #
- # Only when the fast path's candidate IS the dense base, though. A CACHED
- # pre-quant stages no transformer/ shards either -- the plan skips them
- # because the small quantised checkpoint replaces them, not because anything
- # would have to be downloaded -- so reading the empty stage as a verdict here
- # would drop a fast path that costs nothing. The branch above already sent
- # every UNCACHED pre-quant to the GGUF, so what reaches here is free.
- # A lower auto rung whose checkpoint is already cached counts as well. Auto's
- # winner having no hosted prequant does not mean there is none to open: fp8
- # winning while only an int8 checkpoint is published is exactly what the retry
- # below exists for, and declining here would skip past it to the GGUF for a
- # checkpoint that costs nothing. That retry only ever returns a CACHED rung.
- # ... and only on a host where staging those shards could have enabled the
- # quant at all. The unsupported-device checks above run for an EXPLICIT
- # scheme only, so on CPU/MPS, non-bf16 CUDA, a stubbed torchao, or a family
- # this GPU rules out, an AUTO request reached here and was told its shards
- # were not staged -- which is true and irrelevant, since caching them changes
- # nothing. The outcome is the GGUF either way (the re-plan below is gated on
- # dense_transformer_supported), so this is only about not printing a wrong
- # reason on the badge for every Mac and CPU load.
+ # The other half of the same rule, and the half the prefetch decides: ``_transformer_prefetched`` is
+ # False exactly when the plan left the base repo's transformer/ shards out, so taking the dense path
+ # here would pull them from_pretrained() under the load lock, after eviction, where unload cannot
+ # preempt it and progress already reported 100%. Declining keeps the GGUF the user picked, which is
+ # on disk. Only when the fast path's candidate IS the dense base, though. A CACHED pre-quant stages
+ # no transformer/ shards either -- the plan skips them because the small quantised checkpoint
+ # replaces them, not because anything would have to be downloaded -- so reading the empty stage as a
+ # verdict here would drop a fast path that costs nothing. The branch above already sent every
+ # UNCACHED pre-quant to the GGUF, so what reaches here is free. A lower auto rung whose checkpoint
+ # is already cached counts as well. Auto's winner having no hosted prequant does not mean there is
+ # none to open: fp8 winning while only an int8 checkpoint is published is exactly what the retry
+ # below exists for, and declining here would skip past it to the GGUF for a checkpoint that costs
+ # nothing. That retry only ever returns a CACHED rung. ... and only on a host where staging those
+ # shards could have enabled the quant at all. The unsupported-device checks above run for an
+ # EXPLICIT scheme only, so on CPU/MPS, non-bf16 CUDA, a stubbed torchao, or a family this GPU rules
+ # out, an AUTO request reached here and was told its shards were not staged -- which is true and
+ # irrelevant, since caching them changes nothing. The outcome is the GGUF either way (the re-plan
+ # below is gated on dense_transformer_supported), so this is only about not printing a wrong reason
+ # on the badge for every Mac and CPU load.
elif (
not _transformer_prefetched
and dense_transformer_supported(target)
@@ -3757,25 +3680,22 @@ class DiffusionBackend:
and not dense_declined
):
if plan.offload_policy != OFFLOAD_NONE:
- # The GGUF plan picked offload but the quantised artifact is smaller, so re-plan: a resident quant build wins.
candidate = resolve_dense_quant_candidate(
fam = fam,
target = target,
requested = transformer_quant,
base_repo = base,
prequant_path = transformer_prequant_path,
- # A LoRA bake skips the prequant shortcut, so size for the dense build it will run.
force_dense = _has_active_lora(loras),
logger = logger,
)
- # Defined OUTSIDE the candidate check: the retry below rebinds ``candidate``
- # and calls this, and that retry is reached precisely when the first
- # resolve returned None (a free-disk gate skipping the dense download while
- # a lower cached prequant still passes). Nested, the name would be unbound
- # there and the retry would raise UnboundLocalError under the load lock
- # instead of loading the cached rung. Reads ``candidate`` at CALL time, so
- # every call still sizes whichever candidate is current.
+ # Defined OUTSIDE the candidate check: the retry below rebinds ``candidate`` and calls this, and
+ # that retry is reached precisely when the first resolve returned None (a free-disk gate
+ # skipping the dense download while a lower cached prequant still passes). Nested, the name
+ # would be unbound there and the retry would raise UnboundLocalError under the load lock instead
+ # of loading the cached rung. Reads ``candidate`` at CALL time, so every call still sizes
+ # whichever candidate is current.
def _replan_candidate():
return self._plan_memory(
target,
@@ -3792,18 +3712,17 @@ class DiffusionBackend:
),
# Pass the companion estimate so prefetched base shards aren't double-counted.
companion_override_mib = candidate.companions_mib,
- # ... and its text-encoder share, so the planner can still price
- # the streamed-encoder group tier on this path. getattr: a
- # candidate without the split (a duck-typed or older estimate)
- # passes None and keeps the previous decision.
+ # ... and its text-encoder share, so the planner can still price the streamed-encoder
+ # group tier on this path. getattr: a candidate without the split (a duck-typed or older
+ # estimate) passes None and keeps the previous decision.
text_encoder_override_mib = getattr(
candidate, "text_encoders_mib", None
),
)
if candidate is None:
- # No basis to re-plan (no size entry, or the model cache is too full for
- # the artifact), so the GGUF plan's offload stands and the quant cannot.
+ # No basis to re-plan (no size entry, or the model cache is too full for the artifact), so
+ # the GGUF plan's offload stands and the quant cannot.
transformer_quant_decline = (
f"the memory plan picked '{plan.offload_policy}' offload and there "
"is no quantised candidate to re-plan against on this host (see "
@@ -3813,12 +3732,14 @@ class DiffusionBackend:
replanned = _replan_candidate()
if (
replanned.offload_policy != OFFLOAD_NONE
- # Explicit balanced/low_vram picks offload BY MODE, so a fresh snapshot cannot change it.
+ # Explicit balanced/low_vram picks offload BY MODE, so a fresh snapshot cannot change it
and normalize_memory_mode(memory_mode)
not in (MEMORY_MODE_BALANCED, MEMORY_MODE_LOW_VRAM)
and plan_fits_total_capacity(replanned)
):
- # The candidate fits TOTAL capacity but instantaneous free said no, so re-snapshot (settled) and replan once rather than letting a transient foreign allocation force the GGUF fallback.
+ # The candidate fits TOTAL capacity but instantaneous free said no, so re-snapshot
+ # (settled) and replan once rather than letting a transient foreign allocation force the
+ # GGUF fallback.
replanned = _replan_candidate()
if replanned.offload_policy != OFFLOAD_NONE:
logger.info(
@@ -3840,16 +3761,16 @@ class DiffusionBackend:
)
if replanned.offload_policy == OFFLOAD_NONE:
quant_plan = replanned
- # The GGUF plan declined resident; a prequant-sized replan says nothing about the dense build.
+ # The GGUF plan declined resident; a prequant-sized replan says nothing about the dense
+ # build
if candidate.prequant:
dense_fallback_allowed = False
if quant_plan is None:
- # Nothing resident for auto's WINNER, either because it has no candidate
- # at all or because its replan still offloads. Without this the load
- # leaves quant_plan unset and the gate below skips the dense/prequant
- # path entirely, so a lower rung whose checkpoint is CACHED and would
- # fit never gets looked at. Same retry as the resident branch, just
- # reached from the side where the GGUF plan already wanted offload.
+ # Nothing resident for auto's WINNER, either because it has no candidate at all or because
+ # its replan still offloads. Without this the load leaves quant_plan unset and the gate
+ # below skips the dense/prequant path entirely, so a lower rung whose checkpoint is CACHED
+ # and would fit never gets looked at. Same retry as the resident branch, just reached from
+ # the side where the GGUF plan already wanted offload.
retry = self._auto_prequant_retry_scheme(
target,
fam,
@@ -3890,15 +3811,17 @@ class DiffusionBackend:
if candidate.prequant:
dense_fallback_allowed = False
else:
- # This materialises the dense bf16 transformer, so re-check the fit rather than OOMing after eviction (skipped for a prequant).
+ # This materialises the dense bf16 transformer, so re-check the fit rather than OOMing after
+ # eviction (skipped for a prequant).
scheme = select_transformer_quant_scheme(
target,
transformer_quant, # normalized above
family = getattr(fam, "name", None),
)
- # usable_prequant_source (not resolve_): a missing/non-allowlisted local path must not skip the dense-fit re-check.
+ # usable_prequant_source (not resolve_): a missing/non-allowlisted local path must not skip the
+ # dense-fit re-check
prequant = (
- # A LoRA bake skips the prequant shortcut, so gate the fast path as if no prequant existed.
+ # A LoRA bake skips the prequant shortcut, so gate the fast path as if no prequant existed
None
if _has_active_lora(loras)
else usable_prequant_source(
@@ -3910,18 +3833,17 @@ class DiffusionBackend:
if scheme is not None
else None
)
- # Reads the cache, so it looks where the prefetch WROTE (the mirror when one
- # was swapped in) and carries the staged snapshot: a base served wholly from
- # the import-time root sizes as 0 on the hub id alone, and a 0 skips this
- # fit check, landing the dense build under a GGUF-sized plan.
+ # Reads the cache, so it looks where the prefetch WROTE (the mirror when one was swapped in) and
+ # carries the staged snapshot: a base served wholly from the import-time root sizes as 0 on the
+ # hub id alone, and a 0 skips this fit check, landing the dense build under a GGUF-sized plan.
dense_mib = int(
self._dense_transformer_resident_bytes(fetch_base, _base_local_dir)
// (1024 * 1024)
)
dense_possible = True
- # Why the dense bf16 build is out, in the caller's terms, filled in by
- # whichever branch below rules it out. Only becomes the load's decline
- # reason if the auto retry then fails to find a rung that does fit.
+ # Why the dense bf16 build is out, in the caller's terms, filled in by whichever branch below
+ # rules it out. Only becomes the load's decline reason if the auto retry then fails to find a
+ # rung that does fit.
dense_decline_reason: Optional[str] = None
if dense_mib > 0:
dense_plan = self._plan_memory(
@@ -3935,16 +3857,15 @@ class DiffusionBackend:
repo_id = repo_id,
fetch_base = fetch_base,
transformer_resident_override_mib = dense_mib,
- # No companion_override here, so this one reads the cache: point it
- # at the snapshot the load will read, not the live root alone.
+ # No companion_override here, so this one reads the cache: point it at the snapshot the
+ # load will read, not the live root alone.
base_local_dir = _base_local_dir,
)
- # On unified memory the policy cannot express a misfit -- the planner
- # returns 'none' for ANY size there, because offload shuffles bytes
- # within one pool -- so the check above never fires and the dense build
- # proceeds to be OS-killed. Size it explicitly instead, and decline to
- # the packed GGUF (which already passed the load-level refusal) rather
- # than raising: a fallback that fits is better than no load at all.
+ # On unified memory the policy cannot express a misfit -- the planner returns 'none' for ANY
+ # size there, because offload shuffles bytes within one pool -- so the check above never
+ # fires and the dense build proceeds to be OS-killed. Size it explicitly instead, and
+ # decline to the packed GGUF (which already passed the load-level refusal) rather than
+ # raising: a fallback that fits is better than no load at all.
unified_shortfall = unified_memory_shortfall_message(
dense_plan, family = getattr(fam, "name", None)
)
@@ -3964,26 +3885,24 @@ class DiffusionBackend:
f"'{dense_plan.offload_policy}' offload)"
)
elif not _transformer_prefetched:
- # dense_mib == 0 with nothing staged is not "no information": the
- # prefetch's capacity gate DECLINED the base transformer/ shards, so the
- # dense bf16 build cannot run at all. Reading it as unknown and skipping
- # the retry left the regression alive on the fresh Qwen cache this was
- # written for: fp8 wins, only int8 is published, the load runs fp8 with
- # no prequant and no dense fallback, and drops straight to GGUF.
+ # dense_mib == 0 with nothing staged is not "no information": the prefetch's capacity gate
+ # DECLINED the base transformer/ shards, so the dense bf16 build cannot run at all. Reading
+ # it as unknown and skipping the retry left the regression alive on the fresh Qwen cache
+ # this was written for: fp8 wins, only int8 is published, the load runs fp8 with no prequant
+ # and no dense fallback, and drops straight to GGUF.
dense_possible = False
dense_decline_reason = (
"the base transformer this quant is built from was not staged (the "
"prefetch's capacity gate declined its shards), so the dense bf16 "
"build cannot run on this host"
)
- # A hosted prequant never builds dense, so the check above says nothing
- # about what actually lands. On unified memory that gap is a hole: the
- # OFFLOAD_NONE gate below is satisfied for ANY size there, and an fp8/int8
- # artifact can outweigh the packed GGUF that just passed the load-level
- # refusal (0.55x bf16 against a Q4's ~0.3x), so the load would materialise
- # an unsized transformer and be OS-killed. Size the artifact itself and
- # decline to the GGUF when it does not fit. Unified only: off it, the
- # OFFLOAD_NONE test already carries this and the extra resolve is waste.
+ # A hosted prequant never builds dense, so the check above says nothing about what lands. On
+ # unified memory that gap is a hole: the OFFLOAD_NONE gate below is satisfied for ANY size
+ # there, and an fp8/int8 artifact can outweigh the packed GGUF that just passed the load-level
+ # refusal (0.55x bf16 against a Q4's ~0.3x), so the load would materialise an unsized
+ # transformer and be OS-killed. Size the artifact itself and decline to the GGUF when it does
+ # not fit. Unified only: off it, the OFFLOAD_NONE test already carries this and the extra
+ # resolve is waste.
if (
prequant is not None
and getattr(plan.device_memory, "memory_kind", None) == "unified_memory"
@@ -4013,11 +3932,10 @@ class DiffusionBackend:
transformer_resident_override_mib = (
prequant_candidate.transient_transformer_mib
),
- # companions_mib is the DENSE encoder plus the VAE, but
- # assembly is handed text_encoder_quant and injects the
- # pre-cast encoder when one is configured. Price that
- # share the way the load-level plan does, or a footprint
- # that fits is declined on bytes never materialised.
+ # companions_mib is the DENSE encoder plus the VAE, but assembly is handed
+ # text_encoder_quant and injects the pre-cast encoder when one is configured.
+ # Price that share the way the load-level plan does, or a footprint that fits is
+ # declined on bytes never materialised.
companion_override_mib = (
self._precast_scaled_companions_mib(
prequant_candidate,
@@ -4046,11 +3964,10 @@ class DiffusionBackend:
"loading the GGUF",
prequant_candidate.transient_transformer_mib,
)
- # A dense misfit with a prequant source only forbids the dense fallback; with
- # none, auto could still have picked a DIFFERENT scheme that does have a
- # hosted checkpoint. auto returns one winner, and a winner with no published
- # prequant that also cannot build dense would drop the whole pick to GGUF
- # even though a lower rung would have loaded. Only for auto: an explicit
+ # A dense misfit with a prequant source only forbids the dense fallback; with none, auto could
+ # still have picked a DIFFERENT scheme that does have a hosted checkpoint. auto returns one
+ # winner, and a winner with no published prequant that also cannot build dense would drop the
+ # whole pick to GGUF even though a lower rung would have loaded. Only for auto: an explicit
# scheme is never swapped (same contract as select_transformer_quant_scheme).
if not dense_possible and prequant is None:
retry = self._auto_prequant_retry_scheme(
@@ -4062,11 +3979,10 @@ class DiffusionBackend:
path_override = transformer_prequant_path,
loras = loras,
)
- # Existence is not fit: an int8 checkpoint can outweigh the Q4 GGUF that
- # planned resident, so a host that fits the GGUF need not fit the rung
- # being retried. Size it and replan, exactly as the offload branch does,
- # or the load moves it onto CUDA after eviction under a GGUF-sized
- # budget and OOMs there.
+ # Existence is not fit: an int8 checkpoint can outweigh the Q4 GGUF that planned resident,
+ # so a host that fits the GGUF need not fit the rung being retried. Size it and replan,
+ # exactly as the offload branch does, or the load moves it onto CUDA after eviction under a
+ # GGUF-sized budget and OOMs there.
retry_candidate = (
resolve_dense_quant_candidate(
fam = fam,
@@ -4109,8 +4025,8 @@ class DiffusionBackend:
if (
retry_plan is not None
and retry_plan.offload_policy == OFFLOAD_NONE
- # Same unified caveat as above: 'none' is not a fit there, so the
- # rung being retried has to be sized explicitly before it is pinned.
+ # Same unified caveat as above: 'none' is not a fit there, so the rung being retried has
+ # to be sized explicitly before it is pinned.
and unified_memory_shortfall_message(
retry_plan, family = getattr(fam, "name", None)
)
@@ -4123,16 +4039,14 @@ class DiffusionBackend:
scheme,
retry,
)
- # Pin it: the load below re-selects from this value.
transformer_quant = retry
quant_plan = retry_plan
dense_fallback_allowed = False
else:
- # No prequant source and no retry rung that fits: the fast path is
- # out, so say which of the two halves failed. The reason reaches the
- # caller as the 409 detail on an explicit pin, and as the "Auto: OFF"
- # badge otherwise, so "no hosted checkpoint" alone would leave a user
- # who has one wondering why it was ignored.
+ # No prequant source and no retry rung that fits: the fast path is out, so say which of
+ # the two halves failed. The reason reaches the caller as the 409 detail on an explicit
+ # pin, and as the "Auto: OFF" badge otherwise, so "no hosted checkpoint" alone would
+ # leave a user who has one wondering why it was ignored.
dense_declined = True
transformer_quant_decline = (
f"{dense_decline_reason}, and no hosted pre-quantised "
@@ -4168,14 +4082,14 @@ class DiffusionBackend:
fetch_base = fetch_base,
local_files_only = local_files_only,
)
- except Exception as exc: # noqa: BLE001 — fall back to the GGUF build
+ except Exception as exc: # noqa: BLE001 - fall back to the GGUF build
logger.warning(
"diffusion.transformer_quant_fallback: %s (loading GGUF)", exc
)
pipe = None
transformer_quant_engaged = None
- # Formatted BEFORE the del below, and redacted: this reaches the client both
- # as a status tooltip and (for an explicit ask) as the refusal message.
+ # Formatted BEFORE the del below, and redacted: this reaches the client both as a status tooltip
+ # and (for an explicit ask) as the refusal message.
from utils.native_path_leases import redact_native_paths
transformer_quant_decline = redact_native_paths(
@@ -4183,7 +4097,6 @@ class DiffusionBackend:
)
# Drop the exception before clearing the cache: its traceback pins the dense transformer's VRAM.
del exc
- # Guarded: a sticky CUDA error can raise; the fallback must reach the GGUF build.
try:
clear_gpu_cache()
except Exception: # noqa: BLE001
@@ -4198,7 +4111,8 @@ class DiffusionBackend:
and normalize_transformer_quant(transformer_quant) is not None
and _has_active_lora(loras)
):
- # Adapters were requested BAKED but that build failed, and the GGUF fallback cannot carry them; fail loudly.
+ # Adapters were requested BAKED but that build failed, and the GGUF fallback cannot carry them; fail
+ # loudly.
raise RuntimeError(
"The requested LoRA adapters could not be applied: baking adapters "
"requires the quantized (int8/fp8) transformer build, which was "
@@ -4207,11 +4121,10 @@ class DiffusionBackend:
"adapters, free VRAM, or pick a smaller model."
)
- # Fail closed on a declined EXPLICIT precision. Loading the GGUF here produced a
- # perfectly good image at a precision the caller never asked for, and nothing in
- # the response said so, which is why a successful render could not be taken as
- # proof the requested precision ran. `auto` is untouched: falling down the ladder
- # is what it asks for.
+ # Fail closed on a declined EXPLICIT precision. Loading the GGUF here produced a perfectly good image at
+ # a precision the caller never asked for, and nothing in the response said so, which is why a successful
+ # render could not be taken as proof the requested precision ran. `auto` is untouched: falling down the
+ # ladder is what it asks for.
if (
pipe is None
and transformer_quant_pinned is not None
@@ -4229,16 +4142,16 @@ class DiffusionBackend:
if pipe is None:
if kind == "pipeline":
- # Full diffusers repo: from_pretrained pulls every component and re-applies embedded quant config.
if fam.name == KREA2_FAMILY_NAME:
- # krea ships transformers-5.x configs the 4.x line cannot parse, so assemble per-component; that path never sees pipe_kwargs, so pass the pre-cast TE.
- # Fetches EVERY component from the id given, so it must get the mirror.
+ # krea ships transformers-5.x configs the 4.x line cannot parse, so assemble per-component;
+ # that path never sees pipe_kwargs, so pass the pre-cast TE. Fetches EVERY component from
+ # the id given, so it must get the mirror.
pipe = load_krea2_pipeline(
fetch_base,
dtype,
hf_token = hf_token,
- # The branch never sees pipe_kwargs, so the one keyword that keeps
- # the no-download promise has to be handed over with the rest.
+ # The branch never sees pipe_kwargs, so the one keyword that keeps the no-download
+ # promise has to be handed over with the rest
local_files_only = local_files_only,
text_encoder = te_prequant_pipe_kwargs(
fam,
@@ -4252,7 +4165,8 @@ class DiffusionBackend:
).get("text_encoder"),
)
elif fam.name == IDEOGRAM4_FAMILY_NAME:
- # ideogram ships the same transformers-5.x Qwen stack as krea; assemble per-component too, from the mirror for the same reason.
+ # ideogram ships the same transformers-5.x Qwen stack as krea; assemble per-component too,
+ # from the mirror for the same reason.
pipe = load_ideogram4_pipeline(fetch_base, dtype, hf_token = hf_token)
else:
pipe_kwargs: dict[str, Any] = {
@@ -4263,7 +4177,7 @@ class DiffusionBackend:
if hf_token:
pipe_kwargs["token"] = hf_token
if fam.name == HIDREAM_FAMILY_NAME:
- # The repo names a Llama text_encoder_4 it does not ship; supply it from the open mirror.
+ # The repo names a Llama text_encoder_4 it does not ship; supply it from the open mirror
pipe_kwargs.update(
hidream_te4_kwargs(
dtype,
@@ -4274,7 +4188,8 @@ class DiffusionBackend:
local_files_only = local_files_only,
)
)
- # A hosted pre-cast fp8 text encoder skips the dense TE download; the cast re-applies idempotently.
+ # A hosted pre-cast fp8 text encoder skips the dense TE download; the cast re-applies
+ # idempotently.
pipe_kwargs.update(
te_prequant_pipe_kwargs(
fam,
@@ -4287,17 +4202,17 @@ class DiffusionBackend:
local_files_only = local_files_only,
)
)
- # The prefetched snapshot dir keeps from_pretrained off the hub (24 GB per FLUX.1 otherwise).
+ # The prefetched snapshot dir keeps from_pretrained off the hub (24 GB per FLUX.1 otherwise)
pipe = pipeline_cls.from_pretrained(
_base_local_dir or fetch_base, **pipe_kwargs
)
elif kind == "single_file" and fam.single_file_is_pipeline:
- # A single-file SDXL-style checkpoint is the WHOLE pipeline: load it through the pipeline class with ``config`` on the base repo.
+ # A single-file SDXL-style checkpoint is the WHOLE pipeline: load it through the pipeline class
+ # with ``config`` on the base repo.
sf_pipe_kwargs: dict[str, Any] = {
"local_files_only": local_files_only,
"torch_dtype": dtype,
- # ``config`` is a REPO FETCH ahead of the mirrored load, so a gated id
- # would 401 here first.
+ # ``config`` is a REPO FETCH ahead of the mirrored load, so a gated id would 401 here first
"config": fetch_base,
"cache_dir": hub_cache_dir(),
}
@@ -4308,15 +4223,13 @@ class DiffusionBackend:
# Transformer-only single file; VAE/text-encoder/scheduler come from the base repo.
sf_kwargs: dict[str, Any] = {
"torch_dtype": dtype,
- # Fetched before auth, same ordering as the whole-pipeline branch above.
"config": fetch_base,
"subfolder": "transformer",
"token": hf_token,
"cache_dir": hub_cache_dir(),
- # config is a REPO ID, and diffusers forwards this into the
- # load_config() that resolves it (single_file_model.py), so without the
- # flag this branch reaches the Hub on a load nobody asked for. The
- # pipeline assembly below was already guarded; this call was not.
+ # config is a REPO ID, and diffusers forwards this into the load_config() that resolves it
+ # (single_file_model.py), so without the flag this branch reaches the Hub on a load nobody
+ # asked for. The pipeline assembly below was already guarded; this call was not.
"local_files_only": local_files_only,
}
if kind == "gguf":
@@ -4324,7 +4237,8 @@ class DiffusionBackend:
sf_kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig(
compute_dtype = dtype
)
- # sd.cpp GGUFs prefix tensors with model.diffusion_model.; the FLUX.2 / Qwen converters choke.
+ # sd.cpp GGUFs prefix tensors with model.diffusion_model.; the FLUX.2 / Qwen converters
+ # choke.
_install_gguf_prefix_strip(transformer_cls, logger)
# A safetensors single-file (fp8) carries its own dtype: no GGUF dequant config.
transformer = transformer_cls.from_single_file(
@@ -4337,11 +4251,9 @@ class DiffusionBackend:
dtype,
hf_token = hf_token,
transformer = transformer,
- # Same reason as the full-pipeline branch: the single file supplies
- # only the denoiser, so the encoder, VAE and tokenizer below are
- # still GB this load promised not to fetch.
+ # Same reason as the full-pipeline branch: the single file supplies only the denoiser,
+ # so the encoder, VAE and tokenizer below are still GB this load promised not to fetch.
local_files_only = local_files_only,
- # Same pre-cast TE hand-in as the full-pipeline branch.
text_encoder = te_prequant_pipe_kwargs(
fam,
fetch_base,
@@ -4363,7 +4275,6 @@ class DiffusionBackend:
if hf_token:
pipe_kwargs["token"] = hf_token
if fam.name == HIDREAM_FAMILY_NAME:
- # Same Llama TE4 assembly as the full-pipeline branch above.
pipe_kwargs.update(
hidream_te4_kwargs(
dtype,
@@ -4374,7 +4285,8 @@ class DiffusionBackend:
local_files_only = local_files_only,
)
)
- # Same pre-cast TE injection as above: the GGUF supplies the transformer, so the TE is the big download.
+ # Same pre-cast TE injection as above: the GGUF supplies the transformer, so the TE is the
+ # big download.
pipe_kwargs.update(
te_prequant_pipe_kwargs(
fam,
@@ -4391,16 +4303,18 @@ class DiffusionBackend:
_base_local_dir or fetch_base, **pipe_kwargs
)
- # Effective speed: GGUF defaults to `default` (~2.2x, below the quant noise floor); dense stays bit-identical `off`.
+ # Effective speed: GGUF defaults to `default` (~2.2x, below the quant noise floor); dense stays
+ # bit-identical `off`.
effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf")
- # A torchao-quantized dense transformer must be compiled (eager is ~30x slower, losing to GGUF).
+ # A torchao-quantized dense transformer must be compiled (eager is ~30x slower, losing to GGUF)
if transformer_quant_engaged is not None and effective_speed == SPEED_OFF:
logger.info(
"diffusion.transformer_quant: forcing speed_mode=default "
"(quantized transformer must be compiled; eager is ~30x slower)"
)
effective_speed = SPEED_DEFAULT
- # Deferred speed auto for dense: stay eager and engage `default` on the 3rd image, where compile amortises. Only when speed was unset.
+ # Deferred speed auto for dense: stay eager and engage `default` on the 3rd image, where compile
+ # amortises. Only when speed was unset.
speed_deferred = (
speed_mode is None
and effective_speed == SPEED_OFF
@@ -4409,7 +4323,7 @@ class DiffusionBackend:
)
# Speed optims run BEFORE placement, so snapshot the global backend flags first for unload restore.
backend_flags_before = snapshot_backend_flags()
- # Pick the attention kernel BEFORE compile: auto upgrades to cuDNN fused attention on NVIDIA (~1.18x).
+ # Pick the attention kernel BEFORE compile: auto upgrades to cuDNN fused attention on NVIDIA (~1.18x)
attention_engaged = apply_attention_backend(
pipe,
select_attention_backend(
@@ -4418,8 +4332,9 @@ class DiffusionBackend:
logger = logger,
target = target,
)
- # Step caching (First-Block-Cache), also before compile: reuses the transformer tail across steps (~1.4x on Flux,
- # LPIPS ~0.08) and drops compile fullgraph. Tri-state: unset/auto -> FBCACHE_MIN_STEPS policy; off/fbcache pinned.
+ # Step caching (First-Block-Cache), also before compile: reuses the transformer tail across steps (~1.4x
+ # on Flux, LPIPS ~0.08) and drops compile fullgraph. Tri-state: unset/auto -> FBCACHE_MIN_STEPS policy;
+ # off/fbcache pinned.
cache_request = normalize_transformer_cache(transformer_cache)
cache_auto = transformer_cache is None or cache_request == TC_AUTO
cache_quant_active = transformer_quant_engaged is not None or bool(gguf_filename)
@@ -4437,7 +4352,7 @@ class DiffusionBackend:
quant_active = cache_quant_active,
logger = logger,
)
- # An auto decision can flip at generation time, but only on a cache-capable transformer.
+ # An auto decision can flip at generation time, but only on a cache-capable transformer
cache_may_toggle = cache_auto and callable(
getattr(getattr(pipe, "transformer", None), "enable_cache", None)
)
@@ -4456,8 +4371,9 @@ class DiffusionBackend:
)
else:
cache_reason = "requested"
- # Everything to the _LoadState commit mutates PROCESS-WIDE state; the try/finally below restores it on failure.
- # gguf_transformer: the dense fast path still sets gguf_filename, but pipe.transformer is dense (REGIONAL compile).
+ # Everything to the _LoadState commit mutates PROCESS-WIDE state; the try/finally below restores it on
+ # failure. gguf_transformer: the dense fast path still sets gguf_filename, but pipe.transformer is dense
+ # (REGIONAL compile).
gguf_transformer = kind == "gguf" and transformer_quant_engaged is None
eager_patched = False
@@ -4483,21 +4399,23 @@ class DiffusionBackend:
uninstall_patches()
uninstall_arch_patches()
- # Pre-warmed torch.compile cache: a per-fingerprint inductor dir plus a bundle loaded before the first compiled forward pays the 25-58s compile once. A miss is silent.
+ # Pre-warmed torch.compile cache: a per-fingerprint inductor dir plus a bundle loaded before the
+ # first compiled forward pays the 25-58s compile once.
if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and compile_eligible(
target, is_gguf = gguf_transformer, family = fam
):
compile_ctx = compile_cache.begin(
family = fam.name,
- # U-Net families (SDXL) carry the denoiser as pipe.unet.
transformer = getattr(pipe, "transformer", None)
or getattr(pipe, "unet", None),
dtype = getattr(target, "dtype", None),
- # A GGUF transformer compiles a DIFFERENT graph than a dense load, so it keys its own bundles.
+ # A GGUF transformer compiles a DIFFERENT graph than a dense load, so it keys its own
+ # bundles.
quant = "gguf" if gguf_transformer else transformer_quant_engaged,
attention_backend = attention_engaged,
compile_kwargs = {
- # Mirrors apply_speed_optims' fullgraph decision (a step cache or planned offload graph-breaks), so the bundle keys on the same setting.
+ # Mirrors apply_speed_optims' fullgraph decision (a step cache or planned offload
+ # graph-breaks), so the bundle keys on the same setting.
"fullgraph": cache_engaged is None
and not cache_may_toggle
and plan.offload_policy == OFFLOAD_NONE,
@@ -4516,18 +4434,19 @@ class DiffusionBackend:
family = fam,
speed_mode = effective_speed,
cache_active = cache_engaged is not None or cache_may_toggle,
- # Offload installs compiler-disabled onload hooks, so compile drops fullgraph.
offload_active = plan.offload_policy != OFFLOAD_NONE,
logger = logger,
)
if transformer_quant_engaged is not None and not speed_applied.get("compiled"):
- # Compile could not engage: the quantized transformer runs eager, far slower than the GGUF it replaced.
+ # Compile could not engage: the quantized transformer runs eager, far slower than the GGUF it
+ # replaced
logger.warning(
"diffusion.transformer_quant: %s engaged but the transformer is NOT "
"compiled; eager torchao quant is ~30x slower than GGUF here",
transformer_quant_engaged,
)
- # Quantise the dense companion text encoder(s) before placement so offload moves the smaller weights.
+ # Quantise the dense companion text encoder(s) before placement so offload moves the smaller
+ # weights.
te_outcome = quantize_text_encoders(
pipe,
target,
@@ -4537,12 +4456,11 @@ class DiffusionBackend:
logger = logger,
)
te_quant = te_outcome.mode
- # Same contract for the other half of "the requested precision": an explicit
- # encoder mode that engaged NOTHING leaves a dense bf16 encoder the caller did
- # not ask for, and a PARTIAL cast leaves a pipeline conditioning off a mixture
- # of quantised and dense encoders. A mode that engaged something ELSE
- # (int8 -> fp8) is reported by the badge instead: the encoder IS quantised on
- # every tower, just not the way asked.
+ # Same contract for the other half of "the requested precision": an explicit encoder mode that
+ # engaged NOTHING leaves a dense bf16 encoder the caller did not ask for, and a PARTIAL cast leaves
+ # a pipeline conditioning off a mixture of quantised and dense encoders. A mode that engaged
+ # something ELSE (int8 -> fp8) is reported by the badge instead: the encoder IS quantised on every
+ # tower, just not the way asked.
if (
(te_quant is None or te_outcome.partial)
and normalize_te_quant(text_encoder_quant) is not None
@@ -4558,9 +4476,9 @@ class DiffusionBackend:
)
)
- # Whole-module offload still onloads one complete component for its forward.
- # Refine it from the loaded, possibly quantized weights so an oversized text
- # encoder uses leaf streaming instead of failing during prompt encoding.
+ # Whole-module offload still onloads one complete component for its forward. Refine it from the
+ # loaded, possibly quantized weights so an oversized text encoder uses leaf streaming instead of
+ # failing during prompt encoding.
refined_plan = refine_memory_plan_for_components(pipe, plan)
if refined_plan.offload_policy != plan.offload_policy:
logger.info(
@@ -4571,8 +4489,9 @@ class DiffusionBackend:
)
plan = refined_plan
- # Persistent conditioning cache (UNSLOTH_DIFFUSION_COND_CACHE_DIR): repeated prompts skip the text-encoder
- # forward. After the TE quant so the key reflects the encoders that run; ``base`` keys the companion repo.
+ # Persistent conditioning cache (UNSLOTH_DIFFUSION_COND_CACHE_DIR): repeated prompts skip the
+ # text-encoder forward. After the TE quant so the key reflects the encoders that run; ``base`` keys
+ # the companion repo.
cond_cache.install(
pipe,
family = fam.name,
@@ -4583,19 +4502,21 @@ class DiffusionBackend:
logger = logger,
)
- # Apply the planned placement; apply_memory_plan returns what ACTUALLY engaged so status stays honest.
+ # Apply the planned placement; apply_memory_plan returns what ACTUALLY engaged so status stays
+ # honest.
effective_policy, effective_tiling = apply_memory_plan(
pipe,
plan,
device = device,
- # Indexed when a card was selected: the CPU-offload APIs read the index off
- # this string and default to cuda:0 without one, which would page modules
- # onto a different card than the one generation runs on.
+ # Indexed when a card was selected: the CPU-offload APIs read the index off this string and
+ # default to cuda:0 without one, which would page modules onto a different card than the one
+ # generation runs on.
placement_device = target.torch_device,
logger = logger,
)
- # Per-control provenance for status. cpu_offload=False is the unset default, so only True is explicit.
+ # Per-control provenance for status. cpu_offload=False is the unset default, so only True is
+ # explicit.
resolved = build_resolved_record(
{
"speed_mode": (
@@ -4612,14 +4533,13 @@ class DiffusionBackend:
else "requested",
),
"transformer_quant": (
- # The RAW request, not the tri-state rewrite: an "Auto: OFF" badge
- # and a declined "FP8" must not look the same. Captured before the
- # rewrite, so it is also immune to the auto retries pinning a
- # concrete rung: a pick that fell back to a lower rung is still auto.
+ # The RAW request, not the tri-state rewrite: an "Auto: OFF" badge and a declined "FP8"
+ # must not look the same. Captured before the rewrite, so it is also immune to the auto
+ # retries pinning a concrete rung: a pick that fell back to a lower rung is still auto.
transformer_quant_requested,
transformer_quant_engaged or "off",
- # A recorded decline wins: it names WHY, where the generic strings
- # below only say the GGUF ran.
+ # A recorded decline wins: it names WHY, where the generic strings below only say the
+ # GGUF ran
(
transformer_quant_decline
or (
@@ -4632,8 +4552,8 @@ class DiffusionBackend:
else "re-planned resident for the quantised artifact"
if quant_plan is not None
else "engaged on the dense fast path",
- # Honored when the quant engaged AND when the ask was "off" (a
- # request NOT to quantise, which the GGUF build satisfies).
+ # Honored when the quant engaged AND when the ask was "off" (a request NOT to quantise,
+ # which the GGUF build satisfies)
RESOLVED_APPLIED
if transformer_quant_engaged is not None
or transformer_quant_pinned is None
@@ -4713,8 +4633,6 @@ class DiffusionBackend:
hf_token = hf_token,
resolved = resolved,
gguf_filename = gguf_filename,
- # Built from the artifact this load COMMITTED to, by the same helper
- # _plan_memory used, so the generate-time re-check reuses it verbatim.
variant_hint = _image_variant_hint(
fam.name,
Path(single_file_path).name if single_file_path else None,
@@ -4732,7 +4650,6 @@ class DiffusionBackend:
if eager_patched:
uninstall_patches()
uninstall_arch_patches()
- # Free the half-built pipe's VRAM (uncommitted _state -> nothing else reclaims it).
clear_gpu_cache()
logger.info(
@@ -4802,8 +4719,8 @@ class DiffusionBackend:
# materialise the transformer only to fail at quantize, after eviction. load_pipeline falls back to GGUF.
raise RuntimeError("transformer quant unsupported for this device/scheme")
if fam is not None and not _has_active_lora(lora_specs):
- # A LoRA bake needs the DENSE transformer (adapters attach before quantize_), so skip
- # prequant. Active weights only: an all-zero list bakes nothing.
+ # A LoRA bake needs the DENSE transformer (adapters attach before quantize_), so skip prequant. Active
+ # weights only: an all-zero list bakes nothing.
source = resolve_prequant_source(
fam, scheme, path_override = prequant_path, base_repo = base
)
@@ -4818,15 +4735,15 @@ class DiffusionBackend:
scheme = scheme,
# Reject a checkpoint with a different Linear filter so prequant matches runtime-quant.
min_features = DEFAULT_MIN_LINEAR_FEATURES,
- # A load nobody asked for may not fetch this checkpoint either: it is a
- # multi-GB download like any other, and the switch verified only what the
- # plan listed. A cache miss falls out to the dense path, which is refused
- # for the same reason a line below.
+ # A load nobody asked for may not fetch this checkpoint either: it is a multi-GB download like any
+ # other, and the switch verified only what the plan listed. A cache miss falls out to the dense
+ # path, which is refused for the same reason a line below.
local_files_only = local_files_only,
- # Only enforced when the caller forces fp8 fast-accum; a checkpoint that baked the other choice falls to the dense path.
+ # Only enforced when the caller forces fp8 fast-accum; a checkpoint that baked the other choice
+ # falls to the dense path.
fast_accum = fast_accum,
- # The root _uncached_prequant_repo cleared this load against, so its hit is
- # reused rather than re-downloaded under the import-time constant.
+ # The root _uncached_prequant_repo cleared this load against, so its hit is reused rather than
+ # re-downloaded under the import-time constant.
cache_dir = hub_cache_dir(),
logger = logger,
)
@@ -4849,23 +4766,23 @@ class DiffusionBackend:
# 2. Fallback: materialise the dense bf16 transformer and quantise it on-device.
if not allow_dense_fallback:
- # The plan only budgeted the prequant-sized build, so the dense bf16 transformer would exceed it after eviction.
+ # The plan only budgeted the prequant-sized build, so the dense bf16 transformer would exceed it after
+ # eviction.
raise RuntimeError(
"prequant checkpoint unavailable and the dense transformer does not fit resident"
)
- # Deliberately the hub id, not base_local_dir: diffusers treats a local directory as
- # terminal (_get_model_file raises rather than falling back to the hub) and a sharded load
- # raises per missing shard, so a partial snapshot would drop a build the hub id completes.
- # Off the hub id a base only the other root holds costs a re-download, or 401s into the
- # GGUF fallback, which is what main does today.
+ # Deliberately the hub id, not base_local_dir: diffusers treats a local directory as terminal (_get_model_file
+ # raises rather than falling back to the hub) and a sharded load raises per missing shard, so a partial snapshot
+ # would drop a build the hub id completes. Off the hub id a base only the other root holds costs a re-download,
+ # or 401s into the GGUF fallback, which is what main does today.
transformer = transformer_cls.from_pretrained(
fetch_base,
subfolder = "transformer",
torch_dtype = dtype,
token = hf_token,
cache_dir = hub_cache_dir(),
- # The dense bf16 transformer is the largest single fetch on this path, so an
- # API-initiated load has to be refused here rather than allowed to pull it.
+ # The dense bf16 transformer is the largest single fetch on this path, so an API-initiated load has to be
+ # refused here rather than allowed to pull it.
local_files_only = local_files_only,
)
pipe = self._assemble_pipe(
@@ -4883,8 +4800,9 @@ class DiffusionBackend:
local_files_only = local_files_only,
)
if _has_active_lora(lora_specs):
- # Bake the adapters BEFORE quantize_: peft wraps the dense Linears (post-quant torchao dispatch would TypeError),
- # then quantize_ converts only each wrapper's frozen base_layer while the "lora_" side path stays high precision.
+ # Bake the adapters BEFORE quantize_: peft wraps the dense Linears (post-quant torchao dispatch would
+ # TypeError), then quantize_ converts only each wrapper's frozen base_layer while the "lora_" side path
+ # stays high precision.
baked = self._resolve_lora_set(
[(i, w) for (i, w) in lora_specs if w != 0],
family = getattr(fam, "name", None),
@@ -4957,8 +4875,8 @@ class DiffusionBackend:
hf_token = hf_token,
transformer = transformer,
text_encoder = krea_te,
- # ``base_local_dir`` is None whenever nothing was staged, and then this is a repo
- # id: the same guard the pipe_kwargs below carry for every other family.
+ # ``base_local_dir`` is None whenever nothing was staged, and then this is a repo id: the same guard the
+ # pipe_kwargs below carry for every other family.
local_files_only = local_files_only,
)
pipe.to(device)
@@ -4972,7 +4890,6 @@ class DiffusionBackend:
if hf_token:
pipe_kwargs["token"] = hf_token
if getattr(fam, "name", None) == HIDREAM_FAMILY_NAME:
- # The repo ships no Llama text_encoder_4; assemble it from the open mirror, as above.
pipe_kwargs.update(
hidream_te4_kwargs(
dtype,
@@ -4983,7 +4900,6 @@ class DiffusionBackend:
local_files_only = local_files_only,
)
)
- # Same pre-cast TE injection as the other branches: the dense path supplies only the transformer.
if target is not None:
pipe_kwargs.update(
te_prequant_pipe_kwargs(
@@ -5063,10 +4979,10 @@ class DiffusionBackend:
would be lowered to a number less than half what it loads. On disk is the measured truth
there; only a hub id the table actually recognises earns the substitution."""
try:
- # A whole-pipeline single file (SDXL) carries the U-Net, VAE and text encoders itself,
- # and the base repo is read for config only -- but the plan still adds the base's
- # cached companion weights, so a user who once loaded the full pipeline has those
- # bytes counted twice. Harmless as an offload hint, a rejected load as a hard refusal.
+ # A whole-pipeline single file (SDXL) carries the U-Net, VAE and text encoders itself, and the base repo is
+ # read for config only -- but the plan still adds the base's cached companion weights, so a user who once
+ # loaded the full pipeline has those bytes counted twice. Harmless as an offload hint, a rejected load as a
+ # hard refusal.
if kind == "single_file" and getattr(fam, "single_file_is_pipeline", False):
companion = plan.estimates.get("companion_dense_mib")
current = plan.estimates.get("model_dense_mib")
@@ -5085,12 +5001,11 @@ class DiffusionBackend:
if getattr(target, "dtype", None) not in (torch.bfloat16, torch.float16):
return plan
- # RECOGNISED bases only. The table is keyed on exact upstream ids, so a fine-tune or a
- # renamed mirror the family detector still matches by name would fall through to the
- # coarse family entry -- and for a family carrying two sizes that entry is the smaller
- # one, so a 9B derivative would be lowered to the 4B number and walk past the refusal.
- # Accept the family's own default base and anything with an explicit override; anything
- # else keeps its measured size.
+ # RECOGNISED bases only. The table is keyed on exact upstream ids, so a fine-tune or a renamed mirror the
+ # family detector still matches by name would fall through to the coarse family entry -- and for a family
+ # carrying two sizes that entry is the smaller one, so a 9B derivative would be lowered to the 4B number and
+ # walk past the refusal. Accept the family's own default base and anything with an explicit override;
+ # anything else keeps its measured size.
canonical = canonical_base(base)
if (
base_repo_bf16_components_gb(base) is None
@@ -5100,12 +5015,11 @@ class DiffusionBackend:
table = family_bf16_components_gb(fam, base)
if table is None:
return plan
- # The table's encoder term is the DENSE one. When this pick takes its encoder pre-cast
- # from a hosted fp8 checkpoint the resident encoder is about 0.65x that, and for a
- # heavyweight one (FLUX.2-dev's Mistral-24B, 48 GB) budgeting the dense figure is tens
- # of GB of weights the pipeline never materialises -- enough to refuse a load that fits.
- # Keyed on te_prequant_sources through the shared helper, the same resolver assembly
- # uses, so the budget cannot claim a saving the load does not take.
+ # The table's encoder term is the DENSE one. When this pick takes its encoder pre-cast from a hosted fp8
+ # checkpoint the resident encoder is about 0.65x that, and for a heavyweight one (FLUX.2-dev's Mistral-24B,
+ # 48 GB) budgeting the dense figure is tens of GB of weights the pipeline never materialises -- enough to
+ # refuse a load that fits. Keyed on te_prequant_sources through the shared helper, the same resolver
+ # assembly uses, so the budget cannot claim a saving the load does not take.
from .diffusion_te_prequant import te_prequant_budget_scale
te_scale = te_prequant_budget_scale(
@@ -5124,7 +5038,7 @@ class DiffusionBackend:
plan,
estimates = {**plan.estimates, "model_dense_mib": table_mib},
)
- except Exception: # noqa: BLE001 — sizing aid only; refuse on the plan as built
+ except Exception: # noqa: BLE001 - sizing aid only; refuse on the plan as built
return plan
def declared_footprint_shortfall(
@@ -5240,18 +5154,19 @@ class DiffusionBackend:
sizing an upstream id whose cache is empty folds the VAE/text-encoder to zero and wrongly
picks resident placement. ``base`` and ``repo_id`` keep the upstream identity for the
family/variant checks."""
- # Settled (max-over-reads) on cuda: a transient foreign allocation would make an empty card look full.
+ # Settled (max-over-reads) on cuda: a transient foreign allocation would make an empty card look full
device_memory = settled_snapshot_device_memory(target)
if kind == "pipeline":
- # The whole repo is one cached download, so cached bytes are the resident estimate; a LOCAL path is not cached, so sum its on-disk weights.
+ # The whole repo is one cached download, so cached bytes are the resident estimate; a LOCAL path is not
+ # cached, so sum its on-disk weights.
local_repo = Path(repo_id).expanduser() if repo_id else None
staged_repo = Path(base_local_dir).expanduser() if base_local_dir else None
- # The bytes land under the repo they were STAGED from, so scan the mirror when there is
- # one: an upstream id whose cache is empty folds the whole pipeline to zero.
+ # The bytes land under the repo they were STAGED from, so scan the mirror when there is one: an upstream id
+ # whose cache is empty folds the whole pipeline to zero.
cache_repo = fetch_base or repo_id
- # Largest of every source the load can read this repo from: a staged snapshot can sit
- # under the OTHER root, and a prefetch that split the repo across roots hands back none
- # at all, so the smaller answer would plan a multi-GB pipeline as unknown.
+ # Largest of every source the load can read this repo from: a staged snapshot can sit under the OTHER root,
+ # and a prefetch that split the repo across roots hands back none at all, so the smaller answer would plan a
+ # multi-GB pipeline as unknown.
cached = max(
self._local_dir_weight_bytes(local_repo, exclude_transformer = False)
if local_repo is not None and local_repo.is_dir()
@@ -5272,26 +5187,29 @@ class DiffusionBackend:
and local_repo is not None
and local_repo.is_dir()
):
- # A local fp8 mirror never string-matches base_repo, so detect fp8 from the shard headers (a local nf4 mirror stays compressed).
+ # A local fp8 mirror never string-matches base_repo, so detect fp8 from the shard headers (a local nf4
+ # mirror stays compressed).
is_narrow_base = ideogram4_repo_is_fp8(repo_id)
if is_narrow_base:
table = family_bf16_components_gb(fam, fam.base_repo)
if table is not None:
- # Reserve the bf16 footprint from this network-free constant, else model_dense_mib stays None ("unknown -> resident") and the ~54 GB pipeline OOMs.
+ # Reserve the bf16 footprint from this network-free constant, else model_dense_mib stays None
+ # ("unknown -> resident") and the ~54 GB pipeline OOMs
table_mib = int(sum(table) * (1000.0**3) / (1024.0 * 1024.0))
model_dense_mib = (
table_mib if model_dense_mib is None else max(model_dense_mib, table_mib)
)
companion_mib = None
- # No companion total on this branch (the whole repo IS the model), so there is no
- # split to hand the planner either. None, not a guess: it reproduces the old decision.
+ # No companion total on this branch (the whole repo IS the model), so there is no split to hand the planner
+ # either. None, not a guess: it reproduces the old decision.
text_encoder_mib = None
else:
if transformer_resident_override_mib is not None:
# Planning the dense-quant candidate: the auto-policy estimate replaces the file-size derivation.
transformer_resident = transformer_resident_override_mib
elif kind == "single_file":
- # An fp8 checkpoint upcasts to bf16 on load (~2x resident); detect from the basename. SDXL single-file is already bf16.
+ # An fp8 checkpoint upcasts to bf16 on load (~2x resident); detect from the basename. SDXL single-file
+ # is already bf16.
fp8_upcast = not getattr(fam, "single_file_is_pipeline", False) and (
"fp8" in Path(single_file_path).name.lower() if single_file_path else False
)
@@ -5300,37 +5218,40 @@ class DiffusionBackend:
)
else:
transformer_resident = estimate_gguf_resident_mib(file_size_mib(single_file_path))
- # Companions (VAE + text encoders) load near on-disk size; sum the base-repo cache, or a LOCAL base's on-disk weights.
+ # Companions (VAE + text encoders) load near on-disk size; sum the base-repo cache, or a LOCAL base's
+ # on-disk weights.
if companion_override_mib is not None:
- # Re-planning the dense candidate: the prefetched transformer/ shards land in the same cache, so use the estimate instead of double-counting.
+ # Re-planning the dense candidate: the prefetched transformer/ shards land in the same cache, so use the
+ # estimate instead of double-counting.
companion_mib = companion_override_mib
- # The text-encoder share of that same override, from the same family component
- # table, so the two terms cannot disagree. None when the caller has no split.
+ # The text-encoder share of that same override, from the same family component table, so the two terms
+ # cannot disagree. None when the caller has no split.
text_encoder_mib = text_encoder_override_mib
else:
- # Scan the repo the bytes were staged from, and hand over the staged snapshot too: a
- # base served from the import-time root is invisible to a hub-id scan of the live
- # one, so the VAE + text encoders would load unbudgeted.
+ # Scan the repo the bytes were staged from, and hand over the staged snapshot too: a base served from
+ # the import-time root is invisible to a hub-id scan of the live one, so the VAE + text encoders would
+ # load unbudgeted.
companion = self._companion_cache_bytes(fetch_base or base, base_local_dir)
companion_mib = int(companion // (1024 * 1024)) if companion else None
- # The text-encoder share of that companion total, from the SAME walk over the same
- # trees, so the subtraction the planner does is exact rather than two estimates
- # meeting in the middle. 0 bytes reads as "nothing cached", i.e. no split.
+ # The text-encoder share of that companion total, from the SAME walk over the same trees, so the
+ # subtraction the planner does is exact rather than two estimates meeting in the middle. 0 bytes reads
+ # as "nothing cached", i.e. no split.
text_encoder = self._text_encoder_cache_bytes(fetch_base or base, base_local_dir)
text_encoder_mib = int(text_encoder // (1024 * 1024)) if text_encoder else None
model_dense_mib = None
if transformer_resident is not None:
model_dense_mib = transformer_resident + (companion_mib or 0)
- # Feed the variant hint so estimate_image_runtime_mib sees distilled markers (distilled needs ~15% less headroom).
+ # Feed the variant hint so estimate_image_runtime_mib sees distilled markers (distilled needs ~15% less
+ # headroom).
variant_hint = _image_variant_hint(
fam.name,
Path(single_file_path).name if single_file_path else None,
repo_id,
base,
)
- # No dimensions on purpose: load time genuinely does not know what the user will generate
- # at, so this budgets the 1024x1024 default for PLACEMENT. generate() re-checks the real
- # resolution against the free budget before it samples (raise_on_image_activation_shortfall).
+ # No dimensions on purpose: load time genuinely does not know what the user will generate at, so this budgets
+ # the 1024x1024 default for PLACEMENT. generate() re-checks the real resolution against the free budget before
+ # it samples (raise_on_image_activation_shortfall).
runtime_headroom = estimate_image_runtime_mib(width = None, height = None, family = variant_hint)
return plan_diffusion_memory(
target = target,
@@ -5372,7 +5293,8 @@ class DiffusionBackend:
import diffusers
pipe = self._from_pipe_no_recast(state.pipe, getattr(diffusers, class_name))
- # Publish to the shared aux cache only if THIS load is still current: from_pipe runs without _lock, so an unload can null _state and caching would hand out stale modules.
+ # Publish to the shared aux cache only if THIS load is still current: from_pipe runs without _lock, so an unload
+ # can null _state and caching would hand out stale modules.
with self._lock:
if self._state is state:
self._aux_pipes[class_name] = pipe
@@ -5395,8 +5317,9 @@ class DiffusionBackend:
if cn_model is None:
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
- # resolve_controlnet accepts a bare owner/name without the trust gate and from_pretrained would execute a malicious
- # pickle, so run the same Hub malware preflight. It fails OPEN, so a remote repo also forces safetensors below.
+ # resolve_controlnet accepts a bare owner/name without the trust gate and from_pretrained would execute a
+ # malicious pickle, so run the same Hub malware preflight. It fails OPEN, so a remote repo also forces
+ # safetensors below.
remote_cn = not getattr(resolved_cn, "is_local", False)
if remote_cn:
from utils.security import evaluate_file_security
@@ -5412,7 +5335,8 @@ class DiffusionBackend:
# state.dtype is the display string ("bfloat16"), so pass the real dtype and avoid a float32 load.
cn_dtype = getattr(torch, str(state.dtype).replace("torch.", ""), None)
- # Force safetensors for an untrusted remote repo: if the Hub scan failed open, an embedded pickle would still deserialize.
+ # Force safetensors for an untrusted remote repo: if the Hub scan failed open, an embedded pickle would
+ # still deserialize.
cn_from_pretrained_kwargs: dict[str, Any] = {"cache_dir": hub_cache_dir()}
if remote_cn:
cn_from_pretrained_kwargs["use_safetensors"] = True
@@ -5423,10 +5347,10 @@ class DiffusionBackend:
**cn_from_pretrained_kwargs,
)
if cancel.is_set():
- # An unload raced the blocking download; bail BEFORE placement so we do not allocate onto a just-freed GPU.
del cn_model
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
- # Placement follows the base offload policy (resident base -> resident, offloaded -> group offload). Best-effort.
+ # Placement follows the base offload policy (resident base -> resident, offloaded -> group offload).
+ # Best-effort.
if getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE and (
_offload_controlnet_module(cn_model, state.device, logger)
):
@@ -5434,7 +5358,6 @@ class DiffusionBackend:
else:
cn_model = cn_model.to(state.device)
if cancel.is_set():
- # An unload raced the download and cleared the caches; caching now would pin it.
del cn_model
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
self._cn_models[resolved_cn.id] = cn_model
@@ -5509,8 +5432,6 @@ class DiffusionBackend:
@functools.wraps(original)
def _encode(x: Any, *args: Any, **kwargs: Any) -> Any:
- # Probed per call, not at wrap time: _align_vae_dtype and a txt2img decode
- # both re-cast the VAE. A failed probe forwards the tensor as it arrived.
try:
if isinstance(x, torch.Tensor):
target = next(
@@ -5612,7 +5533,7 @@ class DiffusionBackend:
transformer_quant = state.transformer_quant,
compiled = "compiled" in (getattr(state, "speed_optims", ()) or ()),
):
- # Name only routes the user can actually reach: a GPU host never selects the native engine on its own.
+ # Name only routes the user can actually reach: a GPU host never selects the native engine on its own
raise ValueError(
"LoRA is not supported for this model/quantisation on the diffusers engine "
"(GGUF-via-diffusers, or a torch.compile'd Speed=default/max load). Reload with "
@@ -5689,7 +5610,6 @@ class DiffusionBackend:
if desired == current:
return
if [(n, p) for (n, p, _w) in desired] == [(n, p) for (n, p, _w) in current]:
- # Same adapters, new weights: value-level change on the baked topology.
pipe.set_adapters(
[n for (n, _p, _w) in desired],
adapter_weights = [w for (_n, _p, w) in desired],
@@ -5723,7 +5643,7 @@ class DiffusionBackend:
if callable(reset):
try:
reset()
- except Exception: # noqa: BLE001 — reset is best-effort, never fail a generation
+ except Exception: # noqa: BLE001 - reset is best-effort, never fail a generation
pass
def _engage_deferred_speed(self, state: _LoadState) -> None:
@@ -5754,9 +5674,9 @@ class DiffusionBackend:
target = target,
)
object.__setattr__(state, "attention_backend", attention_engaged)
- # Keep the badge in step with the state it describes: the top-level field moved, so leaving
- # `resolved` on the load-time value would make the panel report an attention backend that is
- # no longer the one running. Same in-place update as the fields above.
+ # Keep the badge in step with the state it describes: the top-level field moved, so leaving `resolved` on the
+ # load-time value would make the panel report an attention backend that is no longer the one running. Same
+ # in-place update as the fields above.
if isinstance(state.resolved, dict) and "attention_backend" in state.resolved:
entry = dict(state.resolved["attention_backend"])
entry["value"] = attention_engaged or "native"
@@ -5826,11 +5746,13 @@ class DiffusionBackend:
guidance: float = 0.0,
seed: Optional[int] = None,
batch_size: int = 1,
- # Batched multi-image generation: ``prompts`` renders one image per prompt (txt2img only), ``seeds`` one per seed,
- # ``batch_size`` alone derives base..base+n-1. Each image gets its OWN Generator, so same-seed repeats are bit-identical.
+ # Batched multi-image generation: ``prompts`` renders one image per prompt (txt2img only), ``seeds`` one per
+ # seed, ``batch_size`` alone derives base..base+n-1. Each image gets its OWN Generator, so same-seed repeats are
+ # bit-identical.
prompts: Optional[list[str]] = None,
seeds: Optional[list[int]] = None,
- # Image-conditioned (base64/data-URL): init alone = img2img, init + mask = inpaint. ``strength`` 0 = keep source, 1 = full redraw.
+ # Image-conditioned (base64/data-URL): init alone = img2img, init + mask = inpaint. ``strength`` 0 = keep
+ # source, 1 = full redraw.
init_image: Optional[str] = None,
mask_image: Optional[str] = None,
strength: Optional[float] = None,
@@ -5838,11 +5760,10 @@ class DiffusionBackend:
upscale: Optional[float] = None,
# Reference (FLUX.2): additional reference images beyond init_image (a list). Ignored elsewhere.
reference_images: Optional[list[str]] = None,
- # LoRA (id, weight) pairs; loaded non-fused and activated for this generation. None/empty clears.
loras: Optional[list[tuple[str, float]]] = None,
# ControlNet (id, control_image_b64, control_type, strength, guidance_start, guidance_end). None = off.
controlnet: Optional[tuple[str, str, str, float, float, float]] = None,
- # load_identity() of the caller's status() read; refuse rather than run a different load (#9448).
+ # load_identity() of the caller's status() read; refuse rather than run a different load (#9448)
expected_load: Optional[LoadIdentity] = None,
) -> dict[str, Any]:
import torch
@@ -5857,20 +5778,18 @@ class DiffusionBackend:
raise RuntimeError(DIFFUSION_NOT_LOADED_MSG)
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
- # The slot admits on a zero fence, which a COMMITTED replacement also satisfies (#9448).
+ # The slot admits on a zero fence, which a COMMITTED replacement also satisfies (#9448)
loaded_id = load_identity(state.repo_id, state.base_repo, state.family.name)
if expected_load is not None and expected_load != loaded_id:
raise DiffusionModelReplacedError(expected_load, loaded_id)
- # Publish an active (step 0) state before the slow pre-denoise setup so a reload mount probe does not read idle.
+ # Publish an active (step 0) state before the slow pre-denoise setup so a reload mount probe does not
+ # read idle.
self._gen = _GenState(total_steps = steps)
try:
- # FIRST, before any device object exists. This worker is not the thread that loaded
- # the pipeline, so until it is pinned the un-indexed state.device below -- and the
- # ControlNet placement further down -- resolve to its own default card while the
- # weights sit on the selected one.
self._state_device_target(state)
- # The local `state` ref keeps the pipe alive even if unload() nulls _state. Resolve the per-image (prompt, seed) jobs
- # up front: N prompts, one prompt x N seeds, or one prompt deriving base..base+batch_size-1 (as the native engine does).
+ # The local `state` ref keeps the pipe alive even if unload() nulls _state. Resolve the per-image
+ # (prompt, seed) jobs up front: N prompts, one prompt x N seeds, or one prompt deriving
+ # base..base+batch_size-1 (as the native engine does).
jobs, seed = resolve_batch_jobs(
prompt = prompt,
prompts = prompts,
@@ -5880,10 +5799,12 @@ class DiffusionBackend:
draw_seed = torch.Generator(device = state.device).seed,
)
- # Deferred speed auto: engage the compile profile on the 3rd image. Best-effort; a failure stays eager and never retries.
- # NOT when a LoRA is requested: a compiled transformer rejects LoRA, breaking every LoRA generation on this load.
+ # Deferred speed auto: engage the compile profile on the 3rd image. Best-effort; a failure stays eager
+ # and never retries. NOT when a LoRA is requested: a compiled transformer rejects LoRA, breaking every
+ # LoRA generation on this load.
lora_requested = _has_active_lora(loras)
- # Also stay eager while a PRIOR generation's adapters are attached: compiling would bake them in permanently.
+ # Also stay eager while a PRIOR generation's adapters are attached: compiling would bake them in
+ # permanently.
loras_attached = bool(getattr(state.pipe, "_unsloth_loras", ()))
if (
state.speed_deferred
@@ -5893,7 +5814,7 @@ class DiffusionBackend:
):
try:
self._engage_deferred_speed(state)
- except Exception as exc: # noqa: BLE001 — speed is best-effort
+ except Exception as exc: # noqa: BLE001 - speed is best-effort
logger.warning(
"diffusion.speed: deferred engagement failed, staying eager: %s",
exc,
@@ -5902,13 +5823,11 @@ class DiffusionBackend:
# Apply/adjust LoRA before picking the workflow pipe; from_pipe pipes share the transformer.
self._apply_loras(state, loras, cancel)
- # Select the workflow pipe: txt2img uses the loaded pipe; img2img/inpaint reuse its modules via from_pipe.
pipe = state.pipe
init_pil = mask_pil = None
control_pil = None
cn_scale = cn_gstart = cn_gend = cn_mode = None
ref_extra: list = []
- # Validate dependencies up front: mask/upscale/reference need an input image, and reference needs a supporting family.
if init_image is None:
if mask_image is not None:
raise ValueError("mask_image requires an input image (init_image).")
@@ -5922,13 +5841,11 @@ class DiffusionBackend:
"model family."
)
if getattr(state.family, "edit", False):
- # Instruction editing: the loaded pipe IS the edit pipeline and always needs an input image; the prompt is the instruction.
if init_image is None:
raise ValueError(
f"{state.family.name} is an image-editing model: provide an input image."
)
if mask_image is not None:
- # The edit family has no inpaint pipeline; a mask would be silently dropped.
raise ValueError(
f"{state.family.name} is an image-editing model and does not "
"support masks (mask_image)."
@@ -5946,7 +5863,8 @@ class DiffusionBackend:
pipe = self._workflow_pipe(state, state.family.img2img_pipeline_class, workflow)
init_pil = decode_b64_image(init_image, mode = "RGB")
iw, ih = init_pil.size
- # Cap the factor, then the absolute output (longest side 2048); round to a multiple of 16 (VAE downsample + patch).
+ # Cap the factor, then the absolute output (longest side 2048); round to a multiple of 16 (VAE
+ # downsample + patch).
factor = max(1.0, min(float(upscale), 4.0))
tw_f, th_f = iw * factor, ih * factor
max_side = 2048
@@ -5964,7 +5882,8 @@ class DiffusionBackend:
if strength is None:
strength = 0.35 # hires-fix default: preserve content, add detail
elif getattr(state.family, "reference", False) and init_image is not None:
- # FLUX.2 reference conditioning: the loaded pipe takes the reference via `image` and generates at the REQUESTED size.
+ # FLUX.2 reference conditioning: the loaded pipe takes the reference via `image` and generates at
+ # the REQUESTED size.
workflow = "reference"
init_pil = decode_b64_image(init_image, mode = "RGB")
# Additional references (FLUX.2 combines a list); capped to bound VRAM.
@@ -6005,7 +5924,8 @@ class DiffusionBackend:
"diffusers engine (needs a bf16 or bnb-4bit load of a family with a "
"ControlNet pipeline; not GGUF-via-diffusers or torchao fp8/int8)."
)
- # Decode + preprocess the control image FIRST so a bad image 400s before any CN download, at the OUTPUT size.
+ # Decode + preprocess the control image FIRST so a bad image 400s before any CN download, at the
+ # OUTPUT size.
src = decode_b64_image(cn_image_b64, mode = "RGB")
control_pil = diffusion_controlnet.preprocess_control(src, cn_type).resize(
(width, height), Image.LANCZOS
@@ -6022,20 +5942,22 @@ class DiffusionBackend:
cn_scale, cn_gstart, cn_gend = cn_strength, cn_gs, cn_ge
# Flux Union CN selects its head by an integer control_mode; map the type.
cn_mode = diffusion_controlnet.union_control_mode(cn_id, cn_type)
- # A prompt LIST batches plain text-to-image only: conditioned workflows take one image per call and a silent broadcast would pair every prompt with it.
+ # A prompt LIST batches plain text-to-image only: conditioned workflows take one image per call and a
+ # silent broadcast would pair every prompt with it.
if uniform_prompt(jobs) is None and workflow != "txt2img":
raise ValueError(
"A prompts list is supported for plain text-to-image only; the "
f"{workflow} workflow takes one prompt per call (seed lists still work)."
)
- # Snap odd-sized inputs (and the mask) to a multiple of 16 where the OUTPUT size comes from the input image.
+ # Snap odd-sized inputs (and the mask) to a multiple of 16 where the OUTPUT size comes from the input
+ # image.
if init_pil is not None and workflow in ("img2img", "inpaint", "edit"):
- # img2img/inpaint take output size from the upload, so bound the longest side to 2048 (a phone photo would OOM).
+ # img2img/inpaint take output size from the upload, so bound the longest side to 2048 (a phone photo
+ # would OOM).
if workflow == "img2img":
- # ...and bound Transform by the REQUESTED size too, so the Resolution
- # control caps the output instead of being inert. img2img only: an
- # inpaint payload is the canvas the mask was painted against (Extend
- # sends a deliberately ENLARGED one), so shrinking it would break them.
+ # ...and bound Transform by the REQUESTED size too, so the Resolution control caps the output
+ # instead of being inert. img2img only: an inpaint payload is the canvas the mask was painted
+ # against (Extend sends a deliberately ENLARGED one), so shrinking it would break them.
init_pil = _fit_within(init_pil, min(2048, width), min(2048, height))
elif workflow == "inpaint":
init_pil = _clamp_max_side(init_pil, 2048)
@@ -6046,8 +5968,8 @@ class DiffusionBackend:
if init_pil is not None:
# Keep the VAE encode dtype consistent with the input image.
self._align_vae_dtype(pipe, state.family.denoiser_attr)
- # ...and with the dtype the PIPELINE hands the encoder, which is neither
- # of the two the line above reconciles.
+ # ...and with the dtype the PIPELINE hands the encoder, which is neither of the two the line above
+ # reconciles.
self._make_vae_encode_dtype_safe(pipe)
# Pipelines vary in accepted kwargs, so gate every optional one on the signature.
@@ -6060,14 +5982,16 @@ class DiffusionBackend:
state.family.cfg_kwarg: guidance,
}
if state.family.name == IDEOGRAM4_FAMILY_NAME:
- # Ideogram 4 drives CFG via EITHER a constant guidance_scale OR a per-step guidance_schedule, never both.
- # At the advertised defaults drop the constant so the recommended 48-step taper engages; else null the schedule.
+ # Ideogram 4 drives CFG via EITHER a constant guidance_scale OR a per-step guidance_schedule, never
+ # both. At the advertised defaults drop the constant so the recommended 48-step taper engages; else
+ # null the schedule.
if steps == 48 and abs(float(guidance) - 7.0) < 1e-6:
kwargs.pop(state.family.cfg_kwarg, None)
else:
kwargs["guidance_schedule"] = None
if state.family.name == LUMINA2_FAMILY_NAME and "cfg_trunc_ratio" in call_params:
- # Lumina 2's card recipe truncates the CFG double-forward to the first quarter (cfg_trunc_ratio=0.25); the 1.0 default oversaturates.
+ # Lumina 2's card recipe truncates the CFG double-forward to the first quarter
+ # (cfg_trunc_ratio=0.25); the 1.0 default oversaturates.
kwargs["cfg_trunc_ratio"] = 0.25
if init_pil is not None:
# Reference passes the whole list (FLUX.2 combines); others take the single image.
@@ -6076,7 +6000,8 @@ class DiffusionBackend:
kwargs["mask_image"] = mask_pil
if strength is not None and "strength" in call_params:
kwargs["strength"] = strength
- # width/height: txt2img uses the slider; image-conditioned pipes must use the INPUT IMAGE's size or the latents mismatch, and many drop them, so pass only when accepted.
+ # width/height: txt2img uses the slider; image-conditioned pipes must use the INPUT IMAGE's size or the
+ # latents mismatch, and many drop them, so pass only when accepted.
if workflow in ("txt2img", "reference", "controlnet"):
# These generate at the REQUESTED size (reference/control image resized to match).
kwargs["width"] = width
@@ -6090,7 +6015,8 @@ class DiffusionBackend:
if negative_prompt and "negative_prompt" in call_params:
kwargs["negative_prompt"] = negative_prompt
if workflow == "controlnet" and control_pil is not None:
- # CN pipeline takes the control map + scale; guidance start/end bound its step range. Every kwarg is signature-gated.
+ # CN pipeline takes the control map + scale; guidance start/end bound its step range. Every kwarg is
+ # signature-gated.
if "control_image" in call_params:
kwargs["control_image"] = control_pil
elif "image" in call_params: # some CN pipelines name it "image"
@@ -6105,32 +6031,25 @@ class DiffusionBackend:
if "control_mode" in call_params and cn_mode is not None:
kwargs["control_mode"] = cn_mode
- # Per-forward chunks: the whole job list in ONE forward by default (batch 32 on 4-step models is ~10-22x over
- # serial engines), bounded by an explicit batch_size cap; the OOM backoff below halves a failed chunk.
+ # Per-forward chunks: the whole job list in ONE forward by default (batch 32 on 4-step models is ~10-22x
+ # over serial engines), bounded by an explicit batch_size cap; the OOM backoff below halves a failed
+ # chunk.
chunks = chunk_jobs(jobs, batch_size)
- # Resolution-aware re-check, with the size this call ACTUALLY runs at. The load-time
- # plan budgeted the 1024x1024 default because load time cannot know the request, so
- # a much larger frame has never been compared against anything. Do it here, before
- # any latent is allocated: weights can be offloaded but activations cannot, so an
- # activation estimate over the free budget overruns at EVERY offload tier, and the
- # refusal is a fact rather than a tuning guess. Best-effort throughout -- a probe
- # that fails must never block a generation that would have worked.
try:
- # The dimensions the forward runs at, not the sliders: img2img / inpaint /
- # upscale / edit take their output size from the input image. Same derivation
- # the compile-cache shape registration uses further down.
+ # The dimensions the forward runs at, not the sliders: img2img / inpaint / upscale / edit take their
+ # output size from the input image. Same derivation the compile-cache shape registration uses
+ # further down.
guard_width, guard_height = _compile_shape_dims(
workflow, init_pil, width, height
)
guard_batch = _activation_guard_batch(chunks)
raise_on_image_activation_shortfall(
- # NOT the settled snapshot the load uses: that one calls empty_cache(),
- # which is right once per load but wrong on a per-generation path -- it
- # releases every cached block, so the next forward re-cudaMallocs all of
- # its activations, defeating the caching allocator on every image. This
- # variant credits the same reclaimable bytes back arithmetically instead,
- # so a warm allocator does not read as a full card and nothing is flushed.
+ # NOT the settled snapshot the load uses: that one calls empty_cache(), which is right once per
+ # load but wrong on a per-generation path -- it releases every cached block, so the next forward
+ # re-cudaMallocs all of its activations, defeating the caching allocator on every image. This
+ # variant credits the same reclaimable bytes back arithmetically instead, so a warm allocator
+ # does not read as a full card and nothing is flushed.
device_memory = reclaimable_snapshot_device_memory(
self._state_device_target(state)
),
@@ -6139,21 +6058,22 @@ class DiffusionBackend:
batch_size = guard_batch,
# The hint the load planned with, so the distilled / edit multipliers match.
family = state.variant_hint,
- # img2img is bounded by the Resolution control (see _fit_within), so
- # "generate at a smaller resolution" is actionable there; the rest size
- # from the upload alone and get the upload-side remedy.
+ # img2img is bounded by the Resolution control (see _fit_within), so "generate at a smaller
+ # resolution" is actionable there; the rest size from the upload alone and get the upload-side
+ # remedy.
source_driven = workflow in ("inpaint", "upscale", "edit"),
logger = logger,
)
except ValueError:
raise # the refusal itself: the route turns this into a 400 with the reason
- except Exception as exc: # noqa: BLE001 — fail OPEN on a broken probe
+ except Exception as exc: # noqa: BLE001 - fail OPEN on a broken probe
logger.warning(
"diffusion.generate: activation headroom re-check skipped (%s)", exc
)
gen = _GenState(total_steps = steps * len(chunks))
- # Steps completed by FINISHED chunks, so the bar spans the whole multi-chunk call (mutable cell for _on_step).
+ # Steps completed by FINISHED chunks, so the bar spans the whole multi-chunk call (mutable cell for
+ # _on_step).
steps_done = [0]
def _on_step(pipe, step_index, timestep, callback_kwargs):
@@ -6165,7 +6085,6 @@ class DiffusionBackend:
gen.eta_seconds = _estimate_eta(
gen.total_steps, gen.step, gen.first_step_at, now
)
- # Preempt a long denoise on unload/superseding load (diffusers checks _interrupt).
if cancel.is_set():
pipe._interrupt = True
return callback_kwargs
@@ -6175,7 +6094,8 @@ class DiffusionBackend:
# Re-check an AUTO cache decision against the ACTUAL step count; explicit choices never toggle.
if state.cache_auto:
- # Key on the EFFECTIVE denoise steps: img2img at strength < 1 denoises a fraction of `steps`, so fold it in to keep FBCache off short trajectories.
+ # Key on the EFFECTIVE denoise steps: img2img at strength < 1 denoises a fraction of `steps`, so
+ # fold it in to keep FBCache off short trajectories.
strength_applied = effective_request_strength(
strength,
init_pil is not None,
@@ -6191,7 +6111,8 @@ class DiffusionBackend:
logger = logger,
)
if toggled != state.transformer_cache:
- # _LoadState is frozen; the one deliberate in-place update, so status() reports the pipe-level toggle.
+ # _LoadState is frozen; the one deliberate in-place update, so status() reports the pipe-level
+ # toggle.
object.__setattr__(state, "transformer_cache", toggled)
entry = (state.resolved or {}).get("transformer_cache")
if isinstance(entry, dict):
@@ -6214,17 +6135,16 @@ class DiffusionBackend:
torch.Generator(device = state.device).manual_seed(s) for _, s in chunk
]
if len(jobs) == 1:
- # Single image: scalar prompt + generator, exactly the pre-batching call shape (checked bit-identical).
chunk_kwargs["prompt"] = shared
chunk_kwargs["generator"] = generators[0]
chunk_kwargs["num_images_per_prompt"] = 1
elif shared is not None:
- # Uniform prompt: encode it ONCE and fan out per-image generators.
chunk_kwargs["prompt"] = shared
chunk_kwargs["generator"] = generators
chunk_kwargs["num_images_per_prompt"] = len(chunk)
else:
- # Distinct prompts: one image per prompt in a single forward. The negative prompt must be broadcast to match, else the pipeline asserts or fails in the txt/img concat.
+ # Distinct prompts: one image per prompt in a single forward. The negative prompt must be
+ # broadcast to match, else the pipeline asserts or fails in the txt/img concat.
chunk_kwargs["prompt"] = [p for p, _ in chunk]
chunk_kwargs["generator"] = generators
chunk_kwargs["num_images_per_prompt"] = 1
@@ -6232,14 +6152,15 @@ class DiffusionBackend:
chunk_kwargs["negative_prompt"] = [
chunk_kwargs["negative_prompt"]
] * len(chunk)
- # Start every forward from a clean step cache: diffusers only resets FBCache after a SUCCESSFUL __call__, so a raised call leaves a residual the next forward trips over.
+ # Start every forward from a clean step cache: diffusers only resets FBCache after a SUCCESSFUL
+ # __call__, so a raised call leaves a residual the next forward trips over.
if state.transformer_cache:
self._reset_step_cache(state.pipe)
try:
# inference_mode is faster than no_grad and numerically identical here.
with torch.inference_mode():
out = pipe(**chunk_kwargs).images
- except Exception as exc: # noqa: BLE001 — reraised unless a splittable OOM
+ except Exception as exc: # noqa: BLE001 - reraised unless a splittable OOM
if len(chunk) < 2 or not is_oom_error(exc):
raise
# OOM backoff: halve the failed chunk and retry; per-image seeds keep every retry reproducible.
@@ -6256,17 +6177,18 @@ class DiffusionBackend:
len(second_half),
)
continue
- # A cancelled denoise returns a partial image; don't persist it, nor run remaining chunks.
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
images.extend(out)
per_image_seeds.extend(s for _, s in chunk)
chunk_shapes.append(len(chunk))
steps_done[0] += steps
- # Keep progress ACTIVE through the post-denoise work: the route persists the image after this returns, so a mount probe reading idle would refresh the gallery too early.
- # Persist the warm compile bundle; a STATIC compile makes new artifacts per (w,h,batch), so register this shape.
+ # Keep progress ACTIVE through the post-denoise work: the route persists the image after this returns,
+ # so a mount probe reading idle would refresh the gallery too early. Persist the warm compile bundle; a
+ # STATIC compile makes new artifacts per (w,h,batch), so register this shape.
try:
- # Register the dims the forward ACTUALLY compiled with, and every distinct chunk size (a static compile makes one artifact per batch size too).
+ # Register the dims the forward ACTUALLY compiled with, and every distinct chunk size (a static
+ # compile makes one artifact per batch size too).
reg_width, reg_height = _compile_shape_dims(workflow, init_pil, width, height)
static_shapes = "compiled" in (
state.speed_optims or ()
@@ -6278,17 +6200,15 @@ class DiffusionBackend:
static = static_shapes,
)
compile_cache.save(state.compile_cache_ctx, logger = logger)
- except Exception: # noqa: BLE001 — cache persistence is best-effort
+ except Exception: # noqa: BLE001 - cache persistence is best-effort
pass
- # Last word on cancellation, AFTER the post-denoise work: the event stays
- # registered through the compile-cache save (a static compile writes a fresh
- # artifact per shape, so that save is not instant) and the page still shows Stop
- # for as long as progress reads active, so a Stop landing there was answered
- # cancelled = true and then contradicted by the image the route persisted.
- # Check and deregister under the cancellation lock, which cancel_generate takes, so the
- # two cannot interleave: a cancel that saw this event registered ran strictly
- # before the check, and one that arrives after finds nothing to set and answers
- # false. The finally below repeats the clear for every other exit.
+ # Last word on cancellation, AFTER the post-denoise work: the event stays registered through the
+ # compile-cache save (a static compile writes a fresh artifact per shape, so that save is not instant)
+ # and the page still shows Stop for as long as progress reads active, so a Stop landing there was
+ # answered cancelled = true and then contradicted by the image the route persisted. Check and deregister
+ # under the cancellation lock, which cancel_generate takes, so the two cannot interleave: a cancel that
+ # saw this event registered ran strictly before the check, and one that arrives after finds nothing to
+ # set and answers false. The finally below repeats the clear for every other exit.
with self._generation_cancel_lock:
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
@@ -6296,35 +6216,37 @@ class DiffusionBackend:
self._active_generate_cancel = None
# Count the finished generation (drives deferred speed); a batch is one generation.
object.__setattr__(state, "generation_count", state.generation_count + 1)
- # Return the PIL images unencoded; the route embeds recipes and persists them. ``seeds`` records each image's own seed.
return {
"images": list(images),
"seed": int(seed),
"seeds": [int(s) for s in per_image_seeds],
"repo_id": state.repo_id,
- # The BUILD this ran on, not just the repo id: a GGUF quant and a torchao scheme each change the pixels.
+ # The BUILD this ran on, not just the repo id: a GGUF quant and a torchao scheme each change the
+ # pixels.
"model_kind": state.kind,
"gguf_filename": state.gguf_filename,
"transformer_quant": state.transformer_quant,
- # Read off the ENGAGED state, never the request: a recipe that claimed a
- # precision the load declined is the same lie the status badges just fixed.
+ # Read off the ENGAGED state, never the request: a recipe that claimed a precision the load declined
+ # is the same lie the status badges just fixed.
"text_encoder_quant": state.text_encoder_quant,
"memory_mode": state.memory_mode,
"offload_policy": state.offload_policy,
- # Adapters baked in at LOAD time: disabling them at generate time is not the same build, so they belong to the build record.
+ # Adapters baked in at LOAD time: disabling them at generate time is not the same build, so they
+ # belong to the build record.
"baked_loras": _baked_lora_names(state.pipe),
# The adapters ACTUALLY attached, at non-zero weight, for this generation.
"active_loras": _active_lora_pairs(state.pipe),
- # The workflow this generation ACTUALLY ran, so a conditioned image is not replayed as a plain Create recipe.
+ # The workflow this generation ACTUALLY ran, so a conditioned image is not replayed as a plain
+ # Create recipe.
"workflow": workflow,
}
finally:
- # Deregister so a later unload/load can't poke a finished generation (if still ours).
with self._generation_cancel_lock:
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
with self._lock:
- # Sole clear of the published progress state, on every exit, so a crashed generation never leaves the UI stuck.
+ # Sole clear of the published progress state, on every exit, so a crashed generation never leaves
+ # the UI stuck.
self._gen = None
def generate_progress(self) -> dict[str, Any]:
@@ -6364,7 +6286,6 @@ class DiffusionBackend:
active.set()
return True
if self._generation_owns_slot:
- # Images are committed; do not report a cancellation after the last-word check.
return False
# Only teardown or transition ownership makes queued waiters cancellable.
if not self._teardown_waiters and not self._transition_owns_slot:
@@ -6379,17 +6300,16 @@ class DiffusionBackend:
def unload(self) -> dict[str, Any]:
with self._lock:
- # Abort an in-flight (lock-free) download so unload returns promptly. Under the lock, like video.py: begin_load
- # rebinds this attribute, so an unlocked read could set an event the current load no longer watches.
+ # Abort an in-flight (lock-free) download so unload returns promptly. Under the lock, like video.py:
+ # begin_load rebinds this attribute, so an unlocked read could set an event the current load no longer
+ # watches.
self._cancel_event.set()
- # Abort an in-flight denoise via ITS cancel event.
with self._generation_cancel_lock:
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
- # Fence queued generations too: they are intentionally not cancelled by model
- # lifecycle changes, so they must wait and observe the post-teardown state.
+ # Fence queued generations too: they are intentionally not cancelled by model lifecycle changes, so they
+ # must wait and observe the post-teardown state.
self._reserve_teardown_locked()
- # Cancel any in-flight load (its worker checks this token) and drop the marker.
self._load_token += 1
self._loading = None
# Wait for the signalled denoise to exit BEFORE tearing down: _unload_locked uninstalls process-wide state
@@ -6399,8 +6319,9 @@ class DiffusionBackend:
try:
self._unload_locked()
finally:
- # Released in a finally, exactly like begin_load: _unload_locked ends in clear_gpu_cache(), which raises on a
- # sticky CUDA fault, and an un-drained fence would refuse every later generation for the life of the process.
+ # Released in a finally, exactly like begin_load: _unload_locked ends in clear_gpu_cache(), which
+ # raises on a sticky CUDA fault, and an un-drained fence would refuse every later generation for the
+ # life of the process.
self._release_teardown_locked()
return self.status()
@@ -6408,7 +6329,8 @@ class DiffusionBackend:
state = self._state
if state is None:
return
- # Restore the process-wide backend flags this load flipped so the next `off` load is bit-identical. All idempotent.
+ # Restore the process-wide backend flags this load flipped so the next `off` load is bit-identical. All
+ # idempotent.
restore_backend_flags(state.backend_flags_before)
compile_cache.restore(state.compile_cache_ctx)
gguf_compile.uninstall_all()
@@ -6419,10 +6341,9 @@ class DiffusionBackend:
uninstall_patches()
uninstall_arch_patches()
- # Deliberately NOT unload_lora_weights(): the whole pipe is dropped below, freeing any adapters with it.
- # Drop the workflow pipes so they do not pin the freed pipeline modules past unload.
+ # Deliberately NOT unload_lora_weights(): the whole pipe is dropped below, freeing any adapters with it. Drop
+ # the workflow pipes so they do not pin the freed pipeline modules past unload.
self._aux_pipes.clear()
- # Drop any ControlNet models + pipelines so the freed load carries no extra modules.
self._cn_pipes.clear()
self._cn_models.clear()
self._state = None
@@ -6542,9 +6463,9 @@ def _resolve_base_repo(
if not base:
tag = _hf_base_model(repo_id, hf_token)
if tag and _is_trusted_diffusion_repo(tag):
- # A card tag can now name a mirror (they are real unsloth/* repos) and clear the trust
- # bar. Map it back: this becomes status()["base_repo"] and a trained adapter's default
- # base_model, both of which must stay the vendor id. An EXPLICIT base_repo is verbatim.
+ # A card tag can now name a mirror (they are real unsloth/* repos) and clear the trust bar. Map it back:
+ # this becomes status()["base_repo"] and a trained adapter's default base_model, both of which must stay the
+ # vendor id. An EXPLICIT base_repo is verbatim.
base = canonical_base(tag)
# Returns the UPSTREAM id; the swap happens at the fetch sites only.
resolved = resolve_base_repo(fam, base)
@@ -6577,7 +6498,7 @@ def _hf_base_model(repo_id: str, hf_token: Optional[str]) -> Optional[str]:
try:
from huggingface_hub import HfApi
meta = HfApi().model_info(repo_id, token = hf_token).cardData or {}
- except Exception: # noqa: BLE001 — best-effort; fall back to the family default
+ except Exception: # noqa: BLE001 - best-effort; fall back to the family default
return None
base = meta.get("base_model")
if isinstance(base, list):
@@ -6607,7 +6528,7 @@ def _offload_controlnet_module(cn_model: Any, device: str, logger: Any) -> bool:
use_stream = onload.type == "cuda",
)
return True
- except Exception as exc: # noqa: BLE001 — offload is best-effort; resident is the fallback
+ except Exception as exc: # noqa: BLE001 - offload is best-effort; resident is the fallback
if logger is not None:
logger.warning("diffusion.controlnet: group offload failed (%s); loading resident", exc)
return False
@@ -6624,13 +6545,12 @@ def _base_file_downloaded(rfilename: str, *, include_transformer: bool = False)
dense transformer-quant path will fetch them anyway (see
``_dense_quant_prefetch_needed``)."""
if rfilename == "transformer/config.json":
- # The one transformer/ file that is ALWAYS needed, shards or not: from_single_file(config =
- # <repo id>, subfolder = "transformer") resolves it through the Hub, so a load that promised
- # to download nothing cannot keep that promise unless this ~1 KB file is already on disk.
- # Excluding it made the locality gate pass and the load fetch it afterwards, AFTER the
- # resident pipeline was evicted. Counting it here is what lets the gate refuse up front
- # (cleanly, before eviction) or clear a pick that really can load offline. Video keeps the
- # same exception at video.py's snapshot filter.
+ # The one transformer/ file that is ALWAYS needed, shards or not: from_single_file(config = <repo id>, subfolder
+ # = "transformer") resolves it through the Hub, so a load that promised to download nothing cannot keep that
+ # promise unless this ~1 KB file is already on disk. Excluding it made the locality gate pass and the load fetch
+ # it afterwards, AFTER the resident pipeline was evicted. Counting it here is what lets the gate refuse up front
+ # (cleanly, before eviction) or clear a pick that really can load offline. Video keeps the same exception at
+ # video.py's snapshot filter.
return True
if rfilename.startswith("transformer/"):
return include_transformer
@@ -6639,7 +6559,8 @@ def _base_file_downloaded(rfilename: str, *, include_transformer: bool = False)
return not rfilename.startswith("assets/")
-# Weights the base repo need NOT supply when the single file is the whole pipeline (SDXL): from_single_file(config=base) reads only the structure.
+# Weights the base repo need NOT supply when the single file is the whole pipeline (SDXL): from_single_file(config=base)
+# reads only the structure.
_BASE_WEIGHT_EXTS = (
".safetensors",
".bin",
diff --git a/studio/backend/core/inference/diffusion_arch_patches.py b/studio/backend/core/inference/diffusion_arch_patches.py
index 928dcb319e0..db6143385ac 100644
--- a/studio/backend/core/inference/diffusion_arch_patches.py
+++ b/studio/backend/core/inference/diffusion_arch_patches.py
@@ -179,7 +179,9 @@ def _spec_zimage_forward():
return (cls, "forward", _zimage_forward)
-# flux.1: FluxTransformerBlock / FluxSingleTransformerBlock. Block modulation goes via AdaLayerNormZero; here we fuse the inline norm2 modulation and the gated residual adds.
+# flux.1: block modulation goes via AdaLayerNormZero
+# flux.1: FluxTransformerBlock / FluxSingleTransformerBlock. Block modulation goes via AdaLayerNormZero; here we fuse
+# the inline norm2 modulation and the gated residual adds.
def _flux_double_forward(
self,
hidden_states,
@@ -305,7 +307,9 @@ def _spec_flux_single():
return (cls, "forward", _flux_single_forward)
-# flux.2-klein: Flux2TransformerBlock / Flux2SingleTransformerBlock. Modulation is INLINE, so fuse both; scale/shift/gate are [B,1,dim], so no [:, None].
+# flux.2-klein: modulation is INLINE, and scale/shift/gate are [B,1,dim], so no [:, None]
+# flux.2-klein: Flux2TransformerBlock / Flux2SingleTransformerBlock. Modulation is INLINE, so fuse both;
+# scale/shift/gate are [B,1,dim], so no [:, None].
def _flux2_double_forward(
self,
hidden_states,
@@ -482,6 +486,7 @@ def _spec_krea2_forward():
return (cls, "forward", _krea2_block_forward)
+# each entry is a zero-arg resolver returning (cls, attr, new_fn) or None; all COMPILE-SAFE
# registry + lifecycle. Each entry is a zero-arg resolver returning (cls, attr, new_fn) or None. All COMPILE-SAFE.
_SPECS: tuple[Callable[[], Optional[tuple]], ...] = (
_spec_qwen_modulate,
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 2808f3fcce7..58350c1bf39 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -67,10 +67,11 @@ def normalize_attention_backend(value: Optional[str]) -> Optional[str]:
return normalized
-# Backends diffusers validates by package at set time but whose kernels need a specific CUDA arch at run time. Gate by a (min, max-exclusive) capability range: FA3 is Hopper-SM90 only, FA4 is Blackwell+.
+# Backends diffusers validates by package at set time but whose kernels need a specific CUDA arch at run time. Gate by a
+# (min, max-exclusive) capability range: FA3 is Hopper-SM90 only, FA4 is Blackwell+.
_ARCH_CAPABILITY: dict[str, tuple[tuple[int, int], Optional[tuple[int, int]]]] = {
- "_flash_3_hub": ((9, 0), (10, 0)), # FlashAttention 3 -> Hopper (SM90) only
- "flash_4_hub": ((10, 0), None), # FlashAttention 4 -> Blackwell (SM100)+
+ "_flash_3_hub": ((9, 0), (10, 0)), # Hopper (SM90) only
+ "flash_4_hub": ((10, 0), None), # Blackwell (SM100)+
}
@@ -105,24 +106,25 @@ def _is_cuda_nvidia(target: Any) -> bool:
try:
import torch
- # Shared with the stub installer: torch.version.hip alone misreads AMD wheels that only tag
- # __version__, dropping aiter and pointing cuDNN/xformers (stubbed there) at a ROCm card.
+ # torch.version.hip alone misreads AMD wheels that only tag __version__, dropping aiter and pointing
+ # cuDNN/xformers at a ROCm card
+ # Shared with the stub installer: torch.version.hip alone misreads AMD wheels that only tag __version__,
+ # dropping aiter and pointing cuDNN/xformers (stubbed there) at a ROCm card.
from core._torchao_stub import _module_is_rocm
return not _module_is_rocm(torch)
except Exception: # noqa: BLE001
return False
+# torch's flash_sdp_enabled() / mem_efficient_sdp_enabled() report the USER TOGGLE (#8225)
+# ── what the native SDPA dispatch can actually run (#8225) ─────────────────── torch's ``flash_sdp_enabled()`` /
+# ``mem_efficient_sdp_enabled()`` report the USER TOGGLE, not whether a kernel exists for this device. On the ROCm build
+# in #8225 (gfx1200, torch 2.11+rocm7) both answer True while every dispatch to them raises "No available kernel.
+# Aborting execution.", so the dispatcher degrades silently to MATH -- and MATH is the one backend that materialises the
+# whole B x heads x N x N score matrix. That is how a 3.4 GB Q4_K_M video model asked a 16 GB card for a single 66.54
+# GiB allocation 70 seconds into a generation. So do not read the flags. Run one tiny attention per backend and record
+# what happens.
# ── what the native SDPA dispatch can actually run (#8225) ───────────────────
-#
-# torch's ``flash_sdp_enabled()`` / ``mem_efficient_sdp_enabled()`` report the USER TOGGLE, not
-# whether a kernel exists for this device. On the ROCm build in #8225 (gfx1200, torch 2.11+rocm7)
-# both answer True while every dispatch to them raises "No available kernel. Aborting execution.",
-# so the dispatcher degrades silently to MATH -- and MATH is the one backend that materialises the
-# whole B x heads x N x N score matrix. That is how a 3.4 GB Q4_K_M video model asked a 16 GB card
-# for a single 66.54 GiB allocation 70 seconds into a generation.
-#
-# So do not read the flags. Run one tiny attention per backend and record what happens.
SDPA_FLASH = "flash"
SDPA_MEM_EFFICIENT = "mem_efficient"
SDPA_CUDNN = "cudnn"
@@ -132,8 +134,9 @@ SDPA_MATH = "math"
_SDPA_SUBQUADRATIC = (SDPA_FLASH, SDPA_MEM_EFFICIENT, SDPA_CUDNN)
_SDPA_PROBE_LOCK = threading.Lock()
-# (device type, dtype name) -> the kernels that ran. A kernel cannot appear or vanish under a
-# running interpreter, so one probe per device/dtype for the life of the process.
+# a kernel cannot appear or vanish under a running interpreter
+# (device type, dtype name) -> the kernels that ran. A kernel cannot appear or vanish under a running interpreter, so
+# one probe per device/dtype for the life of the process.
_SDPA_PROBE_CACHE: dict[tuple[str, str], tuple[str, ...]] = {}
@@ -148,8 +151,8 @@ def _probe_sdpa_kernels(device: str, dtype: Any) -> tuple[str, ...]:
(SDPA_CUDNN, getattr(SDPBackend, "CUDNN_ATTENTION", None)),
(SDPA_MATH, getattr(SDPBackend, "MATH", None)),
)
- # Small enough to be free, but shaped like real attention: the fused kernels reject head_dim
- # they cannot serve, and a degenerate 1-element tensor would not exercise that.
+ # Small enough to be free, but shaped like real attention: the fused kernels reject head_dim they cannot serve, and
+ # a degenerate 1-element tensor would not exercise that.
q = torch.zeros((1, 2, 8, 64), device = device, dtype = dtype)
available: list[str] = []
for name, backend in candidates:
@@ -159,7 +162,7 @@ def _probe_sdpa_kernels(device: str, dtype: Any) -> tuple[str, ...]:
with sdpa_kernel([backend]):
torch.nn.functional.scaled_dot_product_attention(q, q, q)
available.append(name)
- except Exception: # noqa: BLE001 — "No available kernel" is the answer, not an error
+ except Exception: # noqa: BLE001 - "No available kernel" is the answer, not an error
continue
return tuple(available)
@@ -177,8 +180,8 @@ def available_sdpa_kernels(target: Any) -> tuple[str, ...]:
try:
import torch
- # fp16 rather than fp32: the fused kernels are half-precision only, so probing at fp32
- # would report "math only" on hardware where flash is perfectly healthy.
+ # fp16 rather than fp32: the fused kernels are half-precision only, so probing at fp32 would report "math
+ # only" on hardware where flash is perfectly healthy.
dtype = torch.float16
except Exception: # noqa: BLE001
return ()
@@ -189,13 +192,10 @@ def available_sdpa_kernels(target: Any) -> tuple[str, ...]:
return cached
try:
available = _probe_sdpa_kernels(device, dtype)
- except Exception: # noqa: BLE001 — a probe is a diagnostic; it may never fail a load
+ except Exception: # noqa: BLE001 - a probe is a diagnostic; it may never fail a load
available = ()
- # Memoize only an ANSWER. The "kernels cannot change under a running interpreter" argument
- # justifies caching what the probe found, not caching its failure to run: an empty result means
- # the probe itself could not complete (a transient allocator failure while the device was full
- # -- exactly when this warning matters most), and caching that would disable the warning for
- # the rest of the process even after memory frees.
+ # memoize only an ANSWER: an empty result means the probe itself could not complete (a transient allocator failure
+ # while the device was full, exactly when this warning matters most)
if not available:
return ()
with _SDPA_PROBE_LOCK:
@@ -252,7 +252,8 @@ def select_attention_backend(
backend = _ALIASES[alias]
if backend == "native":
return None
- # AITER is the AMD ROCm kernel: honor it on a ROCm target, else the NVIDIA-only guard below drops the one backend that works there.
+ # AITER is the AMD ROCm kernel: honor it on a ROCm target, else the NVIDIA-only guard below drops the one
+ # backend that works there
if backend == "aiter":
if getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target):
return backend
@@ -267,7 +268,6 @@ def select_attention_backend(
if backend == "_native_cudnn" and not _cudnn_attention_supported():
return None
return backend
- # auto
if speed_active and _is_cuda_nvidia(target) and _cudnn_attention_supported():
return "_native_cudnn"
return None
@@ -280,28 +280,33 @@ def _cudnn_attention_supported() -> bool:
return have is None or have >= (8, 0)
-# Optional kernels installable on demand: dispatcher name -> (probe module, pip package). Wheels only (--only-binary=:all:), since a source build needs a CUDA toolchain the host may lack.
+# dispatcher name -> (probe module, pip package)
+# Optional kernels installable on demand: dispatcher name -> (probe module, pip package). Wheels only
+# (--only-binary=:all:), since a source build needs a CUDA toolchain the host may lack.
_INSTALLABLE_BACKENDS: dict[str, tuple[str, str]] = {
"sage": ("sageattention", "sageattention>=2.1.1"),
"flash": ("flash_attn", "flash-attn"),
"_flash_3_hub": ("kernels", "kernels"), # FA3/FA4 from the HF kernels hub
"flash_4_hub": ("kernels", "kernels"),
- # Never handed to pip as a name -- see _MATCHED_WHEEL_BACKENDS below; the package
- # string survives only for logging and for the _INSTALL_ATTEMPTED bookkeeping.
+ # Never handed to pip as a name -- see _MATCHED_WHEEL_BACKENDS below; the package string survives only for logging
+ # and for the _INSTALL_ATTEMPTED bookkeeping.
"xformers": ("xformers", "xformers"),
}
-# On-demand install gate (mirrors UNSLOTH_DIFFUSION_SD_CPP_INSTALL): auto (default) / 1 installs a missing package when a gated backend is requested; 0 never installs and falls back to native.
+# on-demand install gate: auto (default) / 1 installs a missing package when a gated backend is requested
+# On-demand install gate (mirrors UNSLOTH_DIFFUSION_SD_CPP_INSTALL): auto (default) / 1 installs a missing package when
+# a gated backend is requested; 0 never installs and falls back to native.
_ATTENTION_INSTALL_ENV = "UNSLOTH_DIFFUSION_ATTENTION_INSTALL"
-# Packages a pip install was already attempted for in THIS process. The loader pre-installs outside its locks, so a recorded attempt stops apply re-running the 600s install under _generate_lock.
+# the loader pre-installs outside its locks
+# Packages a pip install was already attempted for in THIS process. The loader pre-installs outside its locks, so a
+# recorded attempt stops apply re-running the 600s install under _generate_lock.
_INSTALL_ATTEMPTED: set[str] = set()
-# Backends whose wheel must be resolved against the RUNNING torch build instead of handed
-# to pip as a name. Only DETERMINISTIC answers are memoised (a URL, or a refusal that
-# depends purely on the resident torch, which cannot change under a running interpreter).
-# A probe timeout is transient and is deliberately NOT cached: caching it would turn one
-# loaded-machine hiccup into "no xFormers for the rest of this Unsloth session".
+# Backends whose wheel must be resolved against the RUNNING torch build instead of handed to pip as a name. Only
+# DETERMINISTIC answers are memoised (a URL, or a refusal that depends purely on the resident torch, which cannot change
+# under a running interpreter). A probe timeout is transient and is NOT cached: caching it would turn one loaded-machine
+# hiccup into "no xFormers for the rest of this Unsloth session".
_MATCHED_WHEEL_BACKENDS = frozenset({"xformers"})
_XFORMERS_WHEEL_TARGET: Optional[tuple[Optional[str], Optional[str]]] = None
_XFORMERS_WHEEL_LOCK = threading.Lock()
@@ -350,16 +355,15 @@ def _xformers_wheel_target() -> tuple[Optional[str], Optional[str]]:
try:
from utils.wheel_utils import probe_torch_wheel_env, xformers_wheel_url
- # include_windows: this is the one resolver that HAS win_amd64 wheels
- # upstream. timeout matches the other probe_torch_wheel_env callers.
+ # include_windows: this is the one resolver that HAS win_amd64 wheels upstream. timeout matches the other
+ # probe_torch_wheel_env callers.
env = probe_torch_wheel_env(timeout = 30, include_windows = True)
except Exception as exc: # noqa: BLE001 -- must never break a model load
return (None, f"the xFormers wheel could not be resolved ({exc})")
if env is None:
- # Ambiguous: a platform wheel_platform_tag() does not name (macOS, Windows on
- # ARM) which is deterministic, or a probe that timed out on a busy box which is
- # transient. Not cached, so the next request can settle it. Linux aarch64 is
- # NOT here -- it gets a platform_tag and so lands on the branch below.
+ # Ambiguous: a platform wheel_platform_tag() does not name (macOS, Windows on ARM) which is deterministic,
+ # or a probe that timed out on a busy box which is transient. Not cached, so the next request can settle it.
+ # Linux aarch64 is NOT here -- it gets a platform_tag and so lands on the branch below.
return (
None,
"torch could not be probed, or this platform has no xFormers wheel "
@@ -367,9 +371,9 @@ def _xformers_wheel_target() -> tuple[Optional[str], Optional[str]]:
)
url = xformers_wheel_url(env)
if url is None:
- # Name the platform. Linux aarch64 reaches here with a perfectly ordinary
- # torch, and reporting only the torch and CUDA would read as "upstream never
- # built this pair" when the truth is "upstream never built it for this arch".
+ # Name the platform. Linux aarch64 reaches here with a perfectly ordinary torch, and reporting only the
+ # torch and CUDA would read as "upstream never built this pair" when the truth is "upstream never built it
+ # for this arch".
target = (
None,
f"no xFormers wheel is published for torch "
@@ -397,13 +401,13 @@ def _pip_requirement(backend: str, package: str) -> str:
from diffusers.models.attention_dispatch import _REQUIRED_SAGE_VERSION as floor
if isinstance(floor, str) and floor.strip():
return f"sageattention>={floor.strip()}"
- except Exception: # noqa: BLE001 — older/newer diffusers may not expose it; keep the static pin
+ except Exception: # noqa: BLE001 - older/newer diffusers may not expose it; keep the static pin
pass
return package
-# The huggingface_hub floor the current `kernels` wheels declare (kernels >= 0.14.1 requires
-# huggingface-hub >= 1.10.0). A (major, minor) pair, compared against the resident hub below.
+# The huggingface_hub floor the current `kernels` wheels declare (kernels >= 0.14.1 requires huggingface-hub >= 1.10.0).
+# A (major, minor) pair, compared against the resident hub below.
_KERNELS_HUB_FLOOR = (1, 10)
@@ -432,7 +436,7 @@ def _kernels_hub_compatible() -> bool:
if m is None:
return True
return (int(m.group(1)), int(m.group(2) or 0)) >= _KERNELS_HUB_FLOOR
- except Exception: # noqa: BLE001 — unknown hub -> keep the previous permissive behaviour
+ except Exception: # noqa: BLE001 - unknown hub -> keep the previous permissive behaviour
return True
@@ -459,10 +463,9 @@ def _ensure_attention_backend_installed(backend: str, logger: Any = None) -> Opt
gate = os.environ.get(_ATTENTION_INSTALL_ENV, "auto").strip().lower()
if gate in ("0", "false", "no", "off"):
return None
- # Refusing is a POLICY decision, not a failed attempt, so it is checked before the
- # _INSTALL_ATTEMPTED memo below and records nothing: a later request on a fixed environment
- # must still be able to install. Scoped to kernels; the sage / flash-attn / xformers wheels
- # do not import huggingface_hub at module scope.
+ # Refusing is a POLICY decision, not a failed attempt, so it is checked before the _INSTALL_ATTEMPTED memo below and
+ # records nothing: a later request on a fixed environment must still be able to install. Scoped to kernels; the sage
+ # / flash-attn / xformers wheels do not import huggingface_hub at module scope.
if package == "kernels" and not _kernels_hub_compatible():
if logger is not None:
logger.warning(
@@ -475,23 +478,20 @@ def _ensure_attention_backend_installed(backend: str, logger: Any = None) -> Opt
return "the resident huggingface_hub is too old for the kernels package"
try:
if importlib.util.find_spec(module) is not None:
- # Present is present, including a MISMATCHED xformers: find_spec sees the
- # package, so nothing below runs and the wrong-CUDA build stays. That is
- # deliberate here. Repairing means reinstalling a package the user may have
- # built or pinned on purpose, and this can run under _generate_lock, so a
- # 100 MB download would block unload and cancel. install.ps1 is where the
- # repair belongs -- it compares cpp_lib.json against the resident torch and
- # passes --reinstall-package, outside any request. What this branch prevents
- # is Unsloth CREATING the mismatch, which is how it got made in the first place.
+ # Present is present, including a MISMATCHED xformers: find_spec sees the package, so nothing below runs and
+ # the wrong-CUDA build stays. That is deliberate here. Repairing means reinstalling a package the user may
+ # have built or pinned, and this can run under _generate_lock, so a 100 MB download would block unload and
+ # cancel. install.ps1 is where the repair belongs -- it compares cpp_lib.json against the resident torch and
+ # passes --reinstall-package, outside any request. What this branch prevents is Unsloth CREATING the
+ # mismatch, which is how it got made in the first place.
return None
- except Exception: # noqa: BLE001 — a broken install probes as missing; try the install
+ except Exception: # noqa: BLE001 - a broken install probes as missing; try the install
pass
- # xFormers ships a compiled extension tied to one exact (torch, CUDA) pair, so the name
- # `xformers` is not a safe thing to hand pip: PyPI serves only the CUDA-12.8 build and
- # --no-deps below deliberately stops pip from ever reading its `Requires-Dist: torch==X`.
- # Resolve the matching wheel URL instead, and REFUSE when there is none -- like the
- # kernels gate above this is policy, so it is checked before the _INSTALL_ATTEMPTED memo
- # and records nothing there (a refused backend never burns its one install attempt).
+ # XFormers ships a compiled extension tied to one exact (torch, CUDA) pair, so the name `xformers` is not a safe
+ # thing to hand pip: PyPI serves only the CUDA-12.8 build and --no-deps below stops pip from ever reading its
+ # `Requires-Dist: torch==X`. Resolve the matching wheel URL instead, and REFUSE when there is none -- like the
+ # kernels gate above this is policy, so it is checked before the _INSTALL_ATTEMPTED memo and records nothing there
+ # (a refused backend never burns its one install attempt).
if backend in _MATCHED_WHEEL_BACKENDS:
wheel_url, refusal = _xformers_wheel_target()
if wheel_url is None:
@@ -507,11 +507,13 @@ def _ensure_attention_backend_installed(backend: str, logger: Any = None) -> Opt
)
return refusal
package = wheel_url
- # What pip gets and what the log gets are not the same string. UNSLOTH_PYTORCH_MIRROR may
- # carry userinfo or a token for a private index, and it is baked into the wheel URL, so
- # logging the URL verbatim writes that secret into the backend log.
+ # UNSLOTH_PYTORCH_MIRROR may carry a token for a private index and is baked into the wheel URL
+ # What pip gets and what the log gets are not the same string. UNSLOTH_PYTORCH_MIRROR may carry userinfo or a token
+ # for a private index, and it is baked into the wheel URL, so logging the URL verbatim writes that secret into the
+ # backend log.
display = _redacted_for_log(package)
- # Attempt each install once per process, else the in-lock apply re-runs it under _generate_lock and blocks unload/cancel.
+ # attempt each install once per process, else the in-lock apply re-runs it under _generate_lock and blocks
+ # unload/cancel
if package in _INSTALL_ATTEMPTED:
return None
_INSTALL_ATTEMPTED.add(package)
@@ -524,7 +526,12 @@ def _ensure_attention_backend_installed(backend: str, logger: Any = None) -> Opt
)
try:
subprocess.run(
- # --no-deps: install ONLY this kernel wheel, since xformers/flash-attn pin an exact torch and normal resolution would replace the running one. It also means pip never reads the wheel's `Requires-Dist: torch==X`, so nothing here would catch an ABI mismatch -- for xformers, whose mismatch is SILENT (the extension fails to load and _cpp_lib.py logs a warning), the URL was resolved against the running torch above precisely so there is nothing left to catch.
+ # --no-deps: install ONLY this kernel wheel
+ # --no-deps: install ONLY this kernel wheel, since xformers/flash-attn pin an exact torch and normal
+ # resolution would replace the running one. It also means pip never reads the wheel's `Requires-Dist:
+ # torch==X`, so nothing here would catch an ABI mismatch -- for xformers, whose mismatch is SILENT (the
+ # extension fails to load and _cpp_lib.py logs a warning), the URL was resolved against the running torch
+ # above precisely so there is nothing left to catch.
[
sys.executable,
"-m",
@@ -539,9 +546,10 @@ def _ensure_attention_backend_installed(backend: str, logger: Any = None) -> Opt
timeout = 600,
check = True,
)
- # The import system caches directory listings, so invalidate the finder caches or the next find_spec can miss the new wheel.
+ # The import system caches directory listings, so invalidate the finder caches or the next find_spec can miss
+ # the new wheel.
importlib.invalidate_caches()
- except Exception as exc: # noqa: BLE001 — no wheel / no network -> native fallback
+ except Exception as exc: # noqa: BLE001 - no wheel / no network -> native fallback
if logger is not None:
# CalledProcessError.str() shows only the exit code; surface stderr so the fallback is diagnosable.
stderr = getattr(exc, "stderr", None)
@@ -551,8 +559,7 @@ def _ensure_attention_backend_installed(backend: str, logger: Any = None) -> Opt
logger.warning(
"diffusion.attention: could not install %s; pip failed with: %s",
display,
- # pip echoes the URL it was given, so redact the body too rather than
- # only the name in front of it.
+ # pip echoes the URL it was given, so redact the body too rather than only the name in front of it.
_redacted_for_log(stderr.strip()) or str(exc),
)
else:
@@ -602,9 +609,9 @@ def apply_attention_backend(
if callable(s)
]
if not setters:
- # A U-Net pipeline (SDXL) exposes no dispatcher setter, so there is no backend to set --
- # but its attention still runs through torch SDPA, so the math-only diagnosis applies
- # exactly as it does to a DiT. Report it here or an SDXL load gets no warning at all.
+ # A U-Net pipeline (SDXL) exposes no dispatcher setter, so there is no backend to set -- but its attention still
+ # runs through torch SDPA, so the math-only diagnosis applies exactly as it does to a DiT. Report it here or an
+ # SDXL load gets no warning at all.
if target is not None:
warn_if_sdpa_math_only(target, logger)
return None
@@ -615,18 +622,20 @@ def apply_attention_backend(
try:
fn(backend)
engaged = True
- except Exception as exc: # noqa: BLE001 — unavailable kernel -> restore native below
+ except Exception as exc: # noqa: BLE001 - unavailable kernel -> restore native below
_warn(logger, backend, exc)
if engaged:
- # set_attention_backend also pins the backend process-wide. Each DiT's processors keep it locally, so reset the global to native ONCE, else a later component inherits this kernel.
+ # set_attention_backend also pins the backend process-wide. Each DiT's processors keep it locally, so reset
+ # the global to native ONCE, else a later component inherits this kernel.
_reset_global_backend_to_native(logger)
if logger is not None:
logger.info("diffusion.attention: backend=%s", backend)
return backend
- # No backend requested, or every set failed: pin native so a stale process-wide backend cannot leak in. One reset covers every fresh DiT.
+ # No backend requested, or every set failed: pin native so a stale process-wide backend cannot leak in. One reset
+ # covers every fresh DiT.
_restore_native_backend(setters[0], logger)
- # Native means torch's SDPA dispatch decides per call, and on a device with no fused kernel
- # that decision is MATH. Say so now; the flags this would otherwise be read off lie (#8225).
+ # Native means torch's SDPA dispatch decides per call, and on a device with no fused kernel that decision is MATH.
+ # Say so now; the flags this would otherwise be read off lie (#8225).
if target is not None:
warn_if_sdpa_math_only(target, logger)
return None
@@ -637,7 +646,7 @@ def _active_attention_backend() -> Optional[str]:
try:
from diffusers.models.attention_dispatch import _AttentionBackendRegistry
- # get_active_backend() returns (AttentionBackendName, fn) or None; read element 0's .value, not the tuple.
+ # get_active_backend() returns (AttentionBackendName, fn) or None; read element 0's .value
active = _AttentionBackendRegistry.get_active_backend()
if active is None:
return None
@@ -659,7 +668,7 @@ def _reset_global_backend_to_native(logger: Any) -> None:
_AttentionBackendRegistry,
)
_AttentionBackendRegistry.set_active_backend(AttentionBackendName.NATIVE)
- except Exception: # noqa: BLE001 — best-effort; leave the global as-is on any change
+ except Exception: # noqa: BLE001 - best-effort; leave the global as-is on any change
pass
@@ -669,7 +678,7 @@ def _restore_native_backend(set_backend_fn: Any, logger: Any) -> None:
return # already native -> avoid redundant work and an extra dispatcher warning
try:
set_backend_fn(ATTN_NATIVE)
- except Exception as exc: # noqa: BLE001 — best-effort restore
+ except Exception as exc: # noqa: BLE001 - best-effort restore
_warn(logger, ATTN_NATIVE, exc)
@@ -678,47 +687,37 @@ def _warn(logger: Any, what: str, exc: Exception) -> None:
logger.warning("diffusion.attention: %s unavailable (%s); using default", what, exc)
-# --------------------------------------------------------------------------------------
-# HunyuanVideo-1.5 joint-attention padding trim (accuracy-exact speed win)
-#
-# HunyuanVideo15AttnProcessor2_0 runs a JOINT [video ; text] self-attention and, on EVERY block
-# and step, materialises a dense [B,1,N,N] boolean mask so the video never attends to padded text.
-# But a dense bool attn_mask costs most of what the fused kernels are for. Established by output
-# identity, not by timing, since dispatch overhead makes timings alone ambiguous: FLASH refuses a
-# non-null mask outright, MATH OOMs on the 75.5 GiB [1,16,N,N] score matrix, and the default
-# dispatch is BITWISE-equal to forced cuDNN (296.11 vs 296.00 ms), so cuDNN is what runs -- on a
-# masked path 20x slower than its own unmasked one. On a B200 at the production shape (N~=50k, 121
-# frames 480p) the SAME attention is 296 ms WITH the dense mask vs 15 ms with attn_mask=None. (An
-# aside worth knowing before optimising here: forced EFFICIENT does the masked attention in 168 ms,
-# so the dispatcher's masked pick is not even the fastest available one.)
-# (torch 2.12; scripts/sdpa_mask_backend_probe.py re-measures it, and also shows MATH OOMing on the
-# 75.5 GiB score matrix and FLASH refusing a dense mask outright). END TO END that is 10.4x: a full
-# 121-frame 832x480 10-step render goes 353.8s -> 33.9s, medians of 3, reproduced across two runs
-# purely to mask padding. And the text is ~99.5% padding: a t2v prompt fills only ~9 of ~1985 slots
-# (image 729 + byt5 256 + mllm 1000, almost all zero-padded).
-#
-# The fix is exact: the model already masks the padded text and DISCARDS its attention output (only
-# the video split feeds proj_out), so removing the padded tokens before attention changes nothing
-# for the video. "Exact" here means no information is discarded, NOT bit-reproducible: swapping
-# masked for fused SDPA perturbs each step at bf16 rounding scale (one DiT forward on identical
-# inputs differs by 6.6e-3 relative, cosine 0.99998) and 10 denoising steps amplify that
-# chaotically, so the finished video is visibly a different sample. That is intrinsic to the kernel
-# change, not to the trim: rendering the SAME dense-mask path under two different exact SDPA kernels
-# diverges MORE (LPIPS 0.303 vs the trim's 0.285, SSIM 0.744 vs 0.767, over 13 sampled frames).
-# Whole-video LPIPS cannot judge a kernel change at this step count; the single-forward relative
-# error is the metric that can.
-# Done in an eager forward pre-hook (outside the compiled blocks): drop the all-zero
-# image stream (t2v), trim the mllm/byt5 streams to their globally-valid columns, and -- when
-# nothing partially-padded remains (the common batch-1 / per-branch call) -- flag the DiT so the
-# processor skips the dense mask and runs the fused path. The only numeric change is the SDPA kernel
-# (masked -> fused), on par with the shipped cuDNN backend swap. Mixed-padding batches fall back to
-# the stock dense mask.
-#
-# SHAPE NOTE: the trimmed text length is prompt-dependent, so the compiled blocks see a new shape
-# per prompt. That is free on the default speed tier (compiled with dynamic=True) but not on ``max``
-# (dynamic=False), where each length is its own graph and a fullgraph region hard-errors once
-# dynamo's recompile limit is reached. The caller therefore only installs the trim on a tier that
+# -------------------------------------------------------------------------------------- HunyuanVideo-1.5
+# joint-attention padding trim (accuracy-exact speed win) HunyuanVideo15AttnProcessor2_0 runs a JOINT [video ; text]
+# self-attention and, on EVERY block and step, materialises a dense [B,1,N,N] boolean mask so the video never attends to
+# padded text. But a dense bool attn_mask costs most of what the fused kernels are for. Established by output identity,
+# not by timing, since dispatch overhead makes timings alone ambiguous: FLASH refuses a non-null mask outright, MATH
+# OOMs on the 75.5 GiB [1,16,N,N] score matrix, and the default dispatch is BITWISE-equal to forced cuDNN (296.11 vs
+# 296.00 ms), so cuDNN is what runs -- on a masked path 20x slower than its own unmasked one. On a B200 at the
+# production shape (N~=50k, 121 frames 480p) the SAME attention is 296 ms WITH the dense mask vs 15 ms with
+# attn_mask=None. (An aside worth knowing before optimising here: forced EFFICIENT does the masked attention in 168 ms,
+# so the dispatcher's masked pick is not even the fastest available one.) (torch 2.12;
+# scripts/sdpa_mask_backend_probe.py re-measures it, and also shows MATH OOMing on the 75.5 GiB score matrix and FLASH
+# refusing a dense mask outright). END TO END that is 10.4x: a full 121-frame 832x480 10-step render goes 353.8s ->
+# 33.9s, medians of 3, reproduced across two runs purely to mask padding. And the text is ~99.5% padding: a t2v prompt
+# fills only ~9 of ~1985 slots (image 729 + byt5 256 + mllm 1000, almost all zero-padded). The fix is exact: the model
+# already masks the padded text and DISCARDS its attention output (only the video split feeds proj_out), so removing the
+# padded tokens before attention changes nothing for the video. "Exact" here means no information is discarded, NOT
+# bit-reproducible: swapping masked for fused SDPA perturbs each step at bf16 rounding scale (one DiT forward on
+# identical inputs differs by 6.6e-3 relative, cosine 0.99998) and 10 denoising steps amplify that chaotically, so the
+# finished video is visibly a different sample. That is intrinsic to the kernel change, not to the trim: rendering the
+# SAME dense-mask path under two different exact SDPA kernels diverges MORE (LPIPS 0.303 vs the trim's 0.285, SSIM 0.744
+# vs 0.767, over 13 sampled frames). Whole-video LPIPS cannot judge a kernel change at this step count; the
+# single-forward relative error is the metric that can. Done in an eager forward pre-hook (outside the compiled blocks):
+# drop the all-zero image stream (t2v), trim the mllm/byt5 streams to their globally-valid columns, and -- when nothing
+# partially-padded remains (the common batch-1 / per-branch call) -- flag the DiT so the processor skips the dense mask
+# and runs the fused path. The only numeric change is the SDPA kernel (masked -> fused), on par with the shipped cuDNN
+# backend swap. Mixed-padding batches fall back to the stock dense mask. SHAPE NOTE: the trimmed text length is
+# prompt-dependent, so the compiled blocks see a new shape per prompt. That is free on the default speed tier (compiled
+# with dynamic=True) but not on ``max`` (dynamic=False), where each length is its own graph and a fullgraph region
+# hard-errors once dynamo's recompile limit is reached. The caller therefore only installs the trim on a tier that
# compiles dynamically; see the call site in video.py.
+# --------------------------------------------------------------------------------------
_HUNYUAN15_TRANSFORMER_CLS = "HunyuanVideo15Transformer3DModel"
_HUNYUAN15_PROCESSOR_CLS = "HunyuanVideo15AttnProcessor2_0"
_NULL_ATTN_FLAG = "_unsloth_null_attn_mask"
@@ -760,8 +759,8 @@ def _null_mask_processor_cls():
attention_mask = None,
image_rotary_emb = None,
):
- # Fast path only when the pre-hook removed all padding (attn_mask redundant); a
- # constant python bool so torch.compile const-folds the branch (no graph break).
+ # Fast path only when the pre-hook removed all padding (attn_mask redundant); a constant python bool so
+ # torch.compile const-folds the branch (no graph break).
if not getattr(attn, _NULL_ATTN_FLAG, False):
return super().__call__(
attn,
@@ -832,9 +831,8 @@ def _null_mask_processor_cls():
if getattr(attn, "to_add_out", None) is not None:
encoder_hidden_states = attn.to_add_out(encoder_hidden_states)
- # Always the 2-tuple, matching the stock processor's return contract (it returns
- # (hidden_states, encoder_hidden_states) outside its own `if`), so the calling block
- # unpacks identically on either path.
+ # Always the 2-tuple, matching the stock processor's return contract (it returns (hidden_states,
+ # encoder_hidden_states) outside its own `if`), so the calling block unpacks identically on either path.
return hidden_states, encoder_hidden_states
_NULL_PROCESSOR_CACHE["cls"] = _HunyuanNullMaskProcessor
@@ -853,8 +851,8 @@ def _trim_stream(states, mask):
states = states[:, keep]
mask = mask[:, keep]
mb = mb[:, keep]
- # All remaining slots valid for every element (vacuously True for a 0-length stream, fine
- # for an unused secondary stream e.g. byt5 in t2v).
+ # All remaining slots valid for every element (vacuously True for a 0-length stream, fine for an unused secondary
+ # stream e.g. byt5 in t2v).
all_valid = bool(mb.all().item())
return states, mask, all_valid
@@ -889,17 +887,18 @@ def _hunyuan_trim_pre_hook(module, args, kwargs):
image = kwargs.get("image_embeds")
if image is not None and image.numel() > 0 and bool(torch.all(image == 0).item()):
- # All-zero image == "no image" (t2v). Emptying the token axis removes the 729 padded
- # image tokens; is_t2v stays True in forward (all() of empty is vacuously True).
+ # All-zero image == "no image" (t2v). Emptying the token axis removes the 729 padded image tokens; is_t2v
+ # stays True in forward (all() of empty is vacuously True).
kwargs["image_embeds"] = image[:, :0]
for skey, mkey, required in (
("encoder_hidden_states", "encoder_attention_mask", True),
("encoder_hidden_states_2", "encoder_attention_mask_2", False),
):
- # Only touch streams passed by keyword (the pipeline always does); never write back an
- # absent key (a positional encoder_hidden_states would collide). An absent REQUIRED
- # primary stream drops the fast path; an absent optional byt5 is fine.
+ # only touch streams passed by keyword; An absent REQUIRED primary stream drops the fast path
+ # Only touch streams passed by keyword (the pipeline always does); never write back an absent key (a
+ # positional encoder_hidden_states would collide). An absent REQUIRED primary stream drops the fast path; an
+ # absent optional byt5 is fine.
if skey not in kwargs:
null_ok = null_ok and not required
continue
@@ -908,9 +907,8 @@ def _hunyuan_trim_pre_hook(module, args, kwargs):
kwargs[mkey] = mask
null_ok = null_ok and all_valid
- # The primary mllm stream flows through the TokenRefiner's own attention, whose pooling
- # divides by the mask sum; never hand it a 0-length sequence (pathological empty prompt).
- # Revert and take the stock dense-mask path.
+ # The primary mllm stream flows through the TokenRefiner's own attention, whose pooling divides by the mask sum;
+ # never hand it a 0-length sequence (pathological empty prompt). Revert and take the stock dense-mask path.
primary = kwargs.get("encoder_hidden_states")
if primary is not None and primary.dim() == 3 and primary.shape[1] == 0:
kwargs.clear()
@@ -919,9 +917,9 @@ def _hunyuan_trim_pre_hook(module, args, kwargs):
_set_hunyuan_null_mask(module, null_ok)
return args, kwargs
- except Exception: # noqa: BLE001 — optimisation only; never break the forward
- # We may have trimmed some kwargs before failing. Restore the caller's untrimmed inputs so
- # the stock dense-mask path (flag False) runs on exactly what it expects.
+ except Exception: # noqa: BLE001 - optimisation only; never break the forward
+ # We may have trimmed some kwargs before failing. Restore the caller's untrimmed inputs so the stock dense-mask
+ # path (flag False) runs on exactly what it expects.
kwargs.clear()
kwargs.update(original)
_set_hunyuan_null_mask(module, False)
@@ -943,7 +941,7 @@ def _install_null_processors(dit: Any, logger: Any) -> bool:
already-installed run is a no-op). Preserves any pinned attention backend."""
try:
cls = _null_mask_processor_cls()
- except Exception as exc: # noqa: BLE001 — diffusers moved / unavailable -> skip
+ except Exception as exc: # noqa: BLE001 - diffusers moved / unavailable -> skip
_warn(logger, "hunyuan_attn_trim", exc)
return False
installed = 0
@@ -953,17 +951,17 @@ def _install_null_processors(dit: Any, logger: Any) -> bool:
if proc is None:
continue
if isinstance(proc, cls):
- installed += 1 # already ours (idempotent)
+ installed += 1
continue
if type(proc).__name__ != _HUNYUAN15_PROCESSOR_CLS:
- continue # unknown processor -> leave it alone
+ continue
new = cls()
# carry over any backend/parallel config the stock processor already held
new._attention_backend = getattr(proc, "_attention_backend", None)
new._parallel_config = getattr(proc, "_parallel_config", None)
try:
attn.set_processor(new)
- except Exception: # noqa: BLE001 — fall back to direct assignment
+ except Exception: # noqa: BLE001 - fall back to direct assignment
attn.processor = new
installed += 1
return installed > 0
@@ -993,18 +991,17 @@ def install_hunyuan_attention_trim(
continue
if not _install_null_processors(dit, logger):
continue
- # Installation (and every idle period between generations) starts in the conservative
- # state: the flag is only ever True inside the exact forward its pre-hook trimmed.
+ # installation and every idle period start in the conservative state
_set_hunyuan_null_mask(dit, False)
if getattr(dit, "_unsloth_trim_hook", None) is None:
pre_handle = None
try:
pre_handle = dit.register_forward_pre_hook(_hunyuan_trim_pre_hook, with_kwargs = True)
- # always_call: clear the flag even when the forward raises, so an exception can
- # never leave the null-mask authorisation latched for a later direct forward.
+ # always_call: clear the flag even when the forward raises, so an exception can never leave the
+ # null-mask authorisation latched for a later direct forward.
post_handle = dit.register_forward_hook(_hunyuan_trim_post_hook, always_call = True)
dit._unsloth_trim_hook = (pre_handle, post_handle)
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
if pre_handle is not None:
pre_handle.remove()
_set_hunyuan_null_mask(dit, False)
diff --git a/studio/backend/core/inference/diffusion_auto_policy.py b/studio/backend/core/inference/diffusion_auto_policy.py
index e9e333ff2ea..a451c8b303e 100644
--- a/studio/backend/core/inference/diffusion_auto_policy.py
+++ b/studio/backend/core/inference/diffusion_auto_policy.py
@@ -27,6 +27,7 @@ from typing import Any, Optional
_MIB_PER_GB = 1000.0**3 / (1024.0 * 1024.0) # component sizes below are decimal GB
+# int8/fp8 store one byte per param plus per-row scales (~0.52x with slack for bf16 norms/embeddings);
# Steady size of a torchao-quantised transformer relative to bf16: int8/fp8 store one byte per param plus per-row scales
# (~0.52x with slack for bf16 norms/embeddings); nvfp4 packs two per byte plus block scales. Measured on live loads.
_QUANT_STEADY_FACTOR: dict[str, float] = {
@@ -36,8 +37,11 @@ _QUANT_STEADY_FACTOR: dict[str, float] = {
"nvfp4": 0.33,
}
-# bf16-RESIDENT component sizes in decimal GB: (transformer, text encoders, VAE). What they occupy on device after the dtype
-# cast, NOT the download size (Z-Image-Turbo ships fp32: 24.6 GB of shards -> 12.3 GB bf16). From HF sibling metadata.
+# what they occupy on device after the dtype cast, NOT the download size (Z-Image-Turbo ships fp32: 24.6 GB of shards ->
+# 12.3 GB bf16).
+# bf16-RESIDENT component sizes in decimal GB: (transformer, text encoders, VAE). What they occupy on device after the
+# dtype cast, NOT the download size (Z-Image-Turbo ships fp32: 24.6 GB of shards -> 12.3 GB bf16). From HF sibling
+# metadata.
_FAMILY_BF16_GB: dict[str, tuple[float, float, float]] = {
"flux.1": (23.8, 9.8, 0.2),
"flux.1-kontext": (23.8, 9.8, 0.2),
@@ -51,28 +55,33 @@ _FAMILY_BF16_GB: dict[str, tuple[float, float, float]] = {
"lumina-2": (5.2, 5.2, 0.2),
# 17B dual-stream DiT (32.5 GB bf16 on disk) + Qwen2.5-VL 15.5 GB + ByT5 0.8 GB.
"hunyuanimage-2.1": (32.5, 16.3, 0.8),
- # 17B MoE DiT (34.2 GB bf16) + FOUR text encoders: CLIP-L 0.5 + CLIP-G 2.8 + T5-XXL 9.5, plus Llama-3.1-8B (~16 GB bf16) from the open mirror at load time.
+ # 17B MoE DiT + four text encoders (CLIP-L, CLIP-G, T5-XXL), plus Llama-3.1-8B from the open mirror at load time
+ # 17B MoE DiT (34.2 GB bf16) + FOUR text encoders: CLIP-L 0.5 + CLIP-G 2.8 + T5-XXL 9.5, plus Llama-3.1-8B (~16 GB
+ # bf16) from the open mirror at load time.
"hidream-i1": (34.2, 28.8, 0.2),
- # Two ~9.3B DiTs (Ideogram's dual-branch CFG), both resident, plus a Qwen3-VL encoder. The vendor stores raw float8, so each doubles at bf16.
+ # Two ~9.3B DiTs (Ideogram's dual-branch CFG), both resident, plus a Qwen3-VL encoder. The vendor stores raw float8,
+ # so each doubles at bf16.
"ideogram-4": (37.2, 16.3, 0.2),
}
-# Hub DOWNLOAD bytes relative to the bf16-resident sizes above, for families whose published checkpoints are not stored at
-# bf16. The free-disk gate needs what lands in the HF cache, which differs by 2x in either direction here. From HF metadata:
-# z-image Turbo transformer/ 23,479 MiB fp32 -> 11,730 MiB resident (2.00x)
-# lumina-2 transformer/ 9,956 MiB fp32 -> 4,959 MiB resident (2.01x)
-# ideogram-4 transformer/ + unconditional_transformer/ 17,718 MiB fp8 -> 35,477 MiB resident (0.50x)
-# Anything absent ships bf16 and downloads what it occupies (measured 0.99-1.07x).
+# Hub DOWNLOAD bytes relative to the resident sizes above, for families not stored at bf16 Hub DOWNLOAD bytes relative
+# to the bf16-resident sizes above, for families whose published checkpoints are not stored at bf16. The free-disk gate
+# needs what lands in the HF cache, which differs by 2x in either direction here. From HF metadata: z-image Turbo
+# transformer/ 23,479 MiB fp32 -> 11,730 MiB resident (2.00x) lumina-2 transformer/ 9,956 MiB fp32 -> 4,959 MiB resident
+# (2.01x) ideogram-4 transformer/ + unconditional_transformer/ 17,718 MiB fp8 -> 35,477 MiB resident (0.50x) Anything
+# absent ships bf16 and downloads what it occupies (measured 0.99-1.07x).
_FAMILY_HUB_DOWNLOAD_FACTOR: dict[str, float] = {
"z-image": 2.0,
"lumina-2": 2.0,
"ideogram-4": 0.5,
}
-# Per-base overrides of the factor above, for a checkpoint stored at a different precision from its
-# family default. Only Tongyi-MAI/Z-Image needs one: the family key exists because the distilled
-# Turbo publishes fp32 (23,479 MiB), while the undistilled base ships bf16 (11,740 MiB) and so
-# downloads exactly what it occupies. Without it the free-disk gate demands twice the real size.
+# only Tongyi-MAI/Z-Image needs one: the distilled Turbo publishes fp32 while the undistilled base ships bf16 and
+# downloads exactly what it occupies
+# Per-base overrides of the factor above, for a checkpoint stored at a different precision from its family default. Only
+# Tongyi-MAI/Z-Image needs one: the family key exists because the distilled Turbo publishes fp32 (23,479 MiB), while the
+# undistilled base ships bf16 (11,740 MiB) and so downloads exactly what it occupies. Without it the free-disk gate
+# demands twice the real size.
_BASE_REPO_HUB_DOWNLOAD_FACTOR: dict[str, float] = {
"tongyi-mai/z-image": 1.0,
}
@@ -86,9 +95,9 @@ _BASE_RESIDENT_FACTORS: dict = {
"tongyi-mai/z-image": (1.0, 1.0),
"alpha-vllm/lumina-image-2.0": (2.0, 2.0),
"ideogram-ai/ideogram-4-fp8": (0.5, 0.5),
- # SDXL's default variant is fp32 throughout (headers read 2026-08-25); the loader skips the
- # fp16 twins. At 1:1 its 12.9 GB prices against a 6.5 GB bf16 load and refuses on any 16 GB
- # pool. Its denoiser is ``unet/``, so it lands in the companion bucket: both factors halve.
+ # SDXL's default variant is fp32 throughout (headers read 2026-08-25); the loader skips the fp16 twins. At 1:1 its
+ # 12.9 GB prices against a 6.5 GB bf16 load and refuses on any 16 GB pool. Its denoiser is ``unet/``, so it lands in
+ # the companion bucket: both factors halve.
"stabilityai/stable-diffusion-xl-base-1.0": (2.0, 2.0),
"stabilityai/sdxl-turbo": (2.0, 2.0),
}
@@ -149,11 +158,12 @@ def hub_download_factor(fam: Any, base_repo: Optional[str] = None) -> float:
return _FAMILY_HUB_DOWNLOAD_FACTOR.get(getattr(fam, "name", None), 1.0)
+# flux.2-klein ships FOUR checkpoints under one entry
# Base-repo overrides for families offering multiple sizes under one entry (the table carries the family default).
-# flux.2-klein ships FOUR checkpoints under one entry: 4B / base-4B (the family default, Qwen3-4B encoder) and
-# 9B / base-9B (18.2 GB transformer, Qwen3-8B encoder). The 9B pair needs an override on BOTH ids: sizing
-# klein-BASE-9B off the family default understates it by 2.3x, and the base variants are the ones the upstream
-# guidance points fine-tuning at, so it is the likelier of the two to be loaded.
+# flux.2-klein ships FOUR checkpoints under one entry: 4B / base-4B (the family default, Qwen3-4B encoder) and 9B /
+# base-9B (18.2 GB transformer, Qwen3-8B encoder). The 9B pair needs an override on BOTH ids: sizing klein-BASE-9B off
+# the family default understates it by 2.3x, and the base variants are the ones the upstream guidance points fine-tuning
+# at, so it is the likelier of the two to be loaded.
_BASE_REPO_BF16_GB: dict[str, tuple[float, float, float]] = {
"black-forest-labs/flux.2-klein-9b": (18.2, 16.4, 0.2),
"black-forest-labs/flux.2-klein-base-9b": (18.2, 16.4, 0.2),
@@ -200,10 +210,11 @@ class DenseQuantEstimate:
companions_mib: int
prequant: bool
download_transformer_mib: int = 0
- # The TEXT-ENCODER share of ``companions_mib``. The memory planner needs it to price the
- # group tier that streams the encoders instead of keeping them resident, and this table is
- # the only place the split exists on the dense-candidate path. Defaulted so any construction
- # that predates it still works (the planner reads a 0 split as "no split", i.e. no new tier).
+ # the memory planner needs the text-encoder share to price the group tier that streams the encoders;
+ # The TEXT-ENCODER share of ``companions_mib``. The memory planner needs it to price the group tier that streams the
+ # encoders instead of keeping them resident, and this table is the only place the split exists on the
+ # dense-candidate path. Defaulted so any construction that predates it still works (the planner reads a 0 split as
+ # "no split", i.e. no new tier).
text_encoders_mib: int = 0
@property
@@ -240,8 +251,8 @@ def estimate_dense_quant(
companions_mib = companions,
prequant = prequant_available,
download_transformer_mib = int(transformer_gb * hub_factor * _MIB_PER_GB),
- # Same conversion as `companions`, of which this is the text-encoder half, so the planner's
- # `companions - text_encoders` is the VAE and nothing else.
+ # Same conversion as `companions`, of which this is the text-encoder half, so the planner's `companions -
+ # text_encoders` is the VAE and nothing else.
text_encoders_mib = int(text_encoders_gb * _MIB_PER_GB),
)
@@ -295,29 +306,30 @@ def resolve_dense_quant_candidate(
return None
prequant_available = False
prequant_cached = False
- # force_dense: the loader will SKIP the prequant shortcut (e.g. a LoRA bake), so size the candidate for the dense build.
+ # force_dense: the loader will SKIP the prequant shortcut (e.g. a LoRA bake), so size the candidate for the dense
+ # build.
if not force_dense:
try:
from .diffusion_prequant import prequant_checkpoint_cached, usable_prequant_source
- # usable_ (not resolve_): a local path override counts only when the loader will accept it (allowlisted AND present), else it rebuilds dense after eviction.
+ # usable_ (not resolve_): a local path override counts only when the loader will accept it (allowlisted AND
+ # present), else it rebuilds dense after eviction.
src = usable_prequant_source(
fam, scheme, path_override = prequant_path, base_repo = base_repo
)
prequant_available = src is not None
if src is not None and getattr(src, "kind", None) == "path":
- # A local override is the operator's own file on disk: it downloads nothing, so the
- # space gate has no claim on it. prequant_checkpoint_cached only answers for hosted
- # repos (_cached_in_root returns None for any other kind), so asking it here would
- # report False and re-apply the gate to a file that costs no bytes, which is the
- # opposite of what the retry assumes about local paths.
+ # A local override is the operator's own file on disk: it downloads nothing, so the space gate has no
+ # claim on it. prequant_checkpoint_cached only answers for hosted repos (_cached_in_root returns None
+ # for any other kind), so asking it here would report False and re-apply the gate to a file that costs
+ # no bytes, which is the opposite of what the retry assumes about local paths.
prequant_cached = True
elif src is not None:
- # Pin the ACTIVE root, as the retry and the loader both do. Unpinned,
- # cached_checkpoint_path searches only huggingface_hub's import-time constant, so
- # after a cache-folder change the retry proves the checkpoint cached in the live
- # root and this would still call it uncached and re-apply the gate. Imported from
- # utils rather than diffusion.hub_cache_dir, which would be a circular import.
+ # pin the ACTIVE root, as the retry and loader do
+ # Pin the ACTIVE root, as the retry and the loader both do. Unpinned, cached_checkpoint_path searches
+ # only huggingface_hub's import-time constant, so after a cache-folder change the retry proves the
+ # checkpoint cached in the live root and this would still call it uncached and re-apply the gate.
+ # Imported from utils rather than diffusion.hub_cache_dir, which would be a circular import.
from utils.hf_cache_settings import active_hf_hub_cache
prequant_cached = prequant_checkpoint_cached(src, cache_dir = active_hf_hub_cache())
except Exception: # noqa: BLE001 -- prequant probing must never sink the candidate
@@ -336,12 +348,13 @@ def resolve_dense_quant_candidate(
estimate.companions_mib,
prequant_available,
)
- # A cached prequant checkpoint downloads nothing, so the space gate has no claim on it. The
- # gate used to run anyway, which discarded exactly the candidate the auto retry exists to find:
- # that retry only ever proposes a rung whose checkpoint is already cached, so on a low-disk or
- # moved-cache install every retry fell back to the GGUF despite a resident-fit local artifact.
+ # A cached prequant checkpoint downloads nothing, so the space gate has no claim on it. The gate used to run anyway,
+ # which discarded exactly the candidate the auto retry exists to find: that retry only ever proposes a rung whose
+ # checkpoint is already cached, so on a low-disk or moved-cache install every retry fell back to the GGUF despite a
+ # resident-fit local artifact.
if estimate is not None and not (estimate.prequant and prequant_cached):
- # The dense path may DOWNLOAD the artifact into the HF cache, which must never wedge a nearly full disk. Size it by what lands on DISK: a prequant fetches the quantised checkpoint, else the base repo's.
+ # The dense path may DOWNLOAD the artifact into the HF cache, which must never wedge a nearly full disk. Size it
+ # by what lands on DISK: a prequant fetches the quantised checkpoint, else the base repo's.
needed_mib = (
estimate.steady_transformer_mib
if estimate.prequant
@@ -361,12 +374,13 @@ def resolve_dense_quant_candidate(
return estimate
+# an `auto` precision is a delegation, so falling back is the feature working
+# ── strict precision (fail closed on a declined EXPLICIT request) ──────────── An `auto` precision is a delegation, so
+# falling back down the ladder is the feature working. An EXPLICIT scheme is a contract: silently loading the GGUF (or a
+# dense bf16 DiT) instead produced a perfectly good image at a precision nobody asked for, which is exactly what made a
+# successful render worthless as proof that the requested precision ran. Escape hatch for anyone who relied on the old
+# behaviour; unset (the default) fails closed.
# ── strict precision (fail closed on a declined EXPLICIT request) ────────────
-# An `auto` precision is a delegation, so falling back down the ladder is the feature working. An
-# EXPLICIT scheme is a contract: silently loading the GGUF (or a dense bf16 DiT) instead produced a
-# perfectly good image at a precision nobody asked for, which is exactly what made a successful
-# render worthless as proof that the requested precision ran. Escape hatch for anyone who relied on
-# the old behaviour; unset (the default) fails closed.
_PRECISION_FALLBACK_ENV = "UNSLOTH_DIFFUSION_ALLOW_PRECISION_FALLBACK"
@@ -400,10 +414,11 @@ def precision_refusal_message(
return f"{control}='{requested}' could not be used: {reason}. {remedy}"
+# how the engaged value relates to what the caller asked for
+# ── resolved-record (status surface) ───────────────────────────────────────── How the engaged value relates to what
+# the caller asked for. Additive on the status payload: every honored request and every auto decision reports "applied",
+# so a client that ignores the field sees exactly today's behaviour.
# ── resolved-record (status surface) ─────────────────────────────────────────
-# How the engaged value relates to what the caller asked for. Additive on the status payload: every
-# honored request and every auto decision reports "applied", so a client that ignores the field sees
-# exactly today's behaviour.
RESOLVED_APPLIED = "applied" # the ask was honored (or there was no ask)
RESOLVED_FELL_BACK = "fell_back" # an explicit ask was declined and something ELSE engaged
RESOLVED_UNSUPPORTED = "unsupported" # an explicit ask cannot run on this host / model at all
@@ -411,12 +426,10 @@ RESOLVED_UNSUPPORTED = "unsupported" # an explicit ask cannot run on this host
# Requests that mean "do not engage this control", so an "off" engagement HONORS them.
_RESOLVED_OFF_VALUES = frozenset({"", "none", "off", "false", "0"})
-# Controls whose REQUEST and ENGAGED value share one vocabulary, so a mismatch is derivable here
-# rather than trusted from the call site (a decline site that forgets to classify itself still
-# reports the truth). memory_mode is deliberately absent: it requests a MODE ("low_vram") and
-# engages an offload POLICY ("sequential"), so comparing the two would report every honored
-# request as a fallback. attention_backend is absent for the same reason ("cudnn" engages as
-# "_native_cudnn").
+# Controls whose REQUEST and ENGAGED value share one vocabulary, so a mismatch is derivable here rather than trusted
+# from the call site (a decline site that forgets to classify itself still reports the truth). memory_mode is absent: it
+# requests a MODE ("low_vram") and engages an offload POLICY ("sequential"), so comparing the two would report every
+# honored request as a fallback. attention_backend is absent for the same reason ("cudnn" engages as "_native_cudnn").
_RESOLVED_COMPARABLE = frozenset({"transformer_quant", "text_encoder_quant"})
diff --git a/studio/backend/core/inference/diffusion_batched.py b/studio/backend/core/inference/diffusion_batched.py
index 4b59c8d937c..b4eda771f77 100644
--- a/studio/backend/core/inference/diffusion_batched.py
+++ b/studio/backend/core/inference/diffusion_batched.py
@@ -30,10 +30,13 @@ from __future__ import annotations
from typing import Any, Callable, Optional
-# Upper bound on images per generation call (mirrors the route's cap): a longer prompt/seed list is a client error, not an OOM to back off from.
+# cap on images per call: a longer list is a client error, not an OOM to back off from
+# Upper bound on images per generation call (mirrors the route's cap): a longer prompt/seed list is a client error, not
+# an OOM to back off from.
MAX_BATCH_IMAGES = 32
-# Seeds stay in JS's safe-integer range so they round-trip through the JSON gallery recipes (a raw 64-bit seed loses precision).
+# Seeds stay in JS's safe-integer range so they round-trip through the JSON gallery recipes (a raw 64-bit seed loses
+# precision).
SEED_MASK = (1 << 53) - 1
diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.py
index 1e09d622486..b72f5ac78bc 100644
--- a/studio/backend/core/inference/diffusion_cache.py
+++ b/studio/backend/core/inference/diffusion_cache.py
@@ -27,11 +27,16 @@ TC_AUTO = "auto"
TC_FBCACHE = "fbcache"
TC_MODES = (TC_FBCACHE,)
-# FBCache residual thresholds: higher skips more steps (faster, lower quality). Quantised transformers shift the residual distribution, so they need a higher threshold.
+# higher skips more steps (faster, lower quality)
+# FBCache residual thresholds: higher skips more steps (faster, lower quality). Quantised transformers shift the
+# residual distribution, so they need a higher threshold.
DEFAULT_FBCACHE_THRESHOLD = 0.08
QUANT_FBCACHE_THRESHOLD = 0.12
-# Auto step-count bar: FBCache's win scales with step count, so auto engages only at 20+ steps ("dev" schedules qualify, distilled turbo never does).
+# FBCache's win scales with step count, so auto engages only at 20+ ("dev" schedules qualify, distilled turbo never
+# does)
+# Auto step-count bar: FBCache's win scales with step count, so auto engages only at 20+ steps ("dev" schedules qualify,
+# distilled turbo never does).
FBCACHE_MIN_STEPS = 20
@@ -69,7 +74,8 @@ def _invalidate_child_registry_cache(transformer: Any) -> None:
pass
-# diffusers cache hook names whose compute branch we re-point at a compiled inner forward (leader = measuring first block, block = the rest); both share the fn_ref layout.
+# diffusers cache hook names whose compute branch we re-point at a compiled inner forward (leader = measuring first
+# block, block = the rest); both share the fn_ref layout.
_CACHE_HOOK_NAMES = (
"mag_cache_leader_block_hook",
"mag_cache_block_hook",
@@ -115,7 +121,9 @@ def _compile_hooked_block_inners(transformer: Any, logger: Any = None) -> int:
continue
if getattr(orig, "__self__", None) is None:
continue # not a plain bound method; arming would miss the block
- # fullgraph=False / dynamic=True: a cache is active (its decision graph-breaks) and this matches the default tier. Dynamo caches per code object, so re-arming after a toggle is ~free.
+ # fullgraph=False / dynamic=True: a cache's decision graph-breaks
+ # fullgraph=False / dynamic=True: a cache is active (its decision graph-breaks) and this matches the
+ # default tier. Dynamo caches per code object, so re-arming after a toggle is ~free.
fn_ref.original_forward = torch.compile(orig, fullgraph = False, dynamic = True)
hook._unsloth_orig_inner = orig
armed += 1
@@ -197,12 +205,18 @@ def apply_step_cache(
if threshold is not None
else (QUANT_FBCACHE_THRESHOLD if quant_active else DEFAULT_FBCACHE_THRESHOLD)
)
- # Engage only via the native enable_cache (CacheMixin path): the lower-level apply_first_block_cache hook would also install on a non-CacheMixin transformer whose pipeline opens no cache_context, and crash generation.
+ # native enable_cache only: apply_first_block_cache would also install on a non-CacheMixin transformer whose
+ # pipeline opens no cache_context
+ # Engage only via the native enable_cache (CacheMixin path): the lower-level apply_first_block_cache hook would also
+ # install on a non-CacheMixin transformer whose pipeline opens no cache_context, and crash generation.
enable_cache = getattr(transformer, "enable_cache", None)
if not callable(enable_cache):
_warn(logger, mode, RuntimeError("transformer has no cache_context (not a CacheMixin)"))
return None
- # A CacheMixin transformer is necessary but not sufficient: the hook raises "No context is set" unless the PIPELINE wraps its denoise loop in cache_context(...). Flux Kontext / img2img / inpaint / controlnet open none, so run uncached.
+ # the hook raises "No context is set" unless the PIPELINE wraps its denoise loop in cache_context();
+ # A CacheMixin transformer is necessary but not sufficient: the hook raises "No context is set" unless the PIPELINE
+ # wraps its denoise loop in cache_context(...). Flux Kontext / img2img / inpaint / controlnet open none, so run
+ # uncached.
if not _pipeline_opens_cache_context(pipe):
_warn(
logger, mode, RuntimeError("pipeline __call__ opens no cache_context; running uncached")
@@ -216,19 +230,22 @@ def apply_step_cache(
config = FirstBlockCacheConfig(threshold = thr)
enable_cache(config)
- # enable_cache after the pipe ran leaves a stale cached child-registry list, so the new block hooks would never receive the cache context. Must follow every enable_cache.
+ # enable_cache after the pipe ran leaves a stale cached child-registry list, so the new block hooks would never
+ # receive the cache context. Must follow every enable_cache.
_invalidate_child_registry_cache(transformer)
- # If blocks are already regionally compiled (toggle path), re-point the fresh hooks' compute branch at compiled inners; the load path is armed by _compile_repeated_blocks.
+ # already regionally compiled (toggle path): re-point the fresh hooks at compiled inners
+ # If blocks are already regionally compiled (toggle path), re-point the fresh hooks' compute branch at compiled
+ # inners; the load path is armed by _compile_repeated_blocks.
_compile_hooked_block_inners(transformer, logger)
try:
transformer._unsloth_step_cache = f"{mode}@{thr}"
- except Exception: # noqa: BLE001 — marker is best-effort
+ except Exception: # noqa: BLE001 - marker is best-effort
pass
if logger is not None:
logger.info("diffusion.cache: %s engaged (threshold=%s)", mode, thr)
return mode
- except Exception as exc: # noqa: BLE001 — incompatible model -> run uncached
- # enable_cache can fail after hooking some blocks; drop partial hooks so the reported-uncached model is not half-cached. Restore armed compiled inners FIRST (remove_hook splices original_forward back).
+ except Exception as exc: # noqa: BLE001 - incompatible model -> run uncached
+ # enable_cache can fail part-hooked; restore armed compiled inners FIRST
_restore_hooked_block_inners(transformer)
try:
transformer.disable_cache()
@@ -301,7 +318,8 @@ def maybe_toggle_step_cache(
disable_cache = getattr(transformer, "disable_cache", None)
if callable(disable_cache):
try:
- # Restore before remove_hook splices original_forward back, so compiled wrappers do not leak onto the uncached path.
+ # Restore before remove_hook splices original_forward back, so compiled wrappers do not leak onto the
+ # uncached path.
_restore_hooked_block_inners(transformer)
disable_cache()
transformer._unsloth_step_cache = None
@@ -312,7 +330,7 @@ def maybe_toggle_step_cache(
FBCACHE_MIN_STEPS,
)
return None
- except Exception as exc: # noqa: BLE001 — keep the cache rather than crash
+ except Exception as exc: # noqa: BLE001 - keep the cache rather than crash
_warn(logger, "fbcache disable", exc)
return TC_FBCACHE
return TC_FBCACHE if engaged else None
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index 9b88e9f2762..cd2ffb04a3e 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -41,8 +41,9 @@ import time
from pathlib import Path
from typing import Any, Optional
-# UNSLOTH_DIFFUSION_COMPILE_CACHE: auto (default) | 0 | 1. auto loads a matching bundle and saves one after the first compiled generation; 1 also re-saves on a hit; 0 disables it.
-# Measured on Qwen-Image (B200): compile hitch 29.1 -> 22.2 s, bit-identical, 7.9 MB bundle. The rest is dynamo tracing + guards, which Mega-cache does not capture.
+# UNSLOTH_DIFFUSION_COMPILE_CACHE: auto (default) | 0 | 1. auto loads a matching bundle and saves one after the first
+# compiled generation; 1 also re-saves on a hit; 0 disables it. Measured on Qwen-Image (B200): compile hitch 29.1 ->
+# 22.2 s, bit-identical, 7.9 MB bundle. The rest is dynamo tracing + guards, which Mega-cache does not capture.
# UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR: root dir for bundles (default under the workspace).
# UNSLOTH_DIFFUSION_COMPILE_CACHE_SAVE: 0 disables the auto save (load-only); 1 keeps it.
_ENV_MODE = "UNSLOTH_DIFFUSION_COMPILE_CACHE"
@@ -71,7 +72,8 @@ def _save_enabled(mode: str) -> bool:
return False
if mode == "on":
return True
- # auto: save by default (without a saved bundle no user gets a warm restart); the SAVE env overrides: "0" load-only, "1" on.
+ # auto: save by default (without a saved bundle no user gets a warm restart); the SAVE env overrides: "0" load-only,
+ # "1" on.
return (os.environ.get(_ENV_SAVE) or "").strip().lower() not in ("0", "off", "false", "no")
@@ -85,7 +87,7 @@ def _triton_version() -> Optional[str]:
try:
import triton # noqa: PLC0415
return str(getattr(triton, "__version__", None))
- except Exception: # noqa: BLE001 — triton optional
+ except Exception: # noqa: BLE001 - triton optional
return None
@@ -118,14 +120,15 @@ def environment_fingerprint() -> dict[str, Any]:
fp["torch"] = str(torch.__version__)
fp["torch_cuda"] = str(torch.version.cuda)
if torch.cuda.is_available():
- # The CURRENT device, not 0: a load pinned to another card compiles for that
- # architecture, and keying the bundle by GPU 0 lets two cards share or overwrite
- # each other's supposedly architecture-specific artifacts.
+ # the CURRENT device, not 0: keying by GPU 0 lets two cards share or overwrite each other's
+ # architecture-specific artifacts
+ # The CURRENT device, not 0: a load pinned to another card compiles for that architecture, and keying the
+ # bundle by GPU 0 lets two cards share or overwrite each other's supposedly architecture-specific artifacts.
index = torch.cuda.current_device()
fp["gpu_name"] = torch.cuda.get_device_name(index)
cap = torch.cuda.get_device_capability(index)
fp["gpu_capability"] = f"sm_{cap[0]}{cap[1]}"
- except Exception: # noqa: BLE001 — best-effort
+ except Exception: # noqa: BLE001 - best-effort
pass
return fp
@@ -240,7 +243,6 @@ def begin(
mode = mode,
)
- # Isolate inductor's on-disk cache per key so bundles never cross-contaminate.
try:
cdir.mkdir(parents = True, exist_ok = True)
ctx.prev_inductor_dir = os.environ.get("TORCHINDUCTOR_CACHE_DIR")
@@ -253,7 +255,8 @@ def begin(
if ctx.bundle.exists() and ctx.manifest_path.exists():
ctx.hit = _try_load(ctx, logger)
if ctx.hit and mode != "on":
- # Loaded artifacts == on-disk artifacts, so nothing to save. A new static-compile shape re-dirties via register_shape; mode "on" keeps saving.
+ # Loaded artifacts == on-disk artifacts, so nothing to save. A new static-compile shape re-dirties via
+ # register_shape; mode "on" keeps saving.
ctx.saved = True
else:
_info(logger, f"compile-cache: no bundle for key {key} (will compile locally)")
@@ -274,7 +277,7 @@ def register_shape(ctx: Optional[CacheContext], shape: Any, *, static: bool) ->
if key not in ctx.shapes:
ctx.shapes.add(key)
ctx.saved = False
- except Exception: # noqa: BLE001 — bookkeeping only
+ except Exception: # noqa: BLE001 - bookkeeping only
pass
@@ -309,10 +312,9 @@ def _try_load(ctx: CacheContext, logger: Any) -> bool:
if info is None:
_warn(logger, "compile-cache: load_cache_artifacts returned None (no hit)")
return False
- # Static-compile shapes this bundle covers (see register_shape).
try:
ctx.shapes = {tuple(s) for s in manifest.get("shapes", [])}
- except Exception: # noqa: BLE001 — coverage bookkeeping only
+ except Exception: # noqa: BLE001 - coverage bookkeeping only
ctx.shapes = set()
_info(logger, f"compile-cache: loaded bundle for key {ctx.key}")
return True
diff --git a/studio/backend/core/inference/diffusion_cond_cache.py b/studio/backend/core/inference/diffusion_cond_cache.py
index f662fc5a198..5928100351c 100644
--- a/studio/backend/core/inference/diffusion_cond_cache.py
+++ b/studio/backend/core/inference/diffusion_cond_cache.py
@@ -40,7 +40,8 @@ from typing import Any, Optional
_ENV_DIR = "UNSLOTH_DIFFUSION_COND_CACHE_DIR"
-# Bound arguments that never change the returned embeddings: device is a placement detail (the hit is moved there) and no encode path draws RNG.
+# Bound arguments that never change the returned embeddings: device is a placement detail (the hit is moved there) and
+# no encode path draws RNG.
_KEY_EXCLUDED_ARGS = frozenset({"device", "generator"})
@@ -59,7 +60,9 @@ def _json_safe(value: Any) -> bool:
return False
-# Per-slot layout codes for _flatten/_unflatten (the leading int64 tensor): -1 = None, -2 = a bare tensor, n >= 0 = a LIST of n tensors.
+# layout codes for _flatten/_unflatten: -1 = None, -2 = bare tensor, n >= 0 = list of n tensors
+# Per-slot layout codes for _flatten/_unflatten (the leading int64 tensor): -1 = None, -2 = a bare tensor, n >= 0 = a
+# LIST of n tensors.
_SLOT_NONE = -1
_SLOT_TENSOR = -2
@@ -131,17 +134,20 @@ def install(
if not callable(encode):
return False
try:
+ # lazy: core.training imports core.inference, so a module-level import would be circular
# Lazy: core.training imports parts of core.inference, so a module-level import would be circular.
from core.training.diffusion_train_extras import PersistentConditioningCache
signature = inspect.signature(encode)
cache = PersistentConditioningCache(root, family, 0)
- except Exception as exc: # noqa: BLE001 — cache is best-effort
+ except Exception as exc: # noqa: BLE001 - cache is best-effort
if logger is not None:
logger.warning("diffusion.cond_cache: install failed: %s", exc)
return False
- # Everything beyond the call arguments that changes the embedding numerics. A GGUF/single-file checkpoint takes its TEXT ENCODERS from the
- # companion base repo, so the base identity keys the cache too; both carry a revision marker, so an advanced commit or an edited dir misses.
+ # a GGUF/single-file checkpoint takes its TEXT ENCODERS from the base repo
+ # Everything beyond the call arguments that changes the embedding numerics. A GGUF/single-file checkpoint takes its
+ # TEXT ENCODERS from the companion base repo, so the base identity keys the cache too; both carry a revision marker,
+ # so an advanced commit or an edited dir misses.
base_ref = base_repo if base_repo else repo_id
load_fp = {
"repo": str(repo_id),
@@ -167,7 +173,6 @@ def install(
if name not in _KEY_EXCLUDED_ARGS
}
if not all(_json_safe(v) for v in keyed.values()):
- # Tensor/object arguments (pre-supplied embeds, images) are not keyable.
return encode(*args, **kwargs)
payload = json.dumps({"load": load_fp, "args": keyed}, sort_keys = True, default = str)
key = cache.text_key(
@@ -177,7 +182,7 @@ def install(
if hit is not None:
stats["hits"] += 1
return _unflatten(hit, _target_device(pipe, bound))
- except Exception as exc: # noqa: BLE001 — never fail a generation over the cache
+ except Exception as exc: # noqa: BLE001 - never fail a generation over the cache
if logger is not None:
logger.warning("diffusion.cond_cache: lookup failed: %s", exc)
return encode(*args, **kwargs)
@@ -187,7 +192,7 @@ def install(
if flat is not None:
cache.put(key, flat)
stats["misses"] += 1
- except Exception as exc: # noqa: BLE001 — a failed write only skips reuse
+ except Exception as exc: # noqa: BLE001 - a failed write only skips reuse
if logger is not None:
logger.warning("diffusion.cond_cache: store failed: %s", exc)
return result
@@ -231,5 +236,5 @@ def _source_revision(ref: Any) -> str:
try:
from core.training.diffusion_train_extras import source_revision # noqa: PLC0415
return source_revision(ref)
- except Exception: # noqa: BLE001 — best-effort, never block a load
+ except Exception: # noqa: BLE001 - best-effort, never block a load
return "unresolved"
diff --git a/studio/backend/core/inference/diffusion_controlnet.py b/studio/backend/core/inference/diffusion_controlnet.py
index 42a260ec147..22637b040d1 100644
--- a/studio/backend/core/inference/diffusion_controlnet.py
+++ b/studio/backend/core/inference/diffusion_controlnet.py
@@ -24,10 +24,12 @@ from typing import Any, Optional
from utils.paths.storage_roots import studio_root
from utils.paths.path_utils import is_appledouble_metadata
+# "passthrough": the supplied image IS the control map; "canny": derive an edge map here
# Control map types. "passthrough": the supplied image IS the control map. "canny": derive an edge map here.
CONTROL_TYPES = ("passthrough", "canny")
-# Diffusers quant schemes that cannot host ControlNet cleanly (torchao tensor subclasses); gated off like LoRA, along with GGUF-via-diffusers.
+# Diffusers quant schemes that cannot host ControlNet cleanly (torchao tensor subclasses); gated off like LoRA, along
+# with GGUF-via-diffusers.
_DIFFUSERS_BLOCKED_QUANT = ("int8", "fp8", "nvfp4", "mxfp8")
@@ -37,12 +39,12 @@ class ControlNetCatalogEntry:
id: str
display_name: str
- source: str # "local" | "hub"
- families: tuple[str, ...] = () # compatible families (empty = shown, not gated)
- repo_id: Optional[str] = None # source == "hub"
- local_path: Optional[str] = None # source == "local"
+ source: str
+ families: tuple[str, ...] = ()
+ repo_id: Optional[str] = None
+ local_path: Optional[str] = None
control_types: tuple[str, ...] = ("passthrough",)
- is_union: bool = False # one model covering many control modes
+ is_union: bool = False
@dataclass(frozen = True)
@@ -50,11 +52,12 @@ class ResolvedControlNet:
"""A ControlNet resolved to something ``from_pretrained`` can load."""
id: str
- path: str # repo id (hub) or local directory
+ path: str
is_local: bool
-# Curated, family-tagged catalog. Union models (one model, many modes) are the default picks; local dirs and a bare ``owner/name`` repo id also work.
+# Curated, family-tagged catalog. Union models (one model, many modes) are the default picks; local dirs and a bare
+# ``owner/name`` repo id also work.
_CURATED: tuple[ControlNetCatalogEntry, ...] = (
ControlNetCatalogEntry(
id = "flux-union-pro",
@@ -167,7 +170,6 @@ def resolve_controlnet(spec_id: str, *, family: Optional[str] = None) -> Resolve
"""
entry = _catalog_by_id().get(spec_id)
if entry is None:
- # A curated entry named by its full repo id must still hit the family gate below, not slip through the bare-repo fallback.
entry = next((e for e in _CURATED if e.repo_id and e.repo_id == spec_id), None)
if entry is not None:
# A direct API call could send an entry for another family; reject it before any download.
@@ -215,7 +217,7 @@ def union_control_mode(spec_id: str, control_type: str) -> Optional[int]:
returns a 400 instead of running the wrong head. A non-union entry returns None."""
entry = _catalog_by_id().get(spec_id)
if entry is None:
- # A union model may be named by its bare repo id, so match on repo_id too (the catalog is keyed by short id), else the union runs the wrong head.
+ # match on repo_id too (the catalog is keyed by short id), else the union runs the wrong head
entry = next((e for e in _CURATED if e.repo_id and e.repo_id == spec_id), None)
if entry is None or not entry.is_union:
return None
@@ -223,7 +225,7 @@ def union_control_mode(spec_id: str, control_type: str) -> Optional[int]:
if ct in _UNION_CONTROL_MODES:
return _UNION_CONTROL_MODES[ct]
if ct in ("", "passthrough"):
- return 0 # preprocessed map, no intrinsic mode; canny is the default head
+ return 0 # canny is the default head
raise ValueError(
f"Unknown control type {control_type!r} for a union ControlNet. Use one of: "
f"{', '.join(sorted(_UNION_CONTROL_MODES))}, or passthrough."
@@ -247,7 +249,8 @@ def preprocess_control(image: Any, control_type: str) -> Any:
mag = np.hypot(gx, gy)
peak = float(mag.max())
if peak <= 1e-6:
- # A flat image has no edges, so the map is all black; returning the source would condition the ControlNet on raw luminance.
+ # A flat image has no edges, so the map is all black; returning the source would condition the ControlNet on raw
+ # luminance.
return Image.new("RGB", image.size, (0, 0, 0))
mag = mag / peak * 255.0
edges = (mag > 40.0).astype(np.uint8) * 255 # white edges on black (ControlNet convention)
diff --git a/studio/backend/core/inference/diffusion_convrot.py b/studio/backend/core/inference/diffusion_convrot.py
index bbfcd56dfdc..d4b5caedddf 100644
--- a/studio/backend/core/inference/diffusion_convrot.py
+++ b/studio/backend/core/inference/diffusion_convrot.py
@@ -55,23 +55,27 @@ from __future__ import annotations
from functools import lru_cache
from typing import Any, Iterable, Optional
+# the rotation KIND; an unimplemented value is refused
+# ── the metadata contract, carried in the prequant checkpoint's own ``metadata`` dict ────────── The rotation KIND. A
+# value this module does not implement is refused, so a future scheme can be added without a released Unsloth silently
+# treating it as this one.
# ── the metadata contract, carried in the prequant checkpoint's own ``metadata`` dict ──────────
-# The rotation KIND. A value this module does not implement is refused, so a future scheme can be
-# added without a released Unsloth silently treating it as this one.
CONVROT_KIND = "convrot_hadamard_v1"
-# Key naming mirrors the adaLN curve contract next door (``adaln_form`` / ``curve_dim`` /
-# ``curve_grid``): a form tag plus the parameters needed to reproduce the form.
+# mirrors the adaLN curve contract next door: a form tag plus the parameters to reproduce the form
+# Key naming mirrors the adaLN curve contract next door (``adaln_form`` / ``curve_dim`` / ``curve_grid``): a form tag
+# plus the parameters needed to reproduce the form.
ROTATION_KEY = "activation_rotation"
ROTATION_GROUP_KEY = "activation_rotation_group"
ROTATION_FQNS_KEY = "activation_rotation_fqns"
-# The group size the denoiser artifact ships at, and the one the hosted conditioner already uses.
-# 256 beat 64 in weight space on MiniMax-H3 (mean relative quantization error -19.9% vs -17.3%
-# over 200 layers) and is the largest power of 4 that divides every quantized H3 input axis.
+# 256 beat 64 in weight space on MiniMax-H3 (-19.9% vs -17.3% mean relative quantization error over 200 layers)
+# The group size the denoiser artifact ships at, and the one the hosted conditioner already uses. 256 beat 64 in weight
+# space on MiniMax-H3 (mean relative quantization error -19.9% vs -17.3% over 200 layers) and is the largest power of 4
+# that divides every quantized H3 input axis.
DEFAULT_CONVROT_GROUPSIZE = 256
-# Marker set on a transformer whose rotation is installed, so a caller can tell a rotated module
-# from an unrotated one without re-deriving anything. Diagnostic only.
+# Marker set on a transformer whose rotation is installed, so a caller can tell a rotated module from an unrotated one
+# without re-deriving anything. Diagnostic only.
CONVROT_ATTR = "_unsloth_activation_rotation"
@@ -86,14 +90,16 @@ def is_power_of_four(size: Any) -> bool:
n = size
if n < 4 or n & (n - 1):
return False
- # A power of two is a power of four exactly when its single set bit sits at an even index.
+ # a power of two is a power of four exactly when its single set bit sits at an even index
return (n.bit_length() - 1) % 2 == 0
-# ── the rotation itself ───────────────────────────────────────────────────────────────────────
-# Mirrors comfy-kitchen's ``_build_hadamard`` / ``_rotate_activation`` / ``_rotate_weight``, in a
-# few lines of torch rather than a dependency on a wheel Unsloth does not ship.
+# mirrors comfy-kitchen's _build_hadamard / _rotate_activation / _rotate_weight, without the wheel dependency
+# ── the rotation itself ─────────────────────────────────────────────────────────────────────── Mirrors comfy-kitchen's
+# ``_build_hadamard`` / ``_rotate_activation`` / ``_rotate_weight``, in a few lines of torch rather than a dependency on
+# a wheel Unsloth does not ship.
+# ── the rotation itself ───────────────────────────────────────────────────────────────────────
_HADAMARD_CACHE: dict = {}
@@ -189,8 +195,8 @@ def convrot_linear_class() -> Any:
class ConvRotLinear(nn.Linear):
"""``nn.Linear`` whose input is block-Hadamard rotated before the matmul."""
- # Set per instance by ``_install_rotation``; the class default exists only so a
- # half-constructed instance cannot silently rotate at some other group.
+ # set per instance by _install_rotation; the class default exists so a half-constructed instance cannot silently
+ # rotate at some other group
convrot_groupsize: int = DEFAULT_CONVROT_GROUPSIZE
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -246,8 +252,8 @@ def rotation_metadata_error(metadata: Any) -> Optional[str]:
)
fqns = metadata.get(ROTATION_FQNS_KEY)
if not isinstance(fqns, (list, tuple)) or not fqns:
- # An empty list is refused rather than read as "rotate nothing": a builder that failed to
- # record its set would otherwise emit an artifact that loads clean and renders garbage.
+ # An empty list is refused rather than read as "rotate nothing": a builder that failed to record its set would
+ # otherwise emit an artifact that loads clean and renders garbage.
return f"activation rotation records no fqns ({ROTATION_FQNS_KEY} is {fqns!r})"
if not all(isinstance(fqn, str) and fqn for fqn in fqns):
return f"activation rotation {ROTATION_FQNS_KEY} has non-string entries"
@@ -365,9 +371,7 @@ def apply_activation_rotation(
f"activation rotation target {fqn!r} has in_features {module.in_features}, "
f"which the recorded group {group_size} does not divide"
)
- # Every target is validated before ANY is swapped. A partial install is the one outcome worse
- # than either end state: the rotated half still renders, just wrongly, so there is nothing to
- # notice and nothing to fall back from.
+ # validate every target before swapping ANY: a partial install still renders, just wrongly
for fqn in fqns:
_install_rotation(modules[fqn], group_size)
try:
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.py
index 6b688e5c6f5..59c9e56d7d7 100644
--- a/studio/backend/core/inference/diffusion_device.py
+++ b/studio/backend/core/inference/diffusion_device.py
@@ -29,7 +29,9 @@ class DiffusionDeviceTarget:
supports_default_torch_compile: bool
supports_pinned_transfer: bool
supports_float64: bool = True
- # Selected CUDA/ROCm physical index, kept OUT of ``device``: the memory, speed and attention policies compare that string against "cuda", so a "cuda:1" there disables them silently.
+ # kept OUT of `device`: the memory, speed and attention policies compare that string against "cuda"
+ # Selected CUDA/ROCm physical index, kept OUT of ``device``: the memory, speed and attention policies compare that
+ # string against "cuda", so a "cuda:1" there disables them silently.
ordinal: Optional[int] = None
@property
@@ -188,9 +190,8 @@ def resolve_selected_cuda_ordinal(
raise ValueError(f"GPU selection is unavailable on this host: {exc}") from exc
allowed = resolve_requested_gpu_ids(wanted)
visible = get_parent_visible_gpu_ids()
- # Torch enumerates the parent-visible list in order, so its ordinal for a physical id is that
- # id's position in the mask. Unmasked, the layer reports range(physical count) and this is
- # the identity mapping.
+ # Torch enumerates the parent-visible list in order, so its ordinal for a physical id is that id's position in the
+ # mask. Unmasked, the layer reports range(physical count) and this is the identity mapping.
ordinals = [visible.index(gpu_id) for gpu_id in allowed if gpu_id in visible]
if not ordinals:
raise ValueError(
@@ -224,9 +225,8 @@ def diffusion_device_scope(ordinal: Optional[int]):
if ordinal is None:
yield
return
- # Entering the context is what may fail on an unusable index; the BODY's exceptions have to
- # travel untouched, or a yield-after-throw replaces the caller's real refusal with
- # "generator didn't stop after throw()".
+ # Entering the context is what may fail on an unusable index; the BODY's exceptions have to travel untouched, or a
+ # yield-after-throw replaces the caller's real refusal with "generator didn't stop after throw()".
try:
import torch
scope = torch.cuda.device(ordinal)
@@ -332,7 +332,6 @@ def resolve_diffusion_device_target(*, ordinal: Optional[int] = None) -> Diffusi
return _cpu_target(torch)
if _studio_device_is(studio_device, DeviceType, "XPU"):
return _xpu_target(torch)
- # MLX / CPU / else: diffusers uses MPS, so fall through to the torch probe (MPS over CPU).
if torch.cuda.is_available():
return _cuda_or_rocm_target(torch, is_rocm = is_rocm, ordinal = ordinal)
@@ -402,8 +401,8 @@ def _cuda_or_rocm_target(
ordinal: Optional[int] = None,
) -> DiffusionDeviceTarget:
if is_rocm:
- # ROCm lacks NVIDIA's pre-Ampere bf16-emulation quirk, so is_bf16_supported() is trustworthy.
- # It takes no device argument, so the selected card is asked by scoping the current device.
+ # ROCm lacks NVIDIA's pre-Ampere bf16-emulation quirk, so is_bf16_supported() is trustworthy. It takes no device
+ # argument, so the selected card is asked by scoping the current device.
try:
with diffusion_device_scope(ordinal):
bf16_ok = bool(torch.cuda.is_bf16_supported())
@@ -411,8 +410,9 @@ def _cuda_or_rocm_target(
bf16_ok = False
dtype = torch.bfloat16 if bf16_ok else torch.float16
else:
- # NVIDIA: bf16 needs Ampere+ (major >= 8), by capability NOT is_bf16_supported() (pre-Ampere cards emulate bf16 slowly but report it supported).
- # Asked of the SELECTED card, since the argument-less form reports the current device, a different generation on a mixed box; still argument-less without a selection.
+ # NVIDIA: bf16 needs Ampere+ (major >= 8), by capability NOT is_bf16_supported() (pre-Ampere cards emulate bf16
+ # slowly but report it supported). Asked of the SELECTED card, since the argument-less form reports the current
+ # device, a different generation on a mixed box; still argument-less without a selection.
try:
major = (
torch.cuda.get_device_capability()
@@ -475,11 +475,15 @@ def _mps_or_cpu_target(torch: Any) -> DiffusionDeviceTarget:
mps_available = False
if mps_available:
- # torch reads PYTORCH_MPS_HIGH_WATERMARK_RATIO once, at the first MPS allocation (the probe below), so relax it first or
- # the allocator caps at ~1.7x recommendedMaxWorkingSet and can OOM a model that would fit. setdefault respects an override.
+ # torch reads PYTORCH_MPS_HIGH_WATERMARK_RATIO once, at the first MPS allocation (the probe below), so relax it
+ # first or the allocator caps at ~1.7x recommendedMaxWorkingSet and can OOM a model that would fit. setdefault
+ # respects an override.
os.environ.setdefault("PYTORCH_MPS_HIGH_WATERMARK_RATIO", "0.0")
- # Prefer bfloat16, else float32, NEVER silent float16: modern DiTs produce activations far outside fp16's range (Z-Image
- # MLP peaks near 9e5 -> inf -> NaN -> black image). bf16 (macOS 14+) shares fp32's exponent range; older macOS uses fp32.
+ # NEVER silent float16: modern DiTs produce activations far outside fp16's range (Z-Image MLP peaks near 9e5 ->
+ # inf -> NaN -> black image)
+ # Prefer bfloat16, else float32, NEVER silent float16: modern DiTs produce activations far outside fp16's range
+ # (Z-Image MLP peaks near 9e5 -> inf -> NaN -> black image). bf16 (macOS 14+) shares fp32's exponent range;
+ # older macOS uses fp32.
dtype = torch.bfloat16 if _mps_supports_bfloat16(torch) else torch.float32
return DiffusionDeviceTarget(
device = "mps",
diff --git a/studio/backend/core/inference/diffusion_eager_patches.py b/studio/backend/core/inference/diffusion_eager_patches.py
index 5c6ac67932c..b1bf817f6ae 100644
--- a/studio/backend/core/inference/diffusion_eager_patches.py
+++ b/studio/backend/core/inference/diffusion_eager_patches.py
@@ -47,7 +47,6 @@ def _patches_enabled() -> bool:
return (os.environ.get(_ENV_ENABLE) or "").strip().lower() not in ("0", "off", "false", "no")
-# Resolve the diffusers classes we patch; an import failure -> None (skipped at install).
try:
from diffusers.models.normalization import (
AdaLayerNormContinuous as _AdaLayerNormContinuous,
@@ -65,7 +64,8 @@ except Exception: # noqa: BLE001
_NPU = False
-# Patched forwards, each mirroring diffusers semantics with the fused fast path (``addcmul(input, t1, t2) == input + t1 * t2``).
+# Patched forwards, each mirroring diffusers semantics with the fused fast path (``addcmul(input, t1, t2) == input + t1
+# * t2``).
def _adaln_continuous_forward(self, x, conditioning_embedding):
emb = self.linear(self.silu(conditioning_embedding).to(x.dtype))
scale, shift = torch.chunk(emb, 2, dim = 1)
@@ -106,8 +106,10 @@ _orig_rmsnorm_forward: Optional[Callable] = None
def _rmsnorm_forward(self, hidden_states):
- # Fall back to the exact original where F.rms_norm is NOT equivalent to diffusers: NPU / bias / fp32-weight, a tuple `dim` (diffusers reduces
- # only the LAST dim), or a dtype mismatch (diffusers computes variance in fp32 from the ORIGINAL tensor).
+ # fall back where F.rms_norm is NOT equivalent to diffusers
+ # Fall back to the exact original where F.rms_norm is NOT equivalent to diffusers: NPU / bias / fp32-weight, a tuple
+ # `dim` (diffusers reduces only the LAST dim), or a dtype mismatch (diffusers computes variance in fp32 from the
+ # ORIGINAL tensor).
if _NPU or self.bias is not None or _orig_rmsnorm_forward is None or len(tuple(self.dim)) != 1:
return _orig_rmsnorm_forward(self, hidden_states) # type: ignore[misc]
weight = self.weight
@@ -116,13 +118,12 @@ def _rmsnorm_forward(self, hidden_states):
if weight.dtype in (torch.float16, torch.bfloat16) and hidden_states.dtype == weight.dtype:
# Common DiT path (bf16 acts + bf16 weight): F.rms_norm matches diffusers bit-for-bit.
return F.rms_norm(hidden_states, self.dim, weight, self.eps)
- # Mixed dtype / fp32 weight -> exact original.
return _orig_rmsnorm_forward(self, hidden_states) # type: ignore[misc]
-# Install / uninstall via the shared patch backend: the live original is fingerprinted so a changed forward is left UNPATCHED, and stashed for exact restore.
+# Install / uninstall via the shared patch backend: the live original is fingerprinted so a changed forward is left
+# UNPATCHED, and stashed for exact restore.
def _specs():
- # (class, patched_fn)
return [
(_AdaLayerNormContinuous, _adaln_continuous_forward),
(_AdaLayerNormZero, _adaln_zero_forward),
@@ -142,7 +143,7 @@ def install_compile_safe_patches() -> int:
"""
global _orig_rmsnorm_forward
if not _patches_enabled():
- uninstall_patches() # ensure OFF even if a prior call installed them
+ uninstall_patches()
return 0
if _patched:
return len(_patched)
@@ -153,7 +154,6 @@ def install_compile_safe_patches() -> int:
if cls is _RMSNorm and not hasattr(F, "rms_norm"):
logger.info("eager-patch: skipping RMSNorm (this torch has no F.rms_norm)")
continue
- # Capture the live original before patching, for the RMSNorm fast path's fallback.
if cls is _RMSNorm:
_orig_rmsnorm_forward = cls.forward
if apply_patch(cls, "forward", new_fn, match_level = "relaxed"):
diff --git a/studio/backend/core/inference/diffusion_engine_router.py b/studio/backend/core/inference/diffusion_engine_router.py
index 21b5331bcb4..bb3091460c8 100644
--- a/studio/backend/core/inference/diffusion_engine_router.py
+++ b/studio/backend/core/inference/diffusion_engine_router.py
@@ -49,8 +49,10 @@ logger = get_logger(__name__)
_DISABLE_TOKENS = frozenset({"0", "off", "false", "no"})
_ENABLE_TOKENS = frozenset({"1", "on", "true", "yes"})
-# Resolved device backend -> the prebuilt sd-cli accelerator to install, used only for a force-native load on a GPU host:
-# without it the installer defaults to "cpu" and a forced ROCm/Intel generation silently runs on CPU. Unknown -> "auto".
+# without this the installer defaults to "cpu" and a forced ROCm/Intel generation silently runs on CPU
+# Resolved device backend -> the prebuilt sd-cli accelerator to install, used only for a force-native load on a GPU
+# host: without it the installer defaults to "cpu" and a forced ROCm/Intel generation silently runs on CPU. Unknown ->
+# "auto".
_INSTALL_ACCELERATOR = {"rocm": "rocm", "cuda": "cuda", "xpu": "vulkan"}
@@ -95,11 +97,13 @@ def active_engine_name() -> str:
def _activate(name: str, reason: Optional[str]) -> Any:
global _active_engine_name, _fallback_reason
- # Serialize check -> unload -> publish without holding _lock across the slow unload(), closing the window where a second
- # _activate reads the still-old active engine and loads onto the engine this call is unloading.
+ # without holding _lock across the slow unload()
with _transition_lock:
- # Switching engines: unload the deactivated one first, else its model stays resident but unreachable (the evictor only
- # targets the active engine), leaking 10+ GB. The unload is slow, so resolve under _lock but run unload() OUTSIDE it.
+ # unload the deactivated engine first, else its model stays resident but unreachable (the evictor only targets
+ # the active one), leaking 10+ GB
+ # Switching engines: unload the deactivated one first, else its model stays resident but unreachable (the
+ # evictor only targets the active engine), leaking 10+ GB. The unload is slow, so resolve under _lock but run
+ # unload() OUTSIDE it.
engine_to_unload = None
old_name = None
with _lock:
@@ -107,17 +111,20 @@ def _activate(name: str, reason: Optional[str]) -> Any:
engine_to_unload = get_active_diffusion_engine()
old_name = _active_engine_name
else:
- # No engine change: publish the (possibly refreshed) fallback reason now.
_fallback_reason = reason if name == ENGINE_DIFFUSERS else None
if engine_to_unload is not None:
- # Publish the new engine only AFTER the old one unloads: the evictor unloads get_active_diffusion_engine(), so flipping
- # the name first would let a concurrent acquire_for evict the new (empty) engine while the old model frees VRAM.
+ # publish only AFTER the old engine unloads
+ # Publish the new engine only AFTER the old one unloads: the evictor unloads get_active_diffusion_engine(),
+ # so flipping the name first would let a concurrent acquire_for evict the new (empty) engine while the old
+ # model frees VRAM.
try:
engine_to_unload.unload()
except Exception as exc:
- # Do NOT publish the new engine after a failed teardown. The old model (or the resident sd-server) still holds its memory, and flipping the
- # name would hide it from get_active_diffusion_engine(), which the evictor, /images/unload and the next load all resolve through, so the leak
- # would be permanent. Leaving the old engine active keeps it reclaimable and lets the caller retry.
+ # do NOT publish after a failed teardown: the old model still holds its memory
+ # Do NOT publish the new engine after a failed teardown. The old model (or the resident sd-server) still
+ # holds its memory, and flipping the name would hide it from get_active_diffusion_engine(), which the
+ # evictor, /images/unload and the next load all resolve through, so the leak would be permanent. Leaving
+ # the old engine active keeps it reclaimable and lets the caller retry.
logger.error("failed to unload previous engine %s: %s", old_name, exc)
raise RuntimeError(
f"Could not switch the diffusion engine to {name}: unloading the current "
@@ -164,7 +171,6 @@ def select_and_activate_engine(
a usable GPU, MPS is not enabled, the family has no native asset, or the binary is unavailable
-- always BEFORE the slow load, so a fallback never strands a half-native load.
"""
- # Non-GGUF loads run on diffusers only (the native engine consumes single-file GGUF only).
if model_kind and model_kind != "gguf":
return _activate(ENGINE_DIFFUSERS, f"non-GGUF load ({model_kind}) requires diffusers")
@@ -179,15 +185,18 @@ def select_and_activate_engine(
target = resolve_diffusion_device_target()
backend = target.backend
- # Policy: CPU always native-eligible; MPS only when enabled; a GPU backend never, unless forced.
+ # CPU always native-eligible; MPS only when enabled; a GPU backend never, unless forced
policy_eligible = backend == "cpu" or (backend == "mps" and mps_enabled) or prefer_native
fam_ok = family_sd_cpp_supported(fam)
binary = None
server_binary = None
if policy_eligible and fam_ok:
- # Probe the resident sd-server FIRST (the backend prefers it): a server-only install must still route to native and should
- # not pay an sd-cli download. Install the accelerator-matched build so a forced-native GPU load gets the GPU server.
+ # probe the resident sd-server FIRST: a server-only install must route to native without paying an sd-cli
+ # download
+ # Probe the resident sd-server FIRST (the backend prefers it): a server-only install must still route to native
+ # and should not pay an sd-cli download. Install the accelerator-matched build so a forced-native GPU load gets
+ # the GPU server.
server_binary = ensure_sd_server_binary(
allow_install = _install_allowed(),
accelerator = _install_accelerator_for(backend),
@@ -197,8 +206,9 @@ def select_and_activate_engine(
"sd-server at %s is present but not runnable; not using it", server_binary
)
server_binary = None
- # sd-cli is the one-shot fallback: always LOCATE an existing binary, but auto-INSTALL only when there is no usable server.
- # Probe runnability first, else a present but non-runnable binary passes as available and fails inside the background load.
+ # sd-cli is the one-shot fallback: always LOCATE an existing binary, but auto-INSTALL only when there is no
+ # usable server. Probe runnability first, else a present but non-runnable binary passes as available and fails
+ # inside the background load.
binary = ensure_sd_cpp_binary(
allow_install = _install_allowed() and server_binary is None,
accelerator = _install_accelerator_for(backend),
@@ -297,7 +307,7 @@ def family_buildable_here(fam: Optional[DiffusionFamily], *, model_kind: Optiona
return False
if family_pipeline_available(fam):
return True
- # Only a GGUF can go native, and only for a family with the single-file assets sd.cpp needs.
+ # only a GGUF can go native, and only for a family with the single-file assets sd.cpp needs
if model_kind != "gguf" or not family_sd_cpp_supported(fam):
return False
try:
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 9a7061ebaa9..54c32af379c 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -25,7 +25,9 @@ from typing import NamedTuple, Optional, Sequence
from utils.paths.path_utils import is_appledouble_metadata
-# Runtime->route contract: the /images/generate route matches these messages EXACTLY for a 409 (vs a 500), so both engines raise them verbatim.
+# runtime->route contract: /images/generate matches these messages EXACTLY for a 409 (vs a 500)
+# Runtime->route contract: the /images/generate route matches these messages EXACTLY for a 409 (vs a 500), so both
+# engines raise them verbatim.
DIFFUSION_NOT_LOADED_MSG = "No diffusion model is loaded."
DIFFUSION_CANCELLED_MSG = "Diffusion generation was cancelled."
@@ -84,65 +86,83 @@ class DiffusionFamily:
denoiser_attr: str = "transformer"
# True when a single-file ``.safetensors`` is the WHOLE pipeline (SDXL), so the loader calls ``from_single_file``.
single_file_is_pipeline: bool = False
- # True for families needing MULTIPLE denoisers no single file carries (Ideogram 4), so only a full pipeline load is valid.
+ # True for families needing MULTIPLE denoisers no single file carries (Ideogram 4), so only a full pipeline load is
+ # valid.
pipeline_only: bool = False
- # Optional diffusers pipeline classes for image-conditioned workflows, built via ``Pipeline.from_pipe`` (no reload). None = unsupported.
+ # optional pipeline classes for image-conditioned workflows, built via Pipeline.from_pipe
+ # Optional diffusers pipeline classes for image-conditioned workflows, built via ``Pipeline.from_pipe`` (no reload).
+ # None = unsupported.
img2img_pipeline_class: Optional[str] = None
inpaint_pipeline_class: Optional[str] = None
- # ControlNet pipeline + model classes: the model loads via from_pretrained, the pipeline via ``from_pipe``. None on both = no support.
+ # the ControlNet model loads via from_pretrained, the pipeline via from_pipe
+ # ControlNet pipeline + model classes: the model loads via from_pretrained, the pipeline via ``from_pipe``. None on
+ # both = no support.
controlnet_pipeline_class: Optional[str] = None
controlnet_model_class: Optional[str] = None
- # True when the inpaint pipeline keeps the canvas size, so it can also drive outpaint. False for FLUX.2 (it scales >1MP inputs to ~1MP).
+ # True when the inpaint pipeline keeps the canvas size, so it can also drive outpaint. False for FLUX.2 (it scales
+ # >1MP inputs to ~1MP).
inpaint_preserves_size: bool = True
+ # true for instruction-editing families: the pipeline IS the edit pipeline (image + instruction, no plain
+ # text-to-image)
# True for instruction-editing families (Qwen-Image-Edit / FLUX Kontext): the pipeline IS the edit pipeline (image +
# instruction, no plain text-to-image). ``base_repo`` supplies the VAE / text-encoder / processor / scheduler.
edit: bool = False
- # True for families whose text-to-image pipeline ALSO accepts reference image(s) (FLUX.2 ``image``): no ``strength``, size from width/height.
+ # True for families whose text-to-image pipeline ALSO accepts reference image(s) (FLUX.2 ``image``): no
+ # ``strength``, size from width/height.
reference: bool = False
# Extra lowercased substrings (besides ``name``) that map a repo id here.
aliases: tuple[str, ...] = field(default_factory = tuple)
- # True for families whose activations overflow float16 (-> black image); the backend promotes a resolved float16 to float32.
+ # True for families whose activations overflow float16 (-> black image); the backend promotes a resolved float16 to
+ # float32.
fp16_incompatible: bool = False
- # False only for a family whose denoiser block does not compile cleanly with regional torch.compile.
+ # false only for a family whose denoiser block does not compile cleanly with regional torch.compile
supports_torch_compile: bool = True
- # Optional pre-quantized transformer checkpoints as (scheme, repo_id): fetched instead of the dense bf16 (lower load VRAM + download).
+ # Optional pre-quantized transformer checkpoints as (scheme, repo_id): fetched instead of the dense bf16 (lower load
+ # VRAM + download).
prequant_repos: tuple[tuple[str, str], ...] = field(default_factory = tuple)
- # Hosted checkpoints for NON-DEFAULT bases as (base_repo, scheme, repo_id), base_repo lowercased: one family entry covers
- # variants whose weights differ. Resolution prefers an exact variant match, then falls back to ``prequant_repos``.
+ # Hosted checkpoints for NON-DEFAULT bases as (base_repo, scheme, repo_id), base_repo lowercased: one family entry
+ # covers variants whose weights differ. Resolution prefers an exact variant match, then falls back to
+ # ``prequant_repos``.
prequant_variant_repos: tuple[tuple[str, str, str], ...] = field(default_factory = tuple)
- # Bases (lowercased) with NO hosted checkpoint, which must not inherit ``prequant_repos``: the family
- # fallback names an artifact baked from different weights, and planning acts on it before the load's
- # base_model_id check can refuse it. Only for a base whose weights genuinely differ from the default.
+ # Bases (lowercased) with NO hosted checkpoint, which must not inherit ``prequant_repos``: the family fallback names
+ # an artifact baked from different weights, and planning acts on it before the load's base_model_id check can refuse
+ # it. Only for a base whose weights genuinely differ from the default.
prequant_excluded_bases: tuple[str, ...] = field(default_factory = tuple)
- # Preferred checkpoint FILENAME for a scheme, as (scheme, filename), overriding the
- # ``<Model>-<SCHEME>.pt`` name ``prequant_repo_filename`` derives. The derived name stays on as
- # the fallback, so a repo hosting BOTH an old and a new artifact serves the new one to a build
- # that asks for it by name and the old one to every build that does not. That is what lets a
- # rotated (v2) checkpoint ship without regressing an already-installed Unsloth, which would
- # otherwise refuse the v2 tag and fall all the way back to the dense download.
- # A row may also be (scheme, task, filename), which names the artifact for ONE task and beats
- # the task-agnostic row; see ``family_prequant_filename``.
+ # Preferred checkpoint FILENAME for a scheme, as (scheme, filename), overriding the ``<Model>-<SCHEME>.pt`` name
+ # ``prequant_repo_filename`` derives. The derived name stays on as the fallback, so a repo hosting BOTH an old and a
+ # new artifact serves the new one to a build that asks for it by name and the old one to every build that does not.
+ # That is what lets a rotated (v2) checkpoint ship without regressing an already-installed Unsloth, which would
+ # otherwise refuse the v2 tag and fall all the way back to the dense download. A row may also be (scheme, task,
+ # filename), which names the artifact for ONE task and beats the task-agnostic row; see
+ # ``family_prequant_filename``.
prequant_filenames: tuple[tuple[str, ...], ...] = field(default_factory = tuple)
- # Hosted PRE-CAST text-encoder checkpoints as (scheme, component, repo_id). Layerwise-fp8 only: the cast is deterministic, so the artifact is bit-identical while skipping the dense TE download.
+ # Hosted PRE-CAST text-encoder checkpoints as (scheme, component, repo_id). Layerwise-fp8 only: the cast is
+ # deterministic, so the artifact is bit-identical while skipping the dense TE download.
te_prequant_repos: tuple[tuple[str, str, str], ...] = field(default_factory = tuple)
# Native (sd.cpp) single-file assets, used only on the no-GPU sd.cpp engine. The transformer GGUF is shared with
- # diffusers; sd-cli also needs a (repo_id, filename) VAE + text encoder(s), each with a trailing SdCppModelFiles field name.
+ # diffusers; sd-cli also needs a (repo_id, filename) VAE + text encoder(s), each with a trailing SdCppModelFiles
+ # field name.
sd_cpp_vae: Optional[tuple[str, str]] = None
- # VAE latent-format override for sd-cli (--vae-format): "flux2" for FLUX.2, None otherwise.
+ # VAE latent-format override for sd-cli (--vae-format): "flux2" for FLUX.2, None otherwise
sd_cpp_vae_format: Optional[str] = None
sd_cpp_text_encoders: tuple[tuple[str, str, str], ...] = field(default_factory = tuple)
- # Family-specific sd-cli sampler settings so native output matches the model's supported invocation. None leaves sd-cli defaults.
+ # Family-specific sd-cli sampler settings so native output matches the model's supported invocation. None leaves
+ # sd-cli defaults.
sd_cpp_sampling_method: Optional[str] = None
sd_cpp_flow_shift: Optional[float] = None
- # True when Unsloth can TRAIN a LoRA on this family; the training-start path refuses a non-trainable family up front.
+ # True when Unsloth can TRAIN a LoRA on this family; the training-start path refuses a non-trainable family up
+ # front.
trainable: bool = False
- # Recommended base repos to train FROM, most-preferred first (e.g. a QLoRA prequant repo, then bf16). Surfaced by the Train UI.
+ # Recommended base repos to train FROM, most-preferred first (e.g. a QLoRA prequant repo, then bf16). Surfaced by
+ # the Train UI.
train_base_repos: tuple[str, ...] = field(default_factory = tuple)
- # When set, deploying a LoRA trained on this family loads THIS repo instead (Krea: train on Raw, preview on Turbo). Same precision both sides.
+ # When set, deploying a LoRA trained on this family loads THIS repo instead (Krea: train on Raw, preview on Turbo).
+ # Same precision both sides.
deploy_base_repo: Optional[str] = None
- # Variant-specific training-base to inference-base mappings. FLUX.2 Klein trains on an
- # undistilled base and runs the adapter on the matching 4-step checkpoint, so its 4B and 9B
- # variants cannot share the single family-wide deploy_base_repo above.
+ # FLUX.2 Klein trains on an undistilled base and runs the adapter on the matching 4-step checkpoint
+ # Variant-specific training-base to inference-base mappings. FLUX.2 Klein trains on an undistilled base and runs the
+ # adapter on the matching 4-step checkpoint, so its 4B and 9B variants cannot share the single family-wide
+ # deploy_base_repo above.
deploy_base_repos: tuple[tuple[str, str], ...] = field(default_factory = tuple)
def deploy_base_for(self, trained_base: str) -> str:
@@ -154,29 +174,31 @@ class DiffusionFamily:
return self.deploy_base_repo or trained_base
-# Keyed by architecture, not per variant: the base repo is read from the HF base_model tag at load time, so one entry covers Turbo/full, schnell/dev.
+# Keyed by architecture, not per variant: the base repo is read from the HF base_model tag at load time, so one entry
+# covers Turbo/full, schnell/dev.
_FAMILIES: tuple[DiffusionFamily, ...] = (
DiffusionFamily(
name = "flux.1",
pipeline_class = "FluxPipeline",
transformer_class = "FluxTransformer2DModel",
base_repo = "black-forest-labs/FLUX.1-schnell",
- # Hosted pre-quantized DiT checkpoints (gate-validated vs same-seed bf16). The loader verifies the baked base_model_id, so a non-default base falls back to dense-quantize.
prequant_repos = (
("int8", "unsloth/FLUX.1-schnell-FP8"),
("fp8", "unsloth/FLUX.1-schnell-FP8"),
),
- # Checkpoints baked from the dev / Krea-dev weights (same arch, different weights); without these every int8/fp8 load pays the dense download + on-the-fly quantise.
+ # Checkpoints baked from the dev / Krea-dev weights (same arch, different weights); without these every int8/fp8
+ # load pays the dense download + on-the-fly quantise.
prequant_variant_repos = (
("black-forest-labs/flux.1-dev", "int8", "unsloth/FLUX.1-dev-FP8"),
("black-forest-labs/flux.1-dev", "fp8", "unsloth/FLUX.1-dev-FP8"),
("black-forest-labs/flux.1-krea-dev", "int8", "unsloth/FLUX.1-Krea-dev-FP8"),
("black-forest-labs/flux.1-krea-dev", "fp8", "unsloth/FLUX.1-Krea-dev-FP8"),
),
- # Pre-cast T5-XXL (9.52 -> 5.90 GB; CLIP-L stays dense). One artifact serves schnell/dev/Krea-dev (T5 shards are byte-identical).
+ # Pre-cast T5-XXL (9.52 -> 5.90 GB; CLIP-L stays dense). One artifact serves schnell/dev/Krea-dev (T5 shards are
+ # byte-identical).
te_prequant_repos = (("fp8", "text_encoder_2", "unsloth/FLUX.1-schnell-FP8"),),
aliases = ("flux1", "flux-1"),
- # LoRA training targets FLUX.1-dev via the DiT trainer (QLoRA nf4); the dev repo is gated.
+ # LoRA training targets FLUX.1-dev via the DiT trainer (QLoRA nf4); the dev repo is gated
trainable = True,
train_base_repos = ("black-forest-labs/FLUX.1-dev",),
img2img_pipeline_class = "FluxImg2ImgPipeline",
@@ -190,7 +212,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
("unsloth/flux-text-encoders", "t5xxl_fp16.safetensors", "t5xxl"),
),
),
- # FLUX.2-klein is Flux2KleinPipeline (Qwen3 encoder), not the Mistral Flux2Pipeline, so it must precede a generic flux match.
+ # FLUX.2-klein is Flux2KleinPipeline (Qwen3 encoder), not the Mistral Flux2Pipeline, so it must precede a generic
+ # flux match.
DiffusionFamily(
name = "flux.2-klein",
pipeline_class = "Flux2KleinPipeline",
@@ -201,8 +224,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
("fp8", "unsloth/FLUX.2-klein-4B-FP8"),
),
aliases = ("flux2-klein",),
- # Train the undistilled bases, then preview their adapters on the matching 4-step models.
- # Both vendor ids resolve through the ungated mirrors at fetch time.
+ # Train the undistilled bases, then preview their adapters on the matching 4-step models. Both vendor ids
+ # resolve through the ungated mirrors at fetch time.
trainable = True,
train_base_repos = (
"black-forest-labs/FLUX.2-klein-base-4B",
@@ -218,12 +241,12 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
"black-forest-labs/FLUX.2-klein-9B",
),
),
- # Flux2KleinPipeline takes reference image(s) via `image`, so it exposes a "reference" workflow atop text-to-image. Inpaint but no img2img.
+ # Flux2KleinPipeline takes reference image(s) via `image`, so it exposes a "reference" workflow atop
+ # text-to-image. Inpaint but no img2img.
reference = True,
inpaint_pipeline_class = "Flux2KleinInpaintPipeline",
# FLUX.2 scales >1MP inputs to ~1MP, so outpaint can't grow.
inpaint_preserves_size = False,
- # FLUX.2's 32-channel AE needs the latent-format override. The VAE is mirrored on its own because BFL licensed it Apache-2.0 while the rest of FLUX.2-dev is non-commercial. Shares Qwen3-4B with z-image.
sd_cpp_vae = ("unsloth/FLUX.2-VAE", "split_files/vae/flux2-vae.safetensors"),
sd_cpp_vae_format = "flux2",
sd_cpp_text_encoders = (
@@ -234,7 +257,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
),
),
),
- # FLUX.2-dev: full (non-distilled) FLUX.2 on the Mistral Flux2Pipeline, so its own entry. Gated base, text-to-image only; sd-cli takes the Mistral encoder from unsloth/FLUX.2-dev-ComfyUI and the VAE from unsloth/FLUX.2-VAE.
+ # FLUX.2-dev: full (non-distilled) FLUX.2 on the Mistral Flux2Pipeline, so its own entry. Gated base, text-to-image
+ # only; sd-cli takes the Mistral encoder from unsloth/FLUX.2-dev-ComfyUI and the VAE from unsloth/FLUX.2-VAE.
DiffusionFamily(
name = "flux.2-dev",
pipeline_class = "Flux2Pipeline",
@@ -247,7 +271,6 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# Pre-cast Mistral-Small-24B conditioner (bf16 ~48 GB dense, ~24.7 GB pre-cast).
te_prequant_repos = (("fp8", "text_encoder", "unsloth/FLUX.2-dev-FP8"),),
aliases = ("flux2-dev", "flux2dev"),
- # LoRA training via the DiT trainer (QLoRA nf4); the gated base needs an HF token with the FLUX.2-dev license accepted.
trainable = True,
train_base_repos = ("black-forest-labs/FLUX.2-dev",),
sd_cpp_vae = ("unsloth/FLUX.2-VAE", "split_files/vae/flux2-vae.safetensors"),
@@ -261,7 +284,9 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
),
),
DiffusionFamily(
- # FLUX instruction editing: FluxKontextPipeline takes an image + instruction. Specific aliases first so detect_family prefers this over "flux.1".
+ # FLUX instruction editing; specific aliases first so detect_family prefers this over "flux.1"
+ # FLUX instruction editing: FluxKontextPipeline takes an image + instruction. Specific aliases first so
+ # detect_family prefers this over "flux.1".
name = "flux.1-kontext",
pipeline_class = "FluxKontextPipeline",
transformer_class = "FluxTransformer2DModel",
@@ -270,7 +295,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
edit = True,
),
DiffusionFamily(
- # Qwen instruction editing: the 2511 checkpoint ships as QwenImageEditPlusPipeline. Specific aliases first so detect_family prefers this over "qwen-image".
+ # Qwen instruction editing: the 2511 checkpoint ships as QwenImageEditPlusPipeline. Specific aliases first so
+ # detect_family prefers this over "qwen-image".
name = "qwen-image-edit",
pipeline_class = "QwenImageEditPlusPipeline",
transformer_class = "QwenImageTransformer2DModel",
@@ -290,14 +316,13 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
pipeline_class = "QwenImagePipeline",
transformer_class = "QwenImageTransformer2DModel",
base_repo = "Qwen/Qwen-Image",
- # int8 only: no fp8 DiT checkpoint is published for this family yet. fp8 is no longer
- # denied for inference, so adding one here would now be live rather than dead.
+ # int8 only: no fp8 DiT checkpoint is published for this family yet. fp8 is no longer denied for inference, so
+ # adding one here would now be live rather than dead.
prequant_repos = (("int8", "unsloth/Qwen-Image-FP8"),),
# Pre-cast Qwen2.5-VL-7B (16.6 -> 8.8 GB). Always was independent of the DiT scheme rules.
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Qwen-Image-FP8"),),
cfg_kwarg = "true_cfg_scale",
aliases = ("qwen_image", "qwenimage"),
- # LoRA training via the DiT trainer, defaulting to the prequant nf4 repo (QLoRA).
trainable = True,
train_base_repos = ("unsloth/Qwen-Image-2512-unsloth-bnb-4bit", "Qwen/Qwen-Image"),
img2img_pipeline_class = "QwenImageImg2ImgPipeline",
@@ -327,15 +352,17 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
("int8", "unsloth/Z-Image-Turbo-FP8"),
("fp8", "unsloth/Z-Image-Turbo-FP8"),
),
- # Both hosted checkpoints are baked from the distilled Turbo transformer, so the undistilled base has none and must quantize its own dense weights.
+ # Both hosted checkpoints are baked from the distilled Turbo transformer, so the undistilled base has none and
+ # must quantize its own dense weights.
prequant_excluded_bases = ("tongyi-mai/z-image",),
- # Pre-cast Qwen3-4B TE (8.04 -> 4.41 GB), shared with the undistilled base (byte-identical encoder). NOT shared with flux.2-klein-4B: klein TE retrained layer 35 MLP (up/down_proj maxdiff 0.86 vs this checkpoint).
+ # Pre-cast Qwen3-4B TE (8.04 -> 4.41 GB), shared with the undistilled base (byte-identical encoder). NOT shared
+ # with flux.2-klein-4B: klein TE retrained layer 35 MLP (up/down_proj maxdiff 0.86 vs this checkpoint).
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Z-Image-Turbo-FP8"),),
aliases = ("zimage", "z_image"),
- # LoRA training via the DiT trainer (bf16); defaults to the prequant nf4 repo for QLoRA.
+ # LoRA training via the DiT trainer (bf16); defaults to the prequant nf4 repo for QLoRA
trainable = True,
- # The undistilled base is what the upstream DreamBooth recipe trains on. No deploy pairing:
- # its adapters preview on the base itself at the 20-step / guidance 4 recipe.
+ # The undistilled base is what the upstream DreamBooth recipe trains on. No deploy pairing: its adapters preview
+ # on the base itself at the 20-step / guidance 4 recipe.
train_base_repos = (
"unsloth/Z-Image-Turbo-unsloth-bnb-4bit",
"Tongyi-MAI/Z-Image-Turbo",
@@ -343,7 +370,6 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
),
img2img_pipeline_class = "ZImageImg2ImgPipeline",
inpaint_pipeline_class = "ZImageInpaintPipeline",
- # Z-Image's MLP down-projections peak near 9e5, which overflows float16.
fp16_incompatible = True,
# Byte-identical mirror of Comfy-Org/z_image_turbo (AE + Qwen3-4B).
sd_cpp_vae = ("unsloth/Z-Image-Turbo-ComfyUI", "split_files/vae/ae.safetensors"),
@@ -355,7 +381,6 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
),
),
),
- # Krea 2 (diffusers >= 0.39): a ~12B single-stream DiT with a Qwen3-VL-4B encoder and the Qwen-Image VAE. Loaded per-component because the repo ships transformers-5.x configs.
DiffusionFamily(
name = "krea-2",
pipeline_class = "Krea2Pipeline",
@@ -365,15 +390,15 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
("int8", "unsloth/Krea-2-Turbo-FP8"),
("fp8", "unsloth/Krea-2-Turbo-FP8"),
),
- # Pre-cast Qwen3-VL-4B TE (8.88 -> 4.83 GB); handed into load_krea2_pipeline directly (assembly never sees pipe_kwargs).
+ # Pre-cast Qwen3-VL-4B TE (8.88 -> 4.83 GB); handed into load_krea2_pipeline directly (assembly never sees
+ # pipe_kwargs).
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Krea-2-Turbo-FP8"),),
aliases = ("krea2",),
- # LoRA training via the DiT trainer (no prequant repo yet, so nf4 quantizes on the fly). Krea guidance: train on the undistilled Raw, run adapters on Turbo.
+ # no prequant repo yet, so nf4 quantizes on the fly
trainable = True,
train_base_repos = ("krea/Krea-2-Raw", "krea/Krea-2-Turbo"),
# Adapters trained on Raw run on Turbo; deploy previews them there (same bf16 precision).
deploy_base_repo = "krea/Krea-2-Turbo",
- # Exported bf16-only; fp16 unvalidated upstream, so keep the fp16 fallback off like z-image.
fp16_incompatible = True,
),
# Lumina Image 2.0: a 2.6B single-stream DiT with a Gemma2-2B encoder and a standard AutoencoderKL, so the generic
@@ -383,56 +408,57 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
pipeline_class = "Lumina2Pipeline",
transformer_class = "Lumina2Transformer2DModel",
base_repo = "Alpha-VLLM/Lumina-Image-2.0",
- # Gate-validated hosted checkpoints (28/28 pairs each; LPIPS mean 0.146 int8 / 0.116 fp8).
+ # gate-validated hosted checkpoints (28/28 pairs each; LPIPS mean 0.146 int8 / 0.116 fp8)
prequant_repos = (
("int8", "unsloth/Lumina-Image-2.0-FP8"),
("fp8", "unsloth/Lumina-Image-2.0-FP8"),
),
- # Pre-cast Gemma2-2B TE. The Hub stores it fp32 (10.46 GB), so the 3.20 GB artifact is a 3.3x download cut.
+ # pre-cast Gemma2-2B TE; the Hub stores it fp32 (10.46 GB), so the 3.20 GB artifact is a 3.3x download cut
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Lumina-Image-2.0-FP8"),),
aliases = ("lumina-image-2.0", "lumina-image-2", "lumina2"),
- # Published and validated bf16-only upstream; keep the fp16 fallback off like z-image.
fp16_incompatible = True,
),
- # HunyuanImage 2.1 (diffusers >= 0.39): a 17B dual-stream DiT with a Qwen2.5-VL encoder, a ByT5 glyph encoder and the
- # 32x HunyuanImage VAE. 2K-native; CFG runs inside the repo guider and the call knob is distilled_guidance_scale.
+ # HunyuanImage 2.1: a 17B dual-stream DiT with a Qwen2.5-VL encoder, a ByT5 glyph encoder and the 32x HunyuanImage
+ # VAE
+ # HiDream-I1: a 17B MoE DiT with FOUR text encoders. The repos ship CLIP-L/CLIP-G/T5-XXL but NOT the Llama-3.1-8B
+ # text_encoder_4 their model_index names, so the loader assembles it from the open unsloth mirror. One family covers
+ # Full/Dev/Fast.
DiffusionFamily(
name = "hunyuanimage-2.1",
- # Hosted checkpoints, verified bit-identical to on-the-fly quantize (the guider pipeline is not run-to-run deterministic, so same-seed LPIPS mixes trajectory divergence with harness noise).
+ # hosted checkpoints, verified bit-identical to on-the-fly quantize (the guider pipeline is not run-to-run
+ # deterministic, so same-seed LPIPS mixes trajectory divergence with harness noise)
prequant_repos = (
("int8", "unsloth/HunyuanImage-2.1-FP8"),
("fp8", "unsloth/HunyuanImage-2.1-FP8"),
),
- # The Qwen2.5-VL TE is byte-identical to Qwen-Image (verified sha256), so reuse that artifact: 16.58 -> 8.84 GB. ByT5 stays dense.
+ # the Qwen2.5-VL TE is byte-identical to Qwen-Image (verified sha256), so reuse that artifact: 16.58 -> 8.84 GB.
te_prequant_repos = (("fp8", "text_encoder", "unsloth/Qwen-Image-FP8"),),
pipeline_class = "HunyuanImagePipeline",
transformer_class = "HunyuanImageTransformer2DModel",
base_repo = "hunyuanvideo-community/HunyuanImage-2.1-Diffusers",
cfg_kwarg = "distilled_guidance_scale",
aliases = ("hunyuanimage-2.1-diffusers", "hunyuanimage2.1"),
- # Exported bf16-only; keep the fp16 fallback off like z-image / krea-2.
fp16_incompatible = True,
),
- # HiDream-I1: a 17B MoE DiT with FOUR text encoders. The repos ship CLIP-L/CLIP-G/T5-XXL but NOT the Llama-3.1-8B
- # text_encoder_4 their model_index names, so the loader assembles it from the open unsloth mirror. One family covers Full/Dev/Fast.
DiffusionFamily(
name = "hidream-i1",
- # Hosted checkpoints: 28/28 per-case gate pairs per scheme (LPIPS 0.291 int8 / 0.278 fp8); int8 bit-identical to on-the-fly quantize.
+ # Hosted checkpoints: 28/28 per-case gate pairs per scheme (LPIPS 0.291 int8 / 0.278 fp8); int8 bit-identical to
+ # on-the-fly quantize.
prequant_repos = (
("int8", "unsloth/HiDream-I1-Full-FP8"),
("fp8", "unsloth/HiDream-I1-Full-FP8"),
),
- # Pre-cast Llama-3.1-8B TE4 (16.1 -> 8.1 GB). The generic TE pass only covers text_encoder.._3, so TE4 engages via hidream_te4_kwargs.
+ # Pre-cast Llama-3.1-8B TE4 (16.1 -> 8.1 GB). The generic TE pass only covers text_encoder.._3, so TE4 engages
+ # via hidream_te4_kwargs.
te_prequant_repos = (("fp8", "text_encoder_4", "unsloth/HiDream-I1-Full-FP8"),),
pipeline_class = "HiDreamImagePipeline",
transformer_class = "HiDreamImageTransformer2DModel",
base_repo = "HiDream-ai/HiDream-I1-Full",
aliases = ("hidream", "hidream-i1-full", "hidream-i1-dev", "hidream-i1-fast"),
- # Exported bf16-only; keep the fp16 fallback off like the other modern DiTs.
fp16_incompatible = True,
),
- # Ideogram 4 (diffusers >= 0.39): a 34-layer DiT PAIR (conditional + unconditional, ~9B each, so planning counts two)
- # with a Qwen3-VL encoder. No bf16 ships: -fp8 is the family base, -nf4 carries bnb-4bit. CFG takes guidance_scale OR a per-step schedule.
+ # Ideogram 4: a 34-layer DiT PAIR (conditional + unconditional, ~9B each, so planning counts two) with a Qwen3-VL
+ # encoder
DiffusionFamily(
name = "ideogram-4",
pipeline_class = "Ideogram4Pipeline",
@@ -442,7 +468,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# Two DiTs assembled per-component, so no transformer-only single-file / GGUF load.
pipeline_only = True,
),
- # SDXL is the one U-Net family: the denoiser is ``pipe.unet`` and a single-file ``.safetensors`` is the WHOLE pipeline. img2img / inpaint / ControlNet are the standard SDXL pipelines. No GGUF path.
+ # SDXL is the one U-Net family: the denoiser is ``pipe.unet`` and a single-file ``.safetensors`` is the WHOLE
+ # pipeline. img2img / inpaint / ControlNet are the standard SDXL pipelines. No GGUF path.
DiffusionFamily(
name = "sdxl",
pipeline_class = "StableDiffusionXLPipeline",
@@ -455,7 +482,6 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
inpaint_pipeline_class = "StableDiffusionXLInpaintPipeline",
controlnet_pipeline_class = "StableDiffusionXLControlNetPipeline",
controlnet_model_class = "ControlNetModel",
- # SDXL uses the U-Net LoRA trainer.
trainable = True,
train_base_repos = (
"stabilityai/stable-diffusion-xl-base-1.0",
@@ -470,17 +496,18 @@ def trainable_family_names() -> tuple[str, ...]:
return tuple(fam.name for fam in _FAMILIES if fam.trainable)
-# The family whose CFG uses a guidance_scale/guidance_schedule pair. Named here so the two modules cannot drift.
+# the family whose CFG uses a guidance_scale/guidance_schedule pair, named here so the two modules cannot drift
IDEOGRAM4_FAMILY_NAME = "ideogram-4"
-# The family whose generate call carries the card CFG-truncation ratio. Named here so the two modules cannot drift.
+# the family whose generate call carries the card CFG-truncation ratio, named here so the two modules cannot drift
LUMINA2_FAMILY_NAME = "lumina-2"
-# Models Unsloth deliberately does NOT support, reason surfaced verbatim in the load error, keyed by lowercase repo-id substring. The bar is a diffusers pipeline.
+# Models Unsloth deliberately does NOT support, reason surfaced verbatim in the load error, keyed by lowercase repo-id
+# substring. The bar is a diffusers pipeline.
_EXCLUDED_MODELS: tuple[tuple[str, str], ...] = (
(
- # "-3" scoped so a future HunyuanImage 2.x with a diffusers pipeline falls through normally.
+ # "-3" scoped so a future HunyuanImage 2.x with a diffusers pipeline falls through normally
"hunyuanimage-3",
"HunyuanImage-3.0 has no diffusers pipeline (it is an 80B autoregressive MoE "
"that requires trust_remote_code), so Unsloth does not support it.",
@@ -497,7 +524,8 @@ def excluded_model_reason(repo_id: str) -> Optional[str]:
return None
-# Editing / inpaint checkpoints share an arch keyword but need a different pipeline + input image. "layered" rejects Qwen-Image-Layered, whose transformer expects an extra input.
+# Editing / inpaint checkpoints share an arch keyword but need a different pipeline + input image. "layered" rejects
+# Qwen-Image-Layered, whose transformer expects an extra input.
_EDIT_KEYWORDS = ("edit", "kontext", "inpaint", "layered")
@@ -535,7 +563,8 @@ def detect_family(repo_id: str, override: Optional[str] = None) -> Optional[Diff
needle = repo_id.lower()
match = _best_family_match(needle)
if match is not None:
- # Do not let a generic family (qwen-image) swallow a variant it cannot run (qwen-image-LAYERED). Scoped to the LAST path component so a parent folder named `edit` does not reject a valid file.
+ # Do not let a generic family (qwen-image) swallow a variant it cannot run (qwen-image-LAYERED). Scoped to the
+ # LAST path component so a parent folder named `edit` does not reject a valid file.
basename = re.split(r"[/\\]+", needle)[-1]
matched_tokens = (match.name, *match.aliases)
if any(
@@ -641,12 +670,10 @@ def detect_family_for_pick(
text-to-image and then refused as an unsupported family (#8407)."""
fam = None
if not override:
- # The checkpoint's own declaration outranks any guess made from its path: a family keyword
- # in ANY ancestor segment otherwise shadows the index (a QwenImagePipeline under
- # `.../flux.1/checkpoint` matched FLUX and never reached it), so the listing, which reads
- # the index, named one family and the loader another for one directory.
- # Remote picks are unaffected: with no local index this is None and the name-based paths
- # below run as before.
+ # The checkpoint's own declaration outranks any guess made from its path: a family keyword in ANY ancestor
+ # segment otherwise shadows the index (a QwenImagePipeline under `.../flux.1/checkpoint` matched FLUX and never
+ # reached it), so the listing, which reads the index, named one family and the loader another for one directory.
+ # Remote picks are unaffected: with no local index this is None and the name-based paths below run as before.
fam = detect_family_by_pipeline_index(repo_id)
if fam is None:
fam = detect_family(repo_id, override)
@@ -661,12 +688,11 @@ def resolve_base_repo(fam: DiffusionFamily, base_repo: Optional[str]) -> str:
return base or fam.base_repo
-# Byte-identical unsloth mirrors of the vendor bases: a GGUF/FP8 pick ships only the denoiser, so
-# companions come from the base, which is a 401 for gated vendors and a third-party fetch for the
-# rest. Swapped at the fetch sites only, never in ``resolve_base_repo``, whose result keys the
-# UPSTREAM-id tables below (see ``canonical_base``).
-# A mirror stands in for the WHOLE base, bf16 pipeline loads included, so it must be a complete
-# copy: a companions-only repo breaks every pick that needs the transformer.
+# Byte-identical unsloth mirrors of the vendor bases: a GGUF/FP8 pick ships only the denoiser, so companions come from
+# the base, which is a 401 for gated vendors and a third-party fetch for the rest. Swapped at the fetch sites only,
+# never in ``resolve_base_repo``, whose result keys the UPSTREAM-id tables below (see ``canonical_base``). A mirror
+# stands in for the WHOLE base, bf16 pipeline loads included, so it must be a complete copy: a companions-only repo
+# breaks every pick that needs the transformer.
_GATED_MIRROR_PAIRS: tuple[tuple[str, str], ...] = (
("black-forest-labs/FLUX.1-dev", "unsloth/FLUX.1-dev"),
("black-forest-labs/FLUX.1-schnell", "unsloth/FLUX.1-schnell"),
@@ -682,23 +708,20 @@ _GATED_MIRROR_PAIRS: tuple[tuple[str, str], ...] = (
("ideogram-ai/ideogram-4-nf4-diffusers", "unsloth/ideogram-4-nf4-diffusers"),
)
-# Mirrored to drop the third-party fetch, NOT to route around a gate. Every licence here
-# permits redistribution, and each mirror carries the upstream licence text plus the notice
-# that licence prescribes. Qwen-Image-2512 is the one no other redirect could reach: its
-# companions are named by the artifact repo's base_model card tag, not the family table.
-#
-# Kept apart from the gated pairs because the two answer different questions. Both redirect
-# a fetch, but only a GATED upstream justifies overriding a user's existing cache: for these
-# the upstream is reachable without credentials, so a complete local snapshot must keep being
-# used rather than re-pulled from the mirror.
+# Mirrored to drop the third-party fetch, NOT to route around a gate. Every licence here permits redistribution, and
+# each mirror carries the upstream licence text plus the notice that licence prescribes. Qwen-Image-2512 is the one no
+# other redirect could reach: its companions are named by the artifact repo's base_model card tag, not the family table.
+# Kept apart from the gated pairs because the two answer different questions. Both redirect a fetch, but only a GATED
+# upstream justifies overriding a user's existing cache: for these the upstream is reachable without credentials, so a
+# complete local snapshot must keep being used rather than re-pulled from the mirror.
_UNGATED_MIRROR_PAIRS: tuple[tuple[str, str], ...] = (
("Qwen/Qwen-Image-2512", "unsloth/Qwen-Image-2512"),
("Qwen/Qwen-Image", "unsloth/Qwen-Image"),
("Qwen/Qwen-Image-Edit-2511", "unsloth/Qwen-Image-Edit-2511"),
("black-forest-labs/FLUX.2-klein-4B", "unsloth/FLUX.2-klein-4B"),
- # Lookup is by exact id, so every VARIANT a pick can resolve to needs its own row: the base-4B
- # is what `unsloth/FLUX.2-klein-base-4B-GGUF` resolves to from its card tag, and Dev / Fast are
- # offered directly as bf16 pipeline picks. Without these three the table silently misses them.
+ # Lookup is by exact id, so every VARIANT a pick can resolve to needs its own row: the base-4B is what
+ # `unsloth/FLUX.2-klein-base-4B-GGUF` resolves to from its card tag, and Dev / Fast are offered directly as bf16
+ # pipeline picks. Without these three the table silently misses them.
("black-forest-labs/FLUX.2-klein-base-4B", "unsloth/FLUX.2-klein-base-4B"),
("Tongyi-MAI/Z-Image-Turbo", "unsloth/Z-Image-Turbo"),
("Alpha-VLLM/Lumina-Image-2.0", "unsloth/Lumina-Image-2.0"),
@@ -707,10 +730,6 @@ _UNGATED_MIRROR_PAIRS: tuple[tuple[str, str], ...] = (
("HiDream-ai/HiDream-I1-Fast", "unsloth/HiDream-I1-Fast"),
("stabilityai/stable-diffusion-xl-base-1.0", "unsloth/stable-diffusion-xl-base-1.0"),
("stabilityai/sdxl-turbo", "unsloth/sdxl-turbo"),
- # NOT mirrored: hunyuanvideo-community/HunyuanImage-2.1-Diffusers. The Tencent Hunyuan
- # Community License permits distribution "exclusively in the Territory", and the Territory
- # excludes the EU, the UK and South Korea. A public Hub repo distributes worldwide, so that
- # mirror cannot be made compliant and the family keeps fetching upstream.
)
_MIRROR_PAIRS: tuple[tuple[str, str], ...] = _GATED_MIRROR_PAIRS + _UNGATED_MIRROR_PAIRS
_GATED_MIRRORS: dict[str, str] = {u.lower(): m for u, m in _MIRROR_PAIRS}
@@ -742,7 +761,7 @@ def canonical_base(repo_id: Optional[str]) -> str:
return _MIRROR_UPSTREAM.get(base.lower(), base)
-# What counts as a real weight file, vs the stray config an interrupted pull leaves behind.
+# what counts as a real weight file, vs the stray config an interrupted pull leaves behind
_WEIGHT_SUFFIXES = frozenset({".safetensors", ".bin", ".pt", ".ckpt", ".gguf"})
@@ -836,15 +855,15 @@ def _upstream_is_cached(
if other_root:
from huggingface_hub import constants
- # Exactly what ``cache_dir = None`` resolves to, i.e. the root the per-file reuse
- # probe falls back to (file_download reads constants.HF_HUB_CACHE per call).
+ # Exactly what ``cache_dir = None`` resolves to, i.e. the root the per-file reuse probe falls back to
+ # (file_download reads constants.HF_HUB_CACHE per call).
fallback = Path(constants.HF_HUB_CACHE)
if fallback != roots[0]:
roots.append(fallback)
wanted = tuple(files or ())
if wanted and len(roots) > 1:
- # One revision per root, unioned across roots: a name may come from either root, but
- # within a root it has to come from the revision that root's fetch would resolve.
+ # One revision per root, unioned across roots: a name may come from either root, but within a root it has to
+ # come from the revision that root's fetch would resolve.
reachable = {frozenset()}
for root in roots:
covers = _root_revision_coverage(root, repo_id, wanted)
@@ -872,10 +891,10 @@ def cache_holds_files(repo_id: str, files: Sequence[str]) -> bool:
return bool(files) and _upstream_is_cached(repo_id, tuple(files))
-# Lowercased unsloth mirror -> the community repack the tables named before it. The mirrors are
-# byte identical, so an existing install that already pulled the repack holds the very same bytes
-# under the OLD repo key: the HF cache is keyed by repo id, so re-pointing the table alone would
-# re-download up to tens of GB on upgrade and fail outright offline.
+# Lowercased unsloth mirror -> the community repack the tables named before it. The mirrors are byte identical, so an
+# existing install that already pulled the repack holds the very same bytes under the OLD repo key: the HF cache is
+# keyed by repo id, so re-pointing the table alone would re-download up to tens of GB on upgrade and fail outright
+# offline.
_SD_CPP_LEGACY_SOURCES: dict[str, str] = {
"unsloth/flux-text-encoders": "comfyanonymous/flux_text_encoders",
"unsloth/qwen-image-comfyui": "Comfy-Org/Qwen-Image_ComfyUI",
@@ -884,10 +903,9 @@ _SD_CPP_LEGACY_SOURCES: dict[str, str] = {
"unsloth/z-image-turbo-comfyui": "Comfy-Org/z_image_turbo",
"unsloth/flux.2-klein-9b-comfyui": "Comfy-Org/vae-text-encorder-for-flux-klein-9b",
"unsloth/wan2.2-ti2v-5b-gguf": "QuantStack/Wan2.2-TI2V-5B-GGUF",
- # MiniMax-H3, where only PART of the mirror came from the repack: the two VAEs now sit beside
- # the denoisers in the GGUF repo, and the int8 ConvRot conditioner beside the other
- # prequantized checkpoints. That is why every caller passes the exact file list -- a denoiser
- # the repack never carried simply misses the probe and keeps the mirror.
+ # MiniMax-H3, where only PART of the mirror came from the repack: the two VAEs now sit beside the denoisers in the
+ # GGUF repo, and the int8 ConvRot conditioner beside the other prequantized checkpoints. That is why every caller
+ # passes the exact file list -- a denoiser the repack never carried simply misses the probe and keeps the mirror.
"unsloth/minimax-h3-gguf": "Comfy-Org/MiniMax-H3",
"unsloth/minimax-h3-fp8": "Comfy-Org/MiniMax-H3",
}
@@ -958,17 +976,19 @@ def prefer_ungated_mirror(
mirror = mirror_repo(base)
if not mirror or os.environ.get("UNSLOTH_DIFFUSION_NO_MIRROR", "").strip():
return base
- # A local path is never a Hub id: rewriting one sends loads the other sites resolve on disk
- # (``Path(base).exists()``) to the Hub, skipping the copy already downloaded.
+ # a local path is never a Hub id: rewriting one sends loads the other sites resolve on disk to the Hub, skipping the
+ # copy already downloaded
if _is_local_path(base):
return base
return base if _upstream_is_cached(base, files) else mirror
-# Default (steps, guidance) per model for callers that cannot pass them. Matched by substring, most specific first; same values as the UI MODEL_DEFAULTS table, keep in sync.
+# Default (steps, guidance) per model for callers that cannot pass them. Matched by substring, most specific first; same
+# values as the UI MODEL_DEFAULTS table, keep in sync.
_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
("z-image-turbo", 9, 0.0),
- # FLUX.1 Krea dev is a FLUX.1-dev finetune, NOT a Krea-2: 28 steps at guidance 4.5. Must precede the generic "krea" key.
+ # FLUX.1 Krea dev is a FLUX.1-dev finetune, NOT a Krea-2: 28 steps at guidance 4.5. Must precede the generic "krea"
+ # key.
("flux.1-krea", 28, 4.5),
# Krea 2 Raw (undistilled): 52 steps / guidance 3.5. Must precede the generic "krea" key.
("krea-2-raw", 52, 3.5),
@@ -977,8 +997,8 @@ _GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
("flux.1-schnell", 4, 0.0),
("kontext", 28, 2.5), # editing: before the generic flux.1
("flux.1", 28, 3.5),
- # The undistilled base variants need their model-card 50-step CFG recipe. Keep this before
- # the generic distilled key, which covers both 4B and 9B 4-step checkpoints.
+ # The undistilled base variants need their model-card 50-step CFG recipe. Keep this before the generic distilled
+ # key, which covers both 4B and 9B 4-step checkpoints.
("flux.2-klein-base", 50, 4.0),
("flux.2-klein", 4, 1.0),
("flux.2-dev", 28, 4.0), # full (non-distilled)
@@ -986,13 +1006,16 @@ _GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
("z-image", 20, 4.0),
# Lumina Image 2.0 card: 50 steps, guidance 4 (plus cfg_trunc_ratio 0.25, which the loader passes itself).
("lumina", 50, 4.0),
+ # HunyuanImage 2.1 card: guidance feeds distilled_guidance_scale, while real CFG runs inside the guiders
# HunyuanImage 2.1 card: 50 steps; guidance feeds distilled_guidance_scale, while real CFG runs inside the guiders.
("hunyuanimage", 50, 3.25),
# HiDream-I1 upstream: Full 50 steps / guidance 5; the distilled Dev (28) and Fast (16) run guidance-free.
("hidream-i1-dev", 28, 0.0),
("hidream-i1-fast", 16, 0.0),
("hidream", 50, 5.0),
- # Ideogram 4 card: 48 steps, guidance 7 (its schedule tapers the last 3 steps; the loader keeps that taper at these defaults).
+ # Ideogram 4 card: its schedule tapers the last 3 steps
+ # Ideogram 4 card: 48 steps, guidance 7 (its schedule tapers the last 3 steps; the loader keeps that taper at these
+ # defaults).
("ideogram", 48, 7.0),
# SDXL: Turbo distilled; base wants ~30 steps + CFG ~7. "sdxl-turbo" precedes "sdxl".
("sdxl-turbo", 3, 0.0),
@@ -1031,10 +1054,10 @@ def family_prequant_repo(
# Both tables are keyed on lowercased upstream ids.
base = canonical_base(base_repo).lower()
if base:
- # getattr, because the video loader calls this with a VideoFamily, which has no such
- # field. A plain attribute read raises AttributeError, resolve_prequant_source swallows it
- # in its bare except and hands back None, and every video family silently loses its hosted
- # prequant checkpoint to the dense path whenever a base_repo is passed.
+ # getattr, because the video loader calls this with a VideoFamily, which has no such field. A plain attribute
+ # read raises AttributeError, resolve_prequant_source swallows it in its bare except and hands back None, and
+ # every video family silently loses its hosted prequant checkpoint to the dense path whenever a base_repo is
+ # passed.
if base in (getattr(fam, "prequant_excluded_bases", ()) or ()):
return None
for entry_base, entry_scheme, repo_id in fam.prequant_variant_repos:
@@ -1088,16 +1111,15 @@ def family_prequant_filename(
return agnostic
-# The release where diffusers' own requires-python went ">= 3.10.0", which makes 0.36.0 the newest
-# a supported Python 3.9 host can resolve.
+# the release where diffusers' own requires-python went ">= 3.10.0", making 0.36.0 the newest a supported Python 3.9
+# host can resolve
_DIFFUSERS_DROPPED_PY39 = "0.37.0"
-# First diffusers release exporting each pipeline class, read off ``src/diffusers/__init__.py`` at
-# the upstream tags and cross-checked against each release's requires-python on PyPI. An unlisted
-# class gets a version-free "a newer diffusers" instead of a number, since the ones left out are
-# older than any release in play -- and ``family_pipeline_available`` reads one as available, so
-# every class the listing probes belongs here. This exists so the remedy is true: telling a
-# 3.9 host that Z-Image needs Python >= 3.10 sends it to upgrade the interpreter when
+# First diffusers release exporting each pipeline class, read off ``src/diffusers/__init__.py`` at the upstream tags and
+# cross-checked against each release's requires-python on PyPI. An unlisted class gets a version-free "a newer
+# diffusers" instead of a number, since the ones left out are older than any release in play -- and
+# ``family_pipeline_available`` reads one as available, so every class the listing probes belongs here. This exists so
+# the remedy is true: telling a 3.9 host that Z-Image needs Python >= 3.10 sends it to upgrade the interpreter when
# ``pip install -U diffusers`` (0.36.0 there) would have been enough.
_PIPELINE_MIN_DIFFUSERS: dict[str, str] = {
# MiniMax-H3 is judged by its transformer, first available in diffusers 0.40.0.
@@ -1115,16 +1137,16 @@ _PIPELINE_MIN_DIFFUSERS: dict[str, str] = {
"Flux2KleinInpaintPipeline": "0.38.0",
"Ideogram4Pipeline": "0.39.0",
"Krea2Pipeline": "0.39.0",
- # Older than the 0.35 baseline, but listed anyway: the packaging leaves an UNCONSTRAINED
- # diffusers installable below 3.10, so an already-present ancient one satisfies the pin, and
- # quoting the 0.39 floor at a family that has existed since 0.30 is the same wrong remedy.
+ # Older than the 0.35 baseline, but listed anyway: the packaging leaves an UNCONSTRAINED diffusers installable below
+ # 3.10, so an already-present ancient one satisfies the pin, and quoting the 0.39 floor at a family that has existed
+ # since 0.30 is the same wrong remedy.
"QwenImagePipeline": "0.35.0",
"QwenImageImg2ImgPipeline": "0.35.0",
"QwenImageInpaintPipeline": "0.35.0",
"FluxKontextPipeline": "0.35.0",
"HiDreamImagePipeline": "0.34.0",
- # WanPipeline arrived with Wan2.1 in 0.33.0. The shipped Wan2.2 family wants weights only
- # 0.35 carries, but this table answers class presence, like the attribute probe before it.
+ # WanPipeline arrived with Wan2.1 in 0.33.0. The shipped Wan2.2 family wants weights only 0.35 carries, but this
+ # table answers class presence, like the attribute probe before it.
"WanPipeline": "0.33.0",
"Lumina2Pipeline": "0.33.0",
"FluxPipeline": "0.30.0",
@@ -1235,17 +1257,15 @@ def assert_pipeline_class_available(
dummy_backends = _dummy_required_backends(getattr(diffusers, pipeline_class, None))
except Exception as exc: # noqa: BLE001 -- see below: this check must never raise anything but its own ValueError
# Not this check's business under the default: it answers "is the installed diffusers new enough for this
- # family", and with nothing importable there is no version to judge. Refusing would also break the native
- # sd.cpp engine, which serves GGUF picks on a CPU or Apple host without diffusers. A pick that really needs
- # it fails later, in the loader. The one thing that must not happen is a raise of the wrong type:
- # ModuleNotFoundError is not the ValueError the routes map to 400, so it escapes /images/download-plan as a
- # bare 500 with the message lost.
- #
- # The attribute probe is inside the try for the same reason. diffusers' top level is a lazy module, so
- # ``hasattr`` is what actually imports the pipeline's submodule, and when that submodule's own dependencies
- # are unsatisfiable it raises RuntimeError ("Failed to import diffusers.pipelines...") -- which hasattr does
- # NOT swallow, since it only absorbs AttributeError. A partially usable diffusers install therefore escaped
- # this guard exactly the way a missing one used to.
+ # family", and with nothing importable there is no version to judge. Refusing would also break the native sd.cpp
+ # engine, which serves GGUF picks on a CPU or Apple host without diffusers. A pick that really needs it fails
+ # later, in the loader. The one thing that must not happen is a raise of the wrong type: ModuleNotFoundError is
+ # not the ValueError the routes map to 400, so it escapes /images/download-plan as a bare 500 with the message
+ # lost. The attribute probe is inside the try for the same reason. diffusers' top level is a lazy module, so
+ # ``hasattr`` is what actually imports the pipeline's submodule, and when that submodule's own dependencies are
+ # unsatisfiable it raises RuntimeError ("Failed to import diffusers.pipelines...") -- which hasattr does NOT
+ # swallow, since it only absorbs AttributeError. A partially usable diffusers install therefore escaped this
+ # guard exactly the way a missing one used to.
if strict:
raise ValueError(
f"'{family_name}' needs diffusers ({pipeline_class}), which this environment "
@@ -1254,9 +1274,9 @@ def assert_pipeline_class_available(
return
if present and dummy_backends:
- # A placeholder, not the pipeline. Under the default this is left alone like every other
- # unusable install; strict refuses, because the trainer child imports the same placeholder
- # and its from_pretrained raises only after the GPU residents are gone.
+ # A placeholder, not the pipeline. Under the default this is left alone like every other unusable install;
+ # strict refuses, because the trainer child imports the same placeholder and its from_pretrained raises only
+ # after the GPU residents are gone.
if not strict:
return
raise ValueError(
@@ -1346,13 +1366,13 @@ def family_pipeline_available(fam: Optional[DiffusionFamily]) -> bool:
dependencies. The load path remains the final availability check."""
if fam is None:
return False
- # A modular family is judged on its own transformer class, not on the generic
- # ``ModularPipeline`` entry point -- see ``family_probe_class``.
+ # A modular family is judged on its own transformer class, not on the generic ``ModularPipeline`` entry point -- see
+ # ``family_probe_class``.
name = family_probe_class(fam)
- # No class name to probe means the record is not one this helper can judge, which is the same
- # position as a missing diffusers: answer OPEN. ``family_probe_class`` reads the attribute with
- # a default, so a record without ``pipeline_class`` reaches here as "" rather than raising into
- # the guard below, and ``hasattr(diffusers, "")`` is False -- which would hide the model.
+ # No class name to probe means the record is not one this helper can judge, which is the same position as a missing
+ # diffusers: answer OPEN. ``family_probe_class`` reads the attribute with a default, so a record without
+ # ``pipeline_class`` reaches here as "" rather than raising into the guard below, and ``hasattr(diffusers, "")`` is
+ # False -- which would hide the model.
if not name:
return True
if "diffusers" in sys.modules:
@@ -1366,7 +1386,6 @@ def family_pipeline_available(fam: Optional[DiffusionFamily]) -> bool:
except Exception: # noqa: BLE001 -- a probe failure must not hide a model
return True
minimum, _needs_py310 = pipeline_class_requirement(name)
- # Unlisted classes predate the version gates in this table.
if minimum is None:
return True
installed = _installed_diffusers_version()
@@ -1392,8 +1411,8 @@ def family_sd_cpp_supported(fam: DiffusionFamily) -> bool:
return bool(fam.sd_cpp_vae and fam.sd_cpp_text_encoders)
-# FLUX.2-klein 9B pairs with Qwen3-8B, the 4B with the family-default Qwen3-4B (a mismatched encoder fails deep in sd-cli), so pick per variant.
-# Byte-identical mirror of Comfy-Org/vae-text-encorder-for-flux-klein-9b.
+# FLUX.2-klein 9B pairs with Qwen3-8B, the 4B with the family-default Qwen3-4B (a mismatched encoder fails deep in
+# sd-cli), so pick per variant. Byte-identical mirror of Comfy-Org/vae-text-encorder-for-flux-klein-9b.
_FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS = (
(
"unsloth/FLUX.2-klein-9B-ComfyUI",
@@ -1465,24 +1484,24 @@ def sd_cpp_text_encoders_for(
the fallback for the callers that have no header to read (the delete guard reconstructs a
committed load; the plan runs before a byte is fetched)."""
if fam.name == "flux.2-klein":
- # A dim this family does not have (a FLUX.2-dev file, a misread) falls through to the
- # strings rather than guessing, exactly as the base-size guard fails open on one.
+ # A dim this family does not have (a FLUX.2-dev file, a misread) falls through to the strings rather than
+ # guessing, exactly as the base-size guard fails open on one.
if inner_dim == _FLUX2_KLEIN_9B_INNER_DIM:
return _FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS
if inner_dim == _FLUX2_KLEIN_4B_INNER_DIM:
return fam.sd_cpp_text_encoders
identity = f"{repo_id or ''}/{gguf_filename or ''}".lower()
- # Match the size token on its own: klein-BASE-9B is 9B too, and matching "klein-9b"
- # literally handed it the 4B text encoder.
+ # Match the size token on its own: klein-BASE-9B is 9B too, and matching "klein-9b" literally handed it the 4B
+ # text encoder.
if _token_in_needle("9b", identity) or "klein9b" in identity:
return _FLUX2_KLEIN_9B_SD_CPP_TEXT_ENCODERS
return fam.sd_cpp_text_encoders
-# FLUX.2 sizes the adaLN modulation projection as (6 * inner_dim, inner_dim), and inner_dim is the
-# only thing that differs between the klein sizes and dev. sd.cpp keys its own FLUX.2 version
-# detection on this same tensor, and GGUF metadata cannot help: our files and leejet's carry no kv
-# pairs at all, and city96/orabazes write general.architecture = "flux" for FLUX.1 and FLUX.2 alike.
+# FLUX.2 sizes the adaLN modulation projection as (6 * inner_dim, inner_dim), and inner_dim is the only thing that
+# differs between the klein sizes and dev. sd.cpp keys its own FLUX.2 version detection on this same tensor, and GGUF
+# metadata cannot help: our files and leejet's carry no kv pairs at all, and city96/orabazes write general.architecture
+# = "flux" for FLUX.1 and FLUX.2 alike.
_FLUX2_PROBE_TENSOR = "double_stream_modulation_img.lin.weight"
_FLUX2_INNER_DIMS = {
3072: "FLUX.2-klein-4B / klein-base-4B",
@@ -1521,9 +1540,9 @@ def _flux2_inner_dim_from_tensors(tensors) -> Optional[int]:
"""``inner_dim`` from a parsed GGUF tensor table, or None when the probe tensor is absent."""
for t in tensors:
if t.name == _FLUX2_PROBE_TENSOR or t.name.endswith("." + _FLUX2_PROBE_TENSOR):
- # GGUF stores dims reversed relative to torch, so the input dim leads. A missing or
- # non-positive dim is a parse that went wrong, and "0" would be compared against the
- # base as a real answer and refuse a valid pick; say nothing instead.
+ # GGUF stores dims reversed relative to torch, so the input dim leads. A missing or non-positive dim is a
+ # parse that went wrong, and "0" would be compared against the base as a real answer and refuse a valid
+ # pick; say nothing instead.
dim = int(t.shape[0]) if len(t.shape) else 0
return dim if dim > 0 else None
return None
@@ -1579,23 +1598,24 @@ def gguf_flux2_inner_dim_from_header(header: bytes) -> Optional[int]:
return super()._get(offset, dtype, count, override_order)
def _build_tensors(self, start_offs, fields):
- # ``field.name`` is the decoded tensor name and parts[3] its dims, both already
- # read from real bytes by ``_get_tensor_info_field``.
+ # ``field.name`` is the decoded tensor name and parts[3] its dims, both already read from real bytes by
+ # ``_get_tensor_info_field``.
self.tensors = [_HeaderTensor(field.name, field.parts[3]) for field in fields]
- # GGUFReader memory-maps a path, so the prefix has to land on disk. Bounded by the
- # caller's range request, so this is KiB, never the checkpoint.
+ # GGUFReader memory-maps a path, so the prefix has to land on disk. Bounded by the caller's range request, so
+ # this is KiB, never the checkpoint.
fd, tmp_path = tempfile.mkstemp(suffix = ".gguf-header")
with os.fdopen(fd, "wb") as fh:
fh.write(header)
reader = _HeaderOnlyGGUFReader(tmp_path)
- # Belt and braces on top of the refusing ``_get``: ``data_offset`` is where the table
- # ends, so a table that ran past the prefix is not one to read a shape out of. The
- # alignment slack is for a header-only file, whose padding was never written.
+ # `data_offset` is where the table ends, so a table that ran past the prefix is not one to read a shape out of;
+ # Belt and braces on top of the refusing ``_get``: ``data_offset`` is where the table ends, so a table that ran
+ # past the prefix is not one to read a shape out of. The alignment slack is for a header-only file, whose
+ # padding was never written.
if reader.data_offset > len(header) + min(int(reader.alignment), 4096):
return None
return _flux2_inner_dim_from_tensors(reader.tensors)
- except Exception: # noqa: BLE001 — an unreadable header is not a verdict
+ except Exception: # noqa: BLE001 - an unreadable header is not a verdict
return None
finally:
# Drop the mmap before unlinking: Windows refuses to delete a mapped file.
@@ -1640,8 +1660,8 @@ def assert_flux2_gguf_matches_base(fam, base_repo: str, gguf_path) -> None:
before the resident pipeline is torn down, so a mismatch normally never reaches here."""
if gguf_path is None or not str(getattr(fam, "name", "")).startswith("flux.2"):
return
- # Short-circuit on the free half: an unmapped base fails open anyway, so a mirror id must not
- # cost a memory-map of the whole checkpoint.
+ # Short-circuit on the free half: an unmapped base fails open anyway, so a mirror id must not cost a memory-map of
+ # the whole checkpoint.
want = flux2_base_inner_dim(base_repo)
if want is None:
return
@@ -1665,7 +1685,7 @@ def resolve_local_gguf_child(repo_root: Path, gguf_filename: str) -> Path:
rel = PurePosixPath(gguf_filename)
if any(part in ("", ".", "..") for part in rel.parts):
raise ValueError("gguf_filename must not contain '', '.', or '..' segments.")
- # Resolve symlinks before the containment check (the lexical guards miss a symlink escape).
+ # resolve symlinks before the containment check (the lexical guards miss a symlink escape)
repo_real = repo_root.resolve()
child = repo_root.joinpath(*rel.parts).resolve()
if child != repo_real and repo_real not in child.parents:
diff --git a/studio/backend/core/inference/diffusion_gguf_compile.py b/studio/backend/core/inference/diffusion_gguf_compile.py
index ee636c1e2d1..db50a18187f 100644
--- a/studio/backend/core/inference/diffusion_gguf_compile.py
+++ b/studio/backend/core/inference/diffusion_gguf_compile.py
@@ -32,6 +32,7 @@ from typing import Any
from .diffusion_patch_backend import apply_patch, revert_patch
+
# --- kill-switch -------------------------------------------------------------------
_ENV_COMPILE_DEQUANT = "UNSLOTH_DIFFUSION_GGUF_COMPILE_DEQUANT"
@@ -48,13 +49,10 @@ def _gguf_utils():
try:
from diffusers.quantizers.gguf import utils as gguf_utils # noqa: PLC0415
return gguf_utils
- except Exception: # noqa: BLE001 — old/!GGUF diffusers -> accelerator is a no-op
+ except Exception: # noqa: BLE001 - old/!GGUF diffusers -> accelerator is a no-op
return None
-# --- compiled dequant --------------------------------------------------------------
-
-# True while the compiled wrapper is installed (the patch backend stashes the original).
_compiled_dequant_installed = False
_DEQUANT_ATTR = "dequantize_gguf_tensor"
@@ -78,12 +76,13 @@ def install_compiled_dequant(logger: Any = None) -> bool:
import torch # noqa: PLC0415
compiled = torch.compile(gguf_utils.dequantize_gguf_tensor, dynamic = True)
- # force=True: the compiled callable's fingerprint differs from the original, which can_safely_patch would correctly reject.
+ # force=True: the compiled callable's fingerprint differs from the original, which can_safely_patch would
+ # correctly reject.
if apply_patch(gguf_utils, _DEQUANT_ATTR, compiled, force = True):
_compiled_dequant_installed = True
return True
return False
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "install_compiled_dequant", exc)
_compiled_dequant_installed = False
return False
diff --git a/studio/backend/core/inference/diffusion_hidream.py b/studio/backend/core/inference/diffusion_hidream.py
index e2b7f26900f..bf49c90f481 100644
--- a/studio/backend/core/inference/diffusion_hidream.py
+++ b/studio/backend/core/inference/diffusion_hidream.py
@@ -53,10 +53,10 @@ def hidream_te4_kwargs(
import torch # noqa: F401 -- dtype values are torch dtypes; import keeps parity with callers
from transformers import AutoTokenizer, LlamaForCausalLM
- # Pinned to the LIVE hub root: ``encoder_repo_complete`` verifies these assets there, so an
- # unpinned lookup after a mid-session cache-folder change searches huggingface_hub's
- # import-time root instead and fails under local_files_only for a 16 GB encoder that is
- # present, after the resident image pipeline was evicted.
+ # pin the LIVE hub root: an unpinned lookup uses the import-time root and fails under local_files_only
+ # Pinned to the LIVE hub root: ``encoder_repo_complete`` verifies these assets there, so an unpinned lookup after a
+ # mid-session cache-folder change searches huggingface_hub's import-time root instead and fails under
+ # local_files_only for a 16 GB encoder that is present, after the resident image pipeline was evicted.
from utils.hf_cache_settings import active_hf_hub_cache
cache_dir = active_hf_hub_cache()
@@ -110,7 +110,8 @@ def hidream_te4_kwargs(
hf_token = hf_token,
scheme = "fp8",
logger = logger,
- # The Llama TE4 lives in its own standalone repo (config at the root), and the pipeline needs hidden states/attentions from its forward.
+ # The Llama TE4 lives in its own standalone repo (config at the root), and the pipeline needs hidden
+ # states/attentions from its forward.
config_subfolder = "",
config_overrides = {
"output_hidden_states": True,
@@ -143,7 +144,8 @@ def hidream_te4_kwargs(
_cast_fp8(text_encoder_4, cast_target)
logger.info("diffusion.hidream: TE4 layerwise fp8 cast engaged")
except Exception as exc: # noqa: BLE001 -- best-effort like the generic TE pass
- # A mid-pass failure can leave fp8 storage / upcast hooks behind, and a half-cast encoder cannot run dense, so rebuild it fresh.
+ # A mid-pass failure can leave fp8 storage / upcast hooks behind, and a half-cast encoder cannot run dense,
+ # so rebuild it fresh.
logger.warning("diffusion.hidream: TE4 fp8 cast failed, reloading dense: %s", exc)
text_encoder_4 = LlamaForCausalLM.from_pretrained(
HIDREAM_LLAMA_REPO,
diff --git a/studio/backend/core/inference/diffusion_ideogram4.py b/studio/backend/core/inference/diffusion_ideogram4.py
index 60f04f46aaf..14e8f46d85d 100644
--- a/studio/backend/core/inference/diffusion_ideogram4.py
+++ b/studio/backend/core/inference/diffusion_ideogram4.py
@@ -71,7 +71,9 @@ def _patch_create_causal_mask() -> None:
_CAUSAL_MASK_PATCHED = True
-# The fp8 attention is a fused ``qkv`` matrix (Q/K/V stacked, each ``hidden_size`` rows), read from config so a future change cannot mis-split it.
+# fused qkv (Q/K/V stacked, each hidden_size rows), read from config so a future change cannot mis-split it
+# The fp8 attention is a fused ``qkv`` matrix (Q/K/V stacked, each ``hidden_size`` rows), read from config so a future
+# change cannot mis-split it.
_QKV_SPLIT = ("to_q", "to_k", "to_v")
@@ -141,9 +143,9 @@ def _convert_fp8_state_dict(raw: dict, hidden_size: int, dtype) -> dict:
converted[key] = value.to(dtype)
continue
if key.endswith("attention.qkv.weight"):
- fused = dequantize(key) # [3 * hidden_size, hidden_size]
+ fused = dequantize(key)
if fused.shape[0] != 3 * hidden_size:
- # Equal-thirds only holds for full multi-head attention; a GQA export must fail loudly.
+ # equal thirds only holds for full multi-head attention; a GQA export must fail loudly
raise RuntimeError(
f"fused qkv at {key} has {fused.shape[0]} rows, expected "
f"{3 * hidden_size}; cannot split into equal Q/K/V blocks"
@@ -207,7 +209,6 @@ def _text_encoder_is_fp8(repo_id: str, token: Optional[str]) -> bool:
return any(k.endswith("_scale") for k in weight_map)
except Exception: # noqa: BLE001 -- single-file (nf4) text encoder, not fp8
return False
- # Single-file local text encoder: peek the header keys.
import safetensors
single = local_root / "text_encoder" / "model.safetensors"
@@ -263,7 +264,7 @@ def load_ideogram4_text_encoder(
else:
state_dict[key] = value.to(dtype)
- # Construct normally (so __init__ computes the non-persistent rotary inv_freq the checkpoint omits), then copy the dequantized weights in. Build at the target dtype: this ~8B Qwen3-VL scaffold is ~2x at the fp32 default and loads FIRST, so fp32 can OOM a 64 GB host.
+ # build at the target dtype: this ~8B scaffold loads FIRST, so the fp32 default can OOM a 64 GB host
default_dtype = torch.get_default_dtype()
torch.set_default_dtype(dtype)
try:
@@ -320,7 +321,9 @@ def load_ideogram4_transformer(
config = _read_transformer_config(repo_id, subfolder, token)
shard_paths = _transformer_shard_paths(repo_id, subfolder, token)
- # Detect fp8 from shard HEADERS (keys() reads metadata only), checking all shards so a dense-first multi-shard export still routes to the dequant path. Only fp8 materializes tensors; -nf4 goes straight to from_pretrained.
+ # check every shard header: a dense-first multi-shard fp8 export must still route to dequant
+ # Detect fp8 from shard HEADERS (keys() reads metadata only), checking all shards so a dense-first multi-shard
+ # export still routes to the dequant path. Only fp8 materializes tensors; -nf4 goes straight to from_pretrained.
is_fp8 = False
for path in shard_paths:
with safetensors.safe_open(path, "pt") as handle:
@@ -342,7 +345,8 @@ def load_ideogram4_transformer(
config.pop("quantization_config", None)
hidden_size = int(config["attention_head_dim"]) * int(config["num_attention_heads"])
- # from_config materializes the full ~9B module before the weights copy in, and at fp32 that is ~2x the bf16 model while the second DiT builds with the first + encoder resident, so build at the target dtype. Only rotary_emb.inv_freq is absent from the checkpoint.
+ # build at the target dtype: from_config materializes the full ~9B module while the first DiT and the encoder are
+ # resident
default_dtype = torch.get_default_dtype()
torch.set_default_dtype(dtype)
try:
@@ -351,7 +355,8 @@ def load_ideogram4_transformer(
torch.set_default_dtype(default_dtype)
state_dict = _convert_fp8_state_dict(raw, hidden_size, dtype)
missing, unexpected = model.load_state_dict(state_dict, strict = False)
- # rotary_emb.inv_freq is the only expected "missing" key (built in __init__); a real gap or leftover key must fail loudly rather than ship a partly random model.
+ # rotary_emb.inv_freq is the only expected "missing" key (built in __init__); a real gap or leftover key must fail
+ # loudly rather than ship a partly random model.
real_missing = [k for k in missing if not k.endswith("rotary_emb.inv_freq")]
if real_missing or unexpected:
raise RuntimeError(
@@ -381,7 +386,8 @@ def load_ideogram4_pipeline(
text_encoder = load_ideogram4_text_encoder(repo_id, dtype, hf_token = token)
tokenizer = load_krea2_tokenizer(repo_id, hf_token = token)
transformer = load_ideogram4_transformer(repo_id, "transformer", dtype, hf_token = token)
- # The second DiT drives the unconditional branch of Ideogram's dual-branch CFG (same class and size, always required).
+ # The second DiT drives the unconditional branch of Ideogram's dual-branch CFG (same class and size, always
+ # required).
unconditional_transformer = load_ideogram4_transformer(
repo_id, "unconditional_transformer", dtype, hf_token = token
)
diff --git a/studio/backend/core/inference/diffusion_krea2.py b/studio/backend/core/inference/diffusion_krea2.py
index 97e4b195d6c..821d182a1a5 100644
--- a/studio/backend/core/inference/diffusion_krea2.py
+++ b/studio/backend/core/inference/diffusion_krea2.py
@@ -107,8 +107,8 @@ def load_krea2_text_encoder(
def _read_model_index(path: Path, source: str) -> dict[str, Any]:
try:
model_index = json.loads(path.read_text(encoding = "utf-8-sig"))
- # A nesting bomb raises RecursionError, not a ValueError, so it needs naming separately or it
- # stays the one raw traceback left. diffusion_families.pipeline_class_from_index does the same.
+ # A nesting bomb raises RecursionError, not a ValueError, so it needs naming separately or it stays the one raw
+ # traceback left. diffusion_families.pipeline_class_from_index does the same.
except (OSError, UnicodeDecodeError, json.JSONDecodeError, RecursionError) as exc:
raise ValueError(
f"Unable to read valid model_index.json from {source} at {path}: {exc}"
@@ -134,7 +134,8 @@ def _load_model_index(
except OSError:
pass
if is_local_dir:
- # A local checkpoint dir without the file must fail clearly here, else hf_hub_download dies with an opaque HFValidationError.
+ # A local checkpoint dir without the file must fail clearly here, else hf_hub_download dies with an opaque
+ # HFValidationError.
raise FileNotFoundError(f"model_index.json not found in local model dir {repo_id}")
from huggingface_hub import hf_hub_download
@@ -175,7 +176,9 @@ def load_krea2_pipeline(
"""
import diffusers
- # diffusers gained Krea2Pipeline in 0.39; on an older install the getattr chain below dies with a bare AttributeError, so fail first with the fix.
+ # diffusers gained Krea2Pipeline in 0.39; fail here rather than on a bare AttributeError below
+ # diffusers gained Krea2Pipeline in 0.39; on an older install the getattr chain below dies with a bare
+ # AttributeError, so fail first with the fix.
if not hasattr(diffusers, "Krea2Pipeline"):
raise RuntimeError(
f"Krea 2 needs diffusers >= 0.39.0 (Krea2Pipeline); this environment has "
@@ -185,8 +188,9 @@ def load_krea2_pipeline(
token = hf_token or None
cache_dir = _live_cache_dir()
- # A few KB, and it configures the components, so it is read before them: read last, a corrupt
- # index only surfaced after the encoder, the VAE and the 26 GB transformer were already built.
+ # read the index before the 26 GB transformer: a corrupt one used to surface only after everything was built
+ # A few KB, and it configures the components, so it is read before them: read last, a corrupt index only surfaced
+ # after the encoder, the VAE and the 26 GB transformer were already built.
model_index = _load_model_index(repo_id, hf_token = token, local_files_only = local_files_only)
tokenizer = load_krea2_tokenizer(repo_id, hf_token = token, local_files_only = local_files_only)
if text_encoder is None:
diff --git a/studio/backend/core/inference/diffusion_lora.py b/studio/backend/core/inference/diffusion_lora.py
index 1e68c0a909e..8be552369c7 100644
--- a/studio/backend/core/inference/diffusion_lora.py
+++ b/studio/backend/core/inference/diffusion_lora.py
@@ -41,13 +41,13 @@ class LoraCatalogEntry:
id: str
display_name: str
- source: str # "local" | "hub"
- fmt: str # "safetensors" | "gguf"
+ source: str
+ fmt: str
# Compatible family names (empty = unknown, shown but not family-gated).
families: tuple[str, ...] = ()
- repo_id: Optional[str] = None # source == "hub"
+ repo_id: Optional[str] = None
weight_name: Optional[str] = None # file within the repo (hub)
- local_path: Optional[str] = None # source == "local"
+ local_path: Optional[str] = None
size_bytes: int = 0
weight_default: float = 1.0
@@ -63,9 +63,11 @@ class ResolvedLora:
weight: float
-# Curated, family-tagged catalog of known-good diffusion LoRAs (HF repos with a single-file weight). Local discovery and any public HF LoRA repo id also work.
+# curated HF repos with a single-file weight; local discovery and any public HF LoRA repo id also work
+# Curated, family-tagged catalog of known-good diffusion LoRAs (HF repos with a single-file weight). Local discovery and
+# any public HF LoRA repo id also work.
def _krea2_lora(style: str, display_name: str) -> LoraCatalogEntry:
"""One official krea/Krea-2-LoRA-* style adapter (single ``{style}.safetensors``, trained on
Krea-2-Raw for Krea-2-Turbo per Krea's guidance)."""
@@ -128,7 +130,9 @@ def _scan_local() -> list[LoraCatalogEntry]:
for p in children
if p.is_file() and p.suffix.lower() in _ALL_EXTS and not is_appledouble_metadata(p)
]
- # Two files sharing a stem but differing in extension collide on id (== stem), so a colliding stem keeps the full filename.
+ # two files sharing a stem collide on id (== stem)
+ # Two files sharing a stem but differing in extension collide on id (== stem), so a colliding stem keeps the full
+ # filename.
stem_counts: dict[str, int] = {}
for p in files:
stem_counts[p.stem] = stem_counts.get(p.stem, 0) + 1
@@ -140,7 +144,8 @@ def _scan_local() -> list[LoraCatalogEntry]:
except OSError:
size = 0
entry_id = p.name if stem_counts.get(p.stem, 0) > 1 else p.stem
- # A ``<stem>.json`` sidecar (written by the trainer on publish) records the adapter's family + default weight so it is family-gated instead of "unknown". Best-effort.
+ # A ``<stem>.json`` sidecar (written by the trainer on publish) records the adapter's family + default weight so
+ # it is family-gated instead of "unknown". Best-effort.
families, weight_default = _read_lora_sidecar(p)
entries.append(
LoraCatalogEntry(
@@ -233,7 +238,6 @@ def resolve_one(
if not path or not os.path.exists(path):
raise FileNotFoundError(f"LoRA '{spec_id}' is no longer present on disk")
return ResolvedLora(spec_id, sanitize_alias(spec_id), path, entry.fmt, weight)
- # hub catalog entry
if not entry.repo_id or not entry.weight_name:
raise ValueError(f"LoRA '{spec_id}' has no downloadable weight")
path = hf_hub_download_with_xet_fallback(
@@ -246,7 +250,8 @@ def resolve_one(
repo_id, _, weight_name = spec_id.partition(":")
weight_name = weight_name or None
if weight_name is not None:
- # A client-supplied weight file must stay a plain filename inside the repo: reject traversal / absolute paths so it cannot resolve outside the HF cache dir.
+ # A client-supplied weight file must stay a plain filename inside the repo: reject traversal / absolute
+ # paths so it cannot resolve outside the HF cache dir.
if (
".." in weight_name
or weight_name.startswith(("/", "\\", "~"))
@@ -286,8 +291,10 @@ def _pick_repo_weight_file(repo_id: str, hf_token: Optional[str]) -> str:
return f
if safes:
return safes[0]
- # The gguf fallback only: an imatrix is a .gguf holding no adapter and would be picked
- # here, while a .safetensors is never one, so the candidates above stay untouched.
+ # gguf fallback only: an imatrix is a .gguf holding no adapter and would be picked here, while a .safetensors never
+ # is
+ # The gguf fallback only: an imatrix is a .gguf holding no adapter and would be picked here, while a .safetensors is
+ # never one, so the candidates above stay untouched.
ggufs = [
f
for f in files
@@ -301,7 +308,6 @@ def _pick_repo_weight_file(repo_id: str, hf_token: Optional[str]) -> str:
def _scrub_hub_url(msg: str) -> str:
"""Strip embedded http(s) URLs from a Hub error message before it hits a 400 body."""
cleaned = re.sub(r"https?://\S+", "", msg)
- # Collapse the whitespace the URL removal leaves behind.
return re.sub(r"\s{2,}", " ", cleaned).strip()
@@ -391,9 +397,9 @@ def inject_prompt_tags(prompt: str, resolved: list[ResolvedLora]) -> str:
must WIN over any user-typed `<lora:ALIAS:...>`, so strip ALL user tags first (unselected ones
are dead anyway, not in the managed dir) then append the validated ones.
"""
- # Drop every user-typed tag: unselected ones are dead, selected ones must not override the validated weight / 0-2 bounds.
+ # drop every user-typed tag: unselected ones are dead, selected ones must not override the validated weight / 0-2
+ # bounds
cleaned = _TAG_RE.sub("", prompt)
- # Collapse whitespace left by stripped tags.
cleaned = re.sub(r"[ \t]{2,}", " ", cleaned).strip()
tags = [f"<lora:{r.alias}:{_fmt_weight(r.weight)}>" for r in resolved]
if not tags:
@@ -408,7 +414,8 @@ def _fmt_weight(w: float) -> str:
return s or "0"
-# Families sd-cli's LoRA name-conversion supports (Qwen-Image has no branch). Matched by substring against the resolved family name.
+# Families sd-cli's LoRA name-conversion supports (Qwen-Image has no branch). Matched by substring against the resolved
+# family name.
_NATIVE_LORA_FAMILY_TOKENS = (
"flux.1",
"flux.2",
@@ -419,8 +426,10 @@ _NATIVE_LORA_FAMILY_TOKENS = (
"sd3",
"stable-diffusion",
)
+# LoRA path is the load-time BAKE: adapters attach on the dense transformer BEFORE torchao quantize_ + compile
# Diffusers quant schemes whose LoRA path is the load-time BAKE: adapters attach on the dense transformer BEFORE torchao
-# quantize_ + compile (peft's TorchaoLoraLinear dispatch needs quantizer metadata a manual quantize_ lacks). Verified on peft 0.18.1 / torchao 0.17 / torch 2.10: scale 0 reproduces the quantized base bit-exactly.
+# quantize_ + compile (peft's TorchaoLoraLinear dispatch needs quantizer metadata a manual quantize_ lacks). Verified on
+# peft 0.18.1 / torchao 0.17 / torch 2.10: scale 0 reproduces the quantized base bit-exactly.
_DIFFUSERS_LORA_BAKED_QUANT = ("int8", "fp8")
# Prototype schemes with no validated LoRA path (and no shipped families needing one).
_DIFFUSERS_LORA_BLOCKED_QUANT = ("nvfp4", "mxfp8")
@@ -448,7 +457,6 @@ def supports_lora(
fam = (family or "").lower()
if engine == "sd_cpp":
return any(tok in fam for tok in _NATIVE_LORA_FAMILY_TOKENS)
- # diffusers
quant = (transformer_quant or "").lower()
if quant in _DIFFUSERS_LORA_BAKED_QUANT:
return True # load-time bake; adapters ride inside the compiled quantized build
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index bc5323795af..e35fdd4d798 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -33,11 +33,11 @@ MEMORY_MODES = (
MEMORY_MODE_LOW_VRAM,
)
-# none -- all weights resident (fastest; fits only with room).
-# model -- enable_model_cpu_offload(): one top-level module on the GPU at a time.
-# group -- apply_group_offloading() on the transformer: stream a few blocks at a time with a prefetch stream.
-# streaming -- group-offload the transformer and leaf-offload text encoders that cannot fit whole.
-# sequential -- enable_sequential_cpu_offload(): submodule-level (broken for GGUF through diffusers 0.39, kept as an escape hatch).
+# None: all weights resident. model: enable_model_cpu_offload(), one top-level module on the GPU at a time. none -- all
+# weights resident (fastest; fits only with room). model -- enable_model_cpu_offload(): one top-level module on the GPU
+# at a time. group -- apply_group_offloading() on the transformer: stream a few blocks at a time with a prefetch stream.
+# streaming -- group-offload the transformer and leaf-offload text encoders that cannot fit whole. sequential --
+# enable_sequential_cpu_offload(): submodule-level (broken for GGUF through diffusers 0.39, kept as an escape hatch).
OFFLOAD_NONE = "none"
OFFLOAD_MODEL = "model"
OFFLOAD_GROUP = "group"
@@ -49,7 +49,7 @@ DEFAULT_GROUP_BLOCKS = 1
DEFAULT_IMAGE_WIDTH = 1024
DEFAULT_IMAGE_HEIGHT = 1024
-# Flat allowance for fixed pipeline costs (scheduler, embeddings, CUDA context, fragmentation).
+# flat allowance for fixed pipeline costs (scheduler, embeddings, CUDA context, fragmentation)
DEFAULT_BASE_OVERHEAD_MIB = 2048
@@ -105,9 +105,10 @@ class MemoryPlan:
device_memory: DeviceMemory
estimates: dict[str, Optional[int]]
reasons: tuple[str, ...] = ()
- # Under group offload, stream the TEXT ENCODERS alongside the transformer instead of keeping
- # them resident. Defaulted so every existing construction is unchanged; set only where that
- # is what makes group offload fit at all (see plan_diffusion_memory).
+ # defaulted so every existing construction is unchanged
+ # Under group offload, stream the TEXT ENCODERS alongside the transformer instead of keeping them resident.
+ # Defaulted so every existing construction is unchanged; set only where that is what makes group offload fit at all
+ # (see plan_diffusion_memory).
stream_text_encoders: bool = False
@property
@@ -143,7 +144,7 @@ def snapshot_device_memory(target: Any) -> DeviceMemory:
free, total = _xpu_memory()
return DeviceMemory(backend, device, "discrete_vram", free, total)
if device == "mps":
- # Apple Silicon shares one CPU/GPU pool: system memory is the budget, offload pointless.
+ # Apple Silicon shares one CPU/GPU pool: system memory is the budget, offload pointless
total, free = _system_memory_mib()
return DeviceMemory(backend, device, "unified_memory", free, total)
@@ -188,7 +189,7 @@ def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
return snapshot
free = int(snapshot.free_mib) + reclaimable // (1024 * 1024)
if snapshot.total_mib is not None:
- free = min(free, int(snapshot.total_mib)) # never claim more than the card has
+ free = min(free, int(snapshot.total_mib))
return DeviceMemory(
snapshot.backend, snapshot.device, snapshot.memory_kind, free, snapshot.total_mib
)
@@ -245,7 +246,7 @@ def settled_snapshot_device_memory(
empty_cache = getattr(getattr(torch, "mps", None), "empty_cache", None)
if callable(empty_cache):
empty_cache()
- except Exception: # noqa: BLE001 — settle is best-effort; the snapshot below still runs
+ except Exception: # noqa: BLE001 - settle is best-effort; the snapshot below still runs
pass
return snapshot_device_memory(target)
if device != "cuda":
@@ -254,7 +255,7 @@ def settled_snapshot_device_memory(
import torch
torch.cuda.synchronize()
torch.cuda.empty_cache()
- except Exception: # noqa: BLE001 — settle is best-effort; the snapshot below still runs
+ except Exception: # noqa: BLE001 - settle is best-effort; the snapshot below still runs
pass
best = snapshot_device_memory(target)
delay_s = _settle_delay(delay_s)
@@ -278,17 +279,17 @@ def _cuda_memory(backend: str) -> tuple[Optional[int], Optional[int], str]:
try:
import torch
- # Not torch.cuda.mem_get_info directly: on Windows ROCm its free half is an
- # over-report that does not track residency, and this feeds the activation
- # refusal that exists BECAUSE Windows WDDM spills to host RAM instead of
- # raising (#8403). Imported lazily to keep this module free of backend
- # imports at module scope.
+ # Not torch.cuda.mem_get_info directly: on Windows ROCm its free half is an over-report that does not track
+ # residency, and this feeds the activation refusal that exists BECAUSE Windows WDDM spills to host RAM instead
+ # of raising (#8403). Imported lazily to keep this module free of backend imports at module scope.
from utils.hardware import trusted_mem_get_info
free, total = trusted_mem_get_info()
kind = "discrete_vram"
try:
- # Query the CURRENT device (mem_get_info reports it); hardcoding 0 would inspect the wrong GPU and misclassify it.
+ # query the CURRENT device; hardcoding 0 would inspect the wrong GPU and misclassify it
+ # Query the CURRENT device (mem_get_info reports it); hardcoding 0 would inspect the wrong GPU and
+ # misclassify it.
props = torch.cuda.get_device_properties(torch.cuda.current_device())
if bool(getattr(props, "integrated", False) or getattr(props, "is_integrated", False)):
kind = "unified_memory" # e.g. Jetson / integrated SoC
@@ -350,7 +351,7 @@ def estimate_gguf_resident_mib(storage_mib: Optional[int]) -> Optional[int]:
assumed a full unpack that never happens, over-estimating Q2 ~7.6x and forcing needless offload.)"""
if storage_mib is None:
return None
- return int(storage_mib * 1.05) # small margin for allocator + bf16 norms/biases
+ return int(storage_mib * 1.05) # margin for allocator + bf16 norms/biases
def estimate_safetensors_dense_mib(
@@ -435,15 +436,15 @@ def plan_fits_total_capacity(plan: Any) -> bool:
memory = plan.device_memory
total = memory.total_mib
kind = memory.memory_kind
- except Exception: # noqa: BLE001 — malformed plan: no retry
+ except Exception: # noqa: BLE001 - malformed plan: no retry
return False
if required is None or total is None:
return False
return int(required) <= int((int(total) - _reserve_mib(kind, int(total))) * 0.85)
-# Opt-in escape hatch for the unified-memory refusal below: the shortfall check is an
-# estimate, so an operator who believes it is wrong can still attempt the load.
+# Opt-in escape hatch for the unified-memory refusal below: the shortfall check is an estimate, so an operator who
+# believes it is wrong can still attempt the load.
UNIFIED_OVERSIZE_ENV = "UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_LOAD"
@@ -479,9 +480,8 @@ def unified_memory_shortfall_message(plan: Any, *, family: Optional[str] = None)
return None
try:
memory = plan.device_memory
- # ``system_memory`` (plain CPU) is deliberately excluded: it is an opt-in fringe path,
- # it has swap, and it is not what gets Metal-killed. Only the accelerator-on-system-pool
- # case is guarded.
+ # ``system_memory`` (plain CPU) is deliberately excluded: it is an opt-in fringe path, it has swap, and it is
+ # not what gets Metal-killed. Only the accelerator-on-system-pool case is guarded.
if getattr(memory, "memory_kind", None) != "unified_memory":
return None
estimates = plan.estimates
@@ -489,7 +489,7 @@ def unified_memory_shortfall_message(plan: Any, *, family: Optional[str] = None)
weights = estimates.get("model_dense_mib")
overhead = estimates.get("base_overhead_mib")
free = getattr(memory, "free_mib", None)
- except Exception: # noqa: BLE001 — malformed plan: never block the load
+ except Exception: # noqa: BLE001 - malformed plan: never block the load
return None
if budget is None or weights is None:
return None
@@ -585,12 +585,11 @@ def plan_diffusion_memory(
required = _sum_required(model_dense_mib, runtime_headroom_mib, base_overhead_mib)
# The resident floor under group offload: companions stay, the transformer streams.
group_floor = _sum_required(companion_dense_mib, runtime_headroom_mib, base_overhead_mib)
- # A SECOND floor, for the same tier with the text encoders streamed as well. The encoders are
- # the largest companion on most families (Z-Image: 8.0 of 8.2 GB) and they are used exactly
- # once, before step 0, so holding them resident for the whole denoise reserves their bytes for
- # nothing. Streaming them leaves the VAE as the only resident companion. Computed only when
- # BOTH terms are known: an unknown split must reproduce the previous decision, never guess a
- # smaller floor. Clamped at 0 because the two terms can come from different sources.
+ # A SECOND floor, for the same tier with the text encoders streamed as well. The encoders are the largest companion
+ # on most families (Z-Image: 8.0 of 8.2 GB) and they are used exactly once, before step 0, so holding them resident
+ # for the whole denoise reserves their bytes for nothing. Streaming them leaves the VAE as the only resident
+ # companion. Computed only when BOTH terms are known: an unknown split must reproduce the previous decision, never
+ # guess a smaller floor. Clamped at 0 because the two terms can come from different sources.
group_floor_streamed_te = (
_sum_required(
max(0, int(companion_dense_mib) - int(text_encoder_dense_mib)),
@@ -619,16 +618,17 @@ def plan_diffusion_memory(
return group_floor is not None and budget is not None and group_floor <= budget
def _group_fits_streamed_te() -> bool:
- # The same tier once the text encoders stream too; only reachable with a known split.
return (
group_floor_streamed_te is not None
and budget is not None
and group_floor_streamed_te <= budget
)
- # The best tier available when the weights do not fit resident, in speed order: plain group
- # (companions resident) beats group with streamed encoders (one extra host-to-device pass per
- # CALL) beats whole-module offload (every component paged per STEP -- the 48-minute case).
+ # speed order: plain group (companions resident) beats group with streamed encoders (one extra host-to-device pass
+ # per CALL) beats whole-module offload (every component paged per STEP
+ # The best tier available when the weights do not fit resident, in speed order: plain group (companions resident)
+ # beats group with streamed encoders (one extra host-to-device pass per CALL) beats whole-module offload (every
+ # component paged per STEP -- the 48-minute case).
def _offload_tier() -> tuple[str, bool]:
if _group_fits():
return OFFLOAD_GROUP, False
@@ -642,7 +642,8 @@ def plan_diffusion_memory(
)
if not can_offload or device_memory.is_unified:
- # MPS / CPU cannot stream to a separate device; on unified memory offload just shuffles bytes within the same pool.
+ # MPS / CPU cannot stream to a separate device; on unified memory offload just shuffles bytes within the same
+ # pool.
policy = OFFLOAD_NONE
if device_memory.is_unified:
reasons.append("unified/system memory: CPU offload frees no device memory")
@@ -692,8 +693,9 @@ def plan_diffusion_memory(
policy = OFFLOAD_MODEL
reasons.append("explicit cpu_offload overrides resident placement")
- # VAE savers cap the high-res decode spike. Slicing (one image at a time) is EXACT, so enable it on any offload tier. Tiling
- # is only bit-identical for a single tile (<=1MP), so restrict it to the lowest tiers. Group offload keeps the VAE resident.
+ # VAE savers cap the high-res decode spike. Slicing (one image at a time) is EXACT, so enable it on any offload
+ # tier. Tiling is only bit-identical for a single tile (<=1MP), so restrict it to the lowest tiers. Group offload
+ # keeps the VAE resident.
any_offload = policy != OFFLOAD_NONE or device_memory.backend in ("mps", "cpu")
tile = policy in (OFFLOAD_MODEL, OFFLOAD_SEQUENTIAL) or device_memory.backend in ("mps", "cpu")
return MemoryPlan(
@@ -704,7 +706,7 @@ def plan_diffusion_memory(
device_memory = device_memory,
estimates = estimates,
reasons = tuple(reasons),
- # Only ever meaningful under group offload; every other tier already places the encoders.
+ # only ever meaningful under group offload; every other tier already places the encoders
stream_text_encoders = stream_text_encoders and policy == OFFLOAD_GROUP,
)
@@ -782,7 +784,7 @@ def refine_memory_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan
largest_name, largest_mib = max(streamed_sizes.items(), key = lambda item: item[1])
if largest_mib <= int(budget):
return plan
- # What streaming leaves resident, all at once. Over budget here means streaming OOMs too.
+ # what streaming leaves resident, all at once: over budget here means streaming OOMs too
resident_mib = sum(m for n, m in sizes.items() if n not in streamable)
if resident_mib > int(budget):
return plan
@@ -831,7 +833,8 @@ def apply_memory_plan(
_enable_vae_saver(pipe, "enable_vae_slicing", "enable_slicing", logger)
def _fallback_to_model_offload() -> None:
- # The GROUP plan set vae_tiling=False (the VAE stays resident). Dropping to whole-module offload is the low-VRAM case where the decode spike can OOM, so turn tiling on now.
+ # The GROUP plan set vae_tiling=False (the VAE stays resident). Dropping to whole-module offload is the low-VRAM
+ # case where the decode spike can OOM, so turn tiling on now.
nonlocal tiling_engaged
pipe.enable_model_cpu_offload(device = placement)
if not tiling_engaged:
@@ -855,7 +858,7 @@ def apply_memory_plan(
elif policy == OFFLOAD_SEQUENTIAL:
try:
pipe.enable_sequential_cpu_offload(device = placement)
- except Exception as exc: # noqa: BLE001 — keep the model loadable
+ except Exception as exc: # noqa: BLE001 - keep the model loadable
if logger is not None:
logger.warning(
"diffusion.memory: sequential offload failed (%s); "
@@ -879,7 +882,7 @@ def _enable_vae_saver(pipe: Any, pipe_method: str, vae_method: str, logger: Any)
try:
fn()
return True
- except Exception as exc: # noqa: BLE001 — a VAE saver is an optimisation, never fatal
+ except Exception as exc: # noqa: BLE001 - a VAE saver is an optimisation, never fatal
if logger is not None:
logger.warning("diffusion.memory: %s() failed: %s", method, exc)
return False
@@ -901,28 +904,29 @@ def _apply_group_offload(
transformer = getattr(pipe, "transformer", None)
if transformer is None:
return False
- installed = 0 # streamed modules that already carry group-offload hooks
+ installed = 0
try:
import inspect
import torch
from diffusers.hooks import apply_group_offloading
- # A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep only smaller companions resident.
+ # A dual-DiT pipeline (Ideogram 4) carries a second denoiser as large as the first, so stream every DiT and keep
+ # only smaller companions resident.
streamed: dict[str, Any] = {"transformer": transformer}
for extra in ("transformer_2", "unconditional_transformer"):
module = getattr(pipe, extra, None)
if isinstance(module, torch.nn.Module):
streamed[extra] = module
- # The text encoders are streamed SEPARATELY from the DiTs, and tolerantly (see the apply
- # loop below). Kept in their own dict so the resident placement loop still skips them.
+ # The text encoders are streamed SEPARATELY from the DiTs, and tolerantly (see the apply loop below). Kept in
+ # their own dict so the resident placement loop still skips them.
streamed_encoders: dict[str, Any] = {}
if stream_text_encoders:
- # A text encoder runs ONCE, before step 0, so residency buys it nothing while it costs
- # its bytes for every step of the denoise. Streaming it does two things: the resident
- # loop below skips it (it is no longer placed with comp.to(onload)), and group hooks
- # page it in for that single encode. Component names, not attributes, so a family with
- # text_encoder / text_encoder_2 / text_encoder_3 is covered without a per-family list.
+ # A text encoder runs ONCE, before step 0, so residency buys it nothing while it costs its bytes for every
+ # step of the denoise. Streaming it does two things: the resident loop below skips it (it is no longer
+ # placed with comp.to(onload)), and group hooks page it in for that single encode. Component names, not
+ # attributes, so a family with text_encoder / text_encoder_2 / text_encoder_3 is covered without a
+ # per-family list.
for name, comp in getattr(pipe, "components", {}).items():
if name.startswith("text_encoder") and isinstance(comp, torch.nn.Module):
streamed_encoders[name] = comp
@@ -936,7 +940,8 @@ def _apply_group_offload(
"num_blocks_per_group": DEFAULT_GROUP_BLOCKS,
"use_stream": use_stream,
}
- # On the CUDA stream path, overlap each block's H2D copy with compute. Lossless, and gated on the signature so older diffusers still works.
+ # On the CUDA stream path, overlap each block's H2D copy with compute. Lossless, and gated on the signature so
+ # older diffusers still works.
_params = inspect.signature(apply_group_offloading).parameters
if use_stream:
if "non_blocking" in _params:
@@ -944,17 +949,18 @@ def _apply_group_offload(
if "record_stream" in _params:
gkwargs["record_stream"] = True
if stream_text_encoders and "low_cpu_mem_usage" in _params:
- # The streamed path PINS every offloaded parameter in host RAM when a copy stream is
- # in use (diffusers group_offloading `_init_cpu_param_dict`), which is a fine trade
- # when group offload was already the plan. It is not a fine trade here: this tier is
- # only ever reached as a rescue from whole-module offload, which pins nothing, on a
- # card small enough that the companions did not fit. Those hosts are not reliably
- # RAM-rich either, and silently converting a device-memory shortfall into ten-plus GB
- # of unswappable host RAM is how #8188's machine got into trouble in the first place.
- # low_cpu_mem_usage trades a slower host-to-device copy for not pinning; the encoders
- # this tier streams run ONCE per call, so that copy is paid once, not per step.
+ # The streamed path PINS every offloaded parameter in host RAM when a copy stream is in use (diffusers
+ # group_offloading `_init_cpu_param_dict`), which is a fine trade when group offload was already the plan.
+ # It is not a fine trade here: this tier is only ever reached as a rescue from whole-module offload, which
+ # pins nothing, on a card small enough that the companions did not fit. Those hosts are not reliably
+ # RAM-rich either, and silently converting a device-memory shortfall into ten-plus GB of unswappable host
+ # RAM is how #8188's machine got into trouble in the first place. low_cpu_mem_usage trades a slower
+ # host-to-device copy for not pinning; the encoders this tier streams run ONCE per call, so that copy is
+ # paid once, not per step.
gkwargs["low_cpu_mem_usage"] = True
- # Place the smaller components resident BEFORE attaching the transformer group-offload hooks: a companion .to() OOM then returns False with no hooks installed, and diffusers rejects enable_model_cpu_offload once group hooks exist.
+ # Place the smaller components resident BEFORE attaching the transformer group-offload hooks: a companion .to()
+ # OOM then returns False with no hooks installed, and diffusers rejects enable_model_cpu_offload once group
+ # hooks exist.
for name, comp in getattr(pipe, "components", {}).items():
if name in streamed or name in streamed_encoders:
continue
@@ -963,20 +969,17 @@ def _apply_group_offload(
for module in streamed.values():
apply_group_offloading(module, **gkwargs)
installed += 1
- # The encoders come AFTER the DiTs and are applied one by one, each failure absorbed. A
- # text encoder is a far less well-trodden target for block-level group offloading than a
- # DiT (a family whose encoder exposes no recognisable block list can simply refuse), and
- # this tier is a rescue: the alternative to streaming an encoder is keeping it resident,
- # which is what happened before this tier existed. Letting one refusal join the all-or-
- # nothing DiT loop would turn a slow-but-working load into a hard failure, because by then
- # hooks are installed and whole-module offload can no longer be used as a fallback. So a
- # refusal places that encoder resident instead: the plan's floor becomes optimistic by
- # that encoder's bytes, and the load still runs.
- # Leaf level, not the DiTs' block level: an encoder is not a stack of uniform blocks, so
- # _streamable_components and _apply_streaming_offload already classify every text_encoder*
- # that way. Reusing the transformer's kwargs here grouped the whole encoder as one unit,
- # which is the residency the planner's floor was chosen to avoid -- the plan said leaf and
- # the application said block. num_blocks_per_group goes with it: leaf level has no blocks.
+ # The encoders come AFTER the DiTs and are applied one by one, each failure absorbed. A text encoder is a far
+ # less well-trodden target for block-level group offloading than a DiT (a family whose encoder exposes no
+ # recognisable block list can refuse), and this tier is a rescue: the alternative to streaming an encoder is
+ # keeping it resident, which is what happened before this tier existed. Letting one refusal join the all-or-
+ # nothing DiT loop would turn a slow-but-working load into a hard failure, because by then hooks are installed
+ # and whole-module offload can no longer be used as a fallback. So a refusal places that encoder resident
+ # instead: the plan's floor becomes optimistic by that encoder's bytes, and the load still runs. Leaf level, not
+ # the DiTs' block level: an encoder is not a stack of uniform blocks, so _streamable_components and
+ # _apply_streaming_offload already classify every text_encoder* that way. Reusing the transformer's kwargs here
+ # grouped the whole encoder as one unit, which is the residency the planner's floor was chosen to avoid -- the
+ # plan said leaf and the application said block. num_blocks_per_group goes with it: leaf level has no blocks.
ekwargs = {k: v for k, v in gkwargs.items() if k != "num_blocks_per_group"}
ekwargs["offload_type"] = "leaf_level"
for name, module in streamed_encoders.items():
@@ -993,9 +996,11 @@ def _apply_group_offload(
)
module.to(onload)
return True
- except Exception as exc: # noqa: BLE001 — fall back to whole-module offload
+ except Exception as exc: # noqa: BLE001 - fall back to whole-module offload
if installed:
- # An earlier streamed module already has hooks but a later one failed: the pipe is in a PARTIAL group-offload state enable_model_cpu_offload rejects, so propagate the real failure instead of a misleading hook error.
+ # An earlier streamed module already has hooks but a later one failed: the pipe is in a PARTIAL
+ # group-offload state enable_model_cpu_offload rejects, so propagate the real failure instead of a
+ # misleading hook error.
if logger is not None:
logger.warning(
"diffusion.memory: group offload failed after installing hooks on %d "
@@ -1013,14 +1018,15 @@ def _apply_group_offload(
return False
-# ── generate-time activation guard ────────────────────────────────────────────
-# The load-time plan cannot know the output resolution: a model is loaded once and then generates
-# at whatever size the sliders say, so ``_plan_memory`` budgets the 1024x1024 default. That is the
-# right call for PLACEMENT, but it means a request for a much larger frame is never checked against
-# anything. This is the second half: a per-generation re-check, with the real dimensions.
+# the load-time plan cannot know the output resolution
-# Opt-in escape hatch, mirroring the load-time one: the activation estimate is coarse, so an
-# operator who believes it is wrong keeps a way through.
+# ── generate-time activation guard ──────────────────────────────────────────── The load-time plan cannot know the
+# output resolution: a model is loaded once and then generates at whatever size the sliders say, so ``_plan_memory``
+# budgets the 1024x1024 default. That is the right call for PLACEMENT, but it means a request for a much larger frame is
+# never checked against anything. This is the second half: a per-generation re-check, with the real dimensions.
+# Opt-in escape hatch, mirroring the load-time one: the activation estimate is coarse, so an operator who believes it is
+# wrong keeps a way through.
+# ── generate-time activation guard ────────────────────────────────────────────
OVERSIZED_GENERATE_ENV = "UNSLOTH_DIFFUSION_ALLOW_OVERSIZED_GENERATE"
@@ -1086,18 +1092,18 @@ def image_activation_shortfall_message(
if _oversized_generate_override():
return None
try:
- # Unified / system memory: offload moves bytes within one pool, "free" is a moving target
- # shared with the OS, and the load-time unified refusal already owns that device class.
+ # Unified / system memory: offload moves bytes within one pool, "free" is a moving target shared with the OS,
+ # and the load-time unified refusal already owns that device class.
if getattr(device_memory, "is_unified", False):
return None
- # CUDA / ROCm only. ROCm's torch reports device "cuda", so this covers both. XPU / MPS /
- # CPU keep today's behaviour exactly: their allocators and offload semantics differ and
- # this estimate was measured against a discrete VRAM pool.
+ # CUDA / ROCm only. ROCm's torch reports device "cuda", so this covers both. XPU / MPS / CPU keep today's
+ # behaviour exactly: their allocators and offload semantics differ and this estimate was measured against a
+ # discrete VRAM pool.
if getattr(device_memory, "device", None) != "cuda":
return None
free = getattr(device_memory, "free_mib", None)
if free is None:
- return None # no reading -> no verdict
+ return None
budget = _safe_device_budget_mib(device_memory)
if budget is None:
return None
@@ -1107,9 +1113,9 @@ def image_activation_shortfall_message(
batch_size = batch_size,
family = family,
)
- # What the LOAD budgeted: the same estimator at the default resolution, i.e. the exact
- # call _plan_memory makes. Same function and same family hint, so the comparison is
- # between two points on one curve rather than between two different guesses.
+ # What the LOAD budgeted: the same estimator at the default resolution, i.e. the exact call _plan_memory makes.
+ # Same function and same family hint, so the comparison is between two points on one curve rather than between
+ # two different guesses.
planned = estimate_image_runtime_mib(
width = None,
height = None,
@@ -1118,12 +1124,11 @@ def image_activation_shortfall_message(
)
except Exception: # noqa: BLE001 -- a broken probe must never block a generation
return None
- # The flat base overhead rides along with the activations: the CUDA context, the scheduler
- # state and the fragmentation allowance all have to coexist with this pass's tensors, and the
- # load-time plan already sums them additively for exactly that reason. Leaving it out made the
- # guard silent by a few hundred MiB on the very card #8188 was reported from (15.92 GiB:
- # 13,872 MiB of activations against a 14,254 MiB budget). It cannot cause a false refusal at
- # or below the default resolution, because the `needed <= planned` arm already exempts every
+ # The flat base overhead rides along with the activations: the CUDA context, the scheduler state and the
+ # fragmentation allowance all have to coexist with this pass's tensors, and the load-time plan already sums them
+ # additively for exactly that reason. Leaving it out made the guard silent by a few hundred MiB on the very card
+ # #8188 was reported from (15.92 GiB: 13,872 MiB of activations against a 14,254 MiB budget). It cannot cause a
+ # false refusal at or below the default resolution, because the `needed <= planned` arm already exempts every
# request the load itself budgeted for.
if int(needed) + max(0, int(base_overhead_mib)) <= int(budget) or needed <= planned:
return None
@@ -1131,11 +1136,10 @@ def image_activation_shortfall_message(
h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
batch = max(1, int(batch_size or 1))
batch_note = f" at a batch of {batch}" if batch > 1 else ""
- # Two decimals, not one: the refusal is often decided by tens of MiB (the 1088x1920 report
- # needed 13,872 MiB against a 13,822 MiB budget), and one decimal prints both as "13.5 GB".
- # The overhead is part of the comparison above, so it has to be part of the number reported.
- # Quoting the activations alone printed a refusal that contradicted itself: 13.55 GB needed
- # against 13.92 GB usable, refused.
+ # Two decimals, not one: the refusal is often decided by tens of MiB (the 1088x1920 report needed 13,872 MiB against
+ # a 13,822 MiB budget), and one decimal prints both as "13.5 GB". The overhead is part of the comparison above, so
+ # it has to be part of the number reported. Quoting the activations alone printed a refusal that contradicted
+ # itself: 13.55 GB needed against 13.92 GB usable, refused.
total = int(needed) + max(0, int(base_overhead_mib))
return (
f"Generating at {w}x{h}{batch_note} needs about {total / 1024:.2f} GB of working memory "
@@ -1144,9 +1148,8 @@ def image_activation_shortfall_message(
f"{int(free) / 1024:.2f} GB currently free, after reserving room for fragmentation and "
"other processes). Working memory holds this pass's latents and attention buffers, which "
"cannot be offloaded to the CPU the way weights can, so no memory mode recovers this. "
- # Only when the batch is what was budgeted. A refusal measured on ONE image cannot be
- # answered by asking for fewer, and pointing there sends the caller at the one change
- # that provably will not help.
+ # Only when the batch is what was budgeted. A refusal measured on ONE image cannot be answered by asking for
+ # fewer, and pointing there sends the caller at the one change that provably will not help.
f"{'Upload a smaller source image (this workflow takes its output size from the image, not the Resolution setting)' if source_driven else 'Generate at a smaller resolution'}"
f"{' or a smaller batch size' if batch > 1 else ''}, "
"free device memory by closing other applications, or set "
@@ -1206,7 +1209,7 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
if not isinstance(components, dict):
raise RuntimeError("pipeline does not expose its components")
- # Same selection the planner sized against, so what it promised to stream is what streams.
+ # same selection the planner sized against, so what it promised to stream is what streams
streamed = _streamable_components(pipe, torch)
if "transformer" not in streamed:
raise RuntimeError("pipeline has no transformer to stream")
@@ -1216,8 +1219,8 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
params = inspect.signature(apply_group_offloading).parameters
use_stream = onload.type == "cuda" and "low_cpu_mem_usage" in params
- # Keep small companions such as the tiled VAE resident. Every transformer and text
- # encoder remains on CPU behind a granular hook.
+ # Keep small companions such as the tiled VAE resident. Every transformer and text encoder remains on CPU behind
+ # a granular hook.
for name, component in components.items():
if str(name) in streamed:
continue
diff --git a/studio/backend/core/inference/diffusion_patch_backend.py b/studio/backend/core/inference/diffusion_patch_backend.py
index 6663556852b..5c920bae5e1 100644
--- a/studio/backend/core/inference/diffusion_patch_backend.py
+++ b/studio/backend/core/inference/diffusion_patch_backend.py
@@ -22,7 +22,9 @@ from __future__ import annotations
from typing import Any, Callable, Optional
-# Resolved ``unsloth_zoo.temporary_patches.utils`` helpers, memoised per process (None = tried and unavailable): resolution can import ``unsloth``, far too heavy to repeat per call.
+# memoised per process (None = unavailable): resolution can import unsloth, far too heavy to repeat
+# Resolved ``unsloth_zoo.temporary_patches.utils`` helpers, memoised per process (None = tried and unavailable):
+# resolution can import ``unsloth``, far too heavy to repeat per call.
_HELPERS: Optional[dict] = None
@@ -45,11 +47,11 @@ def _retry_could_help(exc: BaseException) -> bool:
xpu = getattr(torch, "xpu", None)
if not (torch.cuda.is_available() or (xpu is not None and xpu.is_available())):
return False
- except Exception: # noqa: BLE001 — an unprobeable device is not one unsloth can use
+ except Exception: # noqa: BLE001 - an unprobeable device is not one unsloth can use
return False
try:
return importlib.util.find_spec("unsloth") is not None
- except Exception: # noqa: BLE001 — an unimportable package cannot set the sentinel either
+ except Exception: # noqa: BLE001 - an unimportable package cannot set the sentinel either
return False
@@ -89,12 +91,12 @@ def _helpers() -> Optional[dict]:
try:
_HELPERS = _load()
return _HELPERS
- except Exception as exc: # noqa: BLE001 — no unsloth_zoo / no-GPU host -> skip the patch
+ except Exception as exc: # noqa: BLE001 - no unsloth_zoo / no-GPU host -> skip the patch
if attempt or not _retry_could_help(exc):
break
try:
- import unsloth # noqa: F401 — sets UNSLOTH_IS_PRESENT for the retry
- except Exception: # noqa: BLE001 — not installed / no accelerator: give up quietly
+ import unsloth # noqa: F401 - sets UNSLOTH_IS_PRESENT for the retry
+ except Exception: # noqa: BLE001 - not installed / no accelerator: give up quietly
break
_HELPERS = {}
return None
@@ -124,7 +126,7 @@ def apply_patch(
return False
try:
return bool(patch_function(target, attr, new_fn, match_level = match_level, force = force))
- except Exception: # noqa: BLE001 — best-effort; leave the original in place
+ except Exception: # noqa: BLE001 - best-effort; leave the original in place
return False
diff --git a/studio/backend/core/inference/diffusion_precision.py b/studio/backend/core/inference/diffusion_precision.py
index 26cb172eb23..b21d4c29291 100644
--- a/studio/backend/core/inference/diffusion_precision.py
+++ b/studio/backend/core/inference/diffusion_precision.py
@@ -49,9 +49,10 @@ _TE_TORCHAO_MODES = frozenset({TE_QUANT_INT8, TE_QUANT_FP8_DYNAMIC, TE_QUANT_NVF
# Pipeline attributes that hold a text encoder, in order.
_TEXT_ENCODER_ATTRS = ("text_encoder", "text_encoder_2", "text_encoder_3")
-# int8 degrades on large text encoders unless the quant-sensitive decoder blocks stay bf16. Per-family (skip_first, skip_last) blocks to keep
-# dense, from measured hidden-state fidelity; absent families have no schedule clearing the bar, so int8 falls back to fp8. qwen-image
-# (Qwen2.5-VL-7B): first+last 6 gives ~0.997 cosine; flux.2-dev (Mistral-Small-24B): first 3 gives ~0.98 (early-layer seeding).
+# int8 degrades on large text encoders unless the quant-sensitive decoder blocks stay bf16. Per-family (skip_first,
+# skip_last) blocks to keep dense, from measured hidden-state fidelity; absent families have no schedule clearing the
+# bar, so int8 falls back to fp8. qwen-image (Qwen2.5-VL-7B): first+last 6 gives ~0.997 cosine; flux.2-dev
+# (Mistral-Small-24B): first 3 gives ~0.98 (early-layer seeding).
_TE_INT8_SKIP: dict[str, tuple[int, int]] = {
"qwen-image": (6, 6),
"qwen-image-edit": (6, 6),
@@ -130,8 +131,8 @@ def te_quant_supported(target: Any, mode: str) -> bool:
Blackwell sm_100+ (nvfp4)."""
if getattr(target, "device", None) != "cuda":
return False
- # Torchao modes cannot use the Windows stub or ROCm's non-SM capability values. Plain fp8 is
- # only a dtype cast and remains supported.
+ # Torchao modes cannot use the Windows stub or ROCm's non-SM capability values. Plain fp8 is only a dtype cast and
+ # remains supported.
if mode in _TE_TORCHAO_MODES and (is_stubbed("torchao") or torch_is_rocm()):
return False
try:
@@ -202,7 +203,8 @@ def quantize_text_encoders(
f"int8 has no measured keep-bf16 schedule for family '{family}' "
"(it degrades large encoders without one), so fp8 was used instead"
)
- # torchao modes produce subclasses that reject Module.to(), which an offload placement uses. Layerwise fp8 streams fine.
+ # torchao modes produce subclasses that reject Module.to(), which an offload placement uses. Layerwise fp8 streams
+ # fine.
if offload_active and mode in (TE_QUANT_INT8, TE_QUANT_FP8_DYNAMIC, TE_QUANT_NVFP4):
_note(
logger,
@@ -244,7 +246,7 @@ def quantize_text_encoders(
try:
caster(encoder, target)
cast.append(attr)
- except Exception as exc: # noqa: BLE001 — leave this encoder dense
+ except Exception as exc: # noqa: BLE001 - leave this encoder dense
failed.append(attr)
_warn(logger, f"{mode}:{attr}", exc)
if not cast:
@@ -254,9 +256,9 @@ def quantize_text_encoders(
RESOLVED_FELL_BACK,
)
if failed:
- # A sibling took the cast, so `mode` DID engage -- but the encoders that did not are still
- # dense bf16 and the prompt is conditioned by both. Reporting "applied" here was the one
- # path where an engaged mode could still be a lie about the build that ran.
+ # A sibling took the cast, so `mode` DID engage -- but the encoders that did not are still dense bf16 and the
+ # prompt is conditioned by both. Reporting "applied" here was the one path where an engaged mode could still be
+ # a lie about the build that ran.
return TEQuantOutcome(
mode,
f"'{mode}' engaged on {', '.join(cast)} but {', '.join(failed)} could not be cast and "
@@ -295,7 +297,8 @@ def _keep_bf16_block_fqns(encoder: Any, skip_first: int, skip_last: int) -> set[
def _cast_int8_selective(encoder: Any, target: Any, skip_first: int, skip_last: int) -> None:
- # torchao dynamic int8 on the FLOP-heavy Linears, keeping the first/last decoder blocks (and vision tower / lm_head / T5 wo) bf16. Reuses the transformer-quant factory so config cannot drift.
+ # torchao dynamic int8 on the FLOP-heavy Linears, keeping the first/last decoder blocks (and vision tower / lm_head
+ # / T5 wo) bf16. Reuses the transformer-quant factory so config cannot drift.
from torchao.quantization import quantize_
from .diffusion_transformer_quant import (
TQ_INT8,
@@ -335,7 +338,8 @@ def _weight_has_zero_output_row(module: Any) -> bool:
def _cast_fp8_dynamic(encoder: Any, target: Any) -> None:
- # torchao dynamic fp8 COMPUTE, per-row (torch._scaled_mm on the fp8 cores). Unlike layerwise `fp8` the matmul stays in fp8, and it is robust across encoder sizes, so only the vision tower / lm_head / T5 wo are excluded.
+ # torchao dynamic fp8 COMPUTE, per-row (torch._scaled_mm on the fp8 cores). Unlike layerwise `fp8` the matmul stays
+ # in fp8, and it is robust across encoder sizes, so only the vision tower / lm_head / T5 wo are excluded.
from torchao.quantization import quantize_
from .diffusion_transformer_quant import (
TQ_FP8,
@@ -349,7 +353,6 @@ def _cast_fp8_dynamic(encoder: Any, target: Any) -> None:
DEFAULT_MIN_LINEAR_FEATURES, _te_exclude_tokens(encoder), require_bf16 = True
)
- # An all-zero output row NaNs under per-row scaling (scale 0 -> 0/0); keep those dense.
def filter_fn(module: Any, fqn: str = "") -> bool:
return base(module, fqn) and not _weight_has_zero_output_row(module)
@@ -362,17 +365,23 @@ def _cast_fp8(encoder: Any, target: Any) -> None:
from diffusers.hooks import apply_layerwise_casting
from diffusers.hooks.layerwise_casting import DEFAULT_SKIP_MODULES_PATTERN
- # Idempotent: a pre-cast encoder arrives with the layerwise hooks installed and re-registering a hook name raises, which would report an engaged cast as failed. Keyed on the completion marker, NOT hook presence, so a mid-pass failure still fails closed.
+ # idempotent, keyed on the completion marker NOT hook presence
+ # Idempotent: a pre-cast encoder arrives with the layerwise hooks installed and re-registering a hook name raises,
+ # which would report an engaged cast as failed. Keyed on the completion marker, NOT hook presence, so a mid-pass
+ # failure still fails closed.
if getattr(encoder, "_unsloth_te_cast_complete", False) and _has_layerwise_hooks(encoder):
return
- # Layerwise casting stores each leaf's weights in fp8 and upcasts per forward. Two things on a transformers encoder push an fp8 weight/activation into an op that cannot handle it, both crashing only at generation, so skip them:
+ # Layerwise casting stores each leaf's weights in fp8 and upcasts per forward. Two things on a transformers encoder
+ # push an fp8 weight/activation into an op that cannot handle it, both crashing only at generation, so skip them:
skip = tuple(DEFAULT_SKIP_MODULES_PATTERN)
- # (1) dtype-sensitive modules the encoder flags. T5 keeps "wo" in fp32: its gated FF reads self.wo.weight.dtype and casts activations to match BEFORE calling wo (transformers#20287), racing the upcast hook. Literal substrings.
+ # (1) dtype-sensitive modules the encoder flags. T5 keeps "wo" in fp32: its gated FF reads self.wo.weight.dtype and
+ # casts activations to match BEFORE calling wo (transformers#20287), racing the upcast hook. Literal substrings.
skip += tuple(re.escape(m) for m in (getattr(encoder, "_keep_in_fp32_modules", None) or ()))
- # (2) an output projection tied to the input embedding. FLUX.2's Qwen3 ties lm_head.weight to embed_tokens.weight, so casting lm_head drags the shared embedding to fp8 and the first RMSNorm crashes. lm_head is unused here anyway.
+ # (2) an output projection tied to the input embedding. FLUX.2's Qwen3 ties lm_head.weight to embed_tokens.weight,
+ # so casting lm_head drags the shared embedding to fp8 and the first RMSNorm crashes. lm_head is unused here anyway.
get_out, get_in = (
getattr(encoder, "get_output_embeddings", None),
getattr(encoder, "get_input_embeddings", None),
@@ -389,20 +398,24 @@ def _cast_fp8(encoder: Any, target: Any) -> None:
storage_dtype = torch.float8_e4m3fn,
compute_dtype = target.dtype,
skip_modules_pattern = skip,
- # Keep token-embedding tables full precision: the diffusers default only skips vision pos/patch embeds, and fp8-ing nn.Embedding puts every prompt token on the coarse fp8 grid.
+ # fp8-ing nn.Embedding would put every prompt token on the coarse fp8 grid
+ # Keep token-embedding tables full precision: the diffusers default only skips vision pos/patch embeds, and
+ # fp8-ing nn.Embedding puts every prompt token on the coarse fp8 grid.
skip_modules_classes = (torch.nn.Embedding,),
)
- # Module.dtype reports the first floating parameter, now fp8 STORAGE, but pipelines derive tensor dtypes from it (Flux2
- # feeds it to randn_tensor, which has no fp8 kernel). Report the compute dtype via a property shadowed on the ORIGINAL class reading a per-instance override; a dynamic __class__ swap breaks transformers' output recording.
+ # Module.dtype reports the first floating parameter, now fp8 STORAGE, but pipelines derive tensor dtypes from it
+ # (Flux2 feeds it to randn_tensor, which has no fp8 kernel). Report the compute dtype via a property shadowed on the
+ # ORIGINAL class reading a per-instance override; a dynamic __class__ swap breaks transformers' output recording.
compute_dtype = getattr(target, "dtype", None)
try:
if compute_dtype is not None:
_install_dtype_override(type(encoder))
encoder._unsloth_te_compute_dtype = compute_dtype
- # Marks the cast COMPLETE (hooks fully installed) for the idempotent early return above. Best-effort: a non-Module double without settable attributes just re-casts.
+ # Marks the cast COMPLETE (hooks fully installed) for the idempotent early return above. Best-effort: a
+ # non-Module double without settable attributes just re-casts.
encoder._unsloth_te_cast_complete = True
- except Exception: # noqa: BLE001 — real HF encoders are heap-type nn.Modules; only doubles fail
+ except Exception: # noqa: BLE001 - real HF encoders are heap-type nn.Modules; only doubles fail
pass
@@ -442,7 +455,8 @@ def _has_layerwise_hooks(encoder: Any) -> bool:
def _cast_nvfp4(encoder: Any, target: Any) -> None:
- # Weight-only NVFP4: linear weights become 4-bit NVFP4 on Blackwell FP4 cores, norms / embeddings untouched. Same exclusions as the int8/fp8 TE modes; require_bf16 skips non-bf16 Linears so the cast engages instead of aborting.
+ # Weight-only NVFP4: linear weights become 4-bit NVFP4 on Blackwell FP4 cores, norms / embeddings untouched. Same
+ # exclusions as the int8/fp8 TE modes; require_bf16 skips non-bf16 Linears so the cast engages instead of aborting.
from torchao.quantization import quantize_
from torchao.prototype.mx_formats import NVFP4WeightOnlyConfig
from .diffusion_transformer_quant import DEFAULT_MIN_LINEAR_FEATURES, make_filter_fn
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index f83590c90bc..b169041a742 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -30,16 +30,15 @@ import threading as _threading
from dataclasses import dataclass
from typing import Any, Optional
-# torch.save dict layout tag; bump on an on-disk change so old/foreign artifacts are rejected.
+# torch.save dict layout tag; bump on an on-disk change so old/foreign artifacts are rejected
PREQUANT_FORMAT = "unsloth_prequant_transformer_state_dict_v1"
-# v2 is v1 plus an ACTIVATION ROTATION (see ``diffusion_convrot``): the weights are stored in a
-# rotated basis and are wrong unless the loader rotates the activations to match. That is the one
-# on-disk change a released Unsloth cannot ignore safely -- an old build reading a rotated artifact
-# as v1 would load it clean, raise nothing and render quietly wrong pixels forever -- so it gets a
-# tag old builds refuse outright, and the load drops to dense instead. Strictly a biconditional:
-# a v2 artifact MUST declare a rotation and a v1 artifact must NOT, both checked below, so neither
-# a hand-edited tag nor a builder that forgot one half can produce something that loads.
+# v2 is v1 plus an ACTIVATION ROTATION (see ``diffusion_convrot``): the weights are stored in a rotated basis and are
+# wrong unless the loader rotates the activations to match. That is the one on-disk change a released Unsloth cannot
+# ignore safely -- an old build reading a rotated artifact as v1 would load it clean, raise nothing and render quietly
+# wrong pixels forever -- so it gets a tag old builds refuse outright, and the load drops to dense instead. Strictly a
+# biconditional: a v2 artifact MUST declare a rotation and a v1 artifact must NOT, both checked below, so neither a
+# hand-edited tag nor a builder that forgot one half can produce something that loads.
PREQUANT_FORMAT_ROTATED = "unsloth_prequant_transformer_state_dict_v2"
PREQUANT_FORMATS = (PREQUANT_FORMAT, PREQUANT_FORMAT_ROTATED)
@@ -51,28 +50,23 @@ def prequant_format_for(metadata: Any) -> str:
return PREQUANT_FORMAT_ROTATED if declares_rotation(metadata) else PREQUANT_FORMAT
-# A request-supplied ``kind == "path"`` is read ONLY inside an operator-configured directory
-# ALLOWLIST: an arbitrary path is an arbitrary MODEL. Not a code-execution gate -- the load is
-# weights_only either way.
+# A request-supplied ``kind == "path"`` is read ONLY inside an operator-configured directory ALLOWLIST: an arbitrary
+# path is an arbitrary MODEL. Not a code-execution gate -- the load is weights_only either way.
ALLOW_LOCAL_PREQUANT_PATH_ENV = "UNSLOTH_ALLOW_LOCAL_PREQUANT_PATH"
-# The constructors a pre-quant checkpoint's pickle may name, on top of what ``weights_only``
-# already permits (storages, dtypes, ``_rebuild_*``, ``OrderedDict``, ``torch.device``,
-# ``_get_layout``). Surveyed across every hosted checkpoint Unsloth resolves (image + video, fp8 +
-# int8, rotated and not) this is the complete set, so the load runs ``weights_only = True`` and a
-# checkpoint naming anything else is refused before one opcode of it executes, hosted or local.
-#
-# Registered under the name the PICKLE records, which for a re-exported class is not the class's
-# own ``__module__`` (``torchao.quantization.Float8Tensor`` really lives in
-# ``...quantize_.workflows.float8.float8_tensor``), so both spellings are listed. Names a given
-# torchao lacks are skipped rather than raised: the set spans every release
-# ``install_python_stack`` pins (0.14, 0.16, 0.17) and an absent class could not have produced a
-# loadable checkpoint here anyway.
-#
-# Adding a scheme means adding its constructors here; forgetting warns and falls back to
-# dense-quantise, never a silent unpickle.
+# constructors a pre-quant checkpoint's pickle may name on top of what weights_only permits;
+# The constructors a pre-quant checkpoint's pickle may name, on top of what ``weights_only`` already permits (storages,
+# dtypes, ``_rebuild_*``, ``OrderedDict``, ``torch.device``, ``_get_layout``). Surveyed across every hosted checkpoint
+# Unsloth resolves (image + video, fp8 + int8, rotated and not) this is the complete set, so the load runs
+# ``weights_only = True`` and a checkpoint naming anything else is refused before one opcode of it executes, hosted or
+# local. Registered under the name the PICKLE records, which for a re-exported class is not the class's own
+# ``__module__`` (``torchao.quantization.Float8Tensor`` really lives in
+# ``...quantize_.workflows.float8.float8_tensor``), so both spellings are listed. Names a given torchao lacks are
+# skipped rather than raised: the set spans every release ``install_python_stack`` pins (0.14, 0.16, 0.17) and an absent
+# class could not have produced a loadable checkpoint here anyway. Adding a scheme means adding its constructors here;
+# forgetting warns and falls back to dense-quantise, never a silent unpickle.
_PREQUANT_SAFE_GLOBALS: tuple[tuple[str, str], ...] = (
- # int8: AffineQuantizedTensor + its plain layout, wrapped for dynamic activation quant.
+ # int8: AffineQuantizedTensor + its plain layout, wrapped for dynamic activation quant
("torchao.dtypes.affine_quantized_tensor", "AffineQuantizedTensor"),
("torchao.dtypes.uintx.plain_layout", "PlainAQTTensorImpl"),
("torchao.dtypes.utils", "PlainLayout"),
@@ -91,16 +85,16 @@ _PREQUANT_SAFE_GLOBALS: tuple[tuple[str, str], ...] = (
("torchao.quantization.granularity", "PerRow"),
("torchao.quantization.granularity", "PerTensor"),
("torchao.float8.inference", "Float8MMConfig"),
- # mxfp8 / nvfp4: no hosted checkpoint uses these, but they are TQ_SCHEMES that
- # scripts/build_prequant_checkpoint.py bakes, so a LOCAL override can be either. torchao only
- # registers them on import of the prototype package, which nothing on this path imports.
+ # mxfp8 / nvfp4: no hosted checkpoint uses these, but they are TQ_SCHEMES that scripts/build_prequant_checkpoint.py
+ # bakes, so a LOCAL override can be either. torchao only registers them on import of the prototype package, which
+ # nothing on this path imports.
("torchao.prototype.mx_formats.mx_tensor", "MXTensor"),
("torchao.prototype.mx_formats.mx_tensor", "QuantizeTensorToMXKwargs"),
("torchao.prototype.mx_formats.config", "ScaleCalculationMode"),
("torchao.prototype.mx_formats.nvfp4_tensor", "NVFP4Tensor"),
("torchao.prototype.mx_formats.nvfp4_tensor", "QuantizeTensorToNVFP4Kwargs"),
- # The version string torch.save stamps into the subclass state: not in torch's default set,
- # and without it every torchao checkpoint refuses to load.
+ # The version string torch.save stamps into the subclass state: not in torch's default set, and without it every
+ # torchao checkpoint refuses to load.
("torch.torch_version", "TorchVersion"),
)
@@ -124,9 +118,9 @@ _SAFE_GLOBALS_REGISTERED: Optional[bool] = None
# Filled in by the registration: which of the names above this install actually resolved.
_RESOLVED_SAFE_GLOBALS: set = set()
-# What a checkpoint of each scheme actually NAMES, read off the artifacts with pickletools rather
-# than assumed: every hosted repo the family tables list, plus a local bake of each scheme for the
-# two nothing hosts. Only these are required, so dropping an unused name does not fail a scheme.
+# What a checkpoint of each scheme actually NAMES, read off the artifacts with pickletools rather than assumed: every
+# hosted repo the family tables list, plus a local bake of each scheme for the two nothing hosts. Only these are
+# required, so dropping an unused name does not fail a scheme.
_SCHEME_REQUIRED_GLOBALS: dict = {
"int8": frozenset(
{
@@ -216,23 +210,21 @@ def _register_prequant_safe_globals() -> bool:
import torch
add = getattr(torch.serialization, "add_safe_globals", None)
- # A STUBBED torchao (Windows ROCm) fabricates a class for every name asked of it, so
- # the allowlist would register fakes and answer yes for an install that cannot
- # rebuild a single quantized tensor.
+ # A STUBBED torchao (Windows ROCm) fabricates a class for every name asked of it, so the allowlist would
+ # register fakes and answer yes for an install that cannot rebuild a single quantized tensor.
if add is not None and not is_stubbed("torchao") and _tuple_safe_globals_supported():
pairs = _prequant_safe_globals()
resolved = {name for _obj, name in pairs}
- # "Some entries resolved" is not "a checkpoint can be opened". The floor is what
- # EVERY artifact needs whatever its scheme: the version stamp plus at least one
- # real torchao tensor class. Per-SCHEME completeness is asked separately, by the
- # caller that knows which scheme it is about to plan for.
+ # "Some entries resolved" is not "a checkpoint can be opened". The floor is what EVERY artifact needs
+ # whatever its scheme: the version stamp plus at least one real torchao tensor class. Per-SCHEME
+ # completeness is asked separately, by the caller that knows which scheme it is about to plan for.
if "torch.torch_version.TorchVersion" in resolved and any(
name.startswith("torchao.") for name in resolved
):
add(pairs)
_RESOLVED_SAFE_GLOBALS.update(resolved)
- # The same derivation the unpickler runs, so a form this torch cannot express
- # fails here rather than under a load a plan was already sized on.
+ # The same derivation the unpickler runs, so a form this torch cannot express fails here rather than
+ # under a load a plan was already sized on.
try:
torch._weights_only_unpickler._get_user_allowed_globals()
except AttributeError: # noqa: BLE001 -- private; absence is not a failure
@@ -304,7 +296,7 @@ def _allowed_prequant_roots() -> list:
continue # a bare on/off value is not a directory
try:
roots.append(os.path.realpath(os.path.expanduser(part)))
- except Exception: # noqa: BLE001 — a bad entry is simply not allowlisted
+ except Exception: # noqa: BLE001 - a bad entry is simply not allowlisted
continue
return roots
@@ -399,24 +391,20 @@ def resolve_prequant_source(
repo_id = family_prequant_repo(fam, scheme, base_repo = base_repo)
preferred = family_prequant_filename(fam, scheme, task = task)
- # What the same call would have resolved WITHOUT a task, which is what decides whether a
- # fallback is safe below. Skipped when no task was asked for, since then the two are the
- # same lookup.
+ # What the same call would have resolved WITHOUT a task, which is what decides whether a fallback is safe below.
+ # Skipped when no task was asked for, since then the two are the same lookup.
agnostic = family_prequant_filename(fam, scheme) if task else preferred
- except Exception: # noqa: BLE001 — a bad family object must not break the load
+ except Exception: # noqa: BLE001 - a bad family object must not break the load
repo_id = None
if repo_id:
derived = prequant_repo_filename(repo_id, scheme)
- # A family may name a SECOND artifact for the same repo and scheme (today: MiniMax-H3's
- # rotated INT8 denoiser). It becomes the primary and the derived name becomes the
- # fallback, so a build that knows the new name gets it and every older build keeps
- # resolving the artifact it already understands. Without an override nothing changes: the
- # derived name is primary and the legacy transformer_<scheme>.pt is the fallback.
- #
- # A TASK-SPECIFIC name gets NO fallback. The other artifacts in the repo are the same
- # family, the same scheme and the same base, so every check the loader makes would pass
- # on them -- the fallback would quietly install another partition's denoiser and generate
- # from the wrong weights, which is precisely what naming the artifact per task prevents.
+ # A family may name a SECOND artifact for the same repo and scheme (today: MiniMax-H3's rotated INT8 denoiser).
+ # It becomes the primary and the derived name becomes the fallback, so a build that knows the new name gets it
+ # and every older build keeps resolving the artifact it already understands. Without an override nothing
+ # changes: the derived name is primary and the legacy transformer_<scheme>.pt is the fallback. A TASK-SPECIFIC
+ # name gets NO fallback. The other artifacts in the repo are the same family, the same scheme and the same base,
+ # so every check the loader makes would pass on them -- the fallback would quietly install another partition's
+ # denoiser and generate from the wrong weights, which is precisely what naming the artifact per task prevents.
# Absent is better than wrong here: no artifact means the released bfloat16 denoiser.
task_specific = preferred is not None and preferred != agnostic
return PrequantSource(
@@ -453,9 +441,9 @@ def local_prequant_scheme(path: str) -> Optional[str]:
try:
real = os.path.expanduser(path)
st = os.stat(real)
- # Nanoseconds, not int(st_mtime): an atomic swap for a same-sized artifact inside the same
- # second would otherwise reuse the previous scheme for the life of the process, and int8
- # and fp8 checkpoints of one model are exactly that shape.
+ # Nanoseconds, not int(st_mtime): an atomic swap for a same-sized artifact inside the same second would
+ # otherwise reuse the previous scheme for the life of the process, and int8 and fp8 checkpoints of one model are
+ # exactly that shape.
key = (real, st.st_mtime_ns, int(st.st_size))
except Exception: # noqa: BLE001 -- unreadable is "unknown", handled by the caller
return None
@@ -539,13 +527,13 @@ def _cached_in_root(
import os
from huggingface_hub import try_to_load_from_cache
- except Exception: # noqa: BLE001 — no cache API to ask: treat as not cached
+ except Exception: # noqa: BLE001 - no cache API to ask: treat as not cached
return None
try:
hit = try_to_load_from_cache(source.location, name, cache_dir = root)
- except Exception: # noqa: BLE001 — a malformed cache entry is not a hit
+ except Exception: # noqa: BLE001 - a malformed cache entry is not a hit
return None
- # A str is the cached path; a miss is None and a known-absent file is a sentinel object.
+ # a str is the cached path; a miss is None and a known-absent file is a sentinel object
return hit if isinstance(hit, str) and os.path.isfile(hit) else None
@@ -636,8 +624,8 @@ def load_prequantized_transformer(
artifact.
"""
try:
- # A request-supplied local path names arbitrary WEIGHTS, a different question from the
- # deserialization one below: allowlisted or not, the file is read weights_only.
+ # A request-supplied local path names arbitrary WEIGHTS, a different question from the deserialization one
+ # below: allowlisted or not, the file is read weights_only.
if source.kind == "path" and not _local_prequant_path_allowed(source.location):
_warn(
logger,
@@ -656,11 +644,10 @@ def load_prequantized_transformer(
if path is None:
return None
- # A torch.save pickle, deserialized under the constructor ALLOWLIST above and never as a
- # free-running one. First-party hosting is no reason to execute whatever bytes arrive: the
- # artifact is mutable, fetched over the network, and reached by loads that never asked for
- # one (auto resolves an unset precision to a hosted checkpoint), so a mutated file must
- # fail to load rather than run.
+ # A torch.save pickle, deserialized under the constructor ALLOWLIST above and never as a free-running one.
+ # First-party hosting is no reason to execute whatever bytes arrive: the artifact is mutable, fetched over the
+ # network, and reached by loads that never asked for one (auto resolves an unset precision to a hosted
+ # checkpoint), so a mutated file must fail to load rather than run.
ckpt = _torch_load_prequant(path, map_location = "cpu")
if not _validate_checkpoint(
ckpt, scheme, base, logger, min_features = min_features, fast_accum = fast_accum
@@ -669,9 +656,9 @@ def load_prequantized_transformer(
state_dict = ckpt["state_dict"]
_pin_kernel_preference(state_dict, logger)
- # Read from the root that actually supplied the checkpoint: after a mid-session cache change
- # the pinned root may be gone or read-only, and load_config's raise is swallowed below into
- # a None return, silently dropping a prequant whose checkpoint is cached and already loaded.
+ # Read from the root that actually supplied the checkpoint: after a mid-session cache change the pinned root may
+ # be gone or read-only, and load_config's raise is swallowed below into a None return, silently dropping a
+ # prequant whose checkpoint is cached and already loaded.
config = _load_transformer_config(
transformer_cls,
base,
@@ -688,48 +675,47 @@ def load_prequantized_transformer(
transformer = transformer_cls.from_config(config)
if prepare_model is not None:
prepare_model(transformer, metadata)
- # assign=True swaps in the loaded tensors instead of copying into meta (a no-op); strict=True since the saved dict is the full state dict of the same class.
transformer.load_state_dict(state_dict, strict = True, assign = True)
if _has_meta_tensors(transformer):
- # Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so they hold real values, then re-assign the quantized weights; dense bf16 never reaches the GPU.
+ # Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so
+ # they hold real values, then re-assign the quantized weights; dense bf16 never reaches the GPU.
transformer = transformer_cls.from_config(config)
- # The retry REPLACES the module, so the hook has to run again: skipping it here would
- # load the same state dict into a differently shaped model, and this branch is the one
- # families with non-persistent buffers always take -- the mismatch would be the norm,
- # not the corner case, and strict=True would surface it as a bare key error.
+ # The retry REPLACES the module, so the hook has to run again: skipping it here would load the same state
+ # dict into a differently shaped model, and this branch is the one families with non-persistent buffers
+ # always take -- the mismatch would be the norm, not the corner case, and strict=True would surface it as a
+ # bare key error.
if prepare_model is not None:
prepare_model(transformer, metadata)
transformer.load_state_dict(state_dict, strict = True, assign = True)
- # The ONLINE half of an activation rotation, applied here rather than in a family's
- # ``prepare_model`` hook so that no route can load a rotated checkpoint without it: the
- # offline half is already baked into the weights that were just assigned, and a rotated
- # weight met by an unrotated activation renders plausible garbage with nothing to catch.
- # A no-op for every artifact that declares no rotation, and a RAISE (caught below into the
- # dense fallback) for one this build cannot honour exactly. After load_state_dict because
- # the meta retry above rebuilds the module; before apply_small_m_padding because padding
- # reparents the Linears and the recorded fqns name the unwrapped tree.
+ # The ONLINE half of an activation rotation, applied here rather than in a family's ``prepare_model`` hook so
+ # that no route can load a rotated checkpoint without it: the offline half is already baked into the weights
+ # that were just assigned, and a rotated weight met by an unrotated activation renders plausible garbage with
+ # nothing to catch. A no-op for every artifact that declares no rotation, and a RAISE (caught below into the
+ # dense fallback) for one this build cannot honour exactly. After load_state_dict because the meta retry above
+ # rebuilds the module; before apply_small_m_padding because padding reparents the Linears and the recorded fqns
+ # name the unwrapped tree.
from .diffusion_convrot import apply_activation_rotation
apply_activation_rotation(transformer, metadata, logger = logger)
transformer = transformer.to(device)
- # Same small-M row padding the runtime quantise path applies, and for the same reason: a
- # checkpoint built under the current exclusion set QUANTISES the family's small-M linears,
- # so without the wrappers they would raise inside _int_mm the moment the compiled scope
- # reaches them. After load_state_dict, since wrapping reparents the Linears; after .to()
- # so the granularity probe reads the device tensors the GEMM will actually see.
+ # Same small-M row padding the runtime quantise path applies, and for the same reason: a checkpoint built under
+ # the current exclusion set QUANTISES the family's small-M linears, so without the wrappers they would raise
+ # inside _int_mm the moment the compiled scope reaches them. After load_state_dict, since wrapping reparents the
+ # Linears; after.to() so the granularity probe reads the device tensors the GEMM will see.
from .diffusion_transformer_quant import apply_small_m_padding
apply_small_m_padding(transformer, scheme, metadata.get("family"), logger = logger)
- # from_config starts in TRAIN mode while the dense/GGUF paths use from_pretrained (eval()'d). Match it so train/eval-sensitive layers cannot make prequant inference diverge.
+ # from_config starts in TRAIN mode while the dense/GGUF paths use from_pretrained (eval()'d). Match it so
+ # train/eval-sensitive layers cannot make prequant inference diverge.
try:
transformer.eval()
- except Exception: # noqa: BLE001 — eval() is best-effort
+ except Exception: # noqa: BLE001 - eval() is best-effort
pass
- try: # diagnostic marker, mirrors the runtime-quant path
+ try:
transformer._unsloth_runtime_quant = scheme
- except Exception: # noqa: BLE001 — marker is best-effort
+ except Exception: # noqa: BLE001 - marker is best-effort
pass
if logger is not None:
logger.info(
@@ -739,7 +725,7 @@ def load_prequantized_transformer(
device,
)
return transformer
- except Exception as exc: # noqa: BLE001 — fall back to the dense-quantise path
+ except Exception as exc: # noqa: BLE001 - fall back to the dense-quantise path
_warn(logger, f"{scheme}:{source.kind}", exc)
return None
@@ -752,7 +738,7 @@ def _entry_not_found_errors() -> tuple:
Private markers on an unexpected layout are raised by nothing, keeping today's behaviour."""
try:
from huggingface_hub.errors import EntryNotFoundError
- except Exception: # noqa: BLE001 — older/newer hub layouts
+ except Exception: # noqa: BLE001 - older/newer hub layouts
class EntryNotFoundError(Exception): # type: ignore[no-redef]
pass
@@ -806,7 +792,7 @@ def _download_checkpoint_name(
if not propagate_missing:
return elsewhere
raise
- except Exception: # noqa: BLE001 — revalidation is a bonus, never a new failure
+ except Exception: # noqa: BLE001 - revalidation is a bonus, never a new failure
return elsewhere
return hf_hub_download(
repo_id = source.location,
@@ -851,7 +837,6 @@ def _resolve_checkpoint_path(
except EntryNotFoundError:
if not has_fallback:
raise
- # Primary genuinely absent: same other-root treatment, with nothing left after it.
return _download_checkpoint_name(
source,
source.fallback_filename,
@@ -875,12 +860,12 @@ def _config_cache_roots(checkpoint_path: str, cache_dir: Optional[str]) -> tuple
import os
try:
- # normcase before comparing: on Windows C:\Users vs c:\users would read as "not under the
- # live root" and silently reverse the order below.
+ # normcase before comparing: on Windows C:\Users vs c:\users would read as "not under the live root" and
+ # silently reverse the order below
root = os.path.normcase(os.path.realpath(cache_dir))
real = os.path.normcase(os.path.realpath(checkpoint_path))
under_live = real == root or real.startswith(root + os.sep)
- except Exception: # noqa: BLE001 — an unresolvable path keeps today's order
+ except Exception: # noqa: BLE001 - an unresolvable path keeps today's order
under_live = True
return (cache_dir, None) if under_live else (None, cache_dir)
@@ -909,7 +894,7 @@ def _load_transformer_config(
cache_dir = root,
local_files_only = local_files_only,
)
- except Exception as exc: # noqa: BLE001 — try the other root before giving up
+ except Exception as exc: # noqa: BLE001 - try the other root before giving up
last = exc
raise last # type: ignore[misc]
@@ -1014,7 +999,8 @@ def _validate_checkpoint(
if meta.get("scheme") != scheme:
_warn(logger, scheme, ValueError(f"checkpoint scheme {meta.get('scheme')!r} != {scheme!r}"))
return False
- # fp8 REQUIRES per-row granularity (per-tensor collapses outlier-heavy DiTs to noise). An old checkpoint omits ``fp8_granularity`` or records non-per-row, so reject and let the loader re-quantise.
+ # fp8 REQUIRES per-row granularity (per-tensor collapses outlier-heavy DiTs to noise). An old checkpoint omits
+ # ``fp8_granularity`` or records non-per-row, so reject and let the loader re-quantise.
from .diffusion_transformer_quant import FP8_GRANULARITY, TQ_FP8
if scheme == TQ_FP8 and meta.get("fp8_granularity") != FP8_GRANULARITY:
@@ -1027,18 +1013,18 @@ def _validate_checkpoint(
),
)
return False
- # fp8 also REQUIRES the activation scale floor, and this is checked on the loaded TENSORS, not
- # on metadata. torchao's per-row activation quantiser divides by each row's amax, so a zero row
- # (qwen's text stream emits them) gives scale 0 and NaN qdata unless activation_value_lb floors
- # it. That floor is serialised per tensor as act_quant_kwargs.hp_value_lb, so an artifact built
- # before the fix stays broken however it is loaded, and it predates any metadata field we could
- # stamp -- and "absent is accepted for back-compat", the convention every check above follows,
- # is exactly wrong here. Reading the tensors is fail-closed and needs no format bump.
+ # fp8 also REQUIRES the activation scale floor, and this is checked on the loaded TENSORS, not on metadata.
+ # torchao's per-row activation quantiser divides by each row's amax, so a zero row (qwen's text stream emits them)
+ # gives scale 0 and NaN qdata unless activation_value_lb floors it. That floor is serialised per tensor as
+ # act_quant_kwargs.hp_value_lb, so an artifact built before the fix stays broken however it is loaded, and it
+ # predates any metadata field we could stamp -- and "absent is accepted for back-compat", the convention every check
+ # above follows, is exactly wrong here. Reading the tensors is fail-closed and needs no format bump.
if scheme == TQ_FP8 and not _fp8_activation_floor_present(ckpt.get("state_dict"), logger):
return False
ckpt_base = meta.get("base_model_id")
if base:
- # Keys matching a different base can load strict=True and generate from the wrong weights. Our builder always records base_model_id, so one omitting it against a requested base is untrustworthy.
+ # Keys matching a different base can load strict=True and generate from the wrong weights. Our builder always
+ # records base_model_id, so one omitting it against a requested base is untrustworthy.
if not ckpt_base:
_warn(
logger,
@@ -1060,12 +1046,15 @@ def _validate_checkpoint(
ValueError(f"checkpoint min_features {ckpt_min!r} != runtime {min_features!r}"),
)
return False
- # The int8 exclusion set is scheme-derived, so a token-list change would leave old checkpoints with a stale baked set that passes scheme+min_features then crashes at the first denoise. Reject a recorded mismatch; absent is accepted.
+ # The int8 exclusion set is scheme-derived, so a token-list change would leave old checkpoints with a stale baked
+ # set that passes scheme+min_features then crashes at the first denoise. Reject a recorded mismatch; absent is
+ # accepted.
ckpt_excludes = meta.get("exclude_name_tokens")
if ckpt_excludes is not None:
from .diffusion_transformer_quant import exclude_tokens_for_scheme
- # The exclude set derives from scheme AND family, so use the recorded family: an artifact baked under an older token list is rejected and re-quantised, not loaded crashing.
+ # The exclude set derives from scheme AND family, so use the recorded family: an artifact baked under an older
+ # token list is rejected and re-quantised, not loaded crashing.
expected = tuple(exclude_tokens_for_scheme(scheme, meta.get("family")))
if tuple(ckpt_excludes) != expected:
_warn(
@@ -1076,7 +1065,8 @@ def _validate_checkpoint(
),
)
return False
- # require_bf16 (skip non-bf16 Linears) is scheme-pinned; recording and verifying it stops a future _REQUIRE_BF16_SCHEMES change loading an old-filter checkpoint. Absent accepted.
+ # require_bf16 (skip non-bf16 Linears) is scheme-pinned; recording and verifying it stops a future
+ # _REQUIRE_BF16_SCHEMES change loading an old-filter checkpoint. Absent accepted.
ckpt_require_bf16 = meta.get("require_bf16")
if ckpt_require_bf16 is not None:
from .diffusion_transformer_quant import _REQUIRE_BF16_SCHEMES
@@ -1090,7 +1080,7 @@ def _validate_checkpoint(
),
)
return False
- # fp8 fast-accum is baked into the saved kernels; only enforce when the caller forces it.
+ # fp8 fast-accum is baked into the saved kernels; only enforce when the caller forces it
if fast_accum is not None:
ckpt_fa = meta.get("fast_accum")
if ckpt_fa is not None and bool(ckpt_fa) != bool(fast_accum):
@@ -1162,8 +1152,8 @@ def pin_prequantized_module(
try:
# Drop the accelerate hook so no pre_forward/offload ever moves this module again ...
target.remove()
- # ... and unlist it, so another component's pre_forward cannot pick it as the thing to
- # evict (which would move it to the CPU with no hook left to bring it back).
+ # ... and unlist it, so another component's pre_forward cannot pick it as the thing to evict (which would
+ # move it to the CPU with no hook left to bring it back).
for hook in hooks:
others = getattr(getattr(hook, "hook", None), "other_hooks", None)
if others:
diff --git a/studio/backend/core/inference/diffusion_quant_pad.py b/studio/backend/core/inference/diffusion_quant_pad.py
index c49b6320173..b969a735652 100644
--- a/studio/backend/core/inference/diffusion_quant_pad.py
+++ b/studio/backend/core/inference/diffusion_quant_pad.py
@@ -52,10 +52,10 @@ from torch import nn
# ``_int_mm`` wants strictly more than 16 rows.
INT_MM_MIN_M = 17
-# Pad to a warp-friendly constant rather than to exactly INT_MM_MIN_M. Two reasons: 32 tiles
-# better than 17, and it pins ONE compiled shape for every activation below it, so prompts of
-# differing token counts (H3's seven eval prompts run at M = 10..19) do not each trigger their
-# own inductor recompile.
+# pad to a warp-friendly 32 rather than exactly INT_MM_MIN_M
+# Pad to a warp-friendly constant rather than to exactly INT_MM_MIN_M. Two reasons: 32 tiles better than 17, and it pins
+# ONE compiled shape for every activation below it, so prompts of differing token counts (H3's seven eval prompts run at
+# M = 10..19) do not each trigger their own inductor recompile.
DEFAULT_PAD_TO = 32
@@ -132,30 +132,30 @@ class PadToMinM(nn.Module):
super().__init__()
self.inner = inner
self.min_m = int(min_m)
- # pad_to may exceed min_m to buy tiling and shape stability; it may never be below it,
- # or the "padded" activation would still be under the floor it exists to clear.
+ # pad_to may exceed min_m to buy tiling and shape stability; it may never be below it, or the "padded"
+ # activation would still be under the floor it exists to clear.
self.pad_to = max(int(pad_to or min_m), self.min_m)
def forward(self, x: torch.Tensor) -> torch.Tensor:
- # Deliberately free of call counters or any other mutable int attribute: dynamo treats an
- # nn.Module's integer attributes as static and guards on their value, so a `+= 1` here
- # would force a recompile on EVERY call until the recompile limit is hit, at which point
- # the module silently drops back to eager. Instrumentation belongs outside.
+ # deliberately free of call counters: dynamo guards on an nn.Module's integer attributes
+ # Deliberately free of call counters or any other mutable int attribute: dynamo treats an nn.Module's integer
+ # attributes as static and guards on their value, so a `+= 1` here would force a recompile on EVERY call until
+ # the recompile limit is hit, at which point the module silently drops back to eager. Instrumentation belongs
+ # outside.
lead = x.shape[:-1]
flat = x.reshape(-1, x.shape[-1])
m = flat.shape[0]
if m == 0:
- # No rows to project, and no row 0 to replicate from. torchao returns a zero-row
- # input UNPROJECTED (a quantized 1472 -> 2048 Linear maps [0, 1472] to [0, 1472]),
- # which then breaks a downstream width-sensitive add, so synthesise the empty result
- # at the right width instead of calling through.
+ # torchao returns a zero-row input UNPROJECTED (a 1472 -> 2048 Linear maps [0, 1472] to [0, 1472])
+ # No rows to project, and no row 0 to replicate from. torchao returns a zero-row input UNPROJECTED (a
+ # quantized 1472 -> 2048 Linear maps [0, 1472] to [0, 1472]), which then breaks a downstream width-sensitive
+ # add, so synthesise the empty result at the right width instead of calling through.
return x.new_empty((*lead, self.inner.out_features))
if m < self.pad_to:
- # Everything below pad_to normalises to pad_to, not just what is below min_m. Clearing
- # the floor takes only the latter, but pinning ONE row count means one inductor graph
- # covers every prompt length in the range instead of one per length, and the extra
- # rows are free at these sizes (measured on H3's 13 modules: 1.57 ms padded from
- # M = 10 against 1.48 ms unpadded at M = 17, on a 2.4 s render).
+ # Everything below pad_to normalises to pad_to, not just what is below min_m. Clearing the floor takes only
+ # the latter, but pinning ONE row count means one inductor graph covers every prompt length in the range
+ # instead of one per length, and the extra rows are free at these sizes (measured on H3's 13 modules: 1.57
+ # ms padded from M = 10 against 1.48 ms unpadded at M = 17, on a 2.4 s render).
flat = torch.cat([flat, flat[:1].expand(self.pad_to - m, -1)], dim = 0)
out = self.inner(flat)[:m]
else:
@@ -163,12 +163,11 @@ class PadToMinM(nn.Module):
return out.reshape(*lead, out.shape[-1])
def __getattr__(self, name: str) -> Any:
- # Callers reach THROUGH a Linear for weight / bias / in_features / out_features: diffusers'
- # attention processors read `to_q.weight.dtype`, and H3's blocks read
- # `context_embedder.weight`. Without this forward the wrapper is a drop-in only until the
- # first such access, which fails at render time rather than at wrap time.
- # nn.Module.__getattr__ runs first, so parameters, buffers and submodules registered on
- # the wrapper itself still win.
+ # Callers reach THROUGH a Linear for weight / bias / in_features / out_features: diffusers' attention processors
+ # read `to_q.weight.dtype`, and H3's blocks read `context_embedder.weight`. Without this forward the wrapper is
+ # a drop-in only until the first such access, which fails at render time rather than at wrap time.
+ # nn.Module.__getattr__ runs first, so parameters, buffers and submodules registered on the wrapper itself still
+ # win.
try:
return super().__getattr__(name)
except AttributeError:
@@ -269,12 +268,11 @@ def wrap_small_m_linears(
parent = model.get_submodule(parent_name) if parent_name else model
module = getattr(parent, leaf)
except AttributeError:
- # A family token that matches nothing on this checkpoint variant is not an error:
- # the pruned and dense H3 trees differ, and callers pass a name list, not a promise.
+ # a family token matching nothing on this variant is not an error
continue
- # Skips a dense Linear (``F.linear`` has no row floor to clear, and there is no
- # granularity to prove) and, by the same gate, an already-wrapped one: ``PadToMinM`` is
- # not an ``nn.Linear``, so re-wrapping cannot nest the padding and double the row count.
+ # Skips a dense Linear (``F.linear`` has no row floor to clear, and there is no granularity to prove) and, by
+ # the same gate, an already-wrapped one: ``PadToMinM`` is not an ``nn.Linear``, so re-wrapping cannot nest the
+ # padding and double the row count.
if not is_quantized_linear(module):
continue
if require_per_row and activation_granularity_is_per_row(module) is not True:
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index 23577d2c774..888a0829c8f 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -53,7 +53,7 @@ def snapshot_backend_flags() -> Optional[dict]:
cuda.matmul on CPU/MPS) still captures the rest, instead of leaking a real mutated flag."""
try:
import torch
- except Exception: # noqa: BLE001 — no torch -> nothing to snapshot/restore
+ except Exception: # noqa: BLE001 - no torch -> nothing to snapshot/restore
return None
state: dict[str, bool] = {}
matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
@@ -80,14 +80,14 @@ def restore_backend_flags(state: Optional[dict]) -> None:
return
try:
import torch
- except Exception: # noqa: BLE001 — no torch -> nothing to restore
+ except Exception: # noqa: BLE001 - no torch -> nothing to restore
return
def _set(obj: Any, attr: str, key: str) -> None:
if obj is not None and key in state and hasattr(obj, attr):
try:
setattr(obj, attr, state[key])
- except Exception: # noqa: BLE001 — best-effort per-flag restore
+ except Exception: # noqa: BLE001 - best-effort per-flag restore
pass
matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
@@ -105,7 +105,7 @@ def _inductor_config() -> Any:
try:
import torch
return getattr(getattr(torch, "_inductor", None), "config", None)
- except Exception: # noqa: BLE001 — no inductor -> nothing to snapshot/set
+ except Exception: # noqa: BLE001 - no inductor -> nothing to snapshot/set
return None
@@ -174,7 +174,7 @@ def compile_eligible(target: Any, *, is_gguf: bool, family: Any) -> bool:
Only on CUDA (incl. ROCm), for a bf16 transformer, on a compile-friendly family, in a process
that can run inductor. ``is_gguf`` no longer disqualifies (GGUF compiles fine and ~2.3x
faster); the param is kept for compat."""
- del is_gguf # GGUF is compile-eligible now; param kept for call-site compat.
+ del is_gguf # GGUF is compile-eligible now; param kept for call-site compat
if not torch_compile_runtime_available():
return False
if not bool(getattr(target, "supports_default_torch_compile", False)):
@@ -219,7 +219,9 @@ def apply_speed_optims(
"compiled_vae_decode": False,
}
mode = normalize_speed_mode(speed_mode)
- # TF32 (max) and cudnn.benchmark (any non-off CUDA load) are process-global; the caller restores them so a later `off` load never inherits them.
+ # TF32 (max) and cudnn.benchmark are process-global
+ # TF32 (max) and cudnn.benchmark (any non-off CUDA load) are process-global; the caller restores them so a later
+ # `off` load never inherits them.
if mode == SPEED_OFF:
return applied
@@ -229,26 +231,25 @@ def apply_speed_optims(
# Lossless: a channels-last VAE speeds up its convs with no numeric change.
applied["channels_last"] = _vae_channels_last(pipe, logger)
- # Near-lossless: cuDNN autotunes the fixed-shape VAE convs (CUDA only). It may pick a different algorithm, so it is a "default"-tier win.
if on_cuda:
applied["cudnn_benchmark"] = _enable_cudnn_benchmark(logger)
- # Consumer-only: fp16 GEMMs accumulate in fp16 (~2x on GeForce-class parts). bf16 loads measured bit-identical with the flag
- # on, but fp16 pipelines drifted 2-5% same-seed, so fp16 compute gets it only under ``max``. Guarded by _FP16_ACCUM_DENY.
if on_cuda:
applied["fp16_accum"] = _enable_fp16_accumulation(
family, logger, dtype = getattr(target, "dtype", None), speed_mode = mode
)
- # --- the compile lever, per tier ---
- # default = LIGHT: GGUF compiles ONLY the dequant op chain (cheap, VRAM-free, resolution-invariant); dense falls back to the
- # regional block compile. max = FULL: regional max-autotune compile of the repeated block. eager = no compile.
+ # default = LIGHT: GGUF compiles ONLY the dequant op chain (cheap, VRAM-free, resolution-invariant)
+ # --- the compile lever, per tier --- default = LIGHT: GGUF compiles ONLY the dequant op chain (cheap, VRAM-free,
+ # resolution-invariant); dense falls back to the regional block compile. max = FULL: regional max-autotune compile
+ # of the repeated block. eager = no compile.
if mode == SPEED_DEFAULT:
# Asked directly: this arm never reaches compile_eligible(), unlike the dense arm below.
if is_gguf and on_cuda and family_allows_compile and torch_compile_runtime_available():
applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)
elif compile_eligible(target, is_gguf = is_gguf, family = family):
- # A U-Net (SDXL) fuses QKV BEFORE its whole-module compile: 36.3 vs 39.3 ms/step (LPIPS 0.033). DiTs were neutral, so they keep the fuse on max only.
+ # A U-Net (SDXL) fuses QKV BEFORE its whole-module compile: 36.3 vs 39.3 ms/step (LPIPS 0.033). DiTs were
+ # neutral, so they keep the fuse on max only.
if _denoiser_unet(pipe) is not None:
applied["fused_qkv"] = _fuse_qkv(pipe, logger)
applied["compiled"] = _compile_repeated_blocks(
@@ -267,12 +268,12 @@ def apply_speed_optims(
offload_active = offload_active,
)
- # A compiled U-Net family also compiles the VAE decode (4.98 to 4.25 s over 4 images, LPIPS unchanged). DiTs skip it. dynamic=True keeps it resolution-robust.
+ # A compiled U-Net family also compiles the VAE decode (4.98 to 4.25 s over 4 images, LPIPS unchanged). DiTs skip
+ # it. dynamic=True keeps it resolution-robust.
if applied["compiled"] and _denoiser_unet(pipe) is not None:
applied["compiled_vae_decode"] = _compile_vae_decode(pipe, logger)
if mode == SPEED_MAX:
- # Near-lossless: TF32 matmul (CUDA only) trades mantissa bits for speed.
if on_cuda:
applied["tf32"] = _enable_tf32(logger)
applied["fused_qkv"] = _fuse_qkv(pipe, logger)
@@ -288,13 +289,14 @@ def _vae_channels_last(pipe: Any, logger: Any) -> bool:
import torch
vae.to(memory_format = torch.channels_last)
return True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "channels_last", exc)
return False
-# U-Net denoisers ship no ``_repeated_blocks``, so the regional compile cannot reach them; these classes get a WHOLE-module STATIC compile.
-# SDXL (B200, 30 steps / 1024px): 26.9 vs 45.9 ms/step, 1.61x end-to-end at LPIPS 0.034. Static means a recompile per (height, width, batch).
+# U-Net denoisers ship no ``_repeated_blocks``, so the regional compile cannot reach them; these classes get a
+# WHOLE-module STATIC compile. SDXL (B200, 30 steps / 1024px): 26.9 vs 45.9 ms/step, 1.61x end-to-end at LPIPS 0.034.
+# Static means a recompile per (height, width, batch).
_UNET_WHOLE_COMPILE: frozenset[str] = frozenset({"UNet2DConditionModel"})
@@ -345,8 +347,9 @@ def _compile_repeated_blocks(
unet = _denoiser_unet(pipe) if not dits else None
if not dits and unet is None:
return False
- # default: dynamic=True, fast cold start, no recompile on resolution change. max: max-autotune-no-cudagraphs + dynamic=False, a few % more for a
- # longer compile and a recompile per resolution (CUDA-graph modes crash on the regional block). fullgraph drops to False under a step cache or offload.
+ # default: dynamic=True, fast cold start, no recompile on resolution change. max: max-autotune-no-cudagraphs +
+ # dynamic=False, a few % more for a longer compile and a recompile per resolution (CUDA-graph modes crash on the
+ # regional block). fullgraph drops to False under a step cache or offload.
kwargs: dict[str, Any] = {
"fullgraph": not (cache_active or offload_active),
"dynamic": not max_autotune,
@@ -356,31 +359,36 @@ def _compile_repeated_blocks(
try:
import torch
- # Heterogeneous-block DiTs (Z-Image needs ~11 graphs) exceed dynamo's default recompile_limit of 8, where a resident load
- # hard-errors under fullgraph, so raise it to 64. NOT force_parameter_static_shapes=False: no win and ~6x slower.
+ # Heterogeneous-block DiTs (Z-Image needs ~11 graphs) exceed dynamo's default recompile_limit of 8, where a
+ # resident load hard-errors under fullgraph, so raise it to 64. NOT force_parameter_static_shapes=False: no win
+ # and ~6x slower.
dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)
if dynamo_cfg is not None:
for _limit_attr in ("recompile_limit", "cache_size_limit"): # name varies by torch ver
if hasattr(dynamo_cfg, _limit_attr):
setattr(dynamo_cfg, _limit_attr, max(getattr(dynamo_cfg, _limit_attr) or 0, 64))
- # Match eager intermediate rounding in inductor's fused pointwise kernels: they keep chains in fp32 where eager materialises bf16 between ops, a per-forward delta a multi-step denoise amplifies.
- # Measured LPIPS vs eager: Qwen-Image 0.019 to 0.006, HunyuanVideo-1.5-720p 0.221 to 0.052, at ~zero cost. Process-global, so snapshot_backend_flags restores it on unload.
+ # Match eager intermediate rounding in inductor's fused pointwise kernels: they keep chains in fp32 where eager
+ # materialises bf16 between ops, a per-forward delta a multi-step denoise amplifies. Measured LPIPS vs eager:
+ # Qwen-Image 0.019 to 0.006, HunyuanVideo-1.5-720p 0.221 to 0.052, at ~zero cost. Process-global, so
+ # snapshot_backend_flags restores it on unload.
inductor_cfg = _inductor_config()
if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):
inductor_cfg.emulate_precision_casts = True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "compile_repeated_blocks", exc)
return False
if unet is not None:
- # Whole-module static compile for the U-Net classes above. fullgraph mirrors the regional decision; dynamic is ALWAYS False,
- # so each new (height, width, batch) pays its own compile. ``Module.compile`` keeps the module identity.
+ # dynamic is ALWAYS False here, so each new (height, width, batch) pays its own compile
+ # Whole-module static compile for the U-Net classes above. fullgraph mirrors the regional decision; dynamic is
+ # ALWAYS False, so each new (height, width, batch) pays its own compile. ``Module.compile`` keeps the module
+ # identity.
unet_kwargs: dict[str, Any] = {"fullgraph": kwargs["fullgraph"], "dynamic": False}
if max_autotune:
unet_kwargs["mode"] = "max-autotune-no-cudagraphs"
try:
unet.compile(**unet_kwargs)
return True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "unet whole-module compile", exc)
return False
# Compile every denoiser DiT (dual-DiT families run both); a per-DiT failure degrades only that one to eager.
@@ -389,15 +397,16 @@ def _compile_repeated_blocks(
try:
transformer.compile_repeated_blocks(**kwargs)
engaged = True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "compile_repeated_blocks", exc)
continue
- # A step cache engaged BEFORE this compile already wrapped each block forward in a disabled hook, so the compute branch would
- # run eager and forfeit the regional compile. Re-point the hooks' inner forward at compiled wrappers (no-op without them).
+ # A step cache engaged BEFORE this compile already wrapped each block forward in a disabled hook, so the compute
+ # branch would run eager and forfeit the regional compile. Re-point the hooks' inner forward at compiled
+ # wrappers (no-op without them).
try:
from .diffusion_cache import _compile_hooked_block_inners
_compile_hooked_block_inners(transformer, logger)
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "cache-hook inner compile", exc)
return engaged
@@ -413,7 +422,7 @@ def _compile_vae_decode(pipe: Any, logger: Any) -> bool:
import torch
vae.decode = torch.compile(decode, fullgraph = False, dynamic = True)
return True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "vae decode compile", exc)
return False
@@ -423,7 +432,7 @@ def _enable_cudnn_benchmark(logger: Any) -> bool:
import torch
torch.backends.cudnn.benchmark = True
return True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "cudnn_benchmark", exc)
return False
@@ -435,12 +444,13 @@ def _enable_tf32(logger: Any) -> bool:
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
return True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "tf32", exc)
return False
-# Families the overflow harness found to produce non-finite activations under fp16 accumulation. Empty by measurement: no overflow across all six families.
+# Families the overflow harness found to produce non-finite activations under fp16 accumulation. Empty by measurement:
+# no overflow across all six families.
_FP16_ACCUM_DENY: frozenset[str] = frozenset()
@@ -482,19 +492,20 @@ def _enable_fp16_accumulation(
return False
matmul.allow_fp16_accumulation = True
return True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fp16_accum", exc)
return False
def _fuse_qkv(pipe: Any, logger: Any) -> bool:
- # Prefer the pipe-level fuse (covers every component); else fuse each denoiser DiT so a dual-DiT family fuses BOTH experts.
+ # Prefer the pipe-level fuse (covers every component); else fuse each denoiser DiT so a dual-DiT family fuses BOTH
+ # experts.
fn = getattr(pipe, "fuse_qkv_projections", None)
if callable(fn):
try:
fn()
return True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fuse_qkv_projections", exc)
return False
engaged = False
@@ -504,7 +515,7 @@ def _fuse_qkv(pipe: Any, logger: Any) -> bool:
try:
tfn()
engaged = True
- except Exception as exc: # noqa: BLE001 — optimisation only
+ except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fuse_qkv_projections", exc)
return engaged
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index c7f5078bcbc..21b62c06178 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -41,30 +41,24 @@ TE_PREQUANT_FORMAT = "unsloth_prequant_text_encoder_state_dict_v1"
# The one scheme hosted in v1 (see module docstring).
TE_PREQUANT_SCHEMES = ("fp8",)
-# Components the pipeline-assembly injection covers (text_encoder_4 is family-assembled separately, see diffusion_hidream.py).
+# Components the pipeline-assembly injection covers (text_encoder_4 is family-assembled separately, see
+# diffusion_hidream.py).
TE_PREQUANT_COMPONENTS = ("text_encoder", "text_encoder_2", "text_encoder_3")
-# Fraction of a bf16 text encoder a PRE-CAST fp8 checkpoint occupies, for memory budgeting.
-#
-# fp8 storage is one byte per parameter against bf16's two, so the floor is 0.5, but the cast
-# deliberately keeps modules dense: nn.Embedding tables, the norms in
-# DEFAULT_SKIP_MODULES_PATTERN, the encoder's own _keep_in_fp32_modules (T5's ``wo``) and an
-# lm_head tied to the input embedding. Measured from Hub file metadata over every published
-# artifact (2026-08-07), as hosted checkpoint bytes over the bf16-EQUIVALENT dense bytes of the
-# same component (an fp32-stored encoder is halved first, since the pipeline loads it bf16):
-#
-# FLUX.2-dev text_encoder Mistral-24B 24,683,130,873 / 48,022,800,560 = 0.514
-# HiDream-I1-Full text_encoder_4 Llama-3.1-8B 8,555,963,320 / 16,060,556,376 = 0.533
-# Qwen-Image text_encoder Qwen2.5-VL-7B 8,839,210,073 / 16,584,414,544 = 0.533
-# LTX-2 text_encoder Gemma3-12B 13,205,302,695 / 24,374,720,836 = 0.542
-# Krea-2-Turbo text_encoder Qwen3-4B 4,831,262,424 / 8,875,715,136 = 0.544
-# Z-Image-Turbo text_encoder Qwen3-4B 4,411,751,967 / 8,044,982,000 = 0.548
-# Lumina-Image-2.0 text_encoder Gemma2-2B 3,204,501,909 / 5,228,699,608 = 0.613
-# FLUX.1-schnell text_encoder_2 T5-XXL 5,900,818,800 / 9,524,648,584 = 0.620
-#
-# The small encoders sit highest: their embedding tables are a large share of the parameters and
-# stay dense. 0.65 is the observed maximum rounded up, so this OVER-states every measured
-# encoder rather than under-stating any. It is a memory budget, and an under-estimate is the
+# Fraction of a bf16 text encoder a PRE-CAST fp8 checkpoint occupies, for memory budgeting. fp8 storage is one byte per
+# parameter against bf16's two, so the floor is 0.5, but the cast deliberately keeps modules dense: nn.Embedding tables,
+# the norms in DEFAULT_SKIP_MODULES_PATTERN, the encoder's own _keep_in_fp32_modules (T5's ``wo``) and an lm_head tied
+# to the input embedding. Measured from Hub file metadata over every published artifact (2026-08-07), as hosted
+# checkpoint bytes over the bf16-EQUIVALENT dense bytes of the same component (an fp32-stored encoder is halved first,
+# since the pipeline loads it bf16): FLUX.2-dev text_encoder Mistral-24B 24,683,130,873 / 48,022,800,560 =
+# 0.514 HiDream-I1-Full text_encoder_4 Llama-3.1-8B 8,555,963,320 / 16,060,556,376 = 0.533 Qwen-Image
+# text_encoder Qwen2.5-VL-7B 8,839,210,073 / 16,584,414,544 = 0.533 LTX-2 text_encoder Gemma3-12B
+# 13,205,302,695 / 24,374,720,836 = 0.542 Krea-2-Turbo text_encoder Qwen3-4B 4,831,262,424 /
+# 8,875,715,136 = 0.544 Z-Image-Turbo text_encoder Qwen3-4B 4,411,751,967 / 8,044,982,000 = 0.548
+# Lumina-Image-2.0 text_encoder Gemma2-2B 3,204,501,909 / 5,228,699,608 = 0.613 FLUX.1-schnell
+# text_encoder_2 T5-XXL 5,900,818,800 / 9,524,648,584 = 0.620 The small encoders sit highest: their
+# embedding tables are a large share of the parameters and stay dense. 0.65 is the observed maximum rounded up, so this
+# OVER-states every measured encoder rather than under-stating any. It is a memory budget, and an under-estimate is the
# expensive direction: it lets an oversized load through to the OS killer.
TE_PREQUANT_BUDGET_SCALE = 0.65
@@ -97,11 +91,15 @@ def te_prequant_budget_scale(
return TE_PREQUANT_BUDGET_SCALE if sources else 1.0
-# Bases whose text-encoder weights are VERIFIED byte-identical (every shard LFS sha256 compared on 2026-07-18), so one hosted artifact serves them all. The validator accepts a base_model_id from the same group; anything else keeps the strict refusal.
+# text-encoder weights VERIFIED byte-identical (every shard LFS sha256 compared 2026-07-18)
+# Bases whose text-encoder weights are VERIFIED byte-identical (every shard LFS sha256 compared on 2026-07-18), so one
+# hosted artifact serves them all. The validator accepts a base_model_id from the same group; anything else keeps the
+# strict refusal.
_TE_EQUIVALENT_BASES: tuple[frozenset[str], ...] = (
- # Qwen2.5-VL-7B text encoder: 4 shards, 16,584,414,544 bytes, identical sha256 set. Qwen-Image-2512
- # republishes the same four shards (re-verified 2026-08-25); refusing it here would pull 16.6 GB
- # of dense encoder the load never opens.
+ # Qwen2.5-VL-7B: 4 shards, identical sha256 set
+ # Qwen2.5-VL-7B text encoder: 4 shards, 16,584,414,544 bytes, identical sha256 set. Qwen-Image-2512 republishes the
+ # same four shards (re-verified 2026-08-25); refusing it here would pull 16.6 GB of dense encoder the load never
+ # opens.
frozenset(
{
"qwen/qwen-image",
@@ -109,15 +107,13 @@ _TE_EQUIVALENT_BASES: tuple[frozenset[str], ...] = (
"hunyuanvideo-community/hunyuanimage-2.1-diffusers",
}
),
- # Qwen3-4B text encoder: 1 shard, 8,875,715,136 bytes, identical sha256 across the Krea-2 pair
- # (compared 2026-08-25); only their transformers differ, as with Z-Image.
+ # Qwen3-4B: identical sha256 across the Krea-2 pair (compared 2026-08-25)
frozenset(
{
"krea/krea-2-turbo",
"krea/krea-2-raw",
}
),
- # T5-XXL (text_encoder_2): 2 shards, 9,524,648,584 bytes, identical sha256 across every FLUX.1 release; HiDream-I1 ships the same bytes as text_encoder_3 (cross-component mapping is not wired yet).
frozenset(
{
"black-forest-labs/flux.1-schnell",
@@ -126,7 +122,9 @@ _TE_EQUIVALENT_BASES: tuple[frozenset[str], ...] = (
"hidream-ai/hidream-i1-full",
}
),
- # Qwen3-4B text encoder: 3 shards, 8,044,982,000 bytes, identical sha256 set across the distilled Z-Image-Turbo and the undistilled base (only their transformers differ).
+ # Qwen3-4B: identical sha256 across the distilled Z-Image-Turbo and the undistilled base
+ # T5-XXL (text_encoder_2): 2 shards, 9,524,648,584 bytes, identical sha256 across every FLUX.1 release; HiDream-I1
+ # ships the same bytes as text_encoder_3 (cross-component mapping is not wired yet).
frozenset(
{
"tongyi-mai/z-image-turbo",
@@ -142,8 +140,8 @@ def te_base_equivalent(ckpt_base: str, base: str) -> bool:
equivalence group above."""
if _same_base_model(ckpt_base, base):
return True
- # The groups hold UPSTREAM ids: an unnormalised mirror id is a different string, so it would be
- # refused and the pre-cast encoder dropped for a dense pull.
+ # The groups hold UPSTREAM ids: an unnormalised mirror id is a different string, so it would be refused and the
+ # pre-cast encoder dropped for a dense pull.
from .diffusion_families import canonical_base
a, b = canonical_base(ckpt_base).lower(), canonical_base(base).lower()
@@ -181,7 +179,7 @@ def family_te_prequant_repo(fam: Any, scheme: str, component: str) -> Optional[s
for entry in getattr(fam, "te_prequant_repos", ()) or ():
try:
entry_scheme, entry_component, repo_id = entry
- except Exception: # noqa: BLE001 — a malformed entry must not break the load
+ except Exception: # noqa: BLE001 - a malformed entry must not break the load
continue
if entry_scheme == scheme and entry_component == component:
return repo_id
@@ -244,7 +242,8 @@ def te_prequant_sources(
if mode != TE_QUANT_FP8:
return {}
family = getattr(fam, "name", None)
- # The per-family TE deny table ships on the video branch precision module (the image branch has no denials), so resolve it lazily and one module serves both.
+ # The per-family TE deny table ships on the video branch precision module (the image branch has no denials), so
+ # resolve it lazily and one module serves both.
denied = getattr(precision, "_te_family_denied", None)
if callable(denied) and denied(family, mode):
return {}
@@ -294,7 +293,11 @@ def te_prequant_sources_for_base(
return compatible
-# Weight files a dense encoder folder holds. Everything else (config.json, the shard index, tokenizer JSON) is kept when the pre-cast checkpoint replaces the weights: the pre-cast loader still meta-inits from the base repo component config.
+# everything else (config.json, the shard index, tokenizer JSON) is kept when the pre-cast checkpoint replaces the
+# weights
+# Weight files a dense encoder folder holds. Everything else (config.json, the shard index, tokenizer JSON) is kept when
+# the pre-cast checkpoint replaces the weights: the pre-cast loader still meta-inits from the base repo component
+# config.
_TE_WEIGHT_SUFFIXES = (".safetensors", ".bin", ".pth", ".pt", ".msgpack", ".h5")
@@ -359,7 +362,8 @@ def load_prequant_text_encoder(
import torch
- # The layerwise-fp8 state dict is plain tensors, so weights_only=True suffices and no pickle code runs even for a local path. A future torchao-subclass scheme needs a format bump AND the DiT module's allowlist.
+ # The layerwise-fp8 state dict is plain tensors, so weights_only=True suffices and no pickle code runs even for
+ # a local path. A future torchao-subclass scheme needs a format bump AND the DiT module's allowlist.
ckpt = torch.load(path, weights_only = True, map_location = "cpu")
if not _validate_checkpoint(ckpt, scheme, component, base, logger):
return None
@@ -385,7 +389,9 @@ def load_prequant_text_encoder(
if subfolder:
config_kwargs["subfolder"] = subfolder
config = transformers.AutoConfig.from_pretrained(base, **config_kwargs)
- # Krea-2 ships transformers-5.x configs whose rope lives under rope_parameters; the runtime component loader remaps it for a 4.x runtime, and the meta-init here must match or the rebuilt encoder forwards with a broken rope.
+ # Krea-2 ships transformers-5.x configs whose rope lives under rope_parameters; the runtime component loader
+ # remaps it for a 4.x runtime, and the meta-init here must match or the rebuilt encoder forwards with a broken
+ # rope.
from .diffusion_krea2 import remap_rope_parameters
remap_rope_parameters(getattr(config, "text_config", config))
@@ -395,19 +401,23 @@ def load_prequant_text_encoder(
with init_empty_weights():
encoder = encoder_cls(config)
- # assign=True swaps in the loaded tensors rather than copying into meta; strict=True since the saved dict is the full state dict of the same class.
encoder.load_state_dict(state_dict, strict = True, assign = True)
if _has_meta_tensors(encoder):
- # Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so they hold real values, then re-assign the cast weights.
+ # Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so
+ # they hold real values, then re-assign the cast weights.
encoder = encoder_cls(config)
encoder.load_state_dict(state_dict, strict = True, assign = True)
- # assign=True swaps in SEPARATE tensors for tied weights (the saved dict carries a copy per key), untying e.g. Qwen3's lm_head from embed_tokens and defeating _cast_fp8's tied-projection skip. Re-tie to the builder-identical structure; a no-op when untied.
+ # assign=True swaps in SEPARATE tensors for tied weights (the saved dict carries a copy per key), untying e.g.
+ # Qwen3's lm_head from embed_tokens and defeating _cast_fp8's tied-projection skip. Re-tie to the
+ # builder-identical structure; a no-op when untied.
tie = getattr(encoder, "tie_weights", None)
if callable(tie):
tie()
encoder.eval()
- # Install the SAME upcast hooks the runtime cast applies. The weight cast inside is idempotent, so this only arms the per-layer upcast; without it the fp8 storage weights would meet bf16 activations at the first forward.
+ # Install the SAME upcast hooks the runtime cast applies. The weight cast inside is idempotent, so this only
+ # arms the per-layer upcast; without it the fp8 storage weights would meet bf16 activations at the first
+ # forward.
from .diffusion_precision import _cast_fp8
class _Target:
@@ -424,7 +434,7 @@ def load_prequant_text_encoder(
source.kind,
)
return encoder
- except Exception as exc: # noqa: BLE001 — fall back to the dense download + cast
+ except Exception as exc: # noqa: BLE001 - fall back to the dense download + cast
_warn(logger, f"{scheme}:{component}:{source.kind}", exc)
return None
@@ -475,7 +485,7 @@ def te_prequant_pipe_kwargs(
if encoder is not None:
injected[component] = encoder
return injected
- except Exception as exc: # noqa: BLE001 — injection is an optimisation, never a blocker
+ except Exception as exc: # noqa: BLE001 - injection is an optimisation, never a blocker
_warn(logger, "pipe_kwargs", exc)
return {}
@@ -529,7 +539,8 @@ def _validate_checkpoint(ckpt: Any, scheme: str, component: str, base: str, logg
return False
ckpt_base = meta.get("base_model_id")
if base:
- # Keys matching a different base can load strict=True and encode prompts with the wrong weights. The builder always records base_model_id, so refuse one that omits it.
+ # Keys matching a different base can load strict=True and encode prompts with the wrong weights. The builder
+ # always records base_model_id, so refuse one that omits it.
if not ckpt_base:
_warn(
logger,
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 52fd54e91eb..5ab0fa753ad 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -39,32 +39,38 @@ TQ_AUTO = "auto"
TQ_SCHEMES = (TQ_INT8, TQ_FP8, TQ_NVFP4, TQ_MXFP8)
TQ_MODES = (TQ_AUTO,) + TQ_SCHEMES
-# Schemes whose torchao path asserts a bf16 weight, so their filter skips non-bf16 Linears rather than aborting the pass. On torchao 0.17 / B200: fp8 per-row and mxfp8 assert bf16; nvfp4 and int8 handle fp32/fp16.
+# Schemes whose torchao path asserts a bf16 weight, so their filter skips non-bf16 Linears rather than aborting the
+# pass. On torchao 0.17 / B200: fp8 per-row and mxfp8 assert bf16; nvfp4 and int8 handle fp32/fp16.
_REQUIRE_BF16_SCHEMES = (TQ_FP8, TQ_MXFP8)
-# fp8 granularity the runtime uses: per-ROW is REQUIRED for correctness on outlier-heavy DiTs. Stamped into prequant metadata, so a stale per-TENSOR checkpoint is rejected and rebuilt.
+# per-ROW is REQUIRED for correctness on outlier-heavy DiTs
+# fp8 granularity the runtime uses: per-ROW is REQUIRED for correctness on outlier-heavy DiTs. Stamped into prequant
+# metadata, so a stale per-TENSOR checkpoint is rejected and rebuilt.
FP8_GRANULARITY = "per_row"
# Skip linears below this feature size: a small FLOP share, so leaving them bf16 costs ~nothing.
DEFAULT_MIN_LINEAR_FEATURES = 512
-# int8-ONLY name exclusions: torch._int_mm needs activation rows M above 16, and a DiT's AdaLN modulation and timestep / guidance / pooled-text
-# embedders run once from a [batch, dim] vector (M = 1), so they crash despite large feature dims. Negligible FLOPs; scaled_mm has no M limit.
+# int8-ONLY name exclusions: torch._int_mm needs activation rows M above 16, and a DiT's AdaLN modulation and timestep /
+# guidance / pooled-text embedders run once from a [batch, dim] vector (M = 1), so they crash despite large feature
+# dims. Negligible FLOPs; scaled_mm has no M limit.
_INT8_EXCLUDE_NAME_TOKENS = (
"norm", # AdaLN modulation .linear
"_mod", # Qwen img_mod / txt_mod
"modulation", # Flux.2 double/single_stream_modulation
"timestep_embed",
"guidance_embed",
- "time_text_embed", # Flux/Qwen (pooled-text + timestep); NOT context_embedder
+ "time_text_embed", # pooled-text + timestep; NOT context_embedder
"pooled",
# Krea 2 time_embed.linear_2 (M = batch); time_mod_proj is caught by "_mod", and the rest fall under min_features.
"time_embed",
)
-# int8 PER-FAMILY exclusions, on top of _INT8_EXCLUDE_NAME_TOKENS. Qwen-Image MMDiT runs every TEXT-stream Linear at M = actual prompt tokens
-# (unpadded, unlike FLUX's 512-token T5), so a short prompt falls under _int_mm's M floor of 16 and the denoise crashes. bf16 there costs ~nothing.
+# Qwen-Image MMDiT runs every TEXT-stream Linear at M = actual prompt tokens (unpadded, unlike FLUX's 512-token T5)
+# int8 PER-FAMILY exclusions, on top of _INT8_EXCLUDE_NAME_TOKENS. Qwen-Image MMDiT runs every TEXT-stream Linear at M =
+# actual prompt tokens (unpadded, unlike FLUX's 512-token T5), so a short prompt falls under _int_mm's M floor of 16 and
+# the denoise crashes. bf16 there costs ~nothing.
_QWENIMAGE_INT8_EXCLUDES = (
"txt_in",
"add_q_proj",
@@ -73,17 +79,15 @@ _QWENIMAGE_INT8_EXCLUDES = (
"to_add_out",
"txt_mlp",
)
-# HunyuanVideo-1.5's attention trim (this PR) shrinks the text / image streams from their padded
-# lengths to the VALID token counts, so every text-stream Linear runs at a tiny M the int8 dynamic
-# path cannot handle. Both failures measured on B200:
-# - M = 0 (t2v byt5 / image streams trim to zero tokens): torchao returns the input UNPROJECTED
-# (a quantized 1472 -> 2048 Linear maps [1, 0, 1472] to [1, 0, 1472]), so the 2048-wide
-# cond-type add crashes -> context_embedder_2 / image_embedder;
-# - M <= 16 (a short prompt, or the empty negative prompt's ~6 tokens): torch._int_mm requires
-# M > 16 and raises -> the TokenRefiner and every block's context-stream projections.
-# These run at M = text tokens (tens) against the video stream's M ~ 32k+, so bf16 here costs
-# nothing measurable and the video-stream linears keep full int8 coverage. "context_embedder"
-# also matches "context_embedder_2" (substring check).
+# HunyuanVideo-1.5's attention trim shrinks the text / image streams to their VALID token counts
+# HunyuanVideo-1.5's attention trim (this PR) shrinks the text / image streams from their padded lengths to the VALID
+# token counts, so every text-stream Linear runs at a tiny M the int8 dynamic path cannot handle. Both failures measured
+# on B200: - M = 0 (t2v byt5 / image streams trim to zero tokens): torchao returns the input UNPROJECTED (a quantized
+# 1472 -> 2048 Linear maps [1, 0, 1472] to [1, 0, 1472]), so the 2048-wide cond-type add crashes -> context_embedder_2 /
+# image_embedder; - M <= 16 (a short prompt, or the empty negative prompt's ~6 tokens): torch._int_mm requires M > 16
+# and raises -> the TokenRefiner and every block's context-stream projections. These run at M = text tokens (tens)
+# against the video stream's M ~ 32k+, so bf16 here costs nothing measurable and the video-stream linears keep full int8
+# coverage. "context_embedder" also matches "context_embedder_2" (substring check).
_HUNYUAN15_INT8_EXCLUDES = (
"context_embedder",
"image_embedder",
@@ -93,35 +97,28 @@ _HUNYUAN15_INT8_EXCLUDES = (
"to_add_out",
"ff_context",
)
-# MiniMax-H3 is the same small-M story with one extra trap. Its adaLN projection is named
-# ``adaln_proj``, which no token in _INT8_EXCLUDE_NAME_TOKENS matches: "norm" is the closest and it
-# does not appear in the name. On the DENSE checkpoint that projection is Linear(2688 -> 96768), so
-# it clears min_features = 512 and gets quantized, then runs at M = 1 and raises
-# ``self.size(0) needs to be greater than 16, but got 1`` at the first denoise. Measured: the
-# offline builder happily bakes it, and torch.compile then dies on that module.
-#
-# The pruned-modulation form hides the bug rather than fixing it: there adaln_proj is
-# Linear(8 -> 96768), which falls under min_features and is skipped for free (verified on the
-# fl2va_pruned build: the filter rejects all 51 of them for min_features, in_features = 8). So the
-# exclusion is what makes the DENSE path correct, and is a no-op on the pruned one.
-#
-# H3's OTHER small-M linears -- context_embedder and the two token_refiner blocks, 13 in total --
-# used to be excluded here for the same reason. They are not any more: they are padded instead,
-# see _INT8_FAMILY_PAD_NAME_TOKENS below.
+# MiniMax-H3 is the same small-M story with one extra trap. Its adaLN projection is named ``adaln_proj``, which no token
+# in _INT8_EXCLUDE_NAME_TOKENS matches: "norm" is the closest and it does not appear in the name. On the DENSE
+# checkpoint that projection is Linear(2688 -> 96768), so it clears min_features = 512 and gets quantized, then runs at
+# M = 1 and raises ``self.size(0) needs to be greater than 16, but got 1`` at the first denoise. Measured: the offline
+# builder happily bakes it, and torch.compile then dies on that module. The pruned-modulation form hides the bug rather
+# than fixing it: there adaln_proj is Linear(8 -> 96768), which falls under min_features and is skipped for free
+# (verified on the fl2va_pruned build: the filter rejects all 51 of them for min_features, in_features = 8). So the
+# exclusion is what makes the DENSE path correct, and is a no-op on the pruned one. H3's OTHER small-M linears --
+# context_embedder and the two token_refiner blocks, 13 in total -- used to be excluded here for the same reason. They
+# are not any more: they are padded instead, see _INT8_FAMILY_PAD_NAME_TOKENS below.
_MINIMAX_H3_INT8_EXCLUDES = ("adaln_proj",)
-# LTX-2 is audiovisual: every block carries an audio-side stream (audio_attn1 / audio_attn2 /
-# audio_ff, fed by audio_proj_in) plus the a2v / v2a cross attentions. A video-only run feeds a
-# MINIMAL audio stream -- one token for a still -- so those linears run at M = 1, under the same
-# _int_mm floor of 16 as the cases above. The audio stream is deliberately inert on a video-only
-# run (isolated modalities, out of the loss, out of the LoRA targets), so leaving it in bf16 costs
-# nothing while the video-stream linears keep full int8 coverage. One token covers every audio-side
-# name, including video_to_audio_attn, whose keys/values are that same one-token stream.
-#
-# The audio token alone is not enough. LTX-2 also carries LTX2AdaLayerNormSingle projections whose
-# names say nothing about audio -- av_cross_attn_video_scale_shift / av_cross_attn_video_a2v_gate
-# (the cross-modality modulation, computed unconditionally in forward, isolate_modalities or not)
-# and prompt_adaln / audio_prompt_adaln. Each is a Linear over a BATCH-sized input, so M = batch =
-# 1 for a default training run: the same _int_mm floor, reached whatever the audio stream does.
+# LTX-2 is audiovisual: a video-only run feeds a MINIMAL audio stream (one token for a still) LTX-2 is audiovisual:
+# every block carries an audio-side stream (audio_attn1 / audio_attn2 / audio_ff, fed by audio_proj_in) plus the a2v /
+# v2a cross attentions. A video-only run feeds a MINIMAL audio stream -- one token for a still -- so those linears run
+# at M = 1, under the same _int_mm floor of 16 as the cases above. The audio stream is inert on a video-only run
+# (isolated modalities, out of the loss, out of the LoRA targets), so leaving it in bf16 costs nothing while the
+# video-stream linears keep full int8 coverage. One token covers every audio-side name, including video_to_audio_attn,
+# whose keys/values are that same one-token stream. The audio token alone is not enough. LTX-2 also carries
+# LTX2AdaLayerNormSingle projections whose names say nothing about audio -- av_cross_attn_video_scale_shift /
+# av_cross_attn_video_a2v_gate (the cross-modality modulation, computed unconditionally in forward, isolate_modalities
+# or not) and prompt_adaln / audio_prompt_adaln. Each is a Linear over a BATCH-sized input, so M = batch = 1 for a
+# default training run: the same _int_mm floor, reached whatever the audio stream does.
_LTX2_INT8_EXCLUDES = ("audio", "av_cross_attn", "adaln")
_INT8_FAMILY_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = {
"qwen-image": _QWENIMAGE_INT8_EXCLUDES,
@@ -134,22 +131,17 @@ _INT8_FAMILY_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = {
}
-# int8 PER-FAMILY row PADDING, the alternative to excluding a small-M Linear. ``_int_mm``'s floor is
-# a GEMM shape constraint, not a numerical one: padding the activation up to 32 rows, running the
-# matmul and slicing the result back returns the caller's rows BITWISE unchanged (torchao's int8
-# dynamic path scales activations per row, so the pad rows cannot reach them -- see
-# diffusion_quant_pad for why that is asserted rather than assumed). So a Linear that would have
-# been left dense can be quantized after all, and its weights halve.
-#
-# On MiniMax-H3 that is context_embedder plus the two token_refiner blocks: 13 Linears, 798 M
-# parameters, 0.80 GB of bf16 weights, 3.9% of the whole int8 checkpoint. Measured on B200,
-# 640x384 x 124 frames (see h3_quant/README_int8.md for the paired numbers).
-#
-# Scoped to minimax-h3 deliberately. qwen-image, qwen-image-edit and hunyuanvideo-1.5 have the same
-# small-M shape and could adopt this, but each has a PUBLISHED int8 prequant checkpoint whose
-# metadata bakes the current exclusion set, and _validate_checkpoint compares that set against
-# exclude_tokens_for_scheme: changing it invalidates those artifacts, so they move only together
-# with a rebuild and republish of each.
+# Int8 PER-FAMILY row PADDING, the alternative to excluding a small-M Linear. ``_int_mm``'s floor is a GEMM shape
+# constraint, not a numerical one: padding the activation up to 32 rows, running the matmul and slicing the result back
+# returns the caller's rows BITWISE unchanged (torchao's int8 dynamic path scales activations per row, so the pad rows
+# cannot reach them -- see diffusion_quant_pad for why that is asserted rather than assumed). So a Linear that would
+# have been left dense can be quantized after all, and its weights halve. On MiniMax-H3 that is context_embedder plus
+# the two token_refiner blocks: 13 Linears, 798 M parameters, 0.80 GB of bf16 weights, 3.9% of the whole int8
+# checkpoint. Measured on B200, 640x384 x 124 frames (see h3_quant/README_int8.md for the paired numbers). Scoped to
+# minimax-h3. qwen-image, qwen-image-edit and hunyuanvideo-1.5 have the same small-M shape and could adopt this, but
+# each has a PUBLISHED int8 prequant checkpoint whose metadata bakes the current exclusion set, and _validate_checkpoint
+# compares that set against exclude_tokens_for_scheme: changing it invalidates those artifacts, so they move only
+# together with a rebuild and republish of each.
_MINIMAX_H3_INT8_PAD_TOKENS = ("context_embedder", "token_refiner")
_INT8_FAMILY_PAD_NAME_TOKENS: dict[str, tuple[str, ...]] = {
"minimax-h3": _MINIMAX_H3_INT8_PAD_TOKENS,
@@ -213,12 +205,13 @@ def exclude_tokens_for_scheme(scheme: str, family: Optional[str] = None) -> tupl
return ()
-# Per-arch preference for ``auto``, best first. On Blackwell fp8 leads: on B200 plain fp8 dynamic is faster AND more accurate at DiT shapes,
-# while mxfp8 block scaling only adds overhead. nvfp4's FP4 GEMM is real with torch>=2.11 but wins only on very large GEMMs (0.81x on Z-Image
-# 1024px, LPIPS 0.166 vs fp8 0.044), so it is kept OUT of the ladder below and stays an explicit opt-in (transformer_quant="nvfp4"): auto must
-# never silently drop to a scheme that is both slower and less accurate. Restore the commented Blackwell tier to re-enable it once the FP4
-# tensor-core GEMM wins at the DiT's real shapes (hidden ~3072, MLP ~12288, M ~4096) and its accuracy is validated by the prequant gate.
-# Consumer / workstation GPUs move int8 first: they halve fp8/fp16 FP32-accumulate.
+# Per-arch preference for ``auto``, best first. On Blackwell fp8 leads: on B200 plain fp8 dynamic is faster AND more
+# accurate at DiT shapes, while mxfp8 block scaling only adds overhead. nvfp4's FP4 GEMM is real with torch>=2.11 but
+# wins only on very large GEMMs (0.81x on Z-Image 1024px, LPIPS 0.166 vs fp8 0.044), so it is kept OUT of the ladder
+# below and stays an explicit opt-in (transformer_quant="nvfp4"): auto must never silently drop to a scheme that is both
+# slower and less accurate. Restore the commented Blackwell tier to re-enable it once the FP4 tensor-core GEMM wins at
+# the DiT's real shapes (hidden ~3072, MLP ~12288, M ~4096) and its accuracy is validated by the prequant gate. Consumer
+# / workstation GPUs move int8 first: they halve fp8/fp16 FP32-accumulate.
_AUTO_LADDER: tuple[tuple[tuple[int, int], tuple[str, ...]], ...] = (
((10, 0), (TQ_FP8, TQ_MXFP8, TQ_INT8)), # Blackwell sm_100+ (nvfp4 is explicit opt-in only)
# ((10, 0), (TQ_FP8, TQ_NVFP4, TQ_MXFP8, TQ_INT8)), # restore to re-enable nvfp4 under auto
@@ -226,34 +219,33 @@ _AUTO_LADDER: tuple[tuple[tuple[int, int], tuple[str, ...]], ...] = (
((8, 0), (TQ_INT8,)), # Ampere sm_80 / sm_86
)
-# Families whose activation ranges break specific schemes at the MODEL level (the smoke probe only proves the GEMM runs). Measured with the
-# 28-pair prequant accuracy gate on B200: qwen-image + mxfp8 does semantic damage at 1024px, + nvfp4 is unusable (LPIPS 0.51). Neither has a
-# known fix, so both stay denied and auto falls through to int8 (excellent on Qwen). The deny also applies to an EXPLICIT request.
-#
-# fp8 was denied here too, for black frames. That cause is gone: it was torchao's fp8 scale chooser having no eps clamp, so qwen's all-zero
-# text rows gave scale 0 and NaN, and ``_make_quant_config`` now floors it with ``activation_value_lb``. Re-measured on B200 with the floor,
-# and BOTH paths the deny governs clear the 0.10 LPIPS bar, which is why it is safe to drop rather than just the checkpoint half:
-# prequant checkpoint, full 28-pair gate -> 28/28 PASS (SSIM 0.87-0.99, LPIPS 0.027-0.228, CLIP delta <= 0.019)
-# on-the-fly quantize_, gate's simple_object prompt at its own seeds -> LPIPS 0.048 / 0.033 / 0.027 / 0.063
-# against pre-floor plain fp8 at LPIPS 0.712 with SSIM 0.016 and mean luma 0. Keeping the deny cost real speed: it pinned qwen to int8 at
-# 1.10x bf16 when compiled fp8 measures 1.21x.
+# Families whose activation ranges break specific schemes at the MODEL level (the smoke probe only proves the GEMM
+# runs). Measured with the 28-pair prequant accuracy gate on B200: qwen-image + mxfp8 does semantic damage at 1024px, +
+# nvfp4 is unusable (LPIPS 0.51). Neither has a known fix, so both stay denied and auto falls through to int8 (excellent
+# on Qwen). The deny also applies to an EXPLICIT request. fp8 was denied here too, for black frames. That cause is gone:
+# it was torchao's fp8 scale chooser having no eps clamp, so qwen's all-zero text rows gave scale 0 and NaN, and
+# ``_make_quant_config`` now floors it with ``activation_value_lb``. Re-measured on B200 with the floor, and BOTH paths
+# the deny governs clear the 0.10 LPIPS bar, so it is safe to drop rather than just the checkpoint half: prequant
+# checkpoint, full 28-pair gate -> 28/28 PASS (SSIM 0.87-0.99, LPIPS 0.027-0.228, CLIP delta <= 0.019) on-the-fly
+# quantize_, gate's simple_object prompt at its own seeds -> LPIPS 0.048 / 0.033 / 0.027 / 0.063 against pre-floor plain
+# fp8 at LPIPS 0.712 with SSIM 0.016 and mean luma 0. Keeping the deny cost real speed: it pinned qwen to int8 at 1.10x
+# bf16 when compiled fp8 measures 1.21x.
_FAMILY_SCHEME_DENY: dict[str, frozenset[str]] = {
"qwen-image": frozenset({TQ_MXFP8, TQ_NVFP4}),
"qwen-image-edit": frozenset({TQ_MXFP8, TQ_NVFP4}), # same DiT
}
-# Schemes denied for TRAINING on top of the inference table. Training holds a stricter bar because
-# the evidence above is rendering evidence: it says a frozen fp8 forward reconstructs the bf16 image,
-# not that a LoRA converges when its frozen linears are fp8. Nobody has run that, so qwen fp8 stays
-# out of the Train UI until someone does. Delete the entry once a training run is measured.
+# Schemes denied for TRAINING on top of the inference table. Training holds a stricter bar because the evidence above is
+# rendering evidence: it says a frozen fp8 forward reconstructs the bf16 image, not that a LoRA converges when its
+# frozen linears are fp8. Nobody has run that, so qwen fp8 stays out of the Train UI until someone does. Delete the
+# entry once a training run is measured.
_FAMILY_TRAIN_SCHEME_DENY: dict[str, frozenset[str]] = {
"qwen-image": frozenset({TQ_FP8}),
"qwen-image-edit": frozenset({TQ_FP8}),
- # MiniMax-H3's packed sequence runs three modalities through one set of linears, so its
- # activation range is not the per-family range the fp8 filter was measured against. The
- # trainer refuses both outright; declaring it here is what keeps /diffusion/info from
- # advertising a start that is guaranteed to 400.
+ # MiniMax-H3's packed sequence runs three modalities through one set of linears, so its activation range is not the
+ # per-family range the fp8 filter was measured against. The trainer refuses both outright; declaring it here is what
+ # keeps /diffusion/info from advertising a start that is guaranteed to 400.
"minimax-h3": frozenset({TQ_FP8, TQ_MXFP8}),
}
@@ -317,18 +309,17 @@ def torchao_unavailable_reason() -> Optional[str]:
return _TORCHAO_UNAVAILABLE[0]
reason: Optional[str] = None
if is_stubbed("torchao"):
- # The Windows-ROCm stub imports fine and quantises nothing, so the import test below passes.
+ # the Windows-ROCm stub imports fine and quantises nothing, so the import test below passes
reason = "this platform ships a torchao stub whose quantize_ is a no-op"
else:
try:
from torchao.quantization import quantize_ # noqa: F401
- except Exception as exc: # noqa: BLE001 — any import failure means the same thing here
+ except Exception as exc: # noqa: BLE001 - any import failure means the same thing here
import logging
- # The full text, paths included, goes to the server log; the client gets it stripped.
- # This string is interpolated into the precision-refusal RuntimeError, which both load
- # routes return verbatim as the 409 detail, and an ImportError routinely names the
- # absolute file of the module that raised it.
+ # The full text, paths included, goes to the server log; the client gets it stripped. This string is
+ # interpolated into the precision-refusal RuntimeError, which both load routes return verbatim as the 409
+ # detail, and an ImportError routinely names the absolute file of the module that raised it.
logging.getLogger(__name__).warning(
"torchao is unusable: %s: %s", type(exc).__name__, exc
)
@@ -337,9 +328,8 @@ def torchao_unavailable_reason() -> Optional[str]:
return reason
-# An absolute POSIX or Windows path, up to the first whitespace / quote / bracket. Deliberately
-# requires a directory separator so dotted module names ("torch.nn.functional") survive intact:
-# those are the actionable half of the message.
+# An absolute POSIX or Windows path, up to the first whitespace / quote / bracket. Deliberately requires a directory
+# separator so dotted module names ("torch.nn.functional") survive intact: those are the actionable half of the message.
_ABS_PATH_RE = _re.compile(r"(?:[A-Za-z]:)?[\\/](?:[^\s'\"()<>|]*[\\/])+[^\s'\"()<>|]*")
@@ -366,32 +356,33 @@ def _smoke_cache_device_key(device: str) -> str:
return device
-# Data-center GPU tokens (un-nerfed FP32 accumulate). Matched as whole tokens of get_device_name() so "A4000" is not mistaken for "A40"; anything else is consumer-class.
+# Data-center GPU tokens (un-nerfed FP32 accumulate). Matched as whole tokens of get_device_name() so "A4000" is not
+# mistaken for "A40"; anything else is consumer-class.
_DATACENTER_GPU_TOKENS = frozenset(
{
"B200",
"B100",
- "B300", # Blackwell Ultra data center
+ "B300",
"GB200",
"GB300",
- "GB10", # Blackwell data center
+ "GB10",
"H200",
"H100",
"H800",
"H20",
- "GH200", # Grace-Hopper superchip (data center)
+ "GH200", # Grace-Hopper superchip
"A100",
"A800",
"A30",
"A40",
"A16",
"A10",
- "A2", # Ampere data center
+ "A2",
"L40",
"L40S",
"L4",
"L20",
- "L2", # Ada data center
+ "L2",
"V100",
"P100",
"P40",
@@ -400,7 +391,7 @@ _DATACENTER_GPU_TOKENS = frozenset(
)
-# Professional parts the backend treats as datacenter-class. Matched as phrases since the marker spans tokens.
+# professional parts the backend treats as datacenter-class, matched as phrases since the marker spans tokens
_PROFESSIONAL_GPU_MARKERS = ("RTX PRO 6000", "RTX 6000 ADA")
@@ -415,7 +406,7 @@ def _is_consumer_gpu(device: Any = None) -> bool:
import torch
name = torch.cuda.get_device_name(device).upper()
- except Exception: # noqa: BLE001 — no torch / no device -> assume consumer
+ except Exception: # noqa: BLE001 - no torch / no device -> assume consumer
return True
if "GEFORCE" in name or "TITAN" in name:
return True
@@ -446,13 +437,12 @@ def dense_transformer_supported(target: Any) -> bool:
dtype (the only config any torchao dynamic scheme accelerates). Cheap loader pre-check."""
if getattr(target, "device", None) != "cuda":
return False
- # The Windows-ROCm torchao stub's quantize_ is a no-op, so the smoke probe passes on a
- # still-dense Linear and the transformer gets MARKED quantised without being quantised,
- # giving the wrong VRAM budget and compile policy.
+ # The Windows-ROCm torchao stub's quantize_ is a no-op, so the smoke probe passes on a still-dense Linear and the
+ # transformer gets MARKED quantised without being quantised, giving the wrong VRAM budget and compile policy.
if is_stubbed("torchao"):
return False
- # ROCm reports gfx versions through CUDA capability APIs, so _AUTO_LADDER's NVIDIA SM floors
- # misclassify AMD GPUs. Keep ROCm on the GGUF path.
+ # ROCm reports gfx versions through CUDA capability APIs, so _AUTO_LADDER's NVIDIA SM floors misclassify AMD GPUs.
+ # Keep ROCm on the GGUF path.
if torch_is_rocm():
return False
try:
@@ -574,19 +564,18 @@ def _scheme_supported(
return False
except Exception:
return False
- # Resolve the whole table in one throwaway child, falling back in-process. Nothing above
- # allocates: the context arrives on the first ALLOCATION, not on is_available() or
- # get_device_capability() (0 MiB after both, 614 MiB after the first tensor). Worth a child
- # because /images/download-plan calls assert_precision_available while the user is only
- # STAGING, so a plan alone cost the backend ~700 MiB it can never give back.
- # _smoke_probe's card-qualified key, and the child is asked about that same card: a spawn
- # starts on ordinal 0 whatever this thread is pinned to, so a bare "cuda" would file card 0's
- # verdict under the card the load runs on (and a bare key hid child verdicts entirely).
+ # Resolve the whole table in one throwaway child, falling back in-process. Nothing above allocates: the context
+ # arrives on the first ALLOCATION, not on is_available() or get_device_capability() (0 MiB after both, 614 MiB after
+ # the first tensor). Worth a child because /images/download-plan calls assert_precision_available while the user is
+ # only STAGING, so a plan alone cost the backend ~700 MiB it can never give back. _smoke_probe's card-qualified key,
+ # and the child is asked about that same card: a spawn starts on ordinal 0 whatever this thread is pinned to, so a
+ # bare "cuda" would file card 0's verdict under the card the load runs on (and a bare key hid child verdicts
+ # entirely).
card = _smoke_cache_device_key(device)
if (scheme, card) not in _SMOKE_CACHE:
with _CHILD_PROBE_LOCK:
- # Re-checked under the lock: the route answers plans concurrently, and a burst of
- # them must not each spawn a child that imports torch.
+ # Re-checked under the lock: the route answers plans concurrently, and a burst of them must not each spawn a
+ # child that imports torch.
if (scheme, card) not in _SMOKE_CACHE:
table = _child_probe_table(card)
if table is not None:
@@ -596,27 +585,24 @@ def _scheme_supported(
_SMOKE_CACHE[(name, card)] = child_verdict
if scheme in table:
if table[scheme] is None:
- # Repeating an OOM probe in the same memory state proves nothing.
return unproven_ok
# The child ran the same probe. Missing entries still fall through below.
return table[scheme]
return _smoke_probe(scheme, device, unproven_ok = unproven_ok)
-# Long enough for a cold interpreter to import torch and torchao and run every probe (measured
-# 3.9 s on a B200 host, so this is ~45x headroom for a cold page cache); short enough that a
-# child wedged in the driver does not hold the route thread forever. Overrunning it is not a
-# verdict either -- the caller falls back in-process.
+# Long enough for a cold interpreter to import torch and torchao and run every probe (measured 3.9 s on a B200 host, so
+# this is ~45x headroom for a cold page cache); short enough that a child wedged in the driver does not hold the route
+# thread forever. Overrunning it is not a verdict either -- the caller falls back in-process.
_CHILD_PROBE_TIMEOUT = 180.0
-# Set once the spawn machinery has been shown not to work here (frozen build without
-# multiprocessing support, a sandbox that refuses to fork/exec), so the fallback is paid once
-# rather than on every miss.
+# Set once the spawn machinery has been shown not to work here (frozen build without multiprocessing support, a sandbox
+# that refuses to fork/exec), so the fallback is paid once rather than on every miss.
_CHILD_PROBE_UNAVAILABLE = False
-# Consecutive OSErrors out of the spawn before that latch is set anyway. An OSError is resource
-# pressure and clears, so it is retried; a host that raises one every time still stops paying
-# for the attempt after a few misses. Reset by the first spawn that works.
+# Consecutive OSErrors out of the spawn before that latch is set anyway. An OSError is resource pressure and clears, so
+# it is retried; a host that raises one every time still stops paying for the attempt after a few misses. Reset by the
+# first spawn that works.
_CHILD_PROBE_SPAWN_ERROR_LIMIT = 3
_CHILD_PROBE_SPAWN_ERRORS = 0
@@ -653,37 +639,37 @@ def _child_probe_table(device: str) -> Optional[dict[str, Optional[bool]]]:
run_without_native_path_secret,
)
- # spawn, never fork: the backend is threaded (uvicorn, the arbiter, download workers) and
- # a forked child inherits those locks, and fork does not exist on Windows at all. A build
- # with no spawn support at all raises here, which is what the fallback is for.
+ # spawn, never fork: the backend is threaded (uvicorn, the arbiter, download workers) and a forked child
+ # inherits those locks, and fork does not exist on Windows at all. A build with no spawn support at all raises
+ # here, which is what the fallback is for.
ctx = mp.get_context("spawn")
with native_path_secret_removed_for_child_start():
- # Inside the scrub, as every other orchestrator here builds theirs. The first
- # spawn-context queue starts multiprocessing's resource tracker, which is exec'd
- # with this environment and outlives every child, so building it above the scrub
- # would strand the lease secret somewhere the child-side scrub cannot reach.
+ # inside the scrub: the first spawn-context queue starts multiprocessing's resource tracker
+ # Inside the scrub, as every other orchestrator here builds theirs. The first spawn-context queue starts
+ # multiprocessing's resource tracker, which is exec'd with this environment and outlives every child, so
+ # building it above the scrub would strand the lease secret somewhere the child-side scrub cannot reach.
queue = ctx.Queue()
- # The same entrypoint the inference worker is spawned through: it scrubs the lease
- # secret from the child and binds the child to this process's lifetime, so a wedged
- # probe cannot outlive the backend.
+ # The same entrypoint the inference worker is spawned through: it scrubs the lease secret from the child and
+ # binds the child to this process's lifetime, so a wedged probe cannot outlive the backend.
proc = ctx.Process(
target = run_without_native_path_secret,
args = (__name__, "_child_probe_entry", {}, device, TQ_SCHEMES, queue),
daemon = True,
)
proc.start()
- # Adopted like every other spawn site: the bind above is the CHILD arming PDEATHSIG,
- # which is Linux only, and the Windows job object can fail to take when Unsloth already
- # runs inside an incompatible host job. This record is what is left in that case.
+ # the bind above is the CHILD arming PDEATHSIG, which is Linux only
+ # Adopted like every other spawn site: the bind above is the CHILD arming PDEATHSIG, which is Linux only, and
+ # the Windows job object can fail to take when Unsloth already runs inside an incompatible host job. This record
+ # is what is left in that case.
_adopt_probe_pid(proc.pid)
_CHILD_PROBE_SPAWN_ERRORS = 0
- except Exception as exc: # noqa: BLE001 — no child here: probe in-process instead
+ except Exception as exc: # noqa: BLE001 - no child here: probe in-process instead
import logging
- # An OSError is momentary pressure on descriptors, process slots or /dev/shm, not a
- # host that cannot spawn; latching it would hold the backend on the in-process probe,
- # and its ~800 MiB, until restart. Only a deterministic failure latches on sight, plus
- # a run of OSErrors so a host that always refuses is paid for only briefly.
+ # An OSError is momentary pressure on descriptors, process slots or /dev/shm, not a host that cannot spawn;
+ # latching it would hold the backend on the in-process probe, and its ~800 MiB, until restart. Only a
+ # deterministic failure latches on sight, plus a run of OSErrors so a host that always refuses is paid for only
+ # briefly.
transient = isinstance(exc, OSError)
if transient:
_CHILD_PROBE_SPAWN_ERRORS += 1
@@ -706,14 +692,14 @@ def _child_probe_table(device: str) -> Optional[dict[str, Optional[bool]]]:
try:
table = queue.get(timeout = 0.5)
break
- except Exception: # noqa: BLE001 — Empty, or a queue torn down under us
+ except Exception: # noqa: BLE001 - Empty, or a queue torn down under us
pass
if not proc.is_alive():
- # A put lands through a feeder thread, so it can still be in flight when the
- # child has already exited; one blocking look before calling it a loss.
+ # A put lands through a feeder thread, so it can still be in flight when the child has already exited;
+ # one blocking look before calling it a loss.
try:
table = queue.get(timeout = 1.0)
- except Exception: # noqa: BLE001 — the child really did die empty
+ except Exception: # noqa: BLE001 - the child really did die empty
table = None
break
if time.monotonic() >= deadline:
@@ -726,10 +712,9 @@ def _child_probe_table(device: str) -> Optional[dict[str, Optional[bool]]]:
# SIGKILL may be OOM and SIGTERM is timeout cleanup, so neither is a scheme verdict.
_PROBE_CRASH_SIGNALS = frozenset({4, 6, 7, 8, 11}) # ILL, ABRT, BUS, FPE, SEGV
-# Windows has no signal here: multiprocessing returns GetExitCodeProcess verbatim, so a fault is
-# the raw NTSTATUS (access violation 0xC0000005, UCRT abort 0xC0000409), never -SIGSEGV. Only
-# TERMINATE is negative, hence the sign test first; the severity bits then separate a fault from
-# a deliberate sys.exit.
+# Windows has no signal here: multiprocessing returns GetExitCodeProcess verbatim, so a fault is the raw NTSTATUS
+# (access violation 0xC0000005, UCRT abort 0xC0000409), never -SIGSEGV. Only TERMINATE is negative, hence the sign test
+# first; the severity bits then separate a fault from a deliberate sys.exit.
_NTSTATUS_ERROR_FLOOR = 0xC0000000
@@ -862,11 +847,11 @@ def _child_probe_entry(device: str, schemes: tuple[str, ...], out: Any) -> None:
for scheme in schemes:
try:
table[scheme] = _run_smoke_probe(scheme, device)
- except Exception: # noqa: BLE001 — _run_smoke_probe already swallows; keep the table whole
+ except Exception: # noqa: BLE001 - _run_smoke_probe already swallows; keep the table whole
table[scheme] = False
try:
out.put(table)
- except Exception: # noqa: BLE001 — parent gave up; nothing left to say
+ except Exception: # noqa: BLE001 - parent gave up; nothing left to say
pass
@@ -892,21 +877,22 @@ def _smoke_probe(
``new_zeros`` out to 226 or 512, so the tail rows reaching the DiT are exactly zero.
Measured on B200, 4096-wide Linear: 412 of 512 rows non-finite without the floor, 0 of 512
with it. Failing here costs one ladder step down to int8 instead."""
- # Keyed by the CARD, not just "cuda": the probe runs on whichever device is current, so on a
- # heterogeneous host a pass on one selected GPU would otherwise stand in for an untested one,
- # and a failure on an older card would reject the scheme on a capable one.
+ # keyed by the CARD, not just "cuda": on a heterogeneous host a pass on one selected GPU would stand in for an
+ # untested one
+ # Keyed by the CARD, not just "cuda": the probe runs on whichever device is current, so on a heterogeneous host a
+ # pass on one selected GPU would otherwise stand in for an untested one, and a failure on an older card would reject
+ # the scheme on a capable one.
key = (scheme, _smoke_cache_device_key(device))
if key in _SMOKE_CACHE:
return _SMOKE_CACHE[key]
ok = _run_smoke_probe(scheme, device)
if ok is None:
- # NOT cached, and NOT a verdict. An out-of-memory says nothing about the scheme, and
- # the probe now runs on the ROUTE thread -- BEFORE the arbiter evicts the resident chat
- # model, which is the whole point of raising the refusal early -- so it meets a full
- # GPU by design. Caching it would refuse every later EXPLICIT request for this scheme
- # for the life of the process, on a host that runs it fine once the eviction has
- # happened; and answering "unsupported" to the pre-eviction gate (``unproven_ok``)
- # refuses THIS load for the same reason the eviction was about to remove.
+ # NOT cached, and NOT a verdict. An out-of-memory says nothing about the scheme, and the probe now runs on the
+ # ROUTE thread -- BEFORE the arbiter evicts the resident chat model, which is the whole point of raising the
+ # refusal early -- so it meets a full GPU by design. Caching it would refuse every later EXPLICIT request for
+ # this scheme for the life of the process, on a host that runs it fine once the eviction has happened; and
+ # answering "unsupported" to the pre-eviction gate (``unproven_ok``) refuses THIS load for the same reason the
+ # eviction was about to remove.
return unproven_ok
_SMOKE_CACHE[key] = ok
return ok
@@ -922,7 +908,7 @@ def _run_smoke_probe(scheme: str, device: str) -> Optional[bool]:
lin = torch.nn.Linear(512, 512, bias = False).to(device = device, dtype = torch.bfloat16)
quantize_(lin, _make_quant_config(scheme), filter_fn = make_filter_fn(0))
- # M stays 32 (scaled_mm wants 16-aligned dims); the zero rows go inside it, not after it.
+ # M stays 32 (scaled_mm wants 16-aligned dims); the zero rows go inside it, not after it
x = torch.randn(32, 512, device = device, dtype = torch.bfloat16)
x[16:] = 0
with torch.no_grad():
@@ -951,7 +937,7 @@ def _is_out_of_memory(exc: BaseException) -> bool:
)
if named and isinstance(exc, named):
return True
- except Exception: # noqa: BLE001 — no torch: the message test below still applies
+ except Exception: # noqa: BLE001 - no torch: the message test below still applies
pass
return isinstance(exc, MemoryError) or "out of memory" in str(exc).lower()
@@ -990,10 +976,12 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
if scheme == TQ_INT8:
return Int8DynamicActivationInt8WeightConfig()
if scheme == TQ_FP8:
- # Per-ROW granularity (per-token activation + per-channel weight scale) is REQUIRED: torchao defaults to per-TENSOR, where one Z-Image outlier
- # near 6.6e4 forces a tensor-wide scale that pushes normal values below fp8 resolution and the denoise collapses to noise. _smoke_probe checks
- # per-row scaled_mm, so a build without it falls to int8. fast accumulate (fp8 only) follows GPU class unless forced: consumer cards run fp8
- # ~2x faster with FP16 accumulate. activation_value_lb floors the per-row scale: an ALL-ZERO row otherwise yields scale 0, NaN qdata, black frames.
+ # Per-ROW granularity (per-token activation + per-channel weight scale) is REQUIRED: torchao defaults to
+ # per-TENSOR, where one Z-Image outlier near 6.6e4 forces a tensor-wide scale that pushes normal values below
+ # fp8 resolution and the denoise collapses to noise. _smoke_probe checks per-row scaled_mm, so a build without
+ # it falls to int8. fast accumulate (fp8 only) follows GPU class unless forced: consumer cards run fp8 ~2x
+ # faster with FP16 accumulate. activation_value_lb floors the per-row scale: an ALL-ZERO row otherwise yields
+ # scale 0, NaN qdata, black frames.
import inspect
from torchao.quantization import PerRow
@@ -1001,15 +989,16 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
config_params = inspect.signature(Float8DynamicActivationFloat8WeightConfig).parameters
if "activation_value_lb" in config_params:
fp8_kwargs["activation_value_lb"] = 1e-12
- # Pin the plain-torch quantize kernel: the default AUTO switches to the MSLK kernel whenever an mslk package is importable, changing fp8 scale rounding BITWISE and breaking the prequant bit-identity invariant.
- # It is also slower compiled on B200 (an opaque extern call blocks inductor's quantize fusion), so the pin costs nothing.
+ # Pin the plain-torch quantize kernel: the default AUTO switches to the MSLK kernel whenever an mslk package is
+ # importable, changing fp8 scale rounding BITWISE and breaking the prequant bit-identity invariant. It is also
+ # slower compiled on B200 (an opaque extern call blocks inductor's quantize fusion), so the pin costs nothing.
if "kernel_preference" in config_params:
try:
from torchao.quantization.quantize_.common.kernel_preference import (
KernelPreference,
)
fp8_kwargs["kernel_preference"] = KernelPreference.TORCH
- except Exception: # noqa: BLE001 — enum moved: keep the library default
+ except Exception: # noqa: BLE001 - enum moved: keep the library default
pass
try:
from torchao.float8 import Float8MMConfig
@@ -1017,12 +1006,13 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
mm_config = Float8MMConfig(use_fast_accum = _resolve_fast_accum(fast_accum)),
**fp8_kwargs,
)
- except Exception: # noqa: BLE001 — older torchao without the explicit mm knob
+ except Exception: # noqa: BLE001 - older torchao without the explicit mm knob
return Float8DynamicActivationFloat8WeightConfig(**fp8_kwargs)
if scheme == TQ_NVFP4:
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
- # Select the CUTLASS FP4 path, not the default Triton kernel (which needs MSLK): on a Blackwell box with CUTLASS FP4 but no MSLK the default fails the smoke probe and falls back to GGUF.
+ # Select the CUTLASS FP4 path, not the default Triton kernel (which needs MSLK): on a Blackwell box with CUTLASS
+ # FP4 but no MSLK the default fails the smoke probe and falls back to GGUF.
try:
return NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False)
except TypeError: # older torchao without the knob
@@ -1035,7 +1025,8 @@ def _make_quant_config(scheme: str, fast_accum: Optional[bool] = None) -> Any:
activation_dtype = torch.float8_e4m3fn, weight_dtype = torch.float8_e4m3fn
)
except (TypeError, AttributeError):
- # TypeError: older torchao without the explicit dtype knobs. AttributeError: a torch build without torch.float8_e4m3fn.
+ # TypeError: older torchao without the explicit dtype knobs. AttributeError: a torch build without
+ # torch.float8_e4m3fn.
return MXDynamicActivationMXWeightConfig()
raise ValueError(f"unknown transformer quant scheme '{scheme}'")
@@ -1117,10 +1108,13 @@ def quantize_transformer(
try:
from torchao.quantization import quantize_
- # int8 skips the M=1 projections; fp8/mxfp8 assert a bf16 weight, so on a mixed-precision DiT they must skip non-bf16 ones. "lora_" keeps a
- # baked adapter's side path high precision. Runtime only: NOT part of exclude_tokens_for_scheme, whose list is baked into prequant metadata.
+ # int8 skips the M=1 projections; "lora_" keeps a baked adapter's side path high precision.
+ # int8 skips the M=1 projections; fp8/mxfp8 assert a bf16 weight, so on a mixed-precision DiT they must skip
+ # non-bf16 ones. "lora_" keeps a baked adapter's side path high precision. Runtime only: NOT part of
+ # exclude_tokens_for_scheme, whose list is baked into prequant metadata.
exclude = exclude_tokens_for_scheme(scheme, family) + ("lora_",)
- # GEMM tiling floors per scheme: scaled_mm needs 16-aligned dims, MX block scaling 32. int8's _int_mm has no such floor and keeps the historical filter.
+ # GEMM tiling floors per scheme: scaled_mm needs 16-aligned dims, MX block scaling 32. int8's _int_mm has no
+ # such floor and keeps the historical filter.
divisible = {TQ_FP8: 16, TQ_NVFP4: 16, TQ_MXFP8: 32}.get(scheme, 0)
quantize_(
transformer,
@@ -1132,17 +1126,17 @@ def quantize_transformer(
require_divisible = divisible,
),
)
- # Pad this family's small-M linears now that the weights are quantized and in place. Not
- # best-effort: a raise here means the transformer is quantized but not safely compilable,
- # so it falls into the except below and the caller loads GGUF.
+ # not best-effort: a raise here means the transformer is quantized but not safely compilable
+ # Pad this family's small-M linears now that the weights are quantized and in place. Not best-effort: a raise
+ # here means the transformer is quantized but not safely compilable, so it falls into the except below and the
+ # caller loads GGUF.
apply_small_m_padding(transformer, scheme, family, logger = logger)
- # Runtime-only diagnostic marker.
try:
transformer._unsloth_runtime_quant = scheme
- except Exception: # noqa: BLE001 — marker is best-effort
+ except Exception: # noqa: BLE001 - marker is best-effort
pass
return scheme
- except Exception as exc: # noqa: BLE001 — leave the transformer dense -> GGUF fallback
+ except Exception as exc: # noqa: BLE001 - leave the transformer dense -> GGUF fallback
_warn(logger, scheme, exc)
return None
diff --git a/studio/backend/core/inference/external_tool_transport.py b/studio/backend/core/inference/external_tool_transport.py
index ba56eb874b8..5a815178577 100644
--- a/studio/backend/core/inference/external_tool_transport.py
+++ b/studio/backend/core/inference/external_tool_transport.py
@@ -23,8 +23,8 @@ from typing import Any
from core.inference.external_provider import ExternalProviderClient
-# /inference/cancel and the model-load path only set a threading.Event, so an
-# asyncio consumer has to poll it. Same interval as the Codex client's watcher.
+# /inference/cancel and the model-load path only set a threading.Event, so an asyncio consumer has to poll it. Same
+# interval as the Codex client's watcher.
_CANCEL_POLL_S = 0.05
@@ -44,10 +44,10 @@ class OAICompatTransport:
"""
heals_text_tool_calls = True
- # ExternalProviderClient sanitizes every raw upstream line at the point it
- # arrives, before any translation. What it yields on top of that is this
- # server's own synthesized frames (a provider-hosted image or web-search
- # result), which a second pass in the loop could no longer tell apart.
+ # the client sanitizes raw upstream lines on arrival
+ # ExternalProviderClient sanitizes every raw upstream line at the point it arrives, before any translation. What it
+ # yields on top of that is this server's own synthesized frames (a provider-hosted image or web-search result),
+ # which a second pass in the loop could no longer tell apart.
sanitizes_provider_frames = True
def __init__(
@@ -71,12 +71,10 @@ class OAICompatTransport:
tool_choice: Any,
cancel_event: threading.Event,
) -> AsyncIterator[str]:
- # "Resume the trailing assistant turn", so it is only ever true of the
- # first request. Once a tool runs the conversation ends with a role="tool"
- # result (or a role="user" no-op note), and vLLM / llama.cpp would splice
- # the generation prompt off the end of *that* message: the model continues
- # the tool output instead of answering it, or the chat template raises and
- # the server 400s. Re-read the tail every turn rather than replaying the
+ # "Resume the trailing assistant turn", so it is only ever true of the first request. Once a tool runs the
+ # conversation ends with a role="tool" result (or a role="user" no-op note), and vLLM / llama.cpp would splice
+ # the generation prompt off the end of *that* message: the model continues the tool output instead of answering
+ # it, or the chat template raises and the server 400s. Re-read the tail every turn rather than replaying the
# flag the transport was constructed with.
continue_final_message = bool(self._continue_final_message) and bool(
messages and isinstance(messages[-1], dict) and messages[-1].get("role") == "assistant"
@@ -132,8 +130,8 @@ class OAICompatTransport:
yield line
finally:
watcher.cancel()
- # The pre-existing teardown: the route or the loop closing this
- # generator still reaches the provider stream through here.
+ # The pre-existing teardown: the route or the loop closing this generator still reaches the provider stream
+ # through here.
aclose = getattr(upstream, "aclose", None)
if aclose is not None:
with contextlib.suppress(RuntimeError, GeneratorExit, StopAsyncIteration):
diff --git a/studio/backend/core/inference/gallery_flags.py b/studio/backend/core/inference/gallery_flags.py
index c08ac6eb09d..7a6b36c6b1e 100644
--- a/studio/backend/core/inference/gallery_flags.py
+++ b/studio/backend/core/inference/gallery_flags.py
@@ -34,8 +34,8 @@ logger = get_logger(__name__)
_SCHEMA_VERSION = 1
_STORE_NAME = ".flags.json"
-# Marks a store written over one whose ITEMS MAP was illegible: the old flags could not be carried
-# forward, so the new file is no proof that nothing is archived. See ``_carry_taint``.
+# Marks a store written over one whose ITEMS MAP was illegible: the old flags could not be carried forward, so the new
+# file is no proof that nothing is archived. See ``_carry_taint``.
_TAINT_KEY = "unreadable"
_lock = threading.RLock()
@@ -100,8 +100,9 @@ def _load(directory: Path) -> tuple[dict[str, Any], bool]:
try:
with open(_store_path(directory), encoding = "utf-8-sig") as f:
data = json.load(f)
- # Validate the shape, not just the version: a hand-edited ``items`` that is not a dict
- # (e.g. ``[]``) would otherwise crash every lookup instead of failing safe.
+ # validate the shape, not just the version
+ # Validate the shape, not just the version: a hand-edited ``items`` that is not a dict (e.g. ``[]``) would
+ # otherwise crash every lookup instead of failing safe.
if (
isinstance(data, dict)
and data.get("version") == _SCHEMA_VERSION
@@ -110,11 +111,11 @@ def _load(directory: Path) -> tuple[dict[str, Any], bool]:
# Written over an illegible store, so what it does NOT say is not evidence.
if data.get(_TAINT_KEY):
return data, False
- # Every ENTRY has to be readable too, not just the container. A malformed value is
- # dropped by the readers below, which reads as "this id is not archived" -- enough for
- # clear() to delete an archived file. So one bad entry costs the store its trust, but
- # the surviving entries are still returned: listing should keep the flags it can read,
- # and only destructive callers need to refuse.
+ # every ENTRY must be readable too: a malformed value is dropped by the readers
+ # Every ENTRY has to be readable too, not just the container. A malformed value is dropped by the readers
+ # below, which reads as "this id is not archived" -- enough for clear() to delete an archived file. So one
+ # bad entry costs the store its trust, but the surviving entries are still returned: listing should keep the
+ # flags it can read, and only destructive callers need to refuse.
if all(_valid_entry(v) for v in data["items"].values()):
return data, True
logger.warning(
@@ -126,7 +127,7 @@ def _load(directory: Path) -> tuple[dict[str, Any], bool]:
)
return _empty(), False
except FileNotFoundError:
- return _empty(), True # no store yet is a legitimate "nothing is flagged"
+ return _empty(), True
except Exception as exc:
logger.warning("gallery_flags.read_failed: %s", exc)
return _empty(), False
@@ -192,10 +193,7 @@ def _file_lock(directory: Path):
pass # locking unavailable; the thread lock still applies
yield locked
finally:
- # Release only what was taken, and never let the release be the thing that fails the call.
- # A filesystem that cannot lock usually cannot unlock either, and by this point the body
- # has already written the flags or deleted the media, so raising here reports a failure for
- # work that landed.
+ # never let the release fail the call: a filesystem that cannot lock usually cannot unlock
try:
if locked:
with contextlib.suppress(Exception):
@@ -288,8 +286,8 @@ def flags_for(items: dict[str, dict[str, Any]], item_id: str) -> dict[str, Any]:
"""The public record fields for one id, from an already-read ``items`` map."""
entry = _entry(items, item_id)
return {
- # Reported through the same conversion the sort uses, so a value the ordering cannot use
- # never shows as a pin the user then cannot explain.
+ # Reported through the same conversion the sort uses, so a value the ordering cannot use never shows as a pin
+ # the user then cannot explain.
"pinned": _pinned_at(entry) is not None,
"archived": bool(entry.get("archived")),
}
@@ -333,10 +331,11 @@ def set_flags_locked(
write land as one step. Separate for the same per-descriptor lock reason as ``forget_locked``."""
import time
- # A write REPAIRS the store rather than preserving what made it untrusted. Merging the bad
- # entry straight back would leave every later clear() refused until someone fixed the file by
- # hand, and refusing here instead would leave the user unable to pin anything at all. Dropping
- # only the unreadable entries keeps the flags that still mean something.
+ # a write REPAIRS the store: merging the bad entry back would leave every later clear() refused
+ # A write REPAIRS the store rather than preserving what made it untrusted. Merging the bad entry straight back would
+ # leave every later clear() refused until someone fixed the file by hand, and refusing here instead would leave the
+ # user unable to pin anything at all. Dropping only the unreadable entries keeps the flags that still mean
+ # something.
data = _carry_taint(*_load(directory))
items: dict[str, Any] = {}
for key, value in data.get("items", {}).items():
@@ -347,20 +346,19 @@ def set_flags_locked(
entry = dict(_entry(items, item_id))
if pinned is not None:
if pinned:
- # Strictly ahead of every stamp stored, not just the wall clock: Windows advances
- # time.time() in ~16 ms steps, so two pins a click apart landed on the same value and
- # "most recently pinned leads" stopped holding for exactly the case the client
- # serializes its PATCHes to preserve.
+ # strictly ahead of every stored stamp, not just the wall clock
+ # Strictly ahead of every stamp stored, not just the wall clock: Windows advances time.time() in ~16 ms
+ # steps, so two pins a click apart landed on the same value and "most recently pinned leads" stopped holding
+ # for exactly the case the client serializes its PATCHes to preserve.
latest = max(
(_pinned_at(v) for v in items.values() if _pinned_at(v) is not None),
default = float("-inf"),
)
now = time.time()
nudged = math.nextafter(latest, math.inf) if latest != float("-inf") else now
- # A store holding the largest finite float nudges to infinity, which json writes and
- # _pinned_at then refuses, so the pin just reported would read back unset AND take the
- # store's trust with it. Tie instead: those two fall back to mtime, which costs an
- # ordering rather than the store.
+ # A store holding the largest finite float nudges to infinity, which json writes and _pinned_at then
+ # refuses, so the pin just reported would read back unset AND take the store's trust with it. Tie instead:
+ # those two fall back to mtime, which costs an ordering rather than the store.
entry["pinned_at"] = (
now if now > latest else (nudged if math.isfinite(nudged) else latest)
)
@@ -396,7 +394,7 @@ def forget_locked(directory: Path, item_ids) -> None:
data = _load(directory)[0]
items = data.get("items", {})
if not any(i in items for i in ids):
- return # nothing stored for these ids: skip the write entirely
+ return
for item_id in ids:
items.pop(item_id, None)
try:
diff --git a/studio/backend/core/inference/generation_timing.py b/studio/backend/core/inference/generation_timing.py
index 3cf6c6b0354..b525ccd37f2 100644
--- a/studio/backend/core/inference/generation_timing.py
+++ b/studio/backend/core/inference/generation_timing.py
@@ -35,8 +35,9 @@ def _wait_for_device(device):
except TypeError: # torch.mps.synchronize takes no device argument
synchronize()
except Exception:
- # An async device fault surfaces here as a RuntimeError. It belongs to generate(), whose
- # caller reports it; a timing stamp must not pre-empt that or skip the cleanup after it.
+ # an async device fault belongs to generate(); a timing stamp must not pre-empt it or skip cleanup
+ # An async device fault surfaces here as a RuntimeError. It belongs to generate(), whose caller reports it; a
+ # timing stamp must not pre-empt that or skip the cleanup after it.
pass
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index 14373dc819b..5f57595debf 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -40,20 +40,25 @@ def _evict_chat() -> None:
from core.inference.llama_cpp import chat_load_active
llama = get_llama_cpp_backend()
- # is_active (process exists), not is_loaded (exists AND healthy): a chat model still starting up holds VRAM but is not healthy. chat_load_active
- # too, since an HF load has no process until its GGUF downloaded. unload_model sets the cancel event the download loop polls, so it aborts.
+ # is_active, not is_loaded: a model still starting holds VRAM
+ # is_active (process exists), not is_loaded (exists AND healthy): a chat model still starting up holds VRAM but is
+ # not healthy. chat_load_active too, since an HF load has no process until its GGUF downloaded. unload_model sets
+ # the cancel event the download loop polls, so it aborts.
if llama.is_active or chat_load_active():
llama.unload_model()
orchestrator = get_inference_backend()
if orchestrator.active_model_name:
orchestrator.unload_model(orchestrator.active_model_name)
- # An in-flight safetensors load has no active_model_name yet (published only on success), so the unload above misses it and it would finish onto
- # the GPU we just granted away. cancel_load discards the loading marker BEFORE tearing the worker down, and runs off the lifecycle gate.
+ # An in-flight safetensors load has no active_model_name yet (published only on success), so the unload above misses
+ # it and it would finish onto the GPU we just granted away. cancel_load discards the loading marker BEFORE tearing
+ # the worker down, and runs off the lifecycle gate.
for pending in list(getattr(orchestrator, "loading_models", ()) or ()):
orchestrator.cancel_load(pending)
# Kill the subprocess too: its base CUDA context holds VRAM diffusion needs.
orchestrator._shutdown_subprocess(timeout = 5.0)
- # The driver reclaims the killed VRAM asynchronously, so wait for it to settle before diffusion allocates, else a warm handoff can transiently OOM.
+ # the driver reclaims killed VRAM asynchronously, else a warm handoff can transiently OOM
+ # The driver reclaims the killed VRAM asynchronously, so wait for it to settle before diffusion allocates, else a
+ # warm handoff can transiently OOM.
llama._wait_for_vram_settle(since_kill = time.monotonic())
@@ -68,7 +73,8 @@ def _evict_video() -> None:
get_video_backend().unload()
-# Patchable in tests via monkeypatch.setitem. Ownership is exclusive, so acquire_for's evict-the-current-owner generalises to any number of owners.
+# Patchable in tests via monkeypatch.setitem. Ownership is exclusive, so acquire_for's evict-the-current-owner
+# generalises to any number of owners.
_EVICTORS = {CHAT: _evict_chat, DIFFUSION: _evict_diffusion, VIDEO: _evict_video}
diff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.py
index cc197bf84d9..81fb905542a 100644
--- a/studio/backend/core/inference/image_gallery.py
+++ b/studio/backend/core/inference/image_gallery.py
@@ -70,7 +70,9 @@ def save(image: Any, meta: dict[str, Any]) -> dict[str, Any]:
image_id = uuid.uuid4().hex
directory = gallery_dir()
final_path = directory / f"{image_id}.png"
- # Write to a dotted temp (skipped by the *.png glob) then atomically rename, so a crash mid-write never leaves a truncated {id}.png in the listing.
+ # dotted temp (skipped by the *.png glob) then atomic rename
+ # Write to a dotted temp (skipped by the *.png glob) then atomically rename, so a crash mid-write never leaves a
+ # truncated {id}.png in the listing.
tmp_path = directory / f".{image_id}.png.tmp"
try:
tmp_path.write_bytes(_png_bytes(image, meta))
@@ -89,7 +91,7 @@ def _record(
meta: dict[str, Any],
flags: Optional[dict[str, dict[str, Any]]] = None,
) -> dict[str, Any]:
- # Flags are library state, not recipe: they come from the sidecar store, never the PNG chunk.
+ # flags are library state, not recipe: they come from the sidecar store, never the PNG chunk
return {
**meta,
"id": image_id,
@@ -105,7 +107,6 @@ def image_path(image_id: str) -> Optional[Path]:
if not _ID_RE.match(image_id):
return None
path = gallery_dir() / f"{image_id}.png"
- # Defence in depth: confirm the resolved path is still inside the gallery.
try:
path.resolve().relative_to(gallery_dir().resolve())
except ValueError:
@@ -120,7 +121,9 @@ def image_b64(image_id: str) -> Optional[str]:
return base64.b64encode(path.read_bytes()).decode("ascii")
-# Required recipe keys (GalleryImage fields minus id/url). A PNG missing any is skipped as foreign, so a hand-dropped or older-schema file cannot 500 the listing.
+# a PNG missing any is skipped as foreign, so a hand-dropped or older-schema file cannot 500 the listing
+# Required recipe keys (GalleryImage fields minus id/url). A PNG missing any is skipped as foreign, so a hand-dropped or
+# older-schema file cannot 500 the listing.
_REQUIRED_META = ("prompt", "width", "height", "steps", "guidance", "seed", "created_at")
@@ -187,20 +190,24 @@ def list_images(
except OSError:
return []
flags = gallery_flags.read(gallery_dir())
- # Both the shelf split and the pin sort run on file stems, BEFORE any recipe is read, so they
- # cost one dict lookup per file and leave the early break below intact.
+ # both run on file stems BEFORE any recipe is read
+ # Both the shelf split and the pin sort run on file stems, BEFORE any recipe is read, so they cost one dict lookup
+ # per file and leave the early break below intact.
paths = [p for p in paths if gallery_flags.is_archived(flags, p.stem) == archived]
paths.sort(key = lambda p: (gallery_flags.pin_rank(flags, p.stem), _mtime(p)), reverse = True)
- # Page over READABLE records, not raw files: filtering a foreign PNG out of an already-sliced window would drop valid images and make has_more
- # wrong. Known limit: this re-reads headers from newest down to `offset+limit` per page, so a deep scroll is O(offset) header-opens.
+ # page over READABLE records: filtering a foreign PNG out of an already-sliced window would drop valid images and
+ # make has_more wrong.
+ # Page over READABLE records, not raw files: filtering a foreign PNG out of an already-sliced window would drop
+ # valid images and make has_more wrong. Known limit: this re-reads headers from newest down to `offset+limit` per
+ # page, so a deep scroll is O(offset) header-opens.
want = None if limit is None else offset + limit
records = []
for path in paths:
meta = _read_meta(path)
- if meta is None: # not one of ours (no recipe chunk)
+ if meta is None:
continue
record = _record(path.stem, meta, flags)
- if valid is not None and not valid(record): # present but schema-invalid
+ if valid is not None and not valid(record):
continue
records.append(record)
if want is not None and len(records) >= want:
@@ -217,8 +224,8 @@ def set_flags(
"""Patch one image's pin/archive flags and return its updated record, or None when the id is
not an Unsloth-owned image. Ownership-gated like delete: a guessed stem for a hand-dropped
foreign PNG must not become flaggable (and so listable under a shelf we own)."""
- # Ownership check and write under one lock, so a concurrent clear cannot delete the file
- # between them and leave this reporting success for an image that is already gone.
+ # Ownership check and write under one lock, so a concurrent clear cannot delete the file between them and leave this
+ # reporting success for an image that is already gone.
with gallery_flags.exclusive(gallery_dir()):
path = owned_image_path(image_id)
if path is None:
@@ -234,7 +241,7 @@ def delete(image_id: str) -> bool:
path = image_path(image_id)
if path is None:
return False
- # Only delete files we own (a readable recipe chunk): a hand-dropped foreign PNG is invisible to list_images, so a guessed id must not destroy it.
+ # a hand-dropped foreign PNG is invisible to list_images, so a guessed id must not destroy it
if _read_meta(path) is None:
return False
try:
@@ -242,8 +249,7 @@ def delete(image_id: str) -> bool:
except OSError as exc:
logger.warning("image_gallery.delete_failed: %s", exc)
return False
- # Drop the flags with the file, so the id cannot hand a stale pin to anything and the store
- # cannot grow forever.
+ # drop the flags with the file, so the id cannot hand out a stale pin and the store cannot grow forever
gallery_flags.forget(gallery_dir(), [image_id])
return True
@@ -262,11 +268,10 @@ def clear(include_archived: bool = False) -> int:
Foreign PNGs are preserved: list_images already hides them, so clear must not destroy them."""
removed = 0
directory = gallery_dir()
- # Hold the flag lock across the whole read-then-delete: an archive landing mid-loop would
- # otherwise be judged active from the stale snapshot and deleted, after its PATCH had already
- # reported success.
+ # Hold the flag lock across the whole read-then-delete: an archive landing mid-loop would otherwise be judged active
+ # from the stale snapshot and deleted, after its PATCH had already reported success.
with gallery_flags.exclusive(directory):
- # Read flags BEFORE listing: nothing is unlinked if the store turns out to be untrusted.
+ # read flags BEFORE listing: nothing is unlinked if the store turns out to be untrusted
flags = {} if include_archived else gallery_flags.read_trusted(directory)
try:
paths = list(directory.glob("*.png"))
@@ -274,7 +279,7 @@ def clear(include_archived: bool = False) -> int:
return 0
cleared: list[str] = []
for path in paths:
- if _read_meta(path) is None: # foreign / not ours
+ if _read_meta(path) is None:
continue
if not include_archived and gallery_flags.is_archived(flags, path.stem):
continue
@@ -284,8 +289,9 @@ def clear(include_archived: bool = False) -> int:
cleared.append(path.stem)
except OSError:
continue
- # Nothing left for an unreadable store to protect once every image we own is gone, so this is
- # where the escape hatch escapes: replace it, or every later default clear still refuses.
+ # once every image we own is gone an unreadable store protects nothing
+ # Nothing left for an unreadable store to protect once every image we own is gone, so this is where the escape
+ # hatch escapes: replace it, or every later default clear still refuses.
if include_archived and not gallery_flags.is_trusted(directory):
gallery_flags.reset_locked(directory)
else:
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index c3ffe241dad..24b87891dce 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -2151,6 +2151,7 @@ class InferenceBackend:
# ── Audio (TTS) Generation ────────────────────────────────────
+ # ── Audio (TTS) Generation ────────────────────────────────────
def generate_audio_response(
self,
text: str,
diff --git a/studio/backend/core/inference/instruction_pin.py b/studio/backend/core/inference/instruction_pin.py
index c541065a643..39553b7d859 100644
--- a/studio/backend/core/inference/instruction_pin.py
+++ b/studio/backend/core/inference/instruction_pin.py
@@ -37,17 +37,16 @@ import re
from core.inference.context_window import estimate_messages_tokens_dense, group_turns
-# 80 characters: someone who typed a paragraph wrote an instruction. Nothing inspects
-# meaning or keywords, which are the heuristics a user trips by accident.
+# 80 characters: someone who typed a paragraph wrote an instruction. Nothing inspects meaning or keywords, which are the
+# heuristics a user trips by accident.
INSTRUCTION_MIN_CHARS = int(os.environ.get("ROLLING_INSTRUCTION_MIN_CHARS", "80"))
PIN_GROUPS = int(os.environ.get("ROLLING_INSTRUCTION_PIN_GROUPS", "0"))
PIN_MAX_TOKENS = int(os.environ.get("ROLLING_INSTRUCTION_PIN_MAX_TOKENS", "1024"))
-# ... and never more than this share of the prompt budget, so the pin stays a minority of
-# the window on a small model as well as a large one.
+# ... and never more than this share of the prompt budget, so the pin stays a minority of the window on a small model as
+# well as a large one.
PIN_MAX_FRACTION = float(os.environ.get("ROLLING_INSTRUCTION_PIN_MAX_FRACTION", "0.10"))
-# A pure REJECT list: it can only stop something being treated as an instruction, never
-# promote one.
+# A pure REJECT list: it can only stop something being treated as an instruction, never promote one.
_CONTINUATIONS = frozenset(
{
"continue",
@@ -84,14 +83,15 @@ _CONTINUATIONS = frozenset(
"then",
}
)
-# U+2026 and U+2025 as well as the ASCII spellings: keyboards autocorrect "..." to one
-# ellipsis character, so `continue…` matched nothing and recall searched for "continue".
+# keyboards autocorrect "..." to U+2026, so `continue...` matched nothing and recall searched for "continue"
+# U+2026 and U+2025 as well as the ASCII spellings: keyboards autocorrect "..." to one ellipsis character, so
+# `continue…` matched nothing and recall searched for "continue".
_PUNCTUATION = re.compile(r"[\s\.,!\?;:\-–—\u2025\u2026]+")
-# "Anaphoric" as a closed list rather than a word count: words that cannot name the
-# subject of a request. "what about it" has nothing to search for; "review billing" names
-# its own subject and keeps its retrieval slots. Negation is left out on purpose, as in
-# `store._ARCHIVE_STOPWORDS`: a missed anchor is cheaper than a wrong one.
+# A closed list, not a word count: words that cannot name a request's subject "Anaphoric" as a closed list rather than a
+# word count: words that cannot name the subject of a request. "what about it" has nothing to search for; "review
+# billing" names its own subject and keeps its retrieval slots. Negation is left out, as in `store._ARCHIVE_STOPWORDS`:
+# a missed anchor is cheaper than a wrong one.
_FUNCTION_WORDS = frozenset(
"""
a about all also am an and another any anything are as at be been being both but by can
@@ -187,7 +187,6 @@ def is_thin_query(text: str, *, min_chars: int = INSTRUCTION_MIN_CHARS) -> bool:
# Short AND anaphoric: "what is ZQXVARA123?" names something and stays the query.
words = normalised.split()
if not words:
- # Punctuation only ("???"): nothing survives tokenisation, so the query is empty.
return True
return all(word in _FUNCTION_WORDS or word in _CONTINUATIONS for word in words)
@@ -210,8 +209,9 @@ def _protected_cost(turns: list[list[dict]], index: int) -> int:
small instruction its pin over tokens the pin never keeps -- which is the case the pin
exists for, since an agent run is exactly where the filler follow-up appears.
"""
- # Dense: 4 chars per token undercharges CJK and emoji ~2x, so a 1056-token turn was
- # charged 276 and cleared a 1024 ceiling. Over-charging only refuses the pin.
+ # 4 chars per token undercharges CJK and emoji ~2x; over-charging only refuses the pin
+ # Dense: 4 chars per token undercharges CJK and emoji ~2x, so a 1056-token turn was charged 276 and cleared a 1024
+ # ceiling. Over-charging only refuses the pin.
return estimate_messages_tokens_dense(turns[index])
@@ -246,8 +246,9 @@ def pinned_instruction_ids(
return set()
turns = group_turns(messages)
- # The newest user group is already protected by the window, and the inline recall path
- # replaces that message with a new dict, so its id would go stale anyway.
+ # the newest user group is already window-protected
+ # The newest user group is already protected by the window, and the inline recall path replaces that message with a
+ # new dict, so its id would go stale anyway.
newest_user = next(
(
index
diff --git a/studio/backend/core/inference/key_exchange.py b/studio/backend/core/inference/key_exchange.py
index 15ebf0e4c0e..15ef30102ee 100644
--- a/studio/backend/core/inference/key_exchange.py
+++ b/studio/backend/core/inference/key_exchange.py
@@ -36,7 +36,6 @@ def init_key_pair() -> None:
"""Generate an RSA-2048 key pair. Called once at server startup."""
global _private_key, _public_key_pem, _public_key_fingerprint
if _private_key is not None:
- # Re-entry invalidates in-flight ciphertext from the old public key; log loudly.
logger.warning(
"init_key_pair called again — replacing existing RSA keypair "
"(previous fingerprint=%s). Any frontend that cached the old "
@@ -109,8 +108,7 @@ def decrypt_api_key(encrypted_b64: str) -> str:
),
)
except Exception as exc:
- # Log state to distinguish key mismatch from padding/algo mismatch or
- # corrupted bytes. RSA-2048 ciphertext is exactly 256 bytes.
+ # RSA-2048 ciphertext is exactly 256 bytes; log state to separate key mismatch from padding
logger.warning(
"decrypt_api_key: RSA decrypt failed (ciphertext_len=%d, expected=256, "
"fingerprint=%s, exc=%s): %s",
diff --git a/studio/backend/core/inference/llama_admission.py b/studio/backend/core/inference/llama_admission.py
index bc50589ff35..1951c64c8cd 100644
--- a/studio/backend/core/inference/llama_admission.py
+++ b/studio/backend/core/inference/llama_admission.py
@@ -21,10 +21,9 @@ from dataclasses import dataclass
from typing import Deque, Optional
-# dataclass(slots = True) halves per-instance overhead. Measured as perf-neutral
-# here, not a speed win: it costs a little on construction and gains it back on
-# access. It is 3.10+ and this package declares >=3.9, so gate it rather than
-# dropping it outright. Empty on 3.9 means a plain dataclass.
+# dataclass(slots = True) halves per-instance overhead. Measured as perf-neutral here, not a speed win: it costs a
+# little on construction and gains it back on access. It is 3.10+ and this package declares >=3.9, so gate it rather
+# than dropping it outright. Empty on 3.9 means a plain dataclass.
_SLOTS = {"slots": True} if sys.version_info >= (3, 10) else {}
@@ -33,13 +32,12 @@ ADMISSION_QUEUE_TIMEOUT_ENV = "UNSLOTH_LLAMA_ADMISSION_QUEUE_TIMEOUT"
ADMISSION_KEEPALIVE_INTERVAL_ENV = "UNSLOTH_LLAMA_ADMISSION_KEEPALIVE_INTERVAL"
ADMISSION_MAX_QUEUE_ENV = "UNSLOTH_LLAMA_ADMISSION_MAX_QUEUE"
ADMISSION_QUEUE_PER_SLOT_ENV = "UNSLOTH_LLAMA_ADMISSION_QUEUE_PER_SLOT"
-# Off restores slot-only admission. The escape hatch for a backend whose reported
-# context length does not match the cache llama-server actually allocated.
+# off restores slot-only admission: the escape hatch for a backend whose reported context length does not match the
+# cache llama-server allocated
ADMISSION_KV_BUDGET_ENV = "UNSLOTH_LLAMA_ADMISSION_KV_BUDGET"
-# The UNSLOTH_OPENAI_COMPAT_* spellings predate this queue being shared with the
-# Anthropic /v1/messages route (same llama-server slots). Still honored; the
-# neutral name above wins when both are set.
+# The UNSLOTH_OPENAI_COMPAT_* spellings predate this queue being shared with the Anthropic /v1/messages route (same
+# llama-server slots). Still honored; the neutral name above wins when both are set.
_LEGACY_ENV = {
ADMISSION_CONTROL_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL",
ADMISSION_QUEUE_TIMEOUT_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT",
@@ -53,21 +51,20 @@ DEFAULT_ADMISSION_QUEUE_TIMEOUT_S = None
DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S = 5.0
# None: no absolute cap, the wait line is sized from the pool instead.
DEFAULT_ADMISSION_MAX_QUEUE = None
-# Wait line = 16 x the serving slots, so it tracks --parallel (4 slots -> 64
-# waiters, 8 -> 128). Purely a memory guard; waiting itself is never timed out.
+# Wait line = 16 x the serving slots, so it tracks --parallel (4 slots -> 64 waiters, 8 -> 128). Purely a memory guard;
+# waiting itself is never timed out.
DEFAULT_ADMISSION_QUEUE_PER_SLOT = 16
-# Floor for the scaled line, so a 1-slot backend (plain `unsloth studio`, or any
-# load downshifted to fit VRAM) keeps the depth it had before scaling existed
-# rather than dropping to 16 and rejecting callers that used to queue.
+# Floor for the scaled line, so a 1-slot backend (plain `unsloth studio`, or any load downshifted to fit VRAM) keeps the
+# depth it had before scaling existed rather than dropping to 16 and rejecting callers that used to queue.
DEFAULT_ADMISSION_MIN_QUEUE = 64
-# Token accounting is on by default. Slot-only admission overcommits a unified KV
-# cache: llama.cpp with --parallel N --kv-unified allocates ONE cache of n_ctx but
-# reports n_ctx_slot = n_ctx to every slot, so N generations are admitted against a
-# cache that may hold one. When they collide, llama.cpp kills every task involved.
+# slot-only admission overcommits a unified KV cache
+# Token accounting is on by default. Slot-only admission overcommits a unified KV cache: llama.cpp with --parallel N
+# --kv-unified allocates ONE cache of n_ctx but reports n_ctx_slot = n_ctx to every slot, so N generations are admitted
+# against a cache that may hold one. When they collide, llama.cpp kills every task involved.
DEFAULT_ADMISSION_KV_BUDGET = True
-# Ceiling on one round's wait for cache room; generous, since a legitimate wait is bounded
-# by the longest round in flight. Bounded at all because a reparker holds the wait line
-# shut for every other caller, so an unbounded wait freezes the queue. See recost_waiting.
+# Ceiling on one round's wait for cache room; generous, since a legitimate wait is bounded by the longest round in
+# flight. Bounded at all because a reparker holds the wait line shut for every other caller, so an unbounded wait
+# freezes the queue. See recost_waiting.
DEFAULT_RECOST_WAIT_TIMEOUT_S = 300.0
@@ -102,15 +99,15 @@ def _max_parked(capacity: int) -> int:
"""
workers = _executor_workers()
spare = workers - _executor_reserve(workers) - max(0, capacity)
- # A quarter of the executor, floored at two while `spare` allows: a quarter of
- # five is one, and one park cannot cover the two simultaneous prompts #7455
- # exists for.
+ # a quarter of five is one, and one park cannot cover the two simultaneous prompts #7455 exists for
+ # A quarter of the executor, floored at two while `spare` allows: a quarter of five is one, and one park cannot
+ # cover the two simultaneous prompts #7455 exists for.
return max(0, min(max(2, workers // 4), spare))
-# Process-wide, not per queue: there is one executor, and base_url takes a fresh
-# port on every load, so a per-queue budget would hand the same allowance to each
-# backend and to every reload, blind to the approvals parked on the old queue.
+# Process-wide, not per queue: there is one executor, and base_url takes a fresh port on every load, so a per-queue
+# budget would hand the same allowance to each backend and to every reload, blind to the approvals parked on the old
+# queue.
_PARK_LOCK = threading.Lock()
_parked_total = 0
@@ -152,8 +149,9 @@ class LlamaAdmissionConfig:
keepalive_interval_s: float = DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S
max_queue: Optional[int] = DEFAULT_ADMISSION_MAX_QUEUE
queue_per_slot: Optional[int] = DEFAULT_ADMISSION_QUEUE_PER_SLOT
- # Unconditional floor on the scaled line. The env path clears it when the
- # operator sets QUEUE_PER_SLOT, so only the default multiplier is floored.
+ # the env path clears this floor when the operator sets QUEUE_PER_SLOT
+ # Unconditional floor on the scaled line. The env path clears it when the operator sets QUEUE_PER_SLOT, so only the
+ # default multiplier is floored.
min_queue: Optional[int] = DEFAULT_ADMISSION_MIN_QUEUE
kv_budget: bool = DEFAULT_ADMISSION_KV_BUDGET
@@ -180,8 +178,8 @@ class LlamaAdmissionSnapshot:
active: int
queued: int
free: int = 0
- # KV tokens held by live leases, and the cache they are drawn from. budget 0
- # means token accounting is off, so committed carries no meaning.
+ # KV tokens held by live leases, and the cache they are drawn from. budget 0 means token accounting is off, so
+ # committed carries no meaning.
committed: int = 0
budget: int = 0
@@ -260,8 +258,8 @@ def _queue_limits_from_env() -> tuple[Optional[int], Optional[int], Optional[int
floor applies only to the default multiplier: setting QUEUE_PER_SLOT means
the operator wants that exact depth, however shallow.
"""
- # Explicit means it parsed, not just that something was set: a typo falls back
- # to the default multiplier, so it has to keep the default's floor too.
+ # Explicit means it parsed, not just that something was set: a typo falls back to the default multiplier, so it has
+ # to keep the default's floor too.
raw_per_slot = _raw_env(ADMISSION_QUEUE_PER_SLOT_ENV)
try:
per_slot = int((raw_per_slot or "").strip())
@@ -304,8 +302,9 @@ class _Waiter:
future: asyncio.Future
cancelled: bool = False
granted_lease: Optional["LlamaAdmissionLease"] = None
- # KV tokens this caller will occupy once admitted. Read while the head of the
- # line is considered, so a large request cannot be overtaken by small ones.
+ # read while the head of the line is considered
+ # KV tokens this caller will occupy once admitted. Read while the head of the line is considered, so a large request
+ # cannot be overtaken by small ones.
tokens: int = 0
@@ -332,8 +331,8 @@ class LlamaAdmissionLease:
self._release_lock = threading.Lock()
self._parked = False
self._budgeted = False
- # Returned to the queue on release, not on park: a parked holder has
- # stopped decoding but llama-server still holds its KV until the task ends.
+ # Returned to the queue on release, not on park: a parked holder has stopped decoding but llama-server still
+ # holds its KV until the task ends.
self._tokens = max(0, int(tokens or 0))
@property
@@ -357,9 +356,8 @@ class LlamaAdmissionLease:
with self._release_lock:
if queue is None or self._released or self._parked:
return False
- # Under the lease lock so the decision and the handover cannot split.
- # Nothing takes the queue lock then a lease lock, so this order is
- # the only one in play.
+ # Under the lease lock so the decision and the handover cannot split. Nothing takes the queue lock then a
+ # lease lock, so this order is the only one in play.
if not queue.try_park(self._slot):
return False
self._parked = True
@@ -412,18 +410,18 @@ class LlamaAdmissionLease:
queue = self._queue
if queue is None or not self._parked:
return
- # Before the wait, not after: the prompt is answered, so this holder is
- # already off the executor and must not keep anyone else off it.
+ # Before the wait, not after: the prompt is answered, so this holder is already off the executor and must not
+ # keep anyone else off it.
self._drop_budget()
slot = await queue.acquire_parked_slot(cancel_event = cancel_event, poll_s = poll_s)
stranded = None
with self._release_lock:
- # release() may have run during the wait; it clears the flag and does
- # the unpark itself, so only the caller that clears it here repeats one.
+ # release() may have run during the wait; it clears the flag and does the unpark itself, so only the caller
+ # that clears it here repeats one.
parked, self._parked = self._parked, False
if self._released:
- # Released while waiting: this lease will never hand the slot
- # back, so return it here rather than strand it for good.
+ # Released while waiting: this lease will never hand the slot back, so return it here rather than strand
+ # it for good.
stranded = slot
else:
self._slot = slot
@@ -440,10 +438,9 @@ class LlamaAdmissionLease:
still stands, and the caller is over its reservation.
"""
want = max(0, int(tokens or 0))
- # Held ACROSS try_recost: a release interleaving between the queue accepting the
- # new figure and this lease recording it would hand back the OLD number and strand
- # the difference as committed for the life of the process. release() takes this
- # lock then the queue's, so this order cannot deadlock against it.
+ # Held ACROSS try_recost: a release interleaving between the queue accepting the new figure and this lease
+ # recording it would hand back the OLD number and strand the difference as committed for the life of the
+ # process. release() takes this lock then the queue's, so this order cannot deadlock against it.
with self._release_lock:
if self._released or self._queue is None:
return True
@@ -507,15 +504,14 @@ class LlamaAdmissionLease:
if queue.try_reclaim_commitment(want):
with self._release_lock:
if self._released:
- # Released while waiting; release() already gave back 0 and
- # will not run again, so hand the commitment straight back.
+ # Released while waiting; release() already gave back 0 and will not run again, so hand the
+ # commitment straight back.
queue.release(None, want)
return True
self._tokens = want
return True
- # Every pass, not only on the two exits below: release() runs from the
- # route's teardown without touching the cancel event, so a Stop would
- # otherwise leave this spinning on a dead lease, wait line held shut.
+ # Every pass, not only on the two exits below: release() runs from the route's teardown without touching
+ # the cancel event, so a Stop would otherwise leave this spinning on a dead lease, wait line held shut.
if self._released:
queue.abandon_repark()
return False
@@ -543,9 +539,7 @@ class LlamaAdmissionLease:
"""
with self._release_lock:
if self._released:
- # release() already ran and gave back the 0 held while parked, and
- # yield_commitment already took `held` off the pool, so restoring it
- # here would strand it as committed for good.
+ # release() already gave back the 0 held while parked
queue.abandon_repark()
return False
self._tokens = held
@@ -685,25 +679,24 @@ class LlamaAdmissionQueue:
self._lock = threading.Lock()
self._capacity = 1
self._free: list[int] = [0]
- # Held slots as a bitmask: one int instead of a set, so the pool costs the
- # same whether it is idle or saturated. _held is its popcount, kept as a
- # counter because int.bit_count() is 3.10+ and this package targets 3.9.
+ # Held slots as a bitmask: one int instead of a set, so the pool costs the same whether it is idle or saturated.
+ # _held is its popcount, kept as a counter because int.bit_count() is 3.10+ and this package targets 3.9.
self._in_use = 0
self._held = 0
self._waiters: Deque[_Waiter] = deque()
- # Holders parked on a tool approval prompt. They hold no slot, so this only
- # keeps the queue off the idle-eviction list while they are away.
+ # Holders parked on a tool approval prompt. They hold no slot, so this only keeps the queue off the
+ # idle-eviction list while they are away.
self._parked = 0
- # FIFO tickets for holders resuming from a park (see acquire_parked_slot). A
- # bare count deadlocked: every approved holder blocked every other one.
+ # FIFO tickets for holders resuming from a park (see acquire_parked_slot). A bare count deadlocked: every
+ # approved holder blocked every other one.
self._unpark_tickets: Deque[int] = deque()
self._unpark_seq = 0
- # KV tokens held by live leases, against the cache size the caller reports.
- # 0 budget disables the check, which is what every pre-existing caller gets.
+ # KV tokens held by live leases, against the cache size the caller reports. 0 budget disables the check, which
+ # is what every pre-existing caller gets.
self._committed = 0
self._budget = 0
- # Holders that gave their commitment back and are waiting to take a bigger one.
- # They still hold a slot, so they are not in _waiters. See yield_commitment.
+ # Holders that gave their commitment back and are waiting to take a bigger one. They still hold a slot, so they
+ # are not in _waiters. See yield_commitment.
self._reparking = 0
def _resize_pool_locked(self, capacity: int) -> None:
@@ -739,14 +732,13 @@ class LlamaAdmissionQueue:
reserved: int,
tokens: int = 0,
) -> bool:
- # Slots still held above a shrunk capacity keep occupying the backend, so
- # count every held slot against the ceiling, not just the ids below it.
- # ``reserved`` holds slots back for approved holders waiting to resume;
+ # Slots still held above a shrunk capacity keep occupying the backend, so count every held slot against the
+ # ceiling, not just the ids below it. ``reserved`` holds slots back for approved holders waiting to resume;
# without it a stream of new arrivals took the next slot, forever.
if not (bool(self._free) and (self._held + reserved) < self._capacity):
return False
- # A free slot is not enough: with --kv-unified every slot reports the full
- # n_ctx, so the pool can hand out more slots than the one cache can serve.
+ # A free slot is not enough: with --kv-unified every slot reports the full n_ctx, so the pool can hand out more
+ # slots than the one cache can serve.
return self._fits_budget_locked(tokens)
def _take_slot_locked(
@@ -791,21 +783,18 @@ class LlamaAdmissionQueue:
loop = asyncio.get_running_loop()
with self._lock:
self._resize_pool_locked(capacity)
- # Re-read every reserve: a reload can relaunch llama-server at a
- # different -c, and a stale budget would keep admitting against a
- # cache that no longer exists.
+ # Re-read every reserve: a reload can relaunch llama-server at a different -c, and a stale budget would keep
+ # admitting against a cache that no longer exists.
self._budget = max(0, int(budget or 0))
self._grant_waiters_locked()
- # A pending repark closes the fast path like a non-empty wait line: the
- # reparker gave its room back to ask for more, so an arrival admitted here
- # would take exactly that, pinning a growing conversation at its opening size
- # for as long as traffic lasts.
+ # A pending repark closes the fast path like a non-empty wait line: the reparker gave its room back to ask
+ # for more, so an arrival admitted here would take exactly that, pinning a growing conversation at its
+ # opening size for as long as traffic lasts.
if not self._waiters and self._reparking == 0:
slot = self._take_slot_locked(len(self._unpark_tickets), cost)
if slot is not None:
- # No snapshot here: callers read it through snapshot_now(),
- # which re-reads the queue, so building one per admitted
- # request would be pure allocation on the hot path.
+ # No snapshot here: callers read it through snapshot_now(), which re-reads the queue, so building
+ # one per admitted request would be pure allocation on the hot path.
return LlamaAdmissionReservation(
queue = self,
lease = LlamaAdmissionLease(self, slot, cost),
@@ -843,10 +832,9 @@ class LlamaAdmissionQueue:
) -> None:
with self._lock:
self._release_slot_locked(slot)
- # Floored at 0 so a double release cannot drive the pool negative and
- # let the budget admit callers the cache cannot hold. The lease's own
- # _released guard makes that unreachable; this keeps it unreachable if
- # a future caller releases by hand.
+ # Floored at 0 so a double release cannot drive the pool negative and let the budget admit callers the cache
+ # cannot hold. The lease's own _released guard makes that unreachable; this keeps it unreachable if a future
+ # caller releases by hand.
self._committed = max(0, self._committed - max(0, int(tokens or 0)))
self._grant_waiters_locked()
@@ -874,8 +862,8 @@ class LlamaAdmissionQueue:
self._grant_waiters_locked()
return True
delta = tokens_to - tokens_from
- # The escape admission uses: a holder that is alone goes past the budget,
- # since refusing it stalls a conversation nothing else can unblock.
+ # The escape admission uses: a holder that is alone goes past the budget, since refusing it stalls a
+ # conversation nothing else can unblock.
alone = (self._committed - tokens_from) <= 0
if alone or self._committed + delta <= self._budget:
self._committed += delta
@@ -911,11 +899,10 @@ class LlamaAdmissionQueue:
return False
self._committed += want
self._reparking = max(0, self._reparking - 1)
- # The decrement above may have dropped the LAST repark barrier, and nothing
- # else re-runs the grant: a release arriving during the repark already found
- # the barrier up and returned. Without this, a reclaim that leaves room and a
- # free slot strands the wait line until the grown run releases, and a queued
- # waiter also closes reserve()'s fast path for every later arrival.
+ # The decrement above may have dropped the LAST repark barrier, and nothing else re-runs the grant: a
+ # release arriving during the repark already found the barrier up and returned. Without this, a reclaim that
+ # leaves room and a free slot strands the wait line until the grown run releases, and a queued waiter also
+ # closes reserve()'s fast path for every later arrival.
self._grant_waiters_locked()
return True
@@ -990,7 +977,6 @@ class LlamaAdmissionQueue:
self._unpark_tickets.remove(ticket)
except ValueError:
pass
- # This ticket was holding a slot back from the wait line.
self._grant_waiters_locked()
def cancel(self, waiter: _Waiter) -> None:
@@ -1008,12 +994,9 @@ class LlamaAdmissionQueue:
try:
waiter.loop.call_soon_threadsafe(waiter.future.cancel)
except RuntimeError:
- # Loop gone. Routes call cancel() from finally blocks, so
- # raising here would both mask their exception and skip the
- # release below, stranding the slot for the process lifetime.
pass
- # A cancel frees no slot and no tokens, so nothing else re-runs
- # admission for the waiters this one was blocking.
+ # a cancel frees no slot and no tokens, so nothing else re-runs admission for the waiters this one was
+ # blocking
self._grant_waiters_locked()
if lease_to_release is not None:
lease_to_release.release()
@@ -1026,21 +1009,19 @@ class LlamaAdmissionQueue:
def is_idle(self) -> bool:
with self._lock:
self._prune_waiters_locked()
- # A parked holder owns no slot but is coming back to this queue, so
- # evicting it here would resume it against a fresh 1-slot pool.
+ # A parked holder owns no slot but is coming back to this queue, so evicting it here would resume it against
+ # a fresh 1-slot pool.
return self._in_use == 0 and not self._waiters and not self._parked
def _grant_waiters_locked(self) -> None:
- # A reparker gave its commitment back mid-conversation, so it takes the room ahead
- # of anything not yet started. Without this a steady arrival rate can hold a
- # growing run at its old size indefinitely.
+ # A reparker gave its commitment back mid-conversation, so it takes the room ahead of anything not yet started.
+ # Without this a steady arrival rate can hold a growing run at its old size indefinitely.
if self._reparking > 0:
return
- # Dead waiters are skipped as they are popped, so no prune is needed here.
- # The head's own cost is what is tested, not a zero: granting past a large
- # waiter whenever a small one fits would starve it for as long as traffic
- # keeps arriving. Head-of-line blocking is the fair trade here, and it
- # matches the FIFO the rest of this queue already promises.
+ # Dead waiters are skipped as they are popped, so no prune is needed here. The head's own cost is what is
+ # tested, not a zero: granting past a large waiter whenever a small one fits would starve it for as long as
+ # traffic keeps arriving. Head-of-line blocking is the fair trade here, and it matches the FIFO the rest of this
+ # queue already promises.
while self._waiters and self._can_admit_locked(
len(self._unpark_tickets), self._waiters[0].tokens
):
@@ -1053,17 +1034,15 @@ class LlamaAdmissionQueue:
try:
waiter.loop.call_soon_threadsafe(self._deliver_lease, waiter, lease)
except RuntimeError:
- # Waiter's loop is gone. Reclaim the slot; leaving the bit set
- # would strand it, since _free is rebuilt from the bitmask. The
- # tokens go back with it: nothing will ever release this lease.
+ # waiter's loop is gone: reclaim the slot, since leaving the bit set would strand it (_free is rebuilt
+ # from the bitmask)
waiter.granted_lease = None
self._release_slot_locked(slot)
self._committed = max(0, self._committed - max(0, waiter.tokens))
def _deliver_lease(self, waiter: _Waiter, lease: LlamaAdmissionLease) -> None:
- # Runs on the waiter's own loop thread, which is also the only thread that
- # cancels that reservation, so waiter state is safe to touch unlocked here.
- # release() may be called from any thread, but only reaches this via
+ # Runs on the waiter's own loop thread, which is also the only thread that cancels that reservation, so waiter
+ # state is safe to touch unlocked here. release() may be called from any thread, but only reaches this via
# call_soon_threadsafe. Cancelling off-loop would need this under _lock.
if waiter.cancelled or waiter.future.done():
waiter.granted_lease = None
@@ -1079,9 +1058,8 @@ class LlamaAdmissionQueue:
lease.release()
def _prune_waiters_locked(self) -> None:
- # Rebuilding the deque on every reserve/release dominated the hot path, so
- # only pay it when a waiter actually died out of band (an externally
- # cancelled future); cancel() already drops its own waiter eagerly.
+ # Rebuilding the deque on every reserve/release dominated the hot path, so only pay it when a waiter actually
+ # died out of band (an externally cancelled future); cancel() already drops its own waiter eagerly.
for waiter in self._waiters:
if waiter.cancelled or waiter.future.done():
break
@@ -1100,12 +1078,12 @@ class LlamaAdmissionQueue:
key = self.key,
capacity = self._capacity,
active = self._held,
- # Resume tickets are approved continuations holding no slot yet; omitting
- # them would show a full, idle-looking queue while one still waits.
+ # Resume tickets are approved continuations holding no slot yet; omitting them would show a full,
+ # idle-looking queue while one still waits.
queued = len(self._waiters) + len(self._unpark_tickets),
- # What another caller could actually take, so free never shows next to queued:
- # after a shrink, ids below the new capacity can be free while holdovers fill
- # the ceiling, and tickets hold slots back exactly as _can_admit_locked does.
+ # What another caller could actually take, so free never shows next to queued: after a shrink, ids below the
+ # new capacity can be free while holdovers fill the ceiling, and tickets hold slots back exactly as
+ # _can_admit_locked does.
free = min(
len(self._free),
max(0, self._capacity - self._held - len(self._unpark_tickets)),
@@ -1125,10 +1103,9 @@ def get_llama_admission_queue(key: str) -> LlamaAdmissionQueue:
if queue is None:
queue = LlamaAdmissionQueue(key)
_QUEUES[key] = queue
- # base_url carries a fresh ephemeral port on every model load, so
- # each load registers a new key. Drop the now-idle queues from prior
- # loads so the registry can't grow without bound on a long-running
- # server. Queues with in-flight requests are kept until they drain.
+ # base_url carries a fresh ephemeral port on every model load, so each load registers a new key. Drop the
+ # now-idle queues from prior loads so the registry can't grow without bound on a long-running server. Queues
+ # with in-flight requests are kept until they drain.
for stale_key in [k for k in _QUEUES if k != key and _QUEUES[k].is_idle()]:
del _QUEUES[stale_key]
return queue
@@ -1145,7 +1122,7 @@ def reset_llama_admission_queues() -> None:
global _parked_total
with _QUEUES_LOCK:
_QUEUES.clear()
- # The budget outlives the queues it was claimed against, so dropping them
- # without it leaks the count and shrinks the budget for good.
+ # The budget outlives the queues it was claimed against, so dropping them without it leaks the count and shrinks the
+ # budget for good.
with _PARK_LOCK:
_parked_total = 0
diff --git a/studio/backend/core/inference/llama_http.py b/studio/backend/core/inference/llama_http.py
index 8aa072e35b8..930ed786a06 100644
--- a/studio/backend/core/inference/llama_http.py
+++ b/studio/backend/core/inference/llama_http.py
@@ -25,9 +25,9 @@ def _new_client() -> httpx.AsyncClient:
return httpx.AsyncClient(limits = _LIMITS, trust_env = False)
-# One client per running event loop: an httpx client binds its transport to the
-# loop it first runs on, so a single global instance breaks across a lifespan
-# restart or a second test loop. Weak keys let a finished loop drop its client.
+# httpx binds its transport to the loop it first ran on
+# One client per running event loop: an httpx client binds its transport to the loop it first runs on, so a single
+# global instance breaks across a lifespan restart or a second test loop. Weak keys let a finished loop drop its client.
_clients: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, httpx.AsyncClient]" = (
weakref.WeakKeyDictionary()
)
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py
index 6a2e9c5d9ed..bef73690938 100644
--- a/studio/backend/core/inference/llama_keepwarm.py
+++ b/studio/backend/core/inference/llama_keepwarm.py
@@ -31,40 +31,39 @@ _inflight = 0
_preview_inflight = 0
# Blocked on the unload gate, not yet in _inflight: the idle loop must not unload while one waits.
_pending = 0
-# Subset of _pending that is /p/ preview traffic, so the busy guard can tell a queued
-# Unsloth request from a queued preview.
+# Subset of _pending that is /p/ preview traffic, so the busy guard can tell a queued Unsloth request from a queued
+# preview.
_preview_pending = 0
-# Non-preview requests past FastAPI auth at the local-inference choke point. The preview
-# busy guard counts these, not raw _inflight, so a pre-auth/unauthenticated tracked request
-# that never touches the model can't starve public previews.
+# non-preview requests past FastAPI auth: the preview busy guard counts these
+# Non-preview requests past FastAPI auth at the local-inference choke point. The preview busy guard counts these, not
+# raw _inflight, so a pre-auth/unauthenticated tracked request that never touches the model can't starve public
+# previews.
_admitted_inference = 0
-# Bumped when a preview swap loads a new checkpoint. A non-preview request captures it before
-# the lifecycle gate; if it advanced by the time the gate is held, a preview swapped the model
-# out from under it and the request is rejected (see the middleware).
+# Bumped when a preview swap loads a new checkpoint. A non-preview request captures it before the lifecycle gate; if it
+# advanced by the time the gate is held, a preview swapped the model out from under it and the request is rejected (see
+# the middleware).
_preview_swap_generation = 0
-# Non-zero while a preview swap is loading (before it takes the lifecycle gate until after it
-# releases). Catches a request that captures the counter AFTER the bump but BEFORE the gate
-# releases: the middleware snapshots this flag at entry and rejects if a swap was in progress.
+# Non-zero while a preview swap is loading (before it takes the lifecycle gate until after it releases). Catches a
+# request that captures the counter AFTER the bump but BEFORE the gate releases: the middleware snapshots this flag at
+# entry and rejects if a swap was in progress.
_preview_swap_inflight = 0
_last_active = time.monotonic()
-# The (id, quant) idle-unload last freed, so an alias/unknown request that would otherwise
-# 503 against an empty backend can reload it (set on unload, cleared on reload). The quant
-# lets the reload restore the exact freed variant.
+# The (id, quant) idle-unload last freed, so an alias/unknown request that would otherwise 503 against an empty backend
+# can reload it (set on unload, cleared on reload). The quant lets the reload restore the exact freed variant.
_last_unloaded_model = None
# Slot KV manifest saved by the idle unload; whoever pops it owns deleting its files.
_kv_resume = None
-# Guards inflight bumps against the idle-check-then-unload race and blocks new inference
-# mid-swap. Process-wide, not per-loop: the backend slot is shared across every event loop,
-# so a per-loop gate would let a request on loop B start while a swap on loop A tears it down.
+# Guards inflight bumps against the idle-check-then-unload race and blocks new inference mid-swap. Process-wide, not
+# per-loop: the backend slot is shared across every event loop, so a per-loop gate would let a request on loop B start
+# while a swap on loop A tears it down.
_lifecycle_lock = threading.Lock()
@contextlib.asynccontextmanager
async def _unload_gate(cancel_event: threading.Event | None = None):
- # Acquire off the loop: non-blocking first (the common uncontended case), else
- # poll a non-blocking acquire off a short sleep. Polling keeps the wait off this
- # loop AND cancellation-safe -- a cancel lands during the sleep, when the gate is
- # not held, so it never leaks (mirrors the auto-switch swap gate).
+ # Acquire off the loop: non-blocking first (the common uncontended case), else poll a non-blocking acquire off a
+ # short sleep. Polling keeps the wait off this loop AND cancellation-safe -- a cancel lands during the sleep, when
+ # the gate is not held, so it never leaks (mirrors the auto-switch swap gate).
acquired = False
try:
while not _lifecycle_lock.acquire(blocking = False):
@@ -85,46 +84,47 @@ _INFERENCE_SUFFIXES = (
"/chat/completions",
"/completions",
"/messages",
- "/messages/count_tokens", # counts via the loaded tokenizer; protect like /messages
+ "/messages/count_tokens",
"/chat/count_tokens",
"/embeddings",
"/responses",
- "/generate/stream", # Unsloth's own streaming route on the same llama-server
+ "/generate/stream",
"/audio/generate", # direct GGUF TTS; can outlive the idle TTL
- "/audio/speech", # /v1/audio/speech (+ /api/inference/audio/speech); same TTS core as /audio/generate
- # Image generation holds a multi-GB pipeline for the whole request; tracking it lets other_inference_request_count() see
- # an in-flight generation so an API-key training start is refused (409). endswith avoids matching *-progress / */cancel.
- "/images/generate", # /api/inference/images/generate
- "/images/generations", # /v1/images/generations (+ /api/inference/images/generations)
- # Video runs as a background job (the POST returns at once), so this covers only the brief accept; the training-start guards also probe generate-progress.
- "/video/generate", # /api/inference/video/generate
+ "/audio/speech",
+ # Image generation holds a multi-GB pipeline for the whole request; tracking it lets other_inference_request_count()
+ # see an in-flight generation so an API-key training start is refused (409). endswith avoids matching *-progress /
+ # */cancel.
+ "/images/generate",
+ "/images/generations",
+ # video runs as a background job, so this covers only the brief accept
+ "/video/generate",
)
-# Matched WHOLE, not by suffix. The suffix tuple above is an endswith test, so a bare
-# "/videos" entry would also class an unrouted /v1/anything/videos as inference: that
-# 404s before any auth dependency, and this middleware only excludes 401/403, so each
-# such probe would refresh the chat model's idle timer and keep it resident for free.
+# matched WHOLE, not by suffix: a bare "/videos" entry would class an unrouted /v1/anything/videos as inference
+# Matched WHOLE, not by suffix. The suffix tuple above is an endswith test, so a bare "/videos" entry would also class
+# an unrouted /v1/anything/videos as inference: that 404s before any auth dependency, and this middleware only excludes
+# 401/403, so each such probe would refresh the chat model's idle timer and keep it resident for free.
_INFERENCE_EXACT_PATHS = frozenset({"/v1/videos", "/api/inference/videos"})
-# Tracked above (they hold the GPU, so the in-flight count must see them) but served by the
-# diffusion/video engines, never the llama slot. A successful one therefore did NOT run against
-# the resident chat model and must not adopt it for Unsloth: clearing the marker on an image or
-# video generation would leave a still-preview-owned checkpoint looking Unsloth-owned, and the
-# next preview for a different checkpoint would 503 on the slot guard.
+# tracked above (they hold the GPU) but served by the diffusion/video engines, never the llama slot
+# Tracked above (they hold the GPU, so the in-flight count must see them) but served by the diffusion/video engines,
+# never the llama slot. A successful one therefore did NOT run against the resident chat model and must not adopt it for
+# Unsloth: clearing the marker on an image or video generation would leave a still-preview-owned checkpoint looking
+# Unsloth-owned, and the next preview for a different checkpoint would 503 on the slot guard.
_NON_LLM_SLOT_SUFFIXES = (
"/images/generate",
"/images/generations",
"/video/generate",
- # The OpenAI videos route (/v1/videos + /api/inference/videos) runs the video
- # backend only, exactly like /video/generate. It is tracked as an inference path,
- # so without this it would claim the slot and clear preview ownership.
+ # the OpenAI videos route runs the video backend only, exactly like /video/generate
+ # The OpenAI videos route (/v1/videos + /api/inference/videos) runs the video backend only, exactly like
+ # /video/generate. It is tracked as an inference path, so without this it would claim the slot and clear preview
+ # ownership.
"/videos",
)
def _is_preview_path(path: str) -> bool:
- # Public checkpoint preview delegates to the chat handler on the same backend,
- # so protect it from idle unload.
+ # Public checkpoint preview delegates to the chat handler on the same backend, so protect it from idle unload.
return path.startswith("/p/") and path.endswith("/v1/chat/completions")
@@ -153,9 +153,7 @@ def _note_unpending(is_preview: bool = False) -> None:
def _note_start(is_preview: bool = False) -> None:
- # Don't stamp _last_active here: while _inflight > 0 the model is already protected
- # (see _is_idle), and stamping on start would let a later-untracked external-provider
- # request still reset the local idle timer.
+ # do not stamp _last_active here: while _inflight > 0 the model is already protected
global _inflight, _pending, _preview_inflight, _preview_pending
with _lock:
_pending = max(0, _pending - 1)
@@ -214,8 +212,8 @@ class InferenceActivityReservation:
def _note_untracked_end(is_preview: bool = False) -> None:
- # Drop a request that never used the local GGUF without stamping activity, so
- # external-provider traffic can't keep the model warm.
+ # Drop a request that never used the local GGUF without stamping activity, so external-provider traffic can't keep
+ # the model warm.
global _inflight, _preview_inflight
with _lock:
_inflight = max(0, _inflight - 1)
@@ -324,9 +322,9 @@ def preview_swapped_since_entry(scope) -> bool:
swap-in-progress flag alone."""
if not isinstance(scope, dict):
return False
- # A preview carries its own ownership and may swap the model in (load_model_for_preview
- # bumps the generation before serving its own chat), so it must never reject itself.
- # Mirrors the middleware, which only flags non-preview scopes.
+ # A preview carries its own ownership and may swap the model in (load_model_for_preview bumps the generation before
+ # serving its own chat), so it must never reject itself. Mirrors the middleware, which only flags non-preview
+ # scopes.
if _is_preview_path(scope.get("path") or ""):
return False
if scope.get(_PREVIEW_SWAP_REJECT_SCOPE_KEY):
@@ -351,35 +349,37 @@ def _claim_non_preview_slot() -> None:
logger.debug("preview-slot claim on completion failed: %s", exc)
-# Set on the scope by a route that proved this request won't touch llama.cpp (e.g. it
-# proxied to an external provider), so the keep-warm count excludes it and the middleware
-# skips its end-decrement.
+# set by a route that proved this request will not touch llama.cpp
+# Set on the scope by a route that proved this request won't touch llama.cpp (e.g. it proxied to an external provider),
+# so the keep-warm count excludes it and the middleware skips its end-decrement.
_UNTRACKED_SCOPE_KEY = "_unsloth_keepwarm_untracked"
-# Set after middleware admission so the preview route can distinguish a real tracked
-# request from direct unit/helper calls that have no counters to move.
+# Set after middleware admission so the preview route can distinguish a real tracked request from direct unit/helper
+# calls that have no counters to move.
_TRACKED_SCOPE_KEY = "_unsloth_keepwarm_tracked"
-# A preview route waits on its own serializer after middleware admission. While queued it
-# must be pending, not active: an Unsloth swap holds the lifecycle gate while draining active
-# requests, and the queued preview needs that same gate after it gets the serializer.
+# while queued on its own serializer a preview must be pending
+# A preview route waits on its own serializer after middleware admission. While queued it must be pending, not active:
+# an Unsloth swap holds the lifecycle gate while draining active requests, and the queued preview needs that same gate
+# after it gets the serializer.
_PREVIEW_SERIALIZER_WAIT_SCOPE_KEY = "_unsloth_keepwarm_preview_serializer_wait"
-# Set by the middleware on a non-preview scope when a preview swap advanced the counter
-# while it waited on the gate; _maybe_auto_switch_model then rejects it rather than serve
-# the swapped-in checkpoint. Deferred to the route (not a middleware 503) so an external-
-# provider request that untracks and returns before that check is never rejected.
+# set by the middleware when a preview swap advanced the counter while a non-preview request waited on the gate;
+# Set by the middleware on a non-preview scope when a preview swap advanced the counter while it waited on the gate;
+# _maybe_auto_switch_model then rejects it rather than serve the swapped-in checkpoint. Deferred to the route (not a
+# middleware 503) so an external- provider request that untracks and returns before that check is never rejected.
_PREVIEW_SWAP_REJECT_SCOPE_KEY = "_unsloth_keepwarm_preview_swap_reject"
-# The swap generation snapshot at middleware entry, on the scope so local-inference
-# admission can also reject a request that passed the gate BEFORE a swap (never got the
-# gate-wait reject flag) but is still pre-auth when a preview swaps in.
+# the swap generation at middleware entry, so local-inference admission can also reject a request that passed the gate
+# BEFORE a swap
+# The swap generation snapshot at middleware entry, on the scope so local-inference admission can also reject a request
+# that passed the gate BEFORE a swap (never got the gate-wait reject flag) but is still pre-auth when a preview swaps
+# in.
_SWAP_GEN_AT_ENTRY_KEY = "_unsloth_keepwarm_swap_gen_at_entry"
-# Set on the scope by a streaming route that failed after its 200 headers (an SSE error
-# chunk, a passthrough relaying a mid-stream error while HTTP stays 200). The claim keys
-# off HTTP status alone, so without this a failed stream would adopt a preview-owned model
-# for Unsloth; the claim skips a flagged response.
+# Set on the scope by a streaming route that failed after its 200 headers (an SSE error chunk, a passthrough relaying a
+# mid-stream error while HTTP stays 200). The claim keys off HTTP status alone, so without this a failed stream would
+# adopt a preview-owned model for Unsloth; the claim skips a flagged response.
_RESPONSE_FAILED_SCOPE_KEY = "_unsloth_keepwarm_response_failed"
@@ -391,9 +391,11 @@ def mark_response_failed(scope) -> None:
scope[_RESPONSE_FAILED_SCOPE_KEY] = True
-# The current request's ASGI scope, set by the middleware so deep streaming error
-# helpers can flag a failure without threading the scope through every yield site. The
-# middleware shares the streaming body's task, so the contextvar reaches those generators.
+# the current request's ASGI scope, so deep streaming error helpers can flag a failure without threading the scope
+# through every yield site
+# The current request's ASGI scope, set by the middleware so deep streaming error helpers can flag a failure without
+# threading the scope through every yield site. The middleware shares the streaming body's task, so the contextvar
+# reaches those generators.
_current_response_scope: contextvars.ContextVar = contextvars.ContextVar(
"_unsloth_current_response_scope", default = None
)
@@ -417,8 +419,8 @@ def untrack_current_request(scope) -> None:
if not isinstance(scope, dict) or scope.get(_UNTRACKED_SCOPE_KEY):
return
scope[_UNTRACKED_SCOPE_KEY] = True
- # Keep the preview subset aligned with _inflight: a /p/ request must drop from both
- # counters, or the busy guard sees phantom traffic.
+ # Keep the preview subset aligned with _inflight: a /p/ request must drop from both counters, or the busy guard sees
+ # phantom traffic.
_note_untracked_end(_is_preview_path(scope.get("path") or ""))
@@ -487,8 +489,8 @@ def cancel_preview_serializer_wait(scope) -> None:
return
scope.pop(_PREVIEW_SERIALIZER_WAIT_SCOPE_KEY, None)
_note_unpending(is_preview = True)
- # Middleware must not run the normal active-request decrement after this pending
- # request was removed, or it would stamp activity for a preview that never ran.
+ # Middleware must not run the normal active-request decrement after this pending request was removed, or it would
+ # stamp activity for a preview that never ran.
scope[_UNTRACKED_SCOPE_KEY] = True
@@ -627,8 +629,8 @@ def _carries_bearer_credentials(scope, path: str = "") -> bool:
return True
headers = scope.get("headers")
if headers is None:
- # A real ASGI server always populates headers; a caller that does not is not a
- # client to second-guess, so keep the protection.
+ # A real ASGI server always populates headers; a caller that does not is not a client to second-guess, so keep
+ # the protection.
return True
for name, value in headers:
if _as_bytes(name).lower() != b"authorization":
@@ -645,33 +647,31 @@ class LlamaKeepWarmMiddleware:
self.app = app
async def __call__(self, scope, receive, send):
- # Inference endpoints are all POST; skipping non-POST avoids counting CORS
- # preflight (OPTIONS). ``or ""`` guards an explicit None path.
+ # inference endpoints are all POST, so skipping non-POST avoids counting CORS preflight
+ # Inference endpoints are all POST; skipping non-POST avoids counting CORS preflight (OPTIONS). ``or ""`` guards
+ # an explicit None path.
path = scope.get("path") or ""
if scope.get("type") != "http" or scope.get("method") != "POST":
await self.app(scope, receive, send)
return
- # An image/video generation gets the same bookkeeping against ITS backend, so the
- # media idle unload cannot free the pipeline this request is about to generate on
- # -- or the load it is about to start. The media load routes are tracked HERE only:
- # they do not use the chat GGUF, so they must not stamp chat activity nor count
- # towards other_inference_request_count().
+ # An image/video generation gets the same bookkeeping against ITS backend, so the media idle unload cannot free
+ # the pipeline this request is about to generate on -- or the load it is about to start. The media load routes
+ # are tracked HERE only: they do not use the chat GGUF, so they must not stamp chat activity nor count towards
+ # other_inference_request_count().
from core.inference import media_keepwarm
media_owner = media_keepwarm.owner_for_path(path)
if media_owner is not None and not _carries_bearer_credentials(scope, path):
- # Cannot reach the backend, so it must not hold it warm (see the helper). The
- # chat count keeps its own rule: /p/{run}/v1/chat/completions is public by
- # design, so a missing bearer there is not proof of anything.
+ # Cannot reach the backend, so it must not hold it warm (see the helper). The chat count keeps its own rule:
+ # /p/{run}/v1/chat/completions is public by design, so a missing bearer there is not proof of anything.
media_owner = None
chat_tracked = _is_inference_path(path)
if not chat_tracked and media_owner is None:
await self.app(scope, receive, send)
return
- # Always track in-flight on inference paths, even when the feature is off, so a
- # stream that starts before idle-unload is enabled can't be unloaded mid-response if
- # the operator turns it on. Mark pending before the gate so the idle loop (which
- # holds the gate while unloading) can't free the model while this request waits.
+ # Always track in-flight on inference paths, even when the feature is off, so a stream that starts before
+ # idle-unload is enabled can't be unloaded mid-response if the operator turns it on. Mark pending before the
+ # gate so the idle loop (which holds the gate while unloading) can't free the model while this request waits.
is_preview = _is_preview_path(path)
if chat_tracked:
set_current_response_scope(scope)
@@ -700,20 +700,16 @@ class LlamaKeepWarmMiddleware:
try:
await media_keepwarm.begin_request(media_owner)
except BaseException:
- # The generate routes are tracked on both sides, and this gate can be held
- # for the length of a teardown. A client that disconnects while waiting on
- # it never reaches the _finish below, so balance the chat count here or it
- # stays positive for the life of the process: chat idle unload would never
- # fire again and every training start would see an inference request.
+ # a client that disconnects while waiting on this gate never reaches _finish
if chat_tracked:
_note_untracked_end(is_preview)
raise
ended = {"done": False}
status = {"code": None}
- # Set once the terminal body frame (more_body False) is sent: only a response that
- # completed cleanly adopts the model for Unsloth. A client disconnect after the 200
- # headers raises before that frame (an OSError that _SameTaskStreamingResponse turns
- # into a CancelledError for the body generator, which finishes the monitor and
+ # set once the terminal body frame is sent: only a cleanly completed response adopts the model for Unsloth.
+ # Set once the terminal body frame (more_body False) is sent: only a response that completed cleanly adopts the
+ # model for Unsloth. A client disconnect after the 200 headers raises before that frame (an OSError that
+ # _SameTaskStreamingResponse turns into a CancelledError for the body generator, which finishes the monitor and
# re-raises without flagging the scope), so a cancelled stream never claims the slot.
completed = {"done": False}
@@ -727,15 +723,13 @@ class LlamaKeepWarmMiddleware:
media_keepwarm.end_request(media_owner, counted = code not in (401, 403))
if not chat_tracked:
return
- # A non-preview 2xx that completed cleanly ran against the local model and adopts
- # it for Unsloth, so clear preview ownership. Skip on a per-route 4xx/5xx (never
- # strand a preview-owned model), count_tokens (tokenize only), a failed/cancelled
- # stream, and an untracked balance-only request. Claim BEFORE dropping the admitted
- # count (and the in-flight count) below: load_model_for_preview's busy guard keys on
- # other_admitted_inference_count(), so decrementing first opens a window where a
- # preview sees no admitted Unsloth traffic and a still-preview-owned slot, swaps in,
- # and this delayed claim then clears the wrong checkpoint; while still counted the
- # guard refuses that swap.
+ # A non-preview 2xx that completed cleanly ran against the local model and adopts it for Unsloth, so clear
+ # preview ownership. Skip on a per-route 4xx/5xx (never strand a preview-owned model), count_tokens
+ # (tokenize only), a failed/cancelled stream, and an untracked balance-only request. Claim BEFORE dropping
+ # the admitted count (and the in-flight count) below: load_model_for_preview's busy guard keys on
+ # other_admitted_inference_count(), so decrementing first opens a window where a preview sees no admitted
+ # Unsloth traffic and a still-preview-owned slot, swaps in, and this delayed claim then clears the wrong
+ # checkpoint; while still counted the guard refuses that swap.
if (
not is_preview
and isinstance(code, int)
@@ -749,22 +743,22 @@ class LlamaKeepWarmMiddleware:
and not scope.get(_UNTRACKED_SCOPE_KEY)
):
_claim_non_preview_slot()
- # Balance note_admitted_inference here (runs in the finally, so it can't leak on any
- # exit path), after the claim above and before the untracked / 401 early returns.
+ # balance note_admitted_inference here (in the finally, so it cannot leak on any exit path), after the claim
+ # Balance note_admitted_inference here (runs in the finally, so it can't leak on any exit path), after the
+ # claim above and before the untracked / 401 early returns.
if scope.get(_ADMITTED_SCOPE_KEY):
_note_admitted_end()
if scope.get(_UNTRACKED_SCOPE_KEY):
return
- # This middleware runs before FastAPI auth, so a 401/403 reaches here without
- # touching llama.cpp. Balance _note_start but do NOT stamp activity, or
- # repeated unauthenticated probes would keep the model warm forever.
+ # This middleware runs before FastAPI auth, so a 401/403 reaches here without touching llama.cpp. Balance
+ # _note_start but do NOT stamp activity, or repeated unauthenticated probes would keep the model warm
+ # forever.
if code in (401, 403):
_note_untracked_end(is_preview)
return
- # A preview that did not return 2xx never served tokens (429, bad-token 404,
- # body-validation 4xx all exit before load_model_for_preview). Drop it like
- # an untracked end so rejected public POSTs can't refresh the idle timer and
- # pin the model in VRAM (a loaded-then-failed preview already stamped at load).
+ # A preview that did not return 2xx never served tokens (429, bad-token 404, body-validation 4xx all exit
+ # before load_model_for_preview). Drop it like an untracked end so rejected public POSTs can't refresh the
+ # idle timer and pin the model in VRAM (a loaded-then-failed preview already stamped at load).
if is_preview and not (isinstance(code, int) and 200 <= code < 300):
_note_untracked_end(is_preview)
return
@@ -778,9 +772,10 @@ class LlamaKeepWarmMiddleware:
"more_body", False
)
await send(message)
- # Claim only after the terminal frame is actually delivered: a client that
- # disconnects on the final write makes send() above raise, so completed stays
- # False and the cut-off stream is not mistaken for a clean completion.
+ # claim only after the terminal frame is delivered
+ # Claim only after the terminal frame is actually delivered: a client that disconnects on the final write
+ # makes send() above raise, so completed stays False and the cut-off stream is not mistaken for a clean
+ # completion.
if is_terminal:
completed["done"] = True
_finish()
@@ -794,9 +789,8 @@ class LlamaKeepWarmMiddleware:
def _loaded_identity(backend):
if not backend.is_loaded or not backend.model_identifier:
return None
- # Third slot is the advertised id (repo id) an auto-switch load sets on the
- # backend; it's the override key, so an idle stash keyed by the concrete load
- # path doesn't drop the user's saved launch flags on the alias reload.
+ # Third slot is the advertised id (repo id) an auto-switch load sets on the backend; it's the override key, so an
+ # idle stash keyed by the concrete load path doesn't drop the user's saved launch flags on the alias reload.
advertised = getattr(backend, "_openai_advertised_id", None) or backend.model_identifier
return (backend.model_identifier, getattr(backend, "hf_variant", None), advertised)
@@ -834,24 +828,22 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
seen_model = None
while True:
await asyncio.sleep(poll_seconds)
- # The image/video half of the tick, in its own guard so neither side can cost the
- # other an iteration. Inert unless the media TTL is set.
+ # The image/video half of the tick, in its own guard so neither side can cost the other an iteration. Inert
+ # unless the media TTL is set.
try:
from core.inference.media_keepwarm import idle_unload_step
await idle_unload_step()
except Exception as exc:
logger.debug("media idle_unload_step failed: %s", exc)
try:
- # Keep SQLite-backed setting reads off the event loop.
ttl = await asyncio.to_thread(get_auto_unload_idle_seconds)
if ttl <= 0:
continue
from routes.inference import get_llama_cpp_backend
backend = get_llama_cpp_backend()
- # Track by (id, variant): a (re)loaded model -- including the same repo
- # at a different quant -- counts as activity so it survives one TTL
- # before its first request (loads bypass the activity middleware).
+ # track by (id, variant): a (re)loaded model counts as activity so it survives one TTL before its first
+ # request
async with _unload_gate():
# Purging the stash mid-reload would race the restore.
current = _loaded_identity(backend)
@@ -861,8 +853,7 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
_note_activity()
_set_last_unloaded(None) # a model is loaded; drop stale stash
if backend.is_loaded and await asyncio.to_thread(_user_pinned, backend):
- # Loaded from the UI, so the user wants it resident; only
- # models the API loaded are freed.
+ # loaded from the UI, so the user wants it resident; only models the API loaded are freed
continue
if backend.is_loaded and _is_idle(ttl):
freed = _loaded_identity(backend)
@@ -875,7 +866,6 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
)
except Exception as exc:
logger.debug("slot save before idle unload failed: %s", exc)
- # Re-read settings: the save can outlive a settings change.
ttl = await asyncio.to_thread(get_auto_unload_idle_seconds)
if (
ttl <= 0
@@ -888,8 +878,8 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
if manifest and not await asyncio.to_thread(get_auto_unload_keep_kv):
_delete_resume_files(manifest)
manifest = None
- # A request may register _pending while an off-loop setting read runs.
- # Recheck idleness before unloading.
+ # A request may register _pending while an off-loop setting read runs. Recheck idleness before
+ # unloading.
if not _is_idle(ttl):
if manifest:
_delete_resume_files(manifest)
@@ -897,7 +887,6 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None:
try:
await asyncio.to_thread(backend.unload_model)
except Exception:
- # Failed unload means nothing will stash the manifest.
if manifest:
_delete_resume_files(manifest)
raise
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index 71ce448d966..f4272b64b32 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -20,9 +20,8 @@ from typing import Any, Iterable, Mapping, Optional
logger = logging.getLogger(__name__)
-# Valid llama-server --parallel range, shared with LoadRequest.n_parallel.
-# Mirrored by callers that cannot import this: run.py and unsloth_cli/commands/
-# studio.py (_PARALLEL_MIN/MAX), per-model-config.ts (N_PARALLEL_MIN/MAX);
+# Valid llama-server --parallel range, shared with LoadRequest.n_parallel. Mirrored by callers that cannot import this:
+# run.py and unsloth_cli/commands/ studio.py (_PARALLEL_MIN/MAX), per-model-config.ts (N_PARALLEL_MIN/MAX);
# test_parallel_slots_per_load.py pins them together.
PARALLEL_MIN = 1
PARALLEL_MAX = 64
@@ -31,24 +30,24 @@ PARALLEL_MAX = 64
BATCH_MIN = 1
BATCH_MAX = 65536
-# Sanity bounds, not upstream ones: a stray keystroke fails here rather than in the
-# child. --cache-ram floors at -1 ("no limit"); 0 disables the cache. Mirrored by
-# CTX_CHECKPOINTS_MAX / CACHE_RAM_MAX in per-model-config.ts.
+# Sanity bounds, not upstream ones: a stray keystroke fails here rather than in the child. --cache-ram floors at -1 ("no
+# limit"); 0 disables the cache. Mirrored by CTX_CHECKPOINTS_MAX / CACHE_RAM_MAX in per-model-config.ts.
CTX_CHECKPOINTS_MAX = 256
CACHE_RAM_MAX_MIB = 1024 * 1024
-# Each group = every alias (short + long) of one hard-denied flag.
-# Extend the matching group when llama.cpp adds a new alias.
+# Each group = every alias (short + long) of one hard-denied flag. Extend the matching group when llama.cpp adds a new
+# alias.
_DENYLIST_GROUPS: tuple[frozenset[str], ...] = (
- # Parallel slots: owned by typer --parallel and LoadRequest.n_parallel; a
- # pass-through would desync the slot bookkeeping from llama-server.
+ # Parallel slots: owned by typer --parallel and LoadRequest.n_parallel; a pass-through would desync the slot
+ # bookkeeping from llama-server.
frozenset({"-np", "--parallel", "--n-parallel"}),
- # Model identity: Unsloth resolves it from LoadRequest; a second -m would
- # load a different model than Unsloth thinks it loaded.
+ # Model identity: a second -m would load a different model than Unsloth thinks it loaded
+ # Model identity: Unsloth resolves it from LoadRequest; a second -m would load a different model than Unsloth thinks
+ # it loaded.
frozenset({"-m", "--model"}),
- # Public model id: Unsloth sets a sanitized --alias so the OpenAI API never
- # exposes the local .gguf path. A user-supplied alias is appended after
- # Unsloth's and, with llama.cpp's last-wins parsing, would reintroduce the
+ # Unsloth sets a sanitized --alias so the OpenAI API never exposes the local .gguf path;
+ # Public model id: Unsloth sets a sanitized --alias so the OpenAI API never exposes the local .gguf path. A
+ # user-supplied alias is appended after Unsloth's and, with llama.cpp's last-wins parsing, would reintroduce the
# path leak this is meant to prevent.
frozenset({"-a", "--alias"}),
frozenset({"-mu", "--model-url"}),
@@ -66,15 +65,13 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = (
frozenset({"--path"}),
frozenset({"--api-prefix"}),
frozenset({"--reuse-port"}),
- # Auth / TLS: Unsloth terminates auth; upstream --api-key / TLS shadows
- # Unsloth's key and breaks the proxy hop.
+ # Auth / TLS: Unsloth terminates auth; upstream --api-key / TLS shadows Unsloth's key and breaks the proxy hop
frozenset({"--api-key"}),
frozenset({"--api-key-file"}),
frozenset({"--ssl-key-file"}),
frozenset({"--ssl-cert-file"}),
- # Built-in web UI. --webui/--no-webui is the legacy spelling; upstream
- # renamed to --ui/--no-ui + --ui-*. Keep both so prebuilt and system
- # llama.cpp binaries match.
+ # Built-in web UI. --webui/--no-webui is the legacy spelling; upstream renamed to --ui/--no-ui + --ui-*. Keep both
+ # so prebuilt and system llama.cpp binaries match.
frozenset({"--webui", "--no-webui"}),
frozenset({"--ui", "--no-ui"}),
frozenset({"--ui-config", "--webui-config"}),
@@ -87,42 +84,40 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = (
# Server-mode flips: --embedding is set from the GGUF pooling type at load, not by hand.
frozenset({"--embedding", "--embeddings"}),
frozenset({"--rerank", "--reranking"}),
- # Pooling decides whether the managed embedding launch is safe. A pass-through
- # override appended after --embedding could switch it to NONE or RANK.
+ # Pooling decides whether the managed embedding launch is safe. A pass-through override appended after --embedding
+ # could switch it to NONE or RANK.
frozenset({"--pooling"}),
- # llama-server's own built-in tools flag would silently stack on top of
- # Unsloth's --enable-tools / --disable-tools policy resolver.
+ # llama-server's own tools flag would silently stack on top of Unsloth's --enable-tools / --disable-tools policy
+ # resolver
+ # llama-server's own built-in tools flag would silently stack on top of Unsloth's --enable-tools / --disable-tools
+ # policy resolver.
frozenset({"--tools"}),
- # --agent is --tools by another name: upstream documents it as "enable CORS
- # proxy and ALL built-in tools", and that set includes exec_shell_command.
- # Denying --tools while allowing this left the same capability one alias away.
+ # --agent is --tools by another name: upstream documents it as "enable CORS proxy and ALL built-in tools", and that
+ # set includes exec_shell_command. Denying --tools while allowing this left the same capability one alias away.
frozenset({"-ag", "--agent", "-no-ag", "--no-agent"}),
- # Where those tools run: docker:/podman: spins up a container, ssh:<target>
- # runs them on another host entirely.
+ # Where those tools run: docker:/podman: spins up a container, ssh:<target> runs them on another host
frozenset({"--tools-runtime"}),
- # MCP servers are tools from a config file or an inline JSON blob; upstream
- # says "do not enable in untrusted environments" for both.
+ # MCP servers are tools from a config file or an inline JSON blob; upstream says "do not enable in untrusted
+ # environments" for both.
frozenset({"--mcp-servers-config"}),
frozenset({"--mcp-servers-json"}),
- # CORS: Unsloth terminates browser access at its own origin, so widening the
- # child's would hand a page past the boundary the proxy exists to hold.
+ # CORS: Unsloth terminates browser access at its own origin, so widening the child's would hand a page past the
+ # boundary the proxy exists to hold.
frozenset({"--cors-origins"}),
frozenset({"--cors-headers"}),
frozenset({"--cors-methods"}),
frozenset({"--cors-credentials", "--no-cors-credentials"}),
# Serves local files over the child's HTTP surface.
frozenset({"--media-path"}),
- # Startup output is how _classify_llama_start_failure tells a bad GGUF from an
- # OOM from a rejected flag; redirecting or silencing it makes every failure
- # the same opaque one.
+ # Startup output is how _classify_llama_start_failure tells a bad GGUF from an OOM from a rejected flag; redirecting
+ # or silencing it makes every failure the same opaque one.
frozenset({"--log-file"}),
frozenset({"--log-disable"}),
- # Slot-state dir: Unsloth owns it for KV persistence across idle unload. Endpoint
- # exposure (--slots, --props) is deliberately NOT denied alongside it: Unsloth
- # reads GET /props and never /slots, so either is the user's own call.
+ # Slot-state dir: Unsloth owns it for KV persistence across idle unload. Endpoint exposure (--slots, --props) is
+ # deliberately NOT denied alongside it: Unsloth reads GET /props and never /slots, so either is the user's own call.
frozenset({"--slot-save-path"}),
- # These print and exit instead of serving, so the load would "succeed" with no
- # server behind it and only time out later.
+ # These print and exit instead of serving, so the load would "succeed" with no server behind it and only time out
+ # later
frozenset({"-h", "--help", "--usage"}),
frozenset({"--version"}),
frozenset({"--list-devices"}),
@@ -132,37 +127,33 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = (
_DENYLIST: frozenset[str] = frozenset().union(*_DENYLIST_GROUPS)
-# Flags that take TWO values rather than one. Scanned out of `llama-server --help`:
-# every other option is `--flag VALUE` or a switch, and this list exists so the
-# positional check below does not refuse a legitimate second value.
+# Flags that take TWO values rather than one. Scanned out of `llama-server --help`: every other option is `--flag VALUE`
+# or a switch, and this list exists so the positional check below does not refuse a legitimate second value.
_TWO_VALUE_FLAGS: frozenset[str] = frozenset({"--control-vector-layer-range"})
-# Flags that take a second value on SOME builds. Today's llama.cpp writes the scale
-# into the value ("--lora-scaled FNAME:SCALE"), and older ones took it as a separate
-# token ("--lora-scaled FNAME SCALE"); both spellings are already handled in
-# _sidecar_weight_files. So the second token is allowed but never required: demanding
-# it would refuse the current syntax, and refusing it broke a list that loaded before
-# the positional check existed.
+# Flags taking a second value on SOME builds (llama.cpp)
+# Flags that take a second value on SOME builds. Today's llama.cpp writes the scale into the value ("--lora-scaled
+# FNAME:SCALE"), and older ones took it as a separate token ("--lora-scaled FNAME SCALE"); both spellings are already
+# handled in _sidecar_weight_files. So the second token is allowed but never required: demanding it would refuse the
+# current syntax, and refusing it broke a list that loaded before the positional check existed.
_OPTIONAL_SECOND_VALUE_FLAGS: frozenset[str] = frozenset(
{"--lora-scaled", "--control-vector-scaled"}
)
-# Shape bounds. Not a security boundary -- the denylist is -- but a pasted file or a
-# runaway generator should fail here, naming the limit, rather than at execve or in
-# llama-server's own parser. Generous enough that a grammar or a JSON schema fits.
+# Shape bounds. Not a security boundary -- the denylist is -- but a pasted file or a runaway generator should fail here,
+# naming the limit, rather than at execve or in llama-server's own parser. Generous enough that a grammar or a JSON
+# schema fits.
MAX_EXTRA_ARG_TOKENS = 256
MAX_EXTRA_ARGS_BYTES = 32 * 1024
-# Windows passes CreateProcess ONE string for the whole command line, capped at 32767
-# characters, and the model path, Unsloth's own flags and the quoting subprocess adds
-# all come out of the same budget. So the extras get a smaller share there: accepting
-# the full 32 KiB would pass every check here and then fail inside Popen, after the
-# load had already begun switching models.
+# Windows passes CreateProcess ONE string for the whole command line, capped at 32767 characters, and the model path,
+# Unsloth's own flags and the quoting subprocess adds all come out of the same budget. So the extras get a smaller share
+# there: accepting the full 32 KiB would pass every check here and then fail inside Popen, after the load had already
+# begun switching models.
MAX_EXTRA_ARGS_BYTES_WINDOWS = 24 * 1024
-# CreateProcess takes the whole command line as ONE string, capped here. The rest of
-# the command (the binary, the model path, Unsloth's own flags) has to fit too, so the
-# extras are checked against the limit minus this reserve.
+# CreateProcess takes the whole command line as ONE string, capped here. The rest of the command (the binary, the model
+# path, Unsloth's own flags) has to fit too, so the extras are checked against the limit minus this reserve.
WINDOWS_COMMAND_LIMIT = 32767
WINDOWS_COMMAND_RESERVE = 8192
@@ -245,11 +236,10 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]:
return []
out: list[str] = []
total_bytes = 0
- # How many following tokens the flag just seen may still claim as values. A
- # switch claims none, so the next bare token has no owner.
+ # How many following tokens the flag just seen may still claim as values. A switch claims none, so the next bare
+ # token has no owner.
pending_values = 0
- # Values still owed to a two-value flag, tracked apart because it is the one
- # arity this module knows for certain.
+ # Values still owed to a two-value flag, tracked apart because it is the one arity this module knows for certain
pending_two_value = 0
two_value_flag = ""
for raw in args:
@@ -258,12 +248,10 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]:
raise ValueError(
f"too many extra llama-server args (limit {MAX_EXTRA_ARG_TOKENS} tokens)"
)
- # A grammar or JSON schema is a legitimately long single token, so the cap
- # is on the whole list rather than per token.
- # Strictly, unlike the sizing below: JSON and the browser can both carry an
- # unpaired surrogate, which survives every check here and then makes
- # subprocess.Popen raise while it encodes argv, long after the load has begun
- # switching models. Refused at the boundary, where it is still a 400.
+ # A grammar or JSON schema is a legitimately long single token, so the cap is on the whole list rather than per
+ # token. Strictly, unlike the sizing below: JSON and the browser can both carry an unpaired surrogate, which
+ # survives every check here and then makes subprocess.Popen raise while it encodes argv, long after the load has
+ # begun switching models. Refused at the boundary, where it is still a 400.
try:
encoded = token.encode("utf-8")
except UnicodeEncodeError as error:
@@ -274,8 +262,8 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]:
limit = max_extra_args_bytes()
if total_bytes > limit:
raise ValueError(f"extra llama-server args are too large (limit {limit} bytes)")
- # execve rejects a NUL outright; the rest would reach the child's parser as
- # invisible characters and be blamed on the flag they are attached to.
+ # execve rejects a NUL outright; the rest would reach the child's parser as invisible characters blamed on the
+ # flag they are attached to
if _has_control_characters(token):
raise ValueError("extra llama-server args cannot contain control characters")
flag = _flag_name(token)
@@ -284,17 +272,19 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]:
f"llama-server flag '{flag}' is managed by Unsloth Studio "
f"and cannot be passed as an extra arg"
)
- # Why (#9510): users reaching for `--parallel 1` to cap concurrent predictions on a
- # local model hit this refusal with no pointer to the supported knob; name it.
+ # #9510: users reaching for `--parallel 1` hit this refusal with no pointer to the supported knob
+ # Why (#9510): users reaching for `--parallel 1` to cap concurrent predictions on a local model hit this
+ # refusal with no pointer to the supported knob; name it.
if flag in {"-np", "--parallel", "--n-parallel"}:
message += "; set n_parallel on the load request (parallel decode slots) instead"
raise ValueError(message)
if flag is None:
- # A token belonging to no flag. Today's llama-server answers "invalid
- # argument" and refuses to start, which is a failed load rather than a
- # 400, and a build that did accept a positional would read it as the
- # model path: that is the one thing the -m / --model denial exists to
- # prevent, and it would sidestep the native-path lease as well.
+ # A token belonging to no flag: llama-server answers "invalid argument" and refuses to start (a failed load,
+ # not a 400)
+ # A token belonging to no flag. Today's llama-server answers "invalid argument" and refuses to start, which
+ # is a failed load rather than a 400, and a build that did accept a positional would read it as the model
+ # path: that is the one thing the -m / --model denial exists to prevent, and it would sidestep the
+ # native-path lease as well.
if pending_values <= 0:
raise ValueError(
"extra llama-server args cannot contain a bare value "
@@ -304,27 +294,19 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]:
if pending_two_value > 0:
pending_two_value -= 1
elif token != token.strip():
- # _flag_name strips before it looks anything up, so a quoted "--top-k "
- # passed the denylist and the arity walk as --top-k and then went to the
- # child with the space still on it. llama.cpp looks the whole token up,
- # so it answers "error: invalid argument: --top-k" (measured on b10342),
- # naming a flag that looks correct in the log. Only flag-shaped tokens:
- # a VALUE may legitimately end in whitespace, a chat template or a
- # grammar being the obvious ones.
+ # _flag_name strips before lookup, so a quoted "--top-k " passed the denylist
raise ValueError(
f"llama-server does not accept the spaces around '{token[:64]}': "
f"write it as '{flag}'"
)
elif "=" in token:
- # llama.cpp looks the WHOLE token up in its option map, folding only the
- # underscore spelling, so "--top-k=20" is not "--top-k" with a value: it
- # is an argument it has never heard of. Measured on b10342 and b10360,
- # where --top-k=20, --ctx-size=4096 and --flash-attn=on each exit with
- # "error: invalid argument". Accepting the GNU spelling here meant the
- # switch tore down the resident model and the child then refused to
- # start, so it is refused while it is still a 400 with somewhere to go.
- # Splitting it here would be a guess: for a switch the value is not one,
- # and this module cannot know an ordinary flag's arity.
+ # llama.cpp looks the WHOLE token up, folding only the underscore spelling (b10342, b10360)
+ # llama.cpp looks the WHOLE token up in its option map, folding only the underscore spelling, so
+ # "--top-k=20" is not "--top-k" with a value: it is an argument it has never heard of. Measured on b10342
+ # and b10360, where --top-k=20, --ctx-size=4096 and --flash-attn=on each exit with "error: invalid
+ # argument". Accepting the GNU spelling here meant the switch tore down the resident model and the child
+ # then refused to start, so it is refused while it is still a 400 with somewhere to go. Splitting it here
+ # would be a guess: for a switch the value is not one, and this module cannot know an ordinary flag's arity.
value = token.partition("=")[2]
raise ValueError(
f"llama-server does not read an attached value: write '{flag}' and "
@@ -335,15 +317,15 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]:
attached = _value_is_attached(token, flag)
if pending_two_value > 0:
raise ValueError(f"llama-server flag '{two_value_flag}' takes two values")
- # An attached value is ONE of the two, not the whole option:
- # "--control-vector-layer-range=1" still owes its END, and
- # llama-server exits on the incomplete option.
+ # An attached value is ONE of the two: "--control-vector-layer-range=1" still owes its END and llama-server
+ # exits on the incomplete option
+ # An attached value is ONE of the two, not the whole option: "--control-vector-layer-range=1" still owes its
+ # END, and llama-server exits on the incomplete option.
if flag in _TWO_VALUE_FLAGS:
pending_values = 1 if attached else 2
pending_two_value = pending_values
elif flag in _OPTIONAL_SECOND_VALUE_FLAGS:
- # Allowed, not owed: pending_two_value stays 0, so nothing here
- # insists on the second token.
+ # Allowed, not owed: pending_two_value stays 0, so nothing insists on the second token
pending_values = 1 if attached else 2
pending_two_value = 0
else:
@@ -352,9 +334,9 @@ def validate_extra_args(args: Optional[Iterable[str]]) -> list[str]:
two_value_flag = flag
out.append(token)
if pending_two_value > 0:
- # Only this shape is checkable: an ordinary flag's arity is unknown here, so
- # a list ending in one is left to llama-server. START without END is a launch
- # that fails on the command line rather than a request that fails here.
+ # Only this shape is checkable: an ordinary flag's arity is unknown here, so a list ending in one is left to
+ # llama-server. START without END is a launch that fails on the command line rather than a request that fails
+ # here.
raise ValueError(f"llama-server flag '{two_value_flag}' takes two values")
if sys.platform == "win32":
# After the per-token walk, because this is a property of the whole list.
@@ -404,7 +386,6 @@ def drop_managed_flags(args: Optional[Iterable[str]]) -> tuple[list[str], list[s
return False
seq = tokens if source is None else source
if source is not None:
- # Called on the last surviving token, whose value was the token just shed.
return True
following = seq[index + 1] if index + 1 < len(seq) else None
return following is not None and _flag_name(following) is None
@@ -421,20 +402,18 @@ def drop_managed_flags(args: Optional[Iterable[str]]) -> tuple[list[str], list[s
dropped.append(flag)
skip_next = _takes_next(index, token, flag)
continue
- # A control character never reached the child as anything but noise, and a
- # NUL never reached it at all (execve refuses). A poisoned VALUE takes its
- # flag with it for the same reason a denied flag takes its value: a flag
+ # A poisoned VALUE takes its flag with it for the same reason a denied flag takes its value
+ # A control character never reached the child as anything but noise, and a NUL never reached it at all (execve
+ # refuses). A poisoned VALUE takes its flag with it for the same reason a denied flag takes its value: a flag
# left expecting one would eat the next token and change what that means.
if _has_control_characters(token) or not _is_spawnable(token):
- # A placeholder either way: this list is joined into a warning log, and
- # the unusable characters are in the token itself, so echoing its name
- # would rewrite whatever is reading that log just as echoing its value
+ # A placeholder either way: this list is joined into a warning log, and the unusable characters are in the
+ # token itself, so echoing its name would rewrite whatever is reading that log just as echoing its value
# would.
dropped.append("<flag>" if flag is not None else "<value>")
if flag is not None:
- # Its value goes too, exactly as a denied flag's does: an orphan left
- # behind is a bare positional, which llama-server reads as the model
- # path.
+ # Its value goes too, exactly as a denied flag's does: an orphan left behind is a bare positional, which
+ # llama-server reads as the model path.
skip_next = _takes_next(index, token, flag)
elif kept:
owner = _flag_name(kept[-1])
@@ -443,18 +422,15 @@ def drop_managed_flags(args: Optional[Iterable[str]]) -> tuple[list[str], list[s
kept.pop()
continue
if flag is not None and "=" in token:
- # An attached value llama-server refuses outright, whatever the flag. Dropped
- # here with the denied names rather than left to the trimming loop below:
- # that loop sheds the TAIL, so one legacy "--top-k=20" in the middle would
- # cost every flag written after it. Nothing to skip, the value is in the
- # token. After the control-character check, so a poisoned name is still
- # logged as a placeholder rather than echoed.
+ # An attached value llama-server refuses outright, whatever the flag. Dropped here with the denied names
+ # rather than left to the trimming loop below: that loop sheds the TAIL, so one legacy "--top-k=20" in the
+ # middle would cost every flag written after it. Nothing to skip, the value is in the token. After the
+ # control-character check, so a poisoned name is still logged as a placeholder rather than echoed.
dropped.append(flag)
continue
if flag is not None and token != token.strip():
- # Refused for the same reason, and its value goes with it: the padding is
- # part of the token llama.cpp looks up, so the flag never arrives and the
- # value it was written for would be left as a bare positional.
+ # Refused for the same reason, and its value goes with it: the padding is part of the token llama.cpp looks
+ # up, so the flag never arrives and the value it was written for would be left as a bare positional.
dropped.append(flag)
skip_next = _takes_next(index, token, flag)
continue
@@ -463,9 +439,8 @@ def drop_managed_flags(args: Optional[Iterable[str]]) -> tuple[list[str], list[s
and _takes_next(index, token, flag)
and (_has_control_characters(tokens[index + 1]) or not _is_spawnable(tokens[index + 1]))
):
- # Recorded, not just skipped: the value is about to be dropped for its
- # control characters, and a flag that vanished without a word in the log
- # is the harder half of that to explain afterwards.
+ # Recorded, not just skipped: a flag that vanished without a word in the log is the harder half to explain
+ # afterwards
dropped.append(flag)
continue
kept.append(token)
@@ -474,23 +449,19 @@ def drop_managed_flags(args: Optional[Iterable[str]]) -> tuple[list[str], list[s
try:
return validate_extra_args(kept), dropped
except ValueError:
- # Only the bounds can still fail here, and they are about length, so the
- # tail is the right thing to shed. A flag whose value has just gone with
- # it goes too: `['--grammar', <33 KiB>]` trimmed to `['--grammar']` is
- # syntactically valid to this validator, which knows the arity of only a
- # few flags, and llama-server then refuses the launch over a flag with
- # no value.
- # Names, not values: this list goes into a log line, and the token that
- # broke the bound is by definition enormous.
+ # Only the bounds can still fail here, and they are about length, so the tail is the right thing to shed. A
+ # flag whose value has just gone with it goes too: `['--grammar', <33 KiB>]` trimmed to `['--grammar']` is
+ # syntactically valid to this validator, which knows the arity of only a few flags, and llama-server then
+ # refuses the launch over a flag with no value. Names, not values: this list goes into a log line, and the
+ # token that broke the bound is by definition enormous.
dropped.append(_flag_name(kept[-1]) or "<value>")
kept = kept[:-1]
last_flag = _flag_name(kept[-1]) if kept else None
if last_flag is not None and _takes_next(len(kept) - 1, kept[-1], last_flag, kept):
dropped.append(last_flag)
kept = kept[:-1]
- # A two-value flag loses the whole option rather than half of it: one
- # value left behind is a command llama-server refuses at startup, which
- # is the failure this trimming exists to avoid.
+ # A two-value flag loses the whole option rather than half of it: one value left behind is a command
+ # llama-server refuses at startup, which is the failure this trimming exists to avoid.
while len(kept) >= 2:
owner = _flag_name(kept[-2])
if (
@@ -518,9 +489,8 @@ def is_managed_flag(flag: str) -> bool:
return normalised is not None and normalised in _DENYLIST
-# Pass-through flags that shadow first-class LoadRequest fields; stripped
-# from inherited extras so they can't last-wins-override an Apply that
-# re-sets the same field.
+# Pass-through flags that shadow first-class LoadRequest fields; stripped from inherited extras so they can't
+# last-wins-override an Apply that re-sets the same field.
_CONTEXT_FLAGS: frozenset[str] = frozenset({"-c", "--ctx-size"})
_CACHE_TYPE_K_FLAGS: frozenset[str] = frozenset({"-ctk", "--cache-type-k"})
_CACHE_TYPE_V_FLAGS: frozenset[str] = frozenset({"-ctv", "--cache-type-v"})
@@ -533,16 +503,14 @@ _SPEC_FLAGS: frozenset[str] = frozenset(
"--spec-ngram-size",
"--draft-min",
"--draft-max",
- # MTP path (llama.cpp #22673). The drafter selectors (local --model-draft
- # and HF --spec-draft-hf aliases) are Unsloth-managed since the separate-
- # drafter support (Gemma 4): an inherited copy must not last-wins-override
- # the auto-detected drafter. Explicit extras for the current load are never
- # stripped. The per-drafter tuning knobs (-ngld, --spec-draft-device) are
- # deliberately NOT stripped: the VRAM budget reads them via the same parsers
- # the child honors, so they stay consistent on inherit, and stripping them
- # would silently move a CPU-offloaded drafter back onto the GPU. The draft
- # cache dtype is in that group too, and has its own toggle used only when
- # spec_draft_cache_type is set, the same rule the batch pair follows.
+ # MTP path (llama.cpp #22673). explicit extras for the current load are never stripped. MTP path (llama.cpp
+ # #22673). The drafter selectors (local --model-draft and HF --spec-draft-hf aliases) are Unsloth-managed since
+ # the separate- drafter support (Gemma 4): an inherited copy must not last-wins-override the auto-detected
+ # drafter. Explicit extras for the current load are never stripped. The per-drafter tuning knobs (-ngld,
+ # --spec-draft-device) are NOT stripped: the VRAM budget reads them via the same parsers the child honors, so
+ # they stay consistent on inherit, and stripping them would silently move a CPU-offloaded drafter back onto the
+ # GPU. The draft cache dtype is in that group too, and has its own toggle used only when spec_draft_cache_type
+ # is set, the same rule the batch pair follows.
"--model-draft",
"-md",
"--spec-draft-model",
@@ -568,38 +536,35 @@ _TEMPLATE_FLAGS: frozenset[str] = frozenset(
"--no-jinja",
}
)
-# Multi-GPU split mode shadows the Tensor Parallelism toggle
-# (--split-mode tensor). Pass-through stays allowed so users keep the
-# row/none/layer modes the toggle doesn't expose, but it's stripped on
-# inherit and reconciled into the round-tripped tensor_parallel state.
-# --tensor-split is coupled to the split mode and is stripped with it: Unsloth
-# owns the tensor-mode split ratios, so an inherited/stale --tensor-split must
-# not last-wins-override Unsloth's computed asymmetric split.
+# Multi-GPU split mode shadows the Tensor Parallelism toggle (--split-mode tensor). Pass-through stays allowed so users
+# keep the row/none/layer modes the toggle doesn't expose, but it's stripped on inherit and reconciled into the
+# round-tripped tensor_parallel state. --tensor-split is coupled to the split mode and is stripped with it: Unsloth owns
+# the tensor-mode split ratios, so an inherited/stale --tensor-split must not last-wins-override Unsloth's computed
+# asymmetric split.
_SPLIT_MODE_FLAGS: frozenset[str] = frozenset({"-sm", "--split-mode"})
_TENSOR_SPLIT_FLAGS: frozenset[str] = frozenset({"-ts", "--tensor-split"})
_SPLIT_SHADOWING_FLAGS: frozenset[str] = _SPLIT_MODE_FLAGS | _TENSOR_SPLIT_FLAGS
-# llama.cpp placement flags. Opt-in (users may pass them under auto-select):
-# stripped only when gpu_ids is set, so they cannot override the selected pool
-# or choose a main GPU outside it (#7188).
+# llama.cpp placement flags (#7188)
+# llama.cpp placement flags. Opt-in (users may pass them under auto-select): stripped only when gpu_ids is set, so they
+# cannot override the selected pool or choose a main GPU outside it (#7188).
_DEVICE_FLAGS: frozenset[str] = frozenset({"--device", "-dev", "--main-gpu", "-mg"})
-# GPU-offload flags. Stripped only when the GPU Memory mode owns offload
-# (manual emits --fit / --gpu-layers / --n-cpu-moe); in auto, a user's
-# inherited -ngl is respected (the offload_overridden path), so this group is
-# opt-in, not default. Layer flags are shared with llama_cpp's override
-# detection; the MoE flags are strip-only (manual's --n-cpu-moe slider owns them).
+# GPU-offload flags. in auto a user's inherited -ngl is respected.
+# GPU-offload flags. Stripped only when the GPU Memory mode owns offload (manual emits --fit / --gpu-layers /
+# --n-cpu-moe); in auto, a user's inherited -ngl is respected (the offload_overridden path), so this group is opt-in,
+# not default. Layer flags are shared with llama_cpp's override detection; the MoE flags are strip-only (manual's
+# --n-cpu-moe slider owns them).
_GPU_LAYER_FLAGS: frozenset[str] = frozenset({"-ngl", "--gpu-layers", "--n-gpu-layers"})
# inherited copies of these shadow n_batch / n_ubatch, stripped only when the field is set
_BATCH_FLAGS: frozenset[str] = frozenset({"-b", "--batch-size"})
_UBATCH_FLAGS: frozenset[str] = frozenset({"-ub", "--ubatch-size"})
-# Same rule for the tuning group: stripped only when its field is supplied.
-# --swa-checkpoints is upstream's older spelling of --ctx-checkpoints.
+# Same rule for the tuning group: stripped only when its field is supplied. --swa-checkpoints is upstream's older
+# spelling of --ctx-checkpoints
_CTX_CHECKPOINTS_FLAGS: frozenset[str] = frozenset(
{"-ctxcp", "--ctx-checkpoints", "--swa-checkpoints"}
)
_CACHE_RAM_FLAGS: frozenset[str] = frozenset({"-cram", "--cache-ram"})
-# One group: the control sets a single dtype, so an inherited pair that split K
-# from V has to go whole.
+# One group: the control sets a single dtype, so an inherited pair that split K from V has to go whole
_SPEC_DRAFT_CACHE_K_FLAGS: frozenset[str] = frozenset(
{"-ctkd", "--cache-type-k-draft", "--spec-draft-type-k"}
)
@@ -608,34 +573,33 @@ _SPEC_DRAFT_CACHE_V_FLAGS: frozenset[str] = frozenset(
)
_SPEC_DRAFT_CACHE_FLAGS: frozenset[str] = _SPEC_DRAFT_CACHE_K_FLAGS | _SPEC_DRAFT_CACHE_V_FLAGS
_FIT_FLAGS: frozenset[str] = frozenset({"-fit", "--fit"})
-# The fitter's per-device margin. Never stripped (llama.cpp is last-wins), so a
-# pass-through value is what the child really keeps free; see fit_target_margin_in.
+# The fitter's per-device margin.
+# The fitter's per-device margin. Never stripped (llama.cpp is last-wins), so a pass-through value is what the child
+# really keeps free; see fit_target_margin_in.
_FIT_TARGET_FLAGS: frozenset[str] = frozenset({"-fitt", "--fit-target"})
_LAYER_OFFLOAD_FLAGS: frozenset[str] = _GPU_LAYER_FLAGS | _FIT_FLAGS
_MOE_OFFLOAD_FLAGS: frozenset[str] = frozenset({"-ncmoe", "--n-cpu-moe", "-cmoe", "--cpu-moe"})
_OFFLOAD_SHADOWING_FLAGS: frozenset[str] = _LAYER_OFFLOAD_FLAGS | _MOE_OFFLOAD_FLAGS
-# Host-memory placement flags. Both are full-model RAM reservations (--mlock pins
-# it, --no-mmap mallocs a copy), so the Model Memory settings own them: stripped
-# only when a toggle vetoes them, never unconditionally.
+# Host-memory placement flags. Both are full-model RAM reservations (--mlock pins it, --no-mmap mallocs a copy), so the
+# Model Memory settings own them: stripped only when a toggle vetoes them, never unconditionally.
_MLOCK_FLAGS: frozenset[str] = frozenset({"--mlock", "-mlock"})
# Modern spelling of both, as an enum value. Takes a value, so NOT boolean.
_LOAD_MODE_FLAGS: frozenset[str] = frozenset({"--load-mode", "-lm"})
_NO_MMAP_FLAGS: frozenset[str] = frozenset({"--no-mmap", "-no-mmap"})
-# Deprecated selectors for the same load-mode enum. Measured: ANY of them
-# trailing the managed flag resets the WHOLE mode and drops the mlock, in both
-# polarities ("--mmap" and "--no-direct-io" do it too). Affirmative dio streams
-# and holds no full copy; the negative spellings are NOT plain mmap, upstream
-# maps them to mode `none` like --no-mmap, so no-reserve must veto those too.
+# Deprecated selectors for the same enum. Affirmative dio streams and holds no full copy;
+# Deprecated selectors for the same load-mode enum. Measured: ANY of them trailing the managed flag resets the WHOLE
+# mode and drops the mlock, in both polarities ("--mmap" and "--no-direct-io" do it too). Affirmative dio streams and
+# holds no full copy; the negative spellings are NOT plain mmap, upstream maps them to mode `none` like --no-mmap, so
+# no-reserve must veto those too.
_DIO_ON_FLAGS: frozenset[str] = frozenset({"--direct-io", "-dio"})
_DIO_OFF_FLAGS: frozenset[str] = frozenset({"--no-direct-io", "-ndio"})
_DIO_FLAGS: frozenset[str] = _DIO_ON_FLAGS | _DIO_OFF_FLAGS
_LOAD_MODE_ALIAS_FLAGS: frozenset[str] = _NO_MMAP_FLAGS | frozenset({"--mmap"}) | _DIO_FLAGS
# Every spelling that asks for a full-model host buffer.
_RAM_RESERVING_FLAGS: frozenset[str] = _NO_MMAP_FLAGS | _DIO_OFF_FLAGS
-# llama.cpp reads these before argv, so an inherited value survives stripping the
-# equivalent tokens. Scrubbed whenever a toggle is on, like the spec/placement
-# env groups, so the setting owns memory placement outright.
+# llama.cpp reads these before argv, so an inherited value survives stripping the equivalent tokens. Scrubbed whenever a
+# toggle is on, like the spec/placement env groups, so the setting owns memory placement outright.
MEMORY_ENV_VARS: tuple[str, ...] = (
"LLAMA_ARG_MLOCK",
"LLAMA_ARG_MMAP",
@@ -724,7 +688,6 @@ def parse_ctx_checkpoints_override(args: Optional[Iterable[str]]) -> Optional[in
try:
parsed = int(str(value).strip())
except ValueError:
- # Malformed extras are refused at the boundary; sizing must not raise here.
return None
return max(0, parsed)
@@ -764,7 +727,6 @@ def matches_explicit_ctx_override(args: Optional[Iterable[str]], n_ctx: Any) ->
try:
return parse_ctx_override(args) == n_ctx
except ValueError:
- # Malformed extras are refused at the boundary; this must not raise here.
return False
@@ -916,8 +878,8 @@ def fit_target_margin_in(
if raw_value is None:
return None
values: list[float] = []
- # Upstream splits on both "," and "/" (common/arg.cpp), so "4096/4096" is a
- # well-formed two-device margin; reading it as one token would wrongly abstain.
+ # Upstream splits on both "," and "/" (common/arg.cpp), so "4096/4096" is a well-formed two-device margin; reading
+ # it as one token would wrongly abstain.
for part in str(raw_value).replace("/", ",").split(","):
part = part.strip()
if not part:
@@ -925,7 +887,6 @@ def fit_target_margin_in(
try:
values.append(float(part))
except ValueError:
- # Upstream rejects the whole list, so abstain rather than price part of it.
return None
return max(values) if values else None
@@ -965,7 +926,6 @@ def split_policy_starves_devices(
if raw_split is None:
return False
holding = 0
- # Upstream splits on both "," and "/" (common/arg.cpp).
for part in str(raw_split).replace("/", ",").split(",")[:n_credited]:
part = part.strip()
if not part:
@@ -974,7 +934,6 @@ def split_policy_starves_devices(
if float(part) > 0.0:
holding += 1
except ValueError:
- # Upstream throws on this and the child never starts: nothing to misprice.
return False
return holding < n_credited
@@ -1191,8 +1150,7 @@ def strip_shadowing_flags(
out.append(tok)
i += 1
continue
- # Drop the flag; also consume the next token unless it's boolean,
- # already inline (`-c=4096`), or another flag.
+ # Drop the flag; also consume the next token unless it's boolean, already inline (`-c=4096`)
if flag in _BOOLEAN_SHADOWING_FLAGS or "=" in tok:
i += 1
elif i + 1 < n and _flag_name(tokens[i + 1]) is None:
@@ -1270,9 +1228,8 @@ def apply_model_memory_policy(
# Settings unavailable (bare unit-test import): behave as before.
return [], list(extra_args or [])
- # One snapshot for both decisions: read separately, a save landing between
- # them strips for one setting and locks for the other, so a saved --mlock
- # could survive a committed no-reserve.
+ # One snapshot for both decisions: read separately, a save landing between them strips for one setting and locks for
+ # the other, so a saved --mlock could survive a committed no-reserve.
keep_resident, no_ram_reserve = get_model_memory_settings()
tokens = list(extra_args or [])
if no_ram_reserve:
@@ -1290,8 +1247,8 @@ def apply_model_memory_policy(
managed: list[str] = []
if keep_resident and not no_ram_reserve and weights_in_host_memory:
- # Before the extras, like the rest of the managed block. mmap+mlock, not
- # bare mlock: it matches what --mlock meant alongside the default mmap.
+ # Before the extras, like the rest of the managed block. mmap+mlock, not bare mlock: it matches what --mlock
+ # meant alongside the default mmap.
managed.extend(["--load-mode", "mmap+mlock"] if supports_load_mode else ["--mlock"])
tokens = strip_shadowing_flags(
tokens,
@@ -1353,16 +1310,14 @@ def apply_load_mode_policy(
)
return [], tokens
if not supports_load_mode:
- # A build predating the enum understands the spellings it replaced, and
- # only for the values that had one.
+ # A build predating the enum understands the spellings it replaced, and only for the values that had one.
legacy = _LEGACY_LOAD_MODE_FLAGS.get(mode)
if not legacy:
logger.info("llama-server has no --load-mode; skipping the requested %r mode.", mode)
return [], tokens
return list(legacy), tokens
- # Emitted BEFORE the extras and stripping nothing, like every other control
- # here: a flag typed for THIS load is appended after and last-wins, which is
- # what the panel's diagnostics promise. An INHERITED copy is a different
+ # Emitted BEFORE the extras and stripping nothing, like every other control here: a flag typed for THIS load is
+ # appended after and last-wins, which is what the panel's diagnostics promise. An INHERITED copy is a different
# thing, and the route drops that one before it ever reaches here.
return ["--load-mode", mode], tokens
@@ -1428,22 +1383,19 @@ def _env_var_locks_or_reserves(name: str, value: str) -> bool:
if name == "LLAMA_ARG_MLOCK":
return normalized in _ENV_TRUE_VALUES
if name in {"LLAMA_ARG_NO_MMAP", "LLAMA_ARG_NO_DIO"}:
- # Presence alone selects mode "none", which is a full host buffer.
return True
if name in {"LLAMA_ARG_MMAP", "LLAMA_ARG_DIO"}:
- # Falsy selects "none"; truthy selects mmap / dio, neither of which
- # holds a full copy.
+ # Falsy selects "none"; truthy selects mmap / dio, neither of which holds a full copy.
return normalized in _ENV_FALSE_VALUES
if name == "LLAMA_ARG_LOAD_MODE":
return normalized in _LOAD_MODE_MLOCK_VALUES or normalized in _LOAD_MODE_RESERVING_VALUES
return False
-# The LLAMA_ARG_* twins of flags the denylist refuses. llama.cpp reads these before
-# argv, so a name refused in extra args is still reachable through the environment
-# Unsloth's own process inherits. Anyone who can set that environment can already do
-# worse, so this is not the boundary -- it just stops a denied flag arriving by the
-# back door and leaving no trace in the recorded command.
+# The LLAMA_ARG_* twins of flags the denylist refuses. llama.cpp reads these before argv, so a name refused in extra
+# args is still reachable through the environment Unsloth's own process inherits. Anyone who can set that environment
+# can already do worse, so this is not the boundary -- it just stops a denied flag arriving by the back door and leaving
+# no trace in the recorded command.
DENIED_ENV_VARS: tuple[str, ...] = (
"LLAMA_ARG_TOOLS",
"LLAMA_ARG_TOOLS_RUNTIME",
@@ -1455,39 +1407,38 @@ DENIED_ENV_VARS: tuple[str, ...] = (
"LLAMA_ARG_CORS_METHODS",
"LLAMA_ARG_CORS_CREDENTIALS",
"LLAMA_ARG_MEDIA_PATH",
- # The twins of --log-file and --log-disable. Unsloth classifies a failed start by
- # reading llama-server's own output, so an inherited redirect leaves every
- # failure looking like the same opaque one; and unlike the flags, Unsloth emits
- # nothing later that would override these. LLAMA_ARG_LOG_DISABLE has no twin in
- # today's builds, and is listed so it cannot arrive as one.
+ # Twins of --log-file and --log-disable: Unsloth classifies a failed start by reading llama-server's output
+ # The twins of --log-file and --log-disable. Unsloth classifies a failed start by reading llama-server's own output,
+ # so an inherited redirect leaves every failure looking like the same opaque one; and unlike the flags, Unsloth
+ # emits nothing later that would override these. LLAMA_ARG_LOG_DISABLE has no twin in today's builds, and is listed
+ # so it cannot arrive as one.
"LLAMA_ARG_LOG_FILE",
"LLAMA_ARG_LOG_DISABLE",
- # --api-prefix moves every endpoint, including the /health Unsloth waits on, so an
- # inherited one turns every load into a timeout.
+ # --api-prefix moves every endpoint, including the /health Unsloth waits on, so an inherited one turns every load
+ # into a timeout.
"LLAMA_ARG_API_PREFIX",
- # --api-key and its file. Unsloth terminates auth itself and sends the child no
- # Authorization header, so an inherited key makes the healthy child refuse every
- # request. The bundled build reads LLAMA_API_KEY for the flag and
- # LLAMA_ARG_API_KEY_FILE for the file; the third spelling is listed because the
- # name has moved between releases and none of them is ours to honour.
+ # --api-key and its file. Unsloth terminates auth itself and sends the child no Authorization header, so an
+ # inherited key makes the healthy child refuse every request. The bundled build reads LLAMA_API_KEY for the flag and
+ # LLAMA_ARG_API_KEY_FILE for the file; the third spelling is listed because the name has moved between releases and
+ # none of them is ours to honour.
"LLAMA_API_KEY",
"LLAMA_ARG_API_KEY",
"LLAMA_ARG_API_KEY_FILE",
- # The twins of --ssl-key-file and --ssl-cert-file. Given both, llama-server
- # listens on https, while Unsloth probes /health and proxies over http against
- # the port it launched: the child comes up healthy and every load times out.
- # Measured on b10360, where an inherited pair turns "listening on
- # http://127.0.0.1:PORT" into "listening on https://...".
+ # Twins of --ssl-key-file / --ssl-cert-file (b10360)
+ # The twins of --ssl-key-file and --ssl-cert-file. Given both, llama-server listens on https, while Unsloth probes
+ # /health and proxies over http against the port it launched: the child comes up healthy and every load times out.
+ # Measured on b10360, where an inherited pair turns "listening on http://127.0.0.1:PORT" into "listening on
+ # https://...".
"LLAMA_ARG_SSL_KEY_FILE",
"LLAMA_ARG_SSL_CERT_FILE",
- # The rest of the twins its --help documents for a denied flag, enumerated from
- # the bundled b10342 help rather than picked one at a time: every "(env: NAME)"
- # whose option this module refuses. Unsloth emits most of these itself and argv
- # wins over the environment, so removing them changes nothing in the ordinary
- # case; they are here for the paths where it does not, and so a flag denied in
- # the box is not reachable through the environment instead. The mapping below
- # records which flag each one belongs to, since the name does not always say
- # (LLAMA_ARG_STATIC_PATH is --path).
+ # The rest of the twins --help documents for a denied flag, enumerated from the bundled b10342 help rather than
+ # picked one at a time.
+ # The rest of the twins its --help documents for a denied flag, enumerated from the bundled b10342 help rather than
+ # picked one at a time: every "(env: NAME)" whose option this module refuses. Unsloth emits most of these itself and
+ # argv wins over the environment, so removing them changes nothing in the ordinary case; they are here for the paths
+ # where it does not, and so a flag denied in the box is not reachable through the environment instead. The mapping
+ # below records which flag each one belongs to, since the name does not always say (LLAMA_ARG_STATIC_PATH is
+ # --path).
"LLAMA_ARG_MODEL",
"LLAMA_ARG_MODEL_URL",
"LLAMA_ARG_DOCKER_REPO",
@@ -1501,33 +1452,30 @@ DENIED_ENV_VARS: tuple[str, ...] = (
"LLAMA_ARG_POOLING",
"LLAMA_ARG_EMBEDDINGS",
"LLAMA_ARG_RERANKING",
- # The web UI and its MCP proxy, which upstream marks as not for untrusted
- # environments, and the directory the child serves files from.
+ # The web UI and its MCP proxy, which upstream marks as not for untrusted environments, and the directory the child
+ # serves files from.
"LLAMA_ARG_UI",
"LLAMA_ARG_UI_CONFIG",
"LLAMA_ARG_UI_CONFIG_FILE",
"LLAMA_ARG_UI_MCP_PROXY",
"LLAMA_ARG_STATIC_PATH",
- # Deliberately absent: LLAMA_ARG_MMPROJ and LLAMA_ARG_MMPROJ_URL. --mmproj is
- # refused in the box because Unsloth resolves the projector itself, but the
- # environment twin is an INPUT here: _launch_has_mmproj reads both to know the
- # launch has a projector at all, which is what keeps the vision and audio state
- # of a model loaded through an inherited one. Only the paravirtual CPU recovery
- # drops them, where an unpinned projector is the corrupt path it is undoing.
- # The pooling twins are absent for the opposite reason: load_model already pops
- # LLAMA_ARG_POOLING / _RERANKING / _EMBEDDINGS itself, next to where it decides
- # what the GGUF header says.
- # The multi-model server mode: a child holding its own model directory, preset
- # and autoload policy is not the single model Unsloth launched and accounts for.
+ # Deliberately absent: LLAMA_ARG_MMPROJ and LLAMA_ARG_MMPROJ_URL. --mmproj is refused in the box because Unsloth
+ # resolves the projector itself, but the environment twin is an INPUT here: _launch_has_mmproj reads both to know
+ # the launch has a projector at all, which is what keeps the vision and audio state of a model loaded through an
+ # inherited one. Only the paravirtual CPU recovery drops them, where an unpinned projector is the corrupt path it is
+ # undoing. The pooling twins are absent for the opposite reason: load_model already pops LLAMA_ARG_POOLING /
+ # _RERANKING / _EMBEDDINGS itself, next to where it decides what the GGUF header says. The multi-model server mode:
+ # a child holding its own model directory, preset and autoload policy is not the single model Unsloth launched and
+ # accounts for.
"LLAMA_ARG_MODELS_DIR",
"LLAMA_ARG_MODELS_PRESET",
"LLAMA_ARG_MODELS_MAX",
"LLAMA_ARG_MODELS_AUTOLOAD",
)
-# Which flag each twin belongs to, for the drift test. Derived by name for most of
-# them, but not all: LLAMA_ARG_STATIC_PATH is --path, LLAMA_API_KEY is --api-key, and
-# a rename upstream would otherwise leave a variable here guarding nothing.
+# Which flag each twin belongs to, for the drift test. Derived by name for most of them, but not all:
+# LLAMA_ARG_STATIC_PATH is --path, LLAMA_API_KEY is --api-key, and a rename upstream would otherwise leave a variable
+# here guarding nothing.
DENIED_ENV_TWIN_FLAGS: dict[str, str] = {
"LLAMA_ARG_STATIC_PATH": "--path",
"LLAMA_API_KEY": "--api-key",
@@ -1620,17 +1568,15 @@ def scrub_memory_env(env: dict) -> list[str]:
return removed
-# The pageable twin of each mode that reads the weights into a buffer it allocates.
-# Upstream sets use_mmap for mmap / mmap+mlock / auto only (llama-model-loader.cpp),
-# so `mlock` is a full host copy that is also locked, and `mmap+mlock` is the same
-# lock over a mapping. `none` has no lock to preserve, so it goes back to the default.
+# The pageable twin of each mode that reads the weights into a buffer it allocates. Upstream sets use_mmap for mmap /
+# mmap+mlock / auto only (llama-model-loader.cpp), so `mlock` is a full host copy that is also locked, and `mmap+mlock`
+# is the same lock over a mapping. `none` has no lock to preserve, so it goes back to the default.
_PAGEABLE_LOAD_MODE: dict[str, Optional[str]] = {"none": None, "mlock": "mmap+mlock"}
-# Modes that already map, so the rewrite has no unmapped copy of its own to fix and
-# leaves them alone. It reaches them only when a LATER reserving selector shadowed the
-# lock (``--load-mode mmap+mlock --no-mmap`` runs unlocked and unmapped, last-wins), and
-# there dropping only the selector hands the child back the lock it had lost -- over the
-# full-size mapping the override just restored, which is the one outcome it exists to
-# prevent. So they are stripped in that state and in no other.
+# Modes that already map, so the rewrite has no unmapped copy of its own to fix and leaves them alone. It reaches them
+# only when a LATER reserving selector shadowed the lock (``--load-mode mmap+mlock --no-mmap`` runs unlocked and
+# unmapped, last-wins), and there dropping only the selector hands the child back the lock it had lost -- over the
+# full-size mapping the override just restored, which is the one outcome it exists to prevent. So they are stripped in
+# that state and in no other.
_SHADOWED_LOCK_LOAD_MODE = frozenset({"mmap+mlock"})
@@ -1668,15 +1614,14 @@ def _pageable_env_value(
"""
normalized = value.strip().lower()
if name == "LLAMA_ARG_MLOCK":
- # Only when it is already shadowed: resurrecting it would page-lock the
- # oversized mapping into the RAM this override exists to keep pageable.
+ # Only when it is already shadowed: resurrecting it would page-lock the oversized mapping into the RAM this
+ # override keeps pageable
return drop_shadowed_mlock and normalized in _ENV_TRUE_VALUES, None
if name in {"LLAMA_ARG_NO_MMAP", "LLAMA_ARG_NO_DIO"}:
# Presence alone selects mode "none", whatever the value says.
return True, None
if name in {"LLAMA_ARG_MMAP", "LLAMA_ARG_DIO"}:
- # Falsy selects "none"; truthy selects mmap / dio, neither of which
- # holds a full copy.
+ # Falsy selects "none"; truthy selects mmap / dio, neither of which holds a full copy.
return normalized in _ENV_FALSE_VALUES, None
if name == "LLAMA_ARG_LOAD_MODE":
return _pageable_mode_replacement(normalized, drop_shadowed_mlock)
@@ -1714,9 +1659,9 @@ def force_pageable_load(
the pre-rewrite state decides, not the tokens that happen to be present.
"""
tokens = [str(a) for a in (argv or [])]
- # What the child runs TODAY, across env and argv in llama.cpp's own resolution
- # order. Only a launch that reserves RAM is rewritten at all, so a pageable one
- # (``dio``, plain ``mmap``) keeps every token it was given, mlock included.
+ # What the child runs TODAY, across env and argv in llama.cpp's own resolution order. Only a launch that reserves
+ # RAM is rewritten at all, so a pageable one (``dio``, plain ``mmap``) keeps every token it was given, mlock
+ # included.
_mlock_now, _reserves_now = resolve_effective_memory_state(tokens, env)
drop_shadowed_mlock = _reserves_now and not _mlock_now
overridden: list[str] = []
@@ -1725,14 +1670,15 @@ def force_pageable_load(
while i < n:
token = tokens[i]
flag = _flag_name(token)
- # --no-mmap / --no-direct-io: upstream's deprecated spellings for "none".
- # Valueless, so the token goes and nothing follows it out.
+ # --no-mmap / --no-direct-io: upstream's deprecated spellings for "none". Valueless, so the token goes and
+ # nothing follows it out.
if flag in _RAM_RESERVING_FLAGS:
overridden.append(token)
i += 1
continue
- # A lock a later selector already cleared. Valueless like the above, and
- # named in `overridden` so the log line describes the whole rewrite.
+ # A lock a later selector already cleared.
+ # A lock a later selector already cleared. Valueless like the above, and named in `overridden` so the log line
+ # describes the whole rewrite.
if drop_shadowed_mlock and flag in _MLOCK_FLAGS:
overridden.append(token)
i += 1
@@ -1745,9 +1691,9 @@ def force_pageable_load(
else:
value, step = "", 1
normalized = value.strip().lower()
- # `--load-mode mlock --no-mmap` is unlocked by the time the child parses
- # it, so mmap+mlock would ADD a lock; `mmap+mlock --no-mmap` is the same
- # shape one spelling further on, and there the selector itself is the lock.
+ # `--load-mode mlock --no-mmap` is unlocked by the time the child parses it, so mmap+mlock would ADD a lock;
+ # `mmap+mlock --no-mmap` is the same shape one spelling further on, and there the selector itself is the
+ # lock.
rewrite_mode, replacement = _pageable_mode_replacement(normalized, drop_shadowed_mlock)
if rewrite_mode:
overridden.append(" ".join(tokens[i : i + step]))
@@ -1775,17 +1721,16 @@ def force_pageable_load(
return out, overridden
-# Mirrors llama_cpp's _LLAMA_ARG_TRUE/FALSE_VALUES; duplicated so this module
-# stays dependency-free (llama_cpp imports from here, not the other way).
+# Mirrors llama_cpp's _LLAMA_ARG_TRUE/FALSE_VALUES; duplicated so this module stays dependency-free (llama_cpp imports
+# from here, not the other way).
_ENV_TRUE_VALUES = frozenset({"on", "enabled", "true", "1"})
_ENV_FALSE_VALUES = frozenset({"off", "disabled", "false", "0"})
-# Every --load-mode value llama-server documents, so an unknown one is dropped
-# here rather than exiting the child. Mirrored by LOAD_MODES in per-model-config.ts.
+# Every --load-mode value llama-server documents, so an unknown one is dropped here rather than exiting the child.
+# Mirrored by LOAD_MODES in per-model-config.ts.
_LOAD_MODE_VALUES = frozenset({"auto", "none", "mmap", "mlock", "mmap+mlock", "dio"})
-# What each mode meant before the enum existed, for a build that predates it.
-# "auto" is the default and needs no flag; "mmap+mlock" is what a bare --mlock
-# asked for alongside the default mmap. There is no pre-enum spelling for plain
+# What each mode meant before the enum existed, for a build that predates it. "auto" is the default and needs no flag;
+# "mmap+mlock" is what a bare --mlock asked for alongside the default mmap. There is no pre-enum spelling for plain
# "mmap" or for "dio", so those are skipped rather than approximated.
_LEGACY_LOAD_MODE_FLAGS: dict[str, list[str]] = {
"none": ["--no-mmap"],
@@ -1793,8 +1738,8 @@ _LEGACY_LOAD_MODE_FLAGS: dict[str, list[str]] = {
"mmap+mlock": ["--mlock"],
}
_LOAD_MODE_MLOCK_VALUES = frozenset({"mlock", "mmap+mlock"})
-# Modes that read the weights into a full host buffer. "dio" streams via
-# DirectIO and "mmap" maps, so neither reserves RAM for the whole model.
+# Modes that read the weights into a full host buffer. "dio" streams via DirectIO and "mmap" maps, so neither reserves
+# RAM for the whole model.
_LOAD_MODE_RESERVING_VALUES = frozenset({"none", "mlock"})
@@ -1810,20 +1755,16 @@ def resolve_effective_memory_state(
env = env or {}
mlock = False
reserves_ram = False
- # Each var runs the SAME handler as its flag, so it assigns the whole mode
- # and a later one overwrites an earlier one, in llama.cpp's registration
- # order. Measured: LLAMA_ARG_MLOCK=1 with LLAMA_ARG_MMAP=on or
- # LLAMA_ARG_DIO=0 leaves the child unlocked.
- # Only the mlock bit, like the argv --mlock below: "mlock" vs "mmap+mlock"
- # is not observable and changes no decision.
+ # Each var runs the SAME handler as its flag, so it assigns the whole mode and a later one overwrites an earlier
+ # one, in llama.cpp's registration order. Measured: LLAMA_ARG_MLOCK=1 with LLAMA_ARG_MMAP=on or LLAMA_ARG_DIO=0
+ # leaves the child unlocked. Only the mlock bit, like the argv --mlock below: "mlock" vs "mmap+mlock" is not
+ # observable and changes no decision.
if str(env.get("LLAMA_ARG_MLOCK", "")).strip().lower() in _ENV_TRUE_VALUES:
mlock = True
- # Every option with a negative form also answers to LLAMA_ARG_NO_<NAME>:
- # upstream rewrites the name and, if that var EXISTS, forces the value
- # falsey whatever it says, before reading the affirmative one. Measured:
- # LLAMA_ARG_NO_MMAP=0 still disables mmap, and it beats LLAMA_ARG_MMAP=on.
- # --mlock has no negative form, so LLAMA_ARG_NO_MLOCK does nothing.
- # LLAMA_ARG_MMAP is whether to mmap, so "off" means mmap disabled ("none").
+ # Every option with a negative form also answers to LLAMA_ARG_NO_<NAME>: upstream rewrites the name and, if that var
+ # EXISTS, forces the value falsey whatever it says, before reading the affirmative one. Measured:
+ # LLAMA_ARG_NO_MMAP=0 still disables mmap, and it beats LLAMA_ARG_MMAP=on. --mlock has no negative form, so
+ # LLAMA_ARG_NO_MLOCK does nothing. LLAMA_ARG_MMAP is whether to mmap, so "off" means mmap disabled ("none").
_mmap_env = "0" if "LLAMA_ARG_NO_MMAP" in env else str(env.get("LLAMA_ARG_MMAP", ""))
_mmap_env = _mmap_env.strip().lower()
if _mmap_env in _ENV_TRUE_VALUES:
@@ -1851,27 +1792,24 @@ def resolve_effective_memory_state(
i += 1
continue
if flag in _MLOCK_FLAGS:
- # Only the mlock bit: the enum has both "mlock" and "mmap+mlock" and
- # which one this maps to is not observable. It changes no decision,
- # since mlock alone already counts as a reservation for no-reserve.
+ # Only the mlock bit: which of "mlock" / "mmap+mlock" this maps to is not observable and changes no decision
mlock = True
i += 1
elif flag in _NO_MMAP_FLAGS:
- # Deprecated selector for the whole "none" mode, so it clears the
- # mlock too: measured, "--mlock --no-mmap" leaves the child unlocked
- # while "--no-mmap --mlock" locks it.
+ # Deprecated selector for the whole "none" mode, so it clears the mlock too: measured, "--mlock --no-mmap"
+ # leaves the child unlocked while "--no-mmap --mlock" locks it.
mlock = False
reserves_ram = True
i += 1
elif flag in _DIO_ON_FLAGS:
- # Deprecated load-mode selector: resets the mode, so the mlock goes.
- # DirectIO streams the weights, so it holds no full host copy.
+ # Deprecated load-mode selector: resets the mode, so the mlock goes. DirectIO streams the weights, so it
+ # holds no full host copy.
mlock = False
reserves_ram = False
i += 1
elif flag in _DIO_OFF_FLAGS:
- # Not "plain mmap": upstream maps these to mode `none`, like
- # --no-mmap, which reads the weights into a full host buffer.
+ # Not "plain mmap": upstream maps these to mode `none`, like --no-mmap, which reads the weights into a full
+ # host buffer.
mlock = False
reserves_ram = True
i += 1
@@ -1931,8 +1869,7 @@ def memory_state_satisfies_settings(
return True
mlock, reserves_ram = state
if get_no_ram_reserve():
- # mlock_applicable only excuses a MISSING lock; a live reservation
- # still has to go, wherever the weights are.
+ # mlock_applicable only excuses a MISSING lock; a live reservation still has to go, wherever the weights are
return not (mlock or reserves_ram)
if get_keep_resident():
return mlock or not mlock_applicable
diff --git a/studio/backend/core/inference/llama_stats.py b/studio/backend/core/inference/llama_stats.py
index cc75684de77..d63e38801ee 100644
--- a/studio/backend/core/inference/llama_stats.py
+++ b/studio/backend/core/inference/llama_stats.py
@@ -90,7 +90,7 @@ class LlamaServerStatsLogger:
if not running or decode_calls != self._last_decode:
self._last_decode = decode_calls
self._stall_since = now if running else None
- self._stall_reported = False # progress re-arms the report
+ self._stall_reported = False
return 0.0
if self._stall_since is None:
self._stall_since = now
@@ -126,13 +126,12 @@ class LlamaServerStatsLogger:
m = self._scrape()
if not m:
misses += 1
- if misses == 3: # transient stall (load/GC); keep polling.
+ if misses == 3: # transient stall (load/GC); keep polling
self._log.debug("engine_stats: /metrics scrape failing, still retrying")
continue # real shutdown is driven by stop() from _kill_process
misses = 0
- # Generation tokens come from tokens_predicted_total (counter) and
- # predicted_tokens_seconds (gauge); n_decode_total counts
- # llama_decode() calls, not tokens, so it must not feed tok/s.
+ # Generation tokens come from tokens_predicted_total (counter) and predicted_tokens_seconds (gauge);
+ # n_decode_total counts llama_decode() calls, not tokens, so it must not feed tok/s.
now = time.monotonic()
predicted = m.get("tokens_predicted_total", 0.0)
prompt = m.get("prompt_tokens_total", 0.0)
@@ -142,18 +141,18 @@ class LlamaServerStatsLogger:
gen_delta = max(0.0, (predicted - prev[1]) / dt)
prompt_delta = max(0.0, (prompt - prev[2]) / dt)
prev = (now, predicted, prompt)
- # Prefer llama.cpp's own throughput gauges; fall back to the counter
- # delta for binaries that expose only the counters.
+ # Prefer llama.cpp's own throughput gauges; fall back to the counter delta for binaries that expose only the
+ # counters.
gen_tps = m.get("predicted_tokens_seconds") or gen_delta
prompt_tps = m.get("prompt_tokens_seconds") or prompt_delta
running, waiting = (
int(m.get("requests_processing", 0)),
int(m.get("requests_deferred", 0)),
)
- # A held slot not calling llama_decode() is a wedge, and its only symptom
- # is an endless run of identical info lines. A build without n_decode_total
- # reads None and never "changes", accumulating the same way, so the message
- # is chosen at report time.
+ # a build without n_decode_total reads None and never "changes", accumulating the same way
+ # A held slot not calling llama_decode() is a wedge, and its only symptom is an endless run of identical
+ # info lines. A build without n_decode_total reads None and never "changes", accumulating the same way, so
+ # the message is chosen at report time.
decode_calls = m.get("n_decode_total")
stalled_for = self._stalled_for(now, running, decode_calls)
if self._stall_timeout and stalled_for >= self._stall_timeout:
@@ -180,11 +179,11 @@ class LlamaServerStatsLogger:
)
-# A week already means "never" for a poll interval or a stall timeout. Bounded by
-# threading.TIMEOUT_MAX as well, because the ceiling is platform specific and much lower
-# than it looks: Linux accepts ~9.2e9 seconds, Windows about 49.7 days, since the timeout
-# becomes a DWORD of milliseconds there. Picking a constant by hand got this wrong once
-# already, so let the platform state its own limit.
+# bounded by threading.TIMEOUT_MAX as well
+# A week already means "never" for a poll interval or a stall timeout. Bounded by threading.TIMEOUT_MAX as well, because
+# the ceiling is platform specific and much lower than it looks: Linux accepts ~9.2e9 seconds, Windows about 49.7 days,
+# since the timeout becomes a DWORD of milliseconds there. Picking a constant by hand got this wrong once already, so
+# let the platform state its own limit.
_MAX_ENV_SECONDS = min(7.0 * 24.0 * 60.0 * 60.0, threading.TIMEOUT_MAX)
@@ -211,9 +210,10 @@ def _env_float(name, default, logger):
)
return default
if value > _MAX_ENV_SECONDS:
- # Event.wait() builds an absolute deadline, and one far enough out raises
- # "timestamp out of range for platform time_t" once the wait is entered, killing
- # the poll thread. Measured: a century still waits, 1e10 seconds does not.
+ # Event.wait() builds an absolute deadline
+ # Event.wait() builds an absolute deadline, and one far enough out raises "timestamp out of range for platform
+ # time_t" once the wait is entered, killing the poll thread. Measured: a century still waits, 1e10 seconds does
+ # not.
logger.warning(
"engine_stats_env_clamped",
variable = name,
@@ -230,8 +230,8 @@ def maybe_start_stats_logger(base_url, logger):
if (os.environ.get("UNSLOTH_STUDIO_ENGINE_STATS", "1") or "").strip().lower() in _OFF:
return None
interval = _env_float("UNSLOTH_STUDIO_ENGINE_STATS_INTERVAL_S", 10.0, logger)
- # Generously above any legitimate pause between decode calls; 0 silences the
- # stall line and keeps the poller as a pure stats logger.
+ # Generously above any legitimate pause between decode calls; 0 silences the stall line and keeps the poller as a
+ # pure stats logger.
stall_timeout = _env_float("UNSLOTH_STUDIO_ENGINE_STALL_TIMEOUT_S", 600.0, logger)
sl = LlamaServerStatsLogger(
base_url,
diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py
index eaca4a593ee..f6c35c319d6 100644
--- a/studio/backend/core/inference/local_model_resolver.py
+++ b/studio/backend/core/inference/local_model_resolver.py
@@ -28,30 +28,30 @@ logger = get_logger(__name__)
@dataclass(frozen = True)
class _LocalGgufEntry:
- loader_id: str # advertised id (repo id / folder name), also the override key
- load_path: str # concrete on-disk dir/file passed to /load so it never downloads
+ loader_id: str
+ load_path: str
variants: tuple[str, ...] # local quant labels; () for a standalone .gguf
is_gguf: bool = True # False routes the load to the inference orchestrator
_CACHE_TTL_S = 5.0
-# Monotonic timestamps are nonnegative, so a negative stamp encodes "additions-only
-# invalidated at -stamp". Keeps that trust inside the atomically published _scan
-# tuple instead of a second global, and keeps it time-bounded like any other.
+# Monotonic timestamps are nonnegative, so a negative stamp encodes "additions-only invalidated at -stamp". Keeps that
+# trust inside the atomically published _scan tuple instead of a second global, and keeps it time-bounded like any
+# other.
_lock = threading.Lock()
_scan: tuple[float, dict[str, _LocalGgufEntry]] = (0.0, {})
# Not _lock: that is held for the whole scan, so the request path would wait on it.
_warm_lock = threading.Lock()
-# Repos that finished downloading but are not in the published index yet: nothing
-# else covers them until the next scan, and the request path must not call them absent.
+# Repos that finished downloading but are not in the published index yet: nothing else covers them until the next scan,
+# and the request path must not call them absent.
_just_downloaded: set[str] = set()
_warming = False
-# An invalidation landing while a warmer owns the slot asks it for another pass, so
-# a snapshot published already-stale is rebuilt off the request path. Callers still
-# pair invalidate_index() with warm_index_soon() for the case where it has retired.
+# An invalidation landing while a warmer owns the slot asks it for another pass, so a snapshot published already-stale
+# is rebuilt off the request path. Callers still pair invalidate_index() with warm_index_soon() for the case where it
+# has retired.
_warm_pending = False
_last_scan_s = 0.0
-# Rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously.
+# rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously
_WARM_DUTY = 10.0
@@ -117,13 +117,12 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
p = Path(path)
try:
if p.is_file():
- # A standalone .gguf loads by its own path; no quant sub-selection. An
- # mmproj companion (vision/audio projector) is not a servable model on
- # its own: _scan_models_dir's standalone-file pass does not filter it
- # the way the directory scan does, so reject it here or /v1/models would
- # advertise a projector and a switch could load it instead of the weights,
- # evicting the loaded model. The directory branch below is already mmproj
- # free (list_local_gguf_variants drops mmproj quants).
+ # An mmproj companion is not a servable model on its own A standalone.gguf loads by its own path; no quant
+ # sub-selection. An mmproj companion (vision/audio projector) is not a servable model on its own:
+ # _scan_models_dir's standalone-file pass does not filter it the way the directory scan does, so reject it
+ # here or /v1/models would advertise a projector and a switch could load it instead of the weights, evicting
+ # the loaded model. The directory branch below is already mmproj free (list_local_gguf_variants drops mmproj
+ # quants).
if p.suffix.lower() != ".gguf" or detect_gguf_model(str(p)) is None:
return None
return _LocalGgufEntry(loader_id, str(p), ())
@@ -132,17 +131,17 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
quants = tuple(v.quant for v in variants if getattr(v, "quant", None))
if not quants:
return None
- # That call orders by descending size, so the head is the biggest quant (often
- # F16). Downstream reads [0], and a bare id must mean whichever quant a plain
- # load would take: answering with the largest can evict a model and then OOM.
+ # that call orders by descending size, and downstream reads [0]
+ # That call orders by descending size, so the head is the biggest quant (often F16). Downstream reads [0], and a
+ # bare id must mean whichever quant a plain load would take: answering with the largest can evict a model and
+ # then OOM.
from core.inference.openai_auto_download import preferred_quant
- # Rank the ROOT checkpoints alone when there are any. A plain local load resolves
- # through non-recursive detect_gguf_model and so always takes the repo root, while
- # preferred_quant ranks on the key text and would hand a bare id an equally-good
- # ``distilled/...`` row that sorts earlier -- the same id serving different weights
- # depending on which resolver answered it. The qualified rows stay advertised; they
- # simply are not what a bare id means.
+ # Rank the ROOT checkpoints alone when there are any. A plain local load resolves through non-recursive
+ # detect_gguf_model and so always takes the repo root, while preferred_quant ranks on the key text and would
+ # hand a bare id an equally-good ``distilled/...`` row that sorts earlier -- the same id serving different
+ # weights depending on which resolver answered it. The qualified rows stay advertised; they are not what a bare
+ # id means.
unqualified = tuple(q for q in quants if "/" not in q)
best = preferred_quant(unqualified or quants)
if best and quants[0] != best:
@@ -152,12 +151,11 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
return None
-# A LoRA directory can carry a copied config.json and tokenizer beside these, and
-# ModelConfig would then resolve its base model and fetch weights this resolver
-# promises never to download.
+# A LoRA directory can carry a copied config.json and tokenizer beside these, and ModelConfig would then resolve its
+# base model and fetch weights this resolver promises never to download.
_ADAPTER_MARKERS = ("adapter_config.json", "adapter_model.safetensors", "adapter_model.bin")
-# The multimodal sub-configs the repo's own vision detector reads, which is what tells
-# a served VLM apart from a plain seq2seq wearing the same architecture suffix.
+# The multimodal sub-configs the repo's own vision detector reads, which is what tells a served VLM apart from a plain
+# seq2seq wearing the same architecture suffix.
_MULTIMODAL_CONFIG_KEYS = (
"vision_config",
"img_processor",
@@ -240,23 +238,22 @@ def _has_safetensors_weights(load_dir) -> bool:
def _is_generative_chat_config(config: dict) -> bool:
"""Whether a config.json describes a checkpoint the chat loader can generate with."""
architectures = config.get("architectures")
- # model_type cannot stand in for the list: transformers' causal mapping lists bert and bart.
+ # model_type cannot stand in for the list: transformers' causal mapping lists bert and bart
if not isinstance(architectures, list) or not architectures:
return False
names = [name for name in architectures if isinstance(name, str)]
- # Not every causal checkpoint wears ForCausalLM: GPT2LMHeadModel and friends are
- # selected from model_type by the loader, and withholding them is the invisibility
- # this whole path exists to remove.
+ # Not every causal checkpoint wears ForCausalLM: GPT2LMHeadModel and friends are selected from model_type by the
+ # loader, and withholding them is the invisibility this whole path exists to remove.
if any(name.endswith(("ForCausalLM", "LMHeadModel")) for name in names):
return True
if not any(name.endswith("ForConditionalGeneration") for name in names):
return False
- # ForConditionalGeneration is overloaded: T5 and BART wear it too, and the serving
- # path has no AutoModelForSeq2SeqLM branch, so require a multimodal sub-config.
+ # ForConditionalGeneration is overloaded: T5 and BART wear it too, and the serving path has no AutoModelForSeq2SeqLM
+ # branch, so require a multimodal sub-config.
if any(key in config for key in _MULTIMODAL_CONFIG_KEYS):
return True
- # whisper is the audio model rather than wearing one, so it carries no such sub-config.
- # the MLX worker refuses ASR and TTS outright, so only a Transformers host serves these.
+ # whisper is the audio model rather than wearing one, so it carries no such sub-config. the MLX worker refuses ASR
+ # and TTS outright, so only a Transformers host serves these.
return not _host_serves_mlx() and _model_type_is_audio(config.get("model_type"))
@@ -280,8 +277,9 @@ def _weights_are_servable(load_dir) -> bool:
return False
if any((load_dir / name).is_file() for name in _ADAPTER_MARKERS):
return False
- # Same marker is_embedding_model reads for a local path, without its memo, which
- # would pin a verdict for the process from one scan.
+ # the same marker is_embedding_model reads for a local path, without its memo
+ # Same marker is_embedding_model reads for a local path, without its memo, which would pin a verdict for the process
+ # from one scan.
return not (load_dir / "modules.json").is_file()
@@ -294,7 +292,7 @@ def _config_is_servable_here(load_dir, config: dict) -> bool:
"""
from utils.security.remote_code_scan import REMOTE_CODE_CONFIG_FILES
- # trust_remote_code needs an approval fingerprint a switch has not got; read as data only.
+ # trust_remote_code needs an approval fingerprint a switch has not got; read as data only
for name in REMOTE_CODE_CONFIG_FILES:
candidate = config if name == "config.json" else _read_json(load_dir / name)
# truthiness like the consent gate's _config_has_auto_map: an empty map runs nothing.
@@ -351,9 +349,9 @@ def local_servable_model(info) -> Optional[tuple[bool, tuple[str, ...]]]:
from pathlib import Path
path = getattr(info, "path", None)
- # Ollama-link entries come from a scanner _build_index intentionally skips (it
- # creates symlinks on the request path), so their advertised ids never resolve.
- # Don't report them as servable, or /v1/models would list unswitchable models.
+ # Ollama-link entries come from a scanner _build_index intentionally skips (it creates symlinks on the request
+ # path), so their advertised ids never resolve. Don't report them as servable, or /v1/models would list unswitchable
+ # models.
if isinstance(path, str) and any(
seg in (".studio_links", "ollama_links") for seg in Path(path).parts
):
@@ -421,19 +419,18 @@ def _build_index() -> dict[str, _LocalGgufEntry]:
if rp in seen_hf:
return []
seen_hf.add(rp)
- # Only the active cache loads by repo id. Say so, or an inactive repo is
- # indexed under an id it cannot load by, and its snapshot basename (what
- # /v1/models advertises once loaded by path) is never a key at all.
- # No format classification here: nothing on this path reads model_format,
- # and its recursive walk would duplicate the one _local_gguf_entry already
- # does per snapshot, on the request path.
+ # only the active cache loads by repo id, else an inactive repo is indexed under an id it cannot load by.
+ # Only the active cache loads by repo id. Say so, or an inactive repo is indexed under an id it cannot load
+ # by, and its snapshot basename (what /v1/models advertises once loaded by path) is never a key at all. No
+ # format classification here: nothing on this path reads model_format, and its recursive walk would
+ # duplicate the one _local_gguf_entry already does per snapshot, on the request path.
return _scan_hf_cache(directory, active_cache = rp == active_root, classify_format = False)
except Exception as exc: # a missing/malformed root must skip, never crash the index
logger.debug("auto-switch: skipping HF cache dir %r: %s", directory, exc)
return []
- # Each source is guarded on its own so one bad root (a permission error, a
- # malformed cache) drops only that source, not the whole index.
+ # Each source is guarded on its own so one bad root (a permission error, a malformed cache) drops only that source,
+ # not the whole index.
found: list = []
try:
found += _scan_models_dir(Path("./models").resolve())
@@ -473,8 +470,8 @@ def _build_index() -> dict[str, _LocalGgufEntry]:
raw_id = getattr(info, "id", None)
if not raw_id:
continue
- # Skip what Unsloth hides from its pickers (validation probe, RAG embed
- # weights): not chat models, so never an auto-switch target.
+ # Skip what Unsloth hides from its pickers (validation probe, RAG embed weights): not chat models, so never an
+ # auto-switch target.
if _is_hidden_model(
raw_id,
getattr(info, "model_id", None),
@@ -486,8 +483,8 @@ def _build_index() -> dict[str, _LocalGgufEntry]:
entry = _local_servable_entry(loader_id, info)
if entry is None:
continue
- # Index every alias (including the path) so a client can resolve by any of
- # them, even though only the non-path loader_id is advertised.
+ # index every alias (including the path) so a client can resolve by any of them, even though only the non-path
+ # loader_id is advertised
for key in (
raw_id,
getattr(info, "model_id", None),
@@ -496,8 +493,8 @@ def _build_index() -> dict[str, _LocalGgufEntry]:
):
if key:
index.setdefault(key.strip().lower(), entry)
- # Other revisions of the same repo resolve to their own weights, so a pin on
- # one keeps working after Hugging Face writes a newer snapshot.
+ # Other revisions of the same repo resolve to their own weights, so a pin on one keeps working after Hugging
+ # Face writes a newer snapshot.
if entry.is_gguf:
for name, sibling_entry in _sibling_revision_entries(raw_id, loader_id):
index.setdefault(name.strip().lower(), sibling_entry)
@@ -577,9 +574,9 @@ def _snapshot_is_trusted(timestamp: float, now: float) -> bool:
return False
-# Bumped by every invalidate_index() call. A cache built on top of this index can key
-# on it and be dropped by the same call that drops the index, rather than each new
-# invalidation site having to remember one more cache to clear.
+# a cache built on top of this index can key on it and be dropped by the same call that drops the index
+# Bumped by every invalidate_index() call. A cache built on top of this index can key on it and be dropped by the same
+# call that drops the index, rather than each new invalidation site having to remember one more cache to clear.
_generation = 0
@@ -601,13 +598,12 @@ def invalidate_index(*, additions_only: bool = False) -> None:
now = time.monotonic()
_generation += 1
timestamp, retained = _scan
- # Publish entries and their trust state together. A lock-free reader sees
- # either the complete old snapshot or the complete invalidated one, never a
- # fresh timestamp paired with already-revoked trust.
+ # Publish entries and their trust state together. A lock-free reader sees either the complete old snapshot or
+ # the complete invalidated one, never a fresh timestamp paired with already-revoked trust.
stamp = -now if additions_only and _snapshot_is_trusted(timestamp, now) else 0.0
_scan = (stamp, retained)
- # This may have waited out a scan on _lock, so the warmer that just published can
- # still own the slot with a snapshot that is stale again. See _warm_pending.
+ # This may have waited out a scan on _lock, so the warmer that just published can still own the slot with a snapshot
+ # that is stale again. See _warm_pending.
with _warm_lock:
if _warming:
_warm_pending = True
@@ -615,19 +611,21 @@ def invalidate_index(*, additions_only: bool = False) -> None:
def _index() -> dict[str, _LocalGgufEntry]:
global _scan
- # Build under the lock so concurrent callers with an expired cache don't all
- # run the (multi-dir) scan at once; the rest wait and reuse the fresh result.
+ # Build under the lock so concurrent callers with an expired cache don't all run the (multi-dir) scan at once; the
+ # rest wait and reuse the fresh result.
with _lock:
now = time.monotonic()
ts, cached = _scan
- # ``ts > 0``: monotonic() counts from boot, so under a TTL of uptime an
- # invalidated stamp reads as recent and would serve what was just revoked.
+ # `ts > 0`: monotonic() counts from boot, so under a TTL of uptime an invalidated stamp reads as recent and
+ # would serve what was just revoked
if ts > 0.0 and now - ts < _CACHE_TTL_S:
return cached
fresh = _build_index()
- # Stamp AFTER the scan, not with the pre-scan ``now``: a multi-root scan on
- # an install with many local models can itself exceed the TTL, which would
- # store the cache already expired and make every request rebuild the index.
+ # stamp AFTER the scan: a multi-root scan on an install with many local models can itself exceed the TTL,
+ # storing the cache already expired
+ # Stamp AFTER the scan, not with the pre-scan ``now``: a multi-root scan on an install with many local models
+ # can itself exceed the TTL, which would store the cache already expired and make every request rebuild the
+ # index.
_scan = (time.monotonic(), fresh)
# The scan supersedes the notes: whatever landed is in the index now.
_just_downloaded.clear()
@@ -697,8 +695,8 @@ def warm_index_soon() -> None:
_warming, released = False, True
return
finally:
- # Only on a BaseException: leaving the slot held would kill background
- # warming for the life of the process and put scans back on requests.
+ # Only on a BaseException: leaving the slot held would kill background warming for the life of the process
+ # and put scans back on requests.
if not released:
with _warm_lock:
_warming = _warm_pending = False
@@ -730,8 +728,8 @@ def resolve_local_gguf(
index = _index() if allow_scan else _scan[1]
return _resolve_from_index(requested, index)
except Exception:
- # Best-effort: any resolver failure falls through to the loaded model,
- # so a malformed name can never turn a servable request into a 500.
+ # Best-effort: any resolver failure falls through to the loaded model, so a malformed name can never turn a
+ # servable request into a 500.
return None
@@ -759,8 +757,8 @@ def _resolve_from_index(
if looks_like_quant(variant):
return None
- # ":latest" or ":8b" names no file, so it means the repo; a real quant that
- # is not on disk still misses, or a swap would serve the wrong weights.
+ # ":latest" or ":8b" names no file, so it means the repo; a real quant that is not on disk still misses, or a
+ # swap would serve the wrong weights.
return entry.load_path, (entry.variants[0] if entry.variants else None), entry.loader_id
except Exception:
return None
@@ -809,8 +807,8 @@ def describe_local_miss(requested: str) -> tuple[str, tuple[str, ...]]:
base, sep, variant = requested.strip().rpartition(":")
from core.inference.openai_auto_download import looks_like_quant
- # Split like the resolver or the two disagree: a tag naming no quant means the
- # repo there, so reporting a missing quant for it would name one nobody asked for.
+ # Split like the resolver or the two disagree: a tag naming no quant means the repo there, so reporting a missing
+ # quant for it would name one nobody asked for.
if not sep or not looks_like_quant(variant):
return MISS_MODEL_NOT_FOUND, ()
try:
diff --git a/studio/backend/core/inference/mcp_config_import.py b/studio/backend/core/inference/mcp_config_import.py
index f1bf086d9d3..fdefa361e72 100644
--- a/studio/backend/core/inference/mcp_config_import.py
+++ b/studio/backend/core/inference/mcp_config_import.py
@@ -26,8 +26,8 @@ _HTTP_REMOTE_TYPES = ("http", "streamableHttp")
@dataclass
class ParsedMcpEntry:
display_name: str
- url: str # joined command (stdio) or http(s) url (remote)
- headers: Optional[dict[str, str]] # env vars (stdio) or http headers (remote)
+ url: str
+ headers: Optional[dict[str, str]]
is_stdio: bool
is_enabled: bool = True
use_oauth: bool = False
diff --git a/studio/backend/core/inference/media_auto_switch.py b/studio/backend/core/inference/media_auto_switch.py
index dbd77e3b1ee..49b9aeee31d 100644
--- a/studio/backend/core/inference/media_auto_switch.py
+++ b/studio/backend/core/inference/media_auto_switch.py
@@ -324,9 +324,7 @@ async def _gated_start_load(
kind = kind,
openai_errors = openai_errors,
)
- # re-resolved under the gate: a concurrent load can activate the other image engine
backend = backend_for(owner)
- # what the drain waited out may have been the very load this request wanted
if satisfied_by(await asyncio.to_thread(backend.status), name, pick):
return True
if not await drain(
@@ -339,7 +337,6 @@ async def _gated_start_load(
openai_errors = openai_errors,
):
raise busy(kind, openai_errors)
- # re-planned because a cache deletion during the drain sees no load to guard against
await _require_local(
owner,
pick,
@@ -348,9 +345,10 @@ async def _gated_start_load(
openai_errors = openai_errors,
hf_token = hf_token,
)
- # given its own task and waited on with a cap: a first-run native install runs for
- # minutes before begin_load, and holding both media gates and chat's that long
- # blocks every unrelated request. On expiry the load keeps going without them.
+ # capped, in its own task: a first-run native install runs for minutes before begin_load
+ # given its own task and waited on with a cap: a first-run native install runs for minutes before
+ # begin_load, and holding both media gates and chat's that long blocks every unrelated request. On expiry
+ # the load keeps going without them.
setup = asyncio.ensure_future(_start_load(owner, pick, current_subject, hf_token))
setup.add_done_callback(_consume_detached_error)
with contextlib.suppress(asyncio.TimeoutError):
@@ -455,7 +453,6 @@ async def maybe_auto_switch_media_model(
handed_over = False
try:
backend = backend_for(owner)
- # re-read under the lock: a concurrent request may have just loaded this model
if satisfied_by(await asyncio.to_thread(backend.status), name, pick):
return
await _require_local(
diff --git a/studio/backend/core/inference/media_keepwarm.py b/studio/backend/core/inference/media_keepwarm.py
index 5fc9d9eed48..965007d7085 100644
--- a/studio/backend/core/inference/media_keepwarm.py
+++ b/studio/backend/core/inference/media_keepwarm.py
@@ -48,11 +48,13 @@ class _Tracker:
self._last_active = time.monotonic()
# The model identity the last tick saw, so a fresh load counts as activity.
self.seen: Any = None
- # Whether the last tick found work in flight, so the tick that sees it end can
- # start the TTL there rather than at the last busy poll.
+ # so the tick that sees work end can start the TTL there rather than at the last busy poll
+ # Whether the last tick found work in flight, so the tick that sees it end can start the TTL there rather than
+ # at the last busy poll.
self.was_busy = False
- # The terminal record of the last finished background job this tracker has seen, so
- # a job no poll ever sampled as busy still dates the TTL from its completion.
+ # terminal record of the last finished job
+ # The terminal record of the last finished background job this tracker has seen, so a job no poll ever sampled
+ # as busy still dates the TTL from its completion.
self.completed: Any = None
def note_pending(self) -> None:
@@ -93,9 +95,9 @@ class _Tracker:
_TRACKERS = {DIFFUSION: _Tracker(DIFFUSION), VIDEO: _Tracker(VIDEO)}
-# Per backend, not per engine object: a diffusers <-> sd.cpp switch replaces that object.
-# Keyed by the target a load was started for, since a load route records this the moment the
-# background load is accepted and that load can still fail with the previous model resident.
+# Per backend, not per engine object: a diffusers <-> sd.cpp switch replaces that object. Keyed by the target a load was
+# started for, since a load route records this the moment the background load is accepted and that load can still fail
+# with the previous model resident.
_LOAD_ORIGINS: dict[str, tuple[tuple[str, str, str], bool]] = {}
_LOAD_ORIGINS_GUARD = threading.Lock()
@@ -116,7 +118,7 @@ def _origin_key(
publishes, and would spare everything.
"""
text = str(target or "").strip()
- # A repo id folds case; a path does not, or /models/Foo and /models/foo share an origin.
+ # a repo id folds case; a path does not, or /models/Foo and /models/foo share an origin
key = os.path.normcase(text) if os.path.isabs(text) else text.lower()
return (key, str(variant or "").strip().lower(), str(partition or "").strip().lower())
@@ -182,23 +184,18 @@ def other_request_count(
return max(0, total - 1) if current_request_counted else total
-# The concrete mounted paths, not any path that ends in one of them: FastAPI answers
-# /v1/anything/images/generations with a 404 without ever running an endpoint, and stamping
-# that as activity would let unauthenticated 404s keep a pipeline resident past every TTL.
-# Only the generate and load routes: */generate-progress and */generate/cancel are polled
-# while the user watches, so they must not count as activity.
-#
-# The load routes are here because a load registers with the backend only part way through
-# its POST, so sampling loading_repo_ids() cannot see one that has been accepted but not yet
-# started. Holding the gate for the whole request closes that window rather than narrowing it.
-#
-# test_media_keepwarm asserts every one of these is a real route on the routers main.py
-# mounts, so a rename cannot silently drop the protection an in-flight generation rides on.
+# The concrete mounted paths, not any path that ends in one of them: FastAPI answers /v1/anything/images/generations
+# with a 404 without ever running an endpoint, and stamping that as activity would let unauthenticated 404s keep a
+# pipeline resident past every TTL. Only the generate and load routes: */generate-progress and */generate/cancel are
+# polled while the user watches, so they must not count as activity. The load routes are here because a load registers
+# with the backend only part way through its POST, so sampling loading_repo_ids() cannot see one that has been accepted
+# but not yet started. Holding the gate for the whole request closes that window rather than narrowing it.
+# test_media_keepwarm asserts every one of these is a real route on the routers main.py mounts, so a rename cannot
+# silently drop the protection an in-flight generation rides on.
_TRACKED_PATHS = {
# studio_router, mounted at /api/inference only.
"/api/inference/images/generate": DIFFUSION,
"/api/inference/images/load": DIFFUSION,
- # video_router, likewise.
"/api/inference/video/generate": VIDEO,
"/api/inference/video/load": VIDEO,
# The OpenAI-compatible route is on inference_router, mounted at both prefixes.
@@ -229,8 +226,9 @@ async def admission_gate(owner: str):
@contextlib.asynccontextmanager
async def _gate(tracker: _Tracker):
- # Polled non-blocking acquire, exactly like llama_keepwarm's: it keeps the wait off this
- # loop AND cancellation-safe, since a cancel lands during the sleep with the gate free.
+ # polled non-blocking acquire, like llama_keepwarm's
+ # Polled non-blocking acquire, exactly like llama_keepwarm's: it keeps the wait off this loop AND cancellation-safe,
+ # since a cancel lands during the sleep with the gate free.
while not tracker.gate.acquire(blocking = False):
await asyncio.sleep(0.02)
try:
@@ -260,9 +258,9 @@ def end_request(owner: str, *, counted: bool = True) -> None:
def _diffusion_engine() -> Any:
- # Through the router: a native sd.cpp selection must unload the sd-server, not the
- # diffusers pipeline. Nothing can be resident before either module is imported, and
- # importing them here just to find that out would drag torch into every idle tick.
+ # Through the router: a native sd.cpp selection must unload the sd-server, not the diffusers pipeline. Nothing can
+ # be resident before either module is imported, and importing them here just to find that out would drag torch into
+ # every idle tick.
if not {"core.inference.diffusion", "core.inference.sd_cpp_backend"} & set(sys.modules):
return None
from core.inference.diffusion_engine_router import get_active_diffusion_engine
@@ -316,11 +314,11 @@ def _probe(backend: Any) -> tuple[bool, Optional[tuple[Any, ...]]]:
return loading or bool(progress.get("active")), _completed_token(progress)
-# What makes one resident build different from another. The repo id is not enough:
-# MiniMax-H3 stages a different denoiser per h3_task, so a cached fl2va -> ref2va reload
-# is a new build under the same id, and the quants are picked per load too. Only fields
-# fixed at load time, so nothing that moves under a resident model (a Speed=Auto compile
-# flips speed_optims mid-life) can be mistaken for a reload and keep it warm forever.
+# the repo id is not enough: MiniMax-H3 stages a different denoiser per h3_task and quants are picked per load.
+# What makes one resident build different from another. The repo id is not enough: MiniMax-H3 stages a different
+# denoiser per h3_task, so a cached fl2va -> ref2va reload is a new build under the same id, and the quants are picked
+# per load too. Only fields fixed at load time, so nothing that moves under a resident model (a Speed=Auto compile flips
+# speed_optims mid-life) can be mistaken for a reload and keep it warm forever.
_IDENTITY_FIELDS = (
"repo_id",
"base_repo",
@@ -346,40 +344,40 @@ async def _tick(tracker: _Tracker, ttl: float) -> None:
status = await asyncio.to_thread(backend.status)
identity = _identity(status)
busy, completed = await asyncio.to_thread(_probe, backend)
- # A job whose whole life fell between two polls is only ever visible as a terminal
- # record this tracker has not seen before. The first tick to see any record spends
- # one TTL on work that may be older, which is the direction that keeps the model.
+ # A job whose whole life fell between two polls is only ever visible as a terminal record this tracker has not
+ # seen before. The first tick to see any record spends one TTL on work that may be older, which is the direction
+ # that keeps the model.
finished = completed is not None and completed != tracker.completed
tracker.completed = completed
if busy:
- # A load or generation in flight is activity: the TTL restarts from the end of
- # the work, and the backend is never torn down under it.
+ # A load or generation in flight is activity: the TTL restarts from the end of the work, and the backend is
+ # never torn down under it.
tracker.note_activity()
tracker.seen = identity
tracker.was_busy = True
return
if tracker.was_busy or finished:
- # The work ended between two ticks. A video generation outlives its POST, so the
- # only activity it stamps after that is the busy polls, and dating the TTL from
- # the last of those spends up to one poll interval of the keep-warm window the
- # user configured before the model was even free. Start it here instead.
+ # a video generation outlives its POST, so dating the TTL from the last busy poll would spend up to one poll
+ # interval of the user's keep-warm window before the model was even free
+ # The work ended between two ticks. A video generation outlives its POST, so the only activity it stamps
+ # after that is the busy polls, and dating the TTL from the last of those spends up to one poll interval of
+ # the keep-warm window the user configured before the model was even free. Start it here instead.
tracker.was_busy = False
tracker.seen = identity
tracker.note_activity()
return
if identity != tracker.seen:
- # A (re)loaded model counts as activity so it survives at least one TTL before
- # its first generation: loads never pass through the request middleware.
+ # A (re)loaded model counts as activity so it survives at least one TTL before its first generation: loads
+ # never pass through the request middleware.
tracker.seen = identity
if identity is not None:
tracker.note_activity()
return
if identity is None or not tracker.is_idle(ttl):
return
- # Re-read the effective setting immediately before the teardown. One step covers both
- # backends and an unload frees several GB, so a residency veto applied while it runs
- # (Model Memory, or the TTL itself moved) would otherwise be ignored by every teardown
- # left in the step, freeing a model the settings page now calls pinned.
+ # Re-read the effective setting immediately before the teardown. One step covers both backends and an unload
+ # frees several GB, so a residency veto applied while it runs (Model Memory, or the TTL itself moved) would
+ # otherwise be ignored by every teardown left in the step, freeing a model the settings page now calls pinned.
ttl = await asyncio.to_thread(_effective_ttl)
if ttl <= 0 or not tracker.is_idle(ttl):
return
@@ -391,13 +389,12 @@ async def _tick(tracker: _Tracker, ttl: float) -> None:
status.get("h3_task"),
):
return
- # A request may register _pending during an off-loop setting read.
- # Recheck idleness before unloading.
+ # A request may register _pending during an off-loop setting read. Recheck idleness before unloading.
if not tracker.is_idle(ttl):
return
await asyncio.to_thread(backend.unload)
- # Drop ownership only if nothing came back meanwhile, and check it under the arbiter
- # lock so a same-owner load that re-registered keeps it. Mirrors /images/unload.
+ # Drop ownership only if nothing came back meanwhile, and check it under the arbiter lock so a same-owner load
+ # that re-registered keeps it. Mirrors /images/unload.
await asyncio.to_thread(
release_if,
tracker.owner,
@@ -427,7 +424,6 @@ def _user_pinned(
async def idle_unload_step() -> None:
"""The media half of one idle_unload_loop tick. Inert when the TTL is off."""
- # Keep this SQLite-backed setting read off the event loop.
ttl = await asyncio.to_thread(_effective_ttl)
if ttl <= 0:
return
diff --git a/studio/backend/core/inference/media_locality.py b/studio/backend/core/inference/media_locality.py
index b9dfe80904c..0529178b0b4 100644
--- a/studio/backend/core/inference/media_locality.py
+++ b/studio/backend/core/inference/media_locality.py
@@ -299,8 +299,9 @@ def _pipeline_components_present(root: Path) -> bool:
# [null, null] marks a component this pipeline deliberately ships without.
if len(entry) not in (2, 3) or not entry[1]:
continue
- # a modular entry is [library, class, spec], and its spec can name another repo,
- # which this directory is never expected to hold but the load still pulls
+ # a modular entry is [library, class, spec]
+ # a modular entry is [library, class, spec], and its spec can name another repo, which this directory is
+ # never expected to hold but the load still pulls
hosted = _hosted_source(entry[2]) if len(entry) == 3 else None
if hosted is not None:
if not _hosted_component_cached(*hosted):
@@ -343,8 +344,9 @@ def _cached_snapshot_root(repo_id: str, revision: str = "") -> Optional[Path]:
repo_dir = Path(hub_cache_dir()) / f"models--{repo_id.replace('/', '--')}"
snapshots = repo_dir / "snapshots"
- # a pinned revision is a commit sha, or a branch or tag the cache records under refs/, and
- # it is the only candidate: falling back to main is how the default snapshot approves a pin
+ # a pinned revision is the only candidate: falling back to main is how the default snapshot approves a pin
+ # a pinned revision is a commit sha, or a branch or tag the cache records under refs/, and it is the only candidate:
+ # falling back to main is how the default snapshot approves a pin
for candidate in [revision] if revision else ["main"]:
pinned = snapshots / candidate
if pinned.is_dir():
@@ -357,7 +359,6 @@ def _cached_snapshot_root(repo_id: str, revision: str = "") -> Optional[Path]:
if resolved is not None and resolved.is_dir():
return resolved
if revision:
- # pinned and not cached under that name: the default snapshot is not what will load
return None
try:
# no ref file means a commit-pinned download, where any cached revision is the one
@@ -382,8 +383,9 @@ def _hosted_component_cached(source: str, subfolder: str, revision: str, variant
snapshot = _cached_snapshot_root(source, revision)
if snapshot is None:
return False
- # the same component rules either way: _upstream_is_cached's no-manifest branch is satisfied
- # by a single weight file, which an interrupted sharded pull leaves behind
+ # _upstream_is_cached's no-manifest branch is satisfied by a single weight file
+ # the same component rules either way: _upstream_is_cached's no-manifest branch is satisfied by a single weight
+ # file, which an interrupted sharded pull leaves behind
return _component_present(snapshot / subfolder if subfolder else snapshot, variant)
@@ -414,22 +416,23 @@ def _component_present(component: Path, variant: str = "") -> bool:
return False
if variant and not any(f".{variant}." in entry.name for entry in files):
return False
- # kept on the full listing: a shard index whose blob is gone must still route here, where
- # _shards_declared reads the unreadable index and refuses, rather than fall through to the
- # weight test below and pass on whichever sibling shard did survive
+ # kept on the full listing: a shard index whose blob is gone must still route here, where _shards_declared reads the
+ # unreadable index and refuses, rather than fall through to the weight test below and pass on whichever sibling
+ # shard did survive
if any(entry.name.endswith(".index.json") for entry in entries):
# an index is proof only once it declares something; an empty weight_map declares nothing
return _shards_declared(component)
- # a weight-bearing component declares config.json; schedulers, tokenizers and processors
- # carry their own *_config.json instead and ship no weights at all
+ # a weight-bearing component declares config.json; schedulers, tokenizers and processors carry their own
+ # *_config.json instead and ship no weights at all
if (component / "config.json").is_file():
return any(entry.suffix.lower() in _WEIGHT_SUFFIXES for entry in files)
- # a tokenizer ships no weights but is still useless without its vocabulary, and which file
- # that is varies by class, so any one of the known spellings answers for all of them
+ # a tokenizer ships no weights but is useless without its vocabulary
+ # a tokenizer ships no weights but is still useless without its vocabulary, and which file that is varies by class,
+ # so any one of the known spellings answers for all of them
if (component / "tokenizer_config.json").is_file():
return any((component / name).is_file() for name in _TOKENIZER_ASSETS)
- # a metadata-only component is its config: a scheduler or processor directory holding
- # anything else at all (a stray README) builds nothing and is fetched at load time
+ # a metadata-only component is its config: a scheduler or processor directory holding anything else at all (a stray
+ # README) builds nothing and is fetched at load time
return any(entry.name.endswith("config.json") for entry in files)
@@ -489,13 +492,12 @@ def missing_download_bytes(
if local_pipeline and not _pipeline_components_present(Path(target.model_path)):
return UNSIZED_MISSING
if owner == DIFFUSION:
- # asked of every image pick, not only local pipelines: a single-file HiDream checkpoint
- # plans clean and its assembly still loads the encoder repo unconditionally
+ # asked of every image pick, not only local pipelines: a single-file HiDream checkpoint plans clean and its
+ # assembly still loads the encoder repo unconditionally
external = _missing_external_encoder(target)
if external is None or external:
return external
if local_pipeline:
- # the pipeline is present, so only a dependency outside it could still be fetched
return 0
try:
ordinal = plan_gpu_ordinal()
@@ -506,10 +508,10 @@ def missing_download_bytes(
model_kind = target.model_kind,
gpu_ordinal = ordinal,
hf_token = hf_token,
- # Only the verdict, not the probe: this asks whether the pick is already on
- # disk, so it must count the SAME files the load will fetch. Clearing the probe
- # drops the pre-cast encoder and the GGUF dense-transformer widening, which is
- # how a "fully downloaded" answer goes wrong.
+ # only the verdict, not the probe: this must count the SAME files the load will fetch
+ # Only the verdict, not the probe: this asks whether the pick is already on disk, so it must count the
+ # SAME files the load will fetch. Clearing the probe drops the pre-cast encoder and the GGUF
+ # dense-transformer widening, which is how a "fully downloaded" answer goes wrong.
memory_verdict = False,
)
or {}
diff --git a/studio/backend/core/inference/media_model_index.py b/studio/backend/core/inference/media_model_index.py
index 072e8f80da3..090d2fdc633 100644
--- a/studio/backend/core/inference/media_model_index.py
+++ b/studio/backend/core/inference/media_model_index.py
@@ -33,7 +33,7 @@ _INDEX_TTL_S = 5.0
_index_lock = threading.Lock()
_index: dict[str, tuple[float, dict[str, "MediaModelPick"]]] = {}
-# the video family whose partitions are a load-time choice rather than a property of the files
+# the video family whose partitions are a load-time choice, not a property of the files
_H3_FAMILY = "minimax-h3"
@@ -144,8 +144,8 @@ def _loader_can_open(load_path: str, filename: str) -> bool:
root = Path(load_path)
if not root.is_dir():
- # a repo id loads from the cache, where the containment rule does not apply but the
- # split set still has to be whole; an uncached child is the download guard's business
+ # a repo id loads from the cache, where the containment rule does not apply but the split set still has to be
+ # whole; an uncached child is the download guard's business
cached = _cached_repo_file(load_path, filename)
return True if cached is None else bool(colocated_split_shards(cached)[1])
from core.inference.diffusion_families import resolve_local_gguf_child
@@ -249,8 +249,8 @@ def _loadable_directory(load_dir: Path) -> bool:
return True
except OSError:
return False
- # a sole checkpoint is reinterpreted as a single_file load, which resolves the name through
- # the same containment check a gguf goes through, so a cache snapshot's symlink is refused
+ # a sole checkpoint is reinterpreted as a single_file load, which resolves the name through the same containment
+ # check a gguf goes through, so a cache snapshot's symlink is refused
sole = resolve_local_single_file(str(load_dir))
return sole is not None and _loader_can_open(str(load_dir), sole)
diff --git a/studio/backend/core/inference/memory_contract.py b/studio/backend/core/inference/memory_contract.py
index 7a6d4a7ea10..6595b5edf76 100644
--- a/studio/backend/core/inference/memory_contract.py
+++ b/studio/backend/core/inference/memory_contract.py
@@ -119,17 +119,12 @@ def build_memory_estimate(
"""
resident = int(getattr(breakdown, "weights_bytes", 0) or 0)
quant = int(quant_file_bytes or 0)
- # Deliberately NOT clamped against `resident`, though the quant file is by
- # definition one of the resident files and so cannot really be larger.
- #
- # The two figures do not come from the same place. `resident` is what the
- # planner measured from the files it opened; `quant` is what resolved the
- # user's chosen file, which may be a listing size or a stat of a different
- # path. They agree in production and diverge whenever anything stubs one
- # side, and a clamp there does not catch a bug -- it silently replaces the
- # caller's real number with an unrelated one. The first draft of this
- # function clamped, and the contract-freeze suite caught it truncating a
- # 4.1 GB quant to 373 bytes.
+ # Deliberately NOT clamped against `resident`, though the quant file is by definition one of the resident files and
+ # so cannot really be larger. The two figures do not come from the same place. `resident` is what the planner
+ # measured from the files it opened; `quant` is what resolved the user's chosen file, which may be a listing size or
+ # a stat of a different path. They agree in production and diverge whenever anything stubs one side, and a clamp
+ # there does not catch a bug -- it silently replaces the caller's real number with an unrelated one. The first draft
+ # of this function clamped, and the contract-freeze suite caught it truncating a 4.1 GB quant to 373 bytes.
return MemoryEstimate(
available = True,
reason = None,
@@ -151,8 +146,8 @@ def build_memory_estimate(
if isinstance(total_bytes, _Unset)
else int(total_bytes or 0)
),
- # Not `or 0`: zero is a real answer (an all-CPU launch) and must survive
- # distinct from None. See the field's own description.
+ # Not `or 0`: zero is a real answer (an all-CPU launch) and must survive distinct from None. See the field's own
+ # description.
gpu_bytes = (
(None if getattr(breakdown, "gpu_bytes", None) is None else int(breakdown.gpu_bytes))
if isinstance(gpu_bytes, _Unset)
diff --git a/studio/backend/core/inference/message_content.py b/studio/backend/core/inference/message_content.py
index 942569dc229..cfd989702fe 100644
--- a/studio/backend/core/inference/message_content.py
+++ b/studio/backend/core/inference/message_content.py
@@ -12,8 +12,9 @@ from __future__ import annotations
from typing import Any
-# The composer sends a long paste as a text attachment wrapped in this tag, so
-# a paste-only turn carries no `content` text at all.
+# a paste-only turn carries no `content` text: the composer wraps long pastes in this tag
+# The composer sends a long paste as a text attachment wrapped in this tag, so a paste-only turn carries no `content`
+# text at all.
_PASTED_TEXT_OPEN = "<pasted_text name="
_PASTED_TEXT_CLOSE = "</pasted_text>"
diff --git a/studio/backend/core/inference/model_ids.py b/studio/backend/core/inference/model_ids.py
index c768858e3dd..fb19b95bafe 100644
--- a/studio/backend/core/inference/model_ids.py
+++ b/studio/backend/core/inference/model_ids.py
@@ -32,7 +32,7 @@ def _looks_like_path(identifier: str) -> bool:
return True
if identifier.startswith(("/", "\\", "./", "../", ".\\", "..\\", "~")):
return True
- if len(identifier) >= 2 and identifier[1] == ":": # Windows drive, e.g. C:\
+ if len(identifier) >= 2 and identifier[1] == ":":
return True
if identifier.count("/") >= 2 or "\\" in identifier:
return True
diff --git a/studio/backend/core/inference/offload_cost_model.py b/studio/backend/core/inference/offload_cost_model.py
index 9183ce3bc97..29973fba7cb 100644
--- a/studio/backend/core/inference/offload_cost_model.py
+++ b/studio/backend/core/inference/offload_cost_model.py
@@ -41,63 +41,57 @@ from enum import Enum
GIB = float(1024**3)
-# Reference host (192 cores, one B200), dense FFN spilled in 16/32/48/65-block
-# steps: least-squares fit gives 5.544 ms per GiB, intercept -0.10 ms. The
-# near-zero intercept means partial spilling is LINEAR, so spill the minimum.
+# Reference host (192 cores, one B200), dense FFN spilled in 16/32/48/65-block steps: least-squares fit gives 5.544 ms
+# per GiB, intercept -0.10 ms. The near-zero intercept means partial spilling is LINEAR, so spill the minimum.
REFERENCE_CONTIGUOUS_MS_PER_GIB = 5.544
-# Reference host thread count the rate above was measured at.
+# reference host thread count the rate above was measured at
REFERENCE_HOST_THREADS = 192
-# Generation penalty vs threads, ms per GiB = a / threads + b. Least squares over
-# 39.81 / 15.86 / 7.248 / 5.498 ms per GiB at 4 / 16 / 64 / 192 threads (full
-# dense FFN spilled) gives a = 138.4, b = 5.57, every point within 14%. Amdahl,
-# not a power law: ``a`` is the CPU backend's parallel quantised matmul, ``b`` the
-# floor (host bandwidth, graph-split sync, non-overlapping GPU work), so 4x the
-# cores buys much less than 4x the speed.
+# Generation penalty vs threads, ms per GiB = a / threads + b. Least squares over 39.81 / 15.86 / 7.248 / 5.498 ms per
+# GiB at 4 / 16 / 64 / 192 threads (full dense FFN spilled) gives a = 138.4, b = 5.57, every point within 14%. Amdahl,
+# not a power law: ``a`` is the CPU backend's parallel quantised matmul, ``b`` the floor (host bandwidth, graph-split
+# sync, non-overlapping GPU work), so 4x the cores buys much less than 4x the speed.
_THREAD_PARALLEL_MS_PER_GIB = 138.4
_THREAD_SERIAL_MS_PER_GIB = 5.57
-# The fit above varies threads on ONE machine; across DIFFERENT machines the
-# slope is steeper (12 cloud vCPUs are ~6 physical cores on one memory
-# controller). Least squares over dense Q4 on 12 and 48 vCPU hosts (24.21 and
-# 6.82 ms per GiB) gives a = 278.2, b = 1.03. Neither fit dominates -- at 192
-# threads cross-machine predicts 2.48 vs 5.498 measured, at 12 one-machine
-# predicts 17.1 vs 24.21 -- so take whichever is MORE expensive: optimism quotes
-# a spill that runs several times slower than promised.
+# across DIFFERENT machines the slope is steeper (12 cloud vCPUs are ~6 physical cores on one memory controller)
+# The fit above varies threads on ONE machine; across DIFFERENT machines the slope is steeper (12 cloud vCPUs are ~6
+# physical cores on one memory controller). Least squares over dense Q4 on 12 and 48 vCPU hosts (24.21 and 6.82 ms per
+# GiB) gives a = 278.2, b = 1.03. Neither fit dominates -- at 192 threads cross-machine predicts 2.48 vs 5.498 measured,
+# at 12 one-machine predicts 17.1 vs 24.21 -- so take whichever is MORE expensive: optimism quotes a spill that runs
+# several times slower than promised.
_CROSS_HOST_PARALLEL_MS_PER_GIB = 278.2
_CROSS_HOST_SERIAL_MS_PER_GIB = 1.03
-# Residual model/measured over 70 runs on T4, L4, A100 and RTX PRO 6000 (2, 8,
-# 12, 48 vCPU), Qwen3.8-27B and Qwen3.6-35B-A3B at Q2_K_XL / Q4_K_XL / Q8_K_XL:
-# medians 1.09 (48 cpu), 0.59 (12), 0.24 (8), 0.24 (2).
-#
-# One case stays materially under-priced: DENSE Q2_K on a very small host. Cost
-# per spilled GiB tracks dequant complexity, not byte count -- at 12 vCPU Q2_K
-# 84.17, Q4_K 24.21, Q8 19.96 ms per GiB (Q8 cheapest despite being largest),
-# compressing to 8.70 / 6.82 / 6.29 at 48 vCPU. A quant multiplier would itself
-# move with core count (3.5x at 12 vCPU, 1.3x at 48), so fitting it from two core
-# counts would be overfitting. Left unmodelled deliberately.
+# residual model/measured over 70 runs (T4, L4, A100, RTX PRO 6000 at 2-48 vCPU)
+# Residual model/measured over 70 runs on T4, L4, A100 and RTX PRO 6000 (2, 8, 12, 48 vCPU), Qwen3.8-27B and
+# Qwen3.6-35B-A3B at Q2_K_XL / Q4_K_XL / Q8_K_XL: medians 1.09 (48 cpu), 0.59 (12), 0.24 (8), 0.24 (2). One case stays
+# materially under-priced: DENSE Q2_K on a very small host. Cost per spilled GiB tracks dequant complexity, not byte
+# count -- at 12 vCPU Q2_K 84.17, Q4_K 24.21, Q8 19.96 ms per GiB (Q8 cheapest despite being largest), compressing to
+# 8.70 / 6.82 / 6.29 at 48 vCPU. A quant multiplier would itself move with core count (3.5x at 12 vCPU, 1.3x at 48), so
+# fitting it from two core counts would be overfitting. Left unmodelled.
class Access(str, Enum):
"""How a tensor group is read, which sets its cost per byte."""
- #: Dense FFN: large contiguous matmuls, best case for the CPU backend.
+ # : Dense FFN: large contiguous matmuls, best case for the CPU backend.
CONTIGUOUS = "contiguous"
- #: lm_head: one tall matvec, usually higher-bit. Parallelises worse than 64
- #: independent FFN blocks and dequantises more bytes per output element.
+ # : lm_head: one tall matvec; parallelises worse than 64 independent FFN blocks and dequantises more bytes per
+ # output element.
+ # : lm_head: one tall matvec, usually higher-bit. Parallelises worse than 64 : independent FFN blocks and
+ # dequantises more bytes per output element.
SINGLE_MATVEC = "single_matvec"
- #: MoE experts: 8-of-256 scattered small matmuls per layer per token.
+ # : MoE experts: 8-of-256 scattered small matmuls per layer per token.
SCATTERED = "scattered"
- #: The attention cache. By far the worst thing to move off the device.
+ # : The attention cache. By far the worst thing to move off the device.
KV_CACHE = "kv_cache"
-# Cost per activated byte, relative to CONTIGUOUS. Derived in the plan doc:
-# lm_head 10.51 / 5.955 = 1.77; MoE expert 14.80 / 5.955 = 2.49
-# KV cache 119.7 / 5.955 = 20.1 (20.4 from 121.2 on a structurally different
-# model -- agreeing to 1.3%, the strongest single calibration point here)
+# Cost per activated byte, relative to CONTIGUOUS. Derived in the plan doc: lm_head 10.51 / 5.955 = 1.77; MoE expert
+# 14.80 / 5.955 = 2.49 KV cache 119.7 / 5.955 = 20.1 (20.4 from 121.2 on a structurally different model -- agreeing to
+# 1.3%, the strongest single calibration point here)
_RATE_RATIOS: dict[Access, float] = {
Access.CONTIGUOUS: 1.00,
Access.SINGLE_MATVEC: 1.77,
@@ -105,13 +99,13 @@ _RATE_RATIOS: dict[Access, float] = {
Access.KV_CACHE: 20.1,
}
-# Prefill streaming bandwidth, GiB/s. Bracketed by the two measurements (dense
-# 49.4, MoE 66.9); that spread is why this is a coarse constant, not a curve.
+# bracketed by the two measurements (dense 49.4, MoE 66.9)
+# Prefill streaming bandwidth, GiB/s. Bracketed by the two measurements (dense 49.4, MoE 66.9); that spread is why this
+# is a coarse constant, not a curve.
PREFILL_STREAM_GIB_S = 55.0
-# Spilling several groups costs MORE than the sum of each alone -- contention,
-# not amortisation. Measured 6% for the dense FFN + lm_head pair. The "costs are
-# sub-additive" reading compares throughput percentages instead of times.
+# Spilling several groups costs MORE than the sum of each alone -- contention, not amortisation. Measured 6% for the
+# dense FFN + lm_head pair. The "costs are sub-additive" reading compares throughput percentages instead of times.
MULTI_GROUP_CONTENTION = 0.06
@@ -122,8 +116,8 @@ class TensorGroup:
name: str
bytes_total: int
access: Access
- #: Fraction of ``bytes_total`` read per token during GENERATION. 1.0 for
- #: dense weights; ``n_expert_used / n_expert`` for MoE experts.
+ # : Fraction of ``bytes_total`` read per token during GENERATION. 1.0 for : dense weights; ``n_expert_used /
+ # n_expert`` for MoE experts.
activation_fraction: float = 1.0
@property
@@ -141,8 +135,8 @@ class HostProfile:
"""
threads: int = REFERENCE_HOST_THREADS
- #: Set for unified-memory hosts (Apple Silicon, AMD APU, Vulkan iGPU) where
- #: "spilling" moves nothing, because the two pools are one pool.
+ # : Set for unified-memory hosts (Apple Silicon, AMD APU, Vulkan iGPU) where : "spilling" moves nothing, because the
+ # two pools are one pool.
unified_memory: bool = False
@property
@@ -172,8 +166,8 @@ class Placement:
"""A candidate: which groups sit in host memory, and where the cache is."""
host_groups: list[TensorGroup] = field(default_factory = list)
- #: Bytes of attention cache forced to host RAM. Only ``--no-kv-offload``
- #: puts anything here; ``-ot`` leaves it resident, ``-ngl`` drags it off.
+ # : Bytes of attention cache forced to host RAM. Only ``--no-kv-offload`` : puts anything here; ``-ot`` leaves it
+ # resident, ``-ngl`` drags it off.
kv_host_bytes: int = 0
@@ -231,12 +225,10 @@ def prefill_penalty_ms_per_token(
host = host or HostProfile()
if host.unified_memory:
return 0.0
- # kv_host_bytes counts here too: leaving it out made a cache-offloaded
- # placement prefill FOR FREE while generation_penalty_ms charged it.
- # UNCALIBRATED lower bound -- counts the cache as streamed once per ubatch
- # like the weights, though attention re-reads it and the read grows with
- # depth. No production path builds kv_host_bytes > 0; this only stops rank()
- # calling a cache spill free.
+ # kv_host_bytes counts here too: leaving it out made a cache-offloaded placement prefill FOR FREE while
+ # generation_penalty_ms charged it. UNCALIBRATED lower bound -- counts the cache as streamed once per ubatch like
+ # the weights, though attention re-reads it and the read grows with depth. No production path builds kv_host_bytes >
+ # 0; this only stops rank() calling a cache spill free.
host_bytes = sum(g.bytes_total for g in placement.host_groups) + placement.kv_host_bytes
if host_bytes <= 0 or n_ubatch <= 0:
return 0.0
diff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.py
index db14cda0958..ce885e1f3fb 100644
--- a/studio/backend/core/inference/offload_layout.py
+++ b/studio/backend/core/inference/offload_layout.py
@@ -17,17 +17,14 @@ from typing import Optional
logger = logging.getLogger(__name__)
-# blk.<N>.<tail>
_BLOCK_RE = re.compile(r"^blk\.(\d+)\.(.+)$")
-# Sparse MoE experts: only expert_used_count of expert_count read per token, so
-# host traffic is a small fraction of their size. The cheap thing to spill.
-# Fused (ffn_gate_up_exps, cohere2moe/deepseek2/dots3note) and chunked
-# (ffn_*_chexps, grovemoe) spellings are experts too: created per expert and
-# dispatched with GGML_OP_MUL_MAT_ID, so read just as sparsely as the split form.
-# NOT ffn_routed_up/down: kimi-k3 creates it {n_embd, n_embd_latent} with no
-# expert axis and plain GGML_OP_MUL_MAT, so every token crosses it -- spilling it
-# would send a hot tensor to the host at the rate reserved for cold ones.
+# Sparse MoE experts: only expert_used_count of expert_count read per token, so host traffic is a small fraction of
+# their size. The cheap thing to spill. Fused (ffn_gate_up_exps, cohere2moe/deepseek2/dots3note) and chunked
+# (ffn_*_chexps, grovemoe) spellings are experts too: created per expert and dispatched with GGML_OP_MUL_MAT_ID, so read
+# just as sparsely as the split form. NOT ffn_routed_up/down: kimi-k3 creates it {n_embd, n_embd_latent} with no expert
+# axis and plain GGML_OP_MUL_MAT, so every token crosses it -- spilling it would send a hot tensor to the host at the
+# rate reserved for cold ones.
_MOE_EXPERT_RE = re.compile(r"^ffn_(up|gate|down|gate_up)_(exps|chexps)\.weight$")
# Dense FFN. Fully activated: every byte crosses the bus every token.
_DENSE_FFN_RE = re.compile(r"^ffn_(up|gate|down)\.weight$")
@@ -40,8 +37,7 @@ class BlockLayout:
index: int
# ffn_*_exps (MoE) or plain ffn_* (dense). Safe to push to host RAM.
spillable_bytes: int
- # attention, norms, routers, shared experts, ssm: on the critical path every
- # token, or the KV cache hangs off them.
+ # attention, norms, routers, shared experts, ssm: on the critical path every token, or the KV cache hangs off them.
resident_bytes: int
@@ -53,47 +49,43 @@ class ModelLayout:
n_layers: int = 0
n_attention_layers: int = 0
blocks: tuple[BlockLayout, ...] = field(default_factory = tuple)
- # Rides the layer list at index n_layer_all, so it is GPU-resident for any
- # -ngl >= 1 and can only be moved with an explicit override.
+ # Rides the layer list at index n_layer_all, so it is GPU-resident for any -ngl >= 1 and can only be moved with an
+ # explicit override.
lm_head_bytes: int = 0
- # llama-model.cpp pins dev_input to the CPU unconditionally, so this is
- # never charged to VRAM. Tracked because it IS charged to host RAM.
+ # llama-model.cpp pins dev_input to the CPU unconditionally, so this is never charged to VRAM. Tracked because it IS
+ # charged to host RAM.
token_embd_bytes: int = 0
# output_norm and friends: GPU-resident, too small to be worth spilling.
other_resident_bytes: int = 0
# Attention cache for ONE token at f16, across the attention layers only.
kv_bytes_per_token_f16: int = 0
- # Mamba conv/SSM state. Context independent; follows the layer, which -ot never moves.
+ # Mamba conv/SSM state; context independent, and follows the layer, which -ot never moves
recurrent_bytes: int = 0
n_ctx_train: int = 0
is_moe: bool = False
- # Sparse-MoE routing: experts read per token is expert_used/expert_count.
- # Offloaded experts move only that fraction per token, a dense FFN all of it.
+ # offloaded experts move only expert_used/expert_count per token, a dense FFN all of it
+ # Sparse-MoE routing: experts read per token is expert_used/expert_count. Offloaded experts move only that fraction
+ # per token, a dense FFN all of it.
n_expert: int = 0
n_expert_used: int = 0
- # ``blocks`` drops the trailing nextn/MTP blk.<N> tensors: block_count counts
- # them (llama-model.cpp reads it into n_layer_all) but the target does not use
- # them. They are real blk.<N>.ffn_* weights (models/qwen35moe.cpp,
- # load_block_mtp), so an unbounded ^blk\.\d+\. spill pattern WOULD match them
- # once a draft is loaded. The planner uses this to stay bounded.
+ # ``blocks`` drops the trailing nextn/MTP blk.<N> tensors: block_count counts them (llama-model.cpp reads it into
+ # n_layer_all) but the target does not use them. They are real blk.<N>.ffn_* weights (models/qwen35moe.cpp,
+ # load_block_mtp), so an unbounded ^blk\.\d+\. spill pattern WOULD match them once a draft is loaded. The planner
+ # uses this to stay bounded.
has_excluded_blocks: bool = False
- # Total bytes of those dropped blocks, so a caller that knows a draft WILL
- # engage can charge them back. Dropping them suits the ordinary load: every
- # trailing block gets TENSOR_SKIP unless load_mtp is set
- # (models/glm4-moe.cpp:42-44, the same gate in every embedded-MTP arch) and
- # TENSOR_SKIP returns before the tensor exists
- # (llama-model-loader.cpp:1123-1131). But ``--spec-type draft-mtp`` sets
- # load_mtp on the TARGET's own params (common/common.cpp:1713), so the block is
- # materialised at its layer's buffer type, and i_gpu_start counting back from
- # n_layer_all (llama-model.cpp:1449) puts those blocks on a GPU FIRST.
- # llama.cpp's own fitter widens its offloadable-layer count the same way
- # (common/fit.cpp:139-142). Zero when nothing was dropped.
+ # Total bytes of those dropped blocks, so a caller that knows a draft WILL engage can charge them back. Dropping
+ # them suits the ordinary load: every trailing block gets TENSOR_SKIP unless load_mtp is set
+ # (models/glm4-moe.cpp:42-44, the same gate in every embedded-MTP arch) and TENSOR_SKIP returns before the tensor
+ # exists (llama-model-loader.cpp:1123-1131). But ``--spec-type draft-mtp`` sets load_mtp on the TARGET's own params
+ # (common/common.cpp:1713), so the block is materialised at its layer's buffer type, and i_gpu_start counting back
+ # from n_layer_all (llama-model.cpp:1449) puts those blocks on a GPU FIRST. llama.cpp's own fitter widens its
+ # offloadable-layer count the same way (common/fit.cpp:139-142). Zero when nothing was dropped.
excluded_block_bytes: int = 0
- # Sliding-window attention: some layers keep a window-sized cache, some the
- # full context (llama-kv-cache-iswa.cpp:69-104 builds two caches and filters
- # each by hparams.is_swa(il)), interleaved per layer. Every layer is still an
- # attention layer, so n_attention_layers does NOT reveal this. A multi-device
- # split has to know WHERE the big caches land, so the planner abstains.
+ # sliding-window attention interleaves window-sized and full-context caches per layer
+ # Sliding-window attention: some layers keep a window-sized cache, some the full context
+ # (llama-kv-cache-iswa.cpp:69-104 builds two caches and filters each by hparams.is_swa(il)), interleaved per layer.
+ # Every layer is still an attention layer, so n_attention_layers does NOT reveal this. A multi-device split has to
+ # know WHERE the big caches land, so the planner abstains.
has_swa: bool = False
# False when a needed quantity could not be read. The planner abstains.
complete: bool = False
@@ -153,12 +145,10 @@ def layout_from_gguf(path: str) -> ModelLayout:
def _layout_from_reader(reader) -> ModelLayout:
- # Split GGUF: llama.cpp loads every sibling shard
- # (llama-model-loader.cpp:590-618), but GGUFReader memmaps only the ONE path
- # it was given. Shard 1 still carries the metadata, so the layout would look
- # complete while undercounting resident and spillable by most of the model --
- # an overstated fit, too few -ot patterns, and a startup OOM with --fit off.
- # Abstain instead; the seam then reproduces --fit on exactly.
+ # Split GGUF: llama.cpp loads every sibling shard (llama-model-loader.cpp:590-618), but GGUFReader memmaps only the
+ # ONE path it was given. Shard 1 still carries the metadata, so the layout would look complete while undercounting
+ # resident and spillable by most of the model -- an overstated fit, too few -ot patterns, and a startup OOM with
+ # --fit off. Abstain instead; the seam then reproduces --fit on exactly.
if int(_field(reader, "split.count") or 0) > 1:
return ModelLayout()
@@ -171,13 +161,13 @@ def _layout_from_reader(reader) -> ModelLayout:
return ModelLayout()
blocks_total = int(blocks_total)
- # llama.cpp keeps embedded MTP blocks out of the target context and prices
- # their cache separately, so the attention count must not include them.
+ # llama.cpp keeps embedded MTP blocks out of the target context and prices their cache separately, so the attention
+ # count must not include them.
nextn = int(_field(reader, f"{arch}.nextn_predict_layers") or 0)
n_layers = max(0, blocks_total - nextn)
- # Hybrid: only 1 in full_attention_interval layers carries a KV cache, the
- # rest are recurrent. Absent (or 0) means every layer is attention.
+ # Hybrid: only 1 in full_attention_interval layers carries a KV cache, the rest are recurrent. Absent (or 0) means
+ # every layer is attention.
fai = int(_field(reader, f"{arch}.full_attention_interval") or 0)
n_attention = -(-n_layers // fai) if fai > 0 else n_layers
n_recurrent = max(0, n_layers - n_attention)
@@ -196,12 +186,13 @@ def _layout_from_reader(reader) -> ModelLayout:
kv_per_token = int(n_attention) * int(n_kv_head) * (int(key_len) + int(val_len)) * 2
- # Charging every layer the full context above is the safe direction for the
- # TOTAL; what it cannot say is which layers hold the big caches.
+ # charging every layer the full context is the safe direction for the TOTAL
+ # Charging every layer the full context above is the safe direction for the TOTAL; what it cannot say is which
+ # layers hold the big caches.
has_swa = bool(_field(reader, f"{arch}.attention.sliding_window") or 0)
- # Mamba conv + SSM state, one f32 copy per sequence. Mirrors llama.cpp's
- # own sizing; zero when the model has no recurrent layers.
+ # Mamba conv + SSM state, one f32 copy per sequence. Mirrors llama.cpp's own sizing; zero when the model has no
+ # recurrent layers.
d_inner = int(_field(reader, f"{arch}.ssm.inner_size") or 0)
d_state = int(_field(reader, f"{arch}.ssm.state_size") or 0)
n_group = int(_field(reader, f"{arch}.ssm.group_count") or 0)
@@ -229,8 +220,8 @@ def _layout_from_reader(reader) -> ModelLayout:
if match:
index = int(match.group(1))
tail = match.group(2)
- # Shared experts (ffn_*_shexp) and routers (ffn_gate_inp*) run on every
- # token: dense-FFN bandwidth for a rounding error of size. Not spillable.
+ # Shared experts (ffn_*_shexp) and routers (ffn_gate_inp*) run on every token: dense-FFN bandwidth for a
+ # rounding error of size. Not spillable.
spillable = _MOE_EXPERT_RE.match(tail) or (not is_moe and _DENSE_FFN_RE.match(tail))
if spillable:
spill[index] = spill.get(index, 0) + nbytes
@@ -247,28 +238,24 @@ def _layout_from_reader(reader) -> ModelLayout:
if not spill and not resident:
return ModelLayout()
- # Tied embeddings duplicate the vocabulary matrix, they do not SAVE it. With no
- # output.weight llama.cpp re-creates the output tensor from token_embd as
- # TENSOR_DUPLICATED (models/llama.cpp:41-45, models/qwen3.cpp:22-25,
- # models/gemma3.cpp:43-47, and ~60 more) and routes a duplicated TOKEN_EMBD
- # through the OUTPUT buffer list (llama-model-loader.cpp:1113-1114). dev_input
- # is CPU-pinned while dev_output follows the layer split (llama-model.cpp:1465,
- # 1474), so the buffer-type contexts differ, the same-context reuse check misses
- # (llama-model-loader.cpp:1309-1314), and ggml_dup_tensor allocates a second
- # full matrix (llama-model-loader.cpp:1318) that load_all_data fills by name
- # with a real host to device copy (:1542, :1583). Counting the one stored tensor
- # as host-only understates VRAM by a whole vocabulary matrix -- the optimistic
- # direction. Resident, not lm_head: the duplicate keeps the name
- # token_embd.weight, so LM_HEAD_PATTERN cannot match and the lm_head rung would
- # credit a spill that moves nothing.
+ # Tied embeddings duplicate the vocabulary matrix, they do not SAVE it. With no output.weight llama.cpp re-creates
+ # the output tensor from token_embd as TENSOR_DUPLICATED (models/llama.cpp:41-45, models/qwen3.cpp:22-25,
+ # models/gemma3.cpp:43-47, and ~60 more) and routes a duplicated TOKEN_EMBD through the OUTPUT buffer list
+ # (llama-model-loader.cpp:1113-1114). dev_input is CPU-pinned while dev_output follows the layer split
+ # (llama-model.cpp:1465, 1474), so the buffer-type contexts differ, the same-context reuse check misses
+ # (llama-model-loader.cpp:1309-1314), and ggml_dup_tensor allocates a second full matrix
+ # (llama-model-loader.cpp:1318) that load_all_data fills by name with a real host to device copy (:1542,:1583).
+ # Counting the one stored tensor as host-only understates VRAM by a whole vocabulary matrix -- the optimistic
+ # direction. Resident, not lm_head: the duplicate keeps the name token_embd.weight, so LM_HEAD_PATTERN cannot match
+ # and the lm_head rung would credit a spill that moves nothing.
if not lm_head and token_embd:
other_resident += token_embd
- # Trailing nextn/MTP blocks are NOT part of the target model and are not loaded
- # unless a draft is engaged, so an -ot naming them moves nothing: measured,
- # spilling only blk.<nextn> leaves the host buffer at exactly token_embd and the
- # device buffer unchanged. Counting them spillable would credit bytes that can
- # never be freed. Unsloth prices the drafter separately anyway.
+ # trailing nextn/MTP blocks are not loaded unless a draft is engaged
+ # Trailing nextn/MTP blocks are NOT part of the target model and are not loaded unless a draft is engaged, so an -ot
+ # naming them moves nothing: measured, spilling only blk.<nextn> leaves the host buffer at exactly token_embd and
+ # the device buffer unchanged. Counting them spillable would credit bytes that can never be freed. Unsloth prices
+ # the drafter separately anyway.
all_block_indices = set(spill) | set(resident)
block_indices = sorted(i for i in all_block_indices if i < n_layers)
has_excluded = any(i >= n_layers for i in all_block_indices)
@@ -314,8 +301,7 @@ def spill_pattern_for(layout: ModelLayout, indices: Optional[list[int]] = None)
``\\.weight$`` likewise keeps ``ffn_(up|gate|down)\\.`` from matching
``ffn_gate_inp.weight``.
"""
- # Same set _MOE_EXPERT_RE selected, or the plan credits itself bytes the
- # emitted pattern never moves.
+ # same set _MOE_EXPERT_RE selected, or the plan credits itself bytes the emitted pattern never moves
body = "ffn_(up|gate|down|gate_up)_(exps|chexps)" if layout.is_moe else "ffn_(up|gate|down)"
if indices is None:
block = r"\d+"
diff --git a/studio/backend/core/inference/offload_planner.py b/studio/backend/core/inference/offload_planner.py
index b3409a7ecc2..ac23c7a1c2a 100644
--- a/studio/backend/core/inference/offload_planner.py
+++ b/studio/backend/core/inference/offload_planner.py
@@ -70,8 +70,9 @@ class SpillOrder(Enum):
split), which would favour FRONT/BACK over LARGEST. Hence configurable.
"""
- # Best-fit-decreasing: fewest blocks AND least overshoot. Overshoot is real
- # bandwidth -- a 209 MiB block for a 50 MiB deficit wastes 159 MiB per token.
+ # best-fit-decreasing: overshoot is real bandwidth, a 209 MiB block for a 50 MiB deficit wastes 159 MiB per token
+ # Best-fit-decreasing: fewest blocks AND least overshoot. Overshoot is real bandwidth -- a 209 MiB block for a 50
+ # MiB deficit wastes 159 MiB per token.
LARGEST_FIRST = "largest_first"
FRONT_FIRST = "front_first"
BACK_FIRST = "back_first"
@@ -79,63 +80,49 @@ class SpillOrder(Enum):
@dataclass(frozen = True)
class PlanOptions:
- # Compute buffer + CUDA context + scratch, charged on every device.
- #
- # 1 GiB was too thin and failed CONSISTENTLY: the planner fills to
- # ``budget - overhead_bytes_per_device``, leaving exactly this much free
- # whatever the budget is, so the dense 27B at depth 32768 died identically at
- # 6, 7, 8 and 10 GiB with
- #
- # ggml_backend_cuda_buffer_type_alloc_buffer: allocating 594.16 MiB
- # on device 0: cudaMalloc failed: out of memory
- #
- # The child needs the PREFILL compute buffer (594 MiB measured) plus its own
- # CUDA primary context, which took the rest of the old 1 GiB. Not benchmark
- # fragmentation: 16, 64 and 1024 MiB hog blocks all reproduced the identical
- # 594.16 MiB failure. 1.5 GiB covers the measured 1.07 GiB with margin -- a
- # measured floor, not a fitted curve, since the steady-state compute buffer is
- # flat in context (493 to 509 MiB from depth 4096 to 32768) but the prefill
- # graph's reservation is not. Erring high costs some spill (linear at
- # 5.544 ms/GiB), erring low costs the whole load.
+ # Compute buffer + CUDA context + scratch, charged on every device. 1 GiB was too thin and failed CONSISTENTLY: the
+ # planner fills to ``budget - overhead_bytes_per_device``, leaving exactly this much free whatever the budget is, so
+ # the dense 27B at depth 32768 died identically at 6, 7, 8 and 10 GiB with
+ # ggml_backend_cuda_buffer_type_alloc_buffer: allocating 594.16 MiB on device 0: cudaMalloc failed: out of memory
+ # The child needs the PREFILL compute buffer (594 MiB measured) plus its own CUDA primary context, which took the
+ # rest of the old 1 GiB. Not benchmark fragmentation: 16, 64 and 1024 MiB hog blocks all reproduced the identical
+ # 594.16 MiB failure. 1.5 GiB covers the measured 1.07 GiB with margin -- a measured floor, not a fitted curve,
+ # since the steady-state compute buffer is flat in context (493 to 509 MiB from depth 4096 to 32768) but the prefill
+ # graph's reservation is not. Erring high costs some spill (linear at 5.544 ms/GiB), erring low costs the whole
+ # load.
overhead_bytes_per_device: int = (3 * GIB) // 2
- # GPU-resident bytes NOT in the layout (a vision projector, an MTP draft
- # reserve), charged once against the pooled budget: the layout only knows the
- # target GGUF's tensor table. Subtracting from the budget also reaches
+ # GPU-resident bytes NOT in the layout (a vision projector, an MTP draft reserve), charged once against the pooled
+ # budget: the layout only knows the target GGUF's tensor table. Subtracting from the budget also reaches
# max_context_for. 0 keeps the pure-layout behaviour.
extra_resident_bytes: int = 0
- # The fixed per-device cost of a LAYER SPLIT, charged once for every device
- # after the first. Separate from overhead_bytes_per_device because it is not
- # per-device in the same sense: the first device's share is already folded
- # into the compute buffer above, which is why every other site in
- # llama_cpp.py applies it as ``max(0, n_gpus - 1) * ...`` and skips it
- # entirely at k=1. Folded into the flat per-device term instead, it withheld
- # a GiB of a single card that nothing was ever going to allocate, which is
- # deficit the planner then spilled real blocks to cover.
+ # The fixed per-device cost of a LAYER SPLIT, charged once for every device after the first. Separate from
+ # overhead_bytes_per_device because it is not per-device in the same sense: the first device's share is already
+ # folded into the compute buffer above, which is why every other site in llama_cpp.py applies it as ``max(0, n_gpus
+ # - 1) * ...`` and skips it entirely at k=1. Folded into the flat per-device term instead, it withheld a GiB of a
+ # single card that nothing was ever going to allocate, which is deficit the planner then spilled real blocks to
+ # cover.
pipeline_overhead_bytes: int = 0
- # Host RAM this planner refuses to spend, so a spill does not push the box
- # into swap.
+ # Host RAM this planner refuses to spend, so a spill does not push the box into swap.
host_ram_headroom_bytes: int = 2 * GIB
context_policy: ContextPolicy = ContextPolicy.NEVER_REDUCE
min_ctx: int = 4096
spill_order: SpillOrder = SpillOrder.LARGEST_FIRST
allow_lm_head_spill: bool = True
- # What the host brings to bear on spilled weights. Spilled generation runs on
- # the CPU backend -- ggml only moves an op to the GPU at batch >= 32
- # (ggml-cuda.cu, op_offload_min_batch_size) and decode is batch 1 -- so the
- # penalty scales with core count: 2.42 / 5.83 / 11.82 / 14.94 t/s at
- # 4 / 16 / 64 / 192 threads.
+ # spilled generation runs on the CPU backend (ggml only moves an op to the GPU at batch >= 32 and decode is batch 1)
+ # What the host brings to bear on spilled weights. Spilled generation runs on the CPU backend -- ggml only moves an
+ # op to the GPU at batch >= 32 (ggml-cuda.cu, op_offload_min_batch_size) and decode is batch 1 -- so the penalty
+ # scales with core count: 2.42 / 5.83 / 11.82 / 14.94 t/s at 4 / 16 / 64 / 192 threads.
host: HostProfile = field(default_factory = HostProfile)
- # q8_0 measured 35% slower generation, and without GGML_CUDA_FA_ALL_QUANTS
- # only four MATCHED K/V combinations are compiled (a mismatched pair falls
- # to CPU and stalls). Off by default; matched pairs only when enabled.
+ # q8_0 measured 35% slower generation, and without GGML_CUDA_FA_ALL_QUANTS only four MATCHED K/V combinations are
+ # compiled (a mismatched pair falls to CPU and stalls). Off by default; matched pairs only when enabled.
allow_kv_quant: bool = False
kv_quant_type: str = "q8_0"
- # The caller passed -nkvo (or a false LLAMA_ARG_KV_OFFLOAD), so llama.cpp puts
- # the WHOLE cache on the host: offload is one scalar and the buffer type falls
- # back to the CPU one for every layer (llama-kv-cache.cpp:210-219), same branch
- # in the recurrent and DSV4 caches. The cache and the recurrent state move out
- # of the VRAM footprint and into the host one; charging them to VRAM anyway
- # would spill FFN blocks for a deficit the child never has.
+ # with -nkvo llama.cpp puts the WHOLE cache on the host (offload is one scalar and the buffer type falls back to CPU
+ # for every layer)
+ # The caller passed -nkvo (or a false LLAMA_ARG_KV_OFFLOAD), so llama.cpp puts the WHOLE cache on the host: offload
+ # is one scalar and the buffer type falls back to the CPU one for every layer (llama-kv-cache.cpp:210-219), same
+ # branch in the recurrent and DSV4 caches. The cache and the recurrent state move out of the VRAM footprint and into
+ # the host one; charging them to VRAM anyway would spill FFN blocks for a deficit the child never has.
kv_on_host: bool = False
@@ -143,8 +130,8 @@ class PlanOptions:
class Plan:
"""What to launch with, and why."""
- # False means "emit nothing new": either the planner abstained or the load
- # needs no help. Always safe, since llama.cpp's own defaults then apply.
+ # False means "emit nothing new": either the planner abstained or the load needs no help. Always safe, since
+ # llama.cpp's own defaults then apply.
changed: bool = False
n_ctx: int = 0
ot_patterns: tuple[str, ...] = field(default_factory = tuple)
@@ -153,14 +140,15 @@ class Plan:
cache_type_v: Optional[str] = None
spilled_blocks: tuple[int, ...] = field(default_factory = tuple)
spilled_lm_head: bool = False
- # No rung fits. mmap has to stay, because it is the only thing that makes an
- # over-commit pageable rather than OOM-killed.
+ # no rung fits; mmap has to stay, since it is the only thing that makes an over-commit pageable rather than
+ # OOM-killed
+ # No rung fits. mmap has to stay, because it is the only thing that makes an over-commit pageable rather than
+ # OOM-killed.
insufficient: bool = False
vram_bytes: int = 0
host_bytes: int = 0
- # Predicted extra ms per generated token versus fully resident, on the host
- # this was planned for. 0.0 when nothing is spilled. Reported so callers can
- # surface the real cost instead of implying a spill is free.
+ # Predicted extra ms per generated token versus fully resident, on the host this was planned for. 0.0 when nothing
+ # is spilled. Reported so callers can surface the real cost instead of implying a spill is free.
predicted_gen_penalty_ms: float = 0.0
reason: str = ""
@@ -195,8 +183,7 @@ def _select_blocks(
while freed < deficit and remaining:
if order is SpillOrder.LARGEST_FIRST:
residual = deficit - freed
- # Prefer the SMALLEST block that closes the gap: the last pick must
- # not overshoot by a whole large block.
+ # Prefer the SMALLEST block that closes the gap: the last pick must not overshoot by a whole large block.
covering = [b for b in remaining if b.spillable_bytes >= residual]
pick = min(covering, key = lambda b: b.spillable_bytes) if covering else remaining[0]
else:
@@ -396,9 +383,8 @@ def plan_placement(
if not layout.complete or not vram_bytes_per_device:
return Plan(reason = "layout or device inventory incomplete, leaving llama.cpp defaults")
if opts.host.unified_memory:
- # One pool: "spilling" renames bytes on the same chips and frees nothing.
- # Metal also keeps mmap zero copy (buffer_from_host_ptr), so the no-mmap
- # rule inverts there too.
+ # One pool: "spilling" renames bytes on the same chips and frees nothing. Metal also keeps mmap zero copy
+ # (buffer_from_host_ptr), so the no-mmap rule inverts there too.
return Plan(reason = "unified memory host, spilling frees no device memory")
budget = _usable_vram(vram_bytes_per_device, opts)
if budget <= 0:
@@ -410,8 +396,8 @@ def plan_placement(
if n_ctx <= 0:
return Plan(reason = "no usable context length")
- # PREFER_RESIDENT gets its say before the ladder: a smaller fully resident
- # context outruns a larger spilled one, when the caller allows it to move.
+ # PREFER_RESIDENT gets its say before the ladder: a smaller fully resident context outruns a larger spilled one,
+ # when the caller allows it to move.
if (
opts.context_policy is ContextPolicy.PREFER_RESIDENT
and all_resident_bytes(
@@ -547,10 +533,10 @@ def _per_device_usage(
) -> tuple[Optional[str], list[int], list[list[int]]]:
if len(vram_bytes_per_device) <= 1:
return None, [], []
- # These three shapes -- recurrent hybrid, n_attention_layers short of
- # n_layers, sliding window -- are only a problem when the cache has to be
- # spread evenly for want of anything better. A vector removes that guess;
- # without one they still abstain.
+ # recurrent hybrid, n_attention_layers short of n_layers
+ # These three shapes -- recurrent hybrid, n_attention_layers short of n_layers, sliding window -- are only a problem
+ # when the cache has to be spread evenly for want of anything better. A vector removes that guess; without one they
+ # still abstain.
uneven_cache = (
layout.recurrent_bytes > 0 or layout.n_attention_layers != layout.n_layers or layout.has_swa
)
@@ -610,8 +596,8 @@ def _per_device_usage(
used += kv_by_layer[row]
if row not in spilled_indices:
used += block.spillable_bytes
- # Everything outside the layout sits on the main device, which is
- # devices[0] once -sm none has already pruned the list.
+ # Everything outside the layout sits on the main device, which is devices[0] once -sm none has already pruned
+ # the list.
if device == 0:
used += max(0, opts.extra_resident_bytes)
usage.append(used)
@@ -854,9 +840,8 @@ def _plan_at(
),
)
- # Every block spilled and still short: lm_head is the last rung. It costs
- # 16% here against 43% if taken first, because FFN offload has already made
- # generation host-bandwidth-bound.
+ # lm_head is the last rung: it costs 16% here against 43% taken first, because FFN offload has already made
+ # generation host-bandwidth-bound
if opts.allow_lm_head_spill and layout.lm_head_bytes:
if freed + layout.lm_head_bytes >= deficit:
uneven = _per_device_shortfall(
@@ -913,12 +898,10 @@ def _finish(
patterns: list[str] = []
indices = sorted(b.index for b in chosen)
if indices:
- # One global pattern when every spillable block is going -- shorter, and
- # the form the benchmarks used. NOT when the GGUF carries blocks the layout
- # dropped: the unbounded \d+ would also match the trailing nextn/MTP
- # blocks, whose ffn_*_exps load the moment a draft is engaged. That moves
- # bytes neither host_bytes nor the deficit counted (so the mmap decision is
- # made on an undercount) and drags the draft FFN onto the CPU backend.
+ # One global pattern when every spillable block is going -- shorter, and the form the benchmarks used. NOT when
+ # the GGUF carries blocks the layout dropped: the unbounded \d+ would also match the trailing nextn/MTP blocks,
+ # whose ffn_*_exps load the moment a draft is engaged. That moves bytes neither host_bytes nor the deficit
+ # counted (so the mmap decision is made on an undercount) and drags the draft FFN onto the CPU backend.
spillable = [b.index for b in layout.blocks if b.spillable_bytes > 0]
all_of_them = set(indices) == set(spillable) and not layout.has_excluded_blocks
patterns.append(spill_pattern_for(layout, None if all_of_them else indices))
@@ -928,13 +911,12 @@ def _finish(
spilled_bytes = sum(b.spillable_bytes for b in chosen) + (
layout.lm_head_bytes if spill_lm_head else 0
)
- # token_embd is host-resident on every launch, so it is host RAM this plan
- # has to be able to pay for even when nothing is spilled.
+ # token_embd is host-resident on every launch, so it is host RAM this plan must pay for even when nothing is spilled
host_bytes = layout.token_embd_bytes + spilled_bytes
if opts.kv_on_host:
- # -nkvo moved the cache and the recurrent state out of VRAM, not out of
- # existence: they are host RAM now, and the mmap decision below has to see
- # them or it answers against a footprint short by the whole cache.
+ # -nkvo moved the cache and recurrent state out of VRAM
+ # -nkvo moved the cache and the recurrent state out of VRAM, not out of existence: they are host RAM now, and
+ # the mmap decision below has to see them or it answers against a footprint short by the whole cache.
host_bytes += (
cache_bytes(layout, n_ctx, kv_quantised = quantised, kv_bytes_floor = kv_bytes_floor)
+ layout.recurrent_bytes
@@ -950,8 +932,8 @@ def _finish(
- spilled_bytes
)
- # mmap costs 2 to 4.6x on host-resident weight reads, so turn it off -- but only
- # when host RAM holds the host side; otherwise mmap keeps an over-commit pageable.
+ # mmap costs 2 to 4.6x on host-resident weight reads, so turn it off -- but only when host RAM holds the host side;
+ # otherwise mmap keeps an over-commit pageable.
if host_ram_bytes is None:
load_mode_none = False
else:
@@ -964,8 +946,8 @@ def _finish(
n_ctx = n_ctx,
ot_patterns = tuple(patterns),
load_mode_none = load_mode_none,
- # Matched pairs only: an unmatched K/V combination is not compiled
- # without GGML_CUDA_FA_ALL_QUANTS and silently falls back to CPU.
+ # matched pairs only: an unmatched K/V combination is not compiled without GGML_CUDA_FA_ALL_QUANTS and silently
+ # falls back to CPU
cache_type_k = cache_type,
cache_type_v = cache_type,
spilled_blocks = tuple(indices),
diff --git a/studio/backend/core/inference/openai_auto_download.py b/studio/backend/core/inference/openai_auto_download.py
index 57d2a43e701..49f6e872521 100644
--- a/studio/backend/core/inference/openai_auto_download.py
+++ b/studio/backend/core/inference/openai_auto_download.py
@@ -38,21 +38,20 @@ logger = get_logger(__name__)
# Keep the Hub probe short so a slow Hub can't stall the request path.
_MODEL_INFO_TIMEOUT_S = 8.0
-# auth_check and hf_hub_download take no timeout of their own and run while the
-# provisional slot is held, so an unresponsive Hub would pin the single flight. The
-# code probe fetches up to three configs, so it gets more room than the auth call.
+# auth_check and hf_hub_download take no timeout of their own and run while the provisional slot is held, so an
+# unresponsive Hub would pin the single flight. The code probe fetches up to three configs, so it gets more room than
+# the auth call.
_CODE_PROBE_TIMEOUT_S = 20.0
# Headroom left free after the download, so filling the disk can't wedge the box.
_DISK_RESERVE_BYTES = 5 * 1024**3
_WATCH_POLL_S = 2.0
# A stalled watcher must not pin the single-flight slot forever.
_MAX_WATCH_S = 24 * 60 * 60
-# Past the watch window the row is resolved, so poll only to see whether the
-# worker is still alive and still owns the slot.
+# past the watch window the row is resolved, so poll only to see whether the worker is alive and still owns the slot
_TIMED_OUT_POLL_S = 60.0
_RETRY_AFTER_S = 30
-# Long enough for a client honouring Retry-After to come back and be told, short
-# enough that one that never returns cannot hold the slot.
+# long enough for a client honouring Retry-After to come back and be told, short enough that one that never returns
+# cannot hold the slot
_FAILED_HOLD_S = 3 * _RETRY_AFTER_S
_MAX_LISTED_VARIANTS = 8
@@ -76,8 +75,9 @@ class _Active:
expected_bytes: int = 0
monitor_id: Optional[str] = None
started_at: float = 0.0
- # Set when the worker failed. Held until a retry surfaces it: Retry-After is far
- # longer than the watcher poll, so the client would restart the same failing download.
+ # held until a retry surfaces it: Retry-After is far longer than the watcher poll
+ # Set when the worker failed. Held until a retry surfaces it: Retry-After is far longer than the watcher poll, so
+ # the client would restart the same failing download.
error: Optional[str] = None
failed_at: float = 0.0
@@ -171,9 +171,9 @@ def looks_like_quant(variant: Optional[str]) -> bool:
return False
# _extract_quant_label can append a bpw modifier (IQ4_XS-3.53bpw); still a quant.
label = re.sub(r"-[0-9]+(?:\.[0-9]+)?bpw$", "", variant.strip(), flags = re.IGNORECASE)
- # A qualified key is one of OUR advertised rows: a path (``distilled/model-Q6_K``) or an H3
- # root stem (``minimax_h3_ref2va_pruned-Q6_K``). Explicit, so it must MISS when absent;
- # falling through served the caller a different checkpoint under the requested id.
+ # A qualified key is one of OUR advertised rows: a path (``distilled/model-Q6_K``) or an H3 root stem
+ # (``minimax_h3_ref2va_pruned-Q6_K``). Explicit, so it must MISS when absent; falling through served the caller a
+ # different checkpoint under the requested id.
normalized = label.replace("\\", "/")
if "/" in normalized or is_h3_denoiser_variant_key(normalized):
return True
@@ -289,16 +289,16 @@ def _gguf_variants(siblings, repo_id: str = "") -> dict[str, int]:
if not name.lower().endswith(".gguf"):
continue
label = _extract_quant_label(name)
- # The identity the PLAN is keyed on. A repo holding several checkpoints at one quant
- # advertises a qualified key per checkpoint, and keying this map on the bare label left
- # every one of those rows a hard miss here: a 404 instead of the download.
+ # The identity the PLAN is keyed on. A repo holding several checkpoints at one quant advertises a qualified key
+ # per checkpoint, and keying this map on the bare label left every one of those rows a hard miss here: a 404
+ # instead of the download.
quant = gguf_variant_key(name)
if not looks_like_quant(quant):
- # With no recognized quant token the extractors part ways: this one takes
- # the last hyphenated segment ("7b" of llama-7b) while the plan and worker
- # key the whole stem, so advertising ours dispatches an unresolvable variant.
+ # With no recognized quant token the extractors part ways: this one takes the last hyphenated segment ("7b"
+ # of llama-7b) while the plan and worker key the whole stem, so advertising ours dispatches an unresolvable
+ # variant.
quant = canonical_quant_label(name) or quant
- # The endian test reads a quant TOKEN, so it gets the label, not the path-qualified key.
+ # the endian test reads a quant TOKEN, so it gets the label, not the path-qualified key
if (
_is_mmproj(name)
or _is_mtp_drafter(name)
@@ -406,9 +406,11 @@ async def _watch(active: _Active, hf_token: Optional[str]) -> None:
state, error = await _job_state(active.repo_id, active.variant)
if state in ("running", "cancelling", "unknown"):
if timed_out:
- # A running worker still owns the slot: releasing on the clock alone
- # would admit a second multi-GB download beside it. "unknown" cannot
- # confirm it is alive, so release then, or a broken probe wedges us.
+ # a running worker still owns the slot, and releasing on the clock alone would admit a second
+ # multi-GB download beside it
+ # A running worker still owns the slot: releasing on the clock alone would admit a second multi-GB
+ # download beside it. "unknown" cannot confirm it is alive, so release then, or a broken probe
+ # wedges us.
if state == "unknown":
return
continue
@@ -429,17 +431,15 @@ async def _watch(active: _Active, hf_token: Optional[str]) -> None:
api_monitor.finish(active.monitor_id, status = "cancelled")
return
if state == "complete":
- # No invalidate here: finalize_worker_exit already dropped the cache and
- # warmed it; a second would mark that fresh scan stale and push a
- # synchronous rescan onto the client's retry.
+ # No invalidate here: finalize_worker_exit already dropped the cache and warmed it; a second would mark
+ # that fresh scan stale and push a synchronous rescan onto the client's retry.
api_monitor.finish(active.monitor_id, status = "completed")
elif state == "idle":
# The job vanished without a terminal state (worker killed).
api_monitor.fail_open(active.monitor_id, "Download did not complete")
else:
api_monitor.fail_open(active.monitor_id, error or f"Download {state}")
- # Keep the slot so the next retry is told it failed instead of
- # silently restarting the same download.
+ # keep the slot so the next retry is told it failed instead of silently restarting the same download
active.error = error or f"Download {state}"
active.failed_at = time.monotonic()
return
@@ -482,9 +482,9 @@ async def _is_downloadable_model(repo_id: str, hf_token: Optional[str]) -> bool:
info = await asyncio.to_thread(_probe)
except Exception:
return False
- # The same filter admission uses, not a bare .gguf test: mmproj, MTP drafters and
- # big-endian builds are companions, not quants. Answering otherwise would hold an
- # ordinary foreign label at model_download_busy for an unrelated download.
+ # The same filter admission uses, not a bare .gguf test: mmproj, MTP drafters and big-endian builds are companions,
+ # not quants. Answering otherwise would hold an ordinary foreign label at model_download_busy for an unrelated
+ # download.
servable = bool(_gguf_variants(getattr(info, "siblings", None), repo_id))
if not servable:
_mark_not_servable(repo_id, hf_token)
@@ -520,7 +520,7 @@ async def maybe_auto_download(
if _is_not_servable(repo_id, hf_token) and not looks_like_quant(wanted_variant):
return None
- # Settle the single-flight slot before the network, so retries during a download stay cheap.
+ # settle the single-flight slot before the network, so retries during a download stay cheap
busy: Optional[_Active] = None
with _lock:
current = _active
@@ -539,9 +539,9 @@ async def maybe_auto_download(
_active = provisional
if busy is not None:
- # Refusing before the probe blocks ordinary drop-in traffic: a namespaced label
- # that is no downloadable GGUF repo (LiteLLM/OpenRouter style) would be told to
- # wait out a multi-hour download. Only a downloadable label is a 2nd download.
+ # Refusing before the probe blocks ordinary drop-in traffic: a namespaced label that is no downloadable GGUF
+ # repo (LiteLLM/OpenRouter style) would be told to wait out a multi-hour download. Only a downloadable label is
+ # a 2nd download.
if not await _is_downloadable_model(repo_id, hf_token):
return None
return AutoDownloadRefusal(
@@ -575,7 +575,6 @@ async def maybe_auto_download(
code = "model_download_failed",
message = f"Downloading '{requested_model}' failed: {error or 'unknown error'}",
)
- # complete/idle/cancelled: the watcher is about to free the slot, so retry once more.
return _downloading_refusal(
_public_label(adopted.repo_id, adopted.variant),
100.0 if state == "complete" else None,
@@ -635,10 +634,8 @@ async def _admit_and_start(
return _gated_refusal(repo_id)
if status == 404:
_mark_not_servable(repo_id, hf_token)
- # Unknown to the Hub reads as a foreign label; only an explicit quant makes it ours.
if not looks_like_quant(wanted_variant):
return None
- # A private repo reads as absent without a token; don't confirm either way.
return AutoDownloadRefusal(
status = 404,
code = "model_not_found",
@@ -659,7 +656,6 @@ async def _admit_and_start(
if getattr(info, "gated", False) and await _bounded_probe(
_auth_denied, repo_id, hf_token, timeout = _MODEL_INFO_TIMEOUT_S, default = False
):
- # Metadata for a gated repo is not file access; unchecked, the config read below lies.
_release(active)
return _gated_refusal(repo_id)
@@ -681,9 +677,9 @@ async def _admit_and_start(
# trust_remote_code gate: _config_has_auto_map is tri-state, so refuse on True and on None.
from utils.security.consent import _config_has_auto_map
- # _hub_token, not the raw token: None lets huggingface_hub fall back to a cached
- # server login, so a caller-named repo would be probed with this server's identity.
- # Defaults to None on timeout, which refuses: unchecked is not cleared.
+ # _hub_token, not the raw token: None lets huggingface_hub fall back to a cached server login, so a caller-named
+ # repo would be probed with this server's identity. Defaults to None on timeout, which refuses: unchecked is not
+ # cleared.
has_auto_map = await _bounded_probe(
_config_has_auto_map,
repo_id,
@@ -775,7 +771,7 @@ def preferred_quant(labels) -> Optional[str]:
"""
from utils.models.model_config import _pick_best_gguf
- # _pick_best_gguf ranks filenames and matches upper-case tokens, so feed "<LABEL>.gguf".
+ # _pick_best_gguf ranks filenames and matches upper-case tokens, so feed "<LABEL>.gguf"
synthetic: dict[str, str] = {}
for name in labels:
synthetic.setdefault(f"{name.upper()}.gguf", name)
@@ -795,8 +791,9 @@ def _bare_quant_alias(wanted: str, lowered: dict[str, str]) -> Optional[str]:
target = (wanted or "").strip().lower()
if not target:
return None
- # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant
- # names both partitions, so it must miss rather than serve one of them.
+ # PATH-qualified keys only: an H3 root stem's bare quant names both partitions
+ # PATH-qualified keys only, not is_qualified_gguf_variant_key: an H3 root stem's bare quant names both partitions,
+ # so it must miss rather than serve one of them.
matches = [
name
for key, name in lowered.items()
@@ -814,26 +811,24 @@ def _match_variant(wanted: Optional[str], variants: dict[str, int]) -> Optional[
manual load, matching what the local resolver does with the same tag.
"""
if wanted:
- # Exact first, whatever shape: a repo of generically named GGUFs has real
- # variants like "llama-13b" that are valid worker keys but not quant-shaped,
- # and defaulting past one would fetch a model nobody asked for.
+ # Exact first, whatever shape: a repo of generically named GGUFs has real variants like "llama-13b" that are
+ # valid worker keys but not quant-shaped, and defaulting past one would fetch a model nobody asked for.
lowered = {name.lower(): name for name in variants}
exact = lowered.get(wanted.strip().lower())
if exact is None:
- # Same bare-quant fallback the plan lookup makes, and it has to be made HERE too:
- # admission rejects against this map first, so a repo that files every quant under
- # one shared container answered a legacy org/repo:Q4_K_M with a 404 and the worker's
- # fallback was never reached. Unambiguous only, for the same reason.
+ # Same bare-quant fallback the plan lookup makes, and it has to be made HERE too: admission rejects against
+ # this map first, so a repo that files every quant under one shared container answered a legacy
+ # org/repo:Q4_K_M with a 404 and the worker's fallback was never reached. Unambiguous only, for the same
+ # reason.
exact = _bare_quant_alias(wanted, lowered)
if exact is not None or looks_like_quant(wanted):
# A quant-shaped suffix that matches nothing is a miss, never a swap.
return exact
- # A BARE org/repo means the ROOT checkpoint, so a qualified sibling must not be ranked
- # against it: preferred_quant is order-sensitive, and once the map carried a key per
- # checkpoint a repo with distilled/model-Q6_K beside model-Q6_K could serve the sibling for
- # a bare id -- the same id that resolves to the root locally. Same filter
- # local_model_resolver._local_gguf_entry applies, so both resolvers answer one id one way.
- # A repo with nothing at the root falls back to the whole set rather than refusing.
+ # A BARE org/repo means the ROOT checkpoint, so a qualified sibling must not be ranked against it: preferred_quant
+ # is order-sensitive, and once the map carried a key per checkpoint a repo with distilled/model-Q6_K beside
+ # model-Q6_K could serve the sibling for a bare id -- the same id that resolves to the root locally. Same filter
+ # local_model_resolver._local_gguf_entry applies, so both resolvers answer one id one way. A repo with nothing at
+ # the root falls back to the whole set rather than refusing.
unqualified = {name: size for name, size in variants.items() if "/" not in name}
return preferred_quant(unqualified or variants)
@@ -872,7 +867,6 @@ async def _dispatch(
_release(active)
status = getattr(exc, "status_code", None)
if status == 409:
- # A manual load or hub download already owns this repo.
return busy
logger.warning("auto-download: could not start %r: %s", label, exc)
return AutoDownloadRefusal(
@@ -881,17 +875,17 @@ async def _dispatch(
message = f"Could not start downloading '{requested_model}'.",
)
- # accepted=False means no worker launched, so report the conflict instead of taking the slot.
+ # accepted=False means no worker launched, so report the conflict instead of taking the slot
if isinstance(dispatched, dict) and not dispatched.get("accepted", True):
_release(active)
logger.info("auto-download: dispatch refused for %s (%s)", label, dispatched.get("state"))
return busy
monitor_id = api_monitor.record_lifecycle(
- # Reason "api" since only /v1 reaches auto-download, but that is not API-key
- # traffic: Unsloth's chat calls /v1 on a JWT, and marking its download would pop
- # the overlay mid-chat. So attribution comes from the request, plus its caller,
- # since the row is shared.
+ # only /v1 reaches auto-download, but that is not API-key traffic
+ # Reason "api" since only /v1 reaches auto-download, but that is not API-key traffic: Unsloth's chat calls /v1
+ # on a JWT, and marking its download would pop the overlay mid-chat. So attribution comes from the request, plus
+ # its caller, since the row is shared.
event = "download",
model = label,
reason = "api",
diff --git a/studio/backend/core/inference/openai_codex_auth.py b/studio/backend/core/inference/openai_codex_auth.py
index fbc4bccb215..a023fc46947 100644
--- a/studio/backend/core/inference/openai_codex_auth.py
+++ b/studio/backend/core/inference/openai_codex_auth.py
@@ -237,17 +237,16 @@ def save_oauth_bundle(provider_id: str, bundle: dict[str, Any]) -> None:
json.dumps(bundle, separators = (",", ":")),
)
if previous and previous.get("account_id") != bundle.get("account_id"):
- # Rebound to a different ChatGPT account, whose plan lists different slugs. No
- # later request is guaranteed to notice: the picker only refreshes while its form
- # is open, and the chat gate only refetches a catalog it does not already have.
- # Imported here because the client module imports this one at load time.
+ # Rebound to a different ChatGPT account, whose plan lists different slugs. No later request is guaranteed to
+ # notice: the picker only refreshes while its form is open, and the chat gate only refetches a catalog it does
+ # not already have. Imported here because the client module imports this one at load time.
from core.inference.openai_codex_client import (
forget_subscription_models,
mark_subscription_catalog_stale,
)
forget_subscription_models(provider_id)
- # The emptiness here is deliberate, not a cold start: until this account's own
- # catalog is read, the saved models still describe the previous one.
+ # deliberate, not a cold start: until this account's catalog is read, the saved models still describe the
+ # previous one
mark_subscription_catalog_stale(provider_id)
@@ -290,8 +289,8 @@ def auth_status(provider_id: str) -> str:
bundle = load_oauth_bundle(provider_id)
if not bundle:
return "disconnected"
- # An expired access token is still usable after refresh. Only a permanent
- # refresh rejection should ask the user to reconnect.
+ # An expired access token is still usable after refresh. Only a permanent refresh rejection should ask the user to
+ # reconnect.
return "reauthorization_required" if bundle.get("reauthorization_required") else "connected"
@@ -427,11 +426,11 @@ async def _start_browser_flow(
break
except OSError:
continue
- # The callback URL must match the verifier exchange even when no listener
- # can bind. Manual completion remains valid with the canonical URL.
+ # The callback URL must match the verifier exchange even when no listener can bind. Manual completion remains valid
+ # with the canonical URL.
callback_port = bound_port or OPENAI_CODEX_LOOPBACK_PORTS[0]
- # OpenAI registers localhost redirect URIs; the listener itself remains
- # pinned to 127.0.0.1 so it can never accept non-loopback traffic.
+ # OpenAI registers localhost redirect URIs; the listener itself remains pinned to 127.0.0.1 so it can never accept
+ # non-loopback traffic.
flow.redirect_uri = f"http://localhost:{callback_port}{OPENAI_CODEX_CALLBACK_PATH}"
flow.authorization_url = (
OPENAI_CODEX_AUTHORIZE_URL
@@ -694,10 +693,9 @@ def delete_oauth_bundle(provider_id: str) -> None:
credential_secrets.OPENAI_CODEX_OAUTH_KIND,
provider_id,
)
- # Disconnecting erases the account identity that a later save would be compared
- # against, so the next authorization cannot tell it is a different account. The saved
- # models outlive the bundle, so mark them unproven now rather than letting them read
- # as cold-start evidence under whoever connects next.
+ # Disconnecting erases the account identity that a later save would be compared against, so the next authorization
+ # cannot tell it is a different account. The saved models outlive the bundle, so mark them unproven now rather than
+ # letting them read as cold-start evidence under whoever connects next.
from core.inference.openai_codex_client import mark_subscription_catalog_stale
mark_subscription_catalog_stale(provider_id)
@@ -784,8 +782,8 @@ async def _persist_terminal_flow(flow: OAuthFlow) -> None:
async with provider_oauth_write_guard(flow.provider_id):
set_oauth_flow_marker_status(flow.provider_id, flow.marker, flow.status, flow.message)
except CodexAuthError:
- # Keep the originating worker's terminal state even if another credential
- # write holds the cross-worker lock long enough for this best-effort update.
+ # Keep the originating worker's terminal state even if another credential write holds the cross-worker lock long
+ # enough for this best-effort update.
return
@@ -865,8 +863,7 @@ async def resolve_access(
if not force_refresh and bundle["expires_at"] > time.time() + _REFRESH_SKEW_SECONDS:
return bundle["access_token"], bundle["account_id"]
- # Multiple Unsloth workers may share the installation DB. Serialize with
- # disconnect/delete and re-read so a completed refresh is reused.
+ # multiple workers may share the installation DB
async with provider_oauth_write_guard(provider_id):
bundle = load_oauth_bundle(provider_id)
if not bundle:
@@ -899,9 +896,9 @@ async def resolve_access(
raise CodexAuthError("ChatGPT connection was disconnected during refresh.")
if current.get("refresh_token") != previous_refresh_token:
return current["access_token"], current["account_id"]
- # A refresh rotates credentials, it does not rebind the connection, so the
- # record of which account the saved models were proven for carries over.
- # _validate_token_payload only knows about the token fields.
+ # a refresh rotates credentials without rebinding the connection
+ # A refresh rotates credentials, it does not rebind the connection, so the record of which account the saved
+ # models were proven for carries over. _validate_token_payload only knows about the token fields.
proof = current.get("catalog_account_id")
if proof is not None and refreshed.get("account_id") == current.get("account_id"):
refreshed["catalog_account_id"] = proof
diff --git a/studio/backend/core/inference/openai_codex_client.py b/studio/backend/core/inference/openai_codex_client.py
index a6a9e1b715d..f05d1b43220 100644
--- a/studio/backend/core/inference/openai_codex_client.py
+++ b/studio/backend/core/inference/openai_codex_client.py
@@ -232,8 +232,9 @@ _MODELS_CACHE_MAX_ENTRIES = 32
_models_cache: dict[str, tuple[float, list[dict[str, Any]]]] = {}
# Outlives the cache TTL: a slug listed for this plan stays saveable afterwards.
_offered_models: dict[str, dict[str, dict[str, Any]]] = {}
-# Which ChatGPT account each cached catalog belongs to; a reauthorization can rebind
-# a connection to a different account whose plan lists different slugs.
+# a reauthorization can rebind a connection to a different account whose plan lists different slugs
+# Which ChatGPT account each cached catalog belongs to; a reauthorization can rebind a connection to a different account
+# whose plan lists different slugs.
_catalog_accounts: dict[str, str] = {}
@@ -246,9 +247,11 @@ def _normalize_subscription_model(item: Any) -> dict[str, Any] | None:
display_name = item.get("display_name")
context_window = item.get("context_window")
modalities = item.get("input_modalities")
- # Every other field here tolerates whatever upstream sends, and this one has to as
- # well: `or []` only covers a falsy value, so a scalar raised TypeError out of the
- # whole call and cost the catalog every other entry too, not just this one.
+ # `or []` only covers a falsy value, so a scalar raised TypeError out of the whole call and cost the catalog every
+ # other entry too
+ # Every other field here tolerates whatever upstream sends, and this one has to as well: `or []` only covers a falsy
+ # value, so a scalar raised TypeError out of the whole call and cost the catalog every other entry too, not just
+ # this one.
levels = item.get("supported_reasoning_levels")
efforts = [
level["effort"]
@@ -258,8 +261,8 @@ def _normalize_subscription_model(item: Any) -> dict[str, Any] | None:
return {
"id": slug,
"display_name": display_name if isinstance(display_name, str) and display_name else slug,
- # bool is a subclass of int, so a JSON `true` would otherwise be reported to the
- # picker as a context length of its own.
+ # bool is a subclass of int, so a JSON `true` would otherwise be reported to the picker as a context length of
+ # its own.
"context_length": (
context_window
if isinstance(context_window, int) and not isinstance(context_window, bool)
@@ -267,10 +270,10 @@ def _normalize_subscription_model(item: Any) -> dict[str, Any] | None:
),
"vision": "image" in modalities if isinstance(modalities, list) else None,
"reasoning_efforts": efforts,
- # "hide" marks a slug no picker should offer (codex-auto-review, and models that
- # age out of the list). It is a presentation flag, not a revocation: the account
- # can still call one it already saved, so the entry is kept and marked instead of
- # dropped, which is what lets callers tell "not offered" from "not on this plan".
+ # "hide" is a presentation flag, not a revocation
+ # "hide" marks a slug no picker should offer (codex-auto-review, and models that age out of the list). It is a
+ # presentation flag, not a revocation: the account can still call one it already saved, so the entry is kept and
+ # marked instead of dropped, which is what lets callers tell "not offered" from "not on this plan".
"listed": item.get("visibility") == "list",
}
@@ -296,21 +299,21 @@ def offered_subscription_model_ids(provider_id: str) -> set[str]:
}
-# Connections whose catalog was dropped because the account behind them changed. The
-# absence is deliberate, so it must not read as "nothing fetched yet" and license the
-# previous account's saved slugs.
+# the absence is deliberate, so it must not read as "nothing fetched yet" and license the previous account's saved slugs
+# Connections whose catalog was dropped because the account behind them changed. The absence is deliberate, so it must
+# not read as "nothing fetched yet" and license the previous account's saved slugs.
_stale_catalogs: set[str] = set()
-# The ticket the newest catalog read for each connection is holding, so a read that was
-# overtaken (by a rebind, a disconnect, or a newer read) cannot commit its result over
-# the one that replaced it.
+# the ticket the newest catalog read holds
+# The ticket the newest catalog read for each connection is holding, so a read that was overtaken (by a rebind, a
+# disconnect, or a newer read) cannot commit its result over the one that replaced it.
_catalog_requests: dict[str, int] = {}
-# Tickets are drawn from one counter shared by every connection rather than counting up
-# per connection. Nothing reads the number itself, only whether it still matches, and a
-# value that is never reissued is what lets forget_subscription_models drop the entry
-# instead of leaving a larger one behind: a read still in flight then finds no ticket at
-# all, and the next read draws a number no earlier read can be holding.
+# one shared counter, not per connection: nothing reads the number, only whether it still matches
+# Tickets are drawn from one counter shared by every connection rather than counting up per connection. Nothing reads
+# the number itself, only whether it still matches, and a value that is never reissued is what lets
+# forget_subscription_models drop the entry instead of leaving a larger one behind: a read still in flight then finds no
+# ticket at all, and the next read draws a number no earlier read can be holding.
_catalog_request_serial = 0
@@ -368,9 +371,8 @@ def offered_subscription_model(provider_id: str, model_id: str) -> dict[str, Any
def forget_subscription_models(provider_id: str) -> None:
- # Retire any read still in flight: its result describes what was just dropped. The
- # ticket is dropped rather than bumped, so this releases the entry instead of
- # replacing it; see the counter above for why that is still safe.
+ # Retire any read still in flight: its result describes what was just dropped. The ticket is dropped rather than
+ # bumped, so this releases the entry instead of replacing it; see the counter above for why that is still safe.
_catalog_requests.pop(provider_id, None)
_models_cache.pop(provider_id, None)
_offered_models.pop(provider_id, None)
@@ -390,8 +392,8 @@ async def list_subscription_models(
rolled out since the last fetch is exactly what that click is asking about.
"""
if _catalog_accounts.get(provider_id) not in (None, account_id):
- # Reauthorized against a different account: the previous plan's catalog says
- # nothing about this one, and nothing else clears it on the reconnect path.
+ # Reauthorized against a different account: the previous plan's catalog says nothing about this one, and nothing
+ # else clears it on the reconnect path.
forget_subscription_models(provider_id)
if not force:
cached = cached_subscription_models(provider_id)
@@ -418,10 +420,9 @@ async def list_subscription_models(
params = {"client_version": OPENAI_CODEX_CLIENT_VERSION},
)
if response.status_code == 401:
- # Upstream can reject a token before its recorded expiry while the refresh
- # credential is still good. The responses transport spends one forced refresh
- # on that; without the same here the editor cannot load a catalog for a
- # connection whose chats recover fine.
+ # Upstream can reject a token before its recorded expiry while the refresh credential is still good. The
+ # responses transport spends one forced refresh on that; without the same here the editor cannot load a
+ # catalog for a connection whose chats recover fine.
try:
access_token, account_id = await codex_auth.resolve_access(
provider_id,
@@ -434,10 +435,10 @@ async def list_subscription_models(
status = 401,
) from exc
except Exception as exc:
- # The refresh did not get an answer, which is retryable. Calling it a
- # reauthorization would send the user to reconnect a connection whose
- # credentials are probably fine, and the responses transport treats the
- # same failure as transient.
+ # the refresh got no answer, which is retryable
+ # The refresh did not get an answer, which is retryable. Calling it a reauthorization would send the
+ # user to reconnect a connection whose credentials are probably fine, and the responses transport treats
+ # the same failure as transient.
raise CodexTransportError("Could not refresh ChatGPT authorization.") from exc
response = await client.get(
url,
@@ -445,13 +446,11 @@ async def list_subscription_models(
params = {"client_version": OPENAI_CODEX_CLIENT_VERSION},
)
if response.status_code == 401:
- # A freshly refreshed token was rejected too, so the connection really is
- # done. Record it against that token the way the streaming path does, or
- # auth_status keeps reporting connected, the editor never offers Reconnect
- # and every later catalog load repeats this.
- # Under the same guard the streaming error path uses: the read and the write
- # inside are one step, so a rotation landing between them cannot be undone by
- # writing the rejected bundle back with the marker on it.
+ # A freshly refreshed token was rejected too, so the connection really is done. Record it against that token
+ # the way the streaming path does, or auth_status keeps reporting connected, the editor never offers
+ # Reconnect and every later catalog load repeats this. Under the same guard the streaming error path uses:
+ # the read and the write inside are one step, so a rotation landing between them cannot be undone by writing
+ # the rejected bundle back with the marker on it.
async with codex_auth.provider_oauth_write_guard(provider_id):
codex_auth.mark_reauthorization_required(
provider_id, expected_access_token = access_token
@@ -484,35 +483,28 @@ async def list_subscription_models(
if model is None:
continue
if model["id"] in seen_slugs:
- # A slug repeated in one payload describes itself twice, and the list and the
- # by-id map built from it below disagree about which description won: the
- # route offers what the first entry said while the chat gate judges by the
- # last, so a duplicate whose second entry is hidden had the picker offering a
- # model every send then refused. First wins, so both read the same entry.
+ # A slug repeated in one payload describes itself twice, and the list and the by-id map built from it below
+ # disagree about which description won: the route offers what the first entry said while the chat gate
+ # judges by the last, so a duplicate whose second entry is hidden had the picker offering a model every send
+ # then refused. First wins, so both read the same entry.
continue
seen_slugs.add(model["id"])
models.append(model)
if not any(model.get("listed") for model in models):
- # Nothing offerable came back. The route answers with the curated seed for this,
- # so committing it as a known catalog would leave the picker offering models that
- # validation and chat, reading that same empty catalog, would then refuse.
return models
if _catalog_requests.get(provider_id) != ticket:
- # Overtaken while this request was out. Committing now would reinstate a catalog
- # for an account this connection may no longer be on, and clear the mark that
- # says so, so hand the caller the models without storing them.
+ # Overtaken while this request was out. Committing now would reinstate a catalog for an account this connection
+ # may no longer be on, and clear the mark that says so, so hand the caller the models without storing them.
return models
- # That counter only sees this process. Rebinding travels through the installation DB,
- # so ask it who owns the connection now before recording this as its catalog.
+ # that counter only sees this process, and rebinding travels through the installation DB
current_bundle = codex_auth.load_oauth_bundle(provider_id)
if current_bundle and current_bundle.get("account_id") != account_id:
return models
if len(_models_cache) >= _MODELS_CACHE_MAX_ENTRIES:
- # Only the TTL response cache is bounded here. The per-account authorization
- # evidence deliberately outlives it: dropping it would make every other
- # connection look cold and license saved slugs their account no longer carries.
- # It is keyed by connection and released by forget_subscription_models, so it is
- # bounded by the connections that exist rather than by fetches.
+ # Only the TTL response cache is bounded here. The per-account authorization evidence outlives it: dropping it
+ # would make every other connection look cold and license saved slugs their account no longer carries. It is
+ # keyed by connection and released by forget_subscription_models, so it is bounded by the connections that exist
+ # rather than by fetches.
_models_cache.clear()
_models_cache[provider_id] = (time.time() + _MODELS_CACHE_TTL_SECONDS, models)
_offered_models[provider_id] = {model["id"]: model for model in models}
@@ -536,18 +528,14 @@ async def ensure_subscription_models(provider_id: str) -> set[str]:
access_token, account_id = await codex_auth.resolve_access(provider_id)
models = await list_subscription_models(provider_id, access_token, account_id)
except (codex_auth.CodexAuthError, CodexReauthorizationError):
- # The connection needs reconnecting, which is a different answer from "this plan
- # does not list that model". Callers turn this into the reauthorization error the
- # user can act on instead of a misleading model-choice rejection.
+ # the connection needs reconnecting, a different answer from "this plan does not list that model"
raise
except Exception:
return set()
listed = offered_subscription_model_ids(provider_id)
if listed:
return listed
- # A read overtaken by a newer one returns its models without storing them. That
- # answer still came from upstream for the account resolved above, so use it rather
- # than reporting an empty plan; only a rebind makes it the wrong account's.
+ # a read overtaken by a newer one still came from upstream for the account resolved above
current_bundle = codex_auth.load_oauth_bundle(provider_id)
if current_bundle and current_bundle.get("account_id") == account_id:
return {model["id"] for model in models if model.get("listed")}
@@ -574,8 +562,8 @@ def _quota_metadata(response: httpx.Response, *, terminal: bool = False) -> dict
}
metadata = {key: value for key, value in values.items() if value}
if terminal:
- # Exhausted, not throttled. Both leave as a 429, so without this a client waits out
- # a delay that no wait can clear.
+ # Exhausted, not throttled. Both leave as a 429, so without this a client waits out a delay that no wait can
+ # clear.
metadata["terminal"] = True
return metadata
@@ -606,8 +594,8 @@ async def _upstream_error_code(response: httpx.Response) -> str | None:
"""The structured error's code or type. _upstream_error_detail prefers the display message
and drops these, which hides a terminal code behind a generic "slow down" sentence."""
try:
- # Same read-then-parse as the sibling above, so this does not depend on being called
- # after it. A body whose read already failed raises StreamError, not HTTPError.
+ # Same read-then-parse as the sibling above, so this does not depend on being called after it. A body whose read
+ # already failed raises StreamError, not HTTPError.
await response.aread()
payload = response.json()
except (ValueError, json.JSONDecodeError, httpx.HTTPError, httpx.StreamError, AttributeError):
@@ -872,9 +860,10 @@ class OpenAICodexClient:
"schema": schema["schema"],
"strict": bool(schema.get("strict", True)),
}
- # Pi intentionally does not forward a token cap here. ChatGPT's Codex
- # Responses endpoint rejects max_output_tokens even though the public
- # Responses API accepts it; the subscription service applies its own cap.
+ # ChatGPT's Codex Responses endpoint rejects max_output_tokens even though the public Responses API accepts it;
+ # Pi intentionally does not forward a token cap here. ChatGPT's Codex Responses endpoint rejects
+ # max_output_tokens even though the public Responses API accepts it; the subscription service applies its own
+ # cap.
if reasoning_effort:
body["reasoning"] = {"effort": reasoning_effort, "summary": "auto"}
if tools:
diff --git a/studio/backend/core/inference/openai_codex_tool_loop.py b/studio/backend/core/inference/openai_codex_tool_loop.py
index aed9aeb8fd7..e67e4baa820 100644
--- a/studio/backend/core/inference/openai_codex_tool_loop.py
+++ b/studio/backend/core/inference/openai_codex_tool_loop.py
@@ -100,11 +100,10 @@ def stream_codex_with_studio_tools(
messages = run.messages,
session_id = run.session_id,
thread_id = run.thread_id,
- # Before this loop was shared, Codex relayed the provider's own usage
- # chunks and they carried the Codex model id. The shared loop sums
- # them into one synthetic chunk instead, so dropping the model here
- # would relabel that accounting "external" and move behaviour the
- # Codex path is meant to keep.
+ # keep the Codex model id: the shared loop sums usage into one chunk
+ # Before this loop was shared, Codex relayed the provider's own usage chunks and they carried the Codex
+ # model id. The shared loop sums them into one synthetic chunk instead, so dropping the model here would
+ # relabel that accounting "external" and move behaviour the Codex path is meant to keep.
model = run.model,
tool_choice = run.tool_choice,
continue_final_message = run.continue_final_message,
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index a6b5f93833d..a2703466d50 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -35,9 +35,10 @@ from core.inference.audio_errors import (
from utils.hardware import get_device, prepare_gpu_selection
from utils.utils import hf_env_offline
-# Re-exported from the shared helper so GGUF, training and inference share one type.
-# Via PEP 562, not a module-level import: resolving the name imports unsloth_zoo, hence
-# torch, and routes/inference.py imports this module at startup only for GenStream*.
+# Re-exported via PEP 562, not a module-level import
+# Re-exported from the shared helper so GGUF, training and inference share one type. Via PEP 562, not a module-level
+# import: resolving the name imports unsloth_zoo, hence torch, and routes/inference.py imports this module at startup
+# only for GenStream*.
DownloadStallError: type
@@ -58,31 +59,29 @@ class _LoadCancelled(Exception):
_CTX = mp.get_context("spawn")
-# Dispatcher timeout constants (seconds)
_DISPATCH_READ_TIMEOUT = 30.0
_DISPATCH_POLL_INTERVAL = 0.5
_DISPATCH_STOP_TIMEOUT = 5.0
_DISPATCH_IDLE_TIMEOUT = 30.0
_DISPATCH_DRAIN_TIMEOUT = 5.0
-# Only bounds the Transformers subprocess path; llama.cpp TTS never reaches here.
-# 120s was tuned against GGUF speeds and killed real work: a safetensors LoRA on a
-# mid-range GPU needs minutes for the same clip a GGUF returns in seconds. A dead
-# worker is already caught every second by _ensure_subprocess_alive, so this only
-# has to bound one that is alive and wedged, and a generous value costs nothing.
+# Only bounds the Transformers subprocess path; llama.cpp TTS never reaches here. 120s was tuned against GGUF speeds and
+# killed real work: a safetensors LoRA on a mid-range GPU needs minutes for the same clip a GGUF returns in seconds. A
+# dead worker is already caught every second by _ensure_subprocess_alive, so this only has to bound one that is alive
+# and wedged, and a generous value costs nothing.
_AUDIO_GENERATION_TIMEOUT = 900.0
_AUDIO_GENERATION_BASE_TOKENS = 2048
AUDIO_GENERATION_MAX_TOKENS = 8192
MOSS_TTS_MAX_FRAMES = 32768
MINIMAX_MUSIC_MAX_FRAMES = 9000
_AUDIO_CANCEL_DRAIN_TIMEOUT = 5.0
-# Before audio_started there is nobody to receive the cancel, and a prefill pass (a 3B TTS
-# model on CPU, or OuteTTS's per-token Python repetition penalty) routinely outlasts the
-# drain window. Tearing down on that budget unloads the model the user just loaded.
+# Before audio_started there is nobody to receive the cancel, and a prefill pass (a 3B TTS model on CPU, or OuteTTS's
+# per-token Python repetition penalty) routinely outlasts the drain window. Tearing down on that budget unloads the
+# model the user just loaded.
_AUDIO_CANCEL_TEARDOWN_TIMEOUT = 30.0
-# Max wait for a cancelled generation to release _gen_lock before unload_model
-# tears the subprocess down. Only bounds a wedged worker.
+# Max wait for a cancelled generation to release _gen_lock before unload_model tears the subprocess down. Only bounds a
+# wedged worker.
_UNLOAD_GEN_LOCK_TIMEOUT = 15.0
@@ -183,17 +182,15 @@ def _summed_tool_loop_stats(total, turn):
usage = dict(turn.get("usage") or {})
completion = (usage.get("completion_tokens") or 0) + (prior_usage.get("completion_tokens") or 0)
usage["completion_tokens"] = completion
- # The prompt is the loop's, not one turn's, so a turn that ended before
- # reporting keeps the last count that arrived. Its details describe that same
- # count and move with it, or cached tokens could outnumber prompt tokens.
+ # The prompt is the loop's, not one turn's, so a turn that ended before reporting keeps the last count that arrived.
+ # Its details describe that same count and move with it, or cached tokens could outnumber prompt tokens.
if not usage.get("prompt_tokens"):
usage["prompt_tokens"] = prior_usage.get("prompt_tokens") or 0
usage.pop("prompt_tokens_details", None)
if prior_usage.get("prompt_tokens_details") is not None:
usage["prompt_tokens_details"] = prior_usage["prompt_tokens_details"]
usage["total_tokens"] = usage["prompt_tokens"] + completion
- # Details describe the completion, so they sum with it rather than describing
- # one turn against every turn's tokens.
+ # Details describe the completion, so they sum with it rather than describing one turn against every turn's tokens
details = dict(prior_usage.get("completion_tokens_details") or {})
for field, value in (usage.get("completion_tokens_details") or {}).items():
details[field] = (details.get(field) or 0) + (value or 0)
@@ -203,13 +200,13 @@ def _summed_tool_loop_stats(total, turn):
summed["usage"] = usage
timings = dict(turn.get("timings") or {})
prior = total.get("timings") or {}
- # Seeded from the turn but folded unconditionally, as the llama.cpp loop does:
- # a turn reporting no timings must not take the loop's totals with it.
+ # Seeded from the turn but folded unconditionally, as the llama.cpp loop does: a turn reporting no timings must not
+ # take the loop's totals with it.
if timings or prior:
for field in ("predicted_ms", "predicted_n"):
timings[field] = (timings.get(field) or 0) + (prior.get(field) or 0)
- # Rates describe the totals above, not the turn they arrived with: leaving
- # the last turn's would report a speed the summed counts contradict.
+ # Rates describe the totals above, not the turn they arrived with: leaving the last turn's would report a speed
+ # the summed counts contradict.
predicted_ms = timings.get("predicted_ms") or 0
predicted_n = timings.get("predicted_n") or 0
timings["predicted_per_token_ms"] = (predicted_ms / predicted_n) if predicted_n else 0.0
@@ -252,56 +249,51 @@ class InferenceOrchestrator:
"""
def __init__(self):
- # Subprocess state
self._proc: Optional[mp.Process] = None
self._cmd_queue: Any = None
self._resp_queue: Any = None
self._subprocess_shutdown_lock = threading.Lock()
- self._cancel_event: Any = None # mp.Event — set to cancel generation
- # Set for the whole unload; the worker never clears it (unlike _cancel_event),
- # so a generate queued behind the cancelled one is skipped, not run.
+ self._cancel_event: Any = None # mp.Event - set to cancel generation
+ # Set for the whole unload; the worker never clears it (unlike _cancel_event), so a generate queued behind the
+ # cancelled one is skipped, not run.
self._drain_event: Any = None
self._gen_lock = threading.Lock() # Serializes generation
- # Cancel event of the request holding _gen_lock: lets a Stop tell whether it owns the
- # running generation or is queued behind it (the worker's event is shared).
+ # Cancel event of the request holding _gen_lock: lets a Stop tell whether it owns the running generation or is
+ # queued behind it (the worker's event is shared).
self._active_cancel_events: list = []
self._executing_cancel_events: list = []
self._active_cancel_lock = threading.Lock()
- # Held across claim + _send_cmd so claim order matches the subprocess dequeue order,
- # which _owns_worker relies on.
+ # Held across claim + _send_cmd so claim order matches the subprocess dequeue order, which _owns_worker relies
+ # on.
self._send_order_lock = threading.Lock()
- # Set during a switch so a generation winning the _gen_lock handoff bails
- # instead of starting on the outgoing model.
+ # Set during a switch so a generation winning the _gen_lock handoff bails instead of starting on the outgoing
+ # model
self._unload_pending = False
- # Blocking TTS has no token response that can safely establish worker ownership
- # while it is queued behind compare-mode work. Reserve the dispatcher admission
- # lane for the whole TTS request so its shared cancel event cannot hit a sibling.
- # Mutated under _dispatcher_lifecycle_lock while holding _gen_lock.
+ # Blocking TTS has no token response that can safely establish worker ownership while it is queued behind
+ # compare-mode work. Reserve the dispatcher admission lane for the whole TTS request so its shared cancel event
+ # cannot hit a sibling. Mutated under _dispatcher_lifecycle_lock while holding _gen_lock.
self._exclusive_tts_pending = False
- # A live VRAM probe is another exclusive worker command. Compare-mode
- # requests bypass _gen_lock, so reserve their admission lane while the
- # parent asks the idle worker for allocator-owned bytes.
self._exclusive_vram_probe_pending = False
- # Dispatcher state for compare mode (adapter-controlled requests):
- # bypass _gen_lock, send commands directly, read from per-request
- # mailboxes routed by a dispatcher thread on request_id.
+ # Dispatcher state for compare mode: bypass _gen_lock
+ # Dispatcher state for compare mode (adapter-controlled requests): bypass _gen_lock, send commands directly,
+ # read from per-request mailboxes routed by a dispatcher thread on request_id.
self._mailboxes: dict[str, queue.Queue] = {}
- # request_id -> cancel event, so the dispatcher can move worker ownership as it routes.
- # Consumers read their mailbox whenever they get to it, so only the dispatcher sees
- # responses in the order the worker produced them.
+ # request_id -> cancel event, so the dispatcher can move worker ownership as it routes. Consumers read their
+ # mailbox whenever they get to it, so only the dispatcher sees responses in the order the worker produced them.
self._request_cancel_events: dict[str, object] = {}
- # Mailboxes for the _gen_lock generations. Kept apart from _mailboxes because that map
- # means "compare requests are in flight" to the unload and distributed paths.
+ # Mailboxes for the _gen_lock generations. Kept apart from _mailboxes because that map means "compare requests
+ # are in flight" to the unload and distributed paths.
self._direct_mailboxes: dict[str, queue.Queue] = {}
self._mailbox_lock = threading.Lock()
self._dispatcher_thread: Optional[threading.Thread] = None
self._dispatcher_stop = threading.Event()
- # Serializes dispatcher start/stop. _generate_dispatched (compare mode) bypasses
- # _gen_lock, so two concurrent compare requests can both reach _start_dispatcher;
- # without this lock both could observe no live dispatcher and each spawn one,
- # orphaning the extra thread (self._dispatcher_thread tracks only the last). The
- # orphan later steals the "unloaded" reply off resp_queue and hangs unload_model.
+ # Serializes dispatcher start/stop. the orphan later steals the "unloaded" reply off resp_queue and hangs
+ # unload_model
+ # Serializes dispatcher start/stop. _generate_dispatched (compare mode) bypasses _gen_lock, so two concurrent
+ # compare requests can both reach _start_dispatcher; without this lock both could observe no live dispatcher and
+ # each spawn one, orphaning the extra thread (self._dispatcher_thread tracks only the last). The orphan later
+ # steals the "unloaded" reply off resp_queue and hangs unload_model.
self._dispatcher_lifecycle_lock = threading.Lock()
# Local state mirrors (updated from subprocess responses)
@@ -310,15 +302,15 @@ class InferenceOrchestrator:
self.loading_models: set = set()
from core.inference.defaults import get_default_models
- # The list depends on detection (chat-only hosts get the GGUF set) and the MLX
- # self-heal re-detects, so unchecked a repaired Mac serves the chat-only list
- # forever. Stamp read BEFORE the list, or a re-detect tags the old list as new.
+ # The list depends on detection (chat-only hosts get the GGUF set) and the MLX self-heal re-detects, so
+ # unchecked a repaired Mac serves the chat-only list forever. Stamp read BEFORE the list, or a re-detect tags
+ # the old list as new.
import utils.hardware.hardware as _hw_mod
self._static_models_generation = _hw_mod.DETECTION_GENERATION
self._static_models = get_default_models()
- # Own lock for the stamp/value pair; the construction lock is held across a
- # build that waits on hardware detection.
+ # Own lock for the stamp/value pair; the construction lock is held across a build that waits on hardware
+ # detection
self._static_models_lock = threading.Lock()
self._top_gguf_cache: Optional[list[str]] = None
self._top_hub_cache: Optional[list[str]] = None
@@ -327,14 +319,12 @@ class InferenceOrchestrator:
atexit.register(self._cleanup)
logger.info("InferenceOrchestrator initialized (subprocess mode)")
- # Deliberately NOT started here: construction now runs on the startup warm thread, so
- # fetching from __init__ would call huggingface.co on every boot. First reader starts it.
+ # Deliberately NOT started here: construction now runs on the startup warm thread, so fetching from __init__
+ # would call huggingface.co on every boot. First reader starts it.
self._top_models_started = False
- # ------------------------------------------------------------------
- # Default models (top GGUFs fetched dynamically from HF)
- # ------------------------------------------------------------------
-
+ # ------------------------------------------------------------------ Default models (top GGUFs fetched dynamically
+ # from HF) ------------------------------------------------------------------
def _refresh_static_models_if_stale(self) -> None:
"""Recompute the curated defaults if hardware was re-detected since."""
import utils.hardware.hardware as _hw_mod
@@ -347,8 +337,8 @@ class InferenceOrchestrator:
# Built outside the lock so readers do not queue behind the torch import.
models = get_default_models()
with self._static_models_lock:
- # Commit only while still the newest: a slow reader storing its older list
- # under a newer stamp would look current for the life of the process.
+ # Commit only while still the newest: a slow reader storing its older list under a newer stamp would look
+ # current for the life of the process
if generation != _hw_mod.DETECTION_GENERATION:
return
if generation <= self._static_models_generation:
@@ -367,8 +357,8 @@ class InferenceOrchestrator:
"""
if self._top_models_started:
return
- # Checked before the latch: claiming it while offline would retire the fetch for the
- # process, so an offline boot or a temporary force_hf_offline() could never recover.
+ # Checked before the latch: claiming it while offline would retire the fetch for the process, so an offline boot
+ # or a temporary force_hf_offline() could never recover.
if hf_env_offline():
logger.info("offline mode requested; skipping the remote top-models ranking")
return
@@ -384,9 +374,8 @@ class InferenceOrchestrator:
self._start_top_models_fetch()
top_gguf = self._top_gguf_cache or []
top_hub = self._top_hub_cache or []
- # Never wait for the remote Hugging Face ranking during startup. Chat's
- # first /api/models/list needs curated defaults immediately; the
- # background fetch backfills extra choices on later calls.
+ # Never wait for the remote Hugging Face ranking during startup. Chat's first /api/models/list needs curated
+ # defaults immediately; the background fetch backfills extra choices on later calls.
result: list[str] = []
seen: set[str] = set()
for m in self._static_models + top_gguf + top_hub:
@@ -419,8 +408,8 @@ class InferenceOrchestrator:
hub_ids = [
m["id"] for m in models if not m.get("id", "").upper().endswith("-GGUF")
][:40]
- # Counts at info, ids at debug: two lists of 40 repo names, one line each,
- # cost ~1.5 KB of every boot to say the catalog fetch worked.
+ # Counts at info, ids at debug: two lists of 40 repo names cost ~1.5 KB of every boot to say the catalog
+ # fetch worked
if gguf_ids:
self._top_gguf_cache = gguf_ids
logger.info("Fetched %d top GGUF models", len(gguf_ids))
@@ -434,18 +423,8 @@ class InferenceOrchestrator:
finally:
self._top_models_ready.set()
- # ------------------------------------------------------------------
- # Subprocess lifecycle
- # ------------------------------------------------------------------
-
def _spawn_subprocess(self, config: dict) -> None:
"""Spawn a new inference subprocess."""
- # Same recheck as the training/export spawns, REPAIR reservations only: a
- # repair swaps without holding the lifecycle gate this load's caller owns,
- # while an install cannot swap until this gate is released (and then its
- # queued-load snapshot aborts it), so tolerating installs here lets the
- # load win instead of failing both sides. Also covers the OpenAI
- # auto-switch path, which enters _load_model_impl without route guards.
from utils.transformers_version import (
SidecarSwapInProgress,
sidecar_swap_kind,
@@ -594,26 +573,21 @@ class InferenceOrchestrator:
self._proc = None
return True
- # 1. Cancel any ongoing generation first (instant via mp.Event)
self._cancel_generation()
time.sleep(0.5)
- # 2. Drain stale responses
self._drain_queue()
- # 3. Send shutdown command
try:
self._cmd_queue.put({"type": "shutdown"})
except (OSError, ValueError):
pass
- # 4. Wait for graceful shutdown
try:
self._proc.join(timeout = timeout)
except Exception:
pass
- # 5. Force kill if still alive
if self._proc is not None and self._proc.is_alive():
logger.warning("Inference subprocess did not exit gracefully, terminating")
try:
@@ -638,8 +612,8 @@ class InferenceOrchestrator:
pass
if self._proc is not None and self._proc.is_alive():
- # Survived SIGKILL (uninterruptible syscall): keep the handle so callers
- # and the pre-swap guard see a live worker rather than a nulled one.
+ # Survived SIGKILL (uninterruptible syscall): keep the handle so callers and the pre-swap guard see a live
+ # worker rather than a nulled one.
logger.error(
"Inference subprocess still alive after terminate/kill; "
"preserving its handle for the pre-swap liveness check"
@@ -722,9 +696,8 @@ class InferenceOrchestrator:
< max(tolerance_mib, int(max(current[index], previous[index]) * 0.02))
for index in current
)
- # Two unchanged low samples can precede delayed driver reclaim.
- # Stability is meaningful only after an upward release was observed;
- # otherwise consume the full bounded window.
+ # Two unchanged low samples can precede delayed driver reclaim. Stability is meaningful only after an upward
+ # release was observed; otherwise consume the full bounded window.
if stable and observed_reclaim and not expected_mib:
return True
previous = current
@@ -790,10 +763,6 @@ class InferenceOrchestrator:
return f"{message} Details: pid={pid}, exitcode={exitcode}."
- # ------------------------------------------------------------------
- # Queue helpers
- # ------------------------------------------------------------------
-
def _send_cmd(self, cmd: dict) -> None:
"""Send a command to the subprocess."""
if self._cmd_queue is None:
@@ -830,8 +799,8 @@ class InferenceOrchestrator:
message, so long-running operations (large downloads, slow loads)
survive as long as the subprocess keeps reporting progress.
"""
- # Local: resolving this name runs the shim's lazy unsloth_zoo load, which pulls torch.
- # The shim caches its pick, so this site and load_model()'s `except` see one class.
+ # Local: resolving this name runs the shim's lazy unsloth_zoo load, which pulls torch. The shim caches its pick,
+ # so this site and load_model()'s `except` see one class.
from utils.hf_xet_fallback import DownloadStallError
deadline = time.monotonic() + timeout
@@ -844,7 +813,6 @@ class InferenceOrchestrator:
resp = self._read_resp(timeout = min(remaining, poll_seconds))
if resp is None:
- # Check subprocess health
if not self._ensure_subprocess_alive():
raise RuntimeError(self._subprocess_crash_message("wait"))
continue
@@ -862,7 +830,6 @@ class InferenceOrchestrator:
if rtype == "status":
logger.info("Subprocess status: %s", resp.get("message", ""))
- # Reset deadline — subprocess is still alive and working
deadline = time.monotonic() + timeout
continue
@@ -871,7 +838,6 @@ class InferenceOrchestrator:
logger.warning("Subprocess reported stall: %s", msg)
raise DownloadStallError(msg)
- # Other response types during wait — skip
logger.debug(
"Skipping response type '%s' while waiting for '%s'",
rtype,
@@ -918,7 +884,6 @@ class InferenceOrchestrator:
pass
thread = self._dispatcher_thread
if thread is not None and thread.is_alive():
- # It owns the queue now, and it routes to us.
try:
return mailbox.get(timeout = timeout)
except queue.Empty:
@@ -932,10 +897,6 @@ class InferenceOrchestrator:
other = self._mailboxes.get(rid) or self._direct_mailboxes.get(rid)
owner = self._request_cancel_events.get(rid)
if other is not None:
- # We beat the dispatcher to this response, so make its ownership move here
- # too. The compare consumer opts out of marking, so nothing else promotes
- # or retires that request: skipping it left this one recorded as the
- # executor, ignoring its Stop and letting a late reset cancel it.
if owner is not None:
if resp.get("type", "") in ("gen_done", "gen_error"):
self._release_worker(owner)
@@ -987,10 +948,8 @@ class InferenceOrchestrator:
return
logger.warning("Timed out waiting for gen_done after cancel")
- # ------------------------------------------------------------------
- # Generation command + token-stream helpers (shared by all paths)
- # ------------------------------------------------------------------
-
+ # ------------------------------------------------------------------ Generation command + token-stream helpers
+ # (shared by all paths) ------------------------------------------------------------------
def _build_generate_cmd(
self,
request_id: str,
@@ -1037,7 +996,6 @@ class InferenceOrchestrator:
cmd["seed"] = seed
if stop:
cmd["stop"] = stop
- # Only forward template kwargs the caller set, for older worker compat.
if use_adapter is not None:
cmd["use_adapter"] = use_adapter
if tools is not None:
@@ -1072,9 +1030,8 @@ class InferenceOrchestrator:
cancel ack from that same source so stale events don't leak into the
next request.
"""
- # Latch this stream's subprocess/queue: if a wedged worker is torn down and a
- # later load spawns a fresh one, bail rather than re-block on the new queue
- # under _gen_lock (deadlock).
+ # Latch this stream's subprocess/queue: if a wedged worker is torn down and a later load spawns a fresh one,
+ # bail rather than re-block on the new queue under _gen_lock (deadlock).
initial_proc = self._proc
initial_resp_queue = self._resp_queue
while True:
@@ -1086,7 +1043,6 @@ class InferenceOrchestrator:
return
resp = read_one(read_timeout)
if resp is None:
- # Check subprocess health
if not self._ensure_subprocess_alive():
yield GenStreamError(
f"Error: {self._subprocess_crash_message(crash_context)}",
@@ -1098,25 +1054,22 @@ class InferenceOrchestrator:
rtype = resp.get("type", "")
if rtype == "status":
continue
- # The worker is answering THIS request, so it is the one executing: only now may its
- # cancel event speak for the shared worker one. The dispatched path opts out: its
- # dispatcher already did this in worker order, which a mailbox read can lag behind.
+ # The worker is answering THIS request, so it is the one executing: only now may its cancel event speak for
+ # the shared worker one. The dispatched path opts out: its dispatcher already did this in worker order,
+ # which a mailbox read can lag behind.
if mark_started:
self._mark_worker_started(cancel_event)
- # Subprocess-level error (no request_id); request-scoped failures
- # arrive as gen_error below.
+ # Subprocess-level error (no request_id); request-scoped failures arrive as gen_error below
if rtype == "error" and not resp.get("request_id"):
yield GenStreamError(f"Error: {resp.get('error', 'Unknown error')}")
return
if rtype == "token":
- # Cancel from route (e.g. SSE connection closed).
if cancel_event is not None and cancel_event.is_set():
- # Same rule as reset_generation_state: the shared worker event may only be set by
- # the generation the worker is running. A dispatched request can still be draining
- # stale mailbox tokens after the dispatcher retired it, and signalling from here
- # would end the next one instead. Tearing this stream down is always safe, so the
- # local drain happens either way.
+ # Same rule as reset_generation_state: the shared worker event may only be set by the generation the
+ # worker is running. A dispatched request can still be draining stale mailbox tokens after the
+ # dispatcher retired it, and signalling from here would end the next one instead. Tearing this
+ # stream down is always safe, so the local drain happens either way.
if self._owns_worker(cancel_event):
self._cancel_generation()
drain_on_cancel()
@@ -1130,10 +1083,8 @@ class InferenceOrchestrator:
yield GenStreamError(f"Error: {resp.get('error', 'Unknown error')}")
return
- # ------------------------------------------------------------------
- # Dispatcher — per-request mailbox routing for compare mode
- # ------------------------------------------------------------------
-
+ # ------------------------------------------------------------------ Dispatcher - per-request mailbox routing for
+ # compare mode ------------------------------------------------------------------
def _start_dispatcher(self) -> bool:
"""Start the dispatcher thread if not already running.
@@ -1147,13 +1098,11 @@ class InferenceOrchestrator:
that actually started a new thread; False if one was already alive.
"""
with self._dispatcher_lifecycle_lock:
- # Refuse to start while an unload is in progress. unload_model sets
- # _unload_pending under this same lock before it stops the idle
- # dispatcher, so a start queued behind that stop observes the unload
- # here and bails. Without this a fresh dispatcher would be spawned
- # after the stop, become the resp_queue reader, and consume the
- # worker's "unloaded" reply (unroutable, so dropped) before
- # unload_model's _wait_response sees it -- hanging the unload 300s.
+ # Refuse to start while an unload is in progress. unload_model sets _unload_pending under this same lock
+ # before it stops the idle dispatcher, so a start queued behind that stop observes the unload here and
+ # bails. Without this a fresh dispatcher would be spawned after the stop, become the resp_queue reader, and
+ # consume the worker's "unloaded" reply (unroutable, so dropped) before unload_model's _wait_response sees
+ # it -- hanging the unload 300s.
if (
self._unload_pending
or self._exclusive_tts_pending
@@ -1202,13 +1151,12 @@ class InferenceOrchestrator:
except (EOFError, OSError, ValueError):
break
- # Sole consumer of the response queue; if it died every in-flight
- # stream would hang, so never let routing kill the dispatcher.
+ # Sole consumer of the response queue; if it died every in-flight stream would hang, so never let routing
+ # kill the dispatcher.
try:
rid = resp.get("request_id")
rtype = resp.get("type", "")
- # Status messages: log and skip
if rtype == "status":
logger.info("Subprocess status: %s", resp.get("message", ""))
continue
@@ -1219,9 +1167,9 @@ class InferenceOrchestrator:
mbox = self._mailboxes.get(rid) or self._direct_mailboxes.get(rid)
owner = self._request_cancel_events.get(rid)
if mbox is not None:
- # Worker order, not consumer order: retire a request the moment its last response
- # is routed. Waiting for the consumer's finally left it owning the worker after
- # the worker moved on, so a late Stop for it cancelled whichever request started next.
+ # Worker order, not consumer order: retire a request the moment its last response is routed.
+ # Waiting for the consumer's finally left it owning the worker after the worker moved on, so a
+ # late Stop for it cancelled whichever request started next.
if owner is not None:
if rtype in ("gen_done", "gen_error"):
self._release_worker(owner)
@@ -1230,7 +1178,6 @@ class InferenceOrchestrator:
mbox.put(resp)
continue
- # No matching mailbox; can't un-get from mp.Queue, so just log.
logger.debug(
"Dispatcher: no mailbox for request_id=%s type=%s, dropping",
rid,
@@ -1278,14 +1225,12 @@ class InferenceOrchestrator:
if not self.active_model_name:
yield GenStreamError("Error: No active model", public = True)
return
- # Latch the target model so the recheck below can detect a switch that completed
- # between _start_dispatcher and mailbox registration (mirrors the locked path's
- # expected_model check).
+ # Latch the target model so the recheck below can detect a switch that completed between _start_dispatcher and
+ # mailbox registration (mirrors the locked path's expected_model check).
expected_model = self.active_model_name
- # Switch in flight (unload waiting on _gen_lock). This path bypasses the lock,
- # so without this early-out a compare request would enqueue a generate on the
- # outgoing model and delay the switch.
+ # Switch in flight (unload waiting on _gen_lock). This path bypasses the lock, so without this early-out a
+ # compare request would enqueue a generate on the outgoing model and delay the switch.
if self._unload_pending:
yield GenStreamError("Error: model is being unloaded", public = True)
return
@@ -1293,18 +1238,17 @@ class InferenceOrchestrator:
yield GenStreamError("Error: audio generation is in progress", public = True)
return
- # Ensure the dispatcher runs. _start_dispatcher serializes concurrent starters under
- # _dispatcher_lifecycle_lock and returns True only for the caller that actually spawned
- # the thread, so at most one dispatcher ever exists even when two compare requests race
- # here. Derive dispatcher_preexisting from that atomic result (not a separate unlocked
- # is_alive() read): if THIS call started the dispatcher and then bails on a racing
- # unload, it must stop it again (see the unloading bail below).
+ # _start_dispatcher serializes concurrent starters and returns True only for the caller that spawned the thread.
+ # Ensure the dispatcher runs. _start_dispatcher serializes concurrent starters under _dispatcher_lifecycle_lock
+ # and returns True only for the caller that spawned the thread, so at most one dispatcher ever exists even when
+ # two compare requests race here. Derive dispatcher_preexisting from that atomic result (not a separate unlocked
+ # is_alive() read): if THIS call started the dispatcher and then bails on a racing unload, it must stop it again
+ # (see the unloading bail below).
started = self._start_dispatcher()
dispatcher_preexisting = not started
request_id = str(uuid.uuid4())
- # Convert PIL Image to base64 if needed
image_b64 = None
if image is not None:
image_b64 = self._pil_to_base64(image)
@@ -1333,18 +1277,12 @@ class InferenceOrchestrator:
seed = seed,
)
- # Create the mailbox BEFORE sending, rechecking _unload_pending under
- # _mailbox_lock: an unload sets _unload_pending before _wait_dispatcher_idle
- # reads _mailboxes under the same lock, so either the idle check sees this
- # mailbox (and tears the dispatcher down) or we see the unload and bail.
- # Registering after would orphan the mailbox and hang the compare stream forever.
mailbox: queue.Queue = queue.Queue()
with self._mailbox_lock:
- # _unload_pending alone is not enough: an unload that ran fully since
- # _start_dispatcher clears it in its finally and stops the dispatcher, so it
- # reads False here though the dispatcher is gone and the model swapped. Also
- # bail when the active model changed or the dispatcher died: a mailbox with no
- # dispatcher to route gen_done/gen_error hangs the compare stream.
+ # _unload_pending alone is not enough: an unload that ran fully since _start_dispatcher clears it in its
+ # finally and stops the dispatcher, so it reads False here though the dispatcher is gone and the model
+ # swapped. Also bail when the active model changed or the dispatcher died: a mailbox with no dispatcher to
+ # route gen_done/gen_error hangs the compare stream.
dispatcher_alive = (
self._dispatcher_thread is not None and self._dispatcher_thread.is_alive()
)
@@ -1360,16 +1298,15 @@ class InferenceOrchestrator:
self._mailboxes[request_id] = mailbox
if cancel_event is not None:
self._request_cancel_events[request_id] = cancel_event
- # When bailing without a mailbox, note whether any OTHER compare request still
- # routes through the dispatcher; if none and this call started it, stop it below.
+ # When bailing without a mailbox, note whether any OTHER compare request still routes through the
+ # dispatcher; if none and this call started it, stop it below.
orphaned_dispatcher = blocked and not dispatcher_preexisting and not self._mailboxes
if blocked:
- # A racing unload can pass its _wait_dispatcher_idle() while the dispatcher was
- # stopped, then set _unload_pending. The one we just started would otherwise
- # linger with no mailboxes, race unload_model's _wait_response for the "unloaded"
- # reply off resp_queue, and drop it as unroutable -- hanging the unload 300s. Stop
- # it here so the unload stays the sole resp_queue reader. Outside _mailbox_lock:
- # _stop_dispatcher joins the dispatcher, which itself takes that lock.
+ # A racing unload can pass its _wait_dispatcher_idle() while the dispatcher was stopped, then set
+ # _unload_pending. The one we just started would otherwise linger with no mailboxes, race unload_model's
+ # _wait_response for the "unloaded" reply off resp_queue, and drop it as unroutable -- hanging the unload
+ # 300s. Stop it here so the unload stays the sole resp_queue reader. Outside _mailbox_lock: _stop_dispatcher
+ # joins the dispatcher, which itself takes that lock.
if orphaned_dispatcher:
self._stop_dispatcher()
detail = (
@@ -1382,10 +1319,10 @@ class InferenceOrchestrator:
yield GenStreamError(detail, public = True)
return
- # Claim before sending, like the locked path: dispatched runs are concurrent by design,
- # so without this a Stop on one saw no owner and reset the worker, ending its siblings.
- # Claim and enqueue under one lock, or two dispatcher threads interleave and claim order
- # stops matching the subprocess's command order, which _owns_worker reads.
+ # Claim before sending, like the locked path: dispatched runs are concurrent by design, so without this a Stop
+ # on one saw no owner and reset the worker, ending its siblings. Claim and enqueue under one lock, or two
+ # dispatcher threads interleave and claim order stops matching the subprocess's command order, which
+ # _owns_worker reads.
try:
with self._send_order_lock:
self._claim_worker(cancel_event)
@@ -1404,7 +1341,6 @@ class InferenceOrchestrator:
except queue.Empty:
return None
- # Read tokens from our private mailbox (the dispatcher owns resp_queue).
try:
yield from self._consume_token_stream(
read_mailbox,
@@ -1416,8 +1352,8 @@ class InferenceOrchestrator:
mark_started = False,
)
finally:
- # Normally already retired by the dispatcher at gen_done; this covers streams that
- # end without one (cancel, disconnect, a dead subprocess).
+ # Normally already retired by the dispatcher at gen_done; this covers streams that end without one (cancel,
+ # disconnect, a dead subprocess).
self._release_worker(cancel_event)
with self._mailbox_lock:
self._mailboxes.pop(request_id, None)
@@ -1468,9 +1404,7 @@ class InferenceOrchestrator:
if cancel_event is not None and cancel_event.is_set():
return False
- # Only stop dispatcher if all mailboxes drained. If compare requests
- # are still active, leave it running so their token routing isn't
- # killed mid-stream.
+ # Only stop the dispatcher if all mailboxes drained
with self._mailbox_lock:
still_active = bool(self._mailboxes)
if still_active:
@@ -1536,20 +1470,17 @@ class InferenceOrchestrator:
raise RuntimeError("Timeout waiting for distributed object share")
- # ------------------------------------------------------------------
- # Public API — same interface as InferenceBackend
- # ------------------------------------------------------------------
-
- # Monotonic count of PUBLISHED loads; lets the install route detect a load
- # (including a same-model reload) that completed while it waited on the gate.
- # Bumped when the load result is published, not at load start: a start-time
- # bump is already visible when the installer snapshots mid-load, so the
- # completed reload would look unchanged and get unloaded by the swap.
+ # Monotonic count of PUBLISHED loads, so the install route can detect a load (including a same-model reload) that
+ # completed
+ # Monotonic count of PUBLISHED loads; lets the install route detect a load (including a same-model reload) that
+ # completed while it waited on the gate. Bumped when the load result is published, not at load start: a start-time
+ # bump is already visible when the installer snapshots mid-load, so the completed reload would look unchanged and
+ # get unloaded by the swap.
load_generation: int = 0
def load_model(
self,
- config, # ModelConfig
+ config,
max_seq_length: int = 2048,
dtype = None,
load_in_4bit: bool = True,
@@ -1572,7 +1503,6 @@ class InferenceOrchestrator:
"""
from utils.transformers_version import needs_transformers_5
- # Same lazy-shim reason as _wait_response(); see the note there.
from utils.hf_xet_fallback import DownloadStallError
model_name = config.identifier
@@ -1585,7 +1515,6 @@ class InferenceOrchestrator:
try:
needed_major = "5" if needs_transformers_5(model_name) else "4"
- # Build config dict for subprocess
sub_config = {
"model_name": model_name,
"max_seq_length": max_seq_length,
@@ -1620,11 +1549,9 @@ class InferenceOrchestrator:
logger.info("Load cancelled before worker teardown: %s", model_name)
return False
- # Recheck the sidecar reservation BEFORE tearing the old worker down,
- # for REPAIRS only: an install holds this same lifecycle gate, so it
- # cannot swap while this load runs, and its queued-load snapshot
- # aborts it after this load publishes -- the load wins cleanly.
- # Raising here (repair) keeps the current model loaded.
+ # Recheck the sidecar reservation BEFORE tearing the old worker down, for REPAIRS only: an install holds
+ # this same lifecycle gate, so it cannot swap while this load runs, and its queued-load snapshot aborts it
+ # after this load publishes -- the load wins cleanly. Raising here (repair) keeps the current model loaded.
from utils.transformers_version import (
SidecarSwapInProgress,
sidecar_swap_kind,
@@ -1636,18 +1563,17 @@ class InferenceOrchestrator:
"retry when it completes."
)
- # Always kill the existing subprocess and spawn fresh: reusing one
- # after unsloth patches torch internals breaks getsource on reload.
+ # Always kill the existing subprocess and spawn fresh: reusing one after unsloth patches torch internals
+ # breaks getsource on reload.
had_worker_handle = self._proc is not None
worker_shutdown_at = 0.0
if self._ensure_subprocess_alive():
self._cancel_generation()
time.sleep(0.3)
if self._shutdown_subprocess() is False:
- # The worker survived terminate/kill (e.g. a wedged CUDA syscall that
- # outlives SIGKILL). Its handle is kept, so is_worker_alive() and the
- # pre-swap guard still see it; do not spawn a second worker over one
- # still holding GPU memory. Fail so the load can retry once it exits.
+ # The worker survived terminate/kill (e.g. a wedged CUDA syscall that outlives SIGKILL). Its handle
+ # is kept, so is_worker_alive() and the pre-swap guard still see it; do not spawn a second worker
+ # over one still holding GPU memory. Fail so the load can retry once it exits.
raise RuntimeError(
"The current inference worker did not exit and still holds GPU "
"memory; not starting a new model over it. Retry shortly."
@@ -1683,11 +1609,6 @@ class InferenceOrchestrator:
)
for attempt in range(2):
- # Stop-loading (/unload -> cancel_load) aborts a load by discarding this
- # model's loading marker. cancel_load only kills a live child; if the cancel
- # lands before any child exists (GPU placement, or between retries) there is
- # nothing to kill, and without this check the loop would spawn a worker and
- # load the model after /unload reported it unloaded. Observe removal and stop.
if model_name not in self.loading_models or (
load_cancel_event is not None and load_cancel_event.is_set()
):
@@ -1710,12 +1631,11 @@ class InferenceOrchestrator:
sub_config["disable_xet"] = disable_xet
self._spawn_subprocess(sub_config)
- # A cancel can land after the pre-spawn recheck but while _spawn_subprocess
- # is still creating the queues/process. cancel_load runs off the lifecycle
- # gate, so its _shutdown_subprocess can see _proc still None and no-op,
- # orphaning this fresh worker; the load would then wait for "loaded" and
- # publish a model /unload reported unloaded, over a live subprocess nothing
- # reaps. Recheck now the child exists and tear it down before publishing.
+ # A cancel can land after the pre-spawn recheck but while _spawn_subprocess is still creating the
+ # queues/process. cancel_load runs off the lifecycle gate, so its _shutdown_subprocess can see _proc
+ # still None and no-op, orphaning this fresh worker; the load would then wait for "loaded" and publish a
+ # model /unload reported unloaded, over a live subprocess nothing reaps. Recheck now the child exists
+ # and tear it down before publishing.
if model_name not in self.loading_models or (
load_cancel_event is not None and load_cancel_event.is_set()
):
@@ -1754,7 +1674,6 @@ class InferenceOrchestrator:
self._shutdown_subprocess(timeout = 5)
disable_xet = True
continue
- # Second stall (or xet already off) -> give up
self._shutdown_subprocess(timeout = 5)
raise RuntimeError(
f"Download stalled for '{model_name}' even with "
@@ -1762,15 +1681,6 @@ class InferenceOrchestrator:
)
if resp.get("success"):
- # A cancel can land while we were parked in _wait_response above.
- # cancel_load (off the lifecycle gate) discards this model's loading
- # marker BEFORE its teardown, so a Stop-loading that fired after the
- # worker queued "loaded" (which we can still consume during cancel_load's
- # shutdown window) shows up here only as the marker's removal. Without
- # this recheck we would publish active_model_name/models for a model
- # /unload reported cancelled, over a subprocess cancel_load just killed;
- # its post-teardown re-clear cannot undo a publish that lands after it
- # returns. Observe the removal and abort; cancel_load owns teardown.
if model_name not in self.loading_models or (
load_cancel_event is not None and load_cancel_event.is_set()
):
@@ -1791,10 +1701,9 @@ class InferenceOrchestrator:
model_info = resp.get("model_info", {})
self.active_model_name = model_info.get("identifier", model_name)
self.load_generation += 1
- # A load always spawns a fresh subprocess holding only this model, so
- # mirror that. A lingering stale name would pass unload_model's "not in
- # self.models" guard, and the worker's absent-name fallback would unload
- # its *active* model, not the already-gone one.
+ # A load always spawns a fresh subprocess holding only this model, so mirror that. A lingering stale
+ # name would pass unload_model's "not in self.models" guard, and the worker's absent-name fallback
+ # would unload its *active* model, not the already-gone one.
self.models = {}
self.models[self.active_model_name] = _mirrored_model_entry(
model_info, model_name
@@ -1802,8 +1711,7 @@ class InferenceOrchestrator:
self.models[self.active_model_name].update(
_mlx_runtime_mirror_fields(model_info)
)
- # Mirror chat_template_info so routes can classify caps
- # without re-entering the subprocess.
+ # Mirror chat_template_info so routes can classify caps without re-entering the subprocess
_tpl_info = model_info.get("chat_template_info")
if isinstance(_tpl_info, dict):
self.models[self.active_model_name]["chat_template_info"] = _tpl_info
@@ -1822,14 +1730,13 @@ class InferenceOrchestrator:
from utils.transformers_version import SidecarSwapInProgress
if isinstance(exc, SidecarSwapInProgress) and self._ensure_subprocess_alive():
- # Raised before the old worker was torn down: the previous model
- # is still live, so keep the mirrors (clearing them would let the
- # installer treat the worker as inactive and kill it unreported).
+ # Raised before the old worker was torn down: the previous model is still live, so keep the mirrors
+ # (clearing them would let the installer treat the worker as inactive and kill it unreported).
raise
self.active_model_name = None
self.models.clear()
- # Reap workers after any failed load, including inactivity timeouts
- # that leave installs and GPU memory alive (#9398).
+ # Reap workers after any failed load, including inactivity timeouts that leave installs and GPU memory alive
+ # (#9398)
try:
self._shutdown_subprocess(timeout = 5)
except Exception as teardown_exc:
@@ -1859,35 +1766,31 @@ class InferenceOrchestrator:
"Cancelling in-flight load for model '%s' by terminating subprocess",
target,
)
- # Discard the loading marker (and clear local state) BEFORE the teardown, not
- # after. cancel_load runs off the lifecycle gate, alongside a load_model that
- # rechecks this marker before each spawn. But _shutdown_subprocess can block (~1s
- # tearing a live child down and joining the dispatcher), so clearing only after
- # leaves a window where load_model reads the marker still set, passes its pre-spawn
- # recheck, and loads the model after /unload reported it cancelled. Clear first.
+ # Discard the loading marker BEFORE the teardown
+ # Discard the loading marker (and clear local state) BEFORE the teardown, not after. cancel_load runs off the
+ # lifecycle gate, alongside a load_model that rechecks this marker before each spawn. But _shutdown_subprocess
+ # can block (~1s tearing a live child down and joining the dispatcher), so clearing only after leaves a window
+ # where load_model reads the marker still set, passes its pre-spawn recheck, and loads the model after /unload
+ # reported it cancelled. Clear first.
self.loading_models.discard(target)
self.active_model_name = None
self.models.clear()
self._shutdown_subprocess(timeout = 0.5)
- # Clear the local mirrors again AFTER the teardown. A racing off-gate load_model
- # may still be parked in _wait_response("loaded"): its worker already queued a
- # "loaded" reply, so during the shutdown window above (the 0.5s settle before the
- # response queue is drained and nulled) that thread can consume it and repopulate
- # active_model_name/models, undoing the pre-teardown clear. _shutdown_subprocess
- # nulls the queue but not the mirrors, so without this second clear /unload reports
- # success while the backend still advertises a killed model. The nulled queue lets
- # no further "loaded" through, so re-clearing here wipes any repopulation.
+ # Clear the local mirrors again AFTER the teardown. A racing off-gate load_model may still be parked in
+ # _wait_response("loaded"): its worker already queued a "loaded" reply, so during the shutdown window above (the
+ # 0.5s settle before the response queue is drained and nulled) that thread can consume it and repopulate
+ # active_model_name/models, undoing the pre-teardown clear. _shutdown_subprocess nulls the queue but not the
+ # mirrors, so without this second clear /unload reports success while the backend still advertises a killed
+ # model. The nulled queue lets no further "loaded" through, so re-clearing here wipes any repopulation.
self.active_model_name = None
self.models.clear()
return True
+ # --- Dictation models ------------------------------------------------- These run in the STT sidecars
+ # (whisper-server, llama-server, and the Transformers spawn child), not the chat worker. Their lifecycle goes
+ # through here all the same, so one object knows everything that is resident and Voice settings and Model Hub cannot
+ # report different things about one model.
# --- Dictation models -------------------------------------------------
- # These run in the STT sidecars (whisper-server, llama-server, and the
- # Transformers spawn child), not the chat worker. Their lifecycle goes
- # through here all the same, so one object knows everything that is
- # resident and Voice settings and Model Hub cannot report different things
- # about one model.
-
def load_stt_model(
self,
model: Optional[str],
@@ -1917,57 +1820,50 @@ class InferenceOrchestrator:
def unload_model(self, model_name: str) -> bool:
"""Unload a model from the subprocess."""
- # active_model_name can differ in case from the client's raw /unload name (the
- # load path canonicalizes casing). Match case-insensitively and use the canonical
- # spelling so the guard, unload command, and cleanup below hit the loaded model.
+ # active_model_name can differ in case from the client's raw /unload name (the load path canonicalizes casing).
+ # Match case-insensitively and use the canonical spelling so the guard, unload command, and cleanup below hit
+ # the loaded model.
if (
self.active_model_name is not None
and model_name != self.active_model_name
and model_name.lower() == self.active_model_name.lower()
):
model_name = self.active_model_name
- # In-flight load: tear its subprocess down (shared loading-cancel logic; no
- # worker command sent).
+ # In-flight load: tear its subprocess down (shared loading-cancel logic; no worker command sent)
if self.cancel_load(model_name):
return True
if not self._ensure_subprocess_alive():
- # No subprocess — clear local state
self.models.pop(model_name, None)
if self.active_model_name == model_name:
self.active_model_name = None
return True
- # Nothing loaded under this name: don't unload a stale model. The worker falls
- # back to unloading its *active* model when the name is absent, so a stale unload
- # (lost a race to a concurrent load) would hit the wrong one.
+ # Nothing loaded under this name: the worker falls back to unloading its *active* model when the name is absent
+ # Nothing loaded under this name: don't unload a stale model. The worker falls back to unloading its *active*
+ # model when the name is absent, so a stale unload (lost a race to a concurrent load) would hit the wrong one.
if model_name != self.active_model_name and model_name not in self.models:
self.models.pop(model_name, None)
return True
- # The subprocess runs commands sequentially, so a bare unload queues behind a
- # running generate (a 2-3 min hang). Cancel first (via the mp.Event the worker
- # polls each token), then take _gen_lock as sole resp_queue reader (like GGUF).
- #
- # Set _unload_pending under _dispatcher_lifecycle_lock so it is ordered ahead of
- # the dispatcher stop that _wait_dispatcher_idle runs under the same lock: a
- # compare request's _start_dispatcher queued behind that stop then observes the
- # unload and refuses to spawn a fresh dispatcher that would eat the "unloaded"
- # reply off resp_queue. This is a standalone acquisition (no _gen_lock held yet),
- # so it keeps the _gen_lock -> _dispatcher_lifecycle_lock order and can't deadlock.
+ # The subprocess runs commands sequentially, so a bare unload queues behind a running generate (a 2-3 min hang).
+ # Cancel first (via the mp.Event the worker polls each token), then take _gen_lock as sole resp_queue reader
+ # (like GGUF). Set _unload_pending under _dispatcher_lifecycle_lock so it is ordered ahead of the dispatcher
+ # stop that _wait_dispatcher_idle runs under the same lock: a compare request's _start_dispatcher queued behind
+ # that stop then observes the unload and refuses to spawn a fresh dispatcher that would eat the "unloaded" reply
+ # off resp_queue. This is a standalone acquisition (no _gen_lock held yet), so it keeps the _gen_lock ->
+ # _dispatcher_lifecycle_lock order and can't deadlock.
with self._dispatcher_lifecycle_lock:
self._unload_pending = True
- # Cancelling only the running generation isn't enough: the worker clears
- # cancel_event at each generate start, so a queued one would clear it and run the
- # outgoing model to completion. drain_event, never cleared, makes any generate
- # dequeued during the unload skip.
+ # Cancelling only the running generation isn't enough: the worker clears cancel_event at each generate start, so
+ # a queued one would clear it and run the outgoing model to completion. drain_event, never cleared, makes any
+ # generate dequeued during the unload skip.
if self._drain_event is not None:
self._drain_event.set()
try:
self._cancel_generation()
acquired = self._gen_lock.acquire(timeout = _UNLOAD_GEN_LOCK_TIMEOUT)
if not acquired:
- # Wedged worker: tear the subprocess down to free the GPU (next load respawns).
logger.warning(
"Unload: generation did not yield %.1fs after cancel; "
"shutting the inference subprocess down to free the model",
@@ -1980,11 +1876,10 @@ class InferenceOrchestrator:
return True
try:
- # Stop the compare-mode dispatcher so it can't consume the "unloaded" reply
- # off resp_queue before we do. A dispatched generation bypasses _gen_lock, so
- # a wedged one slips past the acquire above; if the dispatcher is still active
- # it owns resp_queue and the queued unload hangs _wait_response behind the
- # stuck generate. Mirror the wedged locked path: tear the subprocess down.
+ # Stop the compare-mode dispatcher so it can't consume the "unloaded" reply off resp_queue before we do.
+ # A dispatched generation bypasses _gen_lock, so a wedged one slips past the acquire above; if the
+ # dispatcher is still active it owns resp_queue and the queued unload hangs _wait_response behind the
+ # stuck generate.
if not self._wait_dispatcher_idle():
logger.warning(
"Unload: compare-mode dispatcher still active after idle "
@@ -1995,7 +1890,6 @@ class InferenceOrchestrator:
if self.active_model_name == model_name:
self.active_model_name = None
return True
- # Drop stale tokens so they can't be read as the unload reply.
self._drain_queue()
self._send_cmd(
{
@@ -2010,13 +1904,11 @@ class InferenceOrchestrator:
self.active_model_name = None
logger.info("Model '%s' unloaded from subprocess", model_name)
- # empty_cache in the child cannot return the accelerator context, so an
- # idle worker keeps its high-water mark -- VRAM the GGUF backend cannot
- # see and gpu_arbiter never evicts (both are chat-owned). Nothing left
- # to serve, so drop it; load_model respawns a fresh worker regardless.
+ # empty_cache in the child cannot return the accelerator context, so an idle worker keeps its high-water
+ # mark -- VRAM the GGUF backend cannot see and gpu_arbiter never evicts (both are chat-owned). Nothing
+ # left to serve, so drop it; load_model respawns a fresh worker regardless.
if not self.models and not self.loading_models:
logger.info("No models left resident; shutting the inference subprocess down")
- # The unload already succeeded: a failed teardown must not report it failed.
try:
self._shutdown_subprocess(timeout = 5)
except Exception as exc:
@@ -2025,7 +1917,6 @@ class InferenceOrchestrator:
except Exception as exc:
logger.error("Error unloading model '%s': %s", model_name, exc)
- # Clear local state anyway
self.models.pop(model_name, None)
if self.active_model_name == model_name:
self.active_model_name = None
@@ -2213,9 +2104,8 @@ class InferenceOrchestrator:
turn_stats: dict = {}
def _single_turn(conv: list, *, active_tools: Optional[list[dict]] = None):
- # ``conv`` already carries any system message. ``active_tools`` lets
- # run_safetensors_tool_loop drop one-shot tools (e.g. render_html) from
- # later same-response prompts.
+ # ``conv`` already carries any system message. ``active_tools`` lets run_safetensors_tool_loop drop one-shot
+ # tools (e.g. render_html) from later same-response prompts.
turn_tools = active_tools if active_tools is not None else tools
turn_stats.clear()
common_kwargs = dict(
@@ -2233,11 +2123,10 @@ class InferenceOrchestrator:
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
- # Self-limiting: after a tool call the conversation ends on a tool
- # result, so later turns render as ordinary new turns.
+ # Self-limiting: after a tool call the conversation ends on a tool result, so later turns render as
+ # ordinary new turns.
continue_final_message = continue_final_message,
- # Reported per turn and summed below, since the whole loop answers
- # one request.
+ # Reported per turn and summed below, since the whole loop answers one request.
stats_holder = turn_stats,
presence_penalty = presence_penalty,
seed = seed,
@@ -2267,8 +2156,7 @@ class InferenceOrchestrator:
close()
except Exception:
logger.debug("failed to close errored generation stream", exc_info = True)
- # A turn that never reported -- one a cancel interrupted -- folds in
- # as nothing, leaving the turns that did.
+ # A turn that never reported (one a cancel interrupted) folds in as nothing, leaving the turns that did
if stats_holder is not None:
stats_holder["stats"] = _summed_tool_loop_stats(
stats_holder.get("stats"), turn_stats.get("stats")
@@ -2278,10 +2166,9 @@ class InferenceOrchestrator:
if system_prompt:
initial = [{"role": "system", "content": system_prompt}] + initial
- # Same profile the renderer uses, so the controller never drops a tool over a
- # marker this model does not treat as structure. The controller is also given the
- # catalog safe under every template this turn could select, because the
- # native-template fallback renders with a different profile (#7066).
+ # Same profile the renderer uses, so the controller never drops a tool over a marker this model does not treat
+ # as structure. The controller is also given the catalog safe under every template this turn could select,
+ # because the native-template fallback renders with a different profile (#7066).
from core.inference.chat_template_helpers import (
mapped_chat_template,
markup_for_tokenizer,
@@ -2347,9 +2234,8 @@ class InferenceOrchestrator:
try:
for chunk in stream:
if isinstance(chunk, GenStreamError):
- # Preserve the public/operational flag so the route can surface
- # the real message (e.g. "model is being unloaded") instead of a
- # generic error. Mirrors the safetensors tool loop's _single_turn.
+ # Preserve the public/operational flag so the route can surface the real message (e.g. "model is
+ # being unloaded") instead of a generic error. Mirrors the safetensors tool loop's _single_turn.
raise GenStreamErrorRaised(str(chunk), public = chunk.public)
yield chunk
finally:
@@ -2396,25 +2282,15 @@ class InferenceOrchestrator:
return
expected_model = self.active_model_name
- # Drain any prior compare-mode dispatcher so we can read resp_queue.
self._wait_dispatcher_idle()
- # Serialize generation: two concurrent readers on resp_queue would
- # consume and drop each other's token events. Hold _gen_lock across the
- # cmd build + send + whole stream so we stay the sole resp_queue reader.
+ # Serialize generation: two concurrent readers on resp_queue would consume and drop each other's token events.
+ # Hold _gen_lock across the cmd build + send + whole stream so we stay the sole resp_queue reader.
with self._gen_lock:
- # Recheck under the lock: an unload we raced may have cleared/swapped the model.
- # _unload_pending resets after the lock releases, so it can read False by now;
- # the active-model check catches that handoff and a reload that swapped models,
- # so we never generate on the wrong one.
if self._unload_pending or self.active_model_name != expected_model:
- # Won the lock handoff during a switch; don't start on the outgoing model.
yield GenStreamError("Error: model is being unloaded", public = True)
return
if cancel_event is not None and cancel_event.is_set():
- # Stopped while queued on the lock. Sending anyway occupied the worker with a
- # run the user ended: the cancel is only seen on a token, so a long prefill
- # (or a generation that reaches gen_done without one) held up its siblings.
return
request_id = str(uuid.uuid4())
image_b64 = self._pil_to_base64(image) if image is not None else None
@@ -2442,11 +2318,10 @@ class InferenceOrchestrator:
seed = seed,
)
- # Claim the worker BEFORE sending, so a Stop on some OTHER chat -- still queued on the
- # lock above, having generated nothing -- cannot reset the generation this is starting.
- # Claiming after the send left the command running unclaimed. Released in the finally.
- # Own mailbox: a compare request can start the dispatcher while this is streaming,
- # and it would otherwise consume our responses and drop them.
+ # Claim the worker BEFORE sending, so a Stop on some OTHER chat -- still queued on the lock above, having
+ # generated nothing -- cannot reset the generation this is starting. Claiming after the send left the
+ # command running unclaimed. Released in the finally. Own mailbox: a compare request can start the
+ # dispatcher while this is streaming, and it would otherwise consume our responses and drop them.
read_one, drain, release_mailbox = self._direct_reader(request_id)
try:
try:
@@ -2511,12 +2386,12 @@ class InferenceOrchestrator:
"""
with self._active_cancel_lock:
if not self._active_cancel_events:
- # Nothing in flight at all, so there is no one to protect.
return True
if self._executing_cancel_events:
return any(ev is cancel_event for ev in self._executing_cancel_events)
- # Claimed but nothing has answered yet (A is in prefill). The worker takes commands
- # in order, so the oldest claim is the executor; anyone else here is queued behind it.
+ # Claimed but nothing has answered yet.
+ # Claimed but nothing has answered yet (A is in prefill). The worker takes commands in order, so the oldest
+ # claim is the executor; anyone else here is queued behind it.
return self._active_cancel_events[0] is cancel_event
def reset_generation_state(self, caller_cancel_event = None):
@@ -2540,8 +2415,7 @@ class InferenceOrchestrator:
except RuntimeError:
pass
- # ------------------------------------------------------------------
- # Audio generation — TTS, ASR, audio input
+ # ------------------------------------------------------------------ Audio generation - TTS, ASR, audio input
# ------------------------------------------------------------------
def generate_audio_response(
@@ -2569,10 +2443,9 @@ class InferenceOrchestrator:
raise RuntimeError("No active model")
expected_model = self.active_model_name
- # Serialize under _gen_lock and reserve dispatcher admission before waiting for
- # compare work to drain. A bare idle wait is racy: a compare request can register
- # between the wait and this command, leaving TTS queued without safe ownership of
- # the worker's single shared cancel event.
+ # Serialize under _gen_lock and reserve dispatcher admission before waiting for compare work to drain. A bare
+ # idle wait is racy: a compare request can register between the wait and this command, leaving TTS queued
+ # without safe ownership of the worker's single shared cancel event.
with self._gen_lock:
with self._dispatcher_lifecycle_lock:
self._exclusive_tts_pending = True
@@ -2585,13 +2458,13 @@ class InferenceOrchestrator:
"Cannot start audio generation while compare requests are active"
)
- # Recheck after the dispatcher wait: unload can set its flag without
- # _gen_lock, and a switch may have completed while this call was queued.
+ # Recheck after the dispatcher wait: unload can set its flag without _gen_lock, and a switch may have
+ # completed while this call was queued.
if self._unload_pending or self.active_model_name != expected_model:
raise AudioGenerationCancelledError("model is being unloaded")
- # Bound public API integers before either enqueuing work or
- # calculating the floating-point watchdog deadline.
+ # Bound public API integers before either enqueuing work or calculating the floating-point watchdog
+ # deadline
model_info = self.models.get(expected_model, {})
audio_type = model_info.get("audio_type")
max_token_ceiling = AUDIO_GENERATION_MAX_TOKENS
@@ -2636,8 +2509,8 @@ class InferenceOrchestrator:
# Same shared-queue hazard as _generate_inner: see _direct_reader.
read_one, _drain, release_mailbox = self._direct_reader(request_id)
try:
- # Claim before enqueueing so request-scoped reset ownership follows the same
- # discipline as text and audio-input generation.
+ # Claim before enqueueing so request-scoped reset ownership follows the same discipline as text and
+ # audio-input generation
with self._send_order_lock:
self._claim_worker(cancel_event)
self._send_cmd(cmd)
@@ -2661,12 +2534,12 @@ class InferenceOrchestrator:
and not cancel_signalled
and self._owns_worker(cancel_event)
):
- # audio_started is emitted after the worker clears stale state,
- # so this signal cannot be erased or hit an earlier request.
+ # audio_started is emitted after the worker clears stale state, so this signal cannot be
+ # erased or hit an earlier request.
self._cancel_generation()
cancel_signalled = True
- # The cancel is delivered now, so hold the worker to the drain
- # window from here rather than from when the caller asked.
+ # The cancel is delivered now, so hold the worker to the drain window from here rather than
+ # from when the caller asked
cancel_deadline = time.monotonic() + _AUDIO_CANCEL_DRAIN_TIMEOUT
deadline = min(deadline, cancel_deadline)
remaining = max(0.1, deadline - time.monotonic())
@@ -2713,18 +2586,18 @@ class InferenceOrchestrator:
if rtype == "status":
continue
- # A caller cancellation already spent the drain window polling this
- # request's mailbox. Tear down an unresponsive worker now instead of
- # waiting out the much longer generation watchdog or draining twice.
+ # A caller cancellation already spent the drain window polling this request's mailbox. Tear down an
+ # unresponsive worker now instead of waiting out the much longer generation watchdog or draining
+ # twice.
if cancel_deadline is not None:
if self._shutdown_subprocess(timeout = _AUDIO_CANCEL_DRAIN_TIMEOUT):
self.active_model_name = None
self.models.clear()
raise AudioGenerationCancelledError("Audio generation cancelled")
- # Do not release worker ownership or dispatcher exclusivity over a
- # command that may still be generating. Cancel, consume its terminal
- # response, and tear down a worker that does not acknowledge promptly.
+ # Do not release worker ownership or dispatcher exclusivity over a command that may still be
+ # generating. Cancel, consume its terminal response, and tear down a worker that does not
+ # acknowledge promptly.
self._cancel_generation()
if not _drain(timeout = _AUDIO_CANCEL_DRAIN_TIMEOUT):
if self._shutdown_subprocess(timeout = _AUDIO_CANCEL_DRAIN_TIMEOUT):
@@ -2775,7 +2648,7 @@ class InferenceOrchestrator:
"""Audio input generation (e.g. Gemma 3n) — streams text tokens."""
yield from self._generate_audio_input_inner(
audio_array = audio_array,
- audio_type = None, # worker will use generate_audio_input_response
+ audio_type = None, # worker uses generate_audio_input_response
messages = messages,
system_prompt = system_prompt,
temperature = temperature,
@@ -2821,14 +2694,12 @@ class InferenceOrchestrator:
expected_model = self.active_model_name
with self._gen_lock:
- # Recheck under the lock (see _generate_inner): a raced unload/switch may have
- # cleared or swapped the model while we waited.
+ # Recheck under the lock (see _generate_inner): a raced unload/switch may have cleared or swapped the model
+ # while we waited.
if self._unload_pending or self.active_model_name != expected_model:
- # Won the lock handoff during a switch; don't start on the outgoing model.
yield GenStreamError("Error: model is being unloaded", public = True)
return
if cancel_event is not None and cancel_event.is_set():
- # Stopped while queued on the lock, same as _generate_inner.
return
request_id = str(uuid.uuid4())
@@ -2851,18 +2722,16 @@ class InferenceOrchestrator:
"max_new_tokens": max_new_tokens,
"repetition_penalty": repetition_penalty,
}
- # As in the text path: key stays absent unless the caller selected one.
if use_adapter is not None:
cmd["use_adapter"] = use_adapter
if stop:
cmd["stop"] = stop
- # Same shared-queue hazard as _generate_inner: see _direct_reader.
read_one, drain, release_mailbox = self._direct_reader(request_id)
try:
try:
- # Claim under the send lock, like _generate_inner: unclaimed, a compare request queued
- # behind this looked like the oldest owner, so stopping it killed this one.
+ # Claim under the send lock, like _generate_inner: unclaimed, a compare request queued behind this
+ # looked like the oldest owner, so stopping it killed this one.
with self._send_order_lock:
self._claim_worker(cancel_event)
self._send_cmd(cmd)
@@ -2881,8 +2750,7 @@ class InferenceOrchestrator:
self._release_worker(cancel_event)
release_mailbox()
- # ------------------------------------------------------------------
- # Local helpers (no subprocess needed)
+ # ------------------------------------------------------------------ Local helpers (no subprocess needed)
# ------------------------------------------------------------------
def resize_image(
@@ -2934,9 +2802,9 @@ class InferenceOrchestrator:
# ========== GLOBAL INSTANCE ==========
_inference_backend = None
-# Guards the lazy construction below. The first build runs hardware detection, seconds cold,
-# and first-paint routes call this getter from executor threads. Unlocked, several would see
-# None and each build an orchestrator, orphaning all but the last plus any load on them.
+# Guards the lazy construction below. The first build runs hardware detection, seconds cold, and first-paint routes call
+# this getter from executor threads. Unlocked, several would see None and each build an orchestrator, orphaning all but
+# the last plus any load on them.
_inference_backend_lock = threading.Lock()
@@ -2952,7 +2820,7 @@ def peek_inference_backend() -> Optional["InferenceOrchestrator"]:
def get_inference_backend() -> InferenceOrchestrator:
"""Global inference backend instance (orchestrator)."""
global _inference_backend
- # Double-checked: the cheap read keeps the hot path lock-free, the recheck picks a builder.
+ # Double-checked: the cheap read keeps the hot path lock-free, the recheck picks a builder
if _inference_backend is None:
with _inference_backend_lock:
if _inference_backend is None:
diff --git a/studio/backend/core/inference/passthrough_healing.py b/studio/backend/core/inference/passthrough_healing.py
index 1d18140c7f0..2b0e64242a7 100644
--- a/studio/backend/core/inference/passthrough_healing.py
+++ b/studio/backend/core/inference/passthrough_healing.py
@@ -35,17 +35,16 @@ from core.inference.tool_loop_controller import (
)
from core.tool_healing import parse_tool_calls_from_text
-# Only the formats this healer's parser can promote -- narrower than the loops'
-# broader TOOL_XML_SIGNALS. A loop-only marker (Llama <|python_tag|>, bare
-# [ARGS]) would buffer a streamed call as prose without promoting it, so keep a
+# narrower than the loops' TOOL_XML_SIGNALS
+# Only the formats this healer's parser can promote -- narrower than the loops' broader TOOL_XML_SIGNALS. A loop-only
+# marker (Llama <|python_tag|>, bare [ARGS]) would buffer a streamed call as prose without promoting it, so keep a
# healer-aligned list. Mistral's [TOOL_CALLS] IS promotable, so it stays in.
_HEAL_SIGNALS = (
"<tool_call>",
"<|tool_call>",
"<function=",
"[TOOL_CALLS]",
- # TML Inkling native call marker (leaks as text when the server-side
- # parser misses a narration-then-call turn).
+ # TML Inkling native call marker (leaks as text when the server-side parser misses a narration-then-call turn).
"<|content_invoke_tool_json|>",
)
@@ -56,8 +55,8 @@ def _has_heal_signal(text: str) -> bool:
# Read once at import (same convention as the other UNSLOTH_* switches).
_HEALING_DISABLED = os.environ.get("UNSLOTH_DISABLE_TOOL_CALL_HEALING", "0") == "1"
-# Nudging is OPT-IN: per-request nudge_tool_calls=true, or flip the process
-# default with UNSLOTH_TOOL_CALL_NUDGE=1 (e.g. an `unsloth run` operator).
+# Nudging is OPT-IN: per-request nudge_tool_calls=true, or flip the process default with UNSLOTH_TOOL_CALL_NUDGE=1 (e.g.
+# an `unsloth run` operator).
_NUDGE_DEFAULT = os.environ.get("UNSLOTH_TOOL_CALL_NUDGE", "0") == "1"
@@ -66,8 +65,9 @@ def nudge_enabled(request_flag: Optional[bool]) -> bool:
_MAX_SIGNAL_LEN = max(len(s) for s in _HEAL_SIGNALS)
-# A suspected-but-unclosed tool block larger than this is declared a false
-# alarm and flushed, bounding memory on a model rambling XML-lookalike text.
+# a suspected-but-unclosed block larger than this is a false alarm, bounding memory on XML-lookalike prose
+# A suspected-but-unclosed tool block larger than this is declared a false alarm and flushed, bounding memory on a model
+# rambling XML-lookalike text.
_MAX_HOLD_CHARS = 64 * 1024
@@ -329,14 +329,16 @@ class StreamToolCallHealer:
self._buffer = ""
self._holding = False
self._id_offset = 0
- # Markup span each promoted call was cut from, keyed by its call id.
- # Promotion is destructive -- the span never reaches the client -- so a
- # caller that ends up DISCARDING a promoted call (a turn truncated at
- # finish_reason "length" cannot be executed) has no other way to give
- # the model's own words back. Bounded by _MAX_HOLD_CHARS per span.
+ # promotion is destructive, so a caller DISCARDING a promoted call (a turn truncated at finish_reason "length")
+ # has no other way to give the model's words back
+ # Markup span each promoted call was cut from, keyed by its call id. Promotion is destructive -- the span never
+ # reaches the client -- so a caller that ends up DISCARDING a promoted call (a turn truncated at finish_reason
+ # "length" cannot be executed) has no other way to give the model's own words back. Bounded by _MAX_HOLD_CHARS
+ # per span.
self._promoted_spans: dict[str, str] = {}
- # Structured delta.tool_calls seen upstream: grammar mode already
- # worked, so healing goes dormant and text relays verbatim.
+ # structured delta.tool_calls upstream means grammar mode worked, so healing goes dormant
+ # Structured delta.tool_calls seen upstream: grammar mode already worked, so healing goes dormant and text
+ # relays verbatim.
self.dormant = False
@property
@@ -380,10 +382,6 @@ class StreamToolCallHealer:
events.append(("text", emit))
self._buffer = self._buffer[len(self._buffer) - keep :]
return events
- # HOLD: drain the first contiguous run per pass so events keep document
- # order (a later declared call must not overtake an earlier undeclared one
- # flushing as text). A run is one markup call OR a whole Mistral [TOOL_CALLS]
- # array of contiguous spans, so later calls in it are not stranded as text.
parsed, spans = parse_tool_calls_from_text(
self._buffer,
id_offset = self._id_offset,
@@ -407,8 +405,8 @@ class StreamToolCallHealer:
pos = 0
run_end = spans[0][1]
for order, (call, (start, end)) in enumerate(zip(parsed, spans)):
- # Stop at the first gap or incomplete trailing block: leave it for the
- # next pass to re-hold and stream incrementally, not flush as text early.
+ # Stop at the first gap or incomplete trailing block: leave it for the next pass to re-hold and stream
+ # incrementally, not flush as text early.
if order and start != run_end:
break
promoted = _promote(
@@ -524,10 +522,9 @@ def response_has_promotable_calls(
return False
tool_calls = message.get("tool_calls")
if tool_calls:
- # ALL structured calls must be declared: the caller forwards the whole
- # list (and a parallel cap could keep only the FIRST one), so a mixed
- # response with a single hallucinated name could still hand the client
- # an undeclared tool.
+ # ALL structured calls must be declared: the caller forwards the whole list (and a parallel cap could keep only
+ # the FIRST one), so a mixed response with a single hallucinated name could still hand the client an undeclared
+ # tool.
return all(
isinstance(tc, dict)
and isinstance(tc.get("function"), dict)
diff --git a/studio/backend/core/inference/presence_penalty.py b/studio/backend/core/inference/presence_penalty.py
index c73c5138878..4b855d4c710 100644
--- a/studio/backend/core/inference/presence_penalty.py
+++ b/studio/backend/core/inference/presence_penalty.py
@@ -24,11 +24,11 @@ def apply_presence_penalty(input_ids, scores, penalty: float, prompt_len: int):
if generated.numel() == 0:
continue
seen = torch.unique(generated)
- # Bound generated ids to the valid range [0, vocab_size). Real completion
- # tokens are always in range, so this is a zero-regression safety net that
- # drops any stray out-of-range or negative id before indexing (mirrors the
- # MLX path's bound). Filtering both ends avoids indexing scores with a
- # negative id (which would silently wrap to the wrong row).
+ # drop out-of-range/negative ids before indexing: a negative id would wrap to the wrong row
+ # Bound generated ids to the valid range [0, vocab_size). Real completion tokens are always in range, so this is
+ # a zero-regression safety net that drops any stray out-of-range or negative id before indexing (mirrors the MLX
+ # path's bound). Filtering both ends avoids indexing scores with a negative id (which would silently wrap to the
+ # wrong row).
seen = seen[(seen >= 0) & (seen < vocab_size)]
if seen.numel():
scores[b, seen] = scores[b, seen] - penalty
diff --git a/studio/backend/core/inference/pricing.py b/studio/backend/core/inference/pricing.py
index 6886742a797..add4bbafecb 100644
--- a/studio/backend/core/inference/pricing.py
+++ b/studio/backend/core/inference/pricing.py
@@ -12,8 +12,7 @@ from __future__ import annotations
import datetime as _dt
from typing import Any, Optional
-# Per-MTok base USD. Cache multipliers apply to `input_per_mtok`
-# (not absolute prices), per Anthropic docs.
+# Per-MTok base USD. Cache multipliers apply to `input_per_mtok` (not absolute prices), per Anthropic docs.
ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
"claude-fable-5": {"input_per_mtok": 10.0, "output_per_mtok": 50.0},
"claude-mythos-5": {"input_per_mtok": 10.0, "output_per_mtok": 50.0},
@@ -23,8 +22,7 @@ ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
"claude-opus-4-8": {"input_per_mtok": 5.0, "output_per_mtok": 25.0},
"claude-opus-4-7": {"input_per_mtok": 5.0, "output_per_mtok": 25.0},
"claude-opus-4-6": {"input_per_mtok": 5.0, "output_per_mtok": 25.0},
- # Alias bare + dated id: backend defaults use the bare form, which
- # won't prefix-match the dated key.
+ # Alias bare + dated id: backend defaults use the bare form, which won't prefix-match the dated key.
"claude-opus-4-5": {"input_per_mtok": 5.0, "output_per_mtok": 25.0},
"claude-opus-4-5-20251101": {"input_per_mtok": 5.0, "output_per_mtok": 25.0},
"claude-opus-4-1": {"input_per_mtok": 15.0, "output_per_mtok": 75.0},
@@ -39,9 +37,8 @@ ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
}
OPENAI_PRICING: dict[str, dict[str, float]] = {
- # Verified against developers.openai.com/api/docs/pricing.
- # `long_context_*` keys apply past the threshold (gpt-5.6/5.5/5.4: 272k);
- # families without them ship a single rate.
+ # Verified against developers.openai.com/api/docs/pricing. `long_context_*` keys apply past the threshold
+ # (gpt-5.6/5.5/5.4: 272k); families without them ship a single rate.
"gpt-5.6-sol": {
"input_per_mtok": 5.0,
"output_per_mtok": 30.0,
@@ -85,19 +82,16 @@ OPENAI_PRICING: dict[str, dict[str, float]] = {
# chat-latest aliases gpt-5.5.
"gpt-5.3-chat-latest": {"input_per_mtok": 5.0, "output_per_mtok": 30.0},
"chat-latest": {"input_per_mtok": 5.0, "output_per_mtok": 30.0},
- # o-series / gpt-4.5 left off the pricing page: omit so calculate_cost
- # returns priced=False instead of silently $0.
+ # o-series / gpt-4.5 are left off the pricing page
}
# Shared multipliers (all Anthropic models).
ANTHROPIC_CACHE_5M_WRITE_MULT = 1.25
ANTHROPIC_CACHE_1H_WRITE_MULT = 2.0
ANTHROPIC_CACHE_READ_MULT = 0.1
-# Anthropic fast-mode is Opus 5 / Opus 4.8 only, both at $10/$50 per MTok
-# against a $5/$25 base, i.e. 2x on input + output. Opus 4.6 accepts `speed`
-# but reports `usage.speed: "standard"` and bills standard rates, so it never
-# reaches this multiplier.
-# https://platform.claude.com/docs/en/build-with-claude/fast-mode#pricing
+# Anthropic fast-mode is Opus 5 / Opus 4.8 only, both at $10/$50 per MTok against a $5/$25 base, i.e. 2x on input +
+# output. Opus 4.6 accepts `speed` but reports `usage.speed: "standard"` and bills standard rates, so it never reaches
+# this multiplier. https://platform.claude.com/docs/en/build-with-claude/fast-mode#pricing
ANTHROPIC_FAST_MODE_MULT = 2.0
ANTHROPIC_FAST_MODE_MULT_BY_MODEL = {
"claude-opus-5": 2.0,
@@ -107,23 +101,20 @@ ANTHROPIC_FAST_MODE_MULT_BY_MODEL = {
# OpenAI: cache reads 0.1x; cache writes pay input price.
OPENAI_CACHE_READ_MULT = 0.1
-# Server-tool surcharges. Anthropic code_exec: $0.05/hr marginal
-# (50 free hours/day per org, not shown here).
+# Server-tool surcharges. Anthropic code_exec: $0.05/hr marginal (50 free hours/day per org, not shown here).
ANTHROPIC_WEB_SEARCH_USD_PER_1K = 10.0
ANTHROPIC_CODE_EXEC_USD_PER_HOUR = 0.05
-# OpenAI container bills per memory tier; report the 1g default
-# ($0.09/hr) since the tier isn't surfaced to the ledger.
+# OpenAI container bills per memory tier; report the 1g default ($0.09/hr) since the tier isn't surfaced to the ledger.
OPENAI_WEB_SEARCH_USD_PER_1K = 10.0
OPENAI_CONTAINER_USD_PER_HOUR = 0.09 # 1g default tier
-# Claude Sonnet 5 launched on introductory pricing of $2/$10 per MTok, in force
-# through 2026-08-31; ANTHROPIC_PRICING carries the standard $3/$15 that takes
-# over on 2026-09-01. Billing the standard rate early overstates every Sonnet 5
-# turn by 50%, so overlay the launch rate while it lasts. The entry expires on
-# its own and can be deleted after the cutover.
-# https://platform.claude.com/docs/en/about-claude/pricing
+# Sonnet 5 launched at $2/$10 per MTok through 2026-08-31 while ANTHROPIC_PRICING carries the standard $3/$15
+# Claude Sonnet 5 launched on introductory pricing of $2/$10 per MTok, in force through 2026-08-31; ANTHROPIC_PRICING
+# carries the standard $3/$15 that takes over on 2026-09-01. Billing the standard rate early overstates every Sonnet 5
+# turn by 50%, so overlay the launch rate while it lasts. The entry expires on its own and can be deleted after the
+# cutover. https://platform.claude.com/docs/en/about-claude/pricing
_LAUNCH_PRICING: dict[str, tuple[_dt.date, dict[str, float]]] = {
"claude-sonnet-5": (
_dt.date(2026, 9, 1),
@@ -161,8 +152,8 @@ def _lookup(provider: str, model: str) -> Optional[dict[str, float]]:
return None
if model in table:
return table[model]
- # Longest-prefix match on a dash boundary: dated snapshots inherit
- # canonical prices, but "claude-opus-4-15" won't match "claude-opus-4-1".
+ # Longest-prefix match on a dash boundary: dated snapshots inherit canonical prices, but "claude-opus-4-15" won't
+ # match "claude-opus-4-1".
for key in sorted(table, key = len, reverse = True):
if model.startswith(key) and (len(model) == len(key) or model[len(key)] == "-"):
return table[key]
@@ -190,21 +181,18 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str
"priced": bool(prices),
}
- # Accept raw (input_tokens/output_tokens) and Unsloth chat-style
- # (prompt_tokens/completion_tokens) envelopes. Cache buckets differ:
- # raw Anthropic: input_tokens EXCLUDES cache buckets
- # raw OpenAI: input_tokens INCLUDES cache_read
- # Unsloth Anthropic: prompt_tokens INCLUDES cache_creation + cache_read
- # Unsloth OpenAI: prompt_tokens == raw input_tokens
- # Clamp >=0 so corrupted payloads can't produce a negative bill.
+ # Accept raw (input_tokens/output_tokens) and Unsloth chat-style (prompt_tokens/completion_tokens) envelopes. Cache
+ # buckets differ: raw Anthropic: input_tokens EXCLUDES cache buckets raw OpenAI: input_tokens INCLUDES cache_read
+ # Unsloth Anthropic: prompt_tokens INCLUDES cache_creation + cache_read Unsloth OpenAI: prompt_tokens == raw
+ # input_tokens Clamp >=0 so corrupted payloads can't produce a negative bill.
cache_creation = max(0, int(usage.get("cache_creation_input_tokens") or 0))
cache_read_native_present = (
"cache_read_input_tokens" in usage and usage.get("cache_read_input_tokens") is not None
)
cache_read = max(0, int(usage.get("cache_read_input_tokens") or 0))
- # Fall back to mirrored prompt_tokens_details only when native
- # cache_read_input_tokens is absent; an explicit native 0 is
- # authoritative, so a stale proxy mirror can't inflate cache_read.
+ # an explicit native cache_read_input_tokens of 0 is authoritative
+ # Fall back to mirrored prompt_tokens_details only when native cache_read_input_tokens is absent; an explicit native
+ # 0 is authoritative, so a stale proxy mirror can't inflate cache_read.
if not cache_read_native_present:
details = usage.get("prompt_tokens_details") or {}
if isinstance(details, dict):
@@ -213,22 +201,19 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str
if has_input_tokens:
input_tokens = max(0, int(usage.get("input_tokens") or 0))
else:
- # Chat-style: peel cache buckets back out for Anthropic to get
- # the raw uncached prompt count.
+ # chat-style: peel cache buckets back out for Anthropic to get the raw uncached prompt count
prompt_tokens = max(0, int(usage.get("prompt_tokens") or 0))
if provider == "anthropic":
input_tokens = max(0, prompt_tokens - cache_creation - cache_read)
else:
input_tokens = prompt_tokens
- # Prefer raw output_tokens even when 0 (an `or` would pick a stale
- # completion_tokens).
+ # Prefer raw output_tokens even when 0 (an `or` would pick a stale completion_tokens).
if "output_tokens" in usage and usage.get("output_tokens") is not None:
output_tokens = max(0, int(usage.get("output_tokens") or 0))
else:
output_tokens = max(0, int(usage.get("completion_tokens") or 0))
if provider == "openai":
- # Cached tokens land on input_tokens_details (raw Responses) or
- # prompt_tokens_details (Unsloth chat-style).
+ # cached tokens land on input_tokens_details (raw Responses) or prompt_tokens_details (Unsloth chat-style)
for key in ("input_tokens_details", "prompt_tokens_details"):
details = usage.get(key) or {}
if isinstance(details, dict):
@@ -243,11 +228,10 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str
if not prices:
return out
- # Long-context tier: whole-turn flip (not per-token blend) once
- # billable_input_tokens crosses the threshold. Strictly above: OpenAI bills
- # "prompts with >272K input tokens" at the higher rate, so a prompt of
- # exactly 272,000 stays on the standard rate -- which is also what the
- # "(long-context >{lc_thresh})" label emitted below already claims.
+ # Long-context tier: whole-turn flip (not per-token blend) once billable_input_tokens crosses the threshold.
+ # Strictly above: OpenAI bills "prompts with >272K input tokens" at the higher rate, so a prompt of exactly 272,000
+ # stays on the standard rate -- which is also what the "(long-context >{lc_thresh})" label emitted below already
+ # claims.
lc_thresh = prices.get("long_context_threshold")
in_long_context_tier = (
lc_thresh is not None
@@ -263,9 +247,8 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str
base = prices["input_per_mtok"]
out_per = prices["output_per_mtok"]
- # Anthropic fast-mode multiplier on input + output. Cache multipliers stack
- # on top, so applying once to (base, out_per) flows into the
- # cache_*_usd buckets below.
+ # Anthropic fast-mode multiplier on input + output. Cache multipliers stack on top, so applying once to (base,
+ # out_per) flows into the cache_*_usd buckets below.
if provider == "anthropic" and usage.get("speed") == "fast":
fast_mult = next(
(m for k, m in ANTHROPIC_FAST_MODE_MULT_BY_MODEL.items() if model.startswith(k)),
@@ -280,8 +263,8 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str
out["output_usd"] = (output_tokens / 1_000_000.0) * out_per
if provider == "anthropic":
- # Split cache_creation into 5m / 1h buckets when surfaced.
- # Tolerate non-dict (some proxies fold to an int total).
+ # Split cache_creation into 5m / 1h buckets when surfaced. Tolerate non-dict (some proxies fold to an int
+ # total).
cc_raw = usage.get("cache_creation")
cc_breakdown = cc_raw if isinstance(cc_raw, dict) else {}
cc_5m = max(0, int(cc_breakdown.get("ephemeral_5m_input_tokens") or 0))
@@ -293,7 +276,6 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str
cc_1h / 1_000_000.0
) * base * ANTHROPIC_CACHE_1H_WRITE_MULT
out["cache_read_usd"] = (cache_read / 1_000_000.0) * base * ANTHROPIC_CACHE_READ_MULT
- # Server-tool surcharges.
srv = usage.get("server_tool_use") or {}
if isinstance(srv, dict):
web_searches = int(srv.get("web_search_requests") or 0)
@@ -303,15 +285,13 @@ def calculate_cost(provider: str, model: str, usage: dict[str, Any]) -> dict[str
+ code_exec_hours * ANTHROPIC_CODE_EXEC_USD_PER_HOUR
)
else:
- # OpenAI: cache writes pay base input, only reads get 0.1x.
- # Subtract cached from already-counted input_usd to avoid
- # double-billing (OpenAI folds cache into input_tokens).
+ # OpenAI: cache writes pay base input, only reads get 0.1x. Subtract cached from already-counted input_usd to
+ # avoid double-billing (OpenAI folds cache into input_tokens).
if cache_read > 0:
non_cached_input = max(0, input_tokens - cache_read)
out["input_usd"] = (non_cached_input / 1_000_000.0) * base
out["cache_read_usd"] = (cache_read / 1_000_000.0) * base * OPENAI_CACHE_READ_MULT
- # OpenAI server-tool surcharges arrive under `openai_tool_use`
- # (normalised by the SSE finaliser from output items).
+ # OpenAI server-tool surcharges arrive under `openai_tool_use` (normalised by the SSE finaliser)
srv = usage.get("openai_tool_use") or {}
if isinstance(srv, dict):
web_searches = int(srv.get("web_search_requests") or 0)
@@ -336,8 +316,7 @@ def pricing_snapshot() -> dict[str, Any]:
"""Whole pricing table for the /api/providers/pricing endpoint."""
return {
"anthropic": {
- # Same launch-rate overlay calculate_cost bills on, so the
- # /api/providers/pricing table matches the ledger.
+ # Same launch-rate overlay calculate_cost bills on, so the /api/providers/pricing table matches the ledger.
"models": {
model: _launch_prices("anthropic", model, prices)
for model, prices in ANTHROPIC_PRICING.items()
diff --git a/studio/backend/core/inference/repetition_guard.py b/studio/backend/core/inference/repetition_guard.py
index 12a9ae4ccd4..6f6a30bcd45 100644
--- a/studio/backend/core/inference/repetition_guard.py
+++ b/studio/backend/core/inference/repetition_guard.py
@@ -20,12 +20,12 @@ from __future__ import annotations
import math
-# Below this, a fragment is too short to judge. A sentence cut mid-word can trivially
-# repeat a few tokens and is legitimately continued.
+# Below this, a fragment is too short to judge. A sentence cut mid-word can trivially repeat a few tokens and is
+# legitimately continued.
MIN_FRAGMENT_LENGTH = 400
-# Length of the exact-repeat window. A verbatim repeat this long is well beyond ordinary
-# reuse of phrasing, citations, headings or boilerplate.
+# Length of the exact-repeat window. A verbatim repeat this long is well beyond ordinary reuse of phrasing, citations,
+# headings or boilerplate.
_REPEAT_WINDOW = 60
# A window repeating at least this many times is a signal even in a short fragment.
@@ -34,9 +34,9 @@ _MIN_REPEAT_COUNT = 5
# The share of the fragment repeated windows must cover before it counts as dominated.
_DOMINANCE_RATIO = 0.5
-# Ceiling on distinct windows held while scanning. Every fragment a context window can
-# actually produce stays well under this, so the judgement is unchanged in practice; it
-# exists so the scan cannot grow with an arbitrarily long input.
+# ceiling so the scan cannot grow with an arbitrarily long input
+# Ceiling on distinct windows held while scanning. Every fragment a context window can actually produce stays well under
+# this, so the judgement is unchanged in practice; it exists so the scan cannot grow with an arbitrarily long input.
_MAX_TRACKED_WINDOWS = 100_000
@@ -56,13 +56,16 @@ def is_repetition_dominated(text: str) -> bool:
return True
# Sliding exact-repeat windows, for echoes that do not align to line boundaries.
needed = max(_MIN_REPEAT_COUNT, math.ceil(length * _DOMINANCE_RATIO / _REPEAT_WINDOW))
- # Keyed by HASH, not by the window itself. Retaining the 60-character slices meant one
- # entry per starting offset, so an 800,000-character fragment held roughly 180 MB of
- # substrings alive purely to decide whether to send one more continuation.
+ # keyed by HASH: retaining the 60-char slices held ~180 MB for an 800,000-character fragment
+ # Keyed by HASH, not by the window itself. Retaining the 60-character slices meant one entry per starting offset, so
+ # an 800,000-character fragment held roughly 180 MB of substrings alive purely to decide whether to send one more
+ # continuation.
counts: dict[int, int] = {}
- # Occurrences must not overlap, or a single run of one character counts as many. A
- # 64-character rule inside a 400-character answer yields five overlapping 60-character
- # windows and tripped the threshold at 16 percent coverage, abandoning a valid answer.
+ # occurrences must not overlap, else one repeated character counts as many (a 64-char rule tripped the threshold at
+ # 16% coverage)
+ # Occurrences must not overlap, or a single run of one character counts as many. A 64-character rule inside a
+ # 400-character answer yields five overlapping 60-character windows and tripped the threshold at 16 percent
+ # coverage, abandoning a valid answer.
covered_to: dict[int, int] = {}
# Where each hash was first seen, so a hash collision cannot be counted as a repeat.
first_at: dict[int, int] = {}
@@ -72,14 +75,12 @@ def is_repetition_dominated(text: str) -> bool:
continue
first = first_at.get(key)
if first is None:
- # Bounded even for a fragment far larger than any window can hold. Past the
- # cap, known windows keep counting and new ones are ignored, which can only
- # fail open -- the direction this guard already errs in.
+ # Bounded even for a fragment far larger than any window can hold. Past the cap, known windows keep counting
+ # and new ones are ignored, which can only fail open -- the direction this guard already errs in.
if len(first_at) >= _MAX_TRACKED_WINDOWS:
continue
first_at[key] = index
elif text[first : first + _REPEAT_WINDOW] != text[index : index + _REPEAT_WINDOW]:
- # Two different windows, one hash. Not a repeat.
continue
seen = counts.get(key, 0) + 1
if seen >= needed:
diff --git a/studio/backend/core/inference/sandbox_site/__init__.py b/studio/backend/core/inference/sandbox_site/__init__.py
index eea99aa5264..664b53b6e81 100644
--- a/studio/backend/core/inference/sandbox_site/__init__.py
+++ b/studio/backend/core/inference/sandbox_site/__init__.py
@@ -1,6 +1,4 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-# Package marker only, so wheel builds ship this directory. It goes on the
-# sandbox PYTHONPATH so site machinery imports the sibling ``sitecustomize`` at
-# startup; nothing in the backend imports it directly.
+# Package marker: on the sandbox PYTHONPATH so site machinery imports the sibling sitecustomize.
diff --git a/studio/backend/core/inference/sandbox_site/sitecustomize.py b/studio/backend/core/inference/sandbox_site/sitecustomize.py
index a909bfbb90d..e5aaba4be8b 100644
--- a/studio/backend/core/inference/sandbox_site/sitecustomize.py
+++ b/studio/backend/core/inference/sandbox_site/sitecustomize.py
@@ -33,18 +33,19 @@ import json
import os
import sys
-# Code-interpreter convention prefixes. Remapping is gated on the prefix being
-# ABSENT (see _remap) so a genuine host mount / user dir is never shadowed.
+# remapping is gated on the prefix being ABSENT (see _remap) so a genuine host mount is never shadowed
+# Code-interpreter convention prefixes. Remapping is gated on the prefix being ABSENT (see _remap) so a genuine host
+# mount / user dir is never shadowed.
_PREFIXES = ("/mnt/data", "/mnt/outputs", "/home/sandbox", "/workspace")
# /tmp exists on the host; separate only to note that. The absence gate applies alike.
_CONDITIONAL_PREFIXES = ("/tmp/outputs",)
_notified = False
-# Invented absolute write path -> healed CWD target, so re-writing the same
-# artifact re-serves it instead of tripping the anti-clobber guard.
+# Invented absolute write path -> healed CWD target, so re-writing the same artifact re-serves it instead of tripping
+# the anti-clobber guard.
_remapped_writes: dict = {}
-# Each tool call is a fresh subprocess (in-process map starts empty), so this
-# on-disk sidecar carries the map across runs. It records only sources the
-# fallback healed, so an unrelated same-basename file is never adopted.
+# each tool call is a fresh subprocess, so an on-disk sidecar carries the map across runs;
+# Each tool call is a fresh subprocess (in-process map starts empty), so this on-disk sidecar carries the map across
+# runs. It records only sources the fallback healed, so an unrelated same-basename file is never adopted.
_REMAP_SIDECAR = ".unsloth_sandbox_remap.json"
@@ -164,8 +165,8 @@ def _remap_open(file, mode):
# notify=False: emit the notice only once we commit to the mapping below.
mapped = _remap(file, notify = False)
if mapped is not file:
- # Write always heals; a read only when the mapped target exists (else keep
- # the original path so a missing input stays truthful).
+ # Write always heals; a read only when the mapped target exists (else keep the original path so a missing input
+ # stays truthful).
if creating or os.path.exists(mapped):
# Commit: emit the notice now (the notify=False peek above deferred it).
_remap(file, notify = True)
@@ -185,19 +186,18 @@ def _remap_open(file, mode):
if text == cwd or text.startswith(cwd + os.sep):
return file
parent = os.path.dirname(text)
- # Redirect only when the parent is missing; an existing external directory is
- # a deliberate target and stays truthful (os.path.exists follows symlinks).
+ # an existing external directory is a deliberate target and stays truthful (os.path.exists follows symlinks)
+ # Redirect only when the parent is missing; an existing external directory is a deliberate target and stays truthful
+ # (os.path.exists follows symlinks).
if parent and os.path.exists(parent):
return file
base = os.path.basename(text)
- # A trailing sep or '.'/'..' basename would redirect onto the CWD or its
- # parent; refuse and let open raise.
+ # A trailing sep or '.'/'..' basename would redirect onto the CWD or its parent; refuse and let open raise.
if base in ("", ".", ".."):
return file
remapped = os.path.join(cwd, base)
- # Never clobber an unrelated file sharing this basename (lexists catches
- # dangling symlinks). But a target this fallback already healed for the same
- # invented path (in-process map or cross-run sidecar) is the artifact being
+ # Never clobber an unrelated file sharing this basename (lexists catches dangling symlinks). But a target this
+ # fallback already healed for the same invented path (in-process map or cross-run sidecar) is the artifact being
# re-written, so re-serve it instead of raising on every overwrite.
if os.path.lexists(remapped) and remapped not in (
_remapped_writes.get(text),
@@ -223,8 +223,8 @@ def _remap(path, notify = True):
if not isinstance(text, str):
return path
for prefix in _PREFIXES + _CONDITIONAL_PREFIXES:
- # Heal only while the real prefix directory is absent, so a genuine host
- # mount / user directory at that prefix is never shadowed.
+ # Heal only while the real prefix directory is absent, so a genuine host mount / user directory at that prefix
+ # is never shadowed.
if (text == prefix or text.startswith(prefix + "/")) and not os.path.exists(prefix):
return _map_onto_cwd(prefix, text, notify = notify)
return path
@@ -256,8 +256,8 @@ def _install():
):
return original_io_open(_remap_open(file, mode), mode, *args, **kwargs)
- # mkdir/makedirs get only the prefix remap, never the write-mode fallback:
- # an arbitrary absolute directory can legitimately succeed on the host.
+ # mkdir/makedirs get only the prefix remap, never the write-mode fallback: an arbitrary absolute directory can
+ # legitimately succeed on the host.
def _makedirs(name, *args, **kwargs):
return original_makedirs(_remap(name), *args, **kwargs)
@@ -271,9 +271,9 @@ def _install():
*,
dir_fd = None,
):
- # Path.touch() etc. go through os.open, not builtins.open. Only O_CREAT
- # can create, so only it maps to "creating" mode; O_TRUNC / O_APPEND
- # without O_CREAT still require the file to exist, so behave as a read.
+ # only O_CREAT can create; O_TRUNC / O_APPEND without it still require the file to exist
+ # Path.touch() etc. go through os.open, not builtins.open. Only O_CREAT can create, so only it maps to
+ # "creating" mode; O_TRUNC / O_APPEND without O_CREAT still require the file to exist, so behave as a read.
logical_mode = "w" if (flags & os.O_CREAT) else "r"
mapped = _remap_open(path, logical_mode)
if dir_fd is None:
@@ -281,9 +281,9 @@ def _install():
return original_os_open(mapped, flags, mode, dir_fd = dir_fd)
def _path_mkdir(self, *args, **kwargs):
- # pathlib probes Path.is_dir()/os.stat (unpatched) on FileExistsError, so
- # a bare os.mkdir remap would still raise when the target exists. Remap
- # the receiver up front so parents/exist_ok stays idempotent.
+ # pathlib probes the unpatched Path.is_dir()/os.stat on FileExistsError
+ # pathlib probes Path.is_dir()/os.stat (unpatched) on FileExistsError, so a bare os.mkdir remap would still
+ # raise when the target exists. Remap the receiver up front so parents/exist_ok stays idempotent.
mapped = _remap(self)
target = self if mapped is self else self.__class__(mapped)
return original_path_mkdir(target, *args, **kwargs)
@@ -291,18 +291,17 @@ def _install():
builtins.open = _open
# pathlib.Path.open / write_text / read_text call io.open directly, so patch both.
io.open = _io_open
- # Python < 3.11 only: pathlib's accessor captured the ORIGINAL io.open at
- # import (``_NormalAccessor.open = io.open``), so the io.open patch misses it.
- # Repoint it at the same wrapper (staticmethod to stay unbound); 3.11+ dropped
- # the accessor, so this is a no-op there.
+ # Python < 3.11 only: pathlib's accessor captured the ORIGINAL io.open at import (``_NormalAccessor.open =
+ # io.open``), so the io.open patch misses it. Repoint it at the same wrapper (staticmethod to stay unbound); 3.11+
+ # dropped the accessor, so this is a no-op there.
accessor = getattr(pathlib, "_NormalAccessor", None)
if accessor is not None and hasattr(accessor, "open"):
accessor.open = staticmethod(_io_open)
# Path.touch() and other low-level opens call os.open directly, so patch it too.
os.open = _os_open
os.makedirs = _makedirs
- # Path.mkdir(parents=True) calls os.mkdir per component, so patch os.mkdir;
- # patch Path.mkdir itself too so exist_ok/parents land on the mapped path.
+ # Path.mkdir(parents=True) calls os.mkdir per component, so patch os.mkdir; patch Path.mkdir itself too so
+ # exist_ok/parents land on the mapped path.
os.mkdir = _mkdir
pathlib.Path.mkdir = _path_mkdir
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index 723b32059f8..7e598c1add3 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -24,7 +24,9 @@ from core.inference.diffusion_memory import (
OFFLOAD_SEQUENTIAL,
)
-# Per-family text-encoder flags, in supply order. Keyed by ``DiffusionFamily.name`` so the family registry need not import sd.cpp specifics.
+# in supply order, keyed by DiffusionFamily.name so the family registry need not import sd.cpp specifics
+# Per-family text-encoder flags, in supply order. Keyed by ``DiffusionFamily.name`` so the family registry need not
+# import sd.cpp specifics.
_TE_FLAGS_BY_FAMILY: dict[str, tuple[str, ...]] = {
"z-image": ("--llm",),
"flux.2-klein": ("--llm",),
@@ -39,7 +41,8 @@ def text_encoder_flags_for_family(family_name: str) -> tuple[str, ...]:
return _TE_FLAGS_BY_FAMILY.get(family_name, ())
-# sd-cli's image-gen mode (``img_gen``; older ``txt2img``). img2img is the same mode with --init-img, so one token covers both.
+# sd-cli's image-gen mode (``img_gen``; older ``txt2img``). img2img is the same mode with --init-img, so one token
+# covers both.
DEFAULT_MODE = "img_gen"
@@ -70,7 +73,8 @@ class SdCppGenParams:
prompt: str
negative_prompt: Optional[str] = None
- # None = unset: an image-conditioned run lets sd.cpp derive the size from the input; a plain txt2img falls back to 1024x1024.
+ # None = unset: an image-conditioned run lets sd.cpp derive the size from the input; a plain txt2img falls back to
+ # 1024x1024.
width: Optional[int] = None
height: Optional[int] = None
steps: Optional[int] = None
@@ -79,12 +83,10 @@ class SdCppGenParams:
seed: Optional[int] = None
sampling_method: Optional[str] = None
batch_count: int = 1
- # image-to-image / inpaint / edit
init_img: Optional[str] = None
strength: Optional[float] = None
mask: Optional[str] = None
ref_images: tuple[str, ...] = ()
- # LoRA
lora_dir: Optional[str] = None
lora_apply_mode: Optional[str] = None
@@ -125,7 +127,9 @@ class SdCppUpscaleParams:
tile_size: Optional[int] = None
-# Native (sd.cpp) speed profiles, the engine-side analogue of diffusion_speed. off: nothing. default: --diffusion-fa + --diffusion-conv-direct (numerically exact; direct conv took z-image Q8_0 sampling 56.1 -> 51.3 s). max keeps it (profiles chain).
+# Native (sd.cpp) speed profiles, the engine-side analogue of diffusion_speed. off: nothing. default: --diffusion-fa +
+# --diffusion-conv-direct (numerically exact; direct conv took z-image Q8_0 sampling 56.1 -> 51.3 s). max keeps it
+# (profiles chain).
NATIVE_SPEED_OFF = "off"
NATIVE_SPEED_DEFAULT = "default"
NATIVE_SPEED_MAX = "max"
@@ -171,12 +175,19 @@ def metal_text_encoder_flags() -> list[str]:
return ["--clip-on-cpu"]
-# Everything on the CPU backend. sd.cpp prefers GPU -> integrated GPU -> CPU and only `--backend` changes which backend EXECUTES the graph (`--offload-to-cpu` moves parameters, not compute), so this is the one flag that removes ggml-metal entirely.
+# sd.cpp prefers GPU -> iGPU -> CPU and only --backend changes which backend EXECUTES the graph (--offload-to-cpu moves
+# parameters)
+# Everything on the CPU backend. sd.cpp prefers GPU -> integrated GPU -> CPU and only `--backend` changes which backend
+# EXECUTES the graph (`--offload-to-cpu` moves parameters, not compute), so this is the one flag that removes ggml-metal
+# entirely.
CPU_BACKEND_FLAGS: tuple[str, ...] = ("--backend", "cpu")
-# Graph-cut segmented execution; a negative --max-vram auto-detects free VRAM per device, sparing that many GiB. It segments on its own, so it stands alone.
+# a negative --max-vram auto-detects free VRAM per device
+# Graph-cut segmented execution; a negative --max-vram auto-detects free VRAM per device, sparing that many GiB. It
+# segments on its own, so it stands alone.
GRAPH_CUT_VRAM_FLAGS: tuple[str, ...] = ("--max-vram", "-1")
-# Upstream only honours --stream-layers when the diffusion params backend is CPU, i.e. under --offload-to-cpu; otherwise it warns and ignores the flag.
+# Upstream only honours --stream-layers when the diffusion params backend is CPU, i.e. under --offload-to-cpu; otherwise
+# it warns and ignores the flag.
GRAPH_CUT_STREAM_FLAGS: tuple[str, ...] = ("--stream-layers",)
# The full set, for callers that already offload to CPU.
GRAPH_CUT_AUTO_FLAGS: tuple[str, ...] = GRAPH_CUT_VRAM_FLAGS + GRAPH_CUT_STREAM_FLAGS
@@ -231,7 +242,10 @@ def without_device_backend_flags(flags: Sequence[str]) -> list[str]:
return out
-# The ggml signature for "this graph cannot run on this backend at all": ggml-metal calls GGML_ABORT when ggml_metal_device_supports_op() returns false, since a single-backend graph has nowhere else to put the node. The SIGABRT takes sd-server down mid-generation.
+# ggml-metal calls GGML_ABORT when ggml_metal_device_supports_op() is false
+# The ggml signature for "this graph cannot run on this backend at all": ggml-metal calls GGML_ABORT when
+# ggml_metal_device_supports_op() returns false, since a single-backend graph has nowhere else to put the node. The
+# SIGABRT takes sd-server down mid-generation.
_GGML_UNSUPPORTED_OP_MARKERS = ("unsupported op", "ggml_abort")
@@ -306,10 +320,10 @@ def build_sd_cpp_command(
"""
if not files.diffusion_model:
raise ValueError("diffusion_model path is required")
- # ``(prompt or "")`` so a None prompt is rejected here, not passed as literal "None".
if not (params.prompt or "").strip():
raise ValueError("prompt is required")
- # sd-cli inpaint needs the source image: a --mask with no --init-img is invalid, so reject it rather than fail deep in sd-cli.
+ # sd-cli inpaint needs the source image: a --mask with no --init-img is invalid, so reject it rather than fail deep
+ # in sd-cli.
if params.mask and not params.init_img:
raise ValueError("init_img is required when mask is set (inpaint needs a source image)")
@@ -328,7 +342,6 @@ def build_sd_cpp_command(
cmd += ["--prompt", params.prompt]
if params.negative_prompt:
cmd += ["--negative-prompt", params.negative_prompt]
- # img2img / inpaint / edit conditioning.
if params.init_img:
cmd += ["--init-img", params.init_img]
if params.strength is not None:
@@ -342,7 +355,9 @@ def build_sd_cpp_command(
cmd += ["--lora-model-dir", params.lora_dir]
if params.lora_apply_mode:
cmd += ["--lora-apply-mode", params.lora_apply_mode]
- # Emit explicit dims when given. An image-conditioned run leaving them unset omits the flags so sd.cpp derives the size from the input; a plain txt2img keeps the 1024 default.
+ # leaving them unset lets sd.cpp derive the size from an input image
+ # Emit explicit dims when given. An image-conditioned run leaving them unset omits the flags so sd.cpp derives the
+ # size from the input; a plain txt2img keeps the 1024 default.
if params.width is not None or params.height is not None:
w = int(params.width) if params.width is not None else 1024
h = int(params.height) if params.height is not None else 1024
@@ -360,7 +375,8 @@ def build_sd_cpp_command(
if params.seed is not None:
cmd += ["--seed", str(int(params.seed))]
if params.batch_count and params.batch_count != 1:
- # sd-cli names extra batch images itself (output_2.png, ...) but the runner collects only --output, so a CLI batch drops all but the first. Batches use the sdcpp server API.
+ # sd-cli names extra batch images itself (output_2.png, ...) but the runner collects only --output, so a CLI
+ # batch drops all but the first. Batches use the sdcpp server API.
raise ValueError(
"sd-cli runs are single-image; use the sdcpp server API for batch generation."
)
@@ -425,12 +441,10 @@ def build_sd_cpp_video_command(
)
if len(params.ref_video_audios) > len(params.ref_videos):
raise ValueError("each reference video soundtrack needs a reference video to pair with")
- # Keyframes before the sampling flags, matching the img_gen builder's ordering.
if params.init_img:
cmd += ["--init-img", params.init_img]
if params.end_img:
cmd += ["--end-img", params.end_img]
- # Preserve the model's image, video, soundtrack, then standalone-audio order.
for ref in params.ref_images:
cmd += ["--ref-image", ref]
for ref in params.ref_videos:
@@ -468,10 +482,6 @@ def build_sd_cpp_video_command(
cmd += [f for f in metal_text_encoder_flags() if f not in offload]
if verbose:
cmd.append("-v")
- # Appended verbatim, like every sibling builder: sd.cpp's parser is last-wins, so a power user
- # can override anything. Token-wise de-duplication broke that contract on any flag that takes a
- # value, since it drops the token that already appears earlier: ["--rng", "cuda"] lost --rng and
- # left a bare "cuda" for the parser to choke on.
if extra_args:
cmd += list(extra_args)
return cmd
@@ -633,7 +643,10 @@ def build_img_gen_request(
req["seed"] = int(seed)
if sample_params:
req["sample_params"] = sample_params
- # Structured LoRA list: the API resolves each ``path`` against the server's ``--lora-model-dir`` (prompt-embedded ``<lora:>`` tags are unsupported server-side), so LoRAs are staged here.
+ # the API resolves each `path` against the server's --lora-model-dir (prompt-embedded <lora:> tags are unsupported
+ # server-side)
+ # Structured LoRA list: the API resolves each ``path`` against the server's ``--lora-model-dir`` (prompt-embedded
+ # ``<lora:>`` tags are unsupported server-side), so LoRAs are staged here.
if lora:
req["lora"] = lora
return req
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index d541169b229..1dba00face5 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -92,25 +92,25 @@ from utils.subprocess_compat import windows_hidden_subprocess_kwargs
logger = get_logger(__name__)
-# A sampling-progress line ("4/4", "[ 12/ 28]", "sampling: 50%|...| 14/28"). Only a denominator matching the requested step count is trusted, so a stray "1/100" cannot move the bar.
+# A sampling-progress line ("4/4", "[ 12/ 28]", "sampling: 50%|...| 14/28"). Only a denominator matching the requested
+# step count is trusted, so a stray "1/100" cannot move the bar.
_STEP_RE = re.compile(r"(\d+)\s*/\s*(\d+)")
# Serialises the one-time binary install so concurrent first-loads don't race.
_install_lock = threading.Lock()
-# Admission control over the managed tree, because "is anything running in there?" and "start
-# replacing it" have to be ONE decision. _managed_tree_in_use() alone is a point-in-time sample,
-# and an install spends seconds to minutes downloading before it extracts: a one-shot generation
-# admitted inside that window launches the very sd-cli the extraction then overwrites. Installs are
-# the writers, one-shot sd-cli runs are the readers. Held only across the state change, never
-# across a download or a generation, and the readers never take _install_lock, so there is no cycle.
+# Admission control over the managed tree, because "is anything running in there?" and "start replacing it" have to be
+# ONE decision. _managed_tree_in_use() alone is a point-in-time sample, and an install spends seconds to minutes
+# downloading before it extracts: a one-shot generation admitted inside that window launches the very sd-cli the
+# extraction then overwrites. Installs are the writers, one-shot sd-cli runs are the readers. Held only across the state
+# change, never across a download or a generation, and the readers never take _install_lock, so there is no cycle.
_tree_state = threading.Condition()
_tree_readers = 0
_tree_installing = False
-# A download can legitimately take minutes; wait rather than run a binary that is being replaced.
+# A download can legitimately take minutes; wait rather than run a binary that is being replaced
_TREE_WAIT_TIMEOUT_S = 900.0
-# How often the wait re-checks for cancellation. Nothing notifies the condition when a request is
-# cancelled, so a single long wait would hold the generate lock past an unload.
+# How often the wait re-checks for cancellation. Nothing notifies the condition when a request is cancelled, so a single
+# long wait would hold the generate lock past an unload.
_TREE_WAIT_TICK_S = 0.5
@@ -161,9 +161,8 @@ def _tree_reader(
deadline = time.monotonic() + _TREE_WAIT_TIMEOUT_S
while _tree_installing:
if cancel_event is not None and cancel_event.is_set():
- # The caller's own sentinel: the video path recognises only its own, and an
- # image message reaching it reads as "Video generation failed" for what is an
- # ordinary cancellation.
+ # The caller's own sentinel: the video path recognises only its own, and an image message reaching
+ # it reads as "Video generation failed" for what is an ordinary cancellation.
raise RuntimeError(cancelled_message)
remaining = deadline - time.monotonic()
if remaining <= 0:
@@ -183,7 +182,7 @@ def _tree_reader(
_tree_state.notify_all()
-# Max images per img_gen job; larger Unsloth batches (up to 32) are split into these chunks.
+# Max images per img_gen job; larger Unsloth batches (up to 32) are split into these chunks
_MAX_SERVER_BATCH = 8
@@ -219,9 +218,10 @@ def _server_binary_runnable(binary: str) -> bool:
)
except OSError:
return False # cannot exec at all (wrong arch / no execute bit / missing loader)
- except Exception: # noqa: BLE001 -- don't block on a flaky probe (timeout etc.)
+ except Exception: # noqa: BLE001 -- don't block on a flaky probe
return True
- # Negative return code = signal death (e.g. -4 SIGILL from an incompatible prebuilt): launches then crashes, so treat as unavailable.
+ # Negative return code = signal death (e.g. -4 SIGILL from an incompatible prebuilt): launches then crashes, so
+ # treat as unavailable.
return proc.returncode >= 0 and proc.returncode not in (126, 127)
@@ -278,19 +278,18 @@ def _sd_cpp_probe_output(binary: str, *args: str) -> Optional[str]:
env = runtime_env(binary),
**windows_hidden_subprocess_kwargs(),
)
- except Exception: # noqa: BLE001 -- cannot exec / timeout: treated as "cannot tell"
+ except Exception: # noqa: BLE001 -- cannot exec / timeout: "cannot tell"
return None
if proc.returncode != 0:
return None
return (proc.stdout or "") + (proc.stderr or "")
-# The ``--help`` token that marks a build carrying MiniMax-H3 support. Upstream added the mode's
-# own options (``--ref-video`` / ``--ref-audio``, and the "MiniMax-H3 Ref2VA" wording on
-# ``--ref-image``) in the very commit that added H3 upstream, released as
-# master-812-ea7f0c8, so their presence is exactly the capability signal. ``--version`` cannot
-# stand in for it: the release prebuilts are built without a .git dir and answer "stable-diffusion.cpp
-# version unknown, commit unknown", so there is no tag to compare.
+# The ``--help`` token that marks a build carrying MiniMax-H3 support. Upstream added the mode's own options
+# (``--ref-video`` / ``--ref-audio``, and the "MiniMax-H3 Ref2VA" wording on ``--ref-image``) in the very commit that
+# added H3 upstream, released as master-812-ea7f0c8, so their presence is exactly the capability signal. ``--version``
+# cannot stand in for it: the release prebuilts are built without a.git dir and answer "stable-diffusion.cpp version
+# unknown, commit unknown", so there is no tag to compare.
_H3_HELP_MARKER = "--ref-video"
@@ -328,7 +327,8 @@ def sd_cpp_binary_vets_for_h3(binary: str) -> bool:
return help_text_identifies_sd_cpp(text) and help_text_supports_minimax_h3(text)
-# The ``--help`` tokens marking a build with the graph-cut executor; both are required, since --stream-layers does nothing without --max-vram.
+# The ``--help`` tokens marking a build with the graph-cut executor; both are required, since --stream-layers does
+# nothing without --max-vram.
_GRAPH_CUT_HELP_MARKERS: tuple[str, ...] = ("--max-vram", "--stream-layers")
@@ -382,7 +382,8 @@ def sd_cpp_accelerator_device_verdict(binary: str) -> Optional[bool]:
return any(name.upper() != "CPU" for name in names)
-# ggml device-name prefixes indexed by CUDA physical ordinal (ggml names its HIP backend either way); Vulkan is excluded, its ordinals are another namespace.
+# ggml device-name prefixes indexed by CUDA physical ordinal (ggml names its HIP backend either way); Vulkan is
+# excluded, its ordinals are another namespace.
_PHYSICAL_INDEX_DEVICE_PREFIXES: tuple[str, ...] = ("CUDA", "ROCM")
@@ -404,11 +405,10 @@ def sd_cpp_device_name_for_ordinal(binary: Optional[str], ordinal: Optional[int]
continue
if name[len(head) :] == str(ordinal):
return name
- # Said out loud rather than dropped in silence: the load still runs, on whichever device this
- # build picks for itself, which is what happens today for every native load. Refusing instead
- # would take the GPU selection from "not honoured here" to "cannot load at all" on any build
- # older than the one that added --list-devices, including a user's own SD_CLI_PATH copy, since
- # sd.cpp treats an unknown argument as fatal.
+ # Said out loud rather than dropped in silence: the load still runs, on whichever device this build picks for
+ # itself, which is what happens today for every native load. Refusing instead would take the GPU selection from "not
+ # honoured here" to "cannot load at all" on any build older than the one that added --list-devices, including a
+ # user's own SD_CLI_PATH copy, since sd.cpp treats an unknown argument as fatal.
logger.warning(
"sd_cpp.device_pin_unresolved: this build does not report a CUDA/ROCm device %s "
"(--list-devices %s), so the graph runs on its own default device",
@@ -470,36 +470,32 @@ def ensure_h3_sd_cpp_binary(
binary = ensure_sd_cpp_binary(allow_install = allow_install, accelerator = accelerator)
if not binary:
return binary
- # ONE --help, two questions: is this stable-diffusion.cpp, and does this build carry H3. A
- # second spawn would double the cost of the refusal path and could read a different build than
- # the one just judged. None is "could not tell", which stays conservative on both counts.
- #
- # Conservative HERE means keeping the binary, the opposite of the engine's identity probe, which
- # rejects on an unreadable one. Not an inconsistency to iron out: this decides whether to refuse
- # a binary the user chose, where a probe failure must not take native video away from a working
- # build, while the engine decides whether to ADOPT an ambiguously named PATH candidate on no
- # evidence at all. Opposite questions, so opposite safe defaults.
+ # ONE --help, two questions: is this stable-diffusion.cpp, and does this build carry H3. A second spawn would double
+ # the cost of the refusal path and could read a different build than the one just judged. None is "could not tell",
+ # which stays conservative on both counts. Conservative HERE means keeping the binary, the opposite of the engine's
+ # identity probe, which rejects on an unreadable one. Not an inconsistency to iron out: this decides whether to
+ # refuse a binary the user chose, where a probe failure must not take native video away from a working build, while
+ # the engine decides whether to ADOPT an ambiguously named PATH candidate on no evidence at all. Opposite questions,
+ # so opposite safe defaults.
help_text = _sd_cpp_probe_output(binary, "--help")
if help_text is None:
return binary
- # Identity BEFORE capability, never the marker alone. --ref-video is a plain option name that
- # unrelated reference-video tools also expose, so returning early on it would readmit exactly
- # the class of program #8507 was about -- through SD_CLI_PATH instead of PATH. Upstream added
- # H3 eight months after print_usage started with the project banner, so a genuine H3 build
- # always answers both.
+ # Identity BEFORE capability, never the marker alone. --ref-video is a plain option name that unrelated
+ # reference-video tools also expose, so returning early on it would readmit exactly the class of program #8507 was
+ # about -- through SD_CLI_PATH instead of PATH. Upstream added H3 eight months after print_usage started with the
+ # project banner, so a genuine H3 build always answers both.
identified = help_text_identifies_sd_cpp(help_text)
if identified and help_text_supports_minimax_h3(help_text):
return binary
- # What is wrong with it, for the log lines on the managed path below: a managed copy that is not
- # sd.cpp at all is still deleted and reinstalled, but calling it an old build would be false.
+ # What is wrong with it, for the log lines on the managed path below: a managed copy that is not sd.cpp at all is
+ # still deleted and reinstalled, but calling it an old build would be false.
fault = "does not advertise MiniMax-H3 support" if identified else "is not stable-diffusion.cpp"
if not is_managed_binary(binary):
- # Not an old sd.cpp -- not sd.cpp at all. Worth its own message: the H3 marker is missing
- # from EVERY program that is not stable-diffusion.cpp, so reporting the capability verdict
- # here sent users hunting for a newer build of something they never installed (#8507, where
- # the binary was Debian/Ubuntu's `sd` find-and-replace tool). Discovery already skips an
- # unrelated PATH `sd`, so what reaches this line came from somewhere the identity gate does
- # not cover -- an SD_CLI_PATH / UNSLOTH_SD_CPP_PATH override, an in-tree developer build, or
+ # Not an old sd.cpp -- not sd.cpp at all. Worth its own message: the H3 marker is missing from EVERY program
+ # that is not stable-diffusion.cpp, so reporting the capability verdict here sent users hunting for a newer
+ # build of something they never installed (#8507, where the binary was Debian/Ubuntu's `sd` find-and-replace
+ # tool). Discovery already skips an unrelated PATH `sd`, so what reaches this line came from somewhere the
+ # identity gate does not cover -- an SD_CLI_PATH / UNSLOTH_SD_CPP_PATH override, an in-tree developer build, or
# a PATH `sd-cli`. None of them is ours to overwrite, so all four say so and stop.
if not identified:
raise RuntimeError(
@@ -519,11 +515,10 @@ def ensure_h3_sd_cpp_binary(
# Ours, but replacing it is exactly what auto-install is switched off for.
logger.warning("managed sd.cpp binary %s %s", binary, fault)
return None
- # Deleting it is a WRITE to the managed tree, so it takes the same admission an install does.
- # An image one-shot may be executing this very file: on Linux the running child survives the
- # unlink but the next image in the batch can no longer resolve it, and on Windows the unlink
- # fails outright and the H3 load is refused. Held only across the unlink -- ensure_sd_cpp_binary
- # below claims the tree itself, and the claim is not reentrant.
+ # Deleting it is a WRITE to the managed tree, so it takes the same admission an install does. An image one-shot may
+ # be executing this very file: on Linux the running child survives the unlink but the next image in the batch can no
+ # longer resolve it, and on Windows the unlink fails outright and the H3 load is refused. Held only across the
+ # unlink -- ensure_sd_cpp_binary below claims the tree itself, and the claim is not reentrant.
with _tree_claimed_for_install() as claimed:
if not claimed:
logger.warning(
@@ -561,9 +556,9 @@ def _installer_module():
return install_sd_cpp_prebuilt
-# Accelerators whose upgrade install already failed this process. Without this, a host that asks
-# for a GPU build it has no asset for would re-resolve (and re-download) on every single load,
-# because the wrong-accelerator binary it keeps still does not match the request.
+# Accelerators whose upgrade install already failed this process. Without this, a host that asks for a GPU build it has
+# no asset for would re-resolve (and re-download) on every single load, because the wrong-accelerator binary it keeps
+# still does not match the request.
_failed_accelerator_upgrades: set[str] = set()
@@ -571,7 +566,7 @@ def _note_failed_upgrade(accelerator: str) -> None:
"""Stop retrying an accelerator upgrade that just failed while a usable binary is kept."""
try:
_failed_accelerator_upgrades.add(_installer_module().accelerator_class(accelerator))
- except Exception: # noqa: BLE001 -- best effort
+ except Exception: # noqa: BLE001
pass
@@ -584,7 +579,7 @@ def _incomplete_tree_replacement(exc: BaseException) -> bool:
were the accelerator that was asked for."""
try:
return isinstance(exc, _installer_module().SupersededBinaryError)
- except Exception: # noqa: BLE001 -- cannot tell -> treat as an ordinary failure, as before
+ except Exception: # noqa: BLE001 -- cannot tell -> an ordinary failure
return False
@@ -652,11 +647,8 @@ def _accelerator_changed(binary: str, accelerator: str) -> bool:
want = mod.accelerator_class(accelerator)
if want in _failed_accelerator_upgrades:
return False
- # From the root the binary is actually in, not the current default: an install an older
- # build put beside the Unsloth home keeps its own record, and reading the wrong root would
- # report it unrecorded and re-download a bundle that is already here.
return _record_mismatch(mod, root, want)
- except Exception: # noqa: BLE001 -- cannot tell -> keep the existing binary, as before
+ except Exception: # noqa: BLE001 -- cannot tell -> keep the existing binary
return False
@@ -699,7 +691,7 @@ def _superseded_legacy_server(binary: Optional[str], accelerator: str) -> bool:
if not _record_mismatch(mod, root, want) or _record_mismatch(mod, current, want):
return False
return mod.installed_ships_server(current) is False
- except Exception: # noqa: BLE001 -- cannot tell -> leave the existing behavior alone
+ except Exception: # noqa: BLE001 -- cannot tell
return False
@@ -715,10 +707,9 @@ def _installed_accelerator_of(binary: Optional[str]) -> Optional[str]:
failed, and a load that keeps a usable wrong-accelerator build on purpose would be refused by
it on every single load. What the load needs is narrower -- did the tree it resolved this
binary out of get replaced underneath it."""
- # From the root the binary is actually IN, not the current default. The finder also serves a
- # tree an older build left beside the Unsloth home, and reading the current root for a binary
- # out of that one reports "unrecorded" on both sides of the comparison, so a swap underneath
- # this load reads as no change at all.
+ # From the root the binary is actually IN, not the current default. The finder also serves a tree an older build
+ # left beside the Unsloth home, and reading the current root for a binary out of that one reports "unrecorded" on
+ # both sides of the comparison, so a swap underneath this load reads as no change at all.
root = owning_managed_root(binary)
if root is None:
return None
@@ -742,21 +733,20 @@ def ensure_sd_cpp_binary(*, allow_install: bool = True, accelerator: str = "cpu"
if not allow_install:
return found
with _install_lock:
- # Re-check inside the lock: a concurrent first-load may have installed it.
found = find_sd_cpp_binary()
usable = bool(found) and _usable_or_discard_managed(found)
if usable and not _accelerator_changed(found, accelerator):
return found
- # A usable binary of the wrong accelerator is still better than none, so an install that
- # cannot deliver the right one (no such asset for this host, no network) keeps it.
+ # A usable binary of the wrong accelerator is still better than none, so an install that cannot deliver the
+ # right one (no such asset for this host, no network) keeps it.
fallback = found if usable else None
try:
_install = _installer_module().install
except Exception as exc: # noqa: BLE001 -- import path / module issues are non-fatal
logger.warning("sd-cli installer import failed: %s", exc)
return fallback
- # Claim the tree for the whole install, download included: a point-in-time check would
- # let a generation start during the download and be overwritten by the extraction.
+ # Claim the tree for the whole install, download included: a point-in-time check would let a generation start
+ # during the download and be overwritten by the extraction.
with _tree_claimed_for_install() as claimed:
if not claimed:
return fallback # something is running in there; retry on a later load
@@ -767,16 +757,13 @@ def ensure_sd_cpp_binary(*, allow_install: bool = True, accelerator: str = "cpu"
except Exception as exc: # noqa: BLE001 -- download/extract failure -> fall back
logger.warning("sd-cli auto-install failed: %s", exc)
if _incomplete_tree_replacement(exc):
- # The sweep got part way, and it takes sd-cli before sd-server, so the
- # fallback resolved before the install may be one of the copies it already
- # removed. Re-find, so this returns a file that exists -- often the one the
- # new bundle just extracted -- or None, never a path that is gone.
- #
- # Through the usability gate, not raw: the raise came BEFORE install()'s
- # _make_executable, so on POSIX a freshly extracted copy has no execute bit
- # and find_sd_cpp_binary only checks that the path is a file. The gate probes
- # it and, being ours, removes it if it cannot run, so the next load reinstalls
- # rather than being handed a binary that fails to launch.
+ # The sweep got part way, and it takes sd-cli before sd-server, so the fallback resolved before the
+ # install may be one of the copies it already removed. Re-find, so this returns a file that exists
+ # -- often the one the new bundle just extracted -- or None, never a path that is gone. Through the
+ # usability gate, not raw: the raise came BEFORE install()'s _make_executable, so on POSIX a freshly
+ # extracted copy has no execute bit and find_sd_cpp_binary only checks that the path is a file. The
+ # gate probes it and, being ours, removes it if it cannot run, so the next load reinstalls rather
+ # than being handed a binary that fails to launch.
refound = find_sd_cpp_binary()
return refound if refound and _usable_or_discard_managed(refound) else None
if fallback is not None:
@@ -797,10 +784,10 @@ def ensure_sd_server_binary(
"""
found = find_sd_server_binary()
usable = bool(found) and _usable_or_discard_managed(found)
- # Ahead of _accelerator_changed, which reports "unchanged" while the managed tree is in use
- # (an install would overwrite a running binary) and would hand the mismatched legacy server to
- # a load that has the matching serverless build right here. None IS the answer: the one-shot
- # sd-cli of the right build runs, and the bundle is not downloaded again on every later load.
+ # Ahead of _accelerator_changed, which reports "unchanged" while the managed tree is in use (an install would
+ # overwrite a running binary) and would hand the mismatched legacy server to a load that has the matching serverless
+ # build right here. None IS the answer: the one-shot sd-cli of the right build runs, and the bundle is not
+ # downloaded again on every later load.
if usable and _superseded_legacy_server(found, accelerator):
return None
if usable and not _accelerator_changed(found, accelerator):
@@ -814,7 +801,7 @@ def ensure_sd_server_binary(
return None
if usable and not _accelerator_changed(found, accelerator):
return found
- # Keep a usable wrong-accelerator server if the matching one cannot be fetched.
+ # Keep a usable wrong-accelerator server if the matching one cannot be fetched
fallback = found if usable else None
try:
_install = _installer_module().install
@@ -828,24 +815,23 @@ def ensure_sd_server_binary(
_install(accelerator = accelerator) # extracts sd-cli AND sd-server
except Exception as exc: # noqa: BLE001 -- download/extract failure -> fall back
logger.warning("sd-server auto-install failed: %s", exc)
- # Also when only the CLI survives (a legacy server-less tree): the router probes
- # ensure_sd_cpp_binary immediately after this, and without the record that probe
- # resolves and downloads the same bundle a second time inside one selection.
+ # Also when only the CLI survives (a legacy server-less tree): the router probes ensure_sd_cpp_binary
+ # immediately after this, and without the record that probe resolves and downloads the same bundle a
+ # second time inside one selection.
if _incomplete_tree_replacement(exc):
- # As above: the fallback may name a copy the partial sweep removed, and a
- # freshly extracted one never reached _make_executable.
+ # As above: the fallback may name a copy the partial sweep removed, and a freshly extracted one
+ # never reached _make_executable.
refound = find_sd_server_binary()
return refound if refound and _usable_or_discard_managed(refound) else None
if fallback is not None or find_sd_cpp_binary() is not None:
_note_failed_upgrade(accelerator)
return fallback
installed = find_sd_server_binary()
- # The finder also probes the tree an older build left beside the Unsloth home, so when the
- # bundle just installed ships no sd-server the hit here can be that legacy server, built
- # for a different accelerator. None, not the fallback: an install just completed, so the
- # router's next step resolves the sd-cli it landed, and a one-shot run on the right build
- # beats a resident server on the wrong one. The fallback stays for the failure path above,
- # where no matching binary was fetched at all.
+ # The finder also probes the tree an older build left beside the Unsloth home, so when the bundle just installed
+ # ships no sd-server the hit here can be that legacy server, built for a different accelerator. None, not the
+ # fallback: an install just completed, so the router's next step resolves the sd-cli it landed, and a one-shot
+ # run on the right build beats a resident server on the wrong one. The fallback stays for the failure path
+ # above, where no matching binary was fetched at all.
if installed and _accelerator_changed(installed, accelerator):
return None
return installed or fallback
@@ -874,20 +860,19 @@ class _SdState:
mode: str = "server"
# Token kept so LoRA adapters selected at generate time can be fetched from the Hub.
hf_token: Optional[str] = None
- # The GGUF basename this load committed: some variants pick their encoder by filename, and a local *klein-9B*.gguf carries that keyword only in the basename.
+ # The GGUF basename this load committed: some variants pick their encoder by filename, and a local *klein-9B*.gguf
+ # carries that keyword only in the basename.
gguf_filename: Optional[str] = None
- # The FLUX.2 inner_dim this load read out of the checkpoint's own header, when it could. Kept
- # so the delete guard reconstructs the SAME encoder pick without re-probing under the lock.
+ # The FLUX.2 inner_dim this load read out of the checkpoint's own header, when it could. Kept so the delete guard
+ # reconstructs the SAME encoder pick without re-probing under the lock.
flux2_inner_dim: Optional[int] = None
- # The managed tree's recorded accelerator when this load chose its binary. The one-shot path
- # re-resolves sd-cli per image, so without this it would silently adopt an install that landed
- # between images even when that install is for a DIFFERENT accelerator, while ``device`` and
- # ``offload_flags`` still describe the build the load committed to. None on the server path,
- # which asks the same question at start time against its own local copy.
+ # The managed tree's recorded accelerator when this load chose its binary. The one-shot path re-resolves sd-cli per
+ # image, so without this it would silently adopt an install that landed between images even when that install is for
+ # a DIFFERENT accelerator, while ``device`` and ``offload_flags`` still describe the build the load committed to.
+ # None on the server path, which asks the same question at start time against its own local copy.
sd_accelerator: Optional[str] = None
- # Physical card behind the server's single resolved CUDA/ROCm backend.
- # None for CPU/Metal/Vulkan, an unresolved pin, automatic multi-GPU, and
- # one-shot mode; those shapes cannot safely consume the startup VRAM floor.
+ # Physical card behind the server's single resolved CUDA/ROCm backend. None for CPU/Metal/Vulkan, an unresolved pin,
+ # automatic multi-GPU, and one-shot mode; those shapes cannot safely consume the startup VRAM floor.
physical_gpu_id: Optional[int] = None
@@ -910,7 +895,7 @@ def _resolved_server_physical_gpu_id(
try:
from utils.hardware import get_parent_visible_gpu_ids
visible = [int(i) for i in get_parent_visible_gpu_ids()]
- except Exception:
+ except Exception: # noqa: BLE001 -- don't block on a flaky probe (timeout etc.)
return None
local_ordinal = ordinal
if local_ordinal is None:
@@ -923,10 +908,9 @@ def _resolved_server_physical_gpu_id(
if device_name is None:
return None
if ordinal is not None:
- # An explicit request is attributable only when the committed argv
- # actually contains the resolved per-module pin. If the probe failed,
- # sd.cpp falls back to its own default and the requested ordinal is not
- # evidence of placement.
+ # An explicit request is attributable only when the committed argv actually contains the resolved per-module
+ # pin. If the probe failed, sd.cpp falls back to its own default and the requested ordinal is not evidence of
+ # placement.
specs = [
committed_flags[index + 1]
for index, flag in enumerate(committed_flags[:-1])
@@ -961,7 +945,7 @@ class _SdLoading:
repo_id: str
base_repo: str
- # Companion asset repos (VAE / text encoders) so the delete-cached guard protects them.
+ # Companion asset repos (VAE / text encoders) so the delete-cached guard protects them
asset_repos: tuple[str, ...] = ()
expected_bytes: int = 0
downloaded_bytes: int = 0
@@ -1076,7 +1060,8 @@ class SdCppDiffusionBackend:
self._lock = threading.Lock()
self._generate_lock = threading.Lock()
self._engine = engine # resolved lazily on first load so import stays cheap
- # An injected engine (test seam) pins one-shot mode; a fallback-cached engine must NOT, so a now-available server can still be used next load.
+ # An injected engine (test seam) pins one-shot mode; a fallback-cached engine must NOT, so a now-available
+ # server can still be used next load.
self._engine_injected = engine is not None
self._state: Optional[_SdState] = None
self._loading: Optional[_SdLoading] = None
@@ -1084,17 +1069,19 @@ class SdCppDiffusionBackend:
# Replaced (never cleared) per load, so a cancelled asset pull stays cancelled.
self._cancel_event = threading.Event()
self._active_generate_cancel: Optional[threading.Event] = None
- # sd-server started for an in-flight load, before it commits to _state; tracked so an unload can stop it mid-startup.
+ # sd-server started for an in-flight load, before it commits to _state; tracked so an unload can stop it
+ # mid-startup.
self._pending_server: Optional[SdCppServer] = None
self._gen: Optional[_SdGen] = None
- # Set by _resolve_backend when it had to skip an accelerator install because the managed
- # tree was still in use; the load retries it once the tree is free.
+ # Set by _resolve_backend when it had to skip an accelerator install because the managed tree was still in use;
+ # the load retries it once the tree is free.
self._deferred_accelerator_install = False
- # Servers taken out of _state/_pending_server whose stop() has not returned yet. unload()
- # deliberately stops outside the lock (terminate can take seconds), so between the clear
- # and the stop the fields say idle while the process is still running its own executable.
+ # Servers taken out of _state/_pending_server whose stop() has not returned yet. unload() deliberately stops
+ # outside the lock (terminate can take seconds), so between the clear and the stop the fields say idle while the
+ # process is still running its own executable.
self._stopping_servers = 0
- # Set once this load's graph proved unrunnable on the GPU backend (a ggml unsupported-op abort), so the CPU restart happens once per load. Cleared by each load.
+ # Set once this load's graph proved unrunnable on the GPU backend (a ggml unsupported-op abort), so the CPU
+ # restart happens once per load. Cleared by each load.
self._cpu_backend_forced = False
@property
@@ -1137,10 +1124,9 @@ class SdCppDiffusionBackend:
"""The SdCppEngine, installing the binary on first use. Raises if unusable."""
if self._engine is not None and self._engine.is_available():
return self._engine
- # The accelerator this host resolves to, never the "cpu" default: this is also the
- # one-shot FALLBACK path (a GPU sd-server that would not start lands here), and asking
- # for the CPU build there would reinstall the plain bundle over the working GPU one and
- # run the whole generation on the CPU.
+ # The accelerator this host resolves to, never the "cpu" default: this is also the one-shot FALLBACK path (a GPU
+ # sd-server that would not start lands here), and asking for the CPU build there would reinstall the plain
+ # bundle over the working GPU one and run the whole generation on the CPU.
binary = ensure_sd_cpp_binary(
allow_install = _install_allowed() and not _tree_in_use(self),
accelerator = self._resolved_accelerator(),
@@ -1164,15 +1150,13 @@ class SdCppDiffusionBackend:
if self._engine_injected and self._engine is not None:
return "oneshot", None, self._resolve_engine()
accelerator = self._resolved_accelerator()
- # An accelerator upgrade REPLACES the binaries in the managed tree, and this runs before
- # the load stops the resident server (which is executing its own file) or waits out an
- # in-flight one-shot sd-cli. Linux refuses to open a running executable for writing
- # (ETXTBSY) and Windows locks it, so an install here fails and can leave the tree
- # half-written. _accelerator_changed refuses the upgrade while the tree is in use, whoever
- # asks; record that it did, so this load retries it after its own teardown, when the tree
- # is the only thing that has changed.
- # _managed_tree_in_use covers the singleton for callers that never see this instance (the
- # engine router); this load's own state is the authority for this load.
+ # An accelerator upgrade REPLACES the binaries in the managed tree, and this runs before the load stops the
+ # resident server (which is executing its own file) or waits out an in-flight one-shot sd-cli. Linux refuses to
+ # open a running executable for writing (ETXTBSY) and Windows locks it, so an install here fails and can leave
+ # the tree half-written. _accelerator_changed refuses the upgrade while the tree is in use, whoever asks; record
+ # that it did, so this load retries it after its own teardown, when the tree is the only thing that has changed.
+ # _managed_tree_in_use covers the singleton for callers that never see this instance (the engine router); this
+ # load's own state is the authority for this load.
upgrade_pending = _tree_in_use(self) or _managed_tree_in_use()
self._deferred_accelerator_install = upgrade_pending
server_binary = ensure_sd_server_binary(
@@ -1199,9 +1183,8 @@ class SdCppDiffusionBackend:
return server_binary
try:
accelerator = self._resolved_accelerator()
- # Judge the tree by whatever binary it holds: on a serverless install that is the
- # sd-cli, and without this the retry would reinstall on every deferred load, matching
- # accelerator or not.
+ # Judge the tree by whatever binary it holds: on a serverless install that is the sd-cli, and without this
+ # the retry would reinstall on every deferred load, matching accelerator or not.
probe = server_binary or find_sd_cpp_binary()
if probe is None or not _accelerator_changed(probe, accelerator):
return server_binary
@@ -1220,16 +1203,13 @@ class SdCppDiffusionBackend:
self,
repo_id: str,
*,
- # Same name, position and default as DiffusionBackend.begin_load: the route calls whichever
- # engine was activated through ONE call site and passes this unconditionally, so an engine
- # that does not declare it TypeErrors every load on the hosts that select it (CPU-only,
- # opted-in MPS, UNSLOTH_DIFFUSION_ENGINE=sd_cpp) -- including the ordinary user-initiated
- # ones, which pass False.
- #
- # Covers the MODEL ASSETS only: the GGUF, the VAE and the text encoders this pick fetches
- # from the Hub. It deliberately says nothing about the sd-cli/sd-server BINARY, which is a
- # separate managed tree with its own install policy (_install_allowed / ensure_sd_*_binary);
- # a background load may still install one, exactly as it does today.
+ # Same name, position and default as DiffusionBackend.begin_load: the route calls whichever engine was activated
+ # through ONE call site and passes this unconditionally, so an engine that does not declare it TypeErrors every
+ # load on the hosts that select it (CPU-only, opted-in MPS, UNSLOTH_DIFFUSION_ENGINE=sd_cpp) -- including the
+ # ordinary user-initiated ones, which pass False. Covers the MODEL ASSETS only: the GGUF, the VAE and the text
+ # encoders this pick fetches from the Hub. It deliberately says nothing about the sd-cli/sd-server BINARY, which
+ # is a separate managed tree with its own install policy (_install_allowed / ensure_sd_*_binary); a background
+ # load may still install one, exactly as it does today.
local_files_only: bool = False,
gguf_filename: Optional[str] = None,
base_repo: Optional[str] = None,
@@ -1238,7 +1218,8 @@ class SdCppDiffusionBackend:
cpu_offload: bool = False,
memory_mode: Optional[str] = None,
speed_mode: Optional[str] = None,
- # diffusers-only knobs accepted for a uniform call and ignored (sd.cpp has no torchao quant / SDPA dispatcher / fbcache).
+ # diffusers-only knobs accepted for a uniform call and ignored (sd.cpp has no torchao quant / SDPA dispatcher /
+ # fbcache)
text_encoder_quant: Optional[str] = None,
transformer_quant: Optional[str] = None,
transformer_quant_fast_accum: Optional[bool] = None,
@@ -1246,22 +1227,22 @@ class SdCppDiffusionBackend:
attention_backend: Optional[str] = None,
transformer_cache: Optional[str] = None,
transformer_cache_threshold: Optional[float] = None,
- # Accepted for interface parity; native is GGUF-only (router forces diffusers otherwise).
+ # Accepted for interface parity; native is text-to-image only, so image-conditioned requests are rejected below.
model_kind: Optional[str] = None,
- # Parity with the diffusers load-time LoRA bake; native applies LoRA per generation, so a load-time selection is ignored.
+ # Parity with the diffusers load-time LoRA bake; native applies LoRA per generation, so a load-time selection is
+ # ignored.
loras: Optional[list[tuple[str, float]]] = None,
gpu_ids: Optional[list[int]] = None,
- # The ordinal the ROUTE already ranked, so the preflight and the load agree on one card.
+ # The ordinal the ROUTE already ranked, so the preflight and the load agree on one card
gpu_ordinal: Optional[int] = None,
) -> dict[str, Any]:
"""Validate, then fetch assets on a daemon thread. Returns at once."""
# Empty/whitespace token = "no token"; "" verbatim breaks the anonymous fallback.
hf_token = hf_token.strip() if hf_token and hf_token.strip() else None
- # Same fallback the diffusers and video backends take: the route ranks the selection and
- # passes the winner, but a direct caller (an MCP client, a test, a plugin) hands over
- # gpu_ids alone, and without this the native engine is the one engine that would drop the
- # pick silently. Re-ranked only when nobody has, so a route-resolved winner is never
- # second-guessed against free VRAM that has moved since.
+ # Same fallback the diffusers and video backends take: the route ranks the selection and passes the winner, but
+ # a direct caller (an MCP client, a test, a plugin) hands over gpu_ids alone, and without this the native engine
+ # is the one engine that would drop the pick silently. Re-ranked only when nobody has, so a route-resolved
+ # winner is never second-guessed against free VRAM that has moved since.
if gpu_ordinal is None:
gpu_ordinal = (
resolve_selected_cuda_ordinal(gpu_ids)
@@ -1272,7 +1253,8 @@ class SdCppDiffusionBackend:
raise ValueError(
"gguf_filename is required: the native engine loads single-file GGUF checkpoints only."
)
- # Filename-fallback detector (as the route validated) so a local .gguf whose family keyword lives only in the basename still loads.
+ # Filename-fallback detector (as the route validated) so a local .gguf whose family keyword lives only in the
+ # basename still loads
fam = detect_family_for_pick(repo_id, gguf_filename, family_override)
if fam is None:
raise ValueError(
@@ -1284,22 +1266,15 @@ class SdCppDiffusionBackend:
raise ValueError(f"Family '{fam.name}' has no native sd.cpp asset mapping.")
base = resolve_base_repo(fam, base_repo)
- # Offline-only here, and deliberately so. begin_load returns at once by contract -- the
- # route thread answers the UI with a status and the pull happens on the worker -- so it
- # cannot afford the range request's bound, let alone hold _lock across it and stall
- # status()/unload() for the same span. Memo or on-disk header or nothing. This value only
- # seeds the delete-cached guard's repo list below; the worker re-asks WITH the network and
- # refreshes that list, so the guard converges within one round trip of the load starting.
+ # Offline-only here, deliberately: begin_load returns at once by contract
inner_dim = self._flux2_inner_dim(
repo_id, gguf_filename, fam, hf_token, allow_network = False
)
- # Same link the diffusers resolver records, so the delete guard protects a native pick's
- # companions too -- and here that means the repos _asset_specs actually FETCHES. The
- # native engine does not read the diffusers base: FLUX.2 takes its VAE from
- # unsloth/FLUX.2-VAE and its encoders from another repo again, so recording only the base
- # left every repo the pick really depends on outside the guard, and an unloaded model's
- # encoder could be deleted while its GGUF stayed installed. Best-effort bookkeeping;
- # never fails a load.
+ # Same link the diffusers resolver records, so the delete guard protects a native pick's companions too -- and
+ # here that means the repos _asset_specs actually FETCHES. The native engine does not read the diffusers base:
+ # FLUX.2 takes its VAE from unsloth/FLUX.2-VAE and its encoders from another repo again, so recording only the
+ # base left every repo the pick really depends on outside the guard, and an unloaded model's encoder could be
+ # deleted while its GGUF stayed installed. Best-effort bookkeeping; never fails a load.
try:
from hub.utils.companion_assets import record_companion_link
for asset_repo in dict.fromkeys(
@@ -1314,13 +1289,14 @@ class SdCppDiffusionBackend:
with self._lock:
if self._loading is not None and self._loading.error is None:
raise RuntimeError("A diffusion load is already in progress.")
- # A superseding load must stop any in-flight generation, else the old run can still persist an image after the new load starts.
+ # A superseding load must stop any in-flight generation, else the old run can still persist an image after
+ # the new load starts
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
self._load_token += 1
token = self._load_token
- # A NEW event per load, never a clear() of the shared one: unload() sets the event the running worker holds but also
- # drops _loading, so a clear() here would un-cancel its still-running multi-gigabyte pull.
+ # A NEW event per load, never a clear() of the shared one: unload() sets the event the running worker holds
+ # but also drops _loading, so a clear() here would un-cancel its still-running multi-gigabyte pull.
cancel_event = threading.Event()
self._cancel_event = cancel_event
self._loading = _SdLoading(
@@ -1363,8 +1339,8 @@ class SdCppDiffusionBackend:
base: str,
fam: DiffusionFamily,
hf_token: Optional[str],
- # Cache-only when set: every Hub call below is either skipped or told to resolve from disk,
- # so a load nobody asked for cannot pull bytes. See begin_load for what it does not cover.
+ # Cache-only when set: every Hub call below is either skipped or told to resolve from disk, so a load nobody
+ # asked for cannot pull bytes. See begin_load for what it does not cover.
local_files_only: bool = False,
cpu_offload: bool = False,
memory_mode: Optional[str] = None,
@@ -1373,26 +1349,28 @@ class SdCppDiffusionBackend:
_load_token: int,
_cancel_event: Optional[threading.Event] = None,
) -> None:
- # This load's own event: a later load replaces self._cancel_event rather than clearing it.
+ # This load's own event: a later load replaces self._cancel_event rather than clearing it
cancel_event = _cancel_event if _cancel_event is not None else self._cancel_event
- # The server this load publishes to _pending_server, out here so the backstop below can
- # always unpublish it. A leaked _pending_server reads as "the managed tree is busy" for
- # the rest of the process and blocks every later install.
+ # The server this load publishes to _pending_server, out here so the backstop below can always unpublish it. A
+ # leaked _pending_server reads as "the managed tree is busy" for the rest of the process and blocks every later
+ # install.
started: Optional[SdCppServer] = None
try:
- # Resolve mode (server preferred, one-shot fallback) + binary up front so an install failure surfaces before the multi-GB pull.
+ # Resolve mode + binary up front so an install failure surfaces before the multi-GB pull
+ # Resolve mode (server preferred, one-shot fallback) + binary up front so an install failure surfaces before
+ # the multi-GB pull.
mode, server_binary, engine = self._resolve_backend()
if mode == "server":
- # Probe the server binary before the pull: a present-but-unrunnable build would download everything then fail.
+ # Probe the server binary before the pull: a present-but-unrunnable build would download everything then
+ # fail
assert server_binary is not None
if not _server_binary_runnable(server_binary):
logger.warning(
"sd-server at %s is present but not runnable; trying one-shot sd-cli.",
server_binary,
)
- # Resolve ONCE and keep it: two calls can answer with two different
- # binaries if an install lands between them, and the state below reads the
- # accelerator off whichever object it ends up holding.
+ # Resolve ONCE and keep it: two calls can answer with two different binaries if an install lands
+ # between them, and the state below reads the accelerator off whichever object it ends up holding.
fallback: Optional[SdCppEngine] = None
try:
fallback = self._resolve_engine()
@@ -1402,34 +1380,30 @@ class SdCppDiffusionBackend:
if not usable or fallback is None:
raise RuntimeError("sd-server binary is present but not runnable.")
mode, server_binary, engine = "oneshot", None, fallback
- # The accelerator the managed tree held when THIS binary was chosen, taken where the
- # choice is made rather than sampled again later. The asset download below runs for
- # minutes with no claim on the tree, and an install that lands in that window replaces
- # sd-server in place: same path, still runnable, a different build. "It exists and it
- # runs" is therefore not evidence that it is the build this load resolved its device
- # and offload policy for, so the answer is re-asked under the reader claim.
+ # The accelerator the managed tree held when THIS binary was chosen, taken where the choice is made rather
+ # than sampled again later. The asset download below runs for minutes with no claim on the tree, and an
+ # install that lands in that window replaces sd-server in place: same path, still runnable, a different
+ # build. "It exists and it runs" is therefore not evidence that it is the build this load resolved its
+ # device and offload policy for, so the answer is re-asked under the reader claim.
server_accelerator = _installed_accelerator_of(server_binary)
- # The same pin for the one-shot CLI, and for the same reason. Sampling it only at
- # state construction, after the download, would record whatever an install left in
- # the tree in that window and the first generation -- which re-reads the tree and
- # compares -- would then agree with the replacement, so the check that exists to
+ # The same pin for the one-shot CLI, and for the same reason. Sampling it only at state construction, after
+ # the download, would record whatever an install left in the tree in that window and the first generation --
+ # which re-reads the tree and compares -- would then agree with the replacement, so the check that exists to
# notice a swap could never fire for one that landed during the download.
engine_accelerator = _installed_accelerator_of(getattr(engine, "binary", None))
if mode == "oneshot":
- # version() is None when a present binary can't run; fail now, not on the first generation.
+ # version() is None when a present binary can't run; fail now, not on the first generation
assert engine is not None
if engine.version() is None:
raise RuntimeError("sd-cli binary is present but not runnable.")
- # Swap ONCE so the size probe and the download agree: sizes come from paths-info, which
- # -- unlike model_info -- 401s anonymously on a gated repo, so probing the upstream
- # drops the VAE from the progress total the mirror then pulls.
- # The probe is a RANGE READ off the Hub when the checkpoint is not on disk, so an
- # offline load asks it the way begin_load does: memo or local header or nothing. A
- # None here only falls back to the filename heuristic for the encoder pick, and a
- # cache-only load can fetch nothing the heuristic did not already have.
- # The speech verdict lands here rather than in begin_load, which is offline-only by
- # contract; before _asset_specs, so the refusal precedes any fetch.
+ # Swap ONCE so the size probe and the download agree: sizes come from paths-info, which -- unlike model_info
+ # -- 401s anonymously on a gated repo, so probing the upstream drops the VAE from the progress total the
+ # mirror then pulls. The probe is a RANGE READ off the Hub when the checkpoint is not on disk, so an offline
+ # load asks it the way begin_load does: memo or local header or nothing. A None here only falls back to the
+ # filename heuristic for the encoder pick, and a cache-only load can fetch nothing the heuristic did not
+ # already have. The speech verdict lands here rather than in begin_load, which is offline-only by contract;
+ # before _asset_specs, so the refusal precedes any fetch.
_assert_pick_is_not_speech(
repo_id, gguf_filename, hf_token, allow_network = not local_files_only
)
@@ -1439,20 +1413,16 @@ class SdCppDiffusionBackend:
specs = self._asset_specs(repo_id, gguf_filename, fam, inner_dim)
fetch_repo = _fetch_repo_map(specs, hf_token)
assets = [(fetch_repo[repo], fn, kind) for repo, fn, kind in specs]
- # begin_load could only guess the encoder repos (it may not have had the header yet);
- # now they are known, so publish them before a single byte is fetched. Otherwise
- # delete-cached would happily remove a companion this load is about to write into.
with self._lock:
if self._load_token == _load_token and self._loading is not None:
self._loading.asset_repos = tuple(
dict.fromkeys(r for r, _f, kind in specs if kind != "diffusion_model")
)
- # And record them, from the SAME post-probe specs. begin_load records what it can, but
- # it resolves the header offline, so a remote or renamed FLUX.2-klein 9B checkpoint
- # with no cached probe records the default 4B encoder while this load fetches the 9B
- # one. Whatever that leaves unrecorded is a companion the delete guard would let go
- # while its GGUF is still installed. Recorded on the FETCH ids too, since a gated
- # mirror or a cached community repack is where the bytes actually land.
+ # And record them, from the SAME post-probe specs. begin_load records what it can, but it resolves the
+ # header offline, so a remote or renamed FLUX.2-klein 9B checkpoint with no cached probe records the default
+ # 4B encoder while this load fetches the 9B one. Whatever that leaves unrecorded is a companion the delete
+ # guard would let go while its GGUF is still installed. Recorded on the FETCH ids too, since a gated mirror
+ # or a cached community repack is where the bytes land.
try:
from hub.utils.companion_assets import record_companion_link
for asset_repo in dict.fromkeys(
@@ -1464,20 +1434,19 @@ class SdCppDiffusionBackend:
record_companion_link(repo_id, asset_repo)
except Exception as exc: # noqa: BLE001 -- bookkeeping never fails a load
logger.debug("sd_cpp.companion_link_record_failed: %s", exc)
- # Same preflight the plan runs, on POST-swap repos: catch a gated companion here, not
- # 15 GiB into the prefetch, without refusing one an ungated mirror stands in for. The
- # plan alone is not enough: the images page falls back to this load when it fails.
+ # Same preflight the plan runs, on POST-swap repos: catch a gated companion here, not 15 GiB into the
+ # prefetch, without refusing one an ungated mirror stands in for. The plan alone is not enough: the images
+ # page falls back to this load when it fails.
self._preflight_companion_repos(
self._assets_by_repo(assets),
fetch_repo.get(repo_id, repo_id),
hf_token,
local_files_only = local_files_only,
)
- # Skipped outright offline: the size probe is get_paths_info, a Hub round trip, and its
- # only product is the progress bar's denominator. A cache-only load resolves every
- # asset from disk in milliseconds, so 0 (the value this method already reports for any
- # size the Hub will not answer) costs nothing and asking would be the one network call
- # left on the path.
+ # Skipped outright offline: the size probe is get_paths_info, a Hub round trip, and its only product is the
+ # progress bar's denominator. A cache-only load resolves every asset from disk in milliseconds, so 0 (the
+ # value this method already reports for any size the Hub will not answer) costs nothing and asking would be
+ # the one network call left on the path.
if not local_files_only:
self._set_expected_bytes(assets, hf_token)
paths = self._fetch_assets(
@@ -1497,19 +1466,20 @@ class SdCppDiffusionBackend:
qwen2vl = paths.get("qwen2vl"),
)
device = resolve_diffusion_device_target().device
- # Honor speed everywhere; offload only off-CPU (on CPU weights are resident, so the flags are no-ops).
+ # Honor speed everywhere; offload only off-CPU (on CPU weights are resident, so the flags are no-ops)
offload: tuple[str, ...] = ()
if device != "cpu":
offload = tuple(offload_flags(_memory_policy(memory_mode, cpu_offload)))
- # The device pin is NOT folded in here: the binary can still change below, through the
- # deferred accelerator install, the post-download re-resolve, or a server start that
- # falls back to one-shot, and the ggml device names come from whichever build ends up
- # running. It is added at each point the flags are handed to a binary instead.
+ # The device pin is NOT folded in here: the binary can still change below, through the deferred accelerator
+ # install, the post-download re-resolve, or a server start that falls back to one-shot, and the ggml device
+ # names come from whichever build ends up running. It is added at each point the flags are handed to a
+ # binary instead.
gpu_ordinal = gpu_ordinal if device == "cuda" else None
native_speed = _native_speed_for(speed_mode)
- # Tear down the old model then commit the new one under _generate_lock: abort and WAIT for a generation started during
- # the download, so no stale run persists an image. Taken only now, so the download never serialises generation.
+ # Tear down the old model then commit the new one under _generate_lock: abort and WAIT for a generation
+ # started during the download, so no stale run persists an image. Taken only now, so the download never
+ # serialises generation.
with self._lock:
if self._load_token != _load_token:
return # superseded / cancelled
@@ -1521,44 +1491,39 @@ class SdCppDiffusionBackend:
return # superseded / cancelled while waiting
old_state = self._state
self._state = None # the old model is being torn down
- # Same lock block as the clear, so the tree is never briefly readable as idle.
if old_state is not None and old_state.server is not None:
self._reserve_stop()
if old_state is not None and old_state.server is not None:
self._stop_reserved(old_state.server)
- # The tree is free now, so an install deferred in _resolve_backend can land: this
- # runs under both locks, the old server is stopped, the previous generation has
- # finished and no new one can start. Not gated on the resolved mode: a serverless
- # install resolves to one-shot precisely BECAUSE the deferral suppressed the
- # install, and its sd-cli comes out of the same archive.
+ # The tree is free now, so an install deferred in _resolve_backend can land: this runs under both locks,
+ # the old server is stopped, the previous generation has finished and no new one can start. Not gated on
+ # the resolved mode: a serverless install resolves to one-shot precisely BECAUSE the deferral suppressed
+ # the install, and its sd-cli comes out of the same archive.
if self._deferred_accelerator_install:
self._deferred_accelerator_install = False
upgraded = self._upgrade_server_after_teardown(server_binary)
if mode == "server":
server_binary = upgraded
- # This load's own install just rewrote the tree, under the install claim, so
- # what it left behind IS the decision here -- comparing against the answer
- # from before it would fail the load on the upgrade it asked for. Both pins
- # move: a serverless upgrade lands the sd-cli out of the same archive.
+ # This load's own install just rewrote the tree, under the install claim, so what it left behind IS
+ # the decision here -- comparing against the answer from before it would fail the load on the
+ # upgrade it asked for. Both pins move: a serverless upgrade lands the sd-cli out of the same
+ # archive.
server_accelerator = _installed_accelerator_of(server_binary)
engine_accelerator = _installed_accelerator_of(getattr(engine, "binary", None))
- # A new checkpoint earns a fresh attempt on the GPU backend: the previous abort says nothing about this graph.
+ # A new checkpoint earns a fresh attempt on the GPU backend: the previous abort says nothing about this
+ # graph.
self._cpu_backend_forced = False
server: Optional[SdCppServer] = None
if mode == "server":
assert server_binary is not None
- # _fetch_assets above runs for minutes with no claim on the tree (there is
- # nothing executing in it yet to claim for), so an install can have swept this
- # path between layouts since _resolve_backend picked it. Re-resolve before
- # starting: the stale path would drop the load to one-shot for nothing, or
- # start a build this load did not select.
- #
- # Under the READER, and held until _pending_server is published. Re-resolving
- # alone does not close the race: allow_install=False only declines to install,
- # it does not wait for or claim the tree, so an installer that has already
- # passed its in-use check can sweep this executable between the re-read and
- # the start. Once _pending_server is published, _tree_in_use covers it and the
- # claim is no longer what is holding the installer off.
+ # _fetch_assets above runs for minutes with no claim on the tree (there is nothing executing in it
+ # yet to claim for), so an install can have swept this path between layouts since _resolve_backend
+ # picked it. Re-resolve before starting: the stale path would drop the load to one-shot for nothing,
+ # or start a build this load did not select. Under the READER, and held until _pending_server is
+ # published. Re-resolving alone does not close the race: allow_install=False only declines to
+ # install, it does not wait for or claim the tree, so an installer that has already passed its
+ # in-use check can sweep this executable between the re-read and the start. Once _pending_server is
+ # published, _tree_in_use covers it and the claim is no longer what is holding the installer off.
with _tree_reader(server_binary, cancel_event):
refreshed = ensure_sd_server_binary(
allow_install = False, accelerator = self._resolved_accelerator()
@@ -1571,30 +1536,19 @@ class SdCppDiffusionBackend:
)
server_binary = refreshed
if not server_binary or not _server_binary_runnable(server_binary):
- # Nothing runnable survived the replacement; the one-shot CLI is the
- # documented fallback and _resolve_engine re-resolves it from scratch.
logger.warning(
"sd-server is no longer usable after the asset download; "
"falling back to one-shot sd-cli."
)
mode, server_binary, engine = "oneshot", None, self._resolve_engine()
- # And pin off THIS engine. The one-shot pin above was taken while the
- # mode was still "server", i.e. off an engine of None, so leaving it
- # would compare the sd-cli just resolved against None and refuse the
- # documented fallback on every load that reaches it. Resolved here,
- # inside the claim, so it is vetted at the moment it is pinned.
+ # And pin off THIS engine. The one-shot pin above was taken while the mode was still
+ # "server", i.e. off an engine of None, so leaving it would compare the sd-cli just resolved
+ # against None and refuse the documented fallback on every load that reaches it. Resolved
+ # here, inside the claim, so it is vetted at the moment it is pinned.
engine_accelerator = _installed_accelerator_of(
getattr(engine, "binary", None)
)
elif _installed_accelerator_of(server_binary) != server_accelerator:
- # Runnable, and at the same path -- and still not the build this load
- # resolved. An install that landed during the download (an H3 load
- # putting the CPU fallback in, say) leaves a server that starts
- # perfectly well on a device this load has already committed elsewhere,
- # so it would generate on the CPU while the GPU offload policy and the
- # arbiter's accounting both describe a GPU run. Asked here, inside the
- # claim, where no further install can start: refusing costs a retry
- # that re-resolves device, accelerator and install from scratch.
raise RuntimeError(
"The stable-diffusion.cpp server binary was replaced by an install "
"for a different accelerator while this model was loading. Try the "
@@ -1602,17 +1556,14 @@ class SdCppDiffusionBackend:
)
else:
server = SdCppServer(server_binary)
- # Published INSIDE the claim: _tree_in_use reads _pending_server, so
- # this is the handover from "a reader holds the tree" to "a starting
- # server does", with no gap between them.
- #
- # Cancellation is re-read in the SAME block. The revalidation above
- # can sit for 20s in _server_binary_runnable, and an unload arriving
- # in that window finds no _pending_server to stop, so without this the
- # load would go on to spawn the process anyway and hold the device for
- # the whole start() timeout before the commit below noticed. Asked
- # under the lock that publishes, so an unload either stops this server
- # or is seen here; it cannot fall between the two.
+ # Published INSIDE the claim: _tree_in_use reads _pending_server, so this is the handover
+ # from "a reader holds the tree" to "a starting server does", with no gap between them.
+ # Cancellation is re-read in the SAME block. The revalidation above can sit for 20s in
+ # _server_binary_runnable, and an unload arriving in that window finds no _pending_server to
+ # stop, so without this the load would go on to spawn the process anyway and hold the device
+ # for the whole start() timeout before the commit below noticed. Asked under the lock that
+ # publishes, so an unload either stops this server or is seen here; it cannot fall between
+ # the two.
with self._lock:
if self._load_token != _load_token or cancel_event.is_set():
server = None
@@ -1624,20 +1575,16 @@ class SdCppDiffusionBackend:
if mode == "server":
assert server_binary is not None
assert server is not None
- # ``started`` (set with _pending_server above) is the object to clear below.
- # ``server`` itself is set to None when start() fails and the load falls back
- # to one-shot, and comparing THAT against _pending_server left the stopped
- # server published forever, which reads as "the managed tree is busy" for the
- # rest of the process.
- #
- # A server that DID start stays published until _state takes it over, under the
- # same lock. Clearing it here would leave a window in which the tree reads as
- # idle -- no reader, no pending, no state -- while the process is up and
- # running out of it, and an ensure_* landing in that window admits an install
- # that later overwrites the executable underneath the live server.
+ # ``started`` is the object to clear below ``started`` (set with _pending_server above) is the
+ # object to clear below. ``server`` itself is set to None when start() fails and the load falls back
+ # to one-shot, and comparing THAT against _pending_server left the stopped server published forever,
+ # which reads as "the managed tree is busy" for the rest of the process. A server that DID start
+ # stays published until _state takes it over, under the same lock. Clearing it here would leave a
+ # window in which the tree reads as idle -- no reader, no pending, no state -- while the process is
+ # up and running out of it, and an ensure_* landing in that window admits an install that later
+ # overwrites the executable underneath the live server.
started_ok = False
try:
- # Blocks until the model is loaded and answering; raises with the log tail on failure.
server.start(
files,
vae_format = fam.sd_cpp_vae_format,
@@ -1645,33 +1592,29 @@ class SdCppDiffusionBackend:
offload, server_binary, gpu_ordinal
),
native_speed = native_speed,
- # Pin to physical cores (sd.cpp's default oversubscribes; see _default_threads).
threads = _default_threads(),
)
started_ok = True
except SdCppCancelled:
- # Aborted by unload / superseding load: stop the half-started server and bail.
server.stop()
raise
except Exception as start_exc: # noqa: BLE001
- # Fall back to one-shot sd-cli if usable, else surface the server error.
logger.warning(
"sd-server failed to start (%s); falling back to one-shot sd-cli.",
start_exc,
)
server.stop()
- # Unpublish BEFORE resolving the one-shot engine: _pending_server means "a
- # process is running out of the tree", and leaving this stopped one there
- # would block the very sd-cli install this fallback needs.
+ # Unpublish BEFORE resolving the one-shot engine: _pending_server means "a process is running
+ # out of the tree", and leaving this stopped one there would block the very sd-cli install this
+ # fallback needs.
with self._lock:
if self._pending_server is server:
self._pending_server = None
server = None
- # KEEP the engine this fallback resolved. Discarding it left the local
- # `engine` at the server path's None, so state.sd_accelerator was recorded
- # as None and the first one-shot generation, which re-resolves sd-cli and
- # reads its real accelerator, rejected it as a different-accelerator
- # replacement: the load reports success and then cannot generate.
+ # KEEP the engine this fallback resolved. Discarding it left the local `engine` at the server
+ # path's None, so state.sd_accelerator was recorded as None and the first one-shot generation,
+ # which re-resolves sd-cli and reads its real accelerator, rejected it as a
+ # different-accelerator replacement: the load reports success and then cannot generate.
fallback: Optional[SdCppEngine] = None
try:
fallback = self._resolve_engine()
@@ -1681,9 +1624,9 @@ class SdCppDiffusionBackend:
if not usable or fallback is None:
raise start_exc
engine = fallback
- # Vetted here, so pinned here: this engine was resolved after the
- # download, inside the claim, and holding it to the pre-download answer
- # would refuse the fallback on an install this load already lived through.
+ # Vetted here, so pinned here: this engine was resolved after the download, inside the claim,
+ # and holding it to the pre-download answer would refuse the fallback on an install this load
+ # already lived through.
engine_accelerator = _installed_accelerator_of(
getattr(fallback, "binary", None)
)
@@ -1696,10 +1639,9 @@ class SdCppDiffusionBackend:
if mode == "oneshot" and (
_installed_accelerator_of(getattr(engine, "binary", None)) != engine_accelerator
):
- # Runnable, at the same path, and still not the build this load vetted -- the
- # one-shot half of the check the server path makes just above. Refused at load
- # rather than recorded, because recording the replacement is what makes the
- # per-generation comparison agree with it forever after.
+ # Runnable, at the same path, and still not the build this load vetted -- the one-shot half of the
+ # check the server path makes just above. Refused at load rather than recorded, because recording
+ # the replacement is what makes the per-generation comparison agree with it forever after.
raise RuntimeError(
"The stable-diffusion.cpp binary was replaced by an install for a "
"different accelerator while this model was loading. Try the load again."
@@ -1719,10 +1661,9 @@ class SdCppDiffusionBackend:
files = files,
vae_format = fam.sd_cpp_vae_format,
native_speed = native_speed,
- # Pinned against the binary this load COMMITTED to, which a deferred install or
- # a one-shot fallback may have changed since the policy was built.
+ # Pinned against the binary this load COMMITTED to, which a deferred install or a one-shot fallback
+ # may have changed since the policy was built.
offload_flags = committed_offload_flags,
- # One-shot sd-cli reads this per generation; pin to physical cores.
threads = _default_threads(),
sampling_method = fam.sd_cpp_sampling_method,
flow_shift = fam.sd_cpp_flow_shift,
@@ -1731,8 +1672,8 @@ class SdCppDiffusionBackend:
hf_token = hf_token,
gguf_filename = gguf_filename,
flux2_inner_dim = inner_dim,
- # Only the one-shot path needs to carry it: it re-resolves sd-cli per image,
- # long after this decision, and has nothing else to check the answer against.
+ # Only the one-shot path needs to carry it: it re-resolves sd-cli per image, long after this
+ # decision, and has nothing else to check the answer against.
sd_accelerator = engine_accelerator if mode == "oneshot" else None,
physical_gpu_id = (
_resolved_server_physical_gpu_id(
@@ -1749,9 +1690,9 @@ class SdCppDiffusionBackend:
orphan: Optional[SdCppServer] = None
with self._lock:
if self._load_token != _load_token:
- # Superseded / unloaded while loading: discard the started server so it
- # doesn't leak. Reserved in the SAME block that unpublishes it, so the tree
- # never reads as idle while the process is still coming down.
+ # Superseded / unloaded while loading: discard the started server so it doesn't leak. Reserved
+ # in the SAME block that unpublishes it, so the tree never reads as idle while the process is
+ # still coming down.
superseded = True
if server is not None:
self._reserve_stop()
@@ -1759,8 +1700,8 @@ class SdCppDiffusionBackend:
else:
self._state = state
self._loading = None
- # The exchange the started server stayed published for: it is _state's now, or
- # reserved for the stop below, and either way _tree_in_use still sees it.
+ # The exchange the started server stayed published for: it is _state's now, or reserved for the stop
+ # below, and either way _tree_in_use still sees it.
if self._pending_server is started:
self._pending_server = None
if orphan is not None:
@@ -1773,16 +1714,16 @@ class SdCppDiffusionBackend:
if self._load_token != _load_token:
return
logger.error("sd_cpp.load_failed: %s", exc)
- # Redact filesystem paths before this reaches /images/load-progress (as diffusers does).
+ # Redact filesystem paths before this reaches /images/load-progress (as diffusers does)
from utils.native_path_leases import redact_native_paths
with self._lock:
if self._load_token == _load_token and self._loading is not None:
self._loading.error = redact_native_paths(str(exc))
finally:
- # Backstop for the window the started server is deliberately left published across
- # (start() -> _state). Every path through that window unpublishes it itself; this only
- # catches an unexpected raise in between, which would otherwise wedge the tree as busy.
+ # Backstop for the window the started server is deliberately left published across (start() -> _state).
+ # Every path through that window unpublishes it itself; this only catches an unexpected raise in between,
+ # which would otherwise wedge the tree as busy.
if started is not None:
with self._lock:
if self._pending_server is started:
@@ -1817,7 +1758,7 @@ class SdCppDiffusionBackend:
)
fam = detect_family_for_pick(repo_id, gguf_filename, family_override)
if fam is None or not family_sd_cpp_supported(fam):
- # Unreachable through the route, but a direct caller gets the same message begin_load would raise.
+ # Unreachable through the route, but a direct caller gets the same message begin_load would raise
raise ValueError(f"'{repo_id}' has no native sd.cpp asset mapping.")
# Same reason as the diffusers plan: this is what stages the download.
_assert_pick_is_not_speech(repo_id, gguf_filename, hf_token)
@@ -1830,40 +1771,36 @@ class SdCppDiffusionBackend:
)
by_repo = self._assets_by_repo(specs)
- # STAGED before the load runs, so each entry must name the repo _fetch_assets will pull
- # from: some asset repos are gated (the FLUX.1 VAE lives in black-forest-labs/FLUX.1-schnell)
- # and an anonymous user would 401 at staging, never reaching the swap. Same per-repo file
- # list on both sides, so both take the same decision.
+ # STAGED before the load runs, so each entry must name the repo _fetch_assets will pull from: some asset repos
+ # are gated (the FLUX.1 VAE lives in black-forest-labs/FLUX.1-schnell) and an anonymous user would 401 at
+ # staging, never reaching the swap. Same per-repo file list on both sides, so both take the same decision.
fetch_repo = _fetch_repo_map(specs, hf_token)
- # MERGED, not reassigned: two upstream repos can share one fetch repo (the FLUX.2 VAE and
- # the dev encoders both come from Comfy-Org/flux2-dev once that repack is cached), and a
- # plain comprehension would drop whichever landed first, leaving its files out of both the
- # staged entry and the footprint.
+ # MERGED, not reassigned: two upstream repos can share one fetch repo (the FLUX.2 VAE and the dev encoders both
+ # come from Comfy-Org/flux2-dev once that repack is cached), and a plain comprehension would drop whichever
+ # landed first, leaving its files out of both the staged entry and the footprint.
merged: dict[str, list[str]] = {}
for repo, names in by_repo.items():
into = merged.setdefault(fetch_repo[repo], [])
into.extend(n for n in names if n not in into)
by_repo = merged
fetch_repo_id = fetch_repo.get(repo_id, repo_id)
- # AFTER the swap: preflighting the upstream id would refuse the very picks the ungated
- # mirror exists to rescue.
+ # AFTER the swap: preflighting the upstream id would refuse the very picks the ungated mirror exists to rescue
self._preflight_companion_repos(by_repo, fetch_repo_id, hf_token)
sizes = self._plan_file_sizes(by_repo, hf_token)
entries: list[dict[str, Any]] = []
total = 0
- # Imported here, not at module scope: diffusion.py is the heavier module and the routes
- # already load this one on its own.
+ # Imported here, not at module scope: diffusion.py is the heavier module and the routes already load this one on
+ # its own
from core.inference.diffusion import DiffusionBackend
for repo, names in by_repo.items():
total += int(sum(sizes.get((repo, n), 0) for n in names))
- # Same missing-file filter the diffusers planner applies: _fetch_assets already reads
- # both cache roots, so staging an asset it can resolve re-downloads it for nothing and
- # fails offline. required_bytes keeps the UNFILTERED sum -- it is the disk footprint.
- # Sized, so a republished asset under the same name is a miss rather than a silent
- # inline fetch during the load. Without it the probe trusts the local ref alone.
- # Loadable, not merely cached: a stale live-root copy shadows a good one in the other
- # root, because the fetch only switches roots when the live lookup finds nothing.
+ # Same missing-file filter the diffusers planner applies: _fetch_assets already reads both cache roots, so
+ # staging an asset it can resolve re-downloads it for nothing and fails offline. required_bytes keeps the
+ # UNFILTERED sum -- it is the disk footprint. Sized, so a republished asset under the same name is a miss
+ # rather than a silent inline fetch during the load. Without it the probe trusts the local ref alone.
+ # Loadable, not merely cached: a stale live-root copy shadows a good one in the other root, because the
+ # fetch only switches roots when the live lookup finds nothing.
missing = [
n
for n in names
@@ -1877,12 +1814,15 @@ class SdCppDiffusionBackend:
# A stable scope lets repeated picks adopt an in-flight download.
"files": list(names),
"bytes": int(sum(sizes.get((repo, n), 0) for n in missing)),
- # Only the transformer entry carries the GGUF filename; the VAE / encoder entries are plain single files.
+ # Only the transformer entry carries the GGUF filename; the VAE / encoder entries are plain single
+ # files.
"gguf_filename": gguf_filename if repo == fetch_repo_id else None,
- # Same entry, said plainly for the panel's label: the transformer IS the pick, the
- # VAE / encoders are required assets. Compared against the POST-swap id, because a
- # gated pick staged from its ungated mirror no longer matches the id the caller
- # asked for. Native picks are always single-file, so there is no pipeline case.
+ # Compared against the POST-swap id, because a gated pick staged from its ungated mirror no longer
+ # matches the id the caller asked for.
+ # Same entry, said plainly for the panel's label: the transformer IS the pick, the VAE / encoders
+ # are required assets. Compared against the POST-swap id, because a gated pick staged from its
+ # ungated mirror no longer matches the id the caller asked for. Native picks are always single-file,
+ # so there is no pipeline case.
"checkpoint": repo == fetch_repo_id and gguf_filename in missing,
}
)
@@ -1941,8 +1881,8 @@ class SdCppDiffusionBackend:
for repo, names in by_repo.items():
# Companions only: the picker only lists repos it could already read.
if repo != repo_id and names:
- # Probe an asset THIS pick stages: a VAE-only repo has no pipeline manifest, so the
- # default name would neither verify access nor see the cache.
+ # Probe an asset THIS pick stages: a VAE-only repo has no pipeline manifest, so the default name would
+ # neither verify access nor see the cache.
_assert_base_repo_accessible(repo, hf_token, names[0])
def preflight_base_access(
@@ -1963,7 +1903,7 @@ class SdCppDiffusionBackend:
unloads the resident model first. Nothing to check without a family or checkpoint name."""
if fam is None or not gguf_filename:
return
- # Post-swap, as the plan and the load are: the swap is pure, so all three decide alike.
+ # Post-swap, as the plan and the load are: the swap is pure, so all three decide alike
specs = self._asset_specs(
repo_id,
gguf_filename,
@@ -2029,8 +1969,8 @@ class SdCppDiffusionBackend:
specs: list[tuple[str, str, str]] = [(repo_id, gguf_filename, "diffusion_model")]
if fam.sd_cpp_vae:
specs.append((fam.sd_cpp_vae[0], fam.sd_cpp_vae[1], "vae"))
- # Pick the encoder per variant so a 9B GGUF fetches the right one: from the header when the
- # caller read it, else from the load identity, which a renamed file makes silent.
+ # Pick the encoder per variant so a 9B GGUF fetches the right one: from the header when the caller read it, else
+ # from the load identity, which a renamed file makes silent.
for terepo, tefile, kind in sd_cpp_text_encoders_for(
fam, repo_id, gguf_filename, inner_dim = inner_dim
):
@@ -2080,9 +2020,9 @@ class SdCppDiffusionBackend:
# Callers without a per-load event (tests, direct use) fall back to the current one.
cancel = cancel_event if cancel_event is not None else self._cancel_event
paths: dict[str, str] = {}
- # This backend fetches the ASSET repos, never the base, and some are gated, so the swap goes
- # here. Decided per REPO over its whole file list, exactly as download_plan does, so the
- # load pulls from the repo the manager already staged.
+ # This backend fetches the ASSET repos, never the base, and some are gated, so the swap goes here. Decided per
+ # REPO over its whole file list, exactly as download_plan does, so the load pulls from the repo the manager
+ # already staged.
fetch_repo = _fetch_repo_map(assets, hf_token)
assets = [(fetch_repo[repo], fn, kind) for repo, fn, kind in assets]
for repo, fn, kind in assets:
@@ -2092,9 +2032,9 @@ class SdCppDiffusionBackend:
if kind == "diffusion_model" and local_root.exists():
path = str(resolve_local_gguf_child(local_root, fn))
else:
- # Resolve an asset cached only under huggingface_hub's import-time root through
- # that root, as the preflight does. Pinned to the live root, a cache-folder change
- # re-downloads every moved asset and 401s on an already-downloaded gated base.
+ # Resolve an asset cached only under huggingface_hub's import-time root through that root, as the
+ # preflight does. Pinned to the live root, a cache-folder change re-downloads every moved asset and 401s
+ # on an already-downloaded gated base.
try:
path = hf_hub_download_with_xet_fallback(
repo,
@@ -2105,11 +2045,10 @@ class SdCppDiffusionBackend:
local_files_only = local_files_only,
)
except _local_entry_not_found_error() as exc:
- # Raised by huggingface_hub for exactly "not cached and outgoing traffic is
- # disabled", so it can only fire under local_files_only. Its own text names
- # neither the repo nor the file, and this string is what /images/load-progress
- # toasts, so restate it with both. Re-raised untouched in the (unreachable)
- # online case rather than relabelled, so nothing changes when the flag is off.
+ # Raised by huggingface_hub for exactly "not cached and outgoing traffic is disabled", so it can
+ # only fire under local_files_only. Its own text names neither the repo nor the file, and this
+ # string is what /images/load-progress toasts, so restate it with both. Re-raised untouched in the
+ # (unreachable) online case rather than relabelled, so nothing changes when the flag is off.
if not local_files_only:
raise
raise RuntimeError(
@@ -2170,9 +2109,9 @@ class SdCppDiffusionBackend:
repos = [state.repo_id, state.base_repo]
if fam.sd_cpp_vae:
repos.append(fam.sd_cpp_vae[0])
- # Same per-variant selection as _asset_specs (the header dim this load committed, else
- # repo id AND GGUF filename) so the delete guard protects the encoder repo this load
- # actually downloaded. Read off the state, never re-probed: this runs under _lock.
+ # Same per-variant selection as _asset_specs (the header dim this load committed, else repo id AND GGUF
+ # filename) so the delete guard protects the encoder repo this load actually downloaded. Read off the state,
+ # never re-probed: this runs under _lock.
repos.extend(
terepo
for terepo, _f, _k in sd_cpp_text_encoders_for(
@@ -2197,20 +2136,22 @@ class SdCppDiffusionBackend:
guidance: float = 0.0,
seed: Optional[int] = None,
batch_size: int = 1,
- # Batched prompt/seed lists are diffusers-engine features; accepted for parity and rejected below, since sd-cli would render them serially.
+ # Batched prompt/seed lists are diffusers-engine features; accepted for parity and rejected below, since sd-cli
+ # would render them serially.
prompts: Optional[list[str]] = None,
seeds: Optional[list[int]] = None,
- # Accepted for interface parity; native is text-to-image only, so image-conditioned requests are rejected below.
+ # Accepted for interface parity; native is text-to-image only
init_image: Optional[str] = None,
mask_image: Optional[str] = None,
strength: Optional[float] = None,
upscale: Optional[float] = None, # needs an init image; rejected by the guard below
reference_images: Optional[list[str]] = None, # GPU/diffusers-only (FLUX.2)
- # LoRA (id, weight) pairs; resolved up front then applied per path: prompt tags for one-shot sd-cli, structured `lora` for sd-server.
+ # LoRA (id, weight) pairs; resolved up front then applied per path: prompt tags for one-shot sd-cli, structured
+ # `lora` for sd-server.
loras: Optional[list[tuple[str, float]]] = None,
# ControlNet is diffusers-only; rejected by the guard below (accepted for parity).
controlnet: Optional[tuple[str, str, str, float, float, float]] = None,
- # load_identity() of the caller's status() read; refuse rather than run a different load (#9448).
+ # load_identity() of the caller's status() read; refuse rather than run a different load (#9448)
expected_load: Optional[LoadIdentity] = None,
) -> dict[str, Any]:
import tempfile
@@ -2235,7 +2176,7 @@ class SdCppDiffusionBackend:
"(it renders serially); run on a GPU (diffusers) for batched generation, "
"or use batch_size for a serial native batch."
)
- # strength 0/None disables ControlNet (matches diffusers), so no-op it rather than 400.
+ # strength 0/None disables ControlNet (matches diffusers), so no-op it rather than 400
if controlnet is not None and controlnet[3] in (None, 0, 0.0):
controlnet = None
if controlnet is not None:
@@ -2250,7 +2191,8 @@ class SdCppDiffusionBackend:
state = self._state
if state is None:
raise RuntimeError(DIFFUSION_NOT_LOADED_MSG)
- # A resident server can exit while idle; drop stale state and report not-loaded so the client gets the reload path, not a 500.
+ # A resident server can exit while idle; drop stale state and report not-loaded so the client gets the
+ # reload path
if (
state.mode == "server"
and state.server is not None
@@ -2258,12 +2200,13 @@ class SdCppDiffusionBackend:
):
self._state = None
raise RuntimeError(DIFFUSION_NOT_LOADED_MSG)
- # Same window as the diffusers engine: a replacement can commit while this waits (#9448).
+ # Same window as the diffusers engine: a replacement can commit while this waits (#9448)
loaded_id = load_identity(state.repo_id, state.base_repo, state.family.name)
if expected_load is not None and expected_load != loaded_id:
raise DiffusionModelReplacedError(expected_load, loaded_id)
self._active_generate_cancel = cancel
- # Publish an active (step 0) state before the slow pre-generate setup so a reload probe does not read idle while this holds _generate_lock.
+ # Publish an active (step 0) state before the slow pre-generate setup so a reload probe does not read
+ # idle while this holds _generate_lock.
self._gen = _SdGen(total_steps = int(steps))
try:
if seed is None:
@@ -2271,7 +2214,8 @@ class SdCppDiffusionBackend:
else:
seed = int(seed)
cfg_scale, flux_guidance = _map_guidance(state.family, guidance)
- # Resolve selected LoRAs up front (a bad id gives a clear 400). Drop weight-0 rows BEFORE the support gate so an only-disabled request stays a no-op.
+ # Resolve selected LoRAs up front (a bad id gives a clear 400). Drop weight-0 rows BEFORE the support
+ # gate so an only-disabled request stays a no-op.
lora_resolved: list = []
active_loras = [(i, w) for (i, w) in (loras or []) if w != 0]
if active_loras:
@@ -2321,36 +2265,34 @@ class SdCppDiffusionBackend:
lora_resolved = lora_resolved,
cancel = cancel,
)
- # Check and deregister under _lock, the lock cancel_generate takes, so the two
- # cannot interleave: a cancel that saw this event registered ran strictly before
- # the check and the run unwinds as cancelled, and one arriving after finds nothing
- # to set and answers false. Same critical section as DiffusionBackend.generate;
- # /images/generate/cancel resolves through the engine router, so a native host has
- # to give the same answer. The finally repeats the clear for every other exit.
+ # Check and deregister under _lock, the lock cancel_generate takes, so the two cannot interleave: a
+ # cancel that saw this event registered ran strictly before the check and the run unwinds as cancelled,
+ # and one arriving after finds nothing to set and answers false. Same critical section as
+ # DiffusionBackend.generate; /images/generate/cancel resolves through the engine router, so a native
+ # host has to give the same answer. The finally repeats the clear for every other exit.
with self._lock:
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
- # ``seeds`` is the per-image seed (image i used seed+i) for the route to persist.
return {
"images": images,
"seed": int(seed),
"seeds": seeds,
"repo_id": state.repo_id,
- # The BUILD, for the recipe: the repo id alone does not say WHICH GGUF quant ran, and two quants make different pixels.
+ # The BUILD, for the recipe: the repo id alone does not say WHICH GGUF quant ran, and two quants
+ # make different pixels.
"model_kind": "gguf",
"gguf_filename": state.gguf_filename,
- # The rest of the build the recipe records. The native engine has no dense
- # quant path and no memory-mode planner, so those two are honestly null -- but
- # the offload it ran under is real (sd-cli flags) and status() already derives
- # it the same way. Omitting them here persisted null for every native image and
- # left the recipe unable to say how the picture was produced.
+ # The rest of the build the recipe records. The native engine has no dense quant path and no
+ # memory-mode planner, so those two are honestly null -- but the offload it ran under is real
+ # (sd-cli flags) and status() already derives it the same way. Omitting them here persisted null for
+ # every native image and left the recipe unable to say how the picture was produced.
"transformer_quant": None,
"text_encoder_quant": None,
"memory_mode": None,
- # The POLICY flags only: a --backend pin says which card ran the graph, not
- # that anything was offloaded, and a `fast` load carries no policy flags at all.
+ # The POLICY flags only: a --backend pin says which card ran the graph, not that anything was
+ # offloaded, and a `fast` load carries no policy flags at all.
"offload_policy": (
"active" if without_device_backend_flags(state.offload_flags) else "none"
),
@@ -2407,7 +2349,8 @@ class SdCppDiffusionBackend:
base_seed = int(seed) & ((1 << 63) - 1)
images: list = []
seeds: list[int] = []
- # Stage LoRAs into a per-request subdir of the server lora-model-dir (so a prior request's adapters cannot leak in); removed after.
+ # Stage LoRAs into a per-request subdir of the server lora-model-dir (so a prior request's adapters cannot leak
+ # in)
lora_payload: Optional[list[dict]] = None
lora_stage: Optional[Path] = None
if lora_resolved:
@@ -2422,8 +2365,9 @@ class SdCppDiffusionBackend:
}
for m in materialized
]
- # One deadline for the whole request, shared by its chunks: a batch is chunked only because the server caps images per
- # job, so each chunk gets whatever is left rather than its own full budget and a long batch still ends on time.
+ # One deadline for the whole request, shared by its chunks: a batch is chunked only because the server caps
+ # images per job, so each chunk gets whatever is left rather than its own full budget and a long batch still
+ # ends on time.
deadline = time.monotonic() + NATIVE_GENERATION_TIMEOUT_S
try:
for offset in range(0, total, _MAX_SERVER_BATCH):
@@ -2453,7 +2397,8 @@ class SdCppDiffusionBackend:
total_timeout = max(deadline - time.monotonic(), 1.0),
)
except RuntimeError as exc:
- # A ggml unsupported-op abort killed the server: this graph cannot run on the GPU backend at all, so restart the model on the CPU backend once and retry this chunk. Any other death propagates.
+ # A ggml unsupported-op abort killed the server: this graph cannot run on the GPU backend at all, so
+ # restart the model on the CPU backend once and retry this chunk. Any other death propagates.
server = self._restart_server_on_cpu_backend(state, str(exc), cancel)
if server is None:
raise
@@ -2524,10 +2469,9 @@ class SdCppDiffusionBackend:
server.start(
state.files,
vae_format = state.vae_format,
- # WITHOUT the device pin. sd.cpp joins repeated --backend values into one spec
- # instead of replacing, and an explicit diffusion=CUDA0 outranks the bare `cpu`
- # default, so leaving the pin on would restart the server onto the very backend
- # that just aborted.
+ # WITHOUT the device pin. sd.cpp joins repeated --backend values into one spec instead of replacing, and
+ # an explicit diffusion=CUDA0 outranks the bare `cpu` default, so leaving the pin on would restart the
+ # server onto the very backend that just aborted.
offload = without_device_backend_flags(state.offload_flags),
native_speed = state.native_speed,
threads = state.threads,
@@ -2580,7 +2524,7 @@ class SdCppDiffusionBackend:
images = []
seeds: list[int] = []
with tempfile.TemporaryDirectory(prefix = "sdcpp_gen_") as tmpdir:
- # Materialize LoRAs into a scan dir and inject <lora:ALIAS:w> tags (deduped). Empty -> unchanged.
+ # Materialize LoRAs into a scan dir and inject <lora:ALIAS:w> tags (deduped).
eff_prompt = prompt
lora_dir: Optional[str] = None
if lora_resolved:
@@ -2592,7 +2536,8 @@ class SdCppDiffusionBackend:
for index in range(max(1, int(batch_size))):
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
- # Distinct reproducible seed per image; mask to int64 (53 bits would truncate large explicit seeds and collide them).
+ # Distinct reproducible seed per image; mask to int64 (53 bits would truncate large explicit seeds and
+ # collide them)
seed_i = (seed + index) & ((1 << 63) - 1)
out_path = str(Path(tmpdir) / f"img_{index}.png")
params = SdCppGenParams(
@@ -2609,24 +2554,21 @@ class SdCppDiffusionBackend:
lora_dir = lora_dir,
lora_apply_mode = "auto" if lora_dir else None,
)
- # Each sd-cli run executes out of the managed tree, so hold installs off for its
- # duration (and wait here if one is already extracting).
- # getattr: an INJECTED engine is the unit-test seam / escape hatch and need not
- # name a file at all, and nothing without a path is a binary an install replaces.
+ # Each sd-cli run executes out of the managed tree, so hold installs off for its duration (and wait here
+ # if one is already extracting). getattr: an INJECTED engine is the unit-test seam / escape hatch and
+ # need not name a file at all, and nothing without a path is a binary an install replaces.
with _tree_reader(getattr(engine, "binary", None), cancel):
- # Re-resolve INSIDE the claim. An install that finished while this image was
- # waiting can have put its sd-cli somewhere else and swept the copy resolved
- # above, so the cached path would launch a file that is no longer there. Also
- # covers a batch, which releases the claim between images. Cheap when nothing
- # moved: _resolve_engine returns the cached engine whose binary still exists.
+ # Re-resolve INSIDE the claim. An install that finished while this image was waiting can have put
+ # its sd-cli somewhere else and swept the copy resolved above, so the cached path would launch a
+ # file that is no longer there. Also covers a batch, which releases the claim between images. Cheap
+ # when nothing moved: _resolve_engine returns the cached engine whose binary still exists.
engine = self._resolve_engine()
- # Existence is not identity here either. The install that moved the CLI may
- # have been for a different accelerator (an H3 load putting the CPU fallback
- # in, say), and this state's device and offload policy were chosen for the
- # other one, so running it would either spend unaccounted VRAM or drop the
- # whole generation onto the CPU while the arbiter's accounting says otherwise.
- # The server path refuses exactly this mismatch before it starts; refusing here
- # costs a reload, which re-resolves device, accelerator and install together.
+ # Existence is not identity here either. The install that moved the CLI may have been for a
+ # different accelerator (an H3 load putting the CPU fallback in, say), and this state's device and
+ # offload policy were chosen for the other one, so running it would either spend unaccounted VRAM or
+ # drop the whole generation onto the CPU while the arbiter's accounting says otherwise. The server
+ # path refuses exactly this mismatch before it starts; refusing here costs a reload, which
+ # re-resolves device, accelerator and install together.
if (
_installed_accelerator_of(getattr(engine, "binary", None))
!= state.sd_accelerator
@@ -2699,7 +2641,8 @@ class SdCppDiffusionBackend:
def unload(self) -> dict[str, Any]:
with self._lock:
- # Under the lock: begin_load rebinds this attribute, so an unlocked read could set an event the current load no longer watches.
+ # Under the lock: begin_load rebinds this attribute, so an unlocked read could set an event the current load
+ # no longer watches.
self._cancel_event.set()
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
@@ -2710,9 +2653,8 @@ class SdCppDiffusionBackend:
# Grab a mid-start() uncommitted server too so we can stop it (startup is abortable).
pending = self._pending_server
self._pending_server = None
- # Reserved HERE, not at the stop below: the fields are already empty by then, and a
- # router probe landing in that gap would read an idle tree and reinstall over a
- # process that is still running.
+ # Reserved HERE, not at the stop below: the fields are already empty by then, and a router probe landing in
+ # that gap would read an idle tree and reinstall over a process that is still running.
to_stop = [
srv
for srv in (state.server if state is not None else None, pending)
@@ -2721,17 +2663,20 @@ class SdCppDiffusionBackend:
if pending is not None and state is not None and pending is state.server:
to_stop = to_stop[:1]
self._reserve_stop(len(to_stop))
- # Stop the resident server outside the lock (terminate can take seconds); a mid-flight generation unwinds as the process goes away.
+ # Stop the resident server outside the lock (terminate can take seconds); a mid-flight generation unwinds as the
+ # process goes away.
for srv in to_stop:
self._stop_reserved(srv)
- # Barrier: wait for a signalled one-shot generation to exit before reporting unloaded, since callers treat this return as "device is free".
+ # Barrier: wait for a signalled one-shot generation to exit before reporting unloaded, since callers treat this
+ # return as "device is free".
with self._generate_lock:
pass
return self.status()
def status(self) -> dict[str, Any]:
state = self._state
- # A resident sd-server can exit after load (OOM/crash while idle); drop stale state so clients reload instead of 500ing per generation.
+ # A resident sd-server can exit after load (OOM/crash while idle); drop stale state so clients reload instead of
+ # 500ing per generation.
if (
state is not None
and state.mode == "server"
@@ -2781,8 +2726,8 @@ class SdCppDiffusionBackend:
"gguf_variant": extract_quant_token(state.gguf_filename)
if state.gguf_filename
else None,
- # Reflect the offload flags actually passed to sd-cli (empty on CPU -> "none"), minus
- # the --backend device pin, which is a card choice rather than an offload decision.
+ # Reflect the offload flags actually passed to sd-cli (empty on CPU -> "none"), minus the --backend device
+ # pin, which is a card choice rather than an offload decision.
"cpu_offload": bool(without_device_backend_flags(state.offload_flags)),
"offload_policy": (
"active" if without_device_backend_flags(state.offload_flags) else "none"
@@ -2802,7 +2747,6 @@ class SdCppDiffusionBackend:
model_kind = "gguf",
transformer_quant = None,
),
- # ControlNet is diffusers-only; the native engine's generate() rejects it.
"supports_controlnet": False,
# "server" = resident sd-server (load once); "oneshot" = legacy per-image sd-cli.
"native_mode": state.mode,
diff --git a/studio/backend/core/inference/sd_cpp_engine.py b/studio/backend/core/inference/sd_cpp_engine.py
index 1c2196406ce..a3102caa980 100644
--- a/studio/backend/core/inference/sd_cpp_engine.py
+++ b/studio/backend/core/inference/sd_cpp_engine.py
@@ -46,28 +46,33 @@ logger = logging.getLogger(__name__)
# sd-cli (sd-cli.exe on Windows); older builds shipped ``sd`` -- both probed on PATH.
_BINARY_STEM = "sd-cli"
_LEGACY_STEM = "sd"
-# The first stable-diffusion.cpp release already exposed all three. Together they distinguish its
-# oldest help text (before it printed the project name) from unrelated tools also called ``sd``.
+# The first stable-diffusion.cpp release already exposed all three. Together they distinguish its oldest help text
+# (before it printed the project name) from unrelated tools also called ``sd``.
_LEGACY_HELP_MARKERS = ("--negative-prompt", "--cfg-scale", "--steps")
# The persistent HTTP server target, shipped next to sd-cli in both prebuilt and cmake builds.
_SERVER_STEM = "sd-server"
-# Ownership marker written by install_sd_cpp_prebuilt.install and required by setup.sh / uninstall.sh / uninstall.ps1 before they delete a tree.
+# ownership marker written by install_sd_cpp_prebuilt.install and required by setup.sh / uninstall.sh / uninstall.ps1
+# before they delete a tree
OWNER_MARKER = ".unsloth-studio-owned"
-# Ceiling for one native run. The native engine exists FOR slow CPU hosts: on GPU-less CI runners a 512x512 4-step Q2_K generation took 900 s on Linux and 1465 s on Windows, so a 30-minute cap killed jobs that were still progressing.
-# It matches the Images page's own SETTLE_MAX_MS (6 h), so it only stops a WEDGED process from holding the lock forever; cancel_event is the user-facing abort.
+# the native engine exists FOR slow CPU hosts
+# Ceiling for one native run. The native engine exists FOR slow CPU hosts: on GPU-less CI runners a 512x512 4-step Q2_K
+# generation took 900 s on Linux and 1465 s on Windows, so a 30-minute cap killed jobs that were still progressing. It
+# matches the Images page's own SETTLE_MAX_MS (6 h), so it only stops a WEDGED process from holding the lock forever;
+# cancel_event is the user-facing abort.
NATIVE_GENERATION_TIMEOUT_S = 6 * 60 * 60.0
-# sd-cli redraws its progress bar IN PLACE. Each redraw is one printf + fflush shaped
-# "\r<bar> <step>/<steps> - <speed>\033[K", with a trailing newline only on the final step of a
-# phase. So the carriage return LEADS the record and the erase-to-end-of-line CLOSES it.
+# sd-cli redraws its progress bar IN PLACE. Each redraw is one printf + fflush shaped "\r<bar> <step>/<steps> -
+# <speed>\033[K", with a trailing newline only on the final step of a phase. So the carriage return LEADS the record and
+# the erase-to-end-of-line CLOSES it.
_ANSI_ERASE = "\x1b[K"
-# Any CSI escape (the erase above, plus colour runs some builds emit), stripped before a record
-# reaches on_log / the error tail: an escape in the middle of a line corrupts both.
+# stripped before a record reaches on_log / the error tail: an escape mid-line corrupts both
+# Any CSI escape (the erase above, plus colour runs some builds emit), stripped before a record reaches on_log / the
+# error tail: an escape in the middle of a line corrupts both.
_ANSI_CSI_RE = re.compile(r"\x1b\[[0-9;?]*[ -/]*[@-~]")
-# One read1() per redraw in practice; large enough that a burst of finished lines costs one read.
+# one read1() per redraw in practice; large enough that a burst of finished lines costs one read
_READ_CHUNK = 4096
@@ -165,7 +170,7 @@ def _terminate(proc: "subprocess.Popen") -> None:
proc.kill()
except Exception: # noqa: BLE001 -- best-effort teardown
pass
- # Reap the killed child so it does not linger as a zombie: callers raise right after _terminate, so a burst of cancellations would leak process-table entries.
+ # reap the killed child: callers raise right after _terminate
try:
proc.wait(timeout = 5)
except Exception: # noqa: BLE001 -- best-effort reap; never block teardown
@@ -211,7 +216,6 @@ def _layout_candidates(root: Path, stem: str = _BINARY_STEM) -> list[Path]:
root / "bin" / name,
root / name,
]
- # Newest install first, by mtime (tag strings don't sort numerically).
try:
subdirs = [p for p in root.iterdir() if p.is_dir()]
subdirs.sort(key = lambda p: p.stat().st_mtime, reverse = True)
@@ -233,19 +237,20 @@ def _first_file(paths: list[Path]) -> Optional[str]:
return None
-# Identity verdicts, keyed by the file itself rather than by the path alone, so replacing a binary
-# in place re-probes it while a rebuild elsewhere on PATH is unaffected. Bounded: an Unsloth session
-# sees a handful of candidates, and a runaway key set would only ever come from a path being
-# rewritten under us, which is exactly the case that must not be served from here.
+# keyed by the file itself rather than the path alone
+# Identity verdicts, keyed by the file itself rather than by the path alone, so replacing a binary in place re-probes it
+# while a rebuild elsewhere on PATH is unaffected. Bounded: an Unsloth session sees a handful of candidates, and a
+# runaway key set would only ever come from a path being rewritten under us, which is exactly the case that must not be
+# served from here.
_IDENTITY_MEMO: dict[tuple[str, int, int, int], tuple[bool, float]] = {}
_IDENTITY_MEMO_LOCK = threading.Lock()
_IDENTITY_MEMO_MAX = 32
-# How long a verdict may answer for. The key catches the replacements it can SEE, but no stat tuple
-# is a content hash: on Windows ``st_ctime`` is the CREATION time, which an in-place overwrite
-# preserves, so a same-sized write that also restores mtime is invisible to it. Hashing the file on
-# every lookup would trade the exec this memo exists to avoid for a read of the whole binary, on a
-# path walked for every load. A short life is the cheaper guarantee and it is not platform-specific:
-# whatever the key misses, and whatever nobody has thought of, expires within a minute. Long enough
+# no stat tuple is a content hash: on Windows `st_ctime` is the CREATION time
+# How long a verdict may answer for. The key catches the replacements it can SEE, but no stat tuple is a content hash:
+# on Windows ``st_ctime`` is the CREATION time, which an in-place overwrite preserves, so a same-sized write that also
+# restores mtime is invisible to it. Hashing the file on every lookup would trade the exec this memo exists to avoid for
+# a read of the whole binary, on a path walked for every load. A short life is the cheaper guarantee and it is not
+# platform-specific: whatever the key misses, and whatever nobody has thought of, expires within a minute. Long enough
# for its actual job, which is the several resolutions inside one load sequence.
_IDENTITY_MEMO_TTL_S = 60.0
@@ -333,8 +338,8 @@ def sd_cpp_binary_identifies(binary: str) -> bool:
except (OSError, subprocess.SubprocessError):
help_text = ""
identified = help_text_identifies_sd_cpp(help_text)
- # Identifying output settles it whatever the exit code (old builds print usage and exit 1).
- # Otherwise only a clean exit is evidence of anything; rc 127 from the dynamic loader is not.
+ # Identifying output settles it whatever the exit code (old builds print usage and exit 1). Otherwise only a clean
+ # exit is evidence of anything; rc 127 from the dynamic loader is not.
decisive = identified or returncode == 0
if key is not None and decisive:
with _IDENTITY_MEMO_LOCK:
@@ -505,25 +510,27 @@ def _find_binary(
install root (honors ``UNSLOTH_STUDIO_HOME`` / ``STUDIO_HOME``, else ``~/.unsloth/...``);
(4) ``./stable-diffusion.cpp`` in-tree build; (5) ``path_stems`` on PATH.
"""
- # 1. Direct binary path.
env_bin = os.environ.get(direct_env)
if env_bin and Path(env_bin).is_file():
return env_bin
- # 2. Custom install dir.
custom = os.environ.get("UNSLOTH_SD_CPP_PATH")
if custom:
hit = _first_file(_layout_candidates(Path(custom), layout_stem))
if hit:
return hit
- # 3. Default install root: <studio home>/stable-diffusion.cpp (honors UNSLOTH_STUDIO_HOME / STUDIO_HOME like the installer so side-by-side Unsloth instances stay isolated), else ~/.unsloth/....
+ # default install root honors UNSLOTH_STUDIO_HOME / STUDIO_HOME like the installer
+ # 3. Default install root: <studio home>/stable-diffusion.cpp (honors UNSLOTH_STUDIO_HOME / STUDIO_HOME like the
+ # installer so side-by-side Unsloth instances stay isolated), else ~/.unsloth/....
default_root = managed_install_root()
hit = _first_file(_layout_candidates(default_root, layout_stem))
if hit:
return hit
- # 3b. A tree an older build installed beside the Unsloth home. Marker-gated (see legacy_sibling_install_root), so only a real previous install is picked up here.
+ # a tree an older build installed beside the Unsloth home
+ # 3b. A tree an older build installed beside the Unsloth home. Marker-gated (see legacy_sibling_install_root), so
+ # only a real previous install is picked up here.
legacy_root = legacy_sibling_install_root()
if legacy_root is not None:
hit = _first_file(_layout_candidates(legacy_root, layout_stem))
@@ -537,7 +544,6 @@ def _find_binary(
if hit:
return hit
- # 5. PATH.
for stem in path_stems:
on_path = shutil.which(stem)
if on_path and (stem != _LEGACY_STEM or _is_legacy_sd_cpp_binary(on_path)):
@@ -725,7 +731,7 @@ class SdCppEngine:
def _prepare_out(output_path: str) -> Path:
out = Path(output_path)
out.parent.mkdir(parents = True, exist_ok = True)
- # Drop a stale file so the post-run is_file() check proves THIS run produced the image.
+ # drop a stale file so the post-run is_file() check proves THIS run produced the image
out.unlink(missing_ok = True)
return out
@@ -760,14 +766,17 @@ class SdCppEngine:
env = run_env,
# Own session/process group so cancellation/timeout kills the whole tree (POSIX).
start_new_session = (os.name == "posix"),
- # Bind the child to the parent's lifetime (PR_SET_PDEATHSIG) so a parent crash cannot orphan sd-cli holding VRAM/RAM.
+ # Bind the child to the parent's lifetime (PR_SET_PDEATHSIG) so a parent crash cannot orphan sd-cli holding
+ # VRAM/RAM.
**child_popen_kwargs(),
)
- # The kwargs above are empty on macOS, so record it too: a crash mid-generation
- # would otherwise leave sd-cli holding VRAM with nothing able to find it.
+ # the kwargs above are empty on macOS, so record it too, else a crash mid-generation leaves sd-cli holding VRAM
+ # with nothing able to find it
adopt_pid(proc.pid)
- # Drain stdout on a reader thread so the timeout holds even when the child hangs WITHOUT printing (a plain `for line in proc.stdout` blocks until EOF). Lines, then a None sentinel, go to a queue the main loop polls against a wall-clock deadline.
- # iter_sd_cpp_records also splits sd-cli's in-place progress redraws, which carry no newline of their own, so sampling progress reaches on_log while sampling is still running.
+ # Drain stdout on a reader thread so the timeout holds even when the child hangs WITHOUT printing (a plain `for
+ # line in proc.stdout` blocks until EOF). Lines, then a None sentinel, go to a queue the main loop polls against
+ # a wall-clock deadline. iter_sd_cpp_records also splits sd-cli's in-place progress redraws, which carry no
+ # newline of their own, so sampling progress reaches on_log while sampling is still running.
tail: list[str] = []
line_q: "queue.Queue[Optional[str]]" = queue.Queue()
@@ -813,8 +822,6 @@ class SdCppEngine:
finally:
if proc.poll() is None:
_terminate(proc)
- # Only once it has actually exited: a pid still running has to stay
- # recorded, or the next startup has no handle on it.
if proc.poll() is not None:
forget_pid(proc.pid)
diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.py
index 85f4474abb1..a3d0c1494f0 100644
--- a/studio/backend/core/inference/sd_cpp_server.py
+++ b/studio/backend/core/inference/sd_cpp_server.py
@@ -66,15 +66,15 @@ _TRANSPORT_ERRORS = (
httpx.WriteError,
)
-# Readiness probe: the port binds only after the model loads, so any 200 means ready. Use trivial /v1/models, not the capabilities endpoint (which can block).
+# the port binds only after the model loads, so any 200 means ready
+# Readiness probe: the port binds only after the model loads, so any 200 means ready. Use trivial /v1/models, not the
+# capabilities endpoint (which can block).
_READY_PATH = "/v1/models"
-# Native async sdcpp API.
_IMG_GEN_PATH = "/sdcpp/v1/img_gen"
_JOBS_PATH = "/sdcpp/v1/jobs"
-# Stable parameter residency reported once during startup. Compute-buffer
-# lines are deliberately excluded: those allocations can change between
-# generations, while this floor lives until the server process exits.
+# Stable parameter residency reported once during startup. Compute-buffer lines are deliberately excluded: those
+# allocations can change between generations, while this floor lives until the server process exits.
_TOTAL_PARAMS_VRAM_RE = re.compile(
r"total params memory size\s*=\s*[0-9.]+\s*MB\s*\(\s*VRAM\s+([0-9]+(?:\.[0-9]+)?)\s*MB\b",
re.IGNORECASE,
@@ -118,7 +118,8 @@ def _diagnostic_tail(
return "\n".join(chosen)[:limit]
-# Grace for the best-effort native cancel to show in job status; without the cap a lost cancel would hold the generate lock until the job ends.
+# Grace for the best-effort native cancel to show in job status; without the cap a lost cancel would hold the generate
+# lock until the job ends.
_CANCEL_GRACE_S = 5.0
@@ -175,8 +176,6 @@ class SdCppServer:
self._stopped = False
atexit.register(self.stop)
- # ── lifecycle ────────────────────────────────────────────────────────────
-
@property
def base_url(self) -> str:
return f"http://{self.host}:{self.port}"
@@ -218,12 +217,13 @@ class SdCppServer:
wins), which is how the CPU-backend restart pins the graph off the GPU.
"""
with self._lifecycle_lock:
- # A stop()/unload that raced in before start() took the lock already set _abort and closed the client; honor it rather than leak a spawned process.
+ # A stop()/unload that raced in before start() took the lock already set _abort and closed the client; honor
+ # it rather than leak a spawned process.
if self._stopped or self._abort.is_set():
raise SdCppCancelled("sd-server start was cancelled before launch.")
self._abort.clear()
port = self._find_free_port()
- # Empty scratch dir for sd-server's LoRA/upscaler/embeddings scans (errors if missing).
+ # empty scratch dir for sd-server's LoRA/upscaler/embeddings scans (errors if missing)
self._scratch_dir = tempfile.mkdtemp(prefix = "sdcpp_dirs_")
cmd = build_sd_cpp_server_command(
self.binary,
@@ -248,7 +248,8 @@ class SdCppServer:
self._spawn_error: Optional[Exception] = None
spawned = threading.Event()
- # Spawn INSIDE the long-lived drain thread: child_popen_kwargs() sets PR_SET_PDEATHSIG, bound to the creating thread on Linux, so the creator must outlive the child.
+ # Spawn INSIDE the long-lived drain thread: child_popen_kwargs() sets PR_SET_PDEATHSIG, bound to the
+ # creating thread on Linux, so the creator must outlive the child.
def _own_process() -> None:
try:
proc = subprocess.Popen(
@@ -271,7 +272,6 @@ class SdCppServer:
adopt_pid(proc.pid) # so a global shutdown sweep also reaps it
spawned.set()
self._drain_stdout(proc)
- # stdout closed == process exited; reap it so it is not left a zombie.
try:
proc.wait(timeout = 5)
except Exception: # noqa: BLE001
@@ -306,7 +306,8 @@ class SdCppServer:
deadline = time.monotonic() + timeout
url = f"{self.base_url}{_READY_PATH}"
while time.monotonic() < deadline:
- # A concurrent stop() sets _abort so this wait bails without holding the model load hostage for the full startup_timeout.
+ # a concurrent stop() sets _abort so this wait bails without holding the model load hostage for the full
+ # startup_timeout
if self._abort.is_set():
logger.info("sd-server startup aborted before ready")
return False
@@ -414,7 +415,7 @@ class SdCppServer:
def stop(self) -> None:
"""Terminate the server (SIGTERM -> SIGKILL), join the drain, and release the HTTP
client + atexit handler. Idempotent."""
- # Signal abort BEFORE contending for the lock so a start() readiness wait bails immediately instead of blocking stop().
+ # signal abort BEFORE contending for the lock so a start() readiness wait bails instead of blocking stop()
self._abort.set()
self._stopped = True
with self._lifecycle_lock:
@@ -490,7 +491,6 @@ class SdCppServer:
self._step_listener = on_step
job_id: Optional[str] = None
try:
- # Submit -> 202 Accepted + job id.
try:
resp = self._client.post(
f"{self.base_url}{_IMG_GEN_PATH}", json = payload, timeout = submit_timeout
@@ -517,7 +517,6 @@ class SdCppServer:
if not job_id:
raise RuntimeError(f"sd-server img_gen returned no job id: {job}")
- # Poll the job to a terminal state.
deadline = time.monotonic() + total_timeout
cancel_sent_at: Optional[float] = None
while True:
@@ -526,8 +525,10 @@ class SdCppServer:
self.cancel(job_id)
cancel_sent_at = time.monotonic()
elif time.monotonic() - cancel_sent_at > _CANCEL_GRACE_S:
- # Cancel not reflected within the grace window, so the job is still running and sd-server will not interrupt it. Stop the process before reporting
- # the cancellation: abandoning the poll frees the generate lock while the native job keeps the GPU. Safe, since the backend reloads on the next generate.
+ # Cancel not reflected within the grace window, so the job is still running and sd-server will
+ # not interrupt it. Stop the process before reporting the cancellation: abandoning the poll
+ # frees the generate lock while the native job keeps the GPU. Safe, since the backend reloads on
+ # the next generate.
self.stop()
raise SdCppCancelled("sd-server generation was cancelled.")
if not self.is_alive():
@@ -536,7 +537,8 @@ class SdCppServer:
raise SdCppCancelled("sd-server generation was cancelled.")
raise RuntimeError(self._died_message("img_gen poll", None))
if time.monotonic() > deadline:
- # sd-server will not interrupt an in-flight job, so cancel + stop to free the slot; the backend reloads on the next generate.
+ # sd-server will not interrupt an in-flight job, so cancel + stop to free the slot; the backend
+ # reloads on the next generate.
self.cancel(job_id)
self.stop()
raise RuntimeError(f"sd-server generation timed out after {total_timeout}s")
@@ -546,7 +548,8 @@ class SdCppServer:
time.sleep(poll_interval)
continue
except RuntimeError as exc:
- # A concurrent stop() closes the shared client, giving a plain RuntimeError rather than a transport error; map a cancel to 409.
+ # a concurrent stop() closes the shared client, giving a plain RuntimeError rather than a transport
+ # error
if cancel_event is not None and cancel_event.is_set():
raise SdCppCancelled("sd-server generation was cancelled.") from exc
raise
diff --git a/studio/backend/core/inference/search_images.py b/studio/backend/core/inference/search_images.py
index 8b0c648905f..242baae2948 100644
--- a/studio/backend/core/inference/search_images.py
+++ b/studio/backend/core/inference/search_images.py
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-# web_search image results: a registry keyed by opaque ids plus a thumbnail proxy,
-# so neither the model nor the browser sees an image URL.
+# a registry keyed by opaque ids plus a thumbnail proxy
+# web_search image results: a registry keyed by opaque ids plus a thumbnail proxy, so neither the model nor the browser
+# sees an image URL.
from __future__ import annotations
import io
@@ -26,8 +27,8 @@ MAX_IMAGES_PER_SEARCH = 6
MAX_THUMBNAIL_BYTES = 3 * 1024 * 1024
THUMBNAIL_EDGE_PX = 320
THUMBNAIL_FETCH_TIMEOUT_S = 10
-# Read from the header before decoding. Low because draft() only subsamples JPEG:
-# a 77 KB 6000x4000 PNG decodes to ~100 MB of RGB.
+# Read from the header before decoding. Low because draft() only subsamples JPEG: a 77 KB 6000x4000 PNG decodes to ~100
+# MB of RGB.
MAX_IMAGE_PIXELS = 6_000_000
_ALLOWED_FORMATS = frozenset({"JPEG", "PNG", "GIF", "WEBP"})
@@ -40,27 +41,29 @@ _MAX_CONCURRENT_FETCHES = 4
_registry: dict[str, dict[str, Any]] = {}
_registry_lock = threading.Lock()
-# Bumped by clear_cache. A fetch that started before the clear must not publish its
-# thumbnail after it: the write is done under _registry_lock and skipped if this moved.
+# a fetch that started before a clear must not publish its thumbnail after it
+# Bumped by clear_cache. A fetch that started before the clear must not publish its thumbnail after it: the write is
+# done under _registry_lock and skipped if this moved.
_cache_generation = 0
-# The generation at which a CLEAR-EVERYTHING last ran, and the generation at which each
-# individually reaped id was taken. A selective clear must not abort an in-flight fetch for
-# an id it deliberately spared: thumbnail_bytes would answer None, the endpoint 404s, and
-# SearchImageThumb renders nothing and never retries -- its effect depends only on (id,
-# nearViewport), so "re-fetches on the next request" is not true, there is no next request.
-# Bounded; on overflow the per-id record is dropped and the full-clear generation is moved
-# instead, which over-aborts rather than republishing a thumbnail a clear removed.
+# The generation of the last CLEAR-EVERYTHING, plus the generation each individually reaped id was taken at. The
+# generation at which a CLEAR-EVERYTHING last ran, and the generation at which each individually reaped id was taken. A
+# selective clear must not abort an in-flight fetch for an id it spared: thumbnail_bytes would answer None, the endpoint
+# 404s, and SearchImageThumb renders nothing and never retries -- its effect depends only on (id, nearViewport), so
+# "re-fetches on the next request" is not true, there is no next request. Bounded; on overflow the per-id record is
+# dropped and the full-clear generation is moved instead, which over-aborts rather than republishing a thumbnail a clear
+# removed.
_full_clear_generation = 0
_reaped_at: dict[str, int] = {}
_REAPED_AT_MAX = 4096
-# The newest generation whose per-id records have been dropped to stay under that cap. A fetch
-# that started at or after this is still answered exactly, because nothing covering it was
-# dropped; only one older than every record we still hold has to be given up on. Fetches are
-# bounded by THUMBNAIL_FETCH_TIMEOUT_S, so outliving 4096 reaped images is not a real case.
+# a fetch started at or after this is still answered exactly
+# The newest generation whose per-id records have been dropped to stay under that cap. A fetch that started at or after
+# this is still answered exactly, because nothing covering it was dropped; only one older than every record we still
+# hold has to be given up on. Fetches are bounded by THUMBNAIL_FETCH_TIMEOUT_S, so outliving 4096 reaped images is not a
+# real case.
_reaped_floor_generation = 0
-# Ids whose files a clear could not unlink -- on Windows another process holding the
-# JPEG open is enough. The cache-first read and the sidecar read both go around the
-# registry, so without this they would go on serving a picture the user had cleared.
+# Ids whose files a clear could not unlink -- on Windows another process holding the JPEG open is enough. The
+# cache-first read and the sidecar read both go around the registry, so without this they would go on serving a picture
+# the user had cleared.
_cleared_unservable: set[str] = set()
_fetch_slots = threading.BoundedSemaphore(_MAX_CONCURRENT_FETCHES)
_inflight: dict[str, threading.Lock] = {}
@@ -154,8 +157,9 @@ def register_images(
"thumbnail": thumbnail,
"source": source,
"created": now,
- # Kept with the entry: the proxy fetch happens on a later request, and
- # without it every redirect hop would be re-checked against no policy.
+ # the proxy fetch happens on a later request
+ # Kept with the entry: the proxy fetch happens on a later request, and without it every redirect hop
+ # would be re-checked against no policy.
"policy": website_policy,
}
entry = {
@@ -171,8 +175,8 @@ def register_images(
for image_id, stored, registered_generation in persist:
_persist_entry(image_id, stored, registered_generation)
if persist:
- # Here too, not only after a thumbnail write: a user who never opens a picture
- # would otherwise accumulate sidecars with nothing ever bounding them.
+ # Here too, not only after a thumbnail write: a user who never opens a picture would otherwise accumulate
+ # sidecars with nothing ever bounding them.
_evict_cache()
return public
@@ -235,8 +239,8 @@ def is_image_entry(entry: object) -> bool:
def split_images_envelope(result: str) -> tuple[str, list[dict[str, str]]]:
- # Payload ends at the next "\n__", as _strip_files_sentinel and the frontend do,
- # so a sibling sentinel after ours does not make it unreadable.
+ # Payload ends at the next "\n__", as _strip_files_sentinel and the frontend do, so a sibling sentinel after ours
+ # does not make it unreadable.
start = result.rfind(SEARCH_IMAGES_SENTINEL)
if start == -1:
return result, []
@@ -289,9 +293,8 @@ def _persist_entry(image_id: str, entry: dict[str, Any], generation: int) -> Non
ensure_ascii = True,
)
with _registry_lock:
- # Per id, like the thumbnail write: a selective clear bumps the generation
- # without touching this image, and dropping its sidecar then costs the id its
- # only way back after a restart.
+ # Per id, like the thumbnail write: a selective clear bumps the generation without touching this image, and
+ # dropping its sidecar then costs the id its only way back after a restart.
if _reaped_since_locked(image_id, generation):
return
# writer-unique, like the JPEG: a torn read must not be possible.
@@ -314,26 +317,25 @@ def _load_persisted_entry(image_id: str) -> dict[str, Any] | None:
source = raw.get("source")
if not isinstance(thumbnail, str) or not isinstance(source, str):
return None
- # Re-checked on the way back in: what was public when it was written is not
- # necessarily public now, and this bypasses register_images' own gate.
+ # Re-checked on the way back in: what was public when it was written is not necessarily public now, and this
+ # bypasses register_images' own gate.
if not (_names_public_host(thumbnail) and _names_public_host(source)):
return None
policy = raw.get("policy")
return {
"thumbnail": thumbnail,
"source": source,
- # No TTL: a disk entry follows the cache beside it, which is capped by file
- # count rather than age. time.monotonic() from a previous process is meaningless.
+ # No TTL: a disk entry follows the cache beside it, which is capped by file count rather than age.
+ # time.monotonic() from a previous process is meaningless.
"created": time.monotonic(),
"policy": policy if isinstance(policy, dict) else None,
}
def _evict_cache() -> None:
- # Capped per kind. Metadata is written for every registered image but bytes only
- # for the ones actually viewed, so the sidecars outnumber the JPEGs and need their
- # own bound; and an evicted JPEG keeps its sidecar, which is what lets it be
- # fetched again rather than 404.
+ # Capped per kind. Metadata is written for every registered image but bytes only for the ones actually viewed, so
+ # the sidecars outnumber the JPEGs and need their own bound; and an evicted JPEG keeps its sidecar, which is what
+ # lets it be fetched again rather than 404.
for pattern in ("*.jpg", "*.json"):
try:
files = sorted(_cache_dir().glob(pattern), key = lambda p: p.stat().st_mtime)
@@ -354,8 +356,6 @@ def _evict_cache() -> None:
def _encode_thumbnail(raw: bytes) -> bytes | None:
from PIL import Image
-
- # Pillow's own process-wide bomb check is left at its default.
try:
with Image.open(io.BytesIO(raw)) as im:
if im.format not in _ALLOWED_FORMATS:
@@ -418,7 +418,6 @@ def _drop_if_cleared(image_id: str) -> bool:
def thumbnail_bytes(image_id: str) -> bytes | None:
- # Cache first: it survives the restart the in-memory registry does not.
if not IMAGE_ID_RE.fullmatch(image_id or ""):
return None
# Ahead of that read and of the sidecar below, which both go around the registry.
@@ -430,17 +429,16 @@ def thumbnail_bytes(image_id: str) -> bytes | None:
return path.read_bytes()
except OSError:
pass
- # Generation and entry in ONE acquisition, generation first. Taking them
- # separately let a clear land in the gap: this call would then read the POST-clear
- # generation, the check before the write would match, and the thumbnail the clear
- # had just deleted would be written back. Reading it first is the safe order --
- # a clear after this point leaves us holding a stale value, which fails the check.
+ # Generation and entry in ONE acquisition, generation first. Taking them separately let a clear land in the gap:
+ # this call would then read the POST-clear generation, the check before the write would match, and the thumbnail the
+ # clear had just deleted would be written back. Reading it first is the safe order -- a clear after this point
+ # leaves us holding a stale value, which fails the check.
with _registry_lock:
generation = _cache_generation
entry = _lookup_locked(image_id)
if entry is None:
- # Not in memory: the process may have restarted since the search. The metadata
- # on disk outlives it, the same way the cached bytes do.
+ # Not in memory: the process may have restarted since the search. The metadata on disk outlives it, the same way
+ # the cached bytes do.
entry = _load_persisted_entry(image_id)
if entry is None:
return None
@@ -460,14 +458,11 @@ def thumbnail_bytes(image_id: str) -> bytes | None:
return None
with _registry_lock:
if _reaped_since_locked(image_id, generation):
- # A clear that covered THIS id landed while the fetch was in flight.
- # The chat that asked for it is gone, so publish nothing and hand back
- # nothing -- writing here would restore a thumbnail the clear had
- # removed, and the cache-first path above would keep serving it.
- #
- # Asked per id, not off the bare generation: a selective clear bumps
- # that too, and aborting on it took down fetches for the images the
- # clear had gone out of its way to spare.
+ # A clear that covered THIS id landed while the fetch was in flight. The chat that asked for it is
+ # gone, so publish nothing and hand back nothing -- writing here would restore a thumbnail the clear
+ # had removed, and the cache-first path above would keep serving it. Asked per id, not off the bare
+ # generation: a selective clear bumps that too, and aborting on it took down fetches for the images
+ # the clear had gone out of its way to spare.
return None
try:
# Writer-unique: racing writers must not publish a torn JPEG.
@@ -480,7 +475,6 @@ def thumbnail_bytes(image_id: str) -> bytes | None:
tmp.unlink(missing_ok = True)
except OSError:
pass
- # Outside the lock: a glob plus a stat per file, and nothing here needs it.
_evict_cache()
return data
finally:
@@ -504,11 +498,7 @@ def registered_image_ids() -> set[str] | None:
try:
ids.update(path.stem for path in _cache_dir().glob(pattern))
except OSError:
- # Unreadable dir, so the snapshot is incomplete and cannot bound a reap.
- # None is the sentinel clear_cache reads as "clear everything", which is
- # what a clear did before this snapshot existed. An empty set is NOT that
- # sentinel: it is a selective reap of nothing, and would leave the
- # registry populated and its images still fetchable after a clear.
+ # an unreadable dir means the snapshot cannot bound a reap
return None
return ids
@@ -543,8 +533,8 @@ def snapshot_and_fence_registrations() -> set[str] | None:
try:
ids.update(path.stem for path in _cache_dir().glob(pattern))
except OSError:
- # Same fallback as registered_image_ids: an incomplete snapshot cannot bound a
- # reap, and None is the sentinel clear_cache reads as "clear everything".
+ # Same fallback as registered_image_ids: an incomplete snapshot cannot bound a reap, and None is the
+ # sentinel clear_cache reads as "clear everything".
return None
return ids
@@ -554,7 +544,6 @@ def _reaped_since_locked(image_id: str, generation: int) -> bool:
if _full_clear_generation > generation:
return True
if generation < _reaped_floor_generation:
- # Older than every record still held, so this cannot be shown to have been spared.
return True
return _reaped_at.get(image_id, 0) > generation
@@ -571,8 +560,8 @@ def clear_cache(only_ids: set[str] | None = None) -> None:
spared image's fetch is left alone. Aborting it would 404 a card that never retries.
"""
global _cache_generation, _full_clear_generation, _reaped_floor_generation
- # The unlinks are under the lock too, so an in-flight fetch cannot slip its write
- # in between the bump and the delete and leave a cleared thumbnail on disk.
+ # The unlinks are under the lock too, so an in-flight fetch cannot slip its write in between the bump and the delete
+ # and leave a cleared thumbnail on disk.
with _registry_lock:
if only_ids is None:
_registry.clear()
@@ -581,18 +570,17 @@ def clear_cache(only_ids: set[str] | None = None) -> None:
_registry.pop(image_id, None)
_cache_generation += 1
if only_ids is None:
- # Nothing survives, so every in-flight fetch has to abort. One number says so
- # for all of them, including ids this process has never seen.
+ # Nothing survives, so every in-flight fetch has to abort. One number says so for all of them, including ids
+ # this process has never seen.
_full_clear_generation = _cache_generation
_reaped_at.clear()
else:
if len(_reaped_at) + len(only_ids) > _REAPED_AT_MAX:
- # Out of room. Drop the OLDEST records rather than promoting this to a
- # full clear: doing that aborts every fetch in flight, including ones for
- # images this clear deliberately spared, and an aborted fetch is not a
- # cheap retry -- the card 404s and useSearchThumbnail never asks again.
- # Raising the floor instead gives up only on fetches older than every
- # record still held, which the fetch timeout makes unreachable in practice.
+ # Out of room. Drop the OLDEST records rather than promoting this to a full clear: doing that aborts
+ # every fetch in flight, including ones for images this clear spared, and an aborted fetch is not a
+ # cheap retry -- the card 404s and useSearchThumbnail never asks again. Raising the floor instead gives
+ # up only on fetches older than every record still held, which the fetch timeout makes unreachable in
+ # practice.
keep_from = sorted(_reaped_at.values())[len(_reaped_at) // 2 :]
floor = keep_from[0] - 1 if keep_from else _cache_generation
for stale_id in [key for key, at in _reaped_at.items() if at <= floor]:
@@ -606,20 +594,16 @@ def clear_cache(only_ids: set[str] | None = None) -> None:
except OSError:
continue
for path in paths:
- # `.tmp` stems carry a writer suffix, so they never match a snapshot id
- # and are always swept: one was never servable, and a torn write left
- # behind by a crashed fetch has no owner to spare it for.
+ # `.tmp` stems carry a writer suffix, so they never match a snapshot id and are always swept: one was
+ # never servable, and a torn write left behind by a crashed fetch has no owner to spare it for.
if only_ids is not None and pattern != "*.tmp" and path.stem not in only_ids:
continue
- # Per file, as _evict_cache does: one that cannot be unlinked --
- # a JPEG another process holds open on Windows -- must not leave
- # every later one on disk, where the cache-first read in
- # thumbnail_bytes would go on serving it after a clear.
+ # per file: one that cannot be unlinked (a JPEG another process holds open on Windows) must not leave
+ # every later one on disk
try:
path.unlink(missing_ok = True)
except OSError:
- # Still on disk, so remember the id and refuse to serve it until
- # the unlink does land. `.jpg`/`.json` share a stem; a `.tmp` was
- # never servable, and its stem carries the writer suffix.
+ # Still on disk, so remember the id and refuse to serve it until the unlink does land.
+ # `.jpg`/`.json` share a stem; a `.tmp` was never servable, and its stem carries the writer suffix.
if pattern != "*.tmp":
_cleared_unservable.add(path.stem)
diff --git a/studio/backend/core/inference/sse_control_frames.py b/studio/backend/core/inference/sse_control_frames.py
index 0fe4b9566d1..f565930ede0 100644
--- a/studio/backend/core/inference/sse_control_frames.py
+++ b/studio/backend/core/inference/sse_control_frames.py
@@ -29,8 +29,8 @@ import json
from typing import Any
-# Top-level "type" values the chat client routes away from the transcript.
-# Anything here paints UI on the user's behalf, so only this server may send it.
+# Top-level "type" values the chat client routes away from the transcript. Anything here paints UI on the user's behalf,
+# so only this server may send it.
_CONTROL_TYPES = frozenset(
{
"tool_start",
@@ -43,12 +43,13 @@ _CONTROL_TYPES = frozenset(
}
)
-# Unsloth extensions carried inside a chunk. Not part of any provider's wire
-# format, and read by the client with the same trust as the frames above.
+# unsloth extensions, in no provider's wire format, read with the same trust as the frames above
+# Unsloth extensions carried inside a chunk. Not part of any provider's wire format, and read by the client with the
+# same trust as the frames above.
_CONTROL_KEYS = ("_toolEvent", "_toolStatus", "_diffusionFrame", "_reasoningDurationMs")
-# What is left of a stripped frame is only worth relaying if it still says
-# something in the provider's own vocabulary.
+# a stripped frame is only worth relaying if it still says something in the provider's vocabulary
+# What is left of a stripped frame is only worth relaying if it still says something in the provider's own vocabulary.
_SUBSTANTIVE_KEYS = ("choices", "usage", "error")
@@ -65,19 +66,17 @@ def _normalize_reasoning_deltas(payload: dict[str, Any]) -> bool:
continue
reasoning = delta.get("reasoning")
if not isinstance(reasoning, str) or not reasoning:
- # An empty alias says nothing, and rewriting it would cost a re-encode.
continue
details = delta.get("reasoning_details")
if isinstance(details, list) and any(
isinstance(part, dict) and isinstance(part.get("text"), str) and part["text"]
for part in details
):
- # OpenRouter repeats the thought in reasoning_details and the client
- # concatenates both; details carrying no text are not a second copy.
+ # OpenRouter repeats the thought in reasoning_details and the client concatenates both; details carrying no
+ # text are not a second copy.
continue
canonical = delta.get("reasoning_content")
if canonical is not None and (not isinstance(canonical, str) or canonical.strip()):
- # A canonical value the provider set wins, unless it is blank.
continue
delta["reasoning_content"] = reasoning
delta.pop("reasoning", None)
@@ -109,8 +108,6 @@ def sanitize_provider_sse_line(line: str) -> str | None:
forged_type = isinstance(payload.get("type"), str) and payload["type"] in _CONTROL_TYPES
forged_keys = [key for key in _CONTROL_KEYS if key in payload]
if not normalized_reasoning and not forged_type and not forged_keys:
- # The overwhelmingly common case: relay the provider's own bytes rather
- # than paying a re-encode to reproduce them.
return line
cleaned: dict[str, Any] = {
@@ -119,7 +116,7 @@ def sanitize_provider_sse_line(line: str) -> str | None:
if key not in forged_keys and not (forged_type and key == "type")
}
if not any(key in cleaned for key in _SUBSTANTIVE_KEYS):
- # A pure control frame with the control stripped out is an empty
- # envelope; relaying it would only make the client parse nothing.
+ # A pure control frame with the control stripped out is an empty envelope; relaying it would only make the
+ # client parse nothing.
return None
return "data: " + json.dumps(cleaned, separators = (",", ":"))
diff --git a/studio/backend/core/inference/stream_errors.py b/studio/backend/core/inference/stream_errors.py
index 99895cc1795..b8a662dad88 100644
--- a/studio/backend/core/inference/stream_errors.py
+++ b/studio/backend/core/inference/stream_errors.py
@@ -26,32 +26,30 @@ the problem.
from typing import Any, Optional
-# Two different failures share the word "context" and need different advice.
-#
-# Starvation: the decode could not find KV space because concurrent generations drew on
-# the same unified cache. Nothing about this request was too big, so telling the user to
-# shorten it is wrong. Matched on a substring because the server appends punctuation and,
-# on some paths, batch details.
+# Starvation: concurrent generations drew on the same unified cache Two different failures share the word "context" and
+# need different advice. Starvation: the decode could not find KV space because concurrent generations drew on the same
+# unified cache. Nothing about this request was too big, so telling the user to shorten it is wrong. Matched on a
+# substring because the server appends punctuation and, on some paths, batch details.
_STARVATION_MARKERS = (
"context size has been exceeded",
"failed to find free space in the kv cache",
"failed to find a memory slot",
)
-# Oversize: the request alone did not fit, and the server says so precisely, including
-# both token counts. That text is kept verbatim; only the remedy is added.
+# oversize: the server's precise text (both token counts) is kept verbatim, only the remedy is added
+# Oversize: the request alone did not fit, and the server says so precisely, including both token counts. That text is
+# kept verbatim; only the remedy is added.
_OVERSIZE_MARKERS = (
"exceeds the available context size",
"exceeds the context size",
)
-# Worded around the chat client's own substring test. `chat-adapter.ts::isContextLimitError`
-# matches "context window" (among others) and shows "Context limit reached: the conversation
-# has filled the model's context window ... or start a new chat" -- the advice this message
-# exists to deny, since nothing about the conversation was too long and a new chat fails
-# identically while the other generation is still running. The backend `code` cannot rescue
-# it: `chat-api.ts` rethrows an in-band error chunk as `new Error(message)` and the text is
-# all the client has. "Context Length" names the setting and is not one of its markers.
+# Worded around the chat client's own substring test. `chat-adapter.ts::isContextLimitError` matches "context window"
+# (among others) and shows "Context limit reached: the conversation has filled the model's context window... or start a
+# new chat" -- the advice this message exists to deny, since nothing about the conversation was too long and a new chat
+# fails identically while the other generation is still running. The backend `code` cannot rescue it: `chat-api.ts`
+# rethrows an in-band error chunk as `new Error(message)` and the text is all the client has. "Context Length" names the
+# setting and is not one of its markers.
KV_STARVATION_MESSAGE = (
"The model ran out of context space while generating. This happens when several "
"chats or research runs generate at the same time, because they all draw on one "
@@ -90,9 +88,9 @@ class LlamaStreamError(RuntimeError):
kv_starvation: bool = False,
context_oversize: bool = False,
):
- # str(exc) stays the server's own text where there is one, so the existing
- # token-count regex in _friendly_error still matches an oversize refusal and
- # rewrites it into the established "Message too long" wording.
+ # keep the server's own text so _friendly_error's token-count regex still rewrites an oversize refusal
+ # str(exc) stays the server's own text where there is one, so the existing token-count regex in _friendly_error
+ # still matches an oversize refusal and rewrites it into the established "Message too long" wording.
super().__init__(server_message or friendly)
self.friendly = friendly
self.server_message = server_message
diff --git a/studio/backend/core/inference/stt_ggml_sidecar.py b/studio/backend/core/inference/stt_ggml_sidecar.py
index 42cca6cdd37..a392f026ae8 100644
--- a/studio/backend/core/inference/stt_ggml_sidecar.py
+++ b/studio/backend/core/inference/stt_ggml_sidecar.py
@@ -75,9 +75,10 @@ from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid
logger = get_logger(__name__)
-# Curated GGML checkpoints, one repo per model. Keys match the Transformers
-# sidecar's ids so the frontend reuses one picker; values are the single file
-# inside each repo.
+# one repo per model; keys match the Transformers sidecar's ids so the frontend reuses one picker, values are the single
+# file inside each repo
+# Curated GGML checkpoints, one repo per model. Keys match the Transformers sidecar's ids so the frontend reuses one
+# picker; values are the single file inside each repo.
GGML_STT_REPOS: dict[str, str] = {
"tiny": "unslothai/whisper-tiny-GGUF",
"base": "unslothai/whisper-base-GGUF",
@@ -187,8 +188,8 @@ def _is_runnable(p: Path) -> bool:
try:
return p.is_file() and (sys.platform == "win32" or os.access(p, os.X_OK))
except OSError:
- # is_file() propagates EACCES: an unreadable install dir must read as
- # engine-unavailable, like a missing one, never a 500 out of stt/status.
+ # is_file() propagates EACCES: an unreadable install dir must read as engine-unavailable, never a 500 out of
+ # stt/status
return False
@@ -247,18 +248,15 @@ def slim_runtime_intact(binary: str) -> bool:
for name in runtime_dirs
)
if intact and marker.get("backend") == "rocm":
- # Membership plus required, not equality: hipBLASLt builds no Tensile
- # kernels for gfx1030 and the rest of RDNA2, so llama's ROCm bundle for
- # those ships libhipblaslt with no hipblaslt/ catalog, and the installer
- # wires only the catalogs the bundle has; demanding both read a correct
- # install as broken (#8364). rocblas stays mandatory (the backend module
- # links librocblas directly) and a name outside the pair still means
- # stale wiring. Windows overlays wire no catalogs, so both sets are
- # empty there and this reduces to the old equality.
+ # Membership plus required, not equality: hipBLASLt builds no Tensile kernels for gfx1030 and the rest of RDNA2,
+ # so llama's ROCm bundle for those ships libhipblaslt with no hipblaslt/ catalog, and the installer wires only
+ # the catalogs the bundle has; demanding both read a correct install as broken (#8364). rocblas stays mandatory
+ # (the backend module links librocblas directly) and a name outside the pair still means stale wiring. Windows
+ # overlays wire no catalogs, so both sets are empty there and this reduces to the old equality.
known_runtime_dirs = set() if sys.platform == "win32" else {"hipblaslt", "rocblas"}
required_runtime_dirs = set() if sys.platform == "win32" else {"rocblas"}
- # Any wiring from version 2 on records linked_runtime_directories, so
- # pin the floor, not one version, or an installer bump strands installs.
+ # Any wiring from version 2 on records linked_runtime_directories, so pin the floor, not one version, or an
+ # installer bump strands installs.
wiring_version = marker.get("runtime_wiring_version")
intact = (
isinstance(wiring_version, int)
@@ -275,13 +273,12 @@ def slim_runtime_intact(binary: str) -> bool:
return intact
-# A runtime that starts, answers GET /, and then dies on the first actual inference.
-# Reported on Windows with ROCm on gfx1200, where rocBLAS is missing its TensileLibrary:
-# the marker and the linked libraries are all present, so slim_runtime_intact() is happy
-# and is_available() said yes, which meant _resolve_serving_stt_engine never fell back and
-# every recording 501'd while the UI showed the model as loaded. Only inference can prove
-# this, so it is recorded when inference fails and cleared when one succeeds. Process
-# lifetime by design: a reinstall restarts Unsloth.
+# a runtime that starts, answers GET /, then dies on the first inference
+# A runtime that starts, answers GET /, and then dies on the first actual inference. Reported on Windows with ROCm on
+# gfx1200, where rocBLAS is missing its TensileLibrary: the marker and the linked libraries are all present, so
+# slim_runtime_intact() is happy and is_available() said yes, which meant _resolve_serving_stt_engine never fell back
+# and every recording 501'd while the UI showed the model as loaded. Only inference can prove this, so it is recorded
+# when inference fails and cleared when one succeeds. Process lifetime by design: a reinstall restarts Unsloth.
_runtime_inference_failure: Optional[str] = None
_runtime_failure_lock = threading.Lock()
@@ -340,19 +337,19 @@ def ensure_engine_available() -> str:
return binary
-# ---------------------------------------------------------------------------
-# whisper-server child-process environment
-# ---------------------------------------------------------------------------
-# Build the whisper-server env: prepend the binary dir (co-located libs win, and
-# a backstop where the loader ignores the rpath) and scrub secret-bearing vars the
-# binary never needs. On WSL2 ROCm the system HIP libs go first, since a bundle's
-# bare-metal HIP cannot drive /dev/dxg. A CUDA bundle ships libggml-cuda.so but not
-# libcudart/libcublas (paired with the user's PyTorch), so add the
-# CUDA-from-PyTorch runtime dirs the selection gated on, else the backend cannot
-# resolve a runtime that lives only in wheels. Mirrors llama's binary_env(); the
-# scrub/WSL/dedupe helpers live in utils.prebuilt.
+# build the whisper-server env: prepend the binary dir (co-located libs win, and a backstop where the loader ignores the
+# rpath)
+# --------------------------------------------------------------------------- whisper-server child-process environment
+# --------------------------------------------------------------------------- Build the whisper-server env: prepend the
+# binary dir (co-located libs win, and a backstop where the loader ignores the rpath) and scrub secret-bearing vars the
+# binary never needs. On WSL2 ROCm the system HIP libs go first, since a bundle's bare-metal HIP cannot drive /dev/dxg.
+# A CUDA bundle ships libggml-cuda.so but not libcudart/libcublas (paired with the user's PyTorch), so add the
+# CUDA-from-PyTorch runtime dirs the selection gated on, else the backend cannot resolve a runtime that lives only in
+# wheels. Mirrors llama's binary_env(); the scrub/WSL/dedupe helpers live in utils.prebuilt.
# Module-level aliases keep the historical patch points for tests and callers.
+# ---------------------------------------------------------------------------
+
_wsl_system_rocm_lib_dirs = wsl_system_rocm_lib_dirs
_dedupe_existing_dirs = dedupe_existing_dirs
@@ -365,9 +362,8 @@ def _whisper_server_child_env(binary: str) -> dict[str, str]:
env = scrub_env(os.environ)
isolate_home(env, str(_managed_whisper_cpp_dir() / ".child_home"))
bin_dir = str(Path(binary).parent)
- # A CUDA bundle needs the CUDA-from-PyTorch wheel dirs so libcudart/libcublas
- # resolve at launch when they live only in site-packages/nvidia/*/lib. Placed
- # after bin_dir so co-located libs still win; empty for other bundles.
+ # A CUDA bundle needs the CUDA-from-PyTorch wheel dirs so libcudart/libcublas resolve at launch when they live only
+ # in site-packages/nvidia/*/lib. Placed after bin_dir so co-located libs still win; empty for other bundles.
cuda_runtime_dirs: list[str] = []
bundle_dir = Path(bin_dir)
has_cuda_module = any(
@@ -396,8 +392,8 @@ def _whisper_server_child_env(binary: str) -> dict[str, str]:
return env
-# ---------------------------------------------------------------------------
-# Model file download (single files; deliberately outside the Model Hub flow)
+# --------------------------------------------------------------------------- Model file download (single files;
+# deliberately outside the Model Hub flow) ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
@@ -468,9 +464,10 @@ class _GgmlDownloadState:
"model": self._model_id if downloading else None,
"error": self._error,
"cancelled": self._cancelled,
- # Which model the cancel applies to. "model" goes None once the worker
- # thread stops, so a settled cancellation was indistinguishable from an
- # unrelated one and a deferred load restarted the whole download.
+ # "model" goes None once the worker thread stops
+ # Which model the cancel applies to. "model" goes None once the worker thread stops, so a settled
+ # cancellation was indistinguishable from an unrelated one and a deferred load restarted the whole
+ # download.
"cancelled_model": self._model_id if self._cancelled else None,
"bytes_total": self._total_bytes if downloading else None,
}
@@ -543,7 +540,6 @@ class _GgmlDownloadState:
with self._lock:
if self._thread is not None and self._thread.is_alive():
if self._model_id == model_id:
- # Joining a cancelling run would silently download nothing.
if not self._cancelled:
return
raise SttModelIdError(
@@ -600,8 +596,8 @@ class _GgmlDownloadState:
raise RuntimeError("could not resolve the GGML blob identity")
if total_bytes <= 0:
raise RuntimeError("could not resolve the GGML file size")
- # A cancel during metadata has no child to stop. Without these the
- # run still reserves the repo and rewrites the cache after the stop.
+ # A cancel during metadata has no child to stop. Without these the run still reserves the repo and rewrites
+ # the cache after the stop.
with self._lock:
if self._cancelled:
return
@@ -614,8 +610,7 @@ class _GgmlDownloadState:
self._total_bytes = total_bytes
self._etag = etag
self._revision = revision
- # Out of process so cancel() can terminate it; a thread blocked in
- # hf_hub_download could not be interrupted.
+ # out of process so cancel() can terminate it; a thread blocked in hf_hub_download could not be interrupted
from core.inference.stt_download_worker import (
reap_download,
spawn_download,
@@ -637,11 +632,10 @@ class _GgmlDownloadState:
)
with self._lock:
if self._cancelled:
- # cancel() landed between start() and the spawn.
terminate_download(process)
self._process = process
- # reap_download(), not communicate(): only it drops the adopted PID,
- # which could otherwise be reused and then signalled by terminate_all.
+ # reap_download(), not communicate(): only it drops the adopted PID, which could otherwise be reused and
+ # then signalled by terminate_all.
stderr = reap_download(process)
with self._lock:
if self._process is process:
@@ -693,8 +687,6 @@ def cancel_model_download() -> bool:
# ---------------------------------------------------------------------------
-# WAV packaging
-# ---------------------------------------------------------------------------
def _pcm_to_wav_bytes(decoded_audio) -> bytes:
@@ -713,8 +705,6 @@ def _pcm_to_wav_bytes(decoded_audio) -> bytes:
# ---------------------------------------------------------------------------
-# Sidecar
-# ---------------------------------------------------------------------------
class GgmlSttSidecar:
@@ -729,29 +719,24 @@ class GgmlSttSidecar:
self._idle_timer: Optional[threading.Timer] = None
self._idle_generation = 0
self._keep_alive_seconds = keep_alive_seconds
- # Set while whisper-server starts so training admission can account for
- # the accelerator memory it is about to bind. Read without the lock.
+ # Set while whisper-server starts so training admission can account for the accelerator memory it is about to
+ # bind. Read without the lock.
self._loading = False
- # A still-starting whisper-server is cancellable so training can preempt
- # it before it binds accelerator memory. Assigned inside self._lock but
- # acted on without it: cancel_pending_load() runs while load() holds the
- # lock, so the event is the source of truth and terminating the process
- # is a best-effort fast path.
+ # A still-starting whisper-server is cancellable so training can preempt it before it binds accelerator memory.
+ # Assigned inside self._lock but acted on without it: cancel_pending_load() runs while load() holds the lock, so
+ # the event is the source of truth and terminating the process is a best-effort fast path.
self._load_cancel_event: Optional[threading.Event] = None
self._load_owner_cancel_event: Optional[threading.Event] = None
self._starting_process: Optional[subprocess.Popen] = None
- # Set before the updater waits for _lock, then kept set while it owns
- # the lock and atomically replaces the managed install tree. New loads
- # fail fast instead of starting a process from files being swapped.
+ # set before the updater waits for _lock and kept set while it atomically replaces the managed install tree
self._update_in_progress = False
@property
def loaded_model(self) -> Optional[str]:
- # Lock-free status read (like stt_sidecar.py): transcribe() holds
- # self._lock for the whole inference call (up to
- # _TRANSCRIBE_TIMEOUT_SECONDS), and status polls plus training admission
- # must not block behind it. _process_alive() snapshots self._process
- # before poll(), which subprocess guards with _waitpid_lock, so a
+ # lock-free status read: transcribe() holds self._lock for the whole inference call
+ # Lock-free status read (like stt_sidecar.py): transcribe() holds self._lock for the whole inference call (up to
+ # _TRANSCRIBE_TIMEOUT_SECONDS), and status polls plus training admission must not block behind it.
+ # _process_alive() snapshots self._process before poll(), which subprocess guards with _waitpid_lock, so a
# concurrent unload is safe.
return self._model_id if self._process_alive() else None
@@ -760,8 +745,6 @@ class GgmlSttSidecar:
return "whisper.cpp" if self._process_alive() else None
def is_loading(self) -> bool:
- # True only while whisper-server is starting (seconds to bind its GPU
- # backend); load() sets and clears the flag around that window.
with self._load_state_lock:
return self._loading
@@ -770,9 +753,8 @@ class GgmlSttSidecar:
return self._keep_alive_seconds
def _process_alive(self) -> bool:
- # Snapshot self._process once: a concurrent unload() nulls it under the
- # lock, so lock-free readers would otherwise re-read None between the
- # truthiness check and .poll().
+ # Snapshot self._process once: a concurrent unload() nulls it under the lock, so lock-free readers would
+ # otherwise re-read None between the truthiness check and .poll().
process = self._process
return process is not None and process.poll() is None
@@ -884,11 +866,9 @@ class GgmlSttSidecar:
self._update_in_progress = False
def cancel_pending_load(self) -> bool:
- # Preempt a starting whisper-server so training does not launch while it
- # binds accelerator memory. load() holds self._lock for the whole startup,
- # so act without the lock: signal abort and terminate the starting
- # process. _wait_for_server observes the event and raises, then load()
- # reaps the process and releases the lock.
+ # Preempt a starting whisper-server so training does not launch while it binds accelerator memory. load() holds
+ # self._lock for the whole startup, so act without the lock: signal abort and terminate the starting process.
+ # _wait_for_server observes the event and raises, then load() reaps the process and releases the lock.
with self._load_state_lock:
event = self._load_cancel_event
if not self._loading or event is None:
@@ -918,9 +898,8 @@ class GgmlSttSidecar:
return True
def wait_for_load_to_settle(self) -> None:
- # load() holds self._lock across startup and cancel cleanup, so acquiring
- # it blocks until a cancelled server is killed, reaped, and its
- # accelerator memory released.
+ # load() holds self._lock across startup and cancel cleanup, so acquiring it blocks until a cancelled server is
+ # killed, reaped, and its accelerator memory released.
with self._lock:
pass
@@ -969,14 +948,12 @@ class GgmlSttSidecar:
command = [binary, "-m", model_path, "--host", "127.0.0.1", "--port", str(port)]
marker = _whisper_install_marker(binary)
if _training_active():
- # Keep whisper.cpp off the accelerator during training (like the
- # Transformers sidecar's CPU choice) so a mid-training dictation
- # cannot reclaim the VRAM training just freed.
+ # Keep whisper.cpp off the accelerator during training (like the Transformers sidecar's CPU choice) so a
+ # mid-training dictation cannot reclaim the VRAM training just freed.
command.append("--no-gpu")
elif marker is not None and marker.get("backend") == "cpu":
- # A deliberate CPU install must stay CPU: the slim wiring links
- # every llama ggml backend (including CUDA/ROCm), so without
- # this flag a cpu-selected install would still grab the GPU.
+ # A deliberate CPU install must stay CPU: the slim wiring links every llama ggml backend (including
+ # CUDA/ROCm), so without this flag a cpu-selected install would still grab the GPU.
command.append("--no-gpu")
logger.info(
"Starting whisper-server for STT model %s on 127.0.0.1:%s",
@@ -994,19 +971,18 @@ class GgmlSttSidecar:
if cancel_event.is_set():
raise SttLoadCancelledError("GGUF STT model loading was cancelled.")
self._release_locked()
- # Release the reservation as late as possible: whisper-server
- # binds the port moments after this close.
+ # release the reservation as late as possible: whisper-server binds the port moments after this close
reservation.close()
process = subprocess.Popen(
command,
stdout = subprocess.DEVNULL,
stderr = subprocess.DEVNULL,
stdin = subprocess.DEVNULL,
- # Co-located GPU libs on the loader path (WSL system HIP first),
- # secrets scrubbed from the downloaded binary's env.
+ # Co-located GPU libs on the loader path (WSL system HIP first), secrets scrubbed from the
+ # downloaded binary's env.
env = _whisper_server_child_env(binary),
- # Die with Unsloth (Linux PDEATHSIG, Windows job) so a crash
- # never orphans a server holding the model.
+ # die with Unsloth (Linux PDEATHSIG, Windows job) so a crash never orphans a server holding the
+ # model
**child_popen_kwargs(),
)
with self._load_state_lock:
@@ -1049,10 +1025,9 @@ class GgmlSttSidecar:
"The local transcription runtime exited before becoming "
"ready; the model file may be corrupt or unsupported."
)
- # Require a whisper-server-specific response twice, with the managed
- # child alive around each probe. An arbitrary local process that won
- # the bind race would otherwise be mistaken for the sidecar and
- # receive the user's microphone audio.
+ # Require a whisper-server-specific response twice, with the managed child alive around each probe. An
+ # arbitrary local process that won the bind race would otherwise be mistaken for the sidecar and receive the
+ # user's microphone audio.
if GgmlSttSidecar._probe_is_whisper_server(process, port) and (
GgmlSttSidecar._probe_is_whisper_server(process, port)
):
@@ -1102,8 +1077,8 @@ class GgmlSttSidecar:
raise SttLanguageError(
f"Language '{language}' is not supported by STT model '{model_id}'."
)
- # Reject a missing model before decoding so a long clip does not burn CPU
- # only to 409 (matches the Transformers sidecar's preflight).
+ # Reject a missing model before decoding so a long clip does not burn CPU only to 409 (matches the Transformers
+ # sidecar's preflight).
self._ensure_model_downloaded(model_id)
decoded_audio = _decode_audio_bounded(audio, cancel_event)
if cancel_event is not None and cancel_event.is_set():
@@ -1198,8 +1173,8 @@ class GgmlSttSidecar:
except (SttAudioDecodeError, SttEngineUnavailableError):
raise
except Exception as exc:
- # A cancel closes this socket deliberately, so it is not evidence of a broken
- # runtime and must not disable the engine.
+ # A cancel closes this socket deliberately, so it is not evidence of a broken runtime and must not disable
+ # the engine.
if cancel_event is None or not cancel_event.is_set():
note_runtime_inference_failure(f"{type(exc).__name__}: {exc}")
raise SttEngineUnavailableError(
diff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.py
index aed614b9523..059c2211b36 100644
--- a/studio/backend/core/inference/stt_mtmd_sidecar.py
+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py
@@ -59,8 +59,8 @@ from core.inference.stt_sidecar import (
logger = get_logger(__name__)
-# A blocking unload comes from training claiming the VRAM. Bounded so training is not
-# stalled by a long recording, but long enough that a normal transcription finishes.
+# A blocking unload comes from training claiming the VRAM. Bounded so training is not stalled by a long recording, but
+# long enough that a normal transcription finishes.
_ACTIVE_REQUEST_DRAIN_TIMEOUT = 30.0
@@ -90,19 +90,18 @@ MTMD_STT_MODELS: dict[str, MtmdSttModel] = {
transcript_marker = "<asr_text>",
),
}
-# Voxtral Mini is left out: in chat mode it answers the audio instead of
-# transcribing it, and drops sentences when it complies. Parakeet and Nemotron
-# ASR are too: llama.cpp has the audio graphs but not the text architectures.
+# Voxtral Mini is left out: in chat mode it answers the audio instead of transcribing it, and drops sentences when it
+# complies. Parakeet and Nemotron ASR are too: llama.cpp has the audio graphs but not the text architectures.
_TRANSCRIBE_PROMPT = "Transcribe the audio."
-# Output cap per second of audio. Speech runs about 3 tokens a second in
-# English and more in scripts with no word boundaries, so this is deliberately
-# generous: generation stops at EOS long before it, and the cap only exists so
-# a looping model cannot run to the request timeout.
+# Speech runs about 3 tokens a second in English and more in scripts with no word boundaries Output cap per second of
+# audio. Speech runs about 3 tokens a second in English and more in scripts with no word boundaries, so this is
+# generous: generation stops at EOS long before it, and the cap only exists so a looping model cannot run to the request
+# timeout.
_TRANSCRIPT_TOKENS_PER_SECOND = 30
_MIN_TRANSCRIPT_TOKENS = 512
-# Well under any of these models' trained context, which also has to hold the
-# audio. llama-server is left on its default context (loaded from the model).
+# Well under any of these models' trained context, which also has to hold the audio. llama-server is left on its default
+# context (loaded from the model).
_MAX_TRANSCRIPT_TOKENS = 16384
@@ -145,8 +144,7 @@ def is_available() -> bool:
try:
import av # noqa: F401
except Exception:
- # No PyAV means every transcription 501s on decode, so offering a
- # multi-gigabyte download here would be a waste.
+ # No PyAV means every transcription 501s on decode, so offering a multi-gigabyte download here would be a waste.
return False
return True
@@ -186,8 +184,7 @@ def _reap(process: Optional[subprocess.Popen]) -> None:
except Exception as exc: # noqa: BLE001 - shutdown must not raise
logger.warning("Could not reap llama-server (pid %s): %s", process.pid, exc)
finally:
- # The PID is dead, so drop it before it can be reused by something else
- # that terminate_all would then signal.
+ # the PID is dead, so drop it before it can be reused by something else that terminate_all would then signal
forget_pid(process.pid)
@@ -262,8 +259,8 @@ def _cached_model_paths(
return None
-# The panel polls every model every 750ms, and each answer is two hf_hub_download()
-# calls that stat the snapshot even local-only, so memoise the boolean briefly.
+# The panel polls every model every 750ms, and each answer is two hf_hub_download() calls that stat the snapshot even
+# local-only, so memoise the boolean briefly.
_DOWNLOADED_PROBE_TTL_SECONDS = 2.0
_downloaded_probe_lock = threading.Lock()
_downloaded_probe: dict[str, tuple[float, bool]] = {}
@@ -322,9 +319,10 @@ class _MtmdDownloadState:
"model": model_id if downloading else None,
"error": self._error,
"cancelled": self._cancelled,
- # Which model the cancel applies to. "model" goes None once the worker
- # thread stops, so a settled cancellation was indistinguishable from an
- # unrelated one and a deferred load restarted the whole download.
+ # "model" goes None once the worker thread stops
+ # Which model the cancel applies to. "model" goes None once the worker thread stops, so a settled
+ # cancellation was indistinguishable from an unrelated one and a deferred load restarted the whole
+ # download.
"cancelled_model": self._model_id if self._cancelled else None,
"bytes_total": self._total_bytes if downloading else None,
}
@@ -399,7 +397,6 @@ class _MtmdDownloadState:
with self._lock:
if self._thread is not None and self._thread.is_alive():
if self._model_id == model_id:
- # Joining a cancelling run would silently download nothing.
if not self._cancelled:
return
raise SttModelIdError(
@@ -460,8 +457,8 @@ class _MtmdDownloadState:
blob_key = meta.etag,
)
)
- # A cancel during metadata has no child to stop. Without these the
- # run still reserves the repo and rewrites the cache after the stop.
+ # A cancel during metadata has no child to stop. Without these the run still reserves the repo and rewrites
+ # the cache after the stop.
with self._lock:
if self._cancelled:
return
@@ -554,11 +551,11 @@ class MtmdSttSidecar:
self._model_id: Optional[str] = None
self._binary_path_revision: Optional[int] = None
self._loading = False
- # Published as soon as Popen returns, so a startup can be preempted
- # before _process is assigned (readiness takes up to three minutes).
+ # published as soon as Popen returns, so a startup can be preempted before _process is assigned (readiness takes
+ # up to three minutes)
self._starting_process: Optional[subprocess.Popen] = None
- # Whether the resident server was launched with the GPU pinned off for
- # training. Kept so a dictation after the run does not stay on CPU.
+ # Whether the resident server was launched with the GPU pinned off for training. Kept so a dictation after the
+ # run does not stay on CPU.
self._gpu_disabled = False
self._load_cancel_event: Optional[threading.Event] = None
self._load_owner_cancel_event: Optional[threading.Event] = None
@@ -566,21 +563,19 @@ class MtmdSttSidecar:
self._keep_alive_seconds = keep_alive_seconds
self._idle_timer: Optional[threading.Timer] = None
self._generation = 0
- # Transcription runs outside _lock and can outlast the keep-alive, so
- # the idle timer stays disarmed while any request is in flight.
+ # Transcription runs outside _lock and can outlast the keep-alive, so the idle timer stays disarmed while any
+ # request is in flight.
self._active_requests = 0
@property
def loaded_model(self) -> Optional[str]:
- # Lock-free: _lock is held across reaps and llama.cpp installs, but the status
- # route reads this on the event loop. _process_alive() snapshots _process before
- # poll(), so a concurrent unload is safe.
+ # Lock-free: _lock is held across reaps and llama.cpp installs, but the status route reads this on the event
+ # loop. _process_alive() snapshots _process before poll(), so a concurrent unload is safe.
return self._model_id if self._process_alive() else None
@property
def device(self) -> Optional[str]:
- # Derived, not a second probe: two probes can straddle the publish and
- # report a device with no model.
+ # derived, not a second probe: two probes can straddle the publish and report a device with no model
return "llama.cpp" if self.loaded_model else None
def is_loading(self) -> bool:
@@ -621,8 +616,8 @@ class MtmdSttSidecar:
self._generation += 1
process = self._process
try:
- # Reap before clearing: a lock-free reader mid-reap must still see the
- # model, or training starts while the dying server still holds its VRAM.
+ # Reap before clearing: a lock-free reader mid-reap must still see the model, or training starts while the
+ # dying server still holds its VRAM.
_reap(process)
finally:
self._process = None
@@ -671,25 +666,23 @@ class MtmdSttSidecar:
"""
if not wait and (self.is_loading() or self._active_requests):
return
- # A blocking caller is training claiming the VRAM, so this cannot wait forever, but
- # `wait=True` still must not kill llama-server under a live transcription and throw
- # the recording away. Bounded window, then proceed.
+ # A blocking caller is training claiming the VRAM, so this cannot wait forever, but `wait=True` still must not
+ # kill llama-server under a live transcription and throw the recording away. Bounded window, then proceed.
drain_deadline = time.monotonic() + _ACTIVE_REQUEST_DRAIN_TIMEOUT
if wait:
self._drain_active_requests(drain_deadline)
- # A startup has not assigned _process yet, so releasing alone would let
- # it finish and republish the model that was just unloaded. Cancel and
- # settle outside _lock: load() holds _start_lock across startup and
- # takes _lock inside it, so holding _lock here would invert them.
+ # A startup has not assigned _process yet, so releasing alone would let it finish and republish the model that
+ # was just unloaded. Cancel and settle outside _lock: load() holds _start_lock across startup and takes _lock
+ # inside it, so holding _lock here would invert them.
self.cancel_pending_load()
self.wait_for_load_to_settle()
while True:
if not self._lock.acquire(blocking = wait):
return
try:
- # Recheck under the lock. `transcribe` claims _active_requests while holding
- # it, so a request starting between the drain above and this acquire would
- # otherwise have llama-server killed underneath it and lose the recording.
+ # Recheck under the lock. `transcribe` claims _active_requests while holding it, so a request starting
+ # between the drain above and this acquire would otherwise have llama-server killed underneath it and
+ # lose the recording.
if not self._holds_expected_model(expected_model):
return
busy = bool(self._active_requests)
@@ -707,9 +700,8 @@ class MtmdSttSidecar:
return
finally:
self._lock.release()
- # Drained outside the lock, then the release is retried under it. A blocking
- # unload that found the sidecar busy only after acquiring cannot drain in
- # place without deadlocking the request it is draining.
+ # Drained outside the lock, then the release is retried under it. A blocking unload that found the sidecar
+ # busy only after acquiring cannot drain in place without deadlocking the request it is draining.
self._drain_active_requests(drain_deadline)
def cancel_pending_load(self) -> bool:
@@ -811,8 +803,7 @@ class MtmdSttSidecar:
path_revision = custom_llama_cpp_path_revision()
binary = ensure_engine_available()
- # Startup happens outside _lock (it is slow), so this keeps two callers
- # from each spawning a server and orphaning the first.
+ # startup happens outside _lock, so this keeps two callers from each spawning a server and orphaning the first
with self._start_lock:
if request_cancel_event is not None and request_cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
@@ -842,18 +833,15 @@ class MtmdSttSidecar:
and self._model_id == model_id
and self._binary_path_revision == path_revision
):
- # Same model, so only the offload mode can differ: a server
- # started at -ngl 0 during training would otherwise serve every
- # later dictation on CPU. Restarting for that is an
- # optimisation, never worth killing a running transcription
- # for, so an in-flight request keeps the server it has and the
- # next idle load picks the GPU back up.
+ # Same model, so only the offload mode can differ: a server started at -ngl 0 during training would
+ # otherwise serve every later dictation on CPU. Restarting for that is an optimisation, never worth
+ # killing a running transcription for, so an in-flight request keeps the server it has and the next idle
+ # load picks the GPU back up.
if self._gpu_disabled == training or self._active_requests:
self._schedule_idle_unload_locked()
return
- # Announced before the slow probe and reap: is_loading() is read lock-free,
- # so a training start would otherwise see False and wait out the startup in
- # unload() instead of cancelling this load.
+ # Announced before the slow probe and reap: is_loading() is read lock-free, so a training start would
+ # otherwise see False and wait out the startup in unload() instead of cancelling this load.
cancel_event = (
request_cancel_event if request_cancel_event is not None else threading.Event()
)
@@ -864,11 +852,11 @@ class MtmdSttSidecar:
try:
if cancel_event.is_set():
raise SttLoadCancelledError("Dictation model loading was cancelled.")
- # Before the release: a 409 for a model that is not downloaded
- # must not cost the user the server they were already using.
+ # before the release: a 409 for a model that is not downloaded must not cost the user the server they
+ # were already using
model_path, mmproj_path = self._ensure_model_downloaded(model_id)
- # Only when there is a live server to protect: a request against
- # a server that already died must not block recovery.
+ # Only when there is a live server to protect: a request against a server that already died must not
+ # block recovery.
if self._active_requests and self._process_alive():
raise SttModelBusyError(
"A transcription is still running on the current dictation model. "
@@ -882,11 +870,9 @@ class MtmdSttSidecar:
self._loading = False
self._load_cancel_event = None
self._load_owner_cancel_event = None
- # Re-read last: _release_locked() reaps the old server, which can
- # take seconds, and training admission that already passed its own
- # check cannot come back to cancel this load. Publishing _loading
- # first covers the other order, so between them every training start
- # either cancels this load or is seen by it.
+ # Re-read last: _release_locked() reaps the old server, which can take seconds, and training admission that
+ # already passed its own check cannot come back to cancel this load. Publishing _loading first covers the
+ # other order, so between them every training start either cancels this load or is seen by it.
training = _training_active()
try:
sock, port = self._reserve_free_port()
@@ -900,47 +886,43 @@ class MtmdSttSidecar:
"127.0.0.1",
"--port",
str(port),
- # One short request at a time, so one slot keeps the footprint
- # down next to a loaded chat model.
+ # one short request at a time, so one slot keeps the footprint down next to a loaded chat model
"--parallel",
"1",
- # Keep off the accelerator during training (as whisper.cpp does)
- # so a dictation load cannot reclaim VRAM training just freed.
+ # keep off the accelerator during training so a dictation load cannot reclaim VRAM training just freed
"-ngl",
"0" if training else "99",
]
if training:
- # -ngl 0 covers the main model only. clip.cpp offloads the
- # projector on its own flag, which is what the chat backend's
- # _cmd_has_gpu_companion() treats as a GPU companion whatever
- # --gpu-layers says.
+ # -ngl 0 covers the main model only. clip.cpp offloads the projector on its own flag, which is what the
+ # chat backend's _cmd_has_gpu_companion() treats as a GPU companion whatever --gpu-layers says.
cmd.append("--no-mmproj-offload")
sock.close()
process = subprocess.Popen(
cmd,
- # Nothing reads these, and an undrained pipe blocks llama-server
- # mid-startup once its logs fill the buffer.
+ # nothing reads these, and an undrained pipe blocks llama-server mid-startup once its logs fill the
+ # buffer
stdout = subprocess.DEVNULL,
stderr = subprocess.DEVNULL,
stdin = subprocess.DEVNULL,
- # Bundled libs and pip CUDA runtimes on the loader path, secrets
- # scrubbed, as the chat backend spawns the same binary.
+ # bundled libs and pip CUDA runtimes on the loader path, secrets scrubbed, as the chat backend spawns
+ # the same binary
env = _llama_server_child_env(binary),
# Die with Unsloth, so a crash never orphans a server on the GPU.
**child_popen_kwargs(),
)
- # Published before the wait, so training can preempt a startup that
- # is already allocating; _process is not set for another 180s.
+ # Published before the wait, so training can preempt a startup that is already allocating; _process is not
+ # set for another 180s.
with self._lock:
self._starting_process = process
adopt_pid(process.pid) # terminate_all backstop for graceful exits
if not self._wait_for_server(process, port, cancel_event):
- # Reap it here: _process was never assigned, so unload() cannot
- # reach a child that ignores SIGTERM and keeps port and VRAM.
+ # Reap it here: _process was never assigned, so unload() cannot reach a child that ignores SIGTERM and
+ # keeps port and VRAM.
_reap(process)
if cancel_event.is_set():
- # 409 through the route, like the other sidecars: expected
- # preemption, not a broken or missing runtime (501).
+ # 409 through the route, like the other sidecars: expected preemption, not a broken or missing
+ # runtime (501).
raise SttLoadCancelledError(
"Dictation model loading was cancelled so training could start."
)
@@ -1000,11 +982,10 @@ class MtmdSttSidecar:
model_id = resolve_mtmd_model_id(model)
if cancel_event is not None and cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
- # No training guard here on purpose: load() starts the server with
- # -ngl 0 --no-mmproj-offload while a run is active, so this transcribes
- # on CPU exactly as whisper.cpp and Transformers do. Refusing after a
- # preload that succeeded only discarded the user's recording.
- # Reject a missing model before decoding, matching the other sidecars.
+ # No training guard here on purpose: load() starts the server with -ngl 0 --no-mmproj-offload while a run is
+ # active, so this transcribes on CPU exactly as whisper.cpp and Transformers do. Refusing after a preload that
+ # succeeded only discarded the user's recording. Reject a missing model before decoding, matching the other
+ # sidecars.
self._ensure_model_downloaded(model_id)
decoded_audio = _decode_audio_bounded(audio, cancel_event)
if cancel_event is not None and cancel_event.is_set():
@@ -1016,22 +997,20 @@ class MtmdSttSidecar:
port = self._port
if port is None or not self._process_alive():
raise SttUnavailableError("The dictation server is not running.")
- # Another client can switch models in the gap between that load
- # returning and this lock, and the port read here would then be its
- # server. Refuse rather than transcribe on the wrong model.
+ # Another client can switch models in the gap between that load returning and this lock, and the port read
+ # here would then be its server. Refuse rather than transcribe on the wrong model.
if self._model_id != model_id:
raise SttModelBusyError(
"The dictation model changed while this recording was being "
"prepared. Try again."
)
- # Long audio can outlast the keep-alive, and _post_transcribe runs
- # outside the lock, so disarm the timer rather than let it kill
- # llama-server mid-request and throw the dictation away.
+ # Long audio can outlast the keep-alive, and _post_transcribe runs outside the lock, so disarm the timer
+ # rather than let it kill llama-server mid-request and throw the dictation away.
self._active_requests += 1
self._cancel_idle_unload_locked()
try:
- # Outside the lock: a held lock would block unload, including the
- # one a training run performs, for the whole request timeout.
+ # outside the lock: a held lock would block unload, including a training run's, for the whole request
+ # timeout
text = self._post_transcribe(
port, model_id, wav_bytes, audio_seconds, cancel_event = cancel_event
)
diff --git a/studio/backend/core/inference/stt_registry.py b/studio/backend/core/inference/stt_registry.py
index eeec72c8b3c..fee8f1c9af1 100644
--- a/studio/backend/core/inference/stt_registry.py
+++ b/studio/backend/core/inference/stt_registry.py
@@ -19,8 +19,8 @@ from loggers import get_logger
logger = get_logger(__name__)
-# Every engine a dictation model can be resident on. Order is the order an
-# unload sweeps them, which matters only for logging.
+# Every engine a dictation model can be resident on. Order is the order an unload sweeps them, which matters only for
+# logging.
STT_ENGINES = ("transformers", "gguf", "mtmd")
# Serialises load-then-release so two loads on different engines cannot leave both resident.
@@ -55,11 +55,11 @@ def load(
"""
others = [name for name in STT_ENGINES if name != engine]
with _load_lock:
- # Release the other engines BEFORE allocating, but only once the checkpoint is known
- # to be on disk. Holding two engines across the load is what makes a switch OOM on a
- # device that fits either alone; releasing blind would let a 409 for a model that was
- # never downloaded cost the user the engine they were already using. When the answer
- # is not certain, keep the old order and accept the peak.
+ # release other engines only once the checkpoint is on disk
+ # Release the other engines BEFORE allocating, but only once the checkpoint is known to be on disk. Holding two
+ # engines across the load is what makes a switch OOM on a device that fits either alone; releasing blind would
+ # let a 409 for a model that was never downloaded cost the user the engine they were already using. When the
+ # answer is not certain, keep the old order and accept the peak.
if _model_is_downloaded(engine, model):
unload(others, wait = False)
sidecar_for(engine).load(model, request_cancel_event = request_cancel_event)
diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.py
index a5aa3d00807..76c9ea1a100 100644
--- a/studio/backend/core/inference/stt_sidecar.py
+++ b/studio/backend/core/inference/stt_sidecar.py
@@ -40,8 +40,9 @@ from loggers import get_logger
logger = get_logger(__name__)
-# Multilingual Whisper defaults: stable API/UI id -> Hub repository. A request
-# may instead pass a validated Hugging Face `owner/model` id.
+# stable API/UI id -> Hub repository; a request may instead pass a validated `owner/model` id
+# Multilingual Whisper defaults: stable API/UI id -> Hub repository. A request may instead pass a validated Hugging Face
+# `owner/model` id.
STT_MODELS: dict[str, str] = {
"tiny": "unsloth/whisper-tiny",
"base": "unsloth/whisper-base",
@@ -54,17 +55,14 @@ STT_KEEP_ALIVE_SECONDS = 5 * 60
_HF_REPO_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
_HF_COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$")
-# Bound decoded PCM length so a crafted upload cannot exhaust memory (callers
-# also cap the encoded bytes).
+# Bound decoded PCM length so a crafted upload cannot exhaust memory (callers also cap the encoded bytes).
_MAX_AUDIO_SECONDS = 30 * 60
_TARGET_SAMPLE_RATE = 16000
-# Non-weight files WhisperProcessor/WhisperForConditionalGeneration may load.
-# Weight selection is built from pinned Hub metadata. A custom repo id is
-# attacker-controllable, so only safetensors weights are accepted: a
-# pytorch_model.bin is a pickle and executes code while Transformers
-# deserializes it (see utils/security/file_security.py), and this path skips
-# the malware gate the normal model loader applies.
+# Non-weight files WhisperProcessor/WhisperForConditionalGeneration may load. Weight selection is built from pinned Hub
+# metadata. A custom repo id is attacker-controllable, so only safetensors weights are accepted: a pytorch_model.bin is
+# a pickle and executes code while Transformers deserializes it (see utils/security/file_security.py), and this path
+# skips the malware gate the normal model loader applies.
_STT_SNAPSHOT_SUPPORT_FILES = (
"config.json",
"generation_config.json",
@@ -272,7 +270,6 @@ def _known_whisper_languages() -> Optional[frozenset[str]]:
try:
from transformers.models.whisper.tokenization_whisper import LANGUAGES
except Exception:
- # Transformers unavailable or the constant moved: skip the check.
return None
return frozenset(LANGUAGES)
@@ -548,10 +545,10 @@ def _select_snapshot_files(info, load_index) -> tuple[_SelectedHubFile, ...]:
shards = set(weight_map.values())
if not all(isinstance(shard, str) and shard in siblings for shard in shards):
raise SttModelCompatibilityError(f"Checkpoint index '{index_name}' has missing shards.")
- # The index JSON is attacker-controlled: a safetensors index can name
- # pytorch_model-*.bin shards, which Transformers still loads through
- # torch.load (pickle) since it dispatches per shard by file extension.
- # Require every shard to be safetensors so no pickle file is selected.
+ # The index JSON is attacker-controlled and a safetensors index can name.bin shards The index JSON is
+ # attacker-controlled: a safetensors index can name pytorch_model-*.bin shards, which Transformers still loads
+ # through torch.load (pickle) since it dispatches per shard by file extension. Require every shard to be
+ # safetensors so no pickle file is selected.
if not all(shard.endswith(".safetensors") for shard in shards):
raise SttModelCompatibilityError(
f"Checkpoint index '{index_name}' references non-safetensors shards."
@@ -591,8 +588,10 @@ def validate_remote_model(model: Optional[str], hf_token: Optional[str] = None)
raise SttModelCompatibilityError(
f"Could not resolve an immutable revision for STT model '{model_id}'."
)
- # The commit that was validated; the download pins to it so the repo cannot
- # be swapped between validation and snapshot_download (TOCTOU).
+ # the download pins to the validated commit so the repo cannot be swapped between validation and snapshot_download
+ # (TOCTOU)
+ # The commit that was validated; the download pins to it so the repo cannot be swapped between validation and
+ # snapshot_download (TOCTOU).
return {"model": model_id, "repo": repo, "revision": revision}
@@ -620,14 +619,13 @@ def _snapshot_is_complete(snapshot: Path) -> bool:
tokenizer, and weights directly. is_file() follows cache symlinks, so a
link from an interrupted blob download does not count.
"""
- # Safetensors only: a cached pytorch_model.bin is a pickle load path and is
- # never treated as a usable snapshot (a repo shipping only pickle weights
- # re-resolves and fails closed in _select_snapshot_files).
+ # Safetensors only: a cached pytorch_model.bin is a pickle load path and is never treated as a usable snapshot (a
+ # repo shipping only pickle weights re-resolves and fails closed in _select_snapshot_files).
index = snapshot / _STT_SAFETENSORS_INDEX
if index.is_file():
- # Sharded safetensors checkpoint: every shard must exist and be
- # safetensors (a safe index naming .bin shards would still pickle-load
- # them, matching the _select_snapshot_files guard).
+ # every shard must exist and be safetensors, since a safe index naming .bin shards would still pickle-load them
+ # Sharded safetensors checkpoint: every shard must exist and be safetensors (a safe index naming .bin shards
+ # would still pickle-load them, matching the _select_snapshot_files guard).
weight_map = _read_json_object(index).get("weight_map")
if not isinstance(weight_map, dict) or not weight_map:
return False
@@ -637,8 +635,8 @@ def _snapshot_is_complete(snapshot: Path) -> bool:
has_weights = all((snapshot / shard).is_file() for shard in shards)
else:
has_weights = (snapshot / _STT_SAFETENSORS_WEIGHTS).is_file()
- # WhisperProcessor needs the tokenizer: either the fast tokenizer.json or
- # the slow vocab.json + merges.txt pair.
+ # WhisperProcessor needs either the fast tokenizer.json or the slow vocab.json + merges.txt pair
+ # WhisperProcessor needs the tokenizer: either the fast tokenizer.json or the slow vocab.json + merges.txt pair.
has_tokenizer = (snapshot / "tokenizer.json").is_file() or (
(snapshot / "vocab.json").is_file() and (snapshot / "merges.txt").is_file()
)
@@ -688,9 +686,10 @@ class _SnapshotDownloadState:
"model": self._model_id if downloading else None,
"error": self._error,
"cancelled": self._cancelled,
- # Which model the cancel applies to. "model" goes None once the worker
- # thread stops, so a settled cancellation was indistinguishable from an
- # unrelated one and a deferred load restarted the whole download.
+ # "model" goes None once the worker thread stops
+ # Which model the cancel applies to. "model" goes None once the worker thread stops, so a settled
+ # cancellation was indistinguishable from an unrelated one and a deferred load restarted the whole
+ # download.
"cancelled_model": self._model_id if self._cancelled else None,
"bytes_total": self._total_bytes if show_progress else None,
}
@@ -796,7 +795,6 @@ class _SnapshotDownloadState:
with self._lock:
if self._thread is not None and self._thread.is_alive():
if self._model_id == model_id:
- # Joining a cancelling run would silently download nothing.
if not self._cancelled:
return
raise SttModelIdError(
@@ -850,8 +848,8 @@ class _SnapshotDownloadState:
f"Could not resolve an immutable revision for STT model '{repo}'."
)
- # A cancel during metadata has no child to stop. Without these the
- # run still reserves the repo and rewrites the cache after the stop.
+ # A cancel during metadata has no child to stop. Without these the run still reserves the repo and rewrites
+ # the cache after the stop.
with self._lock:
if self._cancelled:
return
@@ -978,8 +976,8 @@ def _pick_device():
try:
import torch
- # New loads use CPU during training; a resident GPU model may stay put
- # when the training admission check confirms enough headroom.
+ # new loads use CPU during training; a resident GPU model may stay put when the admission check confirms
+ # headroom
training_active = _training_active()
if not training_active and torch.cuda.is_available():
return "cuda", torch.float16
@@ -1088,8 +1086,7 @@ def _decode_audio_bounded(audio: bytes, cancel_event = None):
layout = "mono",
rate = _TARGET_SAMPLE_RATE,
)
- # Group frames before resampling so short clips need one resampler call
- # rather than one per codec frame.
+ # group frames before resampling so short clips need one resampler call rather than one per codec frame
fifo = av.audio.fifo.AudioFifo()
def write_frame(frame) -> None:
@@ -1113,7 +1110,6 @@ def _decode_audio_bounded(audio: bytes, cancel_event = None):
except StopIteration:
break
except InvalidDataError:
- # Skip a corrupt frame rather than fail the whole transcription.
continue
if cancel_event is not None and cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
@@ -1156,17 +1152,17 @@ class WhisperSttSidecar:
self._keep_alive_seconds = max(0.0, keep_alive_seconds)
self._idle_timer: Optional[threading.Timer] = None
self._idle_generation = 0
- # Held only to keep its memory accounted: a worker that outlived its own
- # kill answers nothing, so a later dictation cannot be handed it.
+ # Held only to keep its memory accounted: a worker that outlived its own kill answers nothing, so a later
+ # dictation cannot be handed it.
self._survivor = False
- # A child that outlived the kill start() gave it, handed from _build_model
- # to the load that called it. Written and read under _lock.
+ # A child that outlived the kill start() gave it, handed from _build_model to the load that called it. Written
+ # and read under _lock.
self._start_survivor = None
@property
def loaded_model(self) -> Optional[str]:
- # A worker that died holds nothing, so reporting its model would make
- # training admission reserve memory for a model that is not there.
+ # a worker that died holds nothing, so reporting its model would make training admission reserve memory for a
+ # model that is not there
engine = self._engine
if engine is not None and not _engine_is_alive(engine):
return None
@@ -1174,8 +1170,9 @@ class WhisperSttSidecar:
@property
def device(self) -> Optional[str]:
- # Reported, so name the backend a user recognises. Torch's ROCm build keeps the
- # "cuda" device name for HIP, which made an AMD box report "Transformers - cuda".
+ # torch's ROCm build keeps the "cuda" device name for HIP
+ # Reported, so name the backend a user recognises. Torch's ROCm build keeps the "cuda" device name for HIP,
+ # which made an AMD box report "Transformers - cuda".
return _reported_device(self._device)
def is_loading(self) -> bool:
@@ -1482,8 +1479,8 @@ class WhisperSttSidecar:
self._raise_if_load_cancelled(cancel_event)
device, dtype = _pick_device()
if not self._release_engine_locked():
- # Starting a second child over one that never exited doubles
- # the memory this release was meant to give back.
+ # starting a second child over one that never exited doubles the memory this release was meant to
+ # give back
raise SttModelBusyError(
"The previous dictation worker did not exit and still holds its "
"memory. Try again shortly."
@@ -1512,11 +1509,10 @@ class WhisperSttSidecar:
if device == "cpu":
raise
if self._start_survivor is not None:
- # The attempt left a child that outlived its own kill and still
- # holds the device. A second child would sit beside it, and
- # installing that one would forget this one, which is what lets
- # training be admitted against memory that is not free. Refuse as
- # a release that could not kill its worker does; the timer retries.
+ # The attempt left a child that outlived its own kill and still holds the device. A second child
+ # would sit beside it, and installing that one would forget this one, which is what lets
+ # training be admitted against memory that is not free. Refuse as a release that could not kill
+ # its worker does; the timer retries.
raise SttModelBusyError(
"The previous dictation worker did not exit and still holds its "
"memory. Try again shortly."
@@ -1524,9 +1520,8 @@ class WhisperSttSidecar:
logger.warning("STT load on %s failed (%s); retrying on CPU", device, exc)
retry_on_cpu = True
if retry_on_cpu:
- # Retry outside the handler: live exception state pins frames
- # referencing the partly loaded model, so leave it before
- # clearing the cache to release that memory.
+ # Retry outside the handler: live exception state pins frames referencing the partly loaded model,
+ # so leave it before clearing the cache to release that memory.
_clear_device_cache(device)
try:
candidate = self._build_model(
@@ -1558,44 +1553,39 @@ class WhisperSttSidecar:
logger.info("STT model %s ready on %s", model_id, device)
return self._engine
except SttLoadCancelledError:
- # cancel_pending_load() does not wait for the model lock, so the cancel
- # can land after start() came back with a live child. Nothing installed
- # the candidate, and dropping the handle does not end the process holding
- # the context training is waiting for, so close it here.
+ # cancel_pending_load() does not wait for the model lock, so the cancel can land after start() came back
+ # with a live child. Nothing installed the candidate, and dropping the handle does not end the process
+ # holding the context training is waiting for, so close it here.
if self._start_survivor is not None:
- # start() ends its own child, so this one outlived terminate and kill
- # inside it and never became a candidate. It holds its memory all the
- # same and this is the only handle on the process, so keep it rather
- # than let the cancel report the memory given back; the timer retries.
+ # start() ends its own child, so this one outlived terminate and kill inside it and never became a
+ # candidate. It holds its memory all the same and this is the only handle on the process, so keep it
+ # rather than let the cancel report the memory given back; the timer retries.
self._keep_survivor_locked(self._start_survivor, model_id, device)
_clear_device_cache(device)
raise
if not _close_engine(candidate):
- # It outlived terminate and kill, so it still holds the memory this
- # cancel was made to free. Keep it, for the same reason
- # _release_engine_locked keeps its own survivor: reporting nothing
- # resident is what lets training be admitted against memory that is
- # not free. Nothing is installed over, since a candidate exists only
- # after the resident was released.
+ # It outlived terminate and kill, so it still holds the memory this cancel was made to free. Keep
+ # it, for the same reason _release_engine_locked keeps its own survivor: reporting nothing resident
+ # is what lets training be admitted against memory that is not free. Nothing is installed over,
+ # since a candidate exists only after the resident was released.
self._keep_survivor_locked(candidate, model_id)
candidate = None
_clear_device_cache(device)
raise
candidate = None
if resident_released:
- # _release_engine_locked already collected, and the candidate was dropped
- # before it ran, so nothing has become garbage since. This second call is
- # only here to empty the cache of the device this LOAD picked, which need
- # not be the resident's, so it keeps the sweep and skips the collection.
+ # _release_engine_locked already collected, and the candidate was dropped before it ran, so nothing
+ # has become garbage since. This second call is only here to empty the cache of the device this LOAD
+ # picked, which need not be the resident's, so it keeps the sweep and skips the collection.
self._release_engine_locked()
_clear_device_cache(device, collect = False)
else:
_clear_device_cache(device)
raise
except BaseException:
- # Same reasoning for any other failed load: this is the only handle on a
- # child that outlived start()'s own kill and still holds its memory, and
- # reporting nothing resident lets training be admitted against it.
+ # Same reasoning for any other failed load: this is the only handle on a child that outlived start()'s
+ # own kill and still holds its memory, and reporting nothing resident lets training be admitted against
+ # it.
if self._start_survivor is not None and self._engine is None:
self._keep_survivor_locked(self._start_survivor, model_id, device)
_clear_device_cache(device)
@@ -1629,8 +1619,8 @@ class WhisperSttSidecar:
effective_generate_kwargs = dict(generate_kwargs)
generation_config = getattr(engine, "generation_config", None)
if getattr(generation_config, "is_multilingual", None) is False:
- # English-only checkpoints fix language and task in their generation
- # config, and Transformers rejects passing them here.
+ # English-only checkpoints fix language and task in their generation config, and Transformers rejects
+ # passing them here.
effective_generate_kwargs.pop("task", None)
effective_generate_kwargs.pop("language", None)
window = 30 * _TARGET_SAMPLE_RATE
@@ -1660,15 +1650,13 @@ class WhisperSttSidecar:
Accepts any container PyAV can decode: wav, mp3, opus/webm, ogg,
m4a/aac. Returns {text, language, duration, model}.
"""
- # Reject a missing runtime up front, before the cache and bounded decode.
ensure_stt_available()
if cancel_event is not None and cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
- # A set language beats auto-detect. API takes BCP-47; Whisper wants short
- # codes like en or fr.
+ # A set language beats auto-detect. API takes BCP-47; Whisper wants short codes like en or fr.
lang = normalize_whisper_language(language)
- # Pin the requested id: another request may switch the resident model
- # mid-transcription, so sidecar state is not this request's identity.
+ # Pin the requested id: another request may switch the resident model mid-transcription, so sidecar state is not
+ # this request's identity.
model_id = resolve_model_id(model)
known_languages = _known_whisper_languages()
if lang is not None and known_languages is not None and lang not in known_languages:
@@ -1683,8 +1671,7 @@ class WhisperSttSidecar:
decoded_audio = _decode_audio_bounded(audio, cancel_event)
if cancel_event is not None and cancel_event.is_set():
raise SttTranscriptionCancelledError("Transcription cancelled.")
- # condition_on_prev_tokens=False stops a fresh clip inheriting prior
- # context, which causes runaway repeats.
+ # condition_on_prev_tokens=False stops a fresh clip inheriting prior context, which causes runaway repeats.
generate_kwargs = {
"task": "transcribe",
"condition_on_prev_tokens": False,
@@ -1695,7 +1682,6 @@ class WhisperSttSidecar:
if fast:
# Short voiced clips: greedy decoding drops beam search for latency.
generate_kwargs["num_beams"] = 1
- # Serialize inference with model switches and unloads.
with self._lock:
try:
if cancel_event is None:
diff --git a/studio/backend/core/inference/stt_transformers_worker.py b/studio/backend/core/inference/stt_transformers_worker.py
index 136821c93b9..6346088c2d0 100644
--- a/studio/backend/core/inference/stt_transformers_worker.py
+++ b/studio/backend/core/inference/stt_transformers_worker.py
@@ -32,24 +32,23 @@ from loggers import get_logger
logger = get_logger(__name__)
-# Spawn, never fork: a forked CUDA context is unusable, and Windows/macOS have no fork.
+# spawn, never fork: a forked CUDA context is unusable, and Windows/macOS have no fork
_CTX = mp.get_context("spawn")
_BACKEND_PATH = str(Path(__file__).resolve().parent.parent.parent)
-# Both bounds only break a hang: a cold-disk large-v3 load and a 30 minute
-# transcription are both legitimately slow.
+# both bounds only break a hang: a cold-disk large-v3 load and a 30 minute transcription are legitimately slow
_LOAD_TIMEOUT_SECONDS = 600.0
_TRANSCRIBE_TIMEOUT_SECONDS = 600.0
-# How long a cancelled command gets before the child is killed. Generation stops
-# within a token, but a load inside from_pretrained never sees the cancel, and
-# training is waiting for that memory.
+# generation stops within a token, but a load inside from_pretrained never sees the cancel
+# How long a cancelled command gets before the child is killed. Generation stops within a token, but a load inside
+# from_pretrained never sees the cancel, and training is waiting for that memory.
_CANCEL_GRACE_SECONDS = 10.0
_SHUTDOWN_TIMEOUT_SECONDS = 10.0
_POLL_SECONDS = 0.1
-# Errors the child may report that the parent must re-raise as themselves; any
-# other failure crosses as a RuntimeError carrying the child's message.
+# Errors the child may report that the parent must re-raise as themselves; any other failure crosses as a RuntimeError
+# carrying the child's message.
_FORWARDED_ERRORS = (
"SttLoadCancelledError",
"SttTranscriptionCancelledError",
@@ -72,8 +71,6 @@ class SttWorkerSpawnError(SttWorkerError):
# ---------------------------------------------------------------------------
-# Child process
-# ---------------------------------------------------------------------------
def _ensure_backend_on_path() -> None:
@@ -114,8 +111,8 @@ def load_whisper(
dtype = getattr(torch, dtype_name, None) or torch.float32
processor = WhisperProcessor.from_pretrained(snapshot_path, local_files_only = True)
_raise_if_cancelled(cancel_event)
- # use_safetensors forces the pickle-free load path even if a pytorch_model.bin
- # reached the cache; the selector and completeness check exclude them upstream.
+ # use_safetensors forces the pickle-free load path even if a pytorch_model.bin reached the cache; the selector and
+ # completeness check exclude them upstream.
model = WhisperForConditionalGeneration.from_pretrained(
snapshot_path, torch_dtype = dtype, local_files_only = True, use_safetensors = True
)
@@ -213,16 +210,13 @@ def run_stt_worker(
except Exception as exc: # noqa: BLE001 - logging setup must not fail dictation
logger.debug("STT worker logging setup failed: %s", exc)
- # Say this interpreter is up before any model work. Without it the parent has
- # only the exit code, and Windows has no signals: a native crash inside the model
- # load ends the child with a positive status there exactly as a child that never
- # bootstrapped does, and reading that crash as a host that cannot spawn moves the
- # same crashing load into the backend, which the backend does not survive.
- #
- # An Event, not a queue message: Queue.put only hands the object to a feeder
- # thread, so a child that faults before that thread drains the buffer never
- # delivers it (measured here, the queued word was lost in 17 of 20 runs). An
- # Event is shared memory, set the moment it returns, and was lost in none.
+ # Say this interpreter is up before any model work. Without it the parent has only the exit code, and Windows has no
+ # signals: a native crash inside the model load ends the child with a positive status there exactly as a child that
+ # never bootstrapped does, and reading that crash as a host that cannot spawn moves the same crashing load into the
+ # backend, which the backend does not survive. An Event, not a queue message: Queue.put only hands the object to a
+ # feeder thread, so a child that faults before that thread drains the buffer never delivers it (measured here, the
+ # queued word was lost in 17 of 20 runs). An Event is shared memory, set the moment it returns, and was lost in
+ # none.
if ready_event is not None:
ready_event.set()
@@ -288,14 +282,11 @@ def run_stt_worker(
except BaseException as exc: # noqa: BLE001 - every failure is reported, then handled
_send(resp_queue, _error_response(exc))
if kind == "load":
- # Nothing is resident after a failed load, and the attempt may
- # already have taken a context; exiting returns it.
+ # nothing is resident after a failed load, and the attempt may already have taken a context
return
# ---------------------------------------------------------------------------
-# Parent process
-# ---------------------------------------------------------------------------
def _raise_worker_error(response: dict) -> None:
@@ -321,18 +312,18 @@ class WhisperWorker:
self._cmd_queue = None
self._resp_queue = None
self._cancel_event = None
- # Set by the child before any model work, so this separates a host that
- # cannot bring a child up from a child that failed at something.
+ # set by the child before any model work, so this separates a host that cannot bring a child up from a child
+ # that failed at something
self._ready_event = None
self._answered = False
- # Set once close() found a child that outlived terminate and kill. The handle
- # is kept so its memory stays accounted, but a child that answered neither
- # signal answers no later command either, and its terminate left the queues
- # liable to corruption, so it must never be handed to a later dictation.
+ # a child that answered neither terminate nor kill answers no later command either
+ # Set once close() found a child that outlived terminate and kill. The handle is kept so its memory stays
+ # accounted, but a child that answered neither signal answers no later command either, and its terminate left
+ # the queues liable to corruption, so it must never be handed to a later dictation.
self.survived_kill = False
self.device: Optional[str] = None
- # Read by the sidecar the way it read the model's, so an English-only
- # checkpoint still drops the task/language kwargs it rejects.
+ # Read by the sidecar the way it read the model's, so an English-only checkpoint still drops the task/language
+ # kwargs it rejects.
self.generation_config = SimpleNamespace(is_multilingual = None)
def start(
@@ -361,8 +352,8 @@ class WhisperWorker:
self._cancel_event = _CTX.Event()
self._ready_event = _CTX.Event()
self._process = _CTX.Process(
- # The shared shim binds the child to this process's lifetime and
- # applies the Hub cache environment before any import.
+ # the shared shim binds the child to this process's lifetime and applies the Hub cache environment
+ # before any import
target = run_without_native_path_secret,
args = ("core.inference.stt_transformers_worker", "run_stt_worker", cache_env),
kwargs = {
@@ -376,7 +367,6 @@ class WhisperWorker:
)
self._process.start()
except Exception as exc: # noqa: BLE001 - any refusal to spawn reads the same
- # Nothing was started, so nothing to kill; raise a class the caller can act on.
self._process = None
self._close_queues()
raise SttWorkerSpawnError(
@@ -507,9 +497,9 @@ class WhisperWorker:
"pid record so the sweep can still reach it",
process.pid,
)
- # Recorded on the handle, not just returned: a cancelled or timed-out
- # command closes the worker from inside _await and raises over this
- # answer, so the sidecar would keep offering the wedged child otherwise.
+ # Recorded on the handle, not just returned: a cancelled or timed-out command closes the worker from
+ # inside _await and raises over this answer, so the sidecar would keep offering the wedged child
+ # otherwise.
self.survived_kill = True
return False
try:
@@ -553,10 +543,6 @@ class WhisperWorker:
if cancel_deadline is None:
cancel_deadline = time.monotonic() + _CANCEL_GRACE_SECONDS
elif time.monotonic() >= cancel_deadline:
- # A load inside from_pretrained never sees the event and the memory
- # is wanted now, so end the process. The grace WAS the graceful
- # shutdown, and a child too busy to read the cancel will not read a
- # shutdown, so terminate rather than wait _SHUTDOWN_TIMEOUT_SECONDS more.
self.close(graceful_timeout = 0.0)
self._raise_cancelled(phase)
try:
@@ -566,9 +552,8 @@ class WhisperWorker:
raise SttWorkerError(self._crash_message(phase))
if time.monotonic() >= deadline:
if cancel_deadline is not None:
- # A cancel landing near the end of the timeout is still a cancel:
- # the caller is owed the phase's own error (409 load, 499
- # transcription), and a child that ignored it ignores a shutdown.
+ # A cancel landing near the end of the timeout is still a cancel: the caller is owed the phase's
+ # own error (409 load, 499 transcription), and a child that ignored it ignores a shutdown.
self.close(graceful_timeout = 0.0)
self._raise_cancelled(phase)
self.close()
diff --git a/studio/backend/core/inference/studio_tool_loop.py b/studio/backend/core/inference/studio_tool_loop.py
index acfa0675644..baf670dce30 100644
--- a/studio/backend/core/inference/studio_tool_loop.py
+++ b/studio/backend/core/inference/studio_tool_loop.py
@@ -105,9 +105,8 @@ _TOOL_TRUNCATED = (
"output limit."
)
-# Card text for a call the controller skipped. The client already painted a card
-# from the provider's own tool_calls delta, so it needs a short result; the long
-# model-facing nudge stays in the conversation.
+# Card text for a call the controller skipped. The client already painted a card from the provider's own tool_calls
+# delta, so it needs a short result; the long model-facing nudge stays in the conversation.
_TOOL_SKIPPED = {
"duplicate": "Unsloth did not run this call because an identical one had already completed.",
"disabled": _TOOL_DISABLED,
@@ -123,16 +122,16 @@ _BUDGET_EXHAUSTED_NUDGE = (
# SSE comment written while a tool blocks, so a proxy cannot idle the stream out.
_SSE_KEEPALIVE = ": keep-alive"
-# Delay before the first approval keepalive so it lands as a separate write and
-# cannot coalesce with the gated card. Without the gap a desktop webview can hold
-# both in one frame and the Allow / Deny buttons appear only after the tool ends.
+# Delay before the first approval keepalive so it lands as a separate write and cannot coalesce with the gated card.
+# Without the gap a desktop webview can hold both in one frame and the Allow / Deny buttons appear only after the tool
+# ends.
_TOOL_APPROVAL_FLUSH_DELAY_S = 0.05
# A stall after a tool ran costs a re-run on retry, so allow one, as locally.
_MAX_POST_TOOL_REPROMPTS = 1
-# Usage sub-objects worth summing rather than overwriting: the frontend reads the
-# cache slice and pricing reads the reasoning slice.
+# Usage sub-objects worth summing rather than overwriting: the frontend reads the cache slice and pricing reads the
+# reasoning slice.
_USAGE_DETAIL_FIELDS = (
"prompt_tokens_details",
"completion_tokens_details",
@@ -160,14 +159,13 @@ def _truncate_for_model(
return text[:limit] + f"{joiner}... [truncated, {len(text) - limit} more characters]"
-# Cap on a call's label. Small next to the result cap: this is the query or the
-# code, not the output.
+# Cap on a call's label.
+# Cap on a call's label. Small next to the result cap: this is the query or the code, not the output.
_HOSTED_ARGUMENT_MAX_CHARS = 2000
-# Provider plumbing hung off ``arguments`` for the frontend and native-history
-# replay, never for the model: Gemini's ``executableCode`` part plus an opaque
-# ``thoughtSignature``, OpenAI's paired reasoning item with its multi-kilobyte
+# Provider plumbing hung off ``arguments`` for the frontend and native-history replay, never for the model: Gemini's
+# ``executableCode`` part plus an opaque ``thoughtSignature``, OpenAI's paired reasoning item with its multi-kilobyte
# ``encrypted_content``. As prose they are base64 cut off mid-token.
_HOSTED_ARGUMENT_PLUMBING_KEYS = frozenset({"google", "_server_tool"})
@@ -251,9 +249,8 @@ def _normalized_call(call: dict[str, Any], fallback_id: str = "") -> dict[str, A
if not isinstance(function, dict):
return None
if not isinstance(call_id, str) or not call_id:
- # The id is Unsloth's correlation key, not the model's contract. Several
- # OpenAI-compatible servers omit it; dropping the call lost a real
- # request with no error, so mint one instead.
+ # The id is Unsloth's correlation key, not the model's contract. Several OpenAI-compatible servers omit it;
+ # dropping the call lost a real request with no error, so mint one instead.
call_id = fallback_id
if not call_id:
return None
@@ -268,11 +265,7 @@ def _normalized_call(call: dict[str, Any], fallback_id: str = "") -> dict[str, A
except (TypeError, ValueError, json.JSONDecodeError):
parsed = {"_raw": arguments}
except RecursionError:
- # Deeply nested but syntactically valid JSON blows the interpreter's
- # stack rather than failing to decode, and RecursionError is not a
- # ValueError. Uncaught it escapes the loop after the provider's delta
- # was already relayed, so the client gets a server error mid-stream
- # instead of a call that was simply refused.
+ # Deeply nested but valid JSON blows the interpreter's stack rather than failing to decode
parsed = {"_raw": arguments}
if not isinstance(parsed, dict):
parsed = {"value": parsed}
@@ -322,15 +315,13 @@ def _tool_names(tools: list[dict[str, Any]] | None) -> set[str]:
class ToolLoopTransport(Protocol):
"""One turn of provider inference, as OpenAI-shaped SSE lines."""
- # Whether the provider may write tool calls as text instead of emitting
- # structured delta.tool_calls. Codex never does; a self-hosted GGUF often does.
+ # Whether the provider may write tool calls as text instead of emitting structured delta.tool_calls. Codex never
+ # does; a self-hosted GGUF often does.
heals_text_tool_calls: bool
- # Whether the transport already stripped Unsloth's control vocabulary from
- # every raw upstream line. A transport that has not is sanitized here; one
- # that has must not be sanitized twice, because by this point its own
- # synthesized frames (a provider-hosted image result, say) are indis-
- # tinguishable from a forged one and would be thrown away.
+ # Whether the transport already stripped Unsloth's control vocabulary from every raw upstream line. A transport that
+ # has not is sanitized here; one that has must not be sanitized twice, because by this point its own synthesized
+ # frames (a provider-hosted image result, say) are indis- tinguishable from a forged one and would be thrown away.
sanitizes_provider_frames: bool
def stream(
@@ -401,7 +392,6 @@ def _split_top_level_json_objects(text: str) -> tuple[list[str], str]:
in_string = False
continue
if depth == 0:
- # Between objects only whitespace, "\r\n" as readily as "\n".
if ch == "{":
depth = 1
start = i
@@ -426,10 +416,8 @@ def _split_top_level_json_objects(text: str) -> tuple[list[str], str]:
parse_int = str,
)
except (ValueError, TypeError):
- # Balanced but invalid: cutting here would invent a call.
return unsplit
except RecursionError:
- # Valid but too deep to decode, and not a ValueError.
return unsplit
complete.append(segment)
start = -1
@@ -511,9 +499,8 @@ class _Turn:
by_index: dict[Any, dict[str, Any]] = field(default_factory = dict)
order: list[Any] = field(default_factory = list)
- # call key each delta index maps to: the index itself until a second call
- # forks off it, then (index, call_id), or (index, "_split", n) for a call
- # that had no id to fork on and was found on a JSON object boundary.
+ # call key each delta index maps to: the index itself until a second call forks off it, then (index, call_id), or
+ # (index, "_split", n) for a call that had no id to fork on and was found on a JSON object boundary.
open_key_by_index: dict[int, Any] = field(default_factory = dict)
last_index: int | None = None
split_seq: int = 0
@@ -523,20 +510,19 @@ class _Turn:
key_by_call_id: dict[str, Any] = field(default_factory = dict)
# Resumable boundary scan per call, keyed the same as ``by_index``.
scan_by_key: dict[Any, _BoundaryScan] = field(default_factory = dict)
- # Forks whose object never closed: reported only once it does, or a stream
- # cut short after '{"a":1}{' runs the tool on half an argument.
+ # Forks whose object never closed: reported only once it does, or a stream cut short after '{"a":1}{' runs the tool
+ # on half an argument.
open_tail_keys: set[Any] = field(default_factory = set)
- # Metadata from a delta repeating the slot's name. That name is either the
- # call's, resent, or the next call to the same tool announcing itself, and
- # only the object that follows tells them apart.
+ # Metadata from a delta repeating the slot's name. That name is either the call's, resent, or the next call to the
+ # same tool announcing itself, and only the object that follows tells them apart.
pending_extra: dict[Any, dict[str, Any]] = field(default_factory = dict)
round: int = 0
healed: list[dict[str, Any]] = field(default_factory = list)
text: list[str] = field(default_factory = list)
reasoning_extra: dict[str, Any] | None = None
finish_reason: str | None = None
- # Results from tools the PROVIDER ran this turn, keyed by call id so a
- # repeated end event cannot record the same result twice.
+ # Results from tools the PROVIDER ran this turn, keyed by call id so a repeated end event cannot record the same
+ # result twice
hosted_results: dict[str, dict[str, Any]] = field(default_factory = dict)
def note_hosted_tool_event(self, event: Any) -> None:
@@ -564,17 +550,15 @@ class _Turn:
name = event.get("tool_name")
if isinstance(name, str) and name:
entry["name"] = name
- # The operation itself: Gemini's language and code, a search's query.
- # Merged across both halves because OpenAI opens an image generation
- # before it knows the prompt and only names it on the end event.
+ # The operation itself: Gemini's language and code, a search's query. Merged across both halves because OpenAI
+ # opens an image generation before it knows the prompt and only names it on the end event.
arguments = _hosted_arguments_for_model(event.get("arguments"))
if arguments:
merged = dict(entry.get("arguments_obj") or {})
merged.update(arguments)
entry["arguments_obj"] = merged
- # Truncated with the same notice a result gets: Anthropic hands the
- # model's whole tool input through, so a file the code wrote lives
- # here and nowhere else, and a silent cut reads as the whole thing.
+ # Truncated with the same notice a result gets: Anthropic hands the model's whole tool input through, so a
+ # file the code wrote lives here and nowhere else, and a silent cut reads as the whole thing.
entry["arguments"] = _truncate_for_model(
json.dumps(merged, separators = (",", ":")),
_HOSTED_ARGUMENT_MAX_CHARS,
@@ -585,27 +569,24 @@ class _Turn:
return
result = event.get("result")
if isinstance(result, str):
- # The call finished, even if it produced nothing: Gemini reports
- # code that printed nothing as an empty string. Recorded apart from
- # the result so that stays distinguishable from a stream that died
- # after the start. A non-string is a malformed frame, not an outcome.
+ # The call finished, even if it produced nothing: Gemini reports code that printed nothing as an empty
+ # string. Recorded apart from the result so that stays distinguishable from a stream that died after the
+ # start. A non-string is a malformed frame, not an outcome.
entry["ended"] = True
if isinstance(result, str) and result.strip():
if _carries_image_sentinel(result):
- # A Gemini plot with no stdout is nothing BUT the sentinel, so
- # stripping leaves an empty string and the entry looks empty.
+ # A Gemini plot with no stdout is nothing BUT the sentinel, so stripping leaves an empty string and the
+ # entry looks empty.
entry["produced_image"] = True
- # Same normalisation local results get: the frontend sentinels carry
- # a full data URI, and replaying one sends megabytes of base64. The
- # tool's name goes with it, as the local path passes it: only the
- # sandbox tools emit __FILES__, so a fetched page ending in a well
- # formed one keeps that line as the content it is.
+ # Same normalisation local results get: the frontend sentinels carry a full data URI, and replaying one
+ # sends megabytes of base64. The tool's name goes with it, as the local path passes it: only the sandbox
+ # tools emit __FILES__, so a fetched page ending in a well formed one keeps that line as the content it is.
stripped = strip_result_for_model(result, entry.get("name"))
if stripped.strip():
entry["result"] = _truncate_for_model(stripped)
if event.get("image_b64"):
- # image_generation reports an empty result and carries the picture
- # apart. Record that it happened rather than the bytes.
+ # image_generation reports an empty result and carries the picture apart. Record that it happened rather
+ # than the bytes.
entry["produced_image"] = True
def hosted_replay_text(self) -> str:
@@ -615,16 +596,14 @@ class _Turn:
result = entry.get("result", "")
produced_image = entry.get("produced_image")
if not result and not produced_image and not entry.get("ended"):
- # A start with no outcome says only that something began.
continue
name = entry.get("name") or "tool"
header = f"[{name} result]"
arguments = entry.get("arguments")
if arguments:
header = f"[{name} {arguments}]"
- # A call that ended with nothing to show still has to appear, as the
- # same "(no output)" local tools report, so the model can tell the
- # code ran from it never having run at all.
+ # A call that ended with nothing to show still has to appear, as the same "(no output)" local tools report,
+ # so the model can tell the code ran from it never having run at all.
body = result or ("(produced an image)" if produced_image else "(no output)")
if result and produced_image:
body = f"{result}\n(produced an image)"
@@ -637,44 +616,41 @@ class _Turn:
continue
index = raw_call.get("index")
if not isinstance(index, int) or isinstance(index, bool):
- # A server that stamps index only on the opening fragment sends
- # the argument fragments bare. Treating those as a new call left
- # the real one with empty arguments and ran the tool anyway, so
- # continue the call that is already open instead.
+ # A server that stamps index only on the opening fragment sends the argument fragments bare. Treating
+ # those as a new call left the real one with empty arguments and ran the tool anyway, so continue the
+ # call that is already open instead.
index = self.last_index if self.last_index is not None else len(self.order)
call_id = raw_call.get("id")
- # continue whichever call owns this index now: index restarts at 0
- # for every tool round, so after a fork the bare argument fragments
- # belong to the newer call.
+ # index restarts at 0 for every tool round
+ # continue whichever call owns this index now: index restarts at 0 for every tool round, so after a fork the
+ # bare argument fragments belong to the newer call.
key: Any = self.open_key_by_index.get(index, index)
if isinstance(call_id, str) and call_id:
- # An id beats the latest-index mapping: a fragment repeating
- # one returns to its own call wherever that call now sits.
+ # An id beats the latest-index mapping: a fragment repeating one returns to its own call wherever that
+ # call now sits
owner = self.key_by_call_id.get(call_id)
if owner is not None:
key = owner
elif self.by_index.get(key, {}).get("id"):
- # Two distinct calls reported at the same index. Merging them
- # concatenates their argument JSON into one unparseable blob
- # and loses an intent, so key the second on its own id.
+ # Two distinct calls at the same index: merging concatenates their argument JSON into one
+ # unparseable blob and loses an intent
+ # Two distinct calls reported at the same index. Merging them concatenates their argument JSON into
+ # one unparseable blob and loses an intent, so key the second on its own id.
key = (index, call_id)
- # A closed object takes no more content, so the next arguments to reach the
- # slot belong to the next parallel call. Forking on the accumulated text
- # alone catches this only once they glue on, and a delta carrying an id
- # would claim the finished call and append to it (issue #9807).
+ # A closed object takes no more content, so the next arguments to reach the slot belong to the next parallel
+ # call. Forking on the accumulated text alone catches this only once they glue on, and a delta carrying an
+ # id would claim the finished call and append to it (issue #9807).
held = self.by_index.get(key)
new_function = raw_call.get("function")
new_arguments = (
new_function.get("arguments") if isinstance(new_function, dict) else None
)
new_name = new_function.get("name") if isinstance(new_function, dict) else None
- # A next call opens with the "{" of its own arguments object, so a
- # fragment starting with anything else is not one: whitespace after
- # a closing brace belongs to the object just closed, and forking on
- # a stray scalar suffix would run the tool twice.
- # An id names a call outright, so one naming a DIFFERENT call opens
- # the next even before its arguments arrive. Alone, or repeating
- # the slot's name, it is that call's id stamped late.
+ # A next call opens with the "{" of its own arguments object, so a fragment starting with anything else is
+ # not one: whitespace after a closing brace belongs to the object just closed, and forking on a stray scalar
+ # suffix would run the tool twice. An id names a call outright, so one naming a DIFFERENT call opens the
+ # next even before its arguments arrive. Alone, or repeating the slot's name, it is that call's id stamped
+ # late.
held_name_now = held["function"]["name"] if held is not None else ""
id_names_another_call = bool(
isinstance(call_id, str)
@@ -682,12 +658,12 @@ class _Turn:
and isinstance(new_name, str)
and new_name
and held_name_now
- # Not a prefix test: a catalog holds both "web" and
- # "web_search", so only the same name reads as the same call.
+ # Not a prefix test: a catalog holds both "web" and "web_search", so only the same name reads as the
+ # same call.
and new_name != held_name_now
)
- # A snapshot provider repeats the finished call verbatim once it
- # has an id. Exact repeats only, so a second call still opens.
+ # A snapshot provider repeats the finished call verbatim once it has an id. Exact repeats only, so a second
+ # call still opens.
resends_this_call = bool(
held is not None
and isinstance(call_id, str)
@@ -698,15 +674,14 @@ class _Turn:
and isinstance(new_arguments, str)
and new_arguments == held["function"]["arguments"]
)
- # A name at a closed slot announces the next call: names grow
- # before the arguments, so nothing is left to extend. A shared
- # prefix is no proof ("web" after "web_search" is a second call).
+ # A name at a closed slot announces the next call: names grow before the arguments, so nothing is left to
+ # extend. A shared prefix is no proof ("web" after "web_search" is a second call).
names_next_call = bool(
isinstance(new_name, str)
and new_name
and held_name_now
- # The same name is that call's, resent: llama-server and vLLM
- # both repeat it, and forking runs one request twice.
+ # The same name is that call's, resent: llama-server and vLLM both repeat it, and forking runs one
+ # request twice.
and new_name != held_name_now
)
opens_next_call = (
@@ -714,9 +689,8 @@ class _Turn:
or id_names_another_call
or names_next_call
) and not resends_this_call
- # An announcement has no object to close, so the rule above cannot reach
- # it. A different name bringing an object is the next call; gluing gave
- # "A_longB", which matches no tool.
+ # An announcement has no object to close, so the rule above cannot reach it. A different name bringing an
+ # object is the next call; gluing gave "A_longB", which matches no tool.
announces_over_announcement = (
held is not None
and (held.get("announced_only") is True or held.get("resend_suspect") is True)
@@ -727,8 +701,8 @@ class _Turn:
and isinstance(new_arguments, str)
and new_arguments.strip().startswith("{")
)
- # A fragment repeating the id this slot holds continues it however
- # complete the arguments look, or two calls end up with one id.
+ # A fragment repeating the id this slot holds continues it however complete the arguments look, or two calls
+ # end up with one id.
names_this_call = (
held is not None
and isinstance(call_id, str)
@@ -740,9 +714,8 @@ class _Turn:
closed, unfinished = self._scan(key, held["function"]["arguments"])
slot_is_closed = bool(closed) and not unfinished
extra = raw_call.get("extra_content")
- # A name repeating a closed slot's says nothing new about that call, so
- # metadata riding it belongs to whichever call the next object opens.
- # Merged here it overwrites that call's signature and leaves the next
+ # A name repeating a closed slot's says nothing new about that call, so metadata riding it belongs to
+ # whichever call the next object opens. Merged here it overwrites that call's signature and leaves the next
# unsigned.
extra_is_ambiguous = bool(
slot_is_closed
@@ -765,21 +738,21 @@ class _Turn:
self.last_index = index
self.open_key_by_index[index] = key
if key not in self.by_index:
- # A second call to the same tool can arrive with no name, the
- # first delta having given it. Nameless is dropped, so inherit.
+ # A second call to the same tool can arrive with no name, the first delta having given it. Nameless is
+ # dropped, so inherit.
opening_name = new_name if isinstance(new_name, str) and new_name else held_name_now
self.by_index[key] = {
"id": "",
"type": "function",
"function": {"name": opening_name, "arguments": ""},
- # A name with no `arguments` field is an announcement; a
- # zero-parameter tool sends "" rather than no field.
+ # A name with no `arguments` field is an announcement; a zero-parameter tool sends "" rather than no
+ # field.
"announced_only": new_arguments is None and bool(opening_name),
- # A fork's guess, or a slot the provider opened itself.
- # Only the provider's own announcement runs unfilled.
+ # A fork's guess, or a slot the provider opened itself. Only the provider's own announcement runs
+ # unfilled.
"from_fork": held is not None,
- # A name extending the one this fork left behind is most likely it, resent.
- # It still opens a slot, but gives way rather than gluing "alpha_longbeta".
+ # A name extending the one this fork left behind is most likely it, resent. It still opens a slot,
+ # but gives way rather than gluing "alpha_longbeta".
"resend_suspect": bool(
opening_name
and held_name_now
@@ -788,8 +761,8 @@ class _Turn:
),
}
self.seq_by_key[key] = self._next_seq()
- # First-seen order, so a negative or out-of-order index cannot
- # reorder parallel calls against what the model actually sent.
+ # First-seen order, so a negative or out-of-order index cannot reorder parallel calls against what the
+ # model sent
self.order.append(key)
current = self.by_index[key]
if new_arguments is not None:
@@ -805,25 +778,20 @@ class _Turn:
**extra,
}
elif isinstance(extra, dict) and extra:
- # Gemini 3 stows this call's thoughtSignature here, and the
- # native translator rejects a replayed functionCall without it.
- # Per call, so it cannot ride along on the delta-level slot.
+ # Gemini 3 stows this call's thoughtSignature here, and the native translator rejects a replayed
+ # functionCall without it. Per call, so it cannot ride along on the delta-level slot.
current["extra_content"] = {**current.get("extra_content", {}), **extra}
function = raw_call.get("function")
if isinstance(function, dict):
- # Two provider dialects, and picking either one alone breaks the
- # other. llama-server re-sends the whole name as it grows ("web"
- # then "web_search"), so appending yields "webweb_search".
- # OpenAI streams it in fragments ("web" then "_search"), so
- # assigning yields "_search". Both then fail the enabled-name
- # check and the call silently never runs. A fragment that already
- # starts with what we have is the whole name resent; anything
- # else continues it.
+ # Two provider dialects, and picking either one alone breaks the other. llama-server re-sends the whole
+ # name as it grows ("web" then "web_search"), so appending yields "webweb_search". OpenAI streams it in
+ # fragments ("web" then "_search"), so assigning yields "_search". Both then fail the enabled-name check
+ # and the call silently never runs. A fragment that already starts with what we have is the whole name
+ # resent; anything else continues it.
fragment = function.get("name")
name_before = current["function"]["name"]
- # Never once the object has closed AND a name is set: that puts one tool's
- # arguments under another's name. Naming a call that has none is not a
- # rename, and servers do send the arguments first.
+ # Never once the object has closed AND a name is set: that puts one tool's arguments under another's
+ # name. Naming a call that has none is not a rename, and servers do send the arguments first.
if isinstance(fragment, str) and fragment and not (slot_is_closed and name_before):
if fragment.startswith(name_before):
current["function"]["name"] = fragment
@@ -832,8 +800,8 @@ class _Turn:
if isinstance(function.get("arguments"), str) and not resends_this_call:
current["function"]["arguments"] += function["arguments"]
if not (isinstance(call_id, str) and call_id):
- # The id fork cannot see this: an id-less stream has no ids to differ on,
- # so appending glued two calls into one blob (issue #9807).
+ # The id fork cannot see this: an id-less stream has no ids to differ on, so appending glued two
+ # calls into one blob (issue #9807).
self._fork_glued_arguments(
index,
key,
@@ -871,18 +839,18 @@ class _Turn:
segments = complete + ([tail] if tail else [])
if len(segments) < 2:
return
- # Below rewrites this slot's arguments rather than extending them, so
- # the resumable scan no longer describes the string it was reading.
+ # Below rewrites this slot's arguments rather than extending them, so the resumable scan no longer describes the
+ # string it was reading.
self.scan_by_key.pop(key, None)
- # The slot keeps its first object, name and id. Nothing per-call rides
- # along: two calls claiming one thoughtSignature is a rejected turn.
+ # The slot keeps its first object, name and id. Nothing per-call rides along: two calls claiming one
+ # thoughtSignature is a rejected turn.
current["function"]["arguments"] = segments[0]
- # A name on this delta names the calls it opened, so the slot keeps its
- # own: merging gave "alphagamma", which matches no enabled tool.
+ # A name on this delta names the calls it opened, so the slot keeps its own: merging gave "alphagamma", which
+ # matches no enabled tool.
born_name = incoming_name or name_before
current["function"]["name"] = name_before or born_name
- # This delta's metadata belongs to the call it closes, the last one:
- # Gemini checks the signature against the call it is replayed on.
+ # This delta's metadata belongs to the call it closes, the last one: Gemini checks the signature against the
+ # call it is replayed on.
if incoming_extra is not None:
if extra_before:
current["extra_content"] = extra_before
@@ -946,8 +914,8 @@ class _Turn:
seen: set[str] = taken if taken is not None else set()
painted: set[str] = cards if cards is not None else set()
out: list[dict[str, Any]] = []
- # Announcement order, so an index opening in between cannot reorder
- # them. On the sequence number alone: comparing calls raises.
+ # Announcement order, so an index opening in between cannot reorder them. On the sequence number alone:
+ # comparing calls raises.
numbered = [
(
self.seq_by_key.get(key, position),
@@ -960,14 +928,11 @@ class _Turn:
ordered = [
(index, call) for _, index, call in sorted(numbered, key = lambda triple: triple[0])
]
- # An announcement another call took over, and a name that only looked
- # like the closed call's resent, were never calls. A lone announcement
- # the provider opened is kept (a zero-parameter tool looks like that).
- # Its metadata goes to the call it was mistaken for: Gemini stows a
- # thought signature there and rejects a replay without one.
+ # An announcement another call took over, and a name that only looked like the closed call's resent, were never
+ # calls. A lone announcement the provider opened is kept (a zero-parameter tool looks like that). Its metadata
+ # goes to the call it was mistaken for: Gemini stows a thought signature there and rejects a replay without one.
for key, waiting in self.pending_extra.items():
- # No object ever came, so the repeated name was that call's after
- # all and so is the metadata that rode it.
+ # No object ever came, so the repeated name was that call's after all and so is the metadata that rode it
held = self.by_index.get(key)
if held is not None and waiting:
held["extra_content"] = {**held.get("extra_content", {}), **waiting}
@@ -987,10 +952,9 @@ class _Turn:
continue
kept.append((index, call))
ordered = kept
- # Provider ids reserved first, so a minted card id never collides. Only
- # the ones _normalized_call keeps: a call it rejects draws no card, the
- # client releases the id it had minted for it, and an id held here that
- # the client has given back puts the two out of step from the next round.
+ # Provider ids reserved first, so a minted card id never collides. Only the ones _normalized_call keeps: a call
+ # it rejects draws no card, the client releases the id it had minted for it, and an id held here that the client
+ # has given back puts the two out of step from the next round.
painted.update(
call["id"]
for _, call in ordered
@@ -1002,20 +966,18 @@ class _Turn:
if normalized is None:
continue
if not (isinstance(streamed_id, str) and streamed_id):
- # No id on the wire: mint the client's spelling for the card
- # events. The conversation id above is untouched.
+ # No id on the wire: mint the client's spelling for the card events. The conversation id above is
+ # untouched.
card_id = _mint_streamed_card_id(painted, index)
painted.add(card_id)
normalized["card_id"] = card_id
if normalized["id"] in seen:
- # The client keyed the card it painted on the id the provider
- # streamed, so keep that one for the events aimed at the card.
- # Never replayed upstream: the conversation carries the
- # de-duplicated id, which is the whole point of the rename.
+ # The client keyed the card it painted on the id the provider streamed, so keep that one for the events
+ # aimed at the card. Never replayed upstream: the conversation carries the de-duplicated id, which is
+ # the whole point of the rename.
normalized["stream_id"] = normalized["id"]
- # The renamed id is itself stored and replayed, so a single-shot
- # rename collides again on the next request. Counting up over a
- # finite ledger terminates and leaves the first attempt as is.
+ # The renamed id is itself stored and replayed, so a single-shot rename collides again on the next
+ # request. Counting up over a finite ledger terminates and leaves the first attempt as is.
renamed = f"{normalized['id']}_{self.round}_{position}"
attempt = 0
while renamed in seen:
@@ -1241,8 +1203,8 @@ async def stream_with_studio_tools(
bypass_permissions = policy.bypass_permissions
rag_scope = policy.rag_scope
- # The promotion allowlist is the selected catalog, never None: an
- # unrestricted parse re-opens markerless tool-call promotion.
+ # The promotion allowlist is the selected catalog, never None: an unrestricted parse re-opens markerless tool-call
+ # promotion.
heal_names = (
heal_gate(policy.auto_heal, tools, tool_choice) if transport.heals_text_tool_calls else None
)
@@ -1265,9 +1227,8 @@ async def stream_with_studio_tools(
executed_any = False
model_name = run.model or "external"
usage_totals: dict[str, Any] = {}
- # Dedup, one-shot tracking and the force-final-answer transition are the same
- # ledger the local loops keep, so an external model cannot spend the budget
- # repeating one call and a terminal no-op still ends the loop.
+ # Dedup, one-shot tracking and the force-final-answer transition are the same ledger the local loops keep, so an
+ # external model cannot spend the budget repeating one call and a terminal no-op still ends the loop.
controller = ToolLoopController(
tools = tools,
auto_heal_tool_calls = policy.auto_heal is not False,
@@ -1278,25 +1239,23 @@ async def stream_with_studio_tools(
last_reprompt_text = ""
provider_turns = 0
used_call_ids: set[str] = _replayed_call_ids(conversation)
- # Card ids handed out this response. The client keeps one list across all
- # rounds, so the numbering carries rather than reopening an earlier card.
+ # Card ids handed out this response.
+ # Card ids handed out this response. The client keeps one list across all rounds, so the numbering carries rather
+ # than reopening an earlier card.
painted_card_ids: set[str] = set()
spent_budget_passes = 0
fruitless_turns = 0
- # One provider call per possible execution, plus headroom for the no-op,
- # nudge and final-answer passes that legitimately execute nothing. The
- # unlimited sentinel keeps its own budget rather than dropping to a smaller
- # fixed number: both local loops run "Max" for as many turns as the model
- # asks for, and fruitless_turns already ends a run that executes nothing, so
- # a lower bound here only cuts a productive run short with no final answer.
+ # One provider call per possible execution, plus headroom for the no-op, nudge and final-answer passes that
+ # legitimately execute nothing. The unlimited sentinel keeps its own budget rather than dropping to a smaller fixed
+ # number: both local loops run "Max" for as many turns as the model asks for, and fruitless_turns already ends a run
+ # that executes nothing, so a lower bound here only cuts a productive run short with no final answer.
max_provider_turns = max(1, remaining) + 2 * MAX_ACT_REPROMPTS + 4
while not cancel_event.is_set():
if provider_turns >= max_provider_turns:
- # Reached only by a model that keeps asking for tools it cannot run
- # (all disabled, or the budget is gone). Executions are already
- # capped; this caps the asking, so the conversation cannot grow
- # without bound when nothing ever executes.
+ # Reached only by a model that keeps asking for tools it cannot run (all disabled, or the budget is gone).
+ # Executions are already capped; this caps the asking, so the conversation cannot grow without bound when
+ # nothing ever executes.
break
provider_turns += 1
turn = _Turn(round = provider_turns)
@@ -1306,13 +1265,12 @@ async def stream_with_studio_tools(
tools_available = (
tool_choice != "none" and bool(active_tools) and (unlimited or remaining > 0)
)
- # Withdrawing the catalog and pinning "none" together: this path owns the
- # tool surface (the route withholds enabled_tools), so there are no
- # provider-hosted builtins left for "none" to revoke, and saying it
- # explicitly stops a model from calling a tool it was just denied.
+ # Withdrawing the catalog and pinning "none" together: this path owns the tool surface (the route withholds
+ # enabled_tools), so there are no provider-hosted builtins left for "none" to revoke, and saying it explicitly
+ # stops a model from calling a tool it was just denied.
turn_tool_choice = tool_choice if tools_available else "none"
- # A forced choice applies until the model actually calls something; the
- # result follow-up must be free to answer in prose.
+ # A forced choice applies until the model actually calls something; the result follow-up must be free to answer
+ # in prose.
if executed_any and turn_tool_choice not in ("auto", "none"):
turn_tool_choice = "auto"
@@ -1325,19 +1283,13 @@ async def stream_with_studio_tools(
try:
async for line in generator:
if _is_done_sentinel(line):
- # Every turn ends with one. Relaying it mid-loop tells a
- # spec-compliant client the response is over, and it stops before
- # the tool cards and the real answer. The route emits the final one.
continue
- # This loop writes its tool cards, badges and the approval
- # handshake onto the same stream the provider's chunks are
- # relayed on, and the client tells them apart only by shape. A
- # provider's copy of that vocabulary would therefore paint a card
- # for a tool that never ran, so strip it before anything below
- # can relay it. Skipped for a transport that already did it at
- # the point the raw bytes arrived: everything reaching here from
- # one of those is this server's own frame, and stripping it drops
- # a retained hosted tool's result after the provider billed it.
+ # This loop writes its tool cards, badges and the approval handshake onto the same stream the provider's
+ # chunks are relayed on, and the client tells them apart only by shape. A provider's copy of that
+ # vocabulary would therefore paint a card for a tool that never ran, so strip it before anything below
+ # can relay it. Skipped for a transport that already did it at the point the raw bytes arrived:
+ # everything reaching here from one of those is this server's own frame, and stripping it drops a
+ # retained hosted tool's result after the provider billed it.
if not transport_sanitizes:
sanitized = sanitize_provider_sse_line(line)
if sanitized is None:
@@ -1347,24 +1299,21 @@ async def stream_with_studio_tools(
if payload is None:
yield line
continue
- # OpenRouter and friends resolve a routing alias to a concrete
- # model and name it on every chunk. That id is more specific than
- # the one the request asked for, so let it win for the summed
- # usage chunk this loop emits once the answer ends.
+ # OpenRouter and friends resolve a routing alias to a concrete model and name it on every chunk. That id
+ # is more specific than the one the request asked for, so let it win for the summed usage chunk this
+ # loop emits once the answer ends.
upstream_model = payload.get("model")
if isinstance(upstream_model, str) and upstream_model:
model_name = upstream_model
if "usage" in payload:
_merge_usage(usage_totals, payload.get("usage"))
if _is_usage_only(payload):
- # Withheld: one summed chunk is sent once the loop ends, so a
- # multi-turn answer does not report a burst of partial counts.
+ # Withheld: one summed chunk is sent once the loop ends, so a multi-turn answer does not report
+ # a burst of partial counts.
continue
- # Some providers hang usage off a chunk that also carries a
- # choice, which cannot be withheld wholesale without losing
- # the content. Drop just the usage: it is already in the
- # totals, and leaving it here makes a client that sums
- # chunks count this turn twice.
+ # Some providers hang usage off a chunk that also carries a choice, which cannot be withheld
+ # wholesale without losing the content. Drop just the usage: it is already in the totals, and
+ # leaving it here makes a client that sums chunks count this turn twice.
payload.pop("usage", None)
line = "data: " + json.dumps(payload, separators = (",", ":"))
choices = payload.get("choices")
@@ -1386,9 +1335,8 @@ async def stream_with_studio_tools(
if isinstance(raw_calls, list) and raw_calls:
if healer is not None and not healer.dormant:
- # Grammar mode worked; stop second-guessing the text stream.
- # Whatever was being held was ordinary prose after all, so it
- # has to reach the client, not just the conversation replay.
+ # Grammar mode worked; stop second-guessing the text stream. Whatever was being held was
+ # ordinary prose after all, so it has to reach the client, not just the conversation replay.
for kind, value in healer.structured_tool_call_seen():
if kind == "text" and value:
turn.text.append(value)
@@ -1413,13 +1361,11 @@ async def stream_with_studio_tools(
if visible:
turn.text.append(visible)
if visible == content:
- # Nothing was held back, which is the case for almost every chunk
- # of ordinary prose. Relay the provider's own bytes rather than
- # paying a re-encode per chunk to reproduce them.
+ # Nothing was held back, the case for almost every chunk of ordinary prose
yield line
continue
- # Withholding everything is normal mid-block. Only drop the chunk when
- # it carries nothing else worth relaying.
+ # Withholding everything is normal mid-block. Only drop the chunk when it carries nothing else worth
+ # relaying.
if visible or turn.finish_reason is not None or len(delta) > 1:
yield _rewrite_content(payload, choice, visible)
@@ -1433,9 +1379,8 @@ async def stream_with_studio_tools(
turn.healed.append(value)
finally:
- # Release the upstream response now rather than leaving it to the
- # async-generator finalisation hook, which runs a tick or more after
- # the route has already closed this loop.
+ # Release the upstream response now rather than leaving it to the async-generator finalisation hook, which
+ # runs a tick or more after the route has already closed this loop.
aclose = getattr(generator, "aclose", None)
if aclose is not None:
try:
@@ -1443,28 +1388,22 @@ async def stream_with_studio_tools(
except (RuntimeError, GeneratorExit):
pass
- # Both of these mean the turn ended before the model finished saying what
- # it wanted: "length" hit the token ceiling, "content_filter" had the
- # output cut by the provider's own filter. Either way a call collected so
- # far may be half-written, so it is described rather than run. "stop" is
- # deliberately not in this set: llama.cpp and vLLM routinely finish a
- # perfectly good tool call with it, and refusing those would disable
- # tool calling on exactly the self-hosted servers this path exists for.
+ # Both mean the turn ended before the model finished Both of these mean the turn ended before the model finished
+ # saying what it wanted: "length" hit the token ceiling, "content_filter" had the output cut by the provider's
+ # own filter. Either way a call collected so far may be half-written, so it is described rather than run. "stop"
+ # is not in this set: llama.cpp and vLLM routinely finish a perfectly good tool call with it, and refusing those
+ # would disable tool calling on exactly the self-hosted servers this path exists for.
truncated = turn.finish_reason in ("length", "content_filter")
if truncated and healer is not None and turn.healed:
- # A call cut off at the token limit must not run: its arguments can
- # be half-written and the model never finished saying what it wanted.
- # But promotion is destructive -- the healer already removed the
- # markup span from the text relayed above -- so discarding the call
- # on its own would take the sentence that introduced it with it, and
- # the user would be left with a stub answer, no card, and no way to
- # tell that anything was attempted. Give the exact removed span back
- # instead. It is the same verbatim flush the healer performs for a
- # block that turns out not to be a declared call, and only the healer
- # can supply it: nothing else in the loop keeps the raw bytes, and
- # re-encoding the parsed call would print something the model never
- # wrote. Released at the end of the turn rather than in document
- # order because "length" is only known once the turn has ended.
+ # A call cut off at the token limit must not run: its arguments can be half-written and the model never
+ # finished saying what it wanted. But promotion is destructive -- the healer already removed the markup span
+ # from the text relayed above -- so discarding the call on its own would take the sentence that introduced
+ # it with it, and the user would be left with a stub answer, no card, and no way to tell that anything was
+ # attempted. Give the exact removed span back instead. It is the same verbatim flush the healer performs for
+ # a block that turns out not to be a declared call, and only the healer can supply it: nothing else in the
+ # loop keeps the raw bytes, and re-encoding the parsed call would print something the model never wrote.
+ # Released at the end of the turn rather than in document order because "length" is only known once the turn
+ # has ended.
for healed_call in turn.healed:
span = healer.promoted_source(healed_call.get("id", ""))
if not span:
@@ -1472,18 +1411,15 @@ async def stream_with_studio_tools(
turn.text.append(span)
yield _sse({"choices": [{"index": 0, "delta": {"content": span}}]})
if truncated:
- # The other half of the same problem. A call the provider streamed as
- # a tool_calls delta was relayed as it arrived, so the client already
- # has a card for it, and refusing to run it leaves that card open for
- # the rest of the response. Close it the way every other unrun call
- # is closed. Structured only: a healed call was never streamed, and
- # the span released just above is what tells the user about that one.
- # Through `calls`, which never executes anything: it is what mints
- # the card id for a call the provider gave none, and the client drew
- # its card under that same spelling. Reading the slots directly left
- # every id-less call out, so the card the deltas painted spun for the
- # rest of the response. Calls it drops -- nameless, or a fork whose
- # object never closed -- have no card of ours to close either.
+ # The other half: a call the provider streamed as a tool_calls delta was relayed as it arrived
+ # The other half of the same problem. A call the provider streamed as a tool_calls delta was relayed as it
+ # arrived, so the client already has a card for it, and refusing to run it leaves that card open for the
+ # rest of the response. Close it the way every other unrun call is closed. Structured only: a healed call
+ # was never streamed, and the span released just above is what tells the user about that one. Through
+ # `calls`, which never executes anything: it is what mints the card id for a call the provider gave none,
+ # and the client drew its card under that same spelling. Reading the slots directly left every id-less call
+ # out, so the card the deltas painted spun for the rest of the response. Calls it drops -- nameless, or a
+ # fork whose object never closed -- have no card of ours to close either.
for raw_call in turn.calls(used_call_ids, painted_card_ids):
truncated_id = (
raw_call.get("card_id") or raw_call.get("stream_id") or raw_call["id"]
@@ -1492,38 +1428,33 @@ async def stream_with_studio_tools(
for card_line in _unrun_call_card(
tool_name = name,
tool_call_id = truncated_id,
- # The arguments are cut off mid-write, so there is nothing
- # well formed to show; the result says what happened.
+ # The arguments are cut off mid-write, so there is nothing well formed to show; the result says what
+ # happened.
arguments = {},
result = _TOOL_TRUNCATED,
provenance = _unrun_provenance(name, round_id + 1),
):
yield card_line
- # tool_choice "none" is an instruction, and a provider that emits a call
- # anyway has not been authorized to run one. Withdrawing the catalog on
- # the way out is not enough on its own: Deep Research sets "none" exactly
- # so the scraped web text in its prompts cannot reach python or terminal,
- # so a naive or compromised endpoint echoing a call back must not be able
- # to execute it here.
+ # tool_choice "none" is an instruction, and a provider that emits a call anyway has not been authorized to run
+ # one. Withdrawing the catalog on the way out is not enough on its own: Deep Research sets "none" exactly so the
+ # scraped web text in its prompts cannot reach python or terminal, so a naive or compromised endpoint echoing a
+ # call back must not be able to execute it here.
calls = (
[]
if (truncated or tool_choice == "none")
else turn.calls(used_call_ids, painted_card_ids)
)
if not calls:
- # The badge clears between iterations, and this turn is over too.
- # Without it a turn whose only call was refused never reaches the
- # empty status below, so the client keeps the card it drew for that
- # call open and a reprompted call merges into it: an upstream ending
- # on [DONE] sends no finish_reason either, so there is no other
- # boundary on this path. Only when a call was streamed: a turn that
- # said nothing at all left the client nothing to close, and a
- # provider that sends no lines still ends the response silently.
+ # The badge clears between iterations, and this turn is over too. Without it a turn whose only call was
+ # refused never reaches the empty status below, so the client keeps the card it drew for that call open and
+ # a reprompted call merges into it: an upstream ending on [DONE] sends no finish_reason either, so there is
+ # no other boundary on this path. Only when a call was streamed: a turn that said nothing at all left the
+ # client nothing to close, and a provider that sends no lines still ends the response silently.
if turn.by_index:
yield _status_sse("")
- # No tool this turn. A model that only said what it was about to do
- # gets one nudge to actually do it, the same recovery the local loops
- # give a stalled small model, then the answer stands as written.
+ # No tool this turn.
+ # No tool this turn. A model that only said what it was about to do gets one nudge to actually do it, the
+ # same recovery the local loops give a stalled small model, then the answer stands as written.
visible_answer = "".join(turn.text)
if (
tools_available
@@ -1537,10 +1468,9 @@ async def stream_with_studio_tools(
last_reprompt_text = visible_answer
stalled_hosted = turn.hosted_replay_text()
if stalled_hosted:
- # A hosted tool did run, the model just did not go on to ask
- # for a local one. The replay below never happens on this
- # path, so the reprompted request would be told to continue
- # from output it can no longer see.
+ # A hosted tool did run, the model just did not go on to ask for a local one. The replay below never
+ # happens on this path, so the reprompted request would be told to continue from output it can no
+ # longer see.
stalled_message: dict[str, Any] = {
"role": "assistant",
"content": (
@@ -1550,16 +1480,14 @@ async def stream_with_studio_tools(
),
}
if turn.reasoning_extra:
- # Gemini 3 stows the text part's thoughtSignature here
- # and its translator pins it back on from this field
- # alone, so a turn replayed without it is rejected.
+ # Gemini 3 stows the text part's thoughtSignature here and its translator pins it back on from
+ # this field alone, so a turn replayed without it is rejected.
stalled_message["extra_content"] = turn.reasoning_extra
append_assistant_turn(
conversation,
stalled_message,
- # A resumed partial is the same turn as what the model
- # just added, so merge rather than append: appending
- # puts a turn boundary mid-sentence.
+ # A resumed partial is the same turn as what the model just added, so merge rather than append:
+ # appending puts a turn boundary mid-sentence.
continue_final_message = run.continue_final_message,
)
_append_user_turn(conversation, reprompt_to_act_message(tool_hint))
@@ -1576,8 +1504,7 @@ async def stream_with_studio_tools(
if cancel_event.is_set():
break
if not unlimited and remaining <= 0:
- # Budget spent. Answer the model so it stops asking, but never
- # execute: the cap is a safety limit, not a hint to the provider.
+ # Budget spent.
for card_line in _unrun_call_card(
tool_name = call["function"]["name"],
tool_call_id = call.get("card_id") or call.get("stream_id") or call["id"],
@@ -1586,16 +1513,13 @@ async def stream_with_studio_tools(
provenance = _unrun_provenance(call["function"]["name"], round_id),
):
yield card_line
- # The result below has to be replayed with its call: only the
- # call that spent the last slot reaches assistant_tool_calls
- # further down, so this one would arrive as an orphan
- # role="tool" message and OpenAI, Anthropic and Gemini all
- # reject that history instead of answering.
+ # The result below has to be replayed with its call: only the call that spent the last slot reaches
+ # assistant_tool_calls further down, so this one would arrive as an orphan role="tool" message and
+ # OpenAI, Anthropic and Gemini all reject that history instead of answering.
exhausted_call: dict[str, Any] = {
"id": call["id"],
"type": "function",
- # Copied: the normalized call also carries a parsed
- # arguments dict that must not reach the provider.
+ # Copied: the normalized call also carries a parsed arguments dict that must not reach the provider
"function": dict(call["function"]),
}
exhausted_extra = call.get("extra_content")
@@ -1612,25 +1536,20 @@ async def stream_with_studio_tools(
)
continue
decision = controller.prepare_call(call)
- # The frontend groups a round's reasoning by this id
- # (codexLocalToolRoundId), so every tool card the loop emits has to
- # carry it, not just the budget-exhausted one built by hand above.
+ # The frontend groups a round's reasoning by this id (codexLocalToolRoundId), so every tool card the loop
+ # emits has to carry it, not just the budget-exhausted one built by hand above.
decision.provenance["round_id"] = round_id
if not decision.should_execute:
completion = controller.record_noop(decision)
noop_messages.append(completion.model_message())
- # The provider's own tool_calls delta for this call was relayed
- # verbatim while it streamed and the client painted a card from
- # it. Nothing else closes that card, so without a terminal event
- # it spins for the rest of the answer and then reads as a tool
- # that ran and returned nothing. Keyed on the streamed id, since
- # a repeated call is exactly the one this loop renames above.
- #
- # Only for a tool the user DID enable. A call for something
- # outside the catalog is not a tool of Unsloth's that declined to
- # run, it is a name this install never offered, and giving it a
- # card would advertise a tool the user switched off. That one is
- # answered in the conversation only.
+ # The provider's tool_calls delta was relayed while it streamed and the client painted a card from it;
+ # The provider's own tool_calls delta for this call was relayed verbatim while it streamed and the
+ # client painted a card from it. Nothing else closes that card, so without a terminal event it spins for
+ # the rest of the answer and then reads as a tool that ran and returned nothing. Keyed on the streamed
+ # id, since a repeated call is exactly the one this loop renames above. Only for a tool the user DID
+ # enable. A call for something outside the catalog is not a tool of Unsloth's that declined to run, it
+ # is a name this install never offered, and giving it a card would advertise a tool the user switched
+ # off. That one is answered in the conversation only.
if decision.action == "disabled":
continue
for card_line in _unrun_call_card(
@@ -1647,16 +1566,14 @@ async def stream_with_studio_tools(
assistant_call = decision.as_assistant_tool_call()
call_extra = call.get("extra_content")
if isinstance(call_extra, dict) and call_extra:
- # Replayed verbatim: Gemini 3 validates the signature that came
- # back with this exact call.
+ # Replayed verbatim: Gemini 3 validates the signature that came back with this exact call
assistant_call["extra_content"] = call_extra
assistant_tool_calls.append(assistant_call)
name = decision.tool_name
arguments = decision.arguments
call_id = decision.tool_call_id
- # Same id for a call the provider named; for one it did not, the
- # card answers to the id the client minted.
+ # Same id for a call the provider named; for one it did not, the card answers to the id the client minted
card_id = decision.card_id
needs_confirmation = (
confirm_tool_calls and not bypass_permissions and permission_mode != "off"
@@ -1686,9 +1603,8 @@ async def stream_with_studio_tools(
)
)
try:
- # Hold the first keepalive back so the gated card is flushed
- # on its own write and the Allow / Deny buttons paint before
- # the stream blocks waiting for the answer.
+ # Hold the first keepalive back so the gated card is flushed on its own write and the Allow /
+ # Deny buttons paint before the stream blocks waiting for the answer.
done, _pending = await asyncio.wait(
{waiter}, timeout = _TOOL_APPROVAL_FLUSH_DELAY_S
)
@@ -1743,19 +1659,16 @@ async def stream_with_studio_tools(
"rag_scope": rag_scope,
"disable_sandbox": bypass_permissions,
}
- # Provider loops share the local catalogue selector, so
- # search_conversation is advertised here too once a thread has an archive
- # and needs the same branch: the stored rows are the whole DAG, and Retry
- # leaves the replaced response in them.
+ # Provider loops share the local catalogue selector, so search_conversation is advertised here too once
+ # a thread has an archive and needs the same branch: the stored rows are the whole DAG, and Retry leaves
+ # the replaced response in them.
if accepts_kwarg(execute_tool, "conversation_branch"):
kwargs["conversation_branch"] = request_branch
- # And a budget, so the tool's clamp is not skipped. Unsloth cannot measure
- # an external model's window, and a custom OpenAI-compatible endpoint can
- # be a small local server, so a model-chosen 8 chunks is roughly 4K tokens
- # replayed on every later call. Unmeasurable means one recall's worth.
- # Explicitly unknowable, not absent: this request is served by an
- # external provider, so the resident GGUF's window says nothing about
- # what it can hold. 0 keeps the default page cap instead of inheriting it.
+ # And a budget, so the tool's clamp is not skipped. Unsloth cannot measure an external model's window,
+ # and a custom OpenAI-compatible endpoint can be a small local server, so a model-chosen 8 chunks is
+ # roughly 4K tokens replayed on every later call. Unmeasurable means one recall's worth. Explicitly
+ # unknowable, not absent: this request is served by an external provider, so the resident GGUF's window
+ # says nothing about what it can hold. 0 keeps the default page cap instead of inheriting it.
if accepts_kwarg(execute_tool, "context_tokens"):
kwargs["context_tokens"] = 0
if accepts_kwarg(execute_tool, "conversation_budget_tokens"):
@@ -1771,8 +1684,8 @@ async def stream_with_studio_tools(
kwargs.update(search_images_kwargs(execute_tool, call.tool_name))
return execute_tool(call.tool_name, call.arguments, **kwargs)
- # The same wrapper the local loops run tools through: live stdout for
- # the card, and a heartbeat so a long call cannot idle the stream out.
+ # The same wrapper the local loops run tools through: live stdout for the card, and a heartbeat so a long
+ # call cannot idle the stream out.
tool_stream = stream_tool_execution(
_invoke,
tool_name = name,
@@ -1784,16 +1697,15 @@ async def stream_with_studio_tools(
try:
while True:
if cancel_event.is_set():
- # A tool that does not watch the cancel event would keep
- # producing heartbeats and hold the answer open forever.
- # Stop asking for them and let the drain below join the
- # worker under its own bounded timeout.
+ # A tool that does not watch the cancel event would keep producing heartbeats and hold the
+ # answer open forever. Stop asking for them and let the drain below join the worker under its
+ # own bounded timeout.
break
step_task = asyncio.create_task(
asyncio.to_thread(_advance_tool_stream, tool_stream, outcome)
)
- # wait, not await: cancelling this coroutine must leave the
- # worker pending so the drain below can still join it.
+ # wait, not await: cancelling this coroutine must leave the worker pending so the drain below can
+ # still join it
await asyncio.wait({step_task})
event = step_task.result()
step_task = None
@@ -1806,11 +1718,9 @@ async def stream_with_studio_tools(
if "result" in outcome:
result = outcome["result"]
elif cancel_event.is_set():
- # Stopped before the tool returned. Defaulting to "" here
- # would record a successful empty result and paint a normal
- # tool_end, so the transcript would claim a tool ran and
- # produced nothing, when in truth it was abandoned partway
- # and its side effects may already have happened.
+ # Stopped before the tool returned. Defaulting to "" here would record a successful empty result and
+ # paint a normal tool_end, so the transcript would claim a tool ran and produced nothing, when in
+ # truth it was abandoned partway and its side effects may already have happened.
result = _TOOL_CANCELLED
else:
result = ""
@@ -1823,20 +1733,17 @@ async def stream_with_studio_tools(
tool_stream.close()
completion = controller.record_result(decision, result)
- # Counted whether or not the tool succeeded: a failing call has
- # already done its work (and possibly its side effects), so letting
- # it run for free would put the budget past max_calls. Counted per
- # call rather than per turn, so parallel calls each spend one.
+ # Counted whether or not the tool succeeded: a failing call has already done its work (and possibly its side
+ # effects), so letting it run for free would put the budget past max_calls. Counted per call rather than per
+ # turn, so parallel calls each spend one.
if not unlimited:
remaining -= 1
turn_executed_real_tool = True
executed_any = True
- # Opens the post-tool phase; carried-over stall text would eat its nudge.
last_reprompt_text = ""
yield _sse(completion.tool_end_event())
tool_messages.append(completion.tool_message())
- # An empty status clears the badge between iterations.
yield _status_sse("")
assistant_message: dict[str, Any] = {
@@ -1848,12 +1755,10 @@ async def stream_with_studio_tools(
}
hosted_text = turn.hosted_replay_text()
if hosted_text:
- # A tool the provider ran itself this turn. Its output went to the
- # client as its own frame but is not otherwise part of this message,
- # so the follow-up would answer from the local results alone.
- # Replayed as text, not native items: the shape differs per provider
- # (Gemini codeExecutionResult, an OpenAI image call), while every
- # provider can read its own prior turn's prose.
+ # A tool the provider ran itself this turn. Its output went to the client as its own frame but is not
+ # otherwise part of this message, so the follow-up would answer from the local results alone. Replayed as
+ # text, not native items: the shape differs per provider (Gemini codeExecutionResult, an OpenAI image call),
+ # while every provider can read its own prior turn's prose.
assistant_message["content"] = (
f"{assistant_message['content']}\n\n{hosted_text}"
if assistant_message["content"]
@@ -1864,15 +1769,14 @@ async def stream_with_studio_tools(
if assistant_tool_calls:
assistant_message["tool_calls"] = assistant_tool_calls
if assistant_message["content"] or assistant_tool_calls:
- # Merges into a resumed partial so a continued tool turn stays one message.
append_assistant_turn(
conversation,
assistant_message,
continue_final_message = run.continue_final_message,
)
conversation.extend(tool_messages)
- # Deferred to after the results so a no-op never splits a call from them,
- # and merged into a trailing user turn so the roles keep alternating.
+ # Deferred to after the results so a no-op never splits a call from them, and merged into a trailing user turn
+ # so the roles keep alternating.
_append_user_turn(
conversation,
"\n\n".join(dict.fromkeys(message["content"] for message in noop_messages)),
@@ -1885,18 +1789,17 @@ async def stream_with_studio_tools(
else:
fruitless_turns += 1
if fruitless_turns >= _MAX_FRUITLESS_TURNS:
- # It asked for tools twice running and none could run. One more
- # pass would only repeat the exchange, so stop asking.
+ # It asked for tools twice running and none could run. One more pass would only repeat the exchange, so
+ # stop asking.
break
if remaining <= 0 and not unlimited and not controller.force_final_answer:
if spent_budget_passes:
- # It was already told the budget is gone and asked for a tool
- # again anyway. One more pass to let it answer, then stop rather
- # than trading turns with it.
+ # It was already told the budget is gone and asked for a tool again anyway. One more pass to let it
+ # answer, then stop rather than trading turns with it.
break
spent_budget_passes += 1
- # The catalog is gone from here on, so say why rather than letting the
- # next pass ask for a tool that is no longer offered.
+ # The catalog is gone from here on, so say why rather than letting the next pass ask for a tool no longer
+ # offered
_append_user_turn(conversation, _BUDGET_EXHAUSTED_NUDGE)
usage_line = _usage_chunk_line(model_name, usage_totals)
diff --git a/studio/backend/core/inference/tensor_fallback.py b/studio/backend/core/inference/tensor_fallback.py
index 3ceb1a268a0..b60fdf848a9 100644
--- a/studio/backend/core/inference/tensor_fallback.py
+++ b/studio/backend/core/inference/tensor_fallback.py
@@ -59,8 +59,8 @@ async def load_with_tensor_fallback(
if success or not tensor_requested:
return success
- # The first attempt returned False because the user cancelled, not because
- # tensor mode is unsupported -- do not relaunch the cancelled load.
+ # The first attempt returned False because the user cancelled, not because tensor mode is unsupported -- do not
+ # relaunch the cancelled load.
if cancelled is not None and cancelled():
return success
@@ -69,8 +69,9 @@ async def load_with_tensor_fallback(
"(this model may not support tensor parallelism)",
label,
)
- # Force --split-mode layer (CLI wins over env) so neither leftover extras nor
- # an inherited LLAMA_ARG_SPLIT_MODE=tensor can re-engage tensor and re-crash
- # the retry; load_model and the child both honor the explicit layer override.
+ # explicit --split-mode layer: leftover extras or an inherited LLAMA_ARG_SPLIT_MODE=tensor would re-crash the retry
+ # Force --split-mode layer (CLI wins over env) so neither leftover extras nor an inherited
+ # LLAMA_ARG_SPLIT_MODE=tensor can re-engage tensor and re-crash the retry; load_model and the child both honor the
+ # explicit layer override.
layer_extras = strip_split_mode_only(extra_args) or []
return await attempt_load(False, [*layer_extras, "--split-mode", "layer"])
diff --git a/studio/backend/core/inference/tool_call_parser.py b/studio/backend/core/inference/tool_call_parser.py
index 45986cf72ab..a1d36074c68 100644
--- a/studio/backend/core/inference/tool_call_parser.py
+++ b/studio/backend/core/inference/tool_call_parser.py
@@ -29,7 +29,7 @@ import json
import re
from typing import Any, Optional
-# Qwen/Hermes, Qwen3.5 XML and Gemma 4 live in core.tool_healing; this module adds the rest.
+# Qwen/Hermes, Qwen3.5 XML and Gemma 4 live in core.tool_healing; this module adds the rest
from core import tool_healing as _tool_healing
@@ -41,8 +41,8 @@ TOOL_XML_SIGNALS = (
"<|python_tag|>",
"[TOOL_CALLS]",
"<|tool_call>",
- # Bare reasoning-rehearsal marker (``name[ARGS]{...}``, no leading [TOOL_CALLS]);
- # keeps a rehearsed call held in the stream so it is promoted, not leaked as prose.
+ # Bare reasoning-rehearsal marker (``name[ARGS]{...}``, no leading [TOOL_CALLS]); keeps a rehearsed call held in the
+ # stream so it is promoted, not leaked as prose.
"[ARGS]",
# DeepSeek R1 / V3 / V3.1 -- 5 opener variants llama.cpp keeps.
"<|tool▁calls▁begin|>",
@@ -51,26 +51,23 @@ TOOL_XML_SIGNALS = (
"<|tool▁calls|>",
"<|tool calls begin|>",
"<|tool\\_calls\\_begin|>",
- # Kimi K2 / Moonshot.
"<|tool_calls_section_begin|>",
"<|tool_call_begin|>",
- # TML Inkling native call marker.
"<|content_invoke_tool_json|>",
)
-# DeepSeek opener variants; shared by parse and strip so a parsed signal is always stripped.
+# DeepSeek opener variants; shared by parse and strip so a parsed signal is always stripped
_DEEPSEEK_OPEN_ALT = (
r"tool▁calls▁begin|tool_calls_begin|tool calls begin|tool\\_calls\\_begin|tool▁calls"
)
_DEEPSEEK_OPEN_RE_SRC = r"<|(?:" + _DEEPSEEK_OPEN_ALT + r")|>"
-# Closed pairs only (mid-stream); _TOOL_ALL_PATS also eats unclosed tails at
-# end-of-turn. ``[\w-]+`` on ``<function=...>`` tracks OpenAI's
-# ``^[a-zA-Z0-9_-]{1,64}$`` so hyphenated MCP names parse like built-ins.
+# Closed pairs only (mid-stream); _TOOL_ALL_PATS also eats unclosed tails at end-of-turn. ``[\w-]+`` on
+# ``<function=...>`` tracks OpenAI's ``^[a-zA-Z0-9_-]{1,64}$`` so hyphenated MCP names parse like built-ins.
_TOOL_CLOSED_PATS = [
re.compile(r"<tool_call>.*?</tool_call>", re.DOTALL),
- # Span to the real ``</function>`` so a literal one inside a value can't truncate the strip.
+ # Span to the real ``</function>`` so a literal one inside a value can't truncate the strip
re.compile(
r'<function(?:=[\w.\-]+|\s+name="[\w.\-]+")>'
r'(?:(?!<function(?:=[\w.\-]+|\s+name="[\w.\-]+")>).)*'
@@ -85,14 +82,14 @@ _TOOL_CLOSED_PATS = [
re.compile(_DEEPSEEK_OPEN_RE_SRC + r".*?<|tool▁calls▁end|>", re.DOTALL),
# Kimi K2: ``<|tool_calls_section_begin|>...<|tool_calls_section_end|>``.
re.compile(r"<\|tool_calls_section_begin\|>.*?<\|tool_calls_section_end\|>", re.DOTALL),
- # Kimi K2 section-less closed call; else the catch-all below eats trailing prose to EOS.
+ # Kimi K2 section-less closed call; else the catch-all below eats trailing prose to EOS
re.compile(r"<\|tool_call_begin\|>.*?<\|tool_call_end\|>", re.DOTALL),
]
_TOOL_ALL_PATS = _TOOL_CLOSED_PATS + [
re.compile(r"<tool_call>.*$", re.DOTALL),
re.compile(r'<function(?:=[\w.\-]+|\s+name="[\w.\-]+")>.*$', re.DOTALL),
- # Bare-word markers drop a trailing truncated call only when a call-shaped start
- # follows; a prose mention (``See [TOOL_CALLS] docs...``) keeps its tail. Bare marker at EOF drops.
+ # Bare-word markers drop a trailing truncated call only when a call-shaped start follows; a prose mention (``See
+ # [TOOL_CALLS] docs...``) keeps its tail. Bare marker at EOF drops.
re.compile(r"<\|tool_call>(?=\s*call\s*:|\s*$).*$", re.DOTALL),
re.compile(
r"\[TOOL_CALLS\](?=\s*(?:[\[{]|[A-Za-z_][\w.\-]*(?:[\[{]|\s*$))|\s*$).*$",
@@ -102,13 +99,12 @@ _TOOL_ALL_PATS = _TOOL_CLOSED_PATS + [
r"<\|python_tag\|>(?=\s*(?:\{|[A-Za-z_][\w.]*\()|\s*$).*$",
re.DOTALL,
),
- # DeepSeek envelopes truncated mid-stream (any opener); same call-shaped lookahead as above.
+ # DeepSeek envelopes truncated mid-stream (any opener); same call-shaped lookahead as above
re.compile(
_DEEPSEEK_OPEN_RE_SRC + r"(?=\s*(?:<|tool▁call▁begin|>|function)|\s*$).*$",
re.DOTALL,
),
re.compile(r"<|tool▁call▁begin|>(?=\s*function|\s*$).*$", re.DOTALL),
- # Kimi K2 envelope truncated.
re.compile(
r"<\|tool_calls_section_begin\|>(?=\s*<\|tool_call_begin\|>|\s*$).*$",
re.DOTALL,
@@ -117,7 +113,7 @@ _TOOL_ALL_PATS = _TOOL_CLOSED_PATS + [
r"<\|tool_call_begin\|>(?=\s*[A-Za-z_][\w.\-]*:\d|\s*$).*$",
re.DOTALL,
),
- # Gemma wrapper-less ``call:NAME{...}`` is handled by ``_strip_gemma_wrapperless_calls`` (enabled-name gate).
+ # Gemma wrapper-less ``call:NAME{...}`` is handled by ``_strip_gemma_wrapperless_calls`` (enabled-name gate)
]
@@ -156,11 +152,11 @@ BUDGET_EXHAUSTED_NUDGE = (
"any more tools."
)
-# The exact-args dup guard misses paraphrased re-searches, so also cap KB searches per turn.
+# The exact-args dup guard misses paraphrased re-searches, so also cap KB searches per turn
RAG_MAX_SEARCHES_PER_TURN = 3
-# Both retrieval tools share that cap. Their top-K passages land in the current
-# exchange, which the rolling window protects and cannot evict, so an uncapped search
-# only ends the turn in a context-length error. Here so both tool loops agree on it.
+# Both retrieval tools share that cap. Their top-K passages land in the current exchange, which the rolling window
+# protects and cannot evict, so an uncapped search only ends the turn in a context-length error. Here so both tool loops
+# agree on it.
RAG_SEARCH_TOOLS = frozenset({"search_knowledge_base", "search_conversation"})
RAG_SEARCH_CAP_NUDGE = (
"You have already searched the knowledge base several times this turn. "
@@ -169,19 +165,19 @@ RAG_SEARCH_CAP_NUDGE = (
)
+# Verbs naming work this turn.
+# ── Plan-without-action re-prompt (shared by the GGUF and safetensors loops) ── Verbs naming work this turn. Narrow on
+# purpose: "install"/"add"/"open" belong to advice for the user, which must not be re-prompted.
# ── Plan-without-action re-prompt (shared by the GGUF and safetensors loops) ──
-# Verbs naming work this turn. Narrow on purpose: "install"/"add"/"open" belong to
-# advice for the user, which must not be re-prompted.
_ACTION_VERB = (
r"(?:search|check|look|find|fetch|get|call|use|run|query|invoke|analy[sz]e"
r"|review|inspect|read|gather|examine|retrieve|browse|consult|verify"
r"|confirm|compute|calculate|determine|identify|render)"
)
-# Offering to help hands control back exactly like "let me know": measured on real
-# turns, "I'll do my best to help" and "allow me to assist" close a clarification
-# request and never precede a tool call. "help you" keeps its plan reading when an
-# action follows it ("I'll help you search the web").
-# name no work of their own, so they only ever sign off a question to the user (#8907)
+# Offering to help hands control back exactly like "let me know": measured on real turns, "I'll do my best to help" and
+# "allow me to assist" close a clarification request and never precede a tool call. "help you" keeps its plan reading
+# when an action follows it ("I'll help you search the web"). name no work of their own, so they only ever sign off a
+# question to the user (#8907)
_SIGN_OFF = r"(?:dig\s+in|help\s+analy[sz]e)\b(?=[^\w]*\Z)" # \Z not $: a later line is still work
_HELP_OFFER = (
r"(?:do(?:ing)?\s+my\s+best|try\s+my\s+best|be\s+(?:able|happy|glad)\s+to\b"
@@ -200,9 +196,9 @@ INTENT_SIGNAL = re.compile(
# bare "assist" still names work here ("let me assist by searching"), so only the sign-offs skip the "to"
r"(?!\s+" + _SIGN_OFF + r")(?!\s+to\s+" + _HELP_OFFER + r")"
r"|"
- # Step/plan framing. "first" must open a sentence and be followed by a plan
- # (pronoun, "my/our plan", or an action verb); otherwise it is prose ("The
- # first line is blank.", "First place went to Alice") or advice to the user.
+ # "first" must open a sentence and be followed by a plan (pronoun, "my/our plan", or an action verb);
+ # Step/plan framing. "first" must open a sentence and be followed by a plan (pronoun, "my/our plan", or an action
+ # verb); otherwise it is prose ("The first line is blank.", "First place went to Alice") or advice to the user.
r"(?:^|[.!?]\s+)\s*(?:the\s+)?first\s+step\b"
r"|(?:^|[.!?]\s+)\s*first\s*[,:–—-]?\s+(?:my|our)\s+(?:plan|approach|step)\b"
r"|(?:^|[.!?]\s+)\s*first\s*[,:–—-]?\s+(?:i|we|let['’]?s|let us)\b"
@@ -213,12 +209,12 @@ INTENT_SIGNAL = re.compile(
r"\b(?:now i|next i)\b"
r")"
)
-# Matches GGUF's established default (llama_cpp.py has re-prompted up to 3
-# times since #5620); safetensors and MLX inherit the same cap from here.
+# Matches GGUF's established default (llama_cpp.py has re-prompted up to 3 times since #5620); safetensors and MLX
+# inherit the same cap from here.
MAX_ACT_REPROMPTS = 3
REPROMPT_MAX_CHARS = 2000
-# Composer badge while a hidden re-prompted turn regenerates, else the UI looks
-# hung. Matched exactly by the frontend (utils/tool-status.ts); keep in sync.
+# Composer badge while a hidden re-prompted turn regenerates, else the UI looks hung. Matched exactly by the frontend
+# (utils/tool-status.ts); keep in sync.
NUDGE_TOOL_CALLS_STATUS = "Nudging tool calls"
@@ -227,8 +223,8 @@ def is_short_intent_without_action(text: str) -> bool:
return 0 < len(stripped) < REPROMPT_MAX_CHARS and INTENT_SIGNAL.search(stripped) is not None
-# Leading marks are kept unless they are quotes or brackets, so ".NET" survives;
-# stripping all non-word chars would collapse "C++" and "C#" to the same token.
+# Leading marks are kept unless they are quotes or brackets, so ".NET" survives; stripping all non-word chars would
+# collapse "C++" and "C#" to the same token.
_REPEAT_TRAIL_PUNCT = ".,;:!?\"'`()[]{}<>‘’“”"
_REPEAT_LEAD_PUNCT = "\"'`([{‘“"
@@ -237,24 +233,22 @@ def _normalize_for_repeat(text: str) -> str:
words = []
for word in text.lower().split():
stripped = word.rstrip(_REPEAT_TRAIL_PUNCT).lstrip(_REPEAT_LEAD_PUNCT)
- # Keep marks-only tokens: "value is 5" and "value is < 5" differ, and
- # dropping the "<" threw the corrected attempt away.
+ # Keep marks-only tokens: "value is 5" and "value is < 5" differ, and dropping the "<" threw the corrected
+ # attempt away.
words.append(stripped or word)
return " ".join(words)
-# A nudge that just gets the same answer back has not worked, so stop there.
-# Exact after normalisation, deliberately. Every relaxation tried here lost a real
-# correction: a similarity ratio is length dependent (one changed token in a 50-word
-# plan still scored 0.98), a set ignores order ("cats not dogs"), and ignoring filler
-# words eats the target itself ("The Who", "OK Go"). A missed repeat costs one nudge
-# out of MAX_ACT_REPROMPTS; a false one strands the plan unexecuted.
+# A nudge that just gets the same answer back has not worked, so stop there. Exact after normalisation. Every relaxation
+# tried here lost a real correction: a similarity ratio is length dependent (one changed token in a 50-word plan still
+# scored 0.98), a set ignores order ("cats not dogs"), and ignoring filler words eats the target itself ("The Who", "OK
+# Go"). A missed repeat costs one nudge out of MAX_ACT_REPROMPTS; a false one strands the plan unexecuted.
def is_reprompt_repeat(text: str, previous: str) -> bool:
return is_reprompt_restatement(text, previous)
-# Same comparison, different decision: this one discards the turn. An appended answer
-# must not match, and deletions flip meaning ("is not supported" -> "is supported").
+# Same comparison, different decision: this one discards the turn. An appended answer must not match, and deletions flip
+# meaning ("is not supported" -> "is supported").
def is_reprompt_restatement(text: str, previous: str) -> bool:
if not previous:
return False
@@ -272,9 +266,9 @@ def reprompt_to_act_message(tool_hint: str) -> str:
)
-# How much of the unfinished thought is carried into the continuation. The point is to
-# resume, not to replay: the whole thought is what filled the window, so putting it back
-# reproduces the same ending. The tail is the part still being worked on.
+# How much of the unfinished thought is carried into the continuation. The point is to resume, not to replay: the whole
+# thought is what filled the window, so putting it back reproduces the same ending. The tail is the part still being
+# worked on.
_LENGTH_PROGRESS_TAIL_CHARS = 600
@@ -320,8 +314,8 @@ def thinking_exhausted_message(context_length: Optional[int] = None) -> str:
identical symptom is likewise to lower reasoning effort, because no amount of
retrying fits a thought that did not fit the first time.
"""
- # Built by substitution rather than `.format` on a pre-spaced fragment, which produced
- # "spent its whole reply of the 4096-token window on reasoning".
+ # Built by substitution rather than `.format` on a pre-spaced fragment, which produced "spent its whole reply of the
+ # 4096-token window on reasoning".
window = f"{context_length}-token " if context_length else ""
return (
f"The model spent the whole of its {window}window on reasoning and had none "
@@ -369,17 +363,16 @@ def continue_after_length_message() -> str:
# Qwen / Hermes ``<tool_call>{json}``.
_TC_JSON_START_RE = re.compile(r"<tool_call>\s*\{")
-# Qwen3.5 ``<function=name>`` and the attribute form ``<function name="name">``
-# (MiniCPM-5, MiniMax-M2); name class ``[\w.\-]+`` lands in group(1) or group(2).
+# Qwen3.5 ``<function=name>`` and the attribute form ``<function name="name">`` (MiniCPM-5, MiniMax-M2); name class
+# ``[\w.\-]+`` lands in group(1) or group(2).
_TC_FUNC_START_RE = re.compile(r'<function(?:=([\w\.\-]+)|\s+name="([\w\.\-]+)")>\s*')
-# Body ends at ``</tool_call>`` (Hermes) or ``</function>`` (Qwen3.5 / MiniCPM-5)
-# so it stops at the close even when prose follows (else prose leaked into args).
+# Body ends at ``</tool_call>`` (Hermes) or ``</function>`` (Qwen3.5 / MiniCPM-5) so it stops at the close even when
+# prose follows (else prose leaked into args).
_TC_END_TAG_RE = re.compile(r"</(?:tool_call|function)>")
# Byte-identical to the healer's; shared so the pair cannot drift.
_TC_FUNC_CLOSE_RE = _tool_healing._TC_FUNC_CLOSE_RE
-# Horizontal whitespace only (``[^\S\n]*``, not ``\s*``) so the wrapping newline +
-# first-line indentation survive; ``_trim_param_value`` trims one newline, preserving
-# code indentation (SGLang qwen3_coder).
+# Horizontal whitespace only (``[^\S\n]*``, not ``\s*``) so the wrapping newline + first-line indentation survive;
+# ``_trim_param_value`` trims one newline, preserving code indentation (SGLang qwen3_coder).
_TC_PARAM_START_RE = re.compile(
r'<(?:parameter|param)(?:=([\w\.\-]+)|\s+name="([\w\.\-]+)")>[^\S\n]*'
)
@@ -390,28 +383,29 @@ _LLAMA3_PYTHON_TAG = "<|python_tag|>"
_LLAMA3_PY_CALL_RE = re.compile(
r"<\|python_tag\|>\s*([\w\.\-]+)\s*\.\s*call\s*\(",
)
-# Anchored at a fixed offset (char after ``<|python_tag|>``) plus the ``; NAME.call(``
-# chain separator; fixed-offset (not a free scan) ignores ``.call(`` inside JSON args.
+# Anchored at a fixed offset (char after ``<|python_tag|>``) plus the ``; NAME.call(`` chain separator; fixed-offset
+# (not a free scan) ignores ``.call(`` inside JSON args.
_LLAMA3_PY_CALL_HEAD_RE = re.compile(r"\s*([\w\.\-]+)\s*\.\s*call\s*\(")
_LLAMA3_CALL_CHAIN_RE = re.compile(r"\s*;\s*([\w\.\-]+)\s*\.\s*call\s*\(")
-# Llama-3 ``.call(k=v)`` kwarg tokens, hand-scanned below (not finditer) to stay
-# linear on a truncated body; finditer retries every offset of a long run (ReDoS).
+# Llama-3 ``.call(k=v)`` kwarg tokens, hand-scanned below (not finditer) to stay linear on a truncated body; finditer
+# retries every offset of a long run (ReDoS).
_LLAMA3_KEY_RE = re.compile(r"\w+")
_LLAMA3_WS_RE = re.compile(r"\s*")
-# ints, decimals (1.5, 1., .5) and sci notation; trailing ``(?![\w.])`` stops a token
-# like ``1.2.3`` being truncated to ``1.2`` (which would mis-parse the remainder).
+# ints, decimals (1.5, 1., .5) and sci notation
+# ints, decimals (1.5, 1., .5) and sci notation; trailing ``(?![\w.])`` stops a token like ``1.2.3`` being truncated to
+# ``1.2`` (which would mis-parse the remainder).
_LLAMA3_NUM_RE = re.compile(r"-?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?(?![\w.])")
_LLAMA3_LIT_RE = re.compile(r"true|false|null")
-# Mistral ``[TOOL_CALLS]`` trigger. v11+ chains them, each followed by a bare name
-# plus ``{json}`` (Magistral) or ``[ARGS]{json}`` (Ministral / Large 3).
+# Mistral ``[TOOL_CALLS]`` trigger. v11+ chains them, each followed by a bare name plus ``{json}`` (Magistral) or
+# ``[ARGS]{json}`` (Ministral / Large 3).
_MISTRAL_TRIGGER = "[TOOL_CALLS]"
_MISTRAL_ARGS_MARKER = "[ARGS]"
-# Mistral Small 3.2 emits ``name[CALL_ID]<id>[ARGS]{json}`` (absent on Ministral /
-# Magistral); llama.cpp distinguishes the two on ``[CALL_ID]`` (common/chat.cpp).
+# Mistral Small 3.2 emits ``name[CALL_ID]<id>[ARGS]{json}`` (absent on Ministral / Magistral); llama.cpp distinguishes
+# the two on ``[CALL_ID]`` (common/chat.cpp).
_MISTRAL_CALL_ID_MARKER = "[CALL_ID]"
-# Magistral wraps reasoning in ``[THINK]...[/THINK]``; a ``[TOOL_CALLS]`` inside
-# that block is chain-of-thought, not a real call.
+# Magistral wraps reasoning in ``[THINK]...[/THINK]``; a ``[TOOL_CALLS]`` inside that block is chain-of-thought, not a
+# real call.
_MISTRAL_THINK_OPEN = "[THINK]"
_MISTRAL_THINK_CLOSE = "[/THINK]"
_MISTRAL_V11_NAME_RE = re.compile(r"\s*([\w\.\-]+)\s*")
@@ -422,26 +416,28 @@ _DEEPSEEK_END = "<|tool▁calls▁end|>"
_DEEPSEEK_CALL_BEGIN = "<|tool▁call▁begin|>"
_DEEPSEEK_SEP = "<|tool▁sep|>"
_DEEPSEEK_CALL_END = "<|tool▁call▁end|>"
-# R1 wraps args in a ```json fence with a ``function`` prefix; V3/V3.1 do not.
-# Scanned with ``str.find`` -- the regex forms are O(N^2) on truncated bodies.
+# R1 wraps args in a ```json fence with a ``function`` prefix; V3/V3.1 do not. Scanned with ``str.find`` -- the regex
+# forms are O(N^2) on truncated bodies.
_DEEPSEEK_R1_FUNC_MARKER = "function" + _DEEPSEEK_SEP
_DEEPSEEK_R1_FENCE = "\n```json\n"
_DEEPSEEK_R1_CLOSE_RE = re.compile(r"```[\s\r\n]*" + re.escape(_DEEPSEEK_CALL_END))
-# GLM 4.5-4.7: ``<tool_call>NAME[\n]<arg_key>K</arg_key>...``; the lookahead also allows a
-# direct ``<arg_key>``/``</tool_call>`` (4.7 drops the newline, zero-arg calls close at once).
-# Name class ``[\w.\-]+`` keeps prose like ``<tool_call>not a call</tool_call>`` unparsed;
-# ``{`` stays with the Qwen JSON parser.
+# GLM 4.5-4.7: the lookahead also allows a direct ``<arg_key>``/``</tool_call>`` (4.7 drops the newline, zero-arg calls
+# close at once).
+# GLM 4.5-4.7: ``<tool_call>NAME[\n]<arg_key>K</arg_key>...``; the lookahead also allows a direct
+# ``<arg_key>``/``</tool_call>`` (4.7 drops the newline, zero-arg calls close at once). Name class ``[\w.\-]+`` keeps
+# prose like ``<tool_call>not a call</tool_call>`` unparsed; ``{`` stays with the Qwen JSON parser.
_GLM_TC_OPEN_RE = re.compile(r"<tool_call>\s*([\w.\-]+)\s*(?=\n|<arg_key>|</tool_call>)")
_GLM_TC_CLOSE = "</tool_call>"
_GLM_ARG_KEY_OPEN = "<arg_key>"
_GLM_ARG_KEY_CLOSE = "</arg_key>"
_GLM_ARG_VAL_OPEN = "<arg_value>"
_GLM_ARG_VAL_CLOSE = "</arg_value>"
-# Strings arrive raw, non-strings via tojson; only unambiguous JSON literals decode
-# (bare ``42``/``true``/``null`` stay strings).
+# Strings arrive raw, non-strings via tojson; only unambiguous JSON literals decode (bare ``42``/``true``/``null`` stay
+# strings).
_GLM_JSON_NUMERIC_RE = re.compile(r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?")
+# Kimi K2 / Moonshot (ASCII pipes).
# Kimi K2 / Moonshot (ASCII pipes). Id ``functions.NAME:IDX`` -- strip ``functions.``/``:N`` for the name.
_KIMI_SECTION_BEGIN = "<|tool_calls_section_begin|>"
_KIMI_SECTION_END = "<|tool_calls_section_end|>"
@@ -456,18 +452,18 @@ _GEMMA_STR_BEGIN = '<|"|>'
_GEMMA_STR_END = '<|"|>'
_GEMMA_TC_END = "<tool_call|>"
-# skip_special_tokens strips the wrapper and ``<|"|>`` markers, so streamed Gemma calls
-# arrive as bare ``call:NAME{k:v, ...}``; ``(?<!\w)`` avoids ``recall:``.
+# skip_special_tokens strips the wrapper and ``<|"|>`` markers, so streamed Gemma calls arrive as bare ``call:NAME{k:v,
+# ...}``
_GEMMA_BARE_TC_RE = re.compile(r"(?<!\w)call\s*:\s*([\w\.\-]+)\s*\{")
-# Partial leading prefix (``call``, ``call :``, ``call : name``) so the streaming buffer
-# holds it instead of leaking visible text.
+# Partial leading prefix (``call``, ``call :``, ``call : name``) so the streaming buffer holds it instead of leaking
+# visible text
_GEMMA_BARE_TC_PREFIX_RE = re.compile(r"(?<!\w)call\s*(?::\s*[\w\.\-]*)?$")
-# Keys start with a letter/underscore so ``10:00, 11:00`` in a value isn't misread as a new key.
+# Keys start with a letter/underscore so ``10:00, 11:00`` in a value isn't misread as a new key
_GEMMA_KEY_RE = re.compile(r"\s*([A-Za-z_][\w.\-]*)\s*:")
-# Shared with the healer, but brackets-only depth: a stray ``}`` must not end the span
-# early and leave the rest of a malformed call on screen.
+# Shared with the healer, but brackets-only depth: a stray ``}`` must not end the span early and leave the rest of a
+# malformed call on screen.
def _balanced_bracket_end(src: str, start: int) -> "int | None":
return _tool_healing._balanced_bracket_end(src, start, braces_count = False)
@@ -524,19 +520,16 @@ def _strip_mistral_closed_calls(text: str) -> str:
i = body_start
while i < n and text[i] in " \t\n\r":
i += 1
- # Array shape: ``[TOOL_CALLS] [...]``.
if i < n and text[i] == "[":
end = _balanced_bracket_end(text, i)
if end is None:
- # Truncated; let caller buffer / final-strip.
out.append(text[idx:])
break
cursor = end + 1
if text.startswith("</s>", cursor):
cursor += len("</s>")
continue
- # Single-object shape ``[TOOL_CALLS] { json }`` (no name/array): the parser
- # accepts it, so the display strip must remove it too (else it leaks).
+ # Single-object shape ``[TOOL_CALLS] { json }``
if i < n and text[i] == "{":
end = _balanced_brace_end(text, i)
if end is None:
@@ -546,7 +539,6 @@ def _strip_mistral_closed_calls(text: str) -> str:
if text.startswith("</s>", cursor):
cursor += len("</s>")
continue
- # Named shape: ``[TOOL_CALLS] name [ARGS]? { json }``.
name_match = _MISTRAL_V11_NAME_RE.match(text, i)
if not name_match:
out.append(text[idx:body_start])
@@ -569,15 +561,15 @@ def _strip_mistral_closed_calls(text: str) -> str:
out.append(text[idx:])
break
cursor = end + 1
- # Consume the optional EOS marker too, mirroring the array shape, so a
- # ``[TOOL_CALLS]name{json}</s>`` tail doesn't leave ``</s>`` as content.
+ # Consume the optional EOS marker too, mirroring the array shape, so a ``[TOOL_CALLS]name{json}</s>`` tail
+ # doesn't leave ``</s>`` as content.
if text.startswith("</s>", cursor):
cursor += len("</s>")
return "".join(out)
-# A real call may follow a reasoning close directly: the strips run per segment after
-# ``strip_outside_think``, so ``</think>call:NAME{..}`` starts its segment.
+# A real call may follow a reasoning close directly: the strips run per segment after ``strip_outside_think``, so
+# ``</think>call:NAME{..}`` starts its segment.
_GEMMA_ANCHOR_CLOSERS = ("</think>", "[/THINK]", "</thinking>")
@@ -609,8 +601,8 @@ def _strip_gemma_wrapperless_calls(text: str, enabled_tool_names: Optional[set]
cursor = _leading_json_value_end(text) or 0
if cursor:
out.append(text[:cursor])
- # Anchor origin, advanced only past calls this strip removed, so ``call:a{} call:b{}``
- # stays a pair while prose after a call does not inherit its anchor.
+ # Anchor origin, advanced only past calls this strip removed, so ``call:a{} call:b{}`` stays a pair while prose
+ # after a call does not inherit its anchor.
floor = cursor
while cursor < n:
m = _GEMMA_BARE_TC_RE.search(text, cursor)
@@ -620,7 +612,7 @@ def _strip_gemma_wrapperless_calls(text: str, enabled_tool_names: Optional[set]
keep = (enabled_tool_names is not None and m.group(1) not in enabled_tool_names) or (
not _gemma_call_is_anchored(text, m.start(), floor)
)
- brace = m.end() - 1 # _GEMMA_BARE_TC_RE consumes through the opening ``{``
+ brace = m.end() - 1
# Same boundary scanner as the parser: strip exactly what it consumed.
end = _gemma_body_brace_end(text, brace)
closed = end is not None
@@ -630,12 +622,11 @@ def _strip_gemma_wrapperless_calls(text: str, enabled_tool_names: Optional[set]
out.append(text[cursor:] if keep else text[cursor : m.start()])
break
if keep:
- # Prose, not a call.
out.append(text[cursor:next_index])
else:
out.append(text[cursor : m.start()])
floor = next_index
- cursor = next_index # already past the matching ``}``
+ cursor = next_index
return "".join(out)
@@ -653,7 +644,7 @@ def _strip_function_xml_calls(text: str, *, final: bool) -> str:
pos = 0
for idx, m in enumerate(starts):
if m.start() < pos:
- continue # opener already inside a previously consumed call span
+ continue # opener already inside a consumed call span
out.append(text[pos : m.start()])
next_start = starts[idx + 1].start() if idx + 1 < len(starts) else len(text)
close = None
@@ -765,11 +756,9 @@ def _strip_glm_calls(text: str, *, final: bool) -> str:
out.append(text[cursor : m.start()])
cursor = close + len(_GLM_TC_CLOSE)
continue
- # Truncated GLM call (no real close yet).
if final:
out.append(text[cursor : m.start()])
cursor = n
- # Non-final: leave the unclosed call (and any tail) buffered as-is.
break
out.append(text[cursor:])
return "".join(out)
@@ -789,16 +778,16 @@ def strip_segment(
end-of-turn arms (markerless Gemma, open tails, trailing partial rehearsal).
"""
seg = _strip_mistral_closed_calls(segment)
- # Bare rehearsal ``name[ARGS]{json}`` and the Mistral name form, through the shared
- # balanced scan. Name-gated: an inactive ``foo[ARGS]{..}`` is prose and is kept.
+ # Bare rehearsal ``name[ARGS]{json}`` and the Mistral name form, through the shared balanced scan. Name-gated: an
+ # inactive ``foo[ARGS]{..}`` is prose and is kept.
seg = _tool_healing._strip_bracket_tag_calls(seg, enabled_tool_names = enabled_tool_names)
if seg_final:
# Markerless Gemma ``call:NAME{...}``, name-gated like the parse gate.
seg = _strip_gemma_wrapperless_calls(seg, enabled_tool_names)
# Scan, not regex, so a literal ``<function=...>`` inside a value stays data.
seg = _strip_function_xml_calls(seg, final = seg_final)
- # GLM 4.x: scan to the call's real </tool_call>, so a literal one inside a value is
- # data. Qwen <tool_call>{json} is left to the regex arms.
+ # GLM 4.x: scan to the call's real </tool_call>, so a literal one inside a value is data. Qwen <tool_call>{json} is
+ # left to the regex arms.
seg = _strip_glm_calls(seg, final = seg_final)
pats = _TOOL_ALL_PATS if seg_final else _TOOL_CLOSED_PATS
required_pairs = (
@@ -822,8 +811,8 @@ def strip_segment(
continue
scan_end = last_close + len(token)
if pat_idx == 3:
- # The Mistral array arm keeps consuming an optional ``\s*</s>`` PAST
- # the ``]`` the bound is taken from; without this the EOS survives.
+ # The Mistral array arm keeps consuming an optional ``\s*</s>`` PAST the ``]`` the bound is taken from;
+ # without this the EOS survives
eos = scan_end
while eos < len(seg) and seg[eos].isspace():
eos += 1
@@ -836,8 +825,7 @@ def strip_segment(
else:
seg = pat.sub("", seg)
if seg_final:
- # Trailing partial rehearsal the balanced scan cannot close; gated so prose
- # ``foo[ARGS] ...`` survives.
+ # Trailing partial rehearsal the balanced scan cannot close; gated so prose ``foo[ARGS] ...`` survives.
seg = _tool_healing.apply_tool_strip_patterns(
seg,
[_tool_healing._REHEARSAL_TAIL_STRIP_RE],
@@ -860,8 +848,8 @@ def strip_tool_markup(
and the markerless Gemma ``call:NAME{...}`` strip. ``None`` strips every closed call.
"""
if final:
- # Drop a leading Magistral ``[THINK]...[/THINK]`` at end-of-turn; its bracket
- # form is not the ``<think>`` the reasoning channel renders.
+ # Drop a leading Magistral ``[THINK]...[/THINK]`` at end-of-turn; its bracket form is not the ``<think>`` the
+ # reasoning channel renders.
text = _strip_mistral_reasoning(text)
def _strip_segment(segment: str, is_last: bool) -> str:
@@ -871,25 +859,24 @@ def strip_tool_markup(
enabled_tool_names = enabled_tool_names,
)
- # ``<think>`` / ``[THINK]`` reasoning is preserved verbatim (a rehearsed call inside it is
- # not executed, so it must not be stripped from display either); a literal think marker
- # inside a real call's arguments is that call's data and is stripped with the call.
+ # ``<think>`` / ``[THINK]`` reasoning is preserved verbatim (a rehearsed call inside it is not executed, so it must
+ # not be stripped from display either); a literal think marker inside a real call's arguments is that call's data
+ # and is stripped with the call.
result = _tool_healing.strip_outside_think(text, _strip_segment)
return result.strip() if final else result
-# Every strip arm above is anchored on one of these literals, so text containing none of
-# them is returned unchanged; ``StreamingMarkupStripper`` uses that to skip the whole scan.
-# ``test_refactor_guard.py`` fuzzes the claim, so an arm added without its literal here
-# breaks that test rather than production.
+# Every strip arm above is anchored on one of these literals, so text containing none of them is returned unchanged;
+# ``StreamingMarkupStripper`` uses that to skip the whole scan. ``test_refactor_guard.py`` fuzzes the claim, so an arm
+# added without its literal here breaks that test rather than production.
_STRIP_SENTINELS = (
"<tool_call", # Qwen/Hermes open + GLM, and the <tool_call|> Gemma closer
"<|tool_call", # Gemma open, Kimi <|tool_call_begin|> / <|tool_calls_section_begin|>
"<function", # Qwen3.5 <function=name> and <function name="...">
- "[TOOL_CALLS]", # Mistral array + name forms
+ "[TOOL_CALLS]",
"[ARGS]", # reasoning-model rehearsal
"<|python_tag|>", # Llama-3 built-in tools
- "<|content_invoke_tool_json|>", # TML Inkling
+ "<|content_invoke_tool_json|>",
"<|", # DeepSeek's full-width pipe opens every one of its markers
"<think", # reasoning segmentation
"</think",
@@ -897,7 +884,7 @@ _STRIP_SENTINELS = (
"[/THINK",
"call", # markerless Gemma; ``call\s*:`` tolerates whitespace before the colon
)
-# Confirmed against ``_GEMMA_BARE_TC_RE``, not taken at face value: see ``_first_sentinel``.
+# Confirmed against ``_GEMMA_BARE_TC_RE``, not taken at face value: see ``_first_sentinel``
_GEMMA_BARE_SENTINEL = "call"
_SENTINEL_MAX_LEN = max(len(sentinel) for sentinel in _STRIP_SENTINELS)
# Bytes sampled from each end when checking that a buffer continues the previous one.
@@ -988,8 +975,6 @@ def _safe_cut(text: str, first: int) -> int:
if first <= 0:
return 0
if len(text) > _tool_healing._MAX_BRACKET_SCAN_CHARS:
- # Past this length ``_strip_bracket_tag_calls`` stands down. Cutting would bring
- # the tail back under the limit and turn the arm back on, so leave it whole.
return 0
cut = first
if text.startswith("[ARGS]", first):
@@ -999,16 +984,14 @@ def _safe_cut(text: str, first: int) -> int:
head = text[:cut]
if "`" in head or "~" in head:
return 0
- # A closer with no opener makes everything before it reasoning, which
- # ``_think_spans_outside_tool_markup`` decides from offset 0 of the segment. Trimming
- # moves offset 0, so cut only when there is nothing before that closer.
+ # A closer with no opener makes everything before it reasoning, which ``_think_spans_outside_tool_markup`` decides
+ # from offset 0 of the segment. Trimming moves offset 0, so cut only when there is nothing before that closer.
close_at = _unmatched_think_closer(text)
if 0 <= close_at and first < close_at:
return 0
- # ``_strip_function_xml_calls`` treats a ``<function>`` opener inside an unclosed
- # ``<parameter>`` as a literal in an argument value, and decides that from the text
- # before it. Cutting there loses the context and the nested markup leaks into the
- # display. The literal test keeps the common case off the scan.
+ # ``_strip_function_xml_calls`` treats a ``<function>`` opener inside an unclosed ``<parameter>`` as a literal in an
+ # argument value, and decides that from the text before it. Cutting there loses the context and the nested markup
+ # leaks into the display. The literal test keeps the common case off the scan.
if "<param" in head and _inside_open_parameter(text, cut):
return 0
return cut
@@ -1063,10 +1046,9 @@ class StreamingMarkupStripper:
seg_final: bool = True,
):
self._enabled_tool_names = enabled_tool_names
- # Whether the last segment gets the end-of-turn arms. The tool-generation loop
- # wants them (partial markup must never render); the final-answer loop strips with
- # ``final = False`` and does not. The incremental machinery is the same either
- # way: fewer arms on the same sentinels, and ``_safe_cut``'s guards stay a superset.
+ # Whether the last segment gets the end-of-turn arms. The tool-generation loop wants them (partial markup must
+ # never render); the final-answer loop strips with ``final = False`` and does not. The incremental machinery is
+ # the same either way: fewer arms on the same sentinels, and ``_safe_cut``'s guards stay a superset.
self._seg_final = seg_final
self._reset()
@@ -1076,18 +1058,16 @@ class StreamingMarkupStripper:
# Settled prefix: text below ``_floor`` is final, and ``_floor_out`` is its output.
self._floor = 0
self._floor_out = ""
- # True while nothing has been removed, which lets the sentinel-free path hand back
- # the caller's own string instead of rebuilding it every token.
+ # True while nothing has been removed, which lets the sentinel-free path hand back the caller's own string
+ # instead of rebuilding it every token
self._floor_identity = True
- # Set when non-reasoning markup sits at offset 0 of the unsettled text. Nothing can
- # then ever settle or be sliced off, so the bookkeeping is pure overhead on a strip
- # that must run in full anyway; this flag skips to it, so the worst case is no
- # slower than before.
+ # Set when non-reasoning markup sits at offset 0 of the unsettled text. Nothing can then ever settle or be
+ # sliced off, so the bookkeeping is pure overhead on a strip that must run in full anyway; this flag skips to
+ # it, so the worst case is no slower than before.
self._degenerate = False
- # Offset of the reasoning opener currently streaming, and how far its body is scanned.
self._open_at = -1
self._open_scanned = 0
- # Length and both end samples of the last buffer, never the buffer itself: see ``_note``.
+ # Length and both end samples of the last buffer, never the buffer itself: see ``_note``
self._seen_len = -1
self._seen_head = ""
self._seen_mid = ""
@@ -1117,18 +1097,16 @@ class StreamingMarkupStripper:
else:
return ""
body_start = first + len(opener)
- # Resume inside the body, overlapping enough that a closer or sentinel split
- # across two appends is still seen.
+ # Resume inside the body, overlapping enough that a closer or sentinel split across two appends is still seen
resume = body_start
if self._open_at == first and self._open_scanned > body_start:
resume = max(body_start, self._open_scanned - _SENTINEL_MAX_LEN + 1)
end = text.find(closer, resume)
if end < 0:
- # Still streaming. An unclosed block runs to EOF verbatim and the text before
- # it is markup-free, so the buffer is untouched unless something else needs
- # stripping. Scan from ``resume``, not from the opener: everything below it
- # was checked on an earlier token, and a reasoning body is most of a reasoning
- # model's answer, so restarting each token is the quadratic this class removes.
+ # Still streaming. An unclosed block runs to EOF verbatim and the text before it is markup-free, so the
+ # buffer is untouched unless something else needs stripping. Scan from ``resume``, not from the opener:
+ # everything below it was checked on an earlier token, and a reasoning body is most of a reasoning model's
+ # answer, so restarting each token is the quadratic this class removes.
if _first_sentinel(text, resume) >= 0:
self._open_at = -1
return ""
@@ -1170,8 +1148,7 @@ class StreamingMarkupStripper:
def _full_strip(self, text: str) -> str:
def _seg(segment: str, is_last: bool) -> str:
- # Streaming has no separate ``final`` pass, so the last segment takes the
- # end-of-turn arms and partial markup never renders.
+ # Streaming has no separate ``final`` pass
return strip_segment(
segment,
seg_final = is_last and self._seg_final,
@@ -1204,8 +1181,8 @@ class StreamingMarkupStripper:
loop at 64k tokens: 1.14s holding the buffer, 0.003s holding these three.
"""
self._seen_len = len(text)
- # A buffer at or under the sample size aliases itself here. Harmless: the copy
- # that forces is bounded by the sample size.
+ # A buffer at or under the sample size aliases itself here. Harmless: the copy that forces is bounded by the
+ # sample size.
self._seen_head = text[:_EXTENSION_SAMPLE]
self._seen_tail = text[-_EXTENSION_SAMPLE:]
mid = len(text) // 2
@@ -1240,12 +1217,11 @@ class StreamingMarkupStripper:
def strip(self, text: str) -> str:
if not self._is_extension(text):
- # Not an extension: the cached prefix no longer applies.
self._reset()
if self._degenerate:
- # Guarded on ``_floor``: with nothing settled the two arms are the same
- # expression, so the check would be two scans that cannot change the answer.
+ # Guarded on ``_floor``: with nothing settled the two arms are the same expression, so the check would be
+ # two scans that cannot change the answer.
out = (
self._floor_out + self._full_strip(text[self._floor :])
if self._floor and not self._needs_whole_buffer(text)
@@ -1254,25 +1230,20 @@ class StreamingMarkupStripper:
self._note(text)
return out
- # Offsets stay absolute so the buffer is never sliced just to look for a sentinel;
- # a slice is linear per token and puts back the cost this class removes. Loop so
- # several reasoning blocks can settle in one call.
while True:
if self._first < 0:
- # Resume where the last scan stopped, overlapping just enough to catch a
- # sentinel straddling two appends. This is what keeps the pre-markup
- # phase from rescanning the whole buffer per token.
+ # Resume where the last scan stopped, overlapping just enough to catch a sentinel straddling two
+ # appends. This is what keeps the pre-markup phase from rescanning the whole buffer per token.
resume = self._scanned_upto - _SENTINEL_MAX_LEN + 1
found = _first_sentinel(text, resume if resume > self._floor else self._floor)
if found < 0:
self._scanned_upto = len(text)
return self._unchanged(text)
if _is_provisional_call(text, found):
- # A ``call`` that only qualifies because the buffer ends inside it.
- # One more token decides it, so commit nothing and resume here next
- # time. Committing would send every later token through the
- # whole-buffer checks below, and the ordinary word ``call`` lands on
- # a token boundary often enough to matter.
+ # A ``call`` that only qualifies because the buffer ends inside it. One more token decides it, so
+ # commit nothing and resume here next time. Committing would send every later token through the
+ # whole-buffer checks below, and the ordinary word ``call`` lands on a token boundary often enough
+ # to matter.
self._scanned_upto = found
return self._unchanged(text)
self._first = found - self._floor
@@ -1283,15 +1254,15 @@ class StreamingMarkupStripper:
break
tail = text[self._floor :]
- # Markup at offset 0 of the unsettled text: the cut is 0 and stays 0 as the buffer
- # grows, and nothing can settle, so skip the bookkeeping from here on.
+ # Markup at offset 0 of the unsettled text: the cut is 0 and stays 0 as the buffer grows, and nothing can
+ # settle, so skip the bookkeeping from here on.
self._degenerate = self._first == 0
- # Recomputed per call, not cached: the sentinel may have been found before the run
- # preceding it finished streaming, so re-deriving keeps the cut exact.
+ # Recomputed per call, not cached: the sentinel may have been found before the run preceding it finished
+ # streaming, so re-deriving keeps the cut exact.
cut = _safe_cut(tail, self._first)
- # After the cut, not before: with nothing settled and nothing to trim, the
- # whole-buffer strip below already is the split one, so these two scans would
- # decide nothing and an answer with markup near its front would pay them per token.
+ # After the cut, not before: with nothing settled and nothing to trim, the whole-buffer strip below already is
+ # the split one, so these two scans would decide nothing and an answer with markup near its front would pay them
+ # per token.
if (cut or self._floor) and self._needs_whole_buffer(text):
out = self._full_strip(text)
self._note(text)
@@ -1304,9 +1275,9 @@ class StreamingMarkupStripper:
def _unchanged(self, text: str) -> str:
"""Result when nothing after the settled prefix needs stripping."""
self._note(text)
- # With nothing removed the answer is the caller's own string, so hand it straight
- # back rather than rebuilding it a token at a time. Returned, not kept: holding a
- # reference is exactly what would stop the caller growing it in place.
+ # With nothing removed the answer is the caller's own string, so hand it straight back rather than rebuilding it
+ # a token at a time. Returned, not kept: holding a reference is exactly what would stop the caller growing it in
+ # place.
return text if self._floor_identity else self._floor_out + text[self._floor :]
@@ -1314,22 +1285,20 @@ def has_tool_signal(text: str) -> bool:
return any(s in text for s in TOOL_XML_SIGNALS)
-# A Qwen/Hermes ``<tool_call>``/``<function=...>`` envelope whose arguments carry literal
-# DeepSeek/Kimi markers must parse as the OUTER call. Detect it opening before the first
-# marker so the pre-pass skips it.
+# A Qwen/Hermes ``<tool_call>``/``<function=...>`` envelope whose arguments carry literal DeepSeek/Kimi markers must
+# parse as the OUTER call. Detect it opening before the first marker so the pre-pass skips it.
_EMBEDDED_MARKER_RE = re.compile(
_DEEPSEEK_OPEN_RE_SRC + "|" + re.escape(_KIMI_SECTION_BEGIN) + "|" + re.escape(_KIMI_CALL_BEGIN)
)
-# Covers ``<function=NAME>`` and the attribute form. ``<|python_tag|>`` is Llama-3's
-# envelope too (built-in ``NAME.call(`` and custom ``{json}``), so a quoted DeepSeek/Kimi
-# example is data; the call-shaped lookahead mirrors the ``_TOOL_ALL_PATS`` python_tag arm
-# so a bare prose ``<|python_tag|>`` mention isn't treated as one.
+# Covers ``<function=NAME>`` and the attribute form. ``<|python_tag|>`` is Llama-3's envelope too (built-in
+# ``NAME.call(`` and custom ``{json}``), so a quoted DeepSeek/Kimi example is data; the call-shaped lookahead mirrors
+# the ``_TOOL_ALL_PATS`` python_tag arm so a bare prose ``<|python_tag|>`` mention isn't treated as one.
_OUTER_ENVELOPE_OPEN_RE = re.compile(
r'<tool_call>|<function(?:=|\s+name=")|<\|tool_call>'
r"|<\|python_tag\|>(?=\s*(?:\{|[A-Za-z_][\w.]*\())"
)
-# CLOSED outer envelopes, each spanning to its REAL final close so a literal
-# ``</tool_call>``/``</function>`` inside a value is data. Wrapped Gemma counts too.
+# CLOSED outer envelopes, each spanning to its REAL final close so a literal ``</tool_call>``/``</function>`` inside a
+# value is data. Wrapped Gemma counts too.
_OUTER_ENVELOPE_CLOSED_PATS = (
re.compile(r"<tool_call>(?:(?!<tool_call>).)*</tool_call>", re.DOTALL),
_TOOL_CLOSED_PATS[1],
@@ -1341,8 +1310,8 @@ def _marker_inside_leading_envelope(content: str, enabled_tool_names: Optional[s
first_marker = _EMBEDDED_MARKER_RE.search(content)
if first_marker is None:
return False
- # A leading bare-JSON or Mistral [TOOL_CALLS] call is an outer envelope too:
- # a DS/Kimi marker in its argument strings is data.
+ # A leading bare-JSON or Mistral [TOOL_CALLS] call is an outer envelope too: a DS/Kimi marker in its argument
+ # strings is data.
i = 0
n = len(content)
while i < n and content[i] in " \t\n\r":
@@ -1352,19 +1321,19 @@ def _marker_inside_leading_envelope(content: str, enabled_tool_names: Optional[s
if end is not None and i < first_marker.start():
name = _top_level_bare_json_name(content[i : end + 1])
if name is not None and (enabled_tool_names is None or name in enabled_tool_names):
- # The closed leading call owns the turn: a marker inside it is argument
- # data, one after it a trailing example (same rule as the XML envelopes below).
+ # The closed leading call owns the turn: a marker inside it is argument data, one after it a trailing
+ # example (same rule as the XML envelopes below).
return True
if name is not None and first_marker.start() <= end:
- # A disabled-name leading object is prose (can't own the turn), but a marker
- # inside its own strings stays data. A marker AFTER it falls through to the pre-pass.
+ # A disabled-name leading object is prose (can't own the turn), but a marker inside its own strings
+ # stays data. A marker AFTER it falls through to the pre-pass.
return True
elif content.startswith(_MISTRAL_TRIGGER, i):
end = _mistral_region_end(content, i)
if end is not None and i < first_marker.start():
return True
- # A closed outer call PRECEDING the first marker owns the turn; the pre-pass must
- # not steal a trailing example or argument data.
+ # A closed outer call PRECEDING the first marker owns the turn; the pre-pass must not steal a trailing example or
+ # argument data.
required_pairs = (
("</tool_call>", "<tool_call>"),
("</function>", "<function"),
@@ -1396,8 +1365,8 @@ def _marker_inside_leading_envelope(content: str, enabled_tool_names: Optional[s
marker = _EMBEDDED_MARKER_RE.search(residue)
if marker is None:
return True
- # A marker still stands; any opener left in the residue is UNCLOSED. One before the
- # marker is a truncated outer call holding the marker as data: skip the pre-pass.
+ # A marker still stands; any opener left in the residue is UNCLOSED. One before the marker is a truncated outer call
+ # holding the marker as data: skip the pre-pass.
opener = _OUTER_ENVELOPE_OPEN_RE.search(residue)
return opener is not None and opener.start() < marker.start()
@@ -1441,10 +1410,9 @@ def _xml_signal_inside_leading_mistral(content: str) -> bool:
first_xml = _first_foreign_tool_signal(content)
if first_xml is not None and first_xml < trig:
return False
- # Only plain prose precedes the trigger: a visible preface must not hand
- # the turn to a later XML literal (preamble-tolerant, like the
- # wrapperless-Gemma guard). Prose that merely mentions the marker has no
- # parseable region and keeps the normal order.
+ # Only plain prose precedes the trigger: a visible preface must not hand the turn to a later XML literal
+ # (preamble-tolerant, like the wrapperless-Gemma guard). Prose that merely mentions the marker has no parseable
+ # region and keeps the normal order.
return _mistral_region_end(content, trig) is not None
@@ -1497,9 +1465,8 @@ def _first_foreign_tool_signal(content: str) -> int | None:
attr = _ATTR_FUNC_OPEN_RE.search(content)
if attr is not None and (first is None or attr.start() < first):
first = attr.start()
- # DeepSeek/Kimi markers are foreign to a JSON envelope too: a marker inside a leading
- # object routes through the same guard (and, if disabled, the drop-and-parse-the-tail
- # recursion, so a real call after the object is still reached).
+ # DeepSeek/Kimi markers are foreign to a JSON envelope too: a marker inside a leading object routes through the same
+ # guard (and, if disabled, the drop-and-parse-the-tail recursion, so a real call after the object is still reached).
marker = _EMBEDDED_MARKER_RE.search(content)
if marker is not None and (first is None or marker.start() < first):
first = marker.start()
@@ -1534,20 +1501,18 @@ def _xml_signal_inside_leading_bare_json(content: str) -> bool:
if end is None:
return False
if _top_level_bare_json_name(content[i : end + 1]) is None:
- # A NAMELESS object that parses as real JSON is a structured answer / envelope too:
- # quoted markup is data, and the decline path drops it and parses the tail.
- # Non-JSON braced prose keeps the old behaviour.
+ # A NAMELESS object that parses as real JSON is a structured answer / envelope too: quoted markup is data, and
+ # the decline path drops it and parses the tail. Non-JSON braced prose keeps the old behaviour.
try:
json.loads(content[i : end + 1])
except ValueError:
return False
first_xml = _first_foreign_tool_signal(content)
- # The Mistral trigger is foreign to a JSON envelope too (its parser runs first).
trig = content.find(_MISTRAL_TRIGGER)
if trig >= 0 and (first_xml is None or trig < first_xml):
first_xml = trig
- # Inside the balanced body the signal is quoted data; after the closed object the
- # leading call still owns the turn (mirrors the leading-Mistral rule).
+ # Inside the balanced body the signal is quoted data; after the closed object the leading call still owns the turn
+ # (mirrors the leading-Mistral rule).
return first_xml is not None and i < first_xml
@@ -1559,14 +1524,13 @@ def _signal_inside_leading_wrapperless_gemma(
Mistral/bare-JSON leading guards). Markerless form, so gated on an enabled
name (``None`` keeps the name-agnostic behaviour)."""
first = _first_foreign_tool_signal(content)
- # The Mistral trigger is foreign to a Gemma call too (its parser runs first).
trig = content.find(_MISTRAL_TRIGGER)
if trig >= 0 and (first is None or trig < first):
first = trig
if first is None:
return False
- # A preamble before ``call:NAME{...}`` is normal; what matters is an ENABLED balanced
- # call beginning before the first foreign signal.
+ # A preamble before ``call:NAME{...}`` is normal; what matters is an ENABLED balanced call beginning before the
+ # first foreign signal.
cursor = 0
while True:
m = _GEMMA_BARE_TC_RE.search(content, cursor)
@@ -1580,8 +1544,8 @@ def _signal_inside_leading_wrapperless_gemma(
return False
if m.end() - 1 < first <= end:
return True
- # An enabled call that CLOSES before the signal still owns the turn (inside-or-after
- # rule, as for closed bare-JSON/Mistral envelopes), gated on an enabled name.
+ # An enabled call that CLOSES before the signal still owns the turn (inside-or-after rule, as for closed
+ # bare-JSON/Mistral envelopes), gated on an enabled name.
return enabled_tool_names is not None and end < first
@@ -1595,7 +1559,6 @@ def _disabled_gemma_call_end_containing_signal(
if enabled_tool_names is None:
return None
first = _first_foreign_tool_signal(content)
- # Mirror the enabled-call guard: the Mistral trigger is foreign here too.
trig = content.find(_MISTRAL_TRIGGER)
if trig >= 0 and (first is None or trig < first):
first = trig
@@ -1633,21 +1596,19 @@ def parse_tool_calls_from_text(
``enabled_tool_names`` gates only the markerless Llama-3.2 bare-JSON form (the
marker-based forms carry an explicit signal, so a disabled-tool name there is a
real call attempt). ``None`` keeps the name-agnostic behaviour."""
- # Drop Magistral [THINK]...[/THINK] BEFORE dispatch: a rehearsed call inside it must
- # never be promoted, and the parse path must agree with the display strip.
+ # Drop Magistral [THINK]...[/THINK] BEFORE dispatch: a rehearsed call inside it must never be promoted, and the
+ # parse path must agree with the display strip.
content = _strip_mistral_reasoning(content)
- # A leading bare-JSON value is decided FIRST: a string argument quoting tool markup
- # (XML or a Mistral trigger) must stay data, so the bare-JSON parser takes the outer
- # call before any other pass. Precedes the Mistral guard, whose preamble tolerance
- # would otherwise claim a trigger quoted inside the leading object.
+ # A leading bare-JSON value is decided FIRST: a string argument quoting tool markup (XML or a Mistral trigger) must
+ # stay data, so the bare-JSON parser takes the outer call before any other pass. Precedes the Mistral guard, whose
+ # preamble tolerance would otherwise claim a trigger quoted inside the leading object.
if _xml_signal_inside_leading_bare_json(content):
calls = _parse_llama3_bare_json(
content, id_offset = id_offset, enabled_tool_names = enabled_tool_names
)
if calls:
return calls
- # Disabled/example name: the leading object is content. Drop it and parse the tail.
i = 0
while i < len(content) and content[i] in " \t\n\r":
i += 1
@@ -1660,9 +1621,8 @@ def parse_tool_calls_from_text(
enabled_tool_names = enabled_tool_names,
)
- # A leading enabled wrapper-less Gemma call is decided BEFORE the Mistral guard: its
- # body reads as prose to the preamble tolerance below, so a quoted [TOOL_CALLS] would
- # otherwise steal the turn.
+ # A leading enabled wrapper-less Gemma call is decided BEFORE the Mistral guard: its body reads as prose to the
+ # preamble tolerance below, so a quoted [TOOL_CALLS] would otherwise steal the turn.
if _signal_inside_leading_wrapperless_gemma(content, enabled_tool_names):
calls = _parse_gemma_tool_calls(
content,
@@ -1673,8 +1633,8 @@ def parse_tool_calls_from_text(
if calls:
return calls
- # A DISABLED wrapper-less Gemma call is prose: drop the span and parse the tail BEFORE
- # the Mistral guard, whose preamble tolerance would otherwise parse a quoted trigger.
+ # A DISABLED wrapper-less Gemma call is prose: drop the span and parse the tail BEFORE the Mistral guard, whose
+ # preamble tolerance would otherwise parse a quoted trigger.
_prose_end = _disabled_gemma_call_end_containing_signal(content, enabled_tool_names)
if _prose_end is not None:
return parse_tool_calls_from_text(
@@ -1684,16 +1644,16 @@ def parse_tool_calls_from_text(
enabled_tool_names = enabled_tool_names,
)
- # A [TOOL_CALLS] call that is the first tool emission owns the turn: XML quoted in its
- # arguments or trailing prose is not promoted over it, nor does a prose preface forfeit it.
+ # A [TOOL_CALLS] call that is the first tool emission owns the turn: XML quoted in its arguments or trailing prose
+ # is not promoted over it, nor does a prose preface forfeit it.
if _xml_signal_inside_leading_mistral(content):
calls = _parse_mistral_tool_calls(
content, id_offset = id_offset, allow_incomplete = allow_incomplete
)
if calls:
- # A bare rehearsal ``name[ARGS]{..}`` after the Mistral call is a peer tool call,
- # not foreign XML the owns-the-turn guard protects against: promote it too so a
- # Mistral call and a rehearsal in one message both parse.
+ # A bare rehearsal ``name[ARGS]{..}`` after the Mistral call is a peer tool call, not foreign XML the
+ # owns-the-turn guard protects against: promote it too so a Mistral call and a rehearsal in one message both
+ # parse.
calls.extend(
_parse_bare_rehearsals(
content,
@@ -1703,11 +1663,11 @@ def parse_tool_calls_from_text(
)
return calls
- # DeepSeek/Kimi markers are unique, so try them first -- unless an outer envelope
- # opens before the first marker (then the marker is argument data).
+ # DeepSeek/Kimi markers are unique, so try them first -- unless an outer envelope opens before the first marker
+ # (then the marker is argument data).
if not _marker_inside_leading_envelope(content, enabled_tool_names):
- # Dispatch by earliest opener so a quoted DS example inside a Kimi call (or vice
- # versa) can't hijack the turn via fixed parser order.
+ # Dispatch by earliest opener so a quoted DS example inside a Kimi call (or vice versa) can't hijack the turn
+ # via fixed parser order
_ds = _DEEPSEEK_BEGIN_RE.search(content)
_ds_pos = _ds.start() if _ds else len(content)
_km_section = content.find(_KIMI_SECTION_BEGIN)
@@ -1723,9 +1683,9 @@ def parse_tool_calls_from_text(
if calls:
return calls
- # A leading MiniCPM/MiniMax attribute-form call owns the turn: tool_healing doesn't know
- # the <function name="..."> wrapper, so a <tool_call> quoted in its parameter would beat
- # the outer call. Any earlier signal keeps normal order.
+ # A leading MiniCPM/MiniMax attribute-form call owns the turn: tool_healing doesn't know the <function name="...">
+ # wrapper, so a <tool_call> quoted in its parameter would beat the outer call. Any earlier signal keeps normal
+ # order.
attr = _ATTR_FUNC_OPEN_RE.search(content)
if attr is not None:
first_other = None
@@ -1746,9 +1706,9 @@ def parse_tool_calls_from_text(
if calls:
return calls
- # A leading Llama-3 ``<|python_tag|>`` call owns the turn like the others: markup quoted
- # in a ``.call(...)`` argument is not promoted. tool_healing does not know the tag, so
- # gate it here. A foreign signal before the tag keeps normal order.
+ # A leading Llama-3 ``<|python_tag|>`` call owns the turn like the others: markup quoted in a ``.call(...)``
+ # argument is not promoted. tool_healing does not know the tag, so gate it here. A foreign signal before the tag
+ # keeps normal order.
py_tag = content.find(_LLAMA3_PYTHON_TAG)
if py_tag >= 0:
first_other = None
@@ -1766,11 +1726,10 @@ def parse_tool_calls_from_text(
if calls:
return calls
- # Qwen/Hermes, Qwen3.5 XML, Gemma 4, plus Mistral [TOOL_CALLS] / bare rehearsal
- # ``name[ARGS]{json}`` use the shared tool_healing parser (strict/Auto-Heal contract +
- # nested-marker, trailing-prose, and ``<|"|>`` quoted-string handling the GGUF path
- # relies on). ``enabled_tool_names`` gates the ambiguous bare-rehearsal form so an
- # inactive ``foo[ARGS]{..}`` stays prose.
+ # Qwen/Hermes, Qwen3.5 XML, Gemma 4, plus Mistral [TOOL_CALLS] / bare rehearsal ``name[ARGS]{json}`` use the shared
+ # tool_healing parser (strict/Auto-Heal contract + nested-marker, trailing-prose, and ``<|"|>`` quoted-string
+ # handling the GGUF path relies on). ``enabled_tool_names`` gates the ambiguous bare-rehearsal form so an inactive
+ # ``foo[ARGS]{..}`` stays prose.
calls = _tool_healing.parse_tool_calls_from_text(
content,
id_offset = id_offset,
@@ -1780,11 +1739,10 @@ def parse_tool_calls_from_text(
if calls:
return calls
- # Formats tool_healing does not cover; these run only after it finds
- # nothing, so a strict-rejected call is never re-healed here. Blank any
- # JSON/Gemma marker coverage first: markup inside a marker's span (even one
- # that failed to parse) is that call's data, not a sibling, so a nested
- # ``<function=...>`` / ``<|python_tag|>`` / ``[TOOL_CALLS]`` must not be promoted.
+ # Formats tool_healing does not cover; these run only after it finds nothing, so a strict-rejected call is never
+ # re-healed here. Blank any JSON/Gemma marker coverage first: markup inside a marker's span (even one that failed to
+ # parse) is that call's data, not a sibling, so a nested ``<function=...>`` / ``<|python_tag|>`` / ``[TOOL_CALLS]``
+ # must not be promoted.
fallback_content = content
coverage = _tool_healing.marker_coverage(content)
if coverage:
@@ -1794,25 +1752,24 @@ def parse_tool_calls_from_text(
chars[i] = " "
fallback_content = "".join(chars)
for parser in (
- _parse_glm_tool_calls, # GLM 4.x <tool_call>name
- _parse_function_xml, # <function name="..."> attribute form
- _parse_llama3_python_tag, # Llama-3 <|python_tag|>
- _parse_mistral_tool_calls, # Mistral [TOOL_CALLS]
+ _parse_glm_tool_calls,
+ _parse_function_xml,
+ _parse_llama3_python_tag,
+ _parse_mistral_tool_calls,
):
calls = parser(fallback_content, id_offset = id_offset, allow_incomplete = allow_incomplete)
if calls:
return calls
- # Llama-3.2 bare ``{"name":..., "parameters":...}`` (strict shape). Only a LEADING call
- # object matches and owns the turn, so an enabled ``call:NAME{...}`` in its arguments
- # stays data (Gemma never starts ``{``).
+ # Llama-3.2 bare ``{"name":..., "parameters":...}`` (strict shape). Only a LEADING call object matches and owns the
+ # turn, so an enabled ``call:NAME{...}`` in its arguments stays data (Gemma never starts ``{``).
calls = _parse_llama3_bare_json(
content, id_offset = id_offset, enabled_tool_names = enabled_tool_names
)
if calls:
return calls
- # Gemma wrapper-less ``call:NAME{...}``: markerless, so the same enabled-name gate applies.
+ # Gemma wrapper-less ``call:NAME{...}``: markerless, so the same enabled-name gate applies
return _parse_gemma_tool_calls(
content,
id_offset = id_offset,
@@ -1833,9 +1790,8 @@ def _parse_tool_call_json(
end = _balanced_brace_end(content, brace_start)
if end is None:
continue
- # Strict mode: a balanced JSON body that never closed its ``<tool_call>``
- # is a truncated call, not a finished one. Trailing prose after the close
- # is still tolerated (matches the GGUF strict path).
+ # Strict mode: a balanced JSON body that never closed its ``<tool_call>`` is a truncated call, not a finished
+ # one. Trailing prose after the close is still tolerated (matches the GGUF strict path).
if not allow_incomplete and not content[end + 1 :].lstrip().startswith("</tool_call>"):
continue
try:
@@ -1843,7 +1799,6 @@ def _parse_tool_call_json(
except (json.JSONDecodeError, ValueError):
continue
name = obj.get("name", "")
- # Accept ``arguments`` (Hermes/Qwen) and ``parameters`` (Llama-3 drift).
args = obj.get("arguments")
if args is None:
args = obj.get("parameters", {})
@@ -1892,8 +1847,8 @@ def _parse_function_xml(
allow_incomplete: bool = True,
) -> list[dict]:
out: list[dict] = []
- # Skip ``<function ...>`` openers that are literals inside an open parameter value,
- # else the nested marker is promoted to a second call and truncates the real argument.
+ # Skip ``<function ...>`` openers that are literals inside an open parameter value, else the nested marker is
+ # promoted to a second call and truncates the real argument.
func_starts = [
fm
for fm in _TC_FUNC_START_RE.finditer(content)
@@ -1904,10 +1859,9 @@ def _parse_function_xml(
func_name = fm.group(1) or fm.group(2)
body_start = fm.end()
next_func = func_starts[idx + 1].start() if idx + 1 < len(func_starts) else len(content)
- # The call ends at the FIRST </function> / </tool_call> not inside an open
- # parameter: a literal close in a code/search argument is skipped as data, and
- # prose after the real close isn't folded into the last argument (mirrors
- # _strip_function_xml_calls and tool_healing._func_close_index).
+ # The call ends at the FIRST </function> / </tool_call> not inside an open parameter: a literal close in a
+ # code/search argument is skipped as data, and prose after the real close isn't folded into the last argument
+ # (mirrors _strip_function_xml_calls and tool_healing._func_close_index).
close_match = None
for cm in _TC_END_TAG_RE.finditer(content, body_start, next_func):
if not _inside_open_parameter(content, cm.start()):
@@ -1954,8 +1908,7 @@ def _parse_function_xml(
val = _TC_PARAM_CLOSE_RE.sub("", raw_val) if param_closed else raw_val
args[pm.group(1) or pm.group(2)] = _trim_param_value(val)
- # Strict mode: a dangling parameter means the call was cut off; a closed
- # zero-parameter call stays valid.
+ # Strict mode: a dangling parameter means the call was cut off; a closed zero-parameter call stays valid.
if not allow_incomplete and param_unclosed:
continue
@@ -1997,8 +1950,8 @@ def _llama3_kv_value(body: str, p: int, n: int) -> tuple[Any, int | None]:
nm = _LLAMA3_NUM_RE.match(body, p)
if nm:
v = nm.group(0)
- # Scientific notation (1e-3, -2E+4, 0.5e2) and decimals decode as float; a bare
- # integer stays int. ``"." in v`` alone missed the exponent forms (1e-3 -> 1).
+ # Scientific notation (1e-3, -2E+4, 0.5e2) and decimals decode as float; a bare integer stays int. ``"." in v``
+ # alone missed the exponent forms (1e-3 -> 1).
return (float(v) if any(c in v for c in ".eE") else int(v)), nm.end() - p
lm = _LLAMA3_LIT_RE.match(body, p)
if lm:
@@ -2044,16 +1997,15 @@ def _parse_llama3_python_tag(
if _LLAMA3_PYTHON_TAG not in content:
return out
- # 1. ``NAME.call(...)`` built-in form, anchored to ``<|python_tag|>`` and optionally
- # ``; ``-chained within one emission. Anchoring to the tag boundary (not a free scan)
- # keeps a literal ``<|python_tag|>x.call(...)`` quoted in a custom-form JSON argument
- # from being mistaken for a real built-in call.
+ # ``NAME.call(...)`` built-in form, anchored to ``<|python_tag|>`` and optionally ``
+ # 1. ``NAME.call(...)`` built-in form, anchored to ``<|python_tag|>`` and optionally ``; ``-chained within one
+ # emission. Anchoring to the tag boundary (not a free scan) keeps a literal ``<|python_tag|>x.call(...)`` quoted in
+ # a custom-form JSON argument from being mistaken for a real built-in call.
pos = content.find(_LLAMA3_PYTHON_TAG)
truncated = False
while pos >= 0 and not truncated:
head = _LLAMA3_PY_CALL_HEAD_RE.match(content, pos + len(_LLAMA3_PYTHON_TAG))
if head is None:
- # Tag is the custom JSON form (``{...}``) or noise -- leave it to step 2.
break
name = head.group(1)
open_idx = head.end()
@@ -2082,8 +2034,8 @@ def _parse_llama3_python_tag(
if depth == 0:
break
i += 1
- # Truncated ``.call(...)`` with no closing paren: reject in strict mode
- # instead of executing a partial.
+ # Truncated ``.call(...)`` with no closing paren
+ # Truncated ``.call(...)`` with no closing paren: reject in strict mode instead of executing a partial.
if not allow_incomplete and depth > 0:
truncated = True
break
@@ -2107,8 +2059,9 @@ def _parse_llama3_python_tag(
# Past the consumed region: a second ``<|python_tag|>`` may carry more calls.
pos = content.find(_LLAMA3_PYTHON_TAG, i + 1)
- # 2. ``<|python_tag|>{"name":..., "parameters":...}``. ``raw_decode`` peels multiple
- # ``; ``-separated objects from one emission.
+ # ``<|python_tag|>{"name":..., "parameters":...}``
+ # 2. ``<|python_tag|>{"name":..., "parameters":...}``. ``raw_decode`` peels multiple ``; ``-separated objects from
+ # one emission.
if not out:
decoder = json.JSONDecoder()
idx = content.find(_LLAMA3_PYTHON_TAG)
@@ -2133,7 +2086,6 @@ def _parse_llama3_python_tag(
continue
name = obj.get("name") or obj.get("function") or ""
args = obj.get("parameters") if "parameters" in obj else obj.get("arguments", {})
- # Skip rather than fabricate ``{"value": args}`` for a non-dict/non-string value.
if isinstance(args, dict):
args_str = json.dumps(args)
elif isinstance(args, str):
@@ -2154,8 +2106,8 @@ def _parse_llama3_python_tag(
return out
-# Llama-3 special-token sentinels (chainable, any order) plus the role label the
-# template inserts between ``<|start_header_id|>`` and ``<|end_header_id|>``.
+# Llama-3 special-token sentinels (chainable, any order) plus the role label the template inserts between
+# ``<|start_header_id|>`` and ``<|end_header_id|>``.
_LLAMA3_BARE_JSON_SENTINELS = (
"<|begin_of_text|>",
"<|eot_id|>",
@@ -2222,12 +2174,12 @@ def _parse_llama3_bare_json(
name = obj.get("name") or obj.get("function") or ""
if not isinstance(name, str) or not name:
break
- # Markerless JSON is ambiguous: treat it as a call only when the name is an enabled
- # tool, else it is an ordinary JSON answer.
+ # Markerless JSON is ambiguous: treat it as a call only when the name is an enabled tool, else it is an ordinary
+ # JSON answer
if enabled_tool_names is not None and name not in enabled_tool_names:
break
- # ``parameters`` must be a dict (Llama-3 spec); ``arguments`` may be a dict or
- # JSON-string of one (OpenAI). Looser would fire on ``{"name":"x","parameters":"sentence"}``.
+ # ``parameters`` must be a dict (Llama-3 spec); ``arguments`` may be a dict or JSON-string of one (OpenAI).
+ # Looser would fire on ``{"name":"x","parameters":"sentence"}``.
if "parameters" in obj:
args = obj.get("parameters")
if not isinstance(args, dict):
@@ -2274,7 +2226,7 @@ def _parse_mistral_tool_calls(
if idx < 0:
return out
- # Disambiguate the first occurrence: array / single object (pre-v11), or bare-name (v11+).
+ # Disambiguate the first occurrence: array / single object (pre-v11), or bare-name (v11+)
j = idx + len(_MISTRAL_TRIGGER)
k = j
while k < len(content) and content[k] in " \t\n\r":
@@ -2297,8 +2249,7 @@ def _parse_mistral_tool_calls(
except (json.JSONDecodeError, ValueError):
pass
- # v11+: walk every ``[TOOL_CALLS]``, parsing ``name{json}`` or
- # ``name[ARGS]{json}`` after each trigger.
+ # v11+: walk every ``[TOOL_CALLS]``, parsing ``name{json}`` or ``name[ARGS]{json}`` after each trigger
pos = idx
while pos >= 0:
cur = pos + len(_MISTRAL_TRIGGER)
@@ -2372,8 +2323,8 @@ def _parse_mistral_array(
if depth == 0:
break
j += 1
- # An unclosed array (no matching ]) is a truncated call. In strict mode reject it
- # instead of recovering objects by hand below.
+ # An unclosed array (no matching ]) is a truncated call. In strict mode reject it instead of recovering objects by
+ # hand below.
if not allow_incomplete and depth != 0:
return out
body = content[start : j + 1] if depth == 0 else content[start:]
@@ -2389,8 +2340,8 @@ def _parse_mistral_array(
if not allow_incomplete:
return out
- # Healing path for unclosed arrays: walk top-level objects, advancing past each balanced
- # ``{...}`` instead of re-scanning from every ``{`` (quadratic ReDoS).
+ # Healing path for unclosed arrays: walk top-level objects, advancing past each balanced ``{...}`` instead of
+ # re-scanning from every ``{`` (quadratic ReDoS).
pos = 0
blen = len(body)
while pos < blen:
@@ -2413,8 +2364,6 @@ def _consume_mistral_call(obj_text: str, out: list[dict], id_offset: int) -> Non
if not isinstance(obj, dict):
return
name = obj.get("name") or ""
- # Mistral uses ``arguments``; accept the ``parameters`` alias too (sibling paths and
- # SGLang's base detector alias it) so an array object keyed on it keeps args.
args = obj.get("arguments")
if args is None:
args = obj.get("parameters", {})
@@ -2485,16 +2434,16 @@ def _parse_gemma_tool_calls(
indistinguishable from prose documenting the syntax, so a disabled/example
name must not be stolen as a call. ``None`` keeps the name-agnostic behaviour."""
out: list[dict] = []
- # The WRAPPED form (strict + nested-marker handling) is tool_healing's, which runs
- # first: defer content with a wrapped opener. A marker literal alone is not enough --
- # a wrapper-less call mentioning ``<|tool_call>`` would be lost if deferred.
+ # The WRAPPED form (strict + nested-marker handling) is tool_healing's, which runs first: defer content with a
+ # wrapped opener. A marker literal alone is not enough -- a wrapper-less call mentioning ``<|tool_call>`` would be
+ # lost if deferred.
if _GEMMA_TC_RE.search(content):
return out
- # A whole-content JSON value is a structured answer: quoted examples must not become calls.
+ # A whole-content JSON value is a structured answer: quoted examples must not become calls
if _whole_content_is_json_value(content):
return out
- # Manual cursor: resume AFTER each consumed balanced body so a nested ``call:OTHER{...}``
- # in an argument is never re-matched. A leading JSON answer's span is data -- scan after it.
+ # Manual cursor: resume AFTER each consumed balanced body so a nested ``call:OTHER{...}`` in an argument is never
+ # re-matched. A leading JSON answer's span is data -- scan after it.
cursor = _leading_json_value_end(content) or 0
while True:
m = _GEMMA_BARE_TC_RE.search(content, cursor)
@@ -2504,11 +2453,10 @@ def _parse_gemma_tool_calls(
body_start = m.end() - 1
end = _gemma_body_brace_end(content, body_start)
if end is None:
- # Unclosed call: nothing parseable follows (mirrors the strip contract);
- # scanning on would promote quoted argument text.
+ # Unclosed call: nothing parseable follows (mirrors the strip contract); scanning on would promote quoted
+ # argument text.
break
cursor = end + 1
- # Markerless: a disabled/example name is prose, not a call.
if enabled_tool_names is not None and name not in enabled_tool_names:
continue
body = content[body_start + 1 : end]
@@ -2588,7 +2536,7 @@ def _top_level_bare_json_name(probe: str) -> Optional[str]:
while i < n and probe[i] in " \t\r\n,":
i += 1
if i >= n or probe[i] == "}":
- # End of the object with no top-level ``"name"``: fall back to a recorded ``"function"`` alias.
+ # End of the object with no top-level ``"name"``: fall back to a recorded ``"function"`` alias
return function_value
if probe[i] != '"':
return None
@@ -2615,8 +2563,7 @@ def _top_level_bare_json_name(probe: str) -> Optional[str]:
return value if isinstance(value, str) else None
return None
if key == "function" and function_value is None and i < n and probe[i] == '"':
- # ``"function"`` aliases the call name. Record it but keep scanning: a top-level
- # ``"name"`` still wins.
+ # ``"function"`` aliases the call name.
try:
value, consumed = decoder.raw_decode(probe[i:])
except (json.JSONDecodeError, ValueError):
@@ -2625,8 +2572,8 @@ def _top_level_bare_json_name(probe: str) -> Optional[str]:
function_value = value
i += consumed
continue
- # Skip a non-name top-level value; a truncated one can't prove a top-level name
- # exists, so return None (keep the text).
+ # Skip a non-name top-level value; a truncated one can't prove a top-level name exists, so return None (keep the
+ # text).
if i < n and probe[i] == "{":
end = _balanced_brace_end(probe, i)
if end is None:
@@ -2643,7 +2590,6 @@ def _top_level_bare_json_name(probe: str) -> Optional[str]:
except (json.JSONDecodeError, ValueError):
return None
i += consumed
- # No top-level ``"name"`` key: fall back to the ``"function"`` alias if seen.
return function_value
@@ -2664,18 +2610,18 @@ def strip_leading_bare_json_call(text: str, enabled_tool_names: Optional[set] =
if not (probe.startswith("{") and ('"name"' in probe or '"function"' in probe)):
return probe.lstrip() if stripped_any else text
if enabled_tool_names is not None:
- # Only suppress when the leading object's TOP-LEVEL name is an enabled tool. A
- # nested ``"name"`` (e.g. {"result":{"name":"web_search",...}}) is data, not the
- # call name, so it must not gate the strip. An un-extractable name is kept.
+ # Only suppress when the leading object's TOP-LEVEL name is an enabled tool. A nested ``"name"`` (e.g.
+ # {"result":{"name":"web_search",...}}) is data, not the call name, so it must not gate the strip. An
+ # un-extractable name is kept.
name = _top_level_bare_json_name(probe)
if name not in enabled_tool_names:
return probe.lstrip() if stripped_any else text
end = _balanced_brace_end(probe, 0)
if end is None:
return "" # truncated bare-JSON call -- nothing recoverable
- # A closed object must have the CALL SHAPE the parser accepts (dict ``parameters``,
- # or dict / JSON-string ``arguments``). An ordinary JSON answer like
- # {"name":"web_search","result":"no call"} is content, so the strip keeps it visible.
+ # A closed object must have the CALL SHAPE the parser accepts (dict ``parameters``, or dict / JSON-string
+ # ``arguments``). An ordinary JSON answer like {"name":"web_search","result":"no call"} is content, so the strip
+ # keeps it visible.
try:
obj = json.loads(probe[: end + 1])
except (json.JSONDecodeError, ValueError):
@@ -2690,8 +2636,8 @@ def _bare_json_call_shaped(obj) -> bool:
"""The shape gate ``_parse_llama3_bare_json`` applies to a decoded object."""
if not isinstance(obj, dict):
return False
- # The parser requires a TOP-LEVEL name; a nested one (e.g. in a "result" value of an
- # ordinary JSON answer) is data, and stripping it name-agnostically would delete content.
+ # The parser requires a TOP-LEVEL name; a nested one (e.g. in a "result" value of an ordinary JSON answer) is data,
+ # and stripping it name-agnostically would delete content.
name = obj.get("name") or obj.get("function") or ""
if not isinstance(name, str) or not name:
return False
@@ -2753,8 +2699,8 @@ def _gemma_parse_value(
if text[i] == "[":
return _gemma_parse_array(text, i)
if text[i] in "\"'":
- # Raw-quoted string: delimiters inside are data (``{city:"New, York"}`` is one
- # value); returned unquoted like the top-level scalar coercion.
+ # Raw-quoted string: delimiters inside are data (``{city:"New, York"}`` is one value); returned unquoted like
+ # the top-level scalar coercion.
quote = text[i]
j = i + 1
n = len(text)
@@ -2766,8 +2712,8 @@ def _gemma_parse_value(
return text[i + 1 : j], j + 1, True
j += 1
return text[i + 1 :], n, False
- # Primitive / unquoted code: same delimiter rules as the top-level scan (bracket depth
- # + contextual quote openers hide commas and closers).
+ # Primitive / unquoted code: same delimiter rules as the top-level scan (bracket depth + contextual quote openers
+ # hide commas and closers)
end = i
n = len(text)
depth = 0
@@ -2798,8 +2744,8 @@ def _gemma_parse_value(
prev_raw = ch
end += 1
if end == i:
- # Stray delimiter where a value was expected: consume one char so callers always
- # advance (no infinite loop on malformed input).
+ # Stray delimiter where a value was expected: consume one char so callers always advance (no infinite loop on
+ # malformed input).
return "", i + 1, True
raw = text[i:end].strip()
if raw == "true":
@@ -2890,13 +2836,11 @@ def _gemma_parse_stripped_body(body: str) -> dict[str, Any]:
vstart = i
depth = 0
quote = ""
- # Contextual quote openers mirror _gemma_body_brace_end.
prev = ":"
prev_raw = ":"
while i < n:
ch = body[i]
if quote:
- # A ``, key:`` shape inside the quoted string is not a boundary.
if ch == "\\" and i + 1 < n:
i += 2
continue
@@ -2917,8 +2861,8 @@ def _gemma_parse_stripped_body(body: str) -> dict[str, Any]:
i += 1
raw_val = body[vstart:i].strip()
if raw_val[:1] in "{[":
- # Nested object/array: accept only a fully consumed, closed parse; a
- # truncated/malformed value falls back to the raw string.
+ # Nested object/array: accept only a fully consumed, closed parse; a truncated/malformed value falls back to
+ # the raw string.
parsed, end, closed = _gemma_parse_value(raw_val, 0)
out[key] = (
_gemma_strip_quoted_leaves(parsed)
@@ -3023,8 +2967,8 @@ def _parse_deepseek_tool_calls(
if not begin:
return out
scan_start = begin.end()
- # Envelope end OUTSIDE JSON strings: an argument may contain the literal end token,
- # and a raw find would truncate the call.
+ # Envelope end OUTSIDE JSON strings: an argument may contain the literal end token, and a raw find would truncate
+ # the call.
end_pos = _find_outside_json_strings(content, _DEEPSEEK_END, scan_start)
# Strict mode: an unclosed envelope is truncated; reject, don't heal to EOF.
if not allow_incomplete and end_pos < 0:
@@ -3032,7 +2976,6 @@ def _parse_deepseek_tool_calls(
scan_end = end_pos if end_pos >= 0 else len(content)
body = content[scan_start:scan_end]
- # R1 path first: ``function<|tool▁sep|>NAME\n```json\n{...}\n```<|tool▁call▁end|>``.
pos = 0
while pos < len(body):
fpos = body.find(_DEEPSEEK_R1_FUNC_MARKER, pos)
@@ -3047,7 +2990,6 @@ def _parse_deepseek_tool_calls(
continue
name = body[name_start:nl].strip()
json_start = nl + len(_DEEPSEEK_R1_FENCE)
- # Walk a balanced ``{`` even if the trailing fence is truncated.
if json_start >= len(body) or body[json_start] != "{":
pos = json_start
continue
@@ -3062,9 +3004,9 @@ def _parse_deepseek_tool_calls(
if not isinstance(args, dict):
pos = brace_end + 1
continue
- # The closing fence + <|tool▁call▁end|> must IMMEDIATELY follow the JSON, else an
- # unbounded search lands on a LATER call's terminator. Absent close: heal past the
- # JSON (strict rejects); later well-formed calls are still kept.
+ # The closing fence + <|tool▁call▁end|> must IMMEDIATELY follow the JSON, else an unbounded search lands on a
+ # LATER call's terminator. Absent close: heal past the JSON (strict rejects); later well-formed calls are still
+ # kept.
after = brace_end + 1
while after < len(body) and body[after] in " \t\r\n":
after += 1
@@ -3087,15 +3029,13 @@ def _parse_deepseek_tool_calls(
if out:
return out
- # V3 / V3.1: name then bare JSON. Use ``str.find`` for the sep marker and walk
- # back for the name (a ``[^\n<]+`` regex search is O(N^2) on truncated bodies).
pos = 0
while pos < len(body):
sep_pos = body.find(_DEEPSEEK_SEP, pos)
if sep_pos < 0:
break
- # Walk left from sep_pos to the name start; stop at ``\n`` (turn boundary), ``<``
- # (tag start), or ``>`` (end of an optional ``<|tool▁call▁begin|>``).
+ # Walk left from sep_pos to the name start; stop at ``\n`` (turn boundary), ``<`` (tag start), or ``>`` (end of
+ # an optional ``<|tool▁call▁begin|>``).
name_start = sep_pos
while name_start > pos and body[name_start - 1] not in "\n<>":
name_start -= 1
@@ -3109,9 +3049,6 @@ def _parse_deepseek_tool_calls(
brace_end = _balanced_brace_end(body, json_start)
if brace_end is None:
break
- # Strict mode: a real V3 call closes with the per-call <|tool▁call▁end|>; without
- # it the call is truncated/merged, so skip it but keep scanning for a later
- # well-formed call (matches Kimi strict).
if not allow_incomplete:
after = brace_end + 1
while after < len(body) and body[after] in " \t\r\n":
@@ -3138,8 +3075,8 @@ def _parse_deepseek_tool_calls(
},
}
)
- # Advance just past the JSON; seeking the optional <|tool▁call▁end|> could land on
- # a LATER call's end marker and skip the call between.
+ # Advance just past the JSON; seeking the optional <|tool▁call▁end|> could land on a LATER call's end marker and
+ # skip the call between
pos = brace_end + 1
return out
@@ -3172,9 +3109,8 @@ def _parse_glm_tool_calls(
args: dict[str, Any] = {}
valid = True
close = -1
- # Walk arg pairs directly against ``content``: a value may contain a literal
- # </tool_call>, so the real close is the </tool_call> before the next <arg_key>.
- # ``str.find`` keeps this linear.
+ # Walk arg pairs directly against ``content``: a value may contain a literal </tool_call>, so the real close is
+ # the </tool_call> before the next <arg_key>. ``str.find`` keeps this linear.
while True:
ks = content.find(_GLM_ARG_KEY_OPEN, apos)
tc = content.find(_GLM_TC_CLOSE, apos)
@@ -3190,24 +3126,23 @@ def _parse_glm_tool_calls(
while vstart < len(content) and content[vstart] in " \t\r\n":
vstart += 1
if not content.startswith(_GLM_ARG_VAL_OPEN, vstart):
- # Key without <arg_value>: strict rejects the call; Auto-Heal skips it.
if not allow_incomplete:
valid = False
apos = ke + len(_GLM_ARG_KEY_CLOSE)
continue
vs = vstart + len(_GLM_ARG_VAL_OPEN)
- # A first-match find on </arg_value> would truncate values containing literal
- # close tags and execute corrupted arguments.
+ # A first-match find on </arg_value> would truncate values containing literal close tags and execute
+ # corrupted arguments
ve = _glm_value_close(content, vs, strict = not allow_incomplete)
key = content[ks + len(_GLM_ARG_KEY_OPEN) : ke].strip()
if ve < 0:
- # Unclosed <arg_value>: strict rejects the whole call; Auto-Heal keeps the
- # partial value (a truncated query is not a no-arg call).
+ # Unclosed <arg_value>: strict rejects the whole call; Auto-Heal keeps the partial value (a truncated
+ # query is not a no-arg call).
if not allow_incomplete:
valid = False
break
- # Bound the healed value at the next structural tag, not EOF, so a value
- # missing only its </arg_value> can't swallow the markup after it.
+ # Bound the healed value at the next structural tag, not EOF, so a value missing only its </arg_value>
+ # can't swallow the markup after it.
nk = content.find(_GLM_ARG_KEY_OPEN, vs)
tc = content.find(_GLM_TC_CLOSE, vs)
bounds = [b for b in (nk, tc) if b >= 0]
@@ -3220,9 +3155,10 @@ def _parse_glm_tool_calls(
continue
raw_val = content[vs:ve]
apos = ve + len(_GLM_ARG_VAL_CLOSE)
- # Decode only unambiguous JSON literals; else keep the value RAW so whitespace
- # in string args survives (matches vLLM glm4_moe). ``"`` is left out of the
- # probe: a verbatim string's quotes are meaningful.
+ # Decode only unambiguous JSON literals; ``"`` is left out of the probe: a verbatim string's quotes are
+ # meaningful
+ # Decode only unambiguous JSON literals; else keep the value RAW so whitespace in string args survives
+ # (matches vLLM glm4_moe). ``"`` is left out of the probe: a verbatim string's quotes are meaningful.
probe = raw_val.strip()
if (
probe[:1] in "{["
@@ -3280,13 +3216,13 @@ def _parse_kimi_tool_calls(
if section_start < 0:
break
scan_start = section_start + len(_KIMI_SECTION_BEGIN)
- # Section end OUTSIDE JSON strings: an argument may contain the literal end token,
- # and a raw find would drop the later valid call.
+ # Section end OUTSIDE JSON strings: an argument may contain the literal end token, and a raw find would drop the
+ # later valid call.
section_end = _find_outside_json_strings(content, _KIMI_SECTION_END, scan_start)
scan_end = section_end if section_end >= 0 else len(content)
body = content[scan_start:scan_end]
- # Truncated tail: parse what we have, then exit. In strict mode a section with no
- # <|tool_calls_section_end|> is truncated; reject it instead.
+ # Truncated tail: parse what we have, then exit. In strict mode a section with no <|tool_calls_section_end|> is
+ # truncated; reject it instead.
if section_end < 0:
if allow_incomplete:
out.extend(
@@ -3302,8 +3238,8 @@ def _parse_kimi_tool_calls(
)
)
- # The section wrapper is optional (llama.cpp): a bare <|tool_call_begin|> call parses
- # as one section when the loop matched nothing.
+ # The section wrapper is optional (llama.cpp): a bare <|tool_call_begin|> call parses as one section when the loop
+ # matched nothing.
if not out and _KIMI_CALL_BEGIN in content:
out.extend(
_parse_kimi_section_body(
@@ -3333,13 +3269,13 @@ def _parse_kimi_section_body(
full_id = body[id_start:arg_begin].strip()
m = _KIMI_ID_RE.match(full_id)
if m:
- # group(1) is the whole name; do NOT split on ``.`` -- a dotted MCP name stays intact.
name = m.group(1)
else:
base = full_id.split(":")[0]
name = base[len("functions.") :] if base.startswith("functions.") else base
- # Drop bare-counter ids (``3``, ``42``) -- matches vLLM; SGLang infers the name
- # from the tool schema, which we don't have here.
+ # Drop bare-counter ids (``3``, ``42``) -- matches vLLM
+ # Drop bare-counter ids (``3``, ``42``) -- matches vLLM; SGLang infers the name from the tool schema, which we
+ # don't have here.
if name.isdigit():
json_start = arg_begin + len(_KIMI_ARG_BEGIN)
brace_end = (
@@ -3353,7 +3289,6 @@ def _parse_kimi_section_body(
pos = brace_end + 1
continue
json_start = arg_begin + len(_KIMI_ARG_BEGIN)
- # Balanced brace lets a truncated trailing end marker still surface a call.
while json_start < len(body) and body[json_start] in " \t\n\r":
json_start += 1
if json_start >= len(body) or body[json_start] != "{":
@@ -3361,8 +3296,8 @@ def _parse_kimi_section_body(
continue
brace_end = _balanced_brace_end(body, json_start)
if brace_end is None:
- # Malformed / truncated JSON: skip this call but keep parsing later ones
- # instead of dropping the rest of the section (vLLM recovers them).
+ # Malformed / truncated JSON: skip this call but keep parsing later ones instead of dropping the rest of the
+ # section (vLLM recovers them)
nxt = body.find(_KIMI_CALL_BEGIN, json_start)
if nxt < 0:
break
@@ -3377,8 +3312,8 @@ def _parse_kimi_section_body(
pos = brace_end + 1
continue
if not allow_incomplete:
- # Strict mode: this call must close with <|tool_call_end|> before the next
- # <|tool_call_begin|>; otherwise it is truncated, so reject it.
+ # Strict mode: this call must close with <|tool_call_end|> before the next <|tool_call_begin|>; otherwise it
+ # is truncated, so reject it.
end_marker = body.find(_KIMI_CALL_END, brace_end + 1)
next_call = body.find(_KIMI_CALL_BEGIN, brace_end + 1)
if end_marker < 0 or (next_call >= 0 and end_marker > next_call):
@@ -3395,7 +3330,7 @@ def _parse_kimi_section_body(
},
}
)
- # Advance past the JSON; seeking <|tool_call_end|> could skip a following call
- # when this one's end marker is missing.
+ # Advance past the JSON; seeking <|tool_call_end|> could skip a following call when this one's end marker is
+ # missing
pos = brace_end + 1
return out
diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py
index 7bbfbd56a69..2022103409b 100644
--- a/studio/backend/core/inference/tool_loop_controller.py
+++ b/studio/backend/core/inference/tool_loop_controller.py
@@ -27,15 +27,14 @@ _CANONICAL_HEAL_ARG = {
"python": "code",
"terminal": "command",
"render_html": "code",
- # Not derivable: web_search declares no REQUIRED argument, because a call carrying
- # only `url` fetches that page without searching. A bare string is still a query,
- # which is what the old catch-all default got right for this tool and only this one.
+ # Not derivable: web_search declares no REQUIRED argument, because a call carrying only `url` fetches that page
+ # without searching. A bare string is still a query, which is what the old catch-all default got right for this tool
+ # and only this one.
"web_search": "query",
}
-# Where a bare string lands when the tool it was sent to has no argument that could hold
-# it. Read by `execute_tool`, which answers with what actually went wrong instead of
-# letting the tool report a missing key it was never given.
+# Where a bare string lands when the tool it was sent to has no argument that could hold it. Read by `execute_tool`,
+# which answers with what actually went wrong instead of letting the tool report a missing key it was never given.
UNPARSED_ARGUMENTS_KEY = "__unsloth_unparsed_arguments__"
@@ -63,17 +62,16 @@ def _looks_like_broken_json(raw: str) -> bool:
except json.JSONDecodeError as error:
if error.msg.startswith("Unterminated string") or error.pos >= len(text):
return True
- # A value cut mid-token reports at the token's START, not at the end of input, so
- # the two tests above miss `{"flag":tru` (Expecting value) and `{"n":1e`
- # (Expecting ',' delimiter). What they share is that everything from the failure
- # to the end is one unfinished token: no delimiter, no quote, no space.
- #
- # Excluded: a bad PROPERTY NAME. After `{` a bare word is malformed rather than
- # cut off, which is what keeps `{oops` and `{not json at all` healable.
+ # A value cut mid-token reports at the token's START, not at the end of input, so the two tests above miss
+ # `{"flag":tru` (Expecting value) and `{"n":1e` (Expecting ',' delimiter). What they share is that everything
+ # from the failure to the end is one unfinished token: no delimiter, no quote, no space. Excluded: a bad
+ # PROPERTY NAME. After `{` a bare word is malformed rather than cut off, which is what keeps `{oops` and `{not
+ # json at all` healable.
if error.msg.startswith("Expecting property name"):
return False
- # Excluded for the opposite reason: a COMPLETE document with something after it.
- # `{"a": 1} trailing` decodes fully and then finds junk, so nothing was lost.
+ # excluded for the opposite reason: `{"a": 1} trailing` decodes fully and then finds junk
+ # Excluded for the opposite reason: a COMPLETE document with something after it. `{"a": 1} trailing` decodes
+ # fully and then finds junk, so nothing was lost.
if error.msg.startswith("Extra data"):
return False
remainder = text[error.pos :]
@@ -154,8 +152,9 @@ def _heal_arg_key(tool_name: str, tool_schemas = None) -> "str | None":
key = _HEAL_ARG_CACHE.get(tool_name)
if key is not None or not tool_schemas:
return key
- # Not cached: the request's tools belong to the request, and caching them by name
- # would let one chat's MCP server decide another chat's healing.
+ # not cached: caching by name would let one chat's MCP server decide another chat's healing
+ # Not cached: the request's tools belong to the request, and caching them by name would let one chat's MCP server
+ # decide another chat's healing.
return _healable_keys_from(tool_schemas).get(tool_name)
@@ -198,9 +197,9 @@ class ToolCallDecision:
tool_name: str
arguments: dict[str, Any]
tool_call_id: str = ""
- # The id the card carries on screen. For an id-less call that is the
- # spelling the client minted, not the id the conversation replays;
- # otherwise the two are the same.
+ # for an id-less call this is the spelling the client minted
+ # The id the card carries on screen. For an id-less call that is the spelling the client minted, not the id the
+ # conversation replays; otherwise the two are the same.
card_call_id: str = ""
key: str = ""
provenance: dict[str, Any] = field(default_factory = dict)
@@ -243,9 +242,8 @@ class ToolCallDecision:
def tool_start_payload(self) -> dict[str, Any]:
"""Build the payload fields for a real tool_start event."""
fragment = self.unparsed_fragment
- # `raw` is the shape this module already uses for arguments it could not read into
- # a schema, so the card shows the model's own text under a name that means
- # something rather than an internal sentinel.
+ # `raw` is the shape this module already uses for arguments it could not read into a schema, so the card shows
+ # the model's own text under a name that means something rather than an internal sentinel.
arguments = {"raw": fragment} if fragment is not None else self.arguments
return {
"tool_name": self.tool_name,
@@ -267,12 +265,11 @@ class ToolCallDecision:
"type": "function",
"function": {
"name": self.tool_name,
- # Whatever goes here MUST parse as JSON. llama-server parses it while
- # rendering the template and answers 500 otherwise, which is what replaying
- # the fragment verbatim caused: it is unparseable by definition, that being
- # why it is here. So the replay is a short valid object that says the call
- # was cut off, and the tool result carries the detail. The fragment itself
- # is not worth resending -- it is the content that overflowed the window.
+ # Whatever goes here MUST parse as JSON. llama-server parses it while rendering the template and answers
+ # 500 otherwise, which is what replaying the fragment verbatim caused: it is unparseable by definition,
+ # that being why it is here. So the replay is a short valid object that says the call was cut off, and
+ # the tool result carries the detail. The fragment itself is not worth resending -- it is the content
+ # that overflowed the window.
"arguments": json.dumps(_unreadable_arguments_summary(fragment))
if fragment is not None
else canonical_arguments_text(self.arguments),
@@ -359,17 +356,18 @@ def canonical_tool_call_key(tool_name: str, arguments: Mapping[str, Any]) -> str
return f"{tool_name}:{canonical_args}"
-# "0"/"1" are left out: a native `0` arrives already typed, so they would mean two things.
+# "0"/"1" are left out: a native `0` arrives already typed, so they would mean two things
_SCHEMA_TRUE_WORDS = frozenset({"true", "yes"})
_SCHEMA_FALSE_WORDS = frozenset({"false", "no"})
-# Not ValueErrors, so a decode-shaped except would let deep model output escape as a 500.
+# not ValueErrors, so a decode-shaped except would let deep model output escape as a 500
_DECODE_ERRORS = (ValueError, RecursionError)
_LITERAL_ERRORS = (*_DECODE_ERRORS, SyntaxError, MemoryError)
-# A JSON string, open or closed; group 1 is the closing quote, which `endswith` cannot stand
-# in for because an open string can end on an escaped one. The `\?$` tail stops a started
-# match from ever failing, which would send `finditer` back over every later quote.
+# group 1 is the closing quote, which `endswith` cannot stand in for because an open string can end on an escaped one;
+# A JSON string, open or closed; group 1 is the closing quote, which `endswith` cannot stand in for because an open
+# string can end on an escaped one. The `\?$` tail stops a started match from ever failing, which would send `finditer`
+# back over every later quote.
_JSON_STRING_RE = re.compile(r'"(?:[^"\\]|\\.)*(?:(")|\\?$)', re.S)
_JSON_CLOSER = {"[": "]", "{": "}"}
@@ -381,7 +379,7 @@ def _balanced(segment: str, opened: "list[str]") -> str:
opened.append(_JSON_CLOSER[ch])
kept.append(ch)
elif ch in "]}":
- # The commonest slip: a closer of the wrong kind becomes the right one.
+ # the commonest slip: a closer of the wrong kind becomes the right one
if opened:
kept.append(opened.pop())
else:
@@ -418,7 +416,7 @@ _READ_KEYWORDS = frozenset(
"additionalProperties",
}
)
-# Cannot move a declaration out of reach: annotations, and constraints that only reject.
+# cannot move a declaration out of reach: annotations, and constraints that only reject
_INERT_KEYWORDS = frozenset(
{
"title",
@@ -543,8 +541,8 @@ def _coerce_declared_value(text: str, declared: str, repair: bool) -> Any:
number = float(stripped)
except (ValueError, OverflowError):
return text
- # "nan"/"inf" parse, but json.dumps writes them bare and the client rejects that.
- # Exactness is unguarded: float64 IS JSON's number type, so a JSON call agrees.
+ # "nan"/"inf" parse, but json.dumps writes them bare and the client rejects that. Exactness is unguarded:
+ # float64 IS JSON's number type, so a JSON call agrees.
if math.isfinite(number) and (declared == "number" or number.is_integer()):
return number
elif declared == "array":
@@ -602,8 +600,8 @@ def _coerce_by_property(value: Any, spec: Any, depth: int, repair: bool) -> Any:
if declared is None:
return value
value = _coerce_declared_value(value, declared, repair)
- # Descent needs the SAME declaration the conversion needs: without one a text value is
- # not decoded, so descending into an already-decoded one would make the syntaxes disagree.
+ # Descent needs the SAME declaration the conversion needs: without one a text value is not decoded, so descending
+ # into an already-decoded one would make the syntaxes disagree.
if declared == "object" and isinstance(value, Mapping):
nested = spec.get("properties")
nested = nested if isinstance(nested, Mapping) else {}
@@ -684,15 +682,13 @@ def coerce_tool_arguments(
except (json.JSONDecodeError, ValueError):
pass
if heal:
- # Healing exists for a model that sends its ONE argument as a bare string
- # instead of an object. Text that opens like JSON and fails to parse is not
- # that -- it is a broken object, usually one cut off mid-stream, and wrapping
- # it whole becomes the argument's value. Observed on `python`, which has a
- # single `code` argument and so was healable: a truncated call arrived as
- # `{"code":"html = ...`, the entire fragment was passed as the PROGRAM, and the
- # model read its own file back as `{"code":"html = ...` and spent the rest of
- # the turn convinced the sandbox had mangled it. Same defect `edit_file` had;
- # having a single string argument only hid it.
+ # Healing exists for a model that sends its ONE argument as a bare string instead of an object. Text that
+ # opens like JSON and fails to parse is not that -- it is a broken object, usually one cut off mid-stream,
+ # and wrapping it whole becomes the argument's value. Observed on `python`, which has a single `code`
+ # argument and so was healable: a truncated call arrived as `{"code":"html = ...`, the entire fragment was
+ # passed as the PROGRAM, and the model read its own file back as `{"code":"html = ...` and spent the rest of
+ # the turn convinced the sandbox had mangled it. Same defect `edit_file` had; having a single string
+ # argument only hid it.
key = (
None
if _looks_like_broken_json(raw_args)
@@ -700,11 +696,10 @@ def coerce_tool_arguments(
)
if key is not None:
return CoercedArguments({key: raw_args}, True)
- # No single argument this text could be. Inventing one used to default to
- # "query", which edit_file -- three required arguments, none of them a
- # query -- then reported as "'old_string' and 'new_string' must both be
- # strings": a type error blaming the model for a key it never sent, on a
- # call whose real problem was that its JSON never finished arriving.
+ # No single argument this text could be. Inventing one used to default to "query", which edit_file -- three
+ # required arguments, none of them a query -- then reported as "'old_string' and 'new_string' must both be
+ # strings": a type error blaming the model for a key it never sent, on a call whose real problem was that
+ # its JSON never finished arriving.
return CoercedArguments({UNPARSED_ARGUMENTS_KEY: raw_args}, False)
return CoercedArguments({"raw": raw_args}, False)
return CoercedArguments({}, False)
@@ -750,15 +745,15 @@ def status_for_tool(tool_name: str, arguments: Mapping[str, Any]) -> str:
if tool_name == "web_search":
url = str(arguments.get("url") or "").strip()
if url:
- # Bare hosts are fetched as https, so normalize first or the badge
- # stays generic for exactly the URLs the fetch layer accepts.
+ # bare hosts are fetched as https, so normalize first or the badge stays generic for exactly the URLs the
+ # fetch layer accepts
from core.inference.tools import _normalize_url_scheme
try:
parsed = urlparse(_normalize_url_scheme(url))
except ValueError:
- # Runs in prepare_call, outside the fetch's exception handler:
- # raising here kills the turn instead of returning "Blocked:".
+ # Runs in prepare_call, outside the fetch's exception handler: raising here kills the turn instead of
+ # returning "Blocked:".
return "Reading page..."
if parsed.scheme in ("http", "https") and parsed.hostname:
host = parsed.hostname
@@ -858,8 +853,8 @@ def _strip_files_sentinel(result: str) -> str:
entries = json.loads(result[payload_start:end])
except (ValueError, TypeError, RecursionError):
return result
- # Every entry, not just the list: the executor emits {"name": str, "size":
- # int | None}, and anything else is a tool that happened to print the marker.
+ # Every entry, not just the list: the executor emits {"name": str, "size": int | None}, and anything else is a tool
+ # that happened to print the marker.
if not isinstance(entries, list) or not all(_is_file_entry(e) for e in entries):
return result
return result[:start] + result[end:]
@@ -874,9 +869,8 @@ def _is_file_entry(entry: object) -> bool:
)
-# Only these emit the file envelope, and only their output is defused first. An
-# MCP tool or a fetched page ending in a well-formed __FILES__ line is content,
-# not an envelope, and stripping it would take that line away from the model.
+# Only these emit the file envelope, and only their output is defused first. An MCP tool or a fetched page ending in a
+# well-formed __FILES__ line is content, not an envelope, and stripping it would take that line away from the model.
_SANDBOX_TOOLS = frozenset({"python", "terminal"})
diff --git a/studio/backend/core/inference/tool_stream_exec.py b/studio/backend/core/inference/tool_stream_exec.py
index 9fa01933e6d..bd9c135508c 100644
--- a/studio/backend/core/inference/tool_stream_exec.py
+++ b/studio/backend/core/inference/tool_stream_exec.py
@@ -66,8 +66,8 @@ def search_images_kwargs(func: Callable[..., str], tool_name: str) -> dict[str,
return {"search_images": True} if search_images_enabled() else {}
-# Cadence of heartbeat events while a tool blocks with no output. Well under
-# common proxy idle caps (Cloudflare ~100 s, nginx default 60 s).
+# Cadence of heartbeat events while a tool blocks with no output. Well under common proxy idle caps (Cloudflare ~100 s,
+# nginx default 60 s).
TOOL_HEARTBEAT_INTERVAL_S = 10.0
# delay before the first approval keepalive so it cannot coalesce with the gated card.
@@ -76,16 +76,14 @@ TOOL_APPROVAL_FLUSH_DELAY_S = 0.05
# How often the wrapper wakes to poll for output / completion / cancellation.
_POLL_INTERVAL_S = 0.25
-# Upper bound on how long teardown waits for the worker once the stream is
-# closed or errors. A cancel-observing tool returns within this after
-# ``cancel_event`` is set; a cancel-ignoring one is a daemon left to finish on
-# its own rather than blocking teardown for the tool's full timeout.
+# Upper bound on how long teardown waits for the worker once the stream is closed or errors. A cancel-observing tool
+# returns within this after ``cancel_event`` is set; a cancel-ignoring one is a daemon left to finish on its own rather
+# than blocking teardown for the tool's full timeout.
_WORKER_JOIN_TIMEOUT_S = 5.0
-# Cap on total streamed live-output characters per tool call, bounding the
-# transient UI stream so a tight print loop cannot flood the SSE channel. Much
-# higher than the model-visible result cap (tools._MAX_OUTPUT_CHARS) since the
-# UI keeps the live stream as the displayed output when the result is truncated.
+# Cap on total streamed live-output characters per tool call, bounding the transient UI stream so a tight print loop
+# cannot flood the SSE channel. Much higher than the model-visible result cap (tools._MAX_OUTPUT_CHARS) since the UI
+# keeps the live stream as the displayed output when the result is truncated.
TOOL_OUTPUT_STREAM_MAX_CHARS = 400_000
_STREAM_CAPPED_NOTICE = "\n... (further live output not streamed)\n"
@@ -152,11 +150,11 @@ def stream_tool_execution(
done_sentinel = object()
outcome: dict[str, Any] = {}
- # Bound accepted output at the PRODUCER boundary: the consumer-side cap alone
- # wouldn't stop a fast worker enqueuing unboundedly while a slow SSE client
- # backpressures. Accept at most one char past the cap (so the consumer still
- # emits the capped notice) and drop the rest. The final result is captured
- # independently, so this never changes the byte-identical result.
+ # bound at the PRODUCER boundary: the consumer-side cap alone would not stop a fast worker enqueuing unboundedly
+ # Bound accepted output at the PRODUCER boundary: the consumer-side cap alone wouldn't stop a fast worker enqueuing
+ # unboundedly while a slow SSE client backpressures. Accept at most one char past the cap (so the consumer still
+ # emits the capped notice) and drop the rest. The final result is captured independently, so this never changes the
+ # byte-identical result.
accepted_output_chars = 0
accepted_output_lock = threading.Lock()
@@ -178,8 +176,8 @@ def stream_tool_execution(
except BaseException as exc: # noqa: BLE001 - re-raised on the caller side
outcome["error"] = exc
finally:
- # Posted after the result/error is recorded; wakes the consumer
- # immediately so fast tools pay no poll-interval latency.
+ # Posted after the result/error is recorded; wakes the consumer immediately so fast tools pay no
+ # poll-interval latency.
output_queue.put(done_sentinel)
worker = threading.Thread(
@@ -189,8 +187,9 @@ def stream_tool_execution(
)
worker.start()
- # Heartbeats are paced by counting idle queue polls rather than a wall clock
- # (tests patch ``time.monotonic`` globally, so the wrapper must not read it).
+ # paced by counting idle polls, not a wall clock: tests patch `time.monotonic` globally
+ # Heartbeats are paced by counting idle queue polls rather than a wall clock (tests patch ``time.monotonic``
+ # globally, so the wrapper must not read it).
idle_polls_per_heartbeat = max(1, int(round(heartbeat_interval_s / poll_interval_s)))
idle_polls = 0
streamed_chars = 0
@@ -226,9 +225,8 @@ def stream_tool_execution(
try:
item = output_queue.get(timeout = poll_interval_s)
except queue.Empty:
- # A disconnect sets cancel_event while the worker is silent;
- # surface a heartbeat this poll so the route regains control and
- # tears down at once, not after a full heartbeat interval.
+ # A disconnect sets cancel_event while the worker is silent; surface a heartbeat this poll so the route
+ # regains control and tears down at once, not after a full heartbeat interval.
if cancel_event is not None and cancel_event.is_set():
yield {"type": "heartbeat"}
continue
@@ -242,10 +240,9 @@ def stream_tool_execution(
break
if stream_capped:
- # Past the cap: drop this chunk and every queued sibling (see
- # _drain_and_drop). Pace with one time.sleep per poll (not
- # time.monotonic -- tests patch the clock), counted as an idle
- # poll so heartbeats keep flowing while the queue stays non-empty.
+ # Past the cap: drop this chunk and every queued sibling (see _drain_and_drop). Pace with one time.sleep
+ # per poll (not time.monotonic -- tests patch the clock), counted as an idle poll so heartbeats keep
+ # flowing while the queue stays non-empty.
_drain_and_drop()
if finished:
break
@@ -256,9 +253,8 @@ def stream_tool_execution(
yield {"type": "heartbeat"}
continue
- # Bound the join to the remaining budget so the crossing batch can't
- # allocate far past the cap (surplus is truncated below anyway); the
- # prefix is long enough that truncation stays byte-identical.
+ # Bound the join to the remaining budget so the crossing batch can't allocate far past the cap (surplus is
+ # truncated below anyway); the prefix is long enough that truncation stays byte-identical.
budget = TOOL_OUTPUT_STREAM_MAX_CHARS - streamed_chars
chunk = item + _drain_pending(max_chars = budget - len(item))
idle_polls = 0
@@ -275,13 +271,11 @@ def stream_tool_execution(
"text": chunk,
}
except BaseException:
- # The loop only raises when the consumer closes us early: an SSE
- # disconnect calls gen.close() (GeneratorExit at the yield) or the route
- # throws in. Signal cancellation so a cancel-observing tool returns; the
- # daemon worker is then abandoned (see finally). Re-raise so the caller
- # sees the real cause (GeneratorExit must not be swallowed). Runs ONLY on
- # abnormal exit, so the shared cancel_event is never set out from under
- # the next tool in a clean multi-tool turn.
+ # The loop only raises when the consumer closes us early: an SSE disconnect calls gen.close() (GeneratorExit at
+ # the yield) or the route throws in. Signal cancellation so a cancel-observing tool returns; the daemon worker
+ # is then abandoned (see finally). Re-raise so the caller sees the real cause (GeneratorExit must not be
+ # swallowed). Runs ONLY on abnormal exit, so the shared cancel_event is never set out from under the next tool
+ # in a clean multi-tool turn.
abnormal_exit = True
if cancel_event is not None:
try:
@@ -290,16 +284,16 @@ def stream_tool_execution(
pass
raise
finally:
- # Clean finish: the worker already recorded its result and queued the
- # sentinel we consumed, so this join returns at once. Abnormal exit:
- # cancel_event is set and the daemon worker abandoned, so join with a zero
- # timeout -- teardown never blocks the caller (the route may close this
- # generator on the event loop), and the daemon cannot outlive the process.
+ # Clean finish: the worker already recorded its result and queued the sentinel we consumed, so this join returns
+ # at once. Abnormal exit: cancel_event is set and the daemon worker abandoned, so join with a zero timeout --
+ # teardown never blocks the caller (the route may close this generator on the event loop), and the daemon cannot
+ # outlive the process.
worker.join(timeout = 0 if abnormal_exit else _WORKER_JOIN_TIMEOUT_S)
error = outcome.get("error")
if error is not None:
raise error
- # Returned verbatim (the loop's record_result handles non-str), so the
- # final tool result is byte-identical to a direct execute_tool call.
+ # returned verbatim, so the final tool result is byte-identical to a direct execute_tool call
+ # Returned verbatim (the loop's record_result handles non-str), so the final tool result is byte-identical to a
+ # direct execute_tool call.
return outcome.get("result")
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index b501702bc7e..a4bac415011 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -1834,6 +1834,7 @@ _SANDBOX_SITE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sa
# decides prompting, and fails closed: anything not provably read-only asks.
# Read-only commands allowed to run without confirmation in auto mode.
+# ── "Approve for me" (permission_mode="auto") safety detection ──────────────
_AUTO_SAFE_TERMINAL_COMMANDS = frozenset(
{
"ls",
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 4a964f8852d..84aaf2ffd9c 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -164,15 +164,17 @@ from .video_minimax_h3_te import (
)
from utils.hardware import clear_gpu_cache
-# Shared with the image backend so both pin every loader call to the same live cache root.
+# Shared with the image backend so both pin every loader call to the same live cache root
from core.inference.diffusion import hub_cache_dir
logger = get_logger(__name__)
-# Load kinds (mirror the image backend): gguf (single-file GGUF DiT + base repo), single_file (safetensors DiT), pipeline (full diffusers repo).
+# Load kinds (mirror the image backend): gguf (single-file GGUF DiT + base repo), single_file (safetensors DiT),
+# pipeline (full diffusers repo)
_MODEL_KINDS = frozenset({"gguf", "single_file", "pipeline"})
-# Vendor base repos allowed to load as full (non-GGUF) artifacts. Exact-match, lowercased, safetensors-only, no remote code.
+# Vendor base repos allowed to load as full (non-GGUF) artifacts. Exact-match, lowercased, safetensors-only, no remote
+# code.
_TRUSTED_NON_GGUF_VIDEO_REPOS = frozenset(
{
"lightricks/ltx-2",
@@ -181,7 +183,7 @@ _TRUSTED_NON_GGUF_VIDEO_REPOS = frozenset(
# Wan2.2 official diffusers base repos: safetensors-only, no remote code.
"wan-ai/wan2.2-ti2v-5b-diffusers",
"wan-ai/wan2.2-t2v-a14b-diffusers",
- # HunyuanVideo-1.5 community Diffusers repacks (tencent's own repo has no model_index.json).
+ # HunyuanVideo-1.5 community Diffusers repacks (tencent's own repo has no model_index.json)
"hunyuanvideo-community/hunyuanvideo-1.5-diffusers-480p_t2v",
"hunyuanvideo-community/hunyuanvideo-1.5-diffusers-720p_t2v",
"minimaxai/minimax-h3",
@@ -228,10 +230,10 @@ def assert_video_precision_available(
te_mode = normalize_te_quant(text_encoder_quant)
if pinned is None and te_mode is None:
return
- # One probe for both checks, asked of the card THIS load will use: the default one can be a
- # different generation, refusing a scheme the selected card supports or passing one it cannot.
- # SCOPED around the probes below, which use argument-less CUDA calls and bare "cuda"
- # allocations, and which the route reaches on a pooled thread that must not keep the pin.
+ # One probe for both checks, asked of the card THIS load will use: the default one can be a different generation,
+ # refusing a scheme the selected card supports or passing one it cannot. SCOPED around the probes below, which use
+ # argument-less CUDA calls and bare "cuda" allocations, and which the route reaches on a pooled thread that must not
+ # keep the pin.
with diffusion_device_scope(gpu_ordinal):
target = (
resolve_diffusion_device_target()
@@ -260,11 +262,10 @@ def _assert_video_precision_for_target(
"""The body of ``assert_video_precision_available``, run with the selected card current."""
pinned = normalize_transformer_quant(transformer_quant)
te_mode = normalize_te_quant(text_encoder_quant)
- # A modular-workflow family (MiniMax-H3) never reaches the memory plan the offload refusals
- # below reason about: load_pipeline hands it to _load_h3_modular_pipeline, which always uses
- # the ComponentsManager auto CPU offload and PINS a pre-quantized denoiser resident, so
- # 'balanced' / 'low_vram' do not put the quantised DiT under a Module.to() offload hook.
- # Refusing those picks here would 409 a combination that loads.
+ # A modular-workflow family (MiniMax-H3) never reaches the memory plan the offload refusals below reason about:
+ # load_pipeline hands it to _load_h3_modular_pipeline, which always uses the ComponentsManager auto CPU offload and
+ # PINS a pre-quantized denoiser resident, so 'balanced' / 'low_vram' do not put the quantised DiT under a
+ # Module.to() offload hook. Refusing those picks here would 409 a combination that loads.
forces_offload = not getattr(fam, "modular_workflow", None) and _memory_request_forces_offload(
memory_mode, False
)
@@ -278,10 +279,9 @@ def _assert_video_precision_for_target(
elif not dense_transformer_supported(target):
reason = dense_transformer_unsupported_reason(target)
elif forces_offload:
- # balanced and low_vram name their offload policy without measuring anything, and
- # offload hooks move modules with Module.to(), which torchao tensors do not survive.
- # load_pipeline therefore skips the dense build and the strict refusal landed after
- # acquire_for and the teardown had already evicted the resident model.
+ # balanced and low_vram name their offload policy without measuring anything, and offload hooks move modules
+ # with Module.to(), which torchao tensors do not survive. load_pipeline therefore skips the dense build and
+ # the strict refusal landed after acquire_for and the teardown had already evicted the resident model.
reason = (
f"'{normalize_memory_mode(memory_mode)}' memory places the DiT under CPU "
"offload, and torchao quantised tensors cannot be moved by the offload hooks"
@@ -296,9 +296,9 @@ def _assert_video_precision_for_target(
)
is None
):
- # An explicit scheme is never swapped for another, so a None means the family's
- # measured deny list, a torchao that cannot run here, or a GPU without the kernels.
- # They read the same to the selector and need different fixes from the user.
+ # An explicit scheme is never swapped for another, so a None means the family's measured deny list, a
+ # torchao that cannot run here, or a GPU without the kernels. They read the same to the selector and need
+ # different fixes from the user.
reason = explain_unusable_scheme(getattr(fam, "name", None), pinned)
if reason is not None:
raise RuntimeError(
@@ -306,22 +306,18 @@ def _assert_video_precision_for_target(
"transformer_quant", pinned, reason, off_label = "Off to run the DiT at bf16"
)
)
- # The mode the loader will ACTUALLY attempt, not the raw request: an explicit int8
- # on a family with no keep-bf16 schedule is rewritten to layerwise fp8 before
- # support is consulted, and that path needs no torchao. Refusing on the raw int8
- # rejected loads the runtime would run and report as fell_back -- Windows ROCm,
- # where the torchao stub kills int8 while fp8 still works.
- # A family with a HOSTED quantized conditioner never touches the generic path this gate
- # reasons about. Unsloth loads that artifact itself: INT8 storage, a Hadamard rotation and an
- # ordinary F.linear, no torchao and no fp8 tensor cores. Left to the code below, the request is
- # first rewritten int8 -> fp8 by effective_te_quant (H3 has no keep-bf16 schedule) and then
- # refused for want of hardware neither the rewrite nor the real loader needs, so a supported
- # CPU load comes back as a 409. Whether the artifact suits THIS base is decided at the seed,
- # which is not a host-level impossibility and so is not this gate's question.
- # PIPELINE loads only. The hosted conditioner is a Diffusers-path artifact; the native GGUF
- # path picks its Q2/Q4 encoder off the denoiser filename and discards text_encoder_quant
- # entirely, so exempting it would let an explicit request through to a load that answers it at
- # an unrelated precision -- the very thing the fail-closed contract exists to stop.
+ # The mode the loader will ACTUALLY attempt, not the raw request: an explicit int8 on a family with no keep-bf16
+ # schedule is rewritten to layerwise fp8 before support is consulted, and that path needs no torchao. Refusing on
+ # the raw int8 rejected loads the runtime would run and report as fell_back -- Windows ROCm, where the torchao stub
+ # kills int8 while fp8 still works. A family with a HOSTED quantized conditioner never touches the generic path this
+ # gate reasons about. Unsloth loads that artifact itself: INT8 storage, a Hadamard rotation and an ordinary
+ # F.linear, no torchao and no fp8 tensor cores. Left to the code below, the request is first rewritten int8 -> fp8
+ # by effective_te_quant (H3 has no keep-bf16 schedule) and then refused for want of hardware neither the rewrite nor
+ # the real loader needs, so a supported CPU load comes back as a 409. Whether the artifact suits THIS base is
+ # decided at the seed, which is not a host-level impossibility and so is not this gate's question. PIPELINE loads
+ # only. The hosted conditioner is a Diffusers-path artifact; the native GGUF path picks its Q2/Q4 encoder off the
+ # denoiser filename and discards text_encoder_quant entirely, so exempting it would let an explicit request through
+ # to a load that answers it at an unrelated precision -- the very thing the fail-closed contract exists to stop.
if (
model_kind == "pipeline"
and getattr(fam, "modular_workflow", None)
@@ -331,7 +327,6 @@ def _assert_video_precision_for_target(
te_effective = effective_te_quant(te_mode, getattr(fam, "name", None))
te_reason = None
if te_quant_needs_resident_weights(te_effective) and not torchao_quantize_importable():
- # Same as the image gate: the casters import torchao only after the pipeline is built.
te_reason = (
"torchao is not importable on this install, and these encoder modes are torchao "
"quantisations"
@@ -342,8 +337,8 @@ def _assert_video_precision_for_target(
"bf16, plus fp8 / int8 / NVFP4 support depending on the mode)"
)
elif te_quant_needs_resident_weights(te_effective) and forces_offload:
- # Same fence on the encoder: the loader reports those modes unsupported once offload is
- # active, and by then the resident model is gone. Layerwise fp8 is a dtype cast.
+ # Same fence on the encoder: the loader reports those modes unsupported once offload is active, and by then the
+ # resident model is gone. Layerwise fp8 is a dtype cast.
te_reason = (
f"'{normalize_memory_mode(memory_mode)}' memory places the text encoder under CPU "
"offload, and torchao quantised tensors cannot be moved by the offload hooks"
@@ -383,7 +378,8 @@ def _picked_gguf_arch(repo_id: str, gguf_filename: str) -> Optional[str]:
path = Path(repo_id).expanduser() / gguf_filename
if not path.is_file():
- # Not a local dir: resolve a cached HUB blob (no network). Probe active, legacy AND default cache roots, or a non-active-root GGUF 400s.
+ # Not a local dir: resolve a cached HUB blob (no network). Probe active, legacy AND default cache roots, or
+ # a non-active-root GGUF 400s.
from huggingface_hub import try_to_load_from_cache
cached = try_to_load_from_cache(repo_id, gguf_filename)
@@ -491,7 +487,8 @@ def _detect_load_family(
else None
)
if fam is None and gguf_filename and not family_override:
- # A renamed GGUF carries no family token, so resolve via general.architecture. No-backend archs still yield None (a 400).
+ # A renamed GGUF carries no family token, so resolve via general.architecture. No-backend archs still yield None
+ # (a 400)
arch = _picked_gguf_arch(repo_id, gguf_filename)
if arch:
fam = detect_video_family(repo_id, override = arch)
@@ -522,12 +519,12 @@ class _VideoLoadState:
dtype: str
kind: str
engine: str = "diffusers"
- # The torch ordinal this pipeline's weights were placed on, or None for an automatic pick.
- # Committed WITH the pipeline, so a load in flight never moves the resident model's card.
+ # The torch ordinal this pipeline's weights were placed on, or None for an automatic pick. Committed WITH the
+ # pipeline, so a load in flight never moves the resident model's card.
gpu_ordinal: Optional[int] = None
- # The card the weights are ACTUALLY on, automatic loads included. Only for re-pinning a worker
- # that a previous pinned load may have left on another card; everything reported reads
- # gpu_ordinal, so an automatic load still resolves a bare device. Mirrors the image backend.
+ # The card the weights are ACTUALLY on, automatic loads included. Only for re-pinning a worker that a previous
+ # pinned load may have left on another card; everything reported reads gpu_ordinal, so an automatic load still
+ # resolves a bare device. Mirrors the image backend.
placed_ordinal: Optional[int] = None
gguf_filename: Optional[str] = None
# Resident MiniMax-H3 denoiser partition, if any.
@@ -540,18 +537,22 @@ class _VideoLoadState:
backend_flags: Optional[dict] = None
attention_backend: Optional[str] = None
transformer_cache: Optional[str] = None
- # AUTO on a cache-capable DiT: generate() toggles FBCache across FBCACHE_MIN_STEPS; an explicit request is never toggled.
+ # AUTO on a cache-capable DiT: generate() toggles FBCache across FBCACHE_MIN_STEPS; an explicit request is never
+ # toggled.
cache_auto: bool = False
# Inputs the generation-time toggle re-applies (quantised threshold + override).
cache_quant_active: bool = False
cache_threshold: Optional[float] = None
- # Dense transformer quant engaged ("int8"|"fp8"|"nvfp4"|"mxfp8") or None. Pipeline-kind only; quantised in place onto the low-precision tensor cores.
+ # Dense transformer quant engaged or None.
+ # Dense transformer quant engaged ("int8"|"fp8"|"nvfp4"|"mxfp8") or None. Pipeline-kind only; quantised in place
+ # onto the low-precision tensor cores.
transformer_quant: Optional[str] = None
- # Text-encoder quant engaged ("fp8"|"fp8_dynamic"|"int8"|"nvfp4") or None. Often the largest resident; shrunk in place.
+ # Text-encoder quant engaged ("fp8"|"fp8_dynamic"|"int8"|"nvfp4") or None. Often the largest resident; shrunk in
+ # place.
text_encoder_quant: Optional[str] = None
- # MiniMax-H3 only: the pre-quantized denoiser was PINNED to the device and taken out of the
- # offload rotation, so it is resident alongside whatever component is running. The memory
- # preflight has to know, because that turns its floor from a max into a sum.
+ # MiniMax-H3 only: the pre-quantized denoiser was PINNED to the device and taken out of the offload rotation, so it
+ # is resident alongside whatever component is running. The memory preflight has to know, because that turns its
+ # floor from a max into a sum.
h3_denoiser_pinned: bool = False
resolved: Optional[dict] = None
@@ -577,8 +578,8 @@ class _VideoLoadingState:
base_repo: str
expected_bytes: Optional[int] = None
error: Optional[str] = None
- # Companion repos this load is ALSO pulling from, beyond repo_id / base_repo (the native H3
- # GGUF and component repos). Mirrors the image backend's _SdLoading.asset_repos.
+ # Companion repos this load is ALSO pulling from, beyond repo_id / base_repo (the native H3 GGUF and component
+ # repos). Mirrors the image backend's _SdLoading.asset_repos.
asset_repos: tuple[str, ...] = ()
@@ -605,21 +606,18 @@ def _h3_te_canonical(repo_id: Optional[str]) -> str:
return canonical_base(repo_id).strip().lower()
-# ── MiniMax-H3 modular precision tri-state ────────────────────────────────────
-# The modular workflow builds each component through its own from_pretrained, so nothing dense can
-# be quantised in place: a hosted checkpoint is the only way to run a component small, which makes
-# "unset" a real choice rather than a no-op.
-#
-# unset / "auto" -> whatever the backend decided for that component
-# "none" / "off" -> the released bfloat16 component, pinned
-# a scheme -> that scheme, or a recorded fallback
-#
-# The two components decided it differently on measured quality (see each site): the CONDITIONER
-# defaults to the hosted quantized artifact, which preserves the sample; the DENOISER keeps the
-# released weights, because both hosted checkpoints re-roll it. These helpers are the one place
-# the tri-state is read, so the plan, the estimate, the pull and the load cannot disagree.
+# MiniMax-H3 modular precision tri-state. unset/"auto" -> whatever the backend decided;
+# ── MiniMax-H3 modular precision tri-state ──────────────────────────────────── The modular workflow builds each
+# component through its own from_pretrained, so nothing dense can be quantised in place: a hosted checkpoint is the only
+# way to run a component small, which makes "unset" a real choice rather than a no-op. unset / "auto" -> whatever the
+# backend decided for that component "none" / "off" -> the released bfloat16 component, pinned a scheme -> that scheme,
+# or a recorded fallback The two components decided it differently on measured quality (see each site): the CONDITIONER
+# defaults to the hosted quantized artifact, which preserves the sample; the DENOISER keeps the released weights,
+# because both hosted checkpoints re-roll it. These helpers are the one place the tri-state is read, so the plan, the
+# estimate, the pull and the load cannot disagree.
+# ── MiniMax-H3 modular precision tri-state ────────────────────────────────────
def _h3_precision_unset(value: Optional[str]) -> bool:
"""True when the caller left this precision to the backend."""
return value is None or str(value).strip().lower() in ("", "auto")
@@ -745,8 +743,8 @@ def _h3_free_device_bytes(device: str) -> Optional[int]:
try:
from utils.hardware import trusted_mem_get_info
- # Windows ROCm over-reports free (#8403); pinning the denoiser on that
- # reading is how a card that is already full still looks roomy.
+ # Windows ROCm over-reports free (#8403); pinning the denoiser on that reading is how a card that is already
+ # full still looks roomy
return int(trusted_mem_get_info()[0])
except Exception: # noqa: BLE001 -- an unreadable card decides nothing
return None
@@ -782,10 +780,9 @@ def _h3_dense_denoiser_fits(sizes: Optional[tuple[int, int]], free_bytes: Option
return int(free_bytes) >= int(denoiser_bytes) + int(others_bytes)
-# The scheme an UNSET transformer_quant falls back to when the released denoiser cannot stay
-# resident. int8 rather than fp8: the two are the same 20.3 GB resident and neither is a faster
-# GEMM, but int8 scored closer to the released denoiser (SSIM 0.49 against 0.43) and, unlike fp8,
-# needs no per-row scale support from the card.
+# The scheme an UNSET transformer_quant falls back to when the released denoiser cannot stay resident. int8 rather than
+# fp8: the two are the same 20.3 GB resident and neither is a faster GEMM, but int8 scored closer to the released
+# denoiser (SSIM 0.49 against 0.43) and, unlike fp8, needs no per-row scale support from the card.
H3_AUTO_FALLBACK_SCHEME = "int8"
@@ -825,20 +822,19 @@ def _h3_auto_denoiser_scheme(
the pre-download decision, which runs while the previous pipeline is still resident.
Never raises, and every unanswerable question keeps the released denoiser."""
- # An explicit speed_mode="off" is a bit-exact contract, and a re-rolled sample is not bit-exact.
- # The conventional loader rewrites an unset precision to "off" under it for exactly this reason;
- # this workflow returns above that rewrite, so it applies the same rule itself.
+ # An explicit speed_mode="off" is a bit-exact contract, and a re-rolled sample is not bit-exact. The conventional
+ # loader rewrites an unset precision to "off" under it for exactly this reason; this workflow returns above that
+ # rewrite, so it applies the same rule itself.
if speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF:
return None
if not _h3_auto_precision_ok(target):
return None
- # The hosted checkpoint is a quantization of MiniMaxAI/MiniMax-H3's OWN denoiser, so the
- # substitution is only equivalent-modulo-precision against that base. Exact identity, not the
- # tolerant tail compare the availability lookup below applies: someone/MiniMax-H3 is a
- # different repo with different weights, and installing the upstream denoiser over a
- # derivative's is not a precision choice, it is a different model. An EXPLICIT request is
- # unaffected; this gate is on the substitution nobody asked for. Same bar, and the same
- # helper, as the conditioner's index gate in the loader.
+ # The hosted checkpoint is a quantization of MiniMaxAI/MiniMax-H3's OWN denoiser, so the substitution is only
+ # equivalent-modulo-precision against that base. Exact identity, not the tolerant tail compare the availability
+ # lookup below applies: someone/MiniMax-H3 is a different repo with different weights, and installing the upstream
+ # denoiser over a derivative's is not a precision choice, it is a different model. An EXPLICIT request is
+ # unaffected; this gate is on the substitution nobody asked for. Same bar, and the same helper, as the conditioner's
+ # index gate in the loader.
canonical = getattr(fam, "base_repo", None)
if not base_repo or not canonical:
return None
@@ -847,28 +843,27 @@ def _h3_auto_denoiser_scheme(
sizes = _h3_planned_denoiser_bytes(fam, te_scheme = te_scheme, dtype = dtype)
free_bytes = (free_reader or _h3_free_device_bytes)(device)
if sizes is None or free_bytes is None:
- # No reading is not evidence of a shortfall, and guessing wrong here changes the picture a
- # user gets. Keep the released denoiser, which is what happens today.
+ # No reading is not evidence of a shortfall, and guessing wrong here changes the picture a user gets. Keep the
+ # released denoiser, which is what happens today.
return None
if not video_family_prequant_available(
fam, H3_AUTO_FALLBACK_SCHEME, task = task, base_repo = base_repo
):
- # Asked per (scheme, PARTITION): a partition with no hosted checkpoint has no fallback, and
- # serving the other partition's would generate the wrong thing.
+ # Asked per (scheme, PARTITION): a partition with no hosted checkpoint has no fallback, and serving the other
+ # partition's would generate the wrong thing.
return None
from .diffusion_prequant import restricted_prequant_load_supported
if not restricted_prequant_load_supported(H3_AUTO_FALLBACK_SCHEME):
- # An install that cannot restrict the deserialization cannot open a checkpoint at all, and
- # this runs BEFORE the download plan: choosing one would drop the dense denoiser shards
- # for an artifact the loader is going to refuse.
+ # An install that cannot restrict the deserialization cannot open a checkpoint at all, and this runs BEFORE the
+ # download plan: choosing one would drop the dense denoiser shards for an artifact the loader is going to
+ # refuse.
return None
- # And the replacement has to fit BEFORE it is chosen. A torchao denoiser cannot ride the offload
- # rotation at all (it does not survive the mid-block move), so taking it means pinning it, which
- # turns the memory floor from a max into a sum: 20.3 GB resident PLUS whatever runs beside it.
- # Under text_encoder_quant="none" that sum is larger than the dense rotation it replaces, so a
- # card that renders today would refuse every generation afterwards. Where the replacement does
- # not fit either, the released denoiser in the rotation is the configuration that still runs.
+ # And the replacement has to fit BEFORE it is chosen. A torchao denoiser cannot ride the offload rotation at all (it
+ # does not survive the mid-block move), so taking it means pinning it, which turns the memory floor from a max into
+ # a sum: 20.3 GB resident PLUS whatever runs beside it. Under text_encoder_quant="none" that sum is larger than the
+ # dense rotation it replaces, so a card that renders today would refuse every generation afterwards. Where the
+ # replacement does not fit either, the released denoiser in the rotation is the configuration that still runs.
hosted_bytes = int(h3_transformer_resident_gb(H3_AUTO_FALLBACK_SCHEME) * 1000.0**3)
if not _h3_dense_denoiser_fits((hosted_bytes, sizes[1]), free_bytes):
return None
@@ -879,11 +874,13 @@ def _progress(phase: Optional[str], **extra: Any) -> dict[str, Any]:
return {"phase": phase, **extra}
-# ── dual-DiT (Wan2.2-A14B MoE) helpers ────────────────────────────────────────
-# The optimisation helpers and the quantiser all act on ``pipe.transformer``. Wan2.2-A14B is a dual-expert MoE
-# (transformer = high-noise, transformer_2 = low-noise), so present the second DiT AS ``pipe.transformer`` via a thin proxy.
+# dual-DiT (Wan2.2-A14B MoE) helpers.
+# ── dual-DiT (Wan2.2-A14B MoE) helpers ──────────────────────────────────────── The optimisation helpers and the
+# quantiser all act on ``pipe.transformer``. Wan2.2-A14B is a dual-expert MoE (transformer = high-noise, transformer_2 =
+# low-noise), so present the second DiT AS ``pipe.transformer`` via a thin proxy.
+# ── dual-DiT (Wan2.2-A14B MoE) helpers ────────────────────────────────────────
def _transformer_names(pipe: Any, fam: VideoFamily) -> tuple[str, ...]:
"""Attribute names of the denoiser(s) on ``pipe`` to optimise. Just
("transformer",) for a single-DiT family; also "transformer_2" for an MoE family
@@ -911,11 +908,11 @@ class _SecondDiTView:
return self._pipe.transformer_2
def __getattr__(self, name: str) -> Any:
- # Only reached for attrs not on the instance/class, so delegate to the real pipe.
return getattr(object.__getattribute__(self, "_pipe"), name)
def __setattr__(self, name: str, value: Any) -> None:
- # Writes land on the real pipe (else a helper reassignment vanishes); ``transformer`` mirrors onto the second expert.
+ # Writes land on the real pipe (else a helper reassignment vanishes); ``transformer`` mirrors onto the second
+ # expert.
pipe = object.__getattribute__(self, "_pipe")
setattr(pipe, "transformer_2" if name == "transformer" else name, value)
@@ -981,31 +978,28 @@ def _log_failed_generation(request_shape: Optional[dict[str, Any]], exc: BaseExc
return
if type(exc).__name__ == "_VideoGenerationCancelled":
return
- # A ValueError is client input feedback here, exactly as _run_generate treats it: it is
- # returned to the caller as a validation message and deliberately gets no
- # video.generate_failed record. Logging the request shape at ERROR for one would turn a
- # rejected resolution or frame count into a server-error entry.
+ # A ValueError is client input feedback here, exactly as _run_generate treats it: it is returned to the caller
+ # as a validation message and deliberately gets no video.generate_failed record. Logging the request shape at
+ # ERROR for one would turn a rejected resolution or frame count into a server-error entry.
if isinstance(exc, ValueError):
return
logger.error(
"video.generate_failed_request: %s",
" ".join(f"{k}={v}" for k, v in request_shape.items()),
)
- # An OOM under the math SDPA backend is the #8225 shape exactly: the score matrix is
- # quadratic in the token count, so the fix is fewer tokens, not a smaller checkpoint.
- # Named here so the next report arrives with the diagnosis already in it.
- #
- # Only when the run was on NATIVE SDPA. A recorded attention_backend means an explicit
- # kernel (AITER, xFormers, ...) engaged and torch's own dispatch was not what ran, so
- # probing native kernels would answer about code this generation never executed -- on a
- # ROCm card with AITER active that turns every OOM into a confident false diagnosis.
+ # An OOM under the math SDPA backend is the #8225 shape exactly: the score matrix is quadratic in the token
+ # count, so the fix is fewer tokens, not a smaller checkpoint. Named here so the next report arrives with the
+ # diagnosis already in it. Only when the run was on NATIVE SDPA. A recorded attention_backend means an explicit
+ # kernel (AITER, xFormers,...) engaged and torch's own dispatch was not what ran, so probing native kernels
+ # would answer about code this generation never executed -- on a ROCm card with AITER active that turns every
+ # OOM into a confident false diagnosis.
if (
_is_out_of_memory(exc)
and request_shape.get("attention_backend") is None
and sdpa_math_only(_probe_target(request_shape))
):
logger.error("video.generate_failed_request: %s", SDPA_MATH_ONLY_MESSAGE)
- except Exception: # noqa: BLE001 — diagnostics never mask the real failure
+ except Exception: # noqa: BLE001 - diagnostics never mask the real failure
pass
@@ -1033,7 +1027,7 @@ def _probe_target(request_shape: dict[str, Any]) -> Any:
candidate = getattr(torch, str(recorded).replace("torch.", ""), None)
if isinstance(candidate, torch.dtype):
dtype = candidate
- except Exception: # noqa: BLE001 — fall back to the probe's own default
+ except Exception: # noqa: BLE001 - fall back to the probe's own default
dtype = None
return types.SimpleNamespace(device = request_shape.get("device"), dtype = dtype)
@@ -1049,18 +1043,20 @@ class VideoBackend:
self._load_token = 0
self._cancel_event = threading.Event()
self._active_generate_cancel: Optional[threading.Event] = None
- # How many unloads / superseding loads are waiting on _generate_lock to free this pipeline. A generation queued behind
- # the active one holds no cancel event yet, so without this fence it could win the lock after an eject and denoise a
- # whole new clip against a pipeline being freed. A count, so concurrent teardowns each own their own release.
+ # How many unloads / superseding loads are waiting on _generate_lock to free this pipeline. A generation queued
+ # behind the active one holds no cancel event yet, so without this fence it could win the lock after an eject
+ # and denoise a whole new clip against a pipeline being freed. A count, so concurrent teardowns each own their
+ # own release.
self._teardown_waiters = 0
# Generation progress, written by the step callback / phase transitions.
self._gen: dict[str, Any] = {"active": False}
- # True from begin_generate() until its worker records a terminal state, so a second call is refused while it runs.
+ # True from begin_generate() until its worker records a terminal state, so a second call is refused while it
+ # runs.
self._generate_job_active = False
- # Which job the flag belongs to. The flag alone cannot tell "my job" from "the job that
- # replaced mine", so finalising is keyed on this. Compared by identity.
+ # Which job the flag belongs to. The flag alone cannot tell "my job" from "the job that replaced mine", so
+ # finalising is keyed on this. Compared by identity.
self._generate_job_token: Optional[object] = None
- # The OpenAI /v1/videos job id this run was started under, or None for a Studio-page run.
+ # The OpenAI /v1/videos job id this run was started under, or None for a Studio-page run
self._gen_video_id: Optional[str] = None
def _device_target(self, ordinal: Optional[int] = None) -> DiffusionDeviceTarget:
@@ -1081,9 +1077,9 @@ class VideoBackend:
touches the loaded pipeline goes through this: the weights are on ``state.gpu_ordinal``
and ``state.device`` is un-indexed, so an unpinned thread resolves to its own card."""
target = self._device_target(state.gpu_ordinal)
- # And an automatic load back onto its own card, in case this thread is a shared one a
- # previous pinned load left elsewhere. The image path reaches this through a pooled
- # executor; here it costs one no-op call to keep the two engines identical.
+ # And an automatic load back onto its own card, in case this thread is a shared one a previous pinned load left
+ # elsewhere. The image path reaches this through a pooled executor; here it costs one no-op call to keep the two
+ # engines identical.
pin_cuda_ordinal(state.placed_ordinal)
return target
@@ -1103,7 +1099,8 @@ class VideoBackend:
) -> VideoFamily:
"""Cheap, network-free validation shared by the route and the load path."""
kind = resolve_video_model_kind(gguf_filename, model_kind)
- # A -GGUF repo picked without a quant filename resolves to pipeline kind and would only fail in from_pretrained after eviction.
+ # A -GGUF repo picked without a quant filename resolves to pipeline kind and would only fail in from_pretrained
+ # after eviction
if kind == "pipeline" and repo_id.strip().lower().rstrip("/").endswith("-gguf"):
raise ValueError(
f"'{repo_id}' is a GGUF repo: pick one of its .gguf files "
@@ -1116,16 +1113,16 @@ class VideoBackend:
f"{', '.join(supported_video_family_names())}. If this is a variant of one "
f"of them, pass family_override with that family name."
)
+ # ── modular-workflow refusals, before anything heavier. Deliberately the FIRST thing after the family resolves:
+ # everything below reaches into diffusers (assert_pipeline_class_available, the transformer_class probe), so a
+ # refusal placed after them would be unreachable on any install whose diffusers cannot even be imported -- and
+ # these two are exactly the picks that cost the most to discover late.
# ── modular-workflow refusals, before anything heavier.
- # Deliberately the FIRST thing after the family resolves: everything below reaches into
- # diffusers (assert_pipeline_class_available, the transformer_class probe), so a refusal
- # placed after them would be unreachable on any install whose diffusers cannot even be
- # imported -- and these two are exactly the picks that cost the most to discover late.
if fam.modular_workflow and kind == "single_file":
- # A modular workflow has no single-file assembly: its components each load through
- # their own from_pretrained from the modular index, and nothing consumes a lone
- # .safetensors DiT. Today that only surfaces inside the loader, i.e. after ~98.7 GB
- # has downloaded AND after the resident pipeline was torn down to make room for it.
+ # A modular workflow has no single-file assembly: its components each load through their own from_pretrained
+ # from the modular index, and nothing consumes a lone .safetensors DiT. Today that only surfaces inside the
+ # loader, i.e. after ~98.7 GB has downloaded AND after the resident pipeline was torn down to make room for
+ # it.
gguf_hint = (
f", or a .gguf checkpoint from '{fam.gguf_repo}' for a quantized single-file load"
if fam.gguf_repo
@@ -1138,11 +1135,10 @@ class VideoBackend:
f"model{gguf_hint}."
)
if fam.modular_workflow and kind == "pipeline":
- # Metal cannot place a modular workflow at all. _load_h3_modular_pipeline hands every
- # non-CPU device to ComponentsManager.enable_auto_cpu_offload, which reads
- # torch.<device>.mem_get_info and raises NotImplementedError for a device module
- # without one; torch.mps has never exposed it. Refuse here, before ~145 GB downloads
- # and the resident pipeline is torn down to make room for it.
+ # Metal cannot place a modular workflow at all. _load_h3_modular_pipeline hands every non-CPU device to
+ # ComponentsManager.enable_auto_cpu_offload, which reads torch.<device>.mem_get_info and raises
+ # NotImplementedError for a device module without one; torch.mps has never exposed it. Refuse here, before
+ # ~145 GB downloads and the resident pipeline is torn down to make room for it.
if resolve_diffusion_device_target().device == "mps":
gguf_hint = (
f" Load a .gguf checkpoint from '{fam.gguf_repo}' instead, which runs on the "
@@ -1156,25 +1152,22 @@ class VideoBackend:
f"torch device exposing mem_get_info, and Metal (MPS) does not have it."
f"{gguf_hint}"
)
- # Same normaliser the load uses, so a malformed value raises the identical message it
- # would below and only a real scheme reaches the availability check.
+ # Same normaliser the load uses, so a malformed value raises the identical message it would below and only a
+ # real scheme reaches the availability check.
requested_scheme = normalize_transformer_quant(transformer_quant)
- # The base the modular load resolves for a pipeline kind IS repo_id, so a
- # variant-keyed checkpoint is judged against the one the load will ask for and
- # validation can never refuse a load that would have worked.
+ # The base the modular load resolves for a pipeline kind IS repo_id, so a variant-keyed checkpoint is judged
+ # against the one the load will ask for and validation can never refuse a load that would have worked.
quant_base = repo_id
- # "auto" is a request for the backend's own choice, not for a specific scheme, so it is
- # never refused: it simply stays on the released bfloat16 components.
+ # "auto" is a request for the backend's own choice, not for a specific scheme, so it is never refused: it
+ # simply stays on the released bfloat16 components.
if requested_scheme is not None and requested_scheme != TQ_AUTO:
- # Asked per (scheme, TASK), not per scheme. A family can host one denoiser per
- # workflow partition in the same repo -- MiniMax-H3 hosts a keyframe (fl2va, which
- # also covers text-only) and a reference (ref2va) checkpoint -- and the two share
- # module shapes, config and base model, so a mismatched one would install, pass
- # every metadata check, and have load_components skip the real denoiser,
- # generating from the wrong partition instead of failing. A pair with no hosted
- # checkpoint is refused here; there is no in-place quantise seam to fall back on,
- # because the workflow's own component loader builds the denoiser and letting the
- # request through would download ~98.7 GB of dense weights and then silently
+ # Asked per (scheme, TASK), not per scheme. A family can host one denoiser per workflow partition in the
+ # same repo -- MiniMax-H3 hosts a keyframe (fl2va, which also covers text-only) and a reference (ref2va)
+ # checkpoint -- and the two share module shapes, config and base model, so a mismatched one would
+ # install, pass every metadata check, and have load_components skip the real denoiser, generating from
+ # the wrong partition instead of failing. A pair with no hosted checkpoint is refused here; there is no
+ # in-place quantise seam to fall back on, because the workflow's own component loader builds the
+ # denoiser and letting the request through would download ~98.7 GB of dense weights and then silently
# ignore it.
if not video_family_prequant_available(
fam, requested_scheme, task = h3_task, base_repo = quant_base
@@ -1185,8 +1178,8 @@ class VideoBackend:
if available
else "Leave transformer_quant unset for this model."
)
- # Name the partition when one was asked for, so "fp8 is unavailable" does not
- # read as a claim about the whole family when it is only true of this task.
+ # Name the partition when one was asked for, so "fp8 is unavailable" does not read as a claim about
+ # the whole family when it is only true of this task.
task_label = f" {h3_task}" if h3_task else ""
raise ValueError(
f"transformer_quant '{requested_scheme}' is unavailable for "
@@ -1222,20 +1215,21 @@ class VideoBackend:
f"Non-GGUF video loads are limited to unsloth/* repos, the official "
f"family base repos, and local paths; '{repo_id}' is neither."
)
- # Companions load with from_pretrained, so a base repo is held to the non-GGUF bar: a GGUF pick must not smuggle in a remote base.
+ # Companions load with from_pretrained, so a base repo is held to the non-GGUF bar: a GGUF pick must not smuggle
+ # in a remote base.
if base_repo and (base_repo or "").strip() and not _is_trusted_video_repo(base_repo):
raise ValueError(
f"base_repo is limited to unsloth/* repos, the official family base "
f"repos, and local paths; '{base_repo}' is neither."
)
- # A local base_repo loads as a full pipeline (needs model_index.json); reject a non-pipeline one here, before the load.
+ # A local base_repo loads as a full pipeline (needs model_index.json); reject a non-pipeline one here, before
+ # the load.
from core.inference.diffusion import _assert_local_base_is_pipeline
_assert_local_base_is_pipeline(base_repo)
if kind in ("gguf", "single_file") and not gguf_filename:
raise ValueError("A gguf/single_file load needs the checkpoint filename.")
if kind in ("gguf", "single_file") and fam.is_moe:
- # A single checkpoint carries one expert; the other would load dense bf16, off-plan.
raise ValueError(
f"'{fam.name}' is a dual-expert model: a single {kind} file covers only "
f"one of its two transformers. Load the diffusers pipeline repo "
@@ -1255,7 +1249,8 @@ class VideoBackend:
f"(expected a .safetensors name; use a .gguf name for a GGUF load)."
)
root = Path(repo_id).expanduser()
- # Path-shaped: "."/".." prefix, a backslash (never in "org/name"), or an absolute path, so a missing local pick fails before the handoff.
+ # Path-shaped: "."/".." prefix, a backslash (never in "org/name"), or an absolute path, so a missing local
+ # pick fails before the handoff.
path_shaped = (
repo_id.startswith(("/", "\\", "~", ".")) or "\\" in repo_id or root.is_absolute()
)
@@ -1266,7 +1261,8 @@ class VideoBackend:
except Exception as exc: # noqa: BLE001 -- surface as client input error
raise ValueError(str(exc)) from exc
elif root.is_file():
- # The loader hands a local FILE straight through (ignoring gguf_filename), so the file's own suffix must match the kind.
+ # The loader hands a local FILE straight through (ignoring gguf_filename), so the file's own suffix must
+ # match the kind.
suffix = root.suffix.lower()
if kind == "gguf" and suffix != ".gguf":
raise ValueError(
@@ -1280,10 +1276,10 @@ class VideoBackend:
)
elif path_shaped:
raise ValueError(f"Local model path '{repo_id}' does not exist.")
- # A local pipeline pick must be a diffusers directory (model_index.json), else it would only fail after eviction.
+ # A local pipeline pick must be a diffusers directory (model_index.json), else it would only fail after eviction
if kind == "pipeline":
root = Path(repo_id).expanduser()
- # Gate on .exists() (not .is_dir()) so a local FILE picked as a pipeline is rejected too.
+ # Gate on .exists() (not .is_dir()) so a local FILE picked as a pipeline is rejected too
indexes = (
("model_index.json", "modular_model_index.json")
if fam.modular_workflow
@@ -1296,10 +1292,11 @@ class VideoBackend:
f"Local pipeline path is not a diffusers directory "
f"(no {' or '.join(indexes)}): {repo_id}"
)
- # Reject a malformed transformer_quant cheaply, before the handoff (pipeline-kind only, matching the image backend).
+ # Reject a malformed transformer_quant cheaply, before the handoff (pipeline-kind only, matching the image
+ # backend)
normalize_transformer_quant(transformer_quant)
- # Reject a malformed text_encoder_quant the same way. Every kind: an unsupported scheme is
- # a bad request regardless of whether this family has a hosted quantized encoder for it.
+ # Reject a malformed text_encoder_quant the same way. Every kind: an unsupported scheme is a bad request
+ # regardless of whether this family has a hosted quantized encoder for it.
normalize_te_quant(text_encoder_quant)
_ensure_mp4_encoder_available()
return fam
@@ -1325,18 +1322,16 @@ class VideoBackend:
model_kind: Optional[str] = None,
h3_task: Optional[str] = None,
gpu_ids: Optional[list[int]] = None,
- # The ordinal the ROUTE already ranked, so the preflight and the load agree on one card.
+ # The ordinal the ROUTE already ranked, so the preflight and the load agree on one card
gpu_ordinal: Optional[int] = None,
) -> dict[str, Any]:
"""Validate, then run the (slow) load on a daemon thread. Returns at once."""
hf_token = (hf_token.strip() if isinstance(hf_token, str) else hf_token) or None
- # Resolved ONCE, here, and carried to the worker: outside it so a bad pick is the route's
- # 400 rather than a load that dies tens of GB later, and only once so free VRAM cannot
- # re-rank the choice after the weights land. Gated on the resolved backend, since XPU /
- # MPS / CPU ignore physical ids and would otherwise 400 a selection the contract drops.
- # Re-ranked only when the caller did not already do it: free VRAM moves between the
- # route's preflight and here, so resolving twice can approve a scheme against one card
- # and place the weights on another.
+ # Resolved ONCE, here, and carried to the worker: outside it so a bad pick is the route's 400 rather than a load
+ # that dies tens of GB later, and only once so free VRAM cannot re-rank the choice after the weights land. Gated
+ # on the resolved backend, since XPU / MPS / CPU ignore physical ids and would otherwise 400 a selection the
+ # contract drops. Re-ranked only when the caller did not already do it: free VRAM moves between the route's
+ # preflight and here, so resolving twice can approve a scheme against one card and place the weights on another.
if gpu_ordinal is None:
gpu_ordinal = (
resolve_selected_cuda_ordinal(gpu_ids)
@@ -1353,11 +1348,11 @@ class VideoBackend:
text_encoder_quant = text_encoder_quant,
h3_task = h3_task,
)
- # Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the
- # route answers 409 with the reason instead of evicting the resident model and pulling
- # tens of GB first. Footprint-dependent declines still surface via load-progress. The
- # ROUTE makes the same call earlier still (see assert_video_precision_available): this one
- # runs inside acquire_for, which has already evicted chat by the time it fires.
+ # Refuse an EXPLICIT precision this host can never honor BEFORE the load starts, so the route answers 409 with
+ # the reason instead of evicting the resident model and pulling tens of GB first. Footprint-dependent declines
+ # still surface via load-progress. The ROUTE makes the same call earlier still (see
+ # assert_video_precision_available): this one runs inside acquire_for, which has already evicted chat by the
+ # time it fires.
assert_video_precision_available(
fam,
model_kind = resolve_video_model_kind(gguf_filename, model_kind),
@@ -1365,10 +1360,10 @@ class VideoBackend:
text_encoder_quant = text_encoder_quant,
gpu_ordinal = gpu_ordinal,
)
- # Resolved out here so the companion claim is published in the SAME locked section as
- # _loading. begin_load returns as soon as the thread is scheduled, and a delete arriving in
- # that gap sees only repo_id and base_repo, passes the guard, and starts removing a
- # companion repo this load needs. A later claim does not revoke a delete already admitted.
+ # Resolved out here so the companion claim is published in the SAME locked section as _loading. begin_load
+ # returns as soon as the thread is scheduled, and a delete arriving in that gap sees only repo_id and base_repo,
+ # passes the guard, and starts removing a companion repo this load needs. A later claim does not revoke a delete
+ # already admitted.
from .video_minimax_h3 import (
H3_COMPONENT_REPO,
H3_GGUF_REPO,
@@ -1377,8 +1372,8 @@ class VideoBackend:
)
h3_native = is_h3_native(fam, resolve_video_model_kind(gguf_filename, model_kind))
- # Both ids: an install may hold the components under either the mirror or the repack it
- # was mirrored from, and whichever one this load reads must be protected.
+ # Both ids: an install may hold the components under either the mirror or the repack it was mirrored from, and
+ # whichever one this load reads must be protected.
claimed_assets = (
(H3_GGUF_REPO, H3_COMPONENT_REPO, H3_LEGACY_COMPONENT_REPO) if h3_native else ()
)
@@ -1388,8 +1383,9 @@ class VideoBackend:
raise RuntimeError("A video load is already in progress.")
self._load_token += 1
token = self._load_token
- # A NEW event per load, never a clear() of the shared one: unload() sets the event the running worker holds but also
- # drops _loading, so the next begin_load would clear the object that worker watches. A fresh object leaves it set.
+ # A NEW event per load, never a clear() of the shared one: unload() sets the event the running worker holds
+ # but also drops _loading, so the next begin_load would clear the object that worker watches. A fresh object
+ # leaves it set.
cancel_event = threading.Event()
self._cancel_event = cancel_event
self._loading = _VideoLoadingState(
@@ -1426,19 +1422,19 @@ class VideoBackend:
def _run_load(self, **kwargs: Any) -> None:
token = kwargs.get("_load_token")
- # This load's own event: a later load replaces self._cancel_event rather than clearing it.
+ # This load's own event: a later load replaces self._cancel_event rather than clearing it
cancel_event = kwargs.pop("_cancel_event", None) or self._cancel_event
- # An API-initiated load promises to download NOTHING: it may only open what is already
- # cached. Read once here and threaded into every helper below -- the metadata probes as
- # much as the fetches, since a model_info call reaches the Hub just as a weight pull does.
- # READ, not popped: load_pipeline takes it too (it is in this thread's kwargs by contract).
+ # An API-initiated load promises to download NOTHING: it may only open what is already cached. Read once here
+ # and threaded into every helper below -- the metadata probes as much as the fetches, since a model_info call
+ # reaches the Hub just as a weight pull does. READ, not popped: load_pipeline takes it too (it is in this
+ # thread's kwargs by contract).
local_files_only = bool(kwargs.get("local_files_only"))
try:
fam = _detect_load_family(
kwargs["repo_id"], kwargs.get("gguf_filename"), kwargs.get("family_override")
)
- # Also on the worker, which a direct begin_load reaches without a plan. Here rather
- # than in validate_load_request, which is network-free by contract.
+ # Also on the worker, which a direct begin_load reaches without a plan. Here rather than in
+ # validate_load_request, which is network-free by contract.
_assert_pick_is_not_speech(
kwargs["repo_id"],
kwargs.get("gguf_filename"),
@@ -1449,8 +1445,8 @@ class VideoBackend:
from .video_minimax_h3 import is_h3_native
if is_h3_native(fam, kind):
- # local_files_only rides in kwargs and is now a NAMED parameter over there, so the
- # native path binds it explicitly instead of swallowing it.
+ # local_files_only rides in kwargs and is now a NAMED parameter over there, so the native path binds it
+ # explicitly instead of swallowing it.
self._run_load_h3_native(
fam = fam,
token = token,
@@ -1467,22 +1463,21 @@ class VideoBackend:
else resolve_video_base_repo(fam, kwargs.get("base_repo"))
)
kwargs["base_repo"] = base
- # An fp8 encoder request loads a hosted pre-cast checkpoint, so neither the estimate nor the pull includes those dense shards.
+ # An fp8 encoder request loads a hosted pre-cast checkpoint, so neither the estimate nor the pull includes
+ # those dense shards.
te_sources = self._te_prequant_sources(
fam,
kwargs.get("text_encoder_quant"),
kwargs.get("gpu_ordinal"),
base = base,
)
- # The same deal for the denoiser: a hosted pre-quantized checkpoint replaces the dense
- # DiT shards, so the estimate and the scoped pull below both drop them. VERIFIED, like
- # the conditioner below and like the plan: this flag both REMOVES 66 GB from the pull
- # and is never re-checked, so an artifact that does not resolve has to keep the dense
- # shards rather than strand the load's own bf16 fallback with nothing to open.
- # An UNSET H3 precision can resolve to a hosted checkpoint too, and that has to be
- # settled HERE rather than only inside the loader: decided late, the pull stages the
- # dense denoiser the load never opens AND the replacement arrives inline, outside this
- # plan's progress, cancel and disk preflight.
+ # The same deal for the denoiser: a hosted pre-quantized checkpoint replaces the dense DiT shards, so the
+ # estimate and the scoped pull below both drop them. VERIFIED, like the conditioner below and like the plan:
+ # this flag both REMOVES 66 GB from the pull and is never re-checked, so an artifact that does not resolve
+ # has to keep the dense shards rather than strand the load's own bf16 fallback with nothing to open. An
+ # UNSET H3 precision can resolve to a hosted checkpoint too, and that has to be settled HERE rather than
+ # only inside the loader: decided late, the pull stages the dense denoiser the load never opens AND the
+ # replacement arrives inline, outside this plan's progress, cancel and disk preflight.
h3_auto_denoiser = self._h3_planned_auto_denoiser_scheme(
fam,
base = base,
@@ -1490,8 +1485,8 @@ class VideoBackend:
text_encoder_quant = kwargs.get("text_encoder_quant"),
speed_mode = kwargs.get("speed_mode"),
h3_task = kwargs.get("h3_task"),
- # This decides the file set AND the memory policy, so it has to read the card the
- # pipeline will land on: the default one can be larger or smaller than the pick.
+ # This decides the file set AND the memory policy, so it has to read the card the pipeline will land on:
+ # the default one can be larger or smaller than the pick.
gpu_ordinal = kwargs.get("gpu_ordinal"),
)
skip_transformer_weights = self._denoiser_prequant_verified(
@@ -1502,17 +1497,16 @@ class VideoBackend:
kwargs.get("hf_token"),
local_files_only = local_files_only,
)
- # Handed to the loader only when the dense shards really are gone from the pull. Then
- # the choice is already committed and the loader must not re-take it against a reading
- # that has moved since -- there would be no dense denoiser left to fall back to. When
- # the plan kept them, this stays None and the loader decides for itself as before.
+ # Handed to the loader only when the dense shards really are gone from the pull. Then the choice is already
+ # committed and the loader must not re-take it against a reading that has moved since -- there would be no
+ # dense denoiser left to fall back to. When the plan kept them, this stays None and the loader decides for
+ # itself as before.
kwargs["_h3_auto_denoiser_planned"] = (
h3_auto_denoiser if h3_auto_denoiser and skip_transformer_weights else None
)
- # And for MiniMax-H3's conditioner, whose hosted quantized artifact replaces the base
- # repo's 62 GB dense text_encoder/ shards outright.
- # Verified, not just name-matched: this scheme decides whether the base pull DROPS the
- # dense encoder, and a derivative that the seed will decline must keep its own.
+ # And for MiniMax-H3's conditioner, whose hosted quantized artifact replaces the base repo's 62 GB dense
+ # text_encoder/ shards outright. Verified, not just name-matched: this scheme decides whether the base pull
+ # DROPS the dense encoder, and a derivative that the seed will decline must keep its own.
h3_te_scheme = self._h3_te_quant_scheme_verified(
fam,
kwargs.get("text_encoder_quant"),
@@ -1535,14 +1529,12 @@ class VideoBackend:
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.base_repo = base
- # The conditioner artifact comes from a THIRD repo, neither repo_id nor
- # base_repo, so without this the delete-cached guard would let it be deleted
- # out from under the in-flight fetch (or out from under the assembly, while
- # the base snapshot that no longer carries a dense encoder is still coming
- # down). Same registration the native H3 path makes for its companions.
- # Both ids, for the same reason the native path claims both: an install
- # whose cache predates the move reads the artifact from the repack, and a
- # claim over only the mirror would leave the entry this load is reading
+ # The conditioner artifact comes from a THIRD repo, neither repo_id nor base_repo, so without this
+ # the delete-cached guard would let it be deleted out from under the in-flight fetch (or out from
+ # under the assembly, while the base snapshot that no longer carries a dense encoder is still coming
+ # down). Same registration the native H3 path makes for its companions. Both ids, for the same
+ # reason the native path claims both: an install whose cache predates the move reads the artifact
+ # from the repack, and a claim over only the mirror would leave the entry this load is reading
# deletable mid-fetch.
if h3_te_scheme:
self._loading.asset_repos = self._loading.asset_repos + (
@@ -1550,7 +1542,8 @@ class VideoBackend:
H3_LEGACY_TE_QUANT_REPO,
)
self._loading.expected_bytes = expected
- # Checkpoint downloads outside the lock so an unload can preempt the multi-GB pull; companions pre-download the same way.
+ # Checkpoint downloads outside the lock so an unload can preempt the multi-GB pull; companions pre-download
+ # the same way.
checkpoint_local: Optional[Path] = None
if kwargs.get("gguf_filename") and not Path(kwargs["repo_id"]).expanduser().exists():
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
@@ -1560,22 +1553,24 @@ class VideoBackend:
kwargs["gguf_filename"],
kwargs.get("hf_token"),
cancel_event = cancel_event,
- # The plan counts a file cached under EITHER root, so the load has to
- # resolve through both or it re-pulls what the planner skipped.
+ # The plan counts a file cached under EITHER root, so the load has to resolve through both or it
+ # re-pulls what the planner skipped.
reuse_other_cache_root = True,
- # An API-initiated load takes the cached checkpoint or fails; it never
- # pulls the multi-GB file itself.
+ # An API-initiated load takes the cached checkpoint or fails; it never pulls the multi-GB file
+ # itself.
local_files_only = local_files_only,
)
)
- # An LTX-2.3 checkpoint supplies the VAEs/vocoder/connectors, so the base pull shrinks to scheduler + TE + tokenizer; recompute the estimate.
+ # An LTX-2.3 checkpoint supplies the VAEs/vocoder/connectors, so the base pull shrinks to scheduler + TE +
+ # tokenizer; recompute the estimate.
ltx23 = False
if fam is not None and fam.name == "ltx-2" and kind != "pipeline":
from .video_ltx2 import is_ltx23_checkpoint
probe = checkpoint_local
if probe is None:
- # Local repos: a bare file, or a dir child via the same resolver load_pipeline uses. Unresolvable keeps the wide pull.
+ # Local repos: a bare file, or a dir child via the same resolver load_pipeline uses. Unresolvable
+ # keeps the wide pull.
root = Path(kwargs["repo_id"]).expanduser()
if root.is_file():
probe = root
@@ -1615,8 +1610,8 @@ class VideoBackend:
local_files_only = local_files_only,
)
kwargs["_te_prequant_skipped"] = te_skipped
- # Same rule for the H3 conditioner: the artifact has to be on disk before the base pull
- # is allowed to leave the dense encoder behind.
+ # Same rule for the H3 conditioner: the artifact has to be on disk before the base pull is allowed to leave
+ # the dense encoder behind
h3_te_skipped = self._fetch_h3_te_quant(
h3_te_scheme,
kwargs.get("hf_token"),
@@ -1630,20 +1625,22 @@ class VideoBackend:
ltx23 = ltx23,
skip_te_components = te_skipped + h3_te_skipped,
skip_transformer_weights = skip_transformer_weights,
- # Picks the H3 denoiser partition: fl2va stages transformer/, ref2va stages the
- # separate 66.28 GB transformer_ref/ that load_components(workflow="ref2va") opens.
+ # Picks the H3 denoiser partition: fl2va stages transformer/, ref2va stages the separate 66.28 GB
+ # transformer_ref/ that load_components(workflow="ref2va") opens
h3_task = kwargs.get("h3_task"),
cancel_event = cancel_event,
local_files_only = local_files_only,
)
- # The 2.3 assembly pulls per component from the hub id (its snapshot lacks the base VAEs), so it only gets the warmed cache.
+ # The 2.3 assembly pulls per component from the hub id (its snapshot lacks the base VAEs), so it only gets
+ # the warmed cache.
kwargs["_base_local_dir"] = None if ltx23 else base_local
self.load_pipeline(**kwargs)
with self._lock:
if self._load_token == token:
self._loading = None
except Exception as exc: # noqa: BLE001 -- surfaced via load_progress
- # A failed/cancelled load never commits _VideoLoadState, so roll back the process-wide speed globals here (token-scoped).
+ # A failed/cancelled load never commits _VideoLoadState, so roll back the process-wide speed globals here
+ # (token-scoped).
self._rollback_precommit_globals(token)
if self._load_token != token:
return
@@ -1670,9 +1667,9 @@ class VideoBackend:
hf_token: Optional[str] = None,
memory_mode: Optional[str] = None,
gpu_ordinal: Optional[int] = None,
- # NAMED, not left to the ``**_`` swallow below: an API-initiated load hands this in
- # through _run_load's kwargs, and swallowed it meant the four-file bundle, the sizing
- # metadata and the sd-cli install were all fetched by a load that promised no downloads.
+ # NAMED, not left to the ``**_`` swallow below: an API-initiated load hands this in through _run_load's kwargs,
+ # and swallowed it meant the four-file bundle, the sizing metadata and the sd-cli install were all fetched by a
+ # load that promised no downloads.
local_files_only: bool = False,
**_: Any,
) -> None:
@@ -1711,106 +1708,83 @@ class VideoBackend:
filename = gguf_filename or ""
qwen_filename = h3_text_encoder_filename(filename)
- # Claimed before anything slow, the preflight included: it can spend minutes installing the
- # sd-cli prebuilt, and asset_repos is what stops the delete-cached guard admitting a delete
- # of the H3 companion repos mid-load, which a later claim cannot revoke. The download list
- # pulls from both, and neither is repo_id or base_repo. loaded_repo_ids() is the committed
- # twin; expected_bytes stays below, where the sizes are known.
- # begin_load publishes the same claim with _loading, covering the gap before this thread is
- # scheduled. This one is for load_pipeline, which never goes through begin_load.
+ # Claimed before anything slow, the preflight included: it can spend minutes installing the sd-cli prebuilt, and
+ # asset_repos is what stops the delete-cached guard admitting a delete of the H3 companion repos mid-load, which
+ # a later claim cannot revoke. The download list pulls from both, and neither is repo_id or base_repo.
+ # loaded_repo_ids() is the committed twin; expected_bytes stays below, where the sizes are known. begin_load
+ # publishes the same claim with _loading, covering the gap before this thread is scheduled. This one is for
+ # load_pipeline, which never goes through begin_load.
with self._lock:
if self._load_token == token and self._loading is not None:
self._loading.base_repo = fam.base_repo
- # Every id a component source can resolve to, which is the triple begin_load
- # already publishes. Naming one resolved answer instead would leave the OTHER
- # deletable, and the two VAEs are resolved independently: an interrupted pre-move
- # pull can leave one on the repack and the other on the mirror.
+ # Every id a component source can resolve to, which is the triple begin_load already publishes. Naming
+ # one resolved answer instead would leave the OTHER deletable, and the two VAEs are resolved
+ # independently: an interrupted pre-move pull can leave one on the repack and the other on the mirror.
self._loading.asset_repos = (
H3_GGUF_REPO,
H3_COMPONENT_REPO,
H3_LEGACY_COMPONENT_REPO,
)
- # BEFORE the download, not after it. The H3-gated ensure, not the plain one: a build that
- # predates H3 runs fine and so clears the version() gate below, then aborts on the first
- # generation. That is the whole reason this gate exists, and running it after the four-file
- # bundle had already been fetched meant the user still paid tens of GB to be told no.
- #
- # Cancel first, though: the ensure below may download and extract the sd-cli prebuilt, and
- # it takes no cancel_event, so a load cancelled before this thread got going would pay for
- # an install nobody is waiting for. The download loop used to be the first check.
if cancel_event.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
target = self._device_target(gpu_ordinal)
- # An install DOWNLOADS and extracts the sd-cli prebuilt, so an offline load may not start
- # one. A build already on disk (managed or user-supplied) is still discovered and used;
- # when there is none, the ensure returns None and the refusal below names it, which is the
- # honest answer for a load that was told not to fetch anything.
+ # An install DOWNLOADS and extracts the sd-cli prebuilt, so an offline load may not start one. A build already
+ # on disk (managed or user-supplied) is still discovered and used; when there is none, the ensure returns None
+ # and the refusal below names it, which is the honest answer for a load that was told not to fetch anything.
allow_install = _install_allowed() and not local_files_only
binary = ensure_h3_sd_cpp_binary(
allow_install = allow_install,
accelerator = _install_accelerator_for(target.backend),
)
native_device = target.device
- # What the accelerator decision below was made on, or None when it was never asked (a CPU
- # or MPS target never consults it). Re-checked under the reader claim, so a replacement
- # that arrives mid-load cannot silently change the answer this device choice rests on.
+ # What the accelerator decision below was made on, or None when it was never asked (a CPU or MPS target never
+ # consults it). Re-checked under the reader claim, so a replacement that arrives mid-load cannot silently change
+ # the answer this device choice rests on.
listed_accelerator: Optional[bool] = None
if target.backend not in ("cpu", "mps"):
- # Under the claim, like the recheck. This probe SPAWNS the managed sd-cli, so leaving
- # it unclaimed lets an install started by another in-process load extract over the
- # executing binary: on Windows that fails on the locked file, on Linux it can leave
- # the replacement half-written. The later claimed recheck cannot undo damage this
- # first probe already allowed.
+ # Under the claim, like the recheck. This probe SPAWNS the managed sd-cli, so leaving it unclaimed lets an
+ # install started by another in-process load extract over the executing binary: on Windows that fails on the
+ # locked file, on Linux it can leave the replacement half-written. The later claimed recheck cannot undo
+ # damage this first probe already allowed.
from .sd_cpp_backend import _tree_reader as _claim_tree
with _claim_tree(binary, cancel_event, VIDEO_CANCELLED_MSG):
listed_accelerator = sd_cpp_lists_accelerator_device(binary)
if target.backend not in ("cpu", "mps") and not listed_accelerator:
- # Upstream currently publishes no Linux CUDA archive. Keep the picker
- # functional with the CPU prebuilt when the user has not supplied a
- # locally compiled CUDA binary through the normal sd.cpp discovery path.
- #
- # The test is what the binary actually offers, not whether one was returned: the
- # accelerator install fails only ONCE, and every later load finds the CPU binary this
- # branch put there and gets it back unchanged. "binary is truthy" therefore skipped
- # the fallback from the second load on, left native_device on the GPU, and applied GPU
- # offload policy and held the VIDEO claim while sd-cli ran wholly on the CPU -- so the
- # next chat/image acquire evicted a model to make room for one that was never there.
+ # Upstream currently publishes no Linux CUDA archive. Keep the picker functional with the CPU prebuilt when
+ # the user has not supplied a locally compiled CUDA binary through the normal sd.cpp discovery path. The
+ # test is what the binary offers, not whether one was returned: the accelerator install fails only ONCE, and
+ # every later load finds the CPU binary this branch put there and gets it back unchanged. "binary is truthy"
+ # therefore skipped the fallback from the second load on, left native_device on the GPU, and applied GPU
+ # offload policy and held the VIDEO claim while sd-cli ran wholly on the CPU -- so the next chat/image
+ # acquire evicted a model to make room for one that was never there.
binary = ensure_h3_sd_cpp_binary(allow_install = allow_install, accelerator = "cpu")
native_device = "cpu"
- # The baseline this branch is compared against is the DECISION, not a fresh probe of
- # what came back. An install can replace the returned CPU binary with a GPU build
- # between that ensure and this line, and probing here would record ITS answer -- after
- # which the re-check under the claim below compares the replacement against itself,
- # passes, and commits CPU resource accounting around a CUDA executable that runs on
- # VRAM nothing accounted for. native_device is "cpu" precisely because the build must
- # offer no accelerator device, so that -- False -- is what the claim has to still find.
+ # The baseline this branch is compared against is the DECISION, not a fresh probe of what came back. An
+ # install can replace the returned CPU binary with a GPU build between that ensure and this line, and
+ # probing here would record ITS answer -- after which the re-check under the claim below compares the
+ # replacement against itself, passes, and commits CPU resource accounting around a CUDA executable that runs
+ # on VRAM nothing accounted for. native_device is "cpu" precisely because the build must offer no
+ # accelerator device, so that -- False -- is what the claim has to still find.
listed_accelerator = False
- # And refuse a preflight that produced nothing, HERE rather than at the claimed re-vet
- # below. ensure_h3_sd_cpp_binary legitimately returns None -- auto-install switched off, an
- # unsupported platform, no network, or a stale managed copy something else is running out
- # of -- and leaving the only `not binary` check after the download loop meant every one of
- # those cases still fetched the four-file bundle first. The re-vet keeps its own check: it
- # guards a replacement arriving mid-download, which is a different question.
+ # And refuse a preflight that produced nothing, HERE rather than at the claimed re-vet below.
+ # ensure_h3_sd_cpp_binary legitimately returns None -- auto-install switched off, an unsupported platform, no
+ # network, or a stale managed copy something else is running out of -- and leaving the only `not binary` check
+ # after the download loop meant every one of those cases still fetched the four-file bundle first. The re-vet
+ # keeps its own check: it guards a replacement arriving mid-download, which is a different question.
if not binary:
raise RuntimeError(
"stable-diffusion.cpp could not be installed or started for MiniMax-H3."
)
- # And again on the way out. The ensure above takes no cancel_event and can spend minutes
- # downloading and extracting the prebuilt, so a cancel arriving during it is already late;
- # without this a CPU or MPS target (which skips the claimed accelerator probe, the only
- # other cancel-aware step here) would go on to make four sequential model_info calls before
- # the download loop finally noticed.
if cancel_event.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
requests = self._h3_native_requests(repo_id, filename, qwen_filename)
total = 0
try:
- # Skipped wholesale offline: model_info is a Hub call, and the number it produces is
- # the size of a download that is not going to happen -- every file below either
- # resolves from the cache or fails the load. total stays 0, which load_progress already
- # reads as "no estimate" and reports as a bare phase.
+ # Skipped wholesale offline: model_info is a Hub call, and the number it produces is the size of a download
+ # that is not going to happen -- every file below either resolves from the cache or fails the load. total
+ # stays 0, which load_progress already reads as "no estimate" and reports as a bare phase.
api = HfApi(token = hf_token or None) if not local_files_only else None
for repo, wanted in requests:
if api is None:
@@ -1846,8 +1820,6 @@ class VideoBackend:
hf_token,
cancel_event = cancel_event,
reuse_other_cache_root = True,
- # Offline this resolves from the cache and raises when the file is not
- # there; h3_download_error below still names the missing repo/file.
local_files_only = local_files_only,
)
)
@@ -1856,12 +1828,6 @@ class VideoBackend:
resolved.append(local)
engine = SdCppEngine(binary)
- # Re-vet and take the identity under ONE reader claim. ensure_h3_sd_cpp_binary checked the
- # file it returned, but an install can start between that return and this line, and an
- # in-place replacement would then become the recorded identity without ever having been
- # checked for H3 support or the selected accelerator -- after which every generation
- # compares the replacement against itself and lets it through. Inside the claim no install
- # can start, so the build that answers --help here is the build whose identity is stored.
from .sd_cpp_backend import (
_tree_reader,
sd_cpp_accelerator_device_verdict,
@@ -1874,44 +1840,36 @@ class VideoBackend:
raise RuntimeError(
"stable-diffusion.cpp could not be installed or started for MiniMax-H3."
)
- # EVERY binary, not just a managed one. The managed-only test was written when the
- # ensure sat right here, so a user-supplied build had been vetted microseconds earlier
- # and only a concurrent INSTALL could have moved underneath it. The preflight now runs
- # before the multi-tens-of-GB download, so the window it has to cover is that whole
- # download -- long enough for a user to rebuild or repoint their own SD_CLI_PATH copy,
- # after which _sd_cli_identity below would record the replacement as the vetted build
- # and every later generation would compare it against itself.
- # Identity AND capability, the same pair the preflight applied. Capability alone is
- # not enough: --ref-video is a plain option name unrelated reference-video tools also
- # expose, so a swap to one of those would clear a marker-only re-check and then be
- # recorded as the vetted identity.
+ # EVERY binary, not just a managed one.
+ # EVERY binary, not just a managed one. The managed-only test was written when the ensure sat right here, so
+ # a user-supplied build had been vetted microseconds earlier and only a concurrent INSTALL could have moved
+ # underneath it. The preflight now runs before the multi-tens-of-GB download, so the window it has to cover
+ # is that whole download -- long enough for a user to rebuild or repoint their own SD_CLI_PATH copy, after
+ # which _sd_cli_identity below would record the replacement as the vetted build and every later generation
+ # would compare it against itself. Identity AND capability, the same pair the preflight applied. Capability
+ # alone is not enough: --ref-video is a plain option name unrelated reference-video tools also expose, so a
+ # swap to one of those would clear a marker-only re-check and then be recorded as the vetted identity.
if not sd_cpp_binary_vets_for_h3(binary):
raise RuntimeError(
"The stable-diffusion.cpp binary changed while this model was loading, and the "
"one now at that path is not an sd.cpp build with MiniMax-H3 support. Try the "
"load again."
)
- # And re-ask the question native_device was decided on. H3 support alone is not
- # enough: a replacement can be H3-capable and still be a different accelerator, and
- # native_device is about to be committed for the life of this runtime. A CPU fallback
- # committed around a CUDA executable runs on VRAM nothing accounted for, next to
- # whatever the arbiter let in on the strength of this load claiming the CPU.
- #
- # Only when the decision actually consulted it. A CPU or MPS target never asked, so
- # there is no answer to have changed, and inventing one would refuse those loads.
- #
- # Ownership does not narrow it, for the same reason the H3 re-vet above dropped that
- # test: the accelerator answer is recorded before the download now, so the window is
- # the whole fetch, and a user's own build can be rebuilt inside it just as an install
- # can land. The harm does not care who owns the file -- what gets committed is
- # native_device, and a GPU one around a CPU binary is offload policy and an arbiter
- # claim written against hardware nothing is running on.
- # Only a DECISIVE re-reading, hence the verdict form rather than
- # sd_cpp_lists_accelerator_device: that one folds "could not tell" into True, which
- # against a recorded False would refuse the very CPU fallback that recorded it.
- # Only when there is a baseline to compare against. A CPU or MPS target never asked
- # the question, so this would spawn --list-devices for an answer the test below cannot
- # use -- on every H3 load, and for the full probe timeout when the build hangs on it.
+ # And re-ask the question native_device was decided on. H3 support alone is not enough: a replacement can be
+ # H3-capable and still be a different accelerator, and native_device is about to be committed for the life
+ # of this runtime. A CPU fallback committed around a CUDA executable runs on VRAM nothing accounted for,
+ # next to whatever the arbiter let in on the strength of this load claiming the CPU. Only when the decision
+ # consulted it. A CPU or MPS target never asked, so there is no answer to have changed, and inventing one
+ # would refuse those loads. Ownership does not narrow it, for the same reason the H3 re-vet above dropped
+ # that test: the accelerator answer is recorded before the download now, so the window is the whole fetch,
+ # and a user's own build can be rebuilt inside it just as an install can land. The harm does not care who
+ # owns the file -- what gets committed is native_device, and a GPU one around a CPU binary is offload policy
+ # and an arbiter claim written against hardware nothing is running on. Only a DECISIVE re-reading, hence the
+ # verdict form rather than sd_cpp_lists_accelerator_device: that one folds "could not tell" into True, which
+ # against a recorded False would refuse the very CPU fallback that recorded it. Only when there is a
+ # baseline to compare against. A CPU or MPS target never asked the question, so this would spawn
+ # --list-devices for an answer the test below cannot use -- on every H3 load, and for the full probe timeout
+ # when the build hangs on it.
fresh_accelerator = (
sd_cpp_accelerator_device_verdict(binary)
if listed_accelerator is not None and binary
@@ -1928,12 +1886,13 @@ class VideoBackend:
"again."
)
binary_identity = _sd_cli_identity(binary)
- # Under the claim like every other probe here, and never on CPU, which allocates from system RAM.
+ # Under the claim like every other probe here; None on the CPU fallback, which has no card to choose
+ # between.
supports_graph_cut = native_device != "cpu" and sd_cpp_supports_graph_cut(binary)
- # Dropped with the accelerator: the CPU fallback runs on no card, so a recorded ordinal
- # would outlive the decision and be committed against a runtime that never used it.
+ # Dropped with the accelerator: the CPU fallback runs on no card, so a recorded ordinal would outlive the
+ # decision and be committed against a runtime that never used it.
native_ordinal = None if native_device == "cpu" else gpu_ordinal
- # Under the claim like every other probe here; None on the CPU fallback, which has no card to choose between.
+ # Under the claim like every other probe here; None on the CPU fallback
native_device_name = (
None
if native_device == "cpu"
@@ -1946,43 +1905,40 @@ class VideoBackend:
"balanced": "group",
"low_vram": "model",
}[requested_mode]
- # NOT --vae-on-cpu. H3 ships an audio VAE, and its 1-D convolutions abort on the CPU
- # path: ggml_conv_1d hardcodes an F16 im2col destination (ggml/src/ggml.c) and
- # ggml_compute_forward_im2col_f16 then asserts the KERNEL is F16, while sd.cpp's
- # audio_conv_weight_type maps only BF16 to F16 and lets F32 through. The result is
- # GGML_ASSERT(src0->type == GGML_TYPE_F16) failed, SIGABRT, exit 134, deterministically.
- # Bisected on the flags: --vae-on-cpu with --audio-vae aborts, without --audio-vae it
- # renders in 95.87s, and converting the audio VAE checkpoint to fp16 does NOT help, so
- # the type is imposed inside sd.cpp rather than by the file and cannot be fixed by
- # shipping a different checkpoint. low_vram is the one mode a small-card user reaches
- # for, so drop the flag rather than the mode; --offload-to-cpu and --clip-on-cpu, which
- # are where the savings actually are, still apply.
- #
- # The abort itself is fixed in the Unsloth sd.cpp fork (it casts F32 conv1d kernels to
- # F16 in-graph on CPU backends), so this is no longer only a crash workaround, and it
- # should not be reverted when that fix reaches the pinned prebuilt. Measured on a build
- # carrying the fix, 640x384, 25 frames, 4 steps, q4_K, with --offload-to-cpu
- # --clip-on-cpu already on: adding --vae-on-cpu moved peak VRAM 12.42 -> 12.42 GiB and
- # wall time 20.9s -> 100.4s. Under --offload-to-cpu the peak is set by the streamed
- # denoiser, so the flag buys nothing and costs 4.8x.
+ # NOT --vae-on-cpu. low_vram is the one mode a small-card user reaches for NOT --vae-on-cpu. H3 ships an audio
+ # VAE, and its 1-D convolutions abort on the CPU path: ggml_conv_1d hardcodes an F16 im2col destination
+ # (ggml/src/ggml.c) and ggml_compute_forward_im2col_f16 then asserts the KERNEL is F16, while sd.cpp's
+ # audio_conv_weight_type maps only BF16 to F16 and lets F32 through. The result is GGML_ASSERT(src0->type ==
+ # GGML_TYPE_F16) failed, SIGABRT, exit 134, deterministically. Bisected on the flags: --vae-on-cpu with
+ # --audio-vae aborts, without --audio-vae it renders in 95.87s, and converting the audio VAE checkpoint to fp16
+ # does NOT help, so the type is imposed inside sd.cpp rather than by the file and cannot be fixed by shipping a
+ # different checkpoint. low_vram is the one mode a small-card user reaches for, so drop the flag rather than the
+ # mode; --offload-to-cpu and --clip-on-cpu, which are where the savings are, still apply. The abort itself is
+ # fixed in the Unsloth sd.cpp fork (it casts F32 conv1d kernels to F16 in-graph on CPU backends), so this is no
+ # longer only a crash workaround, and it should not be reverted when that fix reaches the pinned prebuilt.
+ # Measured on a build carrying the fix, 640x384, 25 frames, 4 steps, q4_K, with --offload-to-cpu --clip-on-cpu
+ # already on: adding --vae-on-cpu moved peak VRAM 12.42 -> 12.42 GiB and wall time 20.9s -> 100.4s. Under
+ # --offload-to-cpu the peak is set by the streamed denoiser, so the flag buys nothing and costs 4.8x.
native_offload = tuple(
offload_flags(policy, vae_tiling = False, diffusion_fa = True, vae_on_cpu = False)
)
- # H3 allocates each module WHOLE on the device (20.5 GB DiT, 17 GB encoder), so --offload-to-cpu alone still cudaMallocs; not gated on memory mode because auto and fast are what OOM.
- # --max-vram segments on its own, but upstream ignores --stream-layers unless the params are on the CPU, so it only rides along with --offload-to-cpu.
+ # H3 allocates each module WHOLE on the device (20.5 GB DiT, 17 GB encoder), so --offload-to-cpu alone still
+ # cudaMallocs; not gated on memory mode because auto and fast are what OOM. --max-vram segments on its own, but
+ # upstream ignores --stream-layers unless the params are on the CPU, so it only rides along with
+ # --offload-to-cpu.
if supports_graph_cut:
native_offload += GRAPH_CUT_VRAM_FLAGS
if "--offload-to-cpu" in native_offload:
native_offload += GRAPH_CUT_STREAM_FLAGS
- # After the policy, so the pin can see which modules it left on the CPU; without it sd.cpp uses ordinal 0 whatever was selected.
+ # After the policy, so the pin can see which modules it left on the CPU; without it sd.cpp uses ordinal 0
+ # whatever was selected.
native_offload += tuple(device_backend_flags(native_device_name, list(native_offload)))
from .video_minimax_h3 import MiniMaxH3NativeRuntime
runtime = MiniMaxH3NativeRuntime(
engine = engine,
- # Pinned under the reader claim above, where this exact file answered --help with the
- # H3 options. Taking it at generation time instead would compare a replacement against
- # itself.
+ # Pinned under the reader claim above, where this exact file answered --help with the H3 options. Taking it
+ # at generation time instead would compare a replacement against itself.
binary_identity = binary_identity,
files = SdCppModelFiles(
diffusion_model = str(resolved[0]),
@@ -2012,8 +1968,8 @@ class VideoBackend:
base_repo = fam.base_repo,
device = native_device,
gpu_ordinal = native_ordinal,
- # The native runtime is sd.cpp, not torch, so there is no thread-local
- # device to put back: the pin it honours is the --backend one.
+ # The native runtime is sd.cpp, not torch, so there is no thread-local device to put back: the
+ # pin it honours is the --backend one.
placed_ordinal = native_ordinal,
dtype = Path(filename).stem.split("-")[-1],
kind = "gguf",
@@ -2038,11 +1994,10 @@ class VideoBackend:
finally:
self._teardown_waiters -= 1
if native_device == "cpu":
- # /video/load acquired the VIDEO GPU claim off the resolved device target, but no
- # accelerator binary was available and this runtime committed to the CPU build, so it
- # holds no VRAM. Drop the stale claim, or the next chat/image acquire evicts a model
- # that is not on the GPU. release_if keeps the token check atomic against a newer load
- # that took ownership already. Mirrors /images/load's release for a CPU-only native load.
+ # /video/load acquired the VIDEO GPU claim off the resolved device target, but no accelerator binary was
+ # available and this runtime committed to the CPU build, so it holds no VRAM. Drop the stale claim, or the
+ # next chat/image acquire evicts a model that is not on the GPU. release_if keeps the token check atomic
+ # against a newer load that took ownership already.
from .gpu_arbiter import VIDEO, release_if
release_if(VIDEO, lambda: token is None or token == self._load_token)
@@ -2065,7 +2020,8 @@ class VideoBackend:
diffusion_gguf_compile.uninstall_all()
- # LTX-2.3 gets DiT/connectors/VAEs/vocoder from the checkpoint + extras, so only the 2.0 base scheduler / text encoder / tokenizer are pulled.
+ # LTX-2.3 gets DiT/connectors/VAEs/vocoder from the checkpoint + extras, so only the 2.0 base scheduler / text
+ # encoder / tokenizer are pulled.
_LTX23_BASE_PREFIXES = ("scheduler/", "text_encoder/", "tokenizer/")
@staticmethod
@@ -2082,8 +2038,8 @@ class VideoBackend:
fam,
base or str(getattr(fam, "base_repo", "") or ""),
te_quant_mode = text_encoder_quant,
- # The selected card: an fp8 encoder the default card cannot take is still hosted
- # pre-cast for the one this load lands on, and vice versa.
+ # The selected card: an fp8 encoder the default card cannot take is still hosted pre-cast for the one this
+ # load lands on
target = (
resolve_diffusion_device_target()
if gpu_ordinal is None
@@ -2225,9 +2181,9 @@ class VideoBackend:
from .diffusion_prequant import restricted_prequant_load_supported
if not restricted_prequant_load_supported(transformer_quant):
- # An install that cannot open a checkpoint keeps the dense shards. This is the
- # decision that COMMITS (it drops 66 GB from the pull) and it runs for an EXPLICIT
- # request too, which the auto selector's gate never sees.
+ # An install that cannot open a checkpoint keeps the dense shards. This is the decision that COMMITS (it
+ # drops 66 GB from the pull) and it runs for an EXPLICIT request too, which the auto selector's gate never
+ # sees.
logger.info(
"video.denoiser_prequant: this install cannot deserialize a pre-quantized "
"checkpoint, so the %s request keeps its dense denoiser shards",
@@ -2283,11 +2239,10 @@ class VideoBackend:
return None
import torch
- # SCOPED, not pinned: the plan route reaches this on a pooled asyncio.to_thread
- # thread, which must not be handed back to the pool set to this request's card.
- # The WHOLE sizing call, not just the target: _h3_device_capacity_bytes and the
- # encoder-scheme probe read the current device, so a scope that closes early measures
- # the pooled thread's default card and stages the wrong denoiser.
+ # SCOPED, not pinned: the plan route reaches this on a pooled asyncio.to_thread thread, which must not be
+ # handed back to the pool set to this request's card. The WHOLE sizing call, not just the target:
+ # _h3_device_capacity_bytes and the encoder-scheme probe read the current device, so a scope that closes
+ # early measures the pooled thread's default card and stages the wrong denoiser.
with diffusion_device_scope(gpu_ordinal):
target = (
resolve_diffusion_device_target()
@@ -2364,7 +2319,6 @@ class VideoBackend:
except Exception: # noqa: BLE001 -- no spec, no substitution
return None
source = getattr(spec, "pretrained_model_name_or_path", None) or getattr(spec, "repo", None)
- # A list means several sources for one component; nothing here can vouch for that.
return source if isinstance(source, str) and source else None
@staticmethod
@@ -2382,12 +2336,11 @@ class VideoBackend:
filename = h3_te_quant_filename(scheme)
if filename is None:
return None, []
- # The repo the fetch will read, so the entry's cached/loadable check asks about the id the
- # bytes are actually under. Its SIZE comes from the mirror: the repack is a source of
- # bytes already on disk, never of network metadata, and one taken down since would make
- # this report no hosted artifact -- after which the plan stages the 62 GB dense
- # text_encoder shards that the load, reading the artifact straight out of that same cache,
- # never opens. The two copies are byte identical, so the number is the same.
+ # The repo the fetch will read, so the entry's cached/loadable check asks about the id the bytes are under. Its
+ # SIZE comes from the mirror: the repack is a source of bytes already on disk, never of network metadata, and
+ # one taken down since would make this report no hosted artifact -- after which the plan stages the 62 GB dense
+ # text_encoder shards that the load, reading the artifact straight out of that same cache, never opens. The two
+ # copies are byte identical, so the number is the same.
repo = h3_te_quant_source(scheme)
try:
info = api.model_info(H3_TE_QUANT_REPO, files_metadata = True)
@@ -2472,10 +2425,10 @@ class VideoBackend:
if not getattr(fam, "modular_workflow", None):
return False
scheme = normalize_transformer_quant(transformer_quant)
- # "auto" leaves the choice to the backend, which keeps the released bfloat16 DENOISER
- # for a modular workflow (the conditioner default is a separate decision, made in
- # _h3_te_quant_scheme), so it must NOT drop the dense shards that load then wants. The
- # hosted checkpoints were measured and left opt-in: see _load_h3_modular_pipeline.
+ # "auto" leaves the choice to the backend, which keeps the released bfloat16 DENOISER for a modular workflow
+ # (the conditioner default is a separate decision, made in _h3_te_quant_scheme), so it must NOT drop the
+ # dense shards that load then wants. The hosted checkpoints were measured and left opt-in: see
+ # _load_h3_modular_pipeline.
if scheme is None or scheme == TQ_AUTO:
return False
return video_family_prequant_available(fam, scheme, task = h3_task, base_repo = base)
@@ -2507,17 +2460,16 @@ class VideoBackend:
try:
from .diffusion_prequant import resolve_prequant_source
- # Task-keyed, like the coverage probe: staging the other partition's checkpoint would
- # advertise the right byte count for the wrong file.
+ # Task-keyed, like the coverage probe: staging the other partition's checkpoint would advertise the right
+ # byte count for the wrong file
source = resolve_prequant_source(fam, scheme, base_repo = base, task = h3_task)
except Exception as exc: # noqa: BLE001 -- a bad registry entry must not sink the plan
logger.warning("video.denoiser_prequant_unresolved: %s", exc)
return None, []
- # A local override is already on disk; only a hosted checkpoint is staged.
if source is None or getattr(source, "kind", None) != "repo":
return None, []
- # The root name first, then the legacy scheme name: resolve_prequant_source hands back both
- # and the load tries them in that order, so the plan must stage whichever one exists.
+ # The root name first, then the legacy scheme name: resolve_prequant_source hands back both and the load tries
+ # them in that order, so the plan must stage whichever one exists.
wanted = [
n
for n in (
@@ -2567,7 +2519,7 @@ class VideoBackend:
except Exception as exc: # noqa: BLE001 -- a bad registry entry keeps the dense shards
logger.warning("video.denoiser_prequant_unresolved: %s", exc)
return None
- # A local override is on disk by definition; only a hosted checkpoint has a cache to probe.
+ # A local override is on disk by definition; only a hosted checkpoint has a cache to probe
if source is None or getattr(source, "kind", None) != "repo":
return None
from core.inference.diffusion import DiffusionBackend
@@ -2578,8 +2530,8 @@ class VideoBackend:
):
if name and DiffusionBackend._hub_file_is_cached(source.location, name):
return source.location
- # No log here: the caller reports the same "keeping its dense denoiser shards" outcome for
- # a miss, and logging it twice would read as two separate decisions.
+ # No log here: the caller reports the same "keeping its dense denoiser shards" outcome for a miss, and logging
+ # it twice would read as two separate decisions.
return None
@staticmethod
@@ -2595,7 +2547,6 @@ class VideoBackend:
whole plan."""
found: dict[str, list[tuple[str, int]]] = {}
for component, source in sources.items():
- # A local path override is already on disk; only a hosted checkpoint is staged.
if getattr(source, "kind", None) != "repo" or not getattr(source, "filename", None):
continue
try:
@@ -2660,10 +2611,9 @@ class VideoBackend:
is_h3_modular = any(s.rfilename == "modular_model_index.json" for s in siblings)
h3_is_references = is_h3_modular and (h3_task or "").strip().lower() == H3_TASK_REFERENCES
h3_denoiser_prefix = "transformer_ref/" if h3_is_references else "transformer/"
- # Which component's dense weight shards ``skip_transformer_weights`` drops. It has to be
- # the partition this load will actually open: a reference load stages transformer_ref/ and
- # nothing else, so dropping "transformer/" drops nothing and the plan carries the full
- # 66.28 GB the pre-quantized checkpoint exists to replace.
+ # Which component's dense weight shards ``skip_transformer_weights`` drops. It has to be the partition this load
+ # will actually open: a reference load stages transformer_ref/ and nothing else, so dropping "transformer/"
+ # drops nothing and the plan carries the full 66.28 GB the pre-quantized checkpoint exists to replace.
h3_skip_component = h3_denoiser_prefix.rstrip("/")
h3_prefixes = (
"audio_scheduler/",
@@ -2683,7 +2633,8 @@ class VideoBackend:
or name.startswith(h3_prefixes)
):
continue
- # .jinja: tokenizer/chat_template.jinja is needed at generation time; a snapshot without it crashes the first generation.
+ # .jinja: tokenizer/chat_template.jinja is needed at generation time; a snapshot without it crashes the
+ # first generation.
if not name.endswith((".safetensors", ".json", ".model", ".txt", ".jinja")):
continue
if "/" not in name and name.endswith(".safetensors"):
@@ -2691,15 +2642,14 @@ class VideoBackend:
if (
kind != "pipeline"
and name.startswith("transformer/")
- # transformer/config.json is the exception: from_single_file(config = <repo id>,
- # subfolder = "transformer") reads it off the Hub, so a load that promised to
- # download nothing needs it staged. Without it the locality gate cleared the pick
- # and the fetch happened after eviction.
+ # transformer/config.json is the exception: from_single_file(config = <repo id>, subfolder =
+ # "transformer") reads it off the Hub, so a load that promised to download nothing needs it staged.
+ # Without it the locality gate cleared the pick and the fetch happened after eviction.
and name != "transformer/config.json"
):
continue
- # is_prequant_covered_weight is component-agnostic (it matches "<component>/" against a
- # weight suffix), so the encoder helper serves the denoiser verbatim.
+ # is_prequant_covered_weight is component-agnostic (it matches "<component>/" against a weight suffix), so
+ # the encoder helper serves the denoiser verbatim.
if skip_transformer_weights and is_prequant_covered_weight(name, (h3_skip_component,)):
continue
if name.startswith("text_encoder/diffusion_pytorch_model"):
@@ -2808,8 +2758,8 @@ class VideoBackend:
from huggingface_hub import HfApi
fam = _detect_load_family(repo_id, gguf_filename, family_override)
- # _detect_load_family resolves from the REPO id first, so a mixed repo answers its media
- # family for every file in it, a csm quant included. Refuse before the plan stages a byte.
+ # _detect_load_family resolves from the REPO id first, so a mixed repo answers its media family for every file
+ # in it, a csm quant included. Refuse before the plan stages a byte.
_assert_pick_is_not_speech(repo_id, gguf_filename, hf_token)
kind = resolve_video_model_kind(gguf_filename, model_kind)
from .video_minimax_h3 import is_h3_native
@@ -2817,9 +2767,9 @@ class VideoBackend:
if is_h3_native(fam, kind):
return self._h3_native_download_plan(repo_id, gguf_filename or "", hf_token = hf_token)
base = repo_id if kind == "pipeline" else resolve_video_base_repo(fam, base_repo)
- # The same resolution the load makes, so an H3 auto pick the loader will serve from a
- # hosted checkpoint is not staged dense here first: that is 66.3 GB the panel downloads
- # and the load never opens, and the replacement would then arrive outside this plan.
+ # The same resolution the load makes, so an H3 auto pick the loader will serve from a hosted checkpoint is not
+ # staged dense here first: that is 66.3 GB the panel downloads and the load never opens, and the replacement
+ # would then arrive outside this plan.
transformer_quant = (
self._h3_planned_auto_denoiser_scheme(
fam,
@@ -2833,15 +2783,17 @@ class VideoBackend:
)
or transformer_quant
)
- # Only the header tells an LTX-2.3 checkpoint from 2.0 and it is not on disk yet, so narrow the base pull by NAME: a wrong guess costs an inline pull, the wide base list costs gigabytes.
+ # Only the header tells an LTX-2.3 checkpoint from 2.0 and it is not on disk yet, so narrow the base pull by
+ # NAME: a wrong guess costs an inline pull, the wide base list costs gigabytes.
ltx23 = self._pick_looks_like_ltx23(fam, repo_id, gguf_filename, kind)
- # Keyed by repo so a 2.3 pick's checkpoint and extras stay ONE scoped job; two entries would collide on the job key.
+ # Keyed by repo so a 2.3 pick's checkpoint and extras stay ONE scoped job; two entries would collide on the job
+ # key.
entries: dict[str, dict[str, Any]] = {}
total = 0
checkpoint_bytes = 0
planned: dict[str, set[str]] = {}
- # Where the loader would resolve each file (live root / other root / nowhere), and the
- # entry labels, both keyed by repo so the staging decision can be made once per repo.
+ # Where the loader would resolve each file (live root / other root / nowhere), and the entry labels, both keyed
+ # by repo so the staging decision can be made once per repo.
located: dict[str, dict[str, tuple[Optional[str], int]]] = {}
labels: dict[str, dict[str, Any]] = {}
@@ -2871,9 +2823,8 @@ class VideoBackend:
repo, name, revision, size, roots = (None,)
):
return None
- # A stale live copy under the right name shadows the good one: reuse_other_cache_root
- # switches roots only when the live lookup finds NOTHING, and presence alone is what
- # it tests, so ask without the size.
+ # A stale live copy under the right name shadows the good one: reuse_other_cache_root switches roots
+ # only when the live lookup finds NOTHING, and presence alone is what it tests, so ask without the size.
if DiffusionBackend._hub_file_is_cached(repo, name, revision, None, roots = (live,)):
return None
return "other"
@@ -2883,8 +2834,8 @@ class VideoBackend:
label = labels.setdefault(repo, {"gguf": None, "checkpoint": False})
if gguf:
label["gguf"] = gguf
- # Only ever raised, never cleared: a repo holding both the checkpoint and companion
- # files is keyed once, so a later companion add would unflag what the checkpoint set.
+ # Only ever raised, never cleared: a repo holding both the checkpoint and companion files is keyed once, so
+ # a later companion add would unflag what the checkpoint set.
label["checkpoint"] = label["checkpoint"] or checkpoint
required = 0
for name, size in files:
@@ -2913,7 +2864,8 @@ class VideoBackend:
checkpoint = True,
)
if ltx23:
- # The 2.3 assembly reads these companion files at load; without them here they would be pulled inline, outside the panel's disk preflight.
+ # The 2.3 assembly reads these companion files at load; without them here they would be pulled
+ # inline, outside the panel's disk preflight.
from .video_ltx2 import ltx23_extras_files, LTX23_EXTRAS_REPO
wanted = set(ltx23_extras_files(gguf_filename))
@@ -2931,7 +2883,7 @@ class VideoBackend:
],
revision = getattr(extras_info, "sha", None),
)
- # Pre-cast encoders first: only a checkpoint that really resolves earns the right to drop the dense shards.
+ # Pre-cast encoders first: only a checkpoint that really resolves earns the right to drop the dense shards
te_sources = self._te_prequant_sources(
fam,
text_encoder_quant,
@@ -2941,20 +2893,18 @@ class VideoBackend:
te_files = self._te_prequant_hub_files(te_sources, api)
for component, files in te_files.items():
total += add(te_sources[component].location, files)
- # The denoiser's replacement artifact, for the same reason: the base entry below drops
- # the dense DiT shards only when this resolves, so it is staged in their place.
+ # The denoiser's replacement artifact, for the same reason: the base entry below drops the dense DiT shards
+ # only when this resolves, so it is staged in their place.
dq_repo, dq_files = self._denoiser_prequant_hub_files(
fam, transformer_quant, base, api, h3_task
)
if dq_repo:
total += add(dq_repo, dq_files)
- # And H3's quantized conditioner, which replaces the base repo's dense text_encoder/.
- # VERIFIED, like the load: this entry both ADDS 27 GB and REMOVES the dense encoder
- # from the base entry below, so a name match that the load will decline gets the plan
- # and the disk preflight wrong in both directions at once -- 27 GB staged and never
- # used, 62 GB needed and never advertised. This function is already the network-bound
- # one (it is nothing but model_info calls, wrapped in the try below), so the few-KB
- # index read is in keeping.
+ # And H3's quantized conditioner, which replaces the base repo's dense text_encoder/. VERIFIED, like the
+ # load: this entry both ADDS 27 GB and REMOVES the dense encoder from the base entry below, so a name match
+ # that the load will decline gets the plan and the disk preflight wrong in both directions at once -- 27 GB
+ # staged and never used, 62 GB needed and never advertised. This function is already the network-bound one
+ # (it is nothing but model_info calls, wrapped in the try below), so the few-KB index read is in keeping.
h3_te_repo, h3_te_files = self._h3_te_quant_hub_files(
self._h3_te_quant_scheme_verified(fam, text_encoder_quant, base, hf_token), api
)
@@ -2971,13 +2921,12 @@ class VideoBackend:
skip_te_components = (
tuple(te_files) + (("text_encoder",) if h3_te_repo else ())
),
- # Gated on the artifact RESOLVING, exactly like the two encoder skips
- # above, not on the registry alone: the registry says a checkpoint should
- # exist, dq_repo says one does. When the task-specific file is absent or
- # its repo cannot be read, dropping the dense shards leaves a plan with
- # NEITHER denoiser, and the bf16 fallback the loader documents has nothing
- # to fall back to offline. The registry probe still runs, because it is
- # the one that refuses a non-modular family and an auto request.
+ # Gated on the artifact RESOLVING, exactly like the two encoder skips above, not on the registry
+ # alone: the registry says a checkpoint should exist, dq_repo says one does. When the
+ # task-specific file is absent or its repo cannot be read, dropping the dense shards leaves a
+ # plan with NEITHER denoiser, and the bf16 fallback the loader documents has nothing to fall
+ # back to offline. The registry probe still runs, because it is the one that refuses a
+ # non-modular family and an auto request.
skip_transformer_weights = (
dq_repo is not None
and self._denoiser_prequant_covered(
@@ -2987,15 +2936,14 @@ class VideoBackend:
h3_task = h3_task,
),
revision = getattr(info, "sha", None),
- # A pipeline pick has no single file: the base IS the selected model (base =
- # repo_id above). Under a GGUF pick the base is a companion, so it stays unflagged.
+ # A pipeline pick has no single file: the base IS the selected model (base = repo_id above). Under a
+ # GGUF pick the base is a companion, so it stays unflagged.
checkpoint = kind == "pipeline",
)
except Exception as exc: # noqa: BLE001 -- an unavailable plan falls back to the inline pull
logger.warning("video.download_plan_failed: %s", exc)
- # Flagged, because zero here means "unknown", not "nothing to fetch": a caller that
- # reads it as verified locality (media auto-switch) would allow the very download
- # the inline fallback then performs.
+ # Flagged, because zero here means "unknown", not "nothing to fetch": a caller that reads it as verified
+ # locality (media auto-switch) would allow the very download the inline fallback then performs.
return {
"entries": [],
"total_bytes": 0,
@@ -3004,13 +2952,12 @@ class VideoBackend:
"plan_failed": True,
}
for repo, where in located.items():
- # Files STRADDLING the two roots cannot be handed to from_pretrained as one snapshot:
- # _predownload_base returns no directory then and the assembly is pinned to
- # hub_cache_dir(), so the other-root subset is re-pulled inline, past this plan's
- # progress, cancel and disk preflight. A repo living WHOLLY in the other root is fine
- # (that is what reuse_other_cache_root is for). A file missing everywhere downloads
- # into the live root, so it counts as live: dropping it would read a mixed repo as
- # unsplit and stage only the missing part, manufacturing the split this avoids.
+ # Files STRADDLING the two roots cannot be handed to from_pretrained as one snapshot: _predownload_base
+ # returns no directory then and the assembly is pinned to hub_cache_dir(), so the other-root subset is
+ # re-pulled inline, past this plan's progress, cancel and disk preflight. A repo living WHOLLY in the other
+ # root is fine (that is what reuse_other_cache_root is for). A file missing everywhere downloads into the
+ # live root, so it counts as live: dropping it would read a mixed repo as unsplit and stage only the missing
+ # part, manufacturing the split this avoids.
split = {w or "live" for w, _size in where.values()} == {"live", "other"}
staged = [
(name, size)
@@ -3020,9 +2967,9 @@ class VideoBackend:
if not staged:
continue
missing_names = [name for name, _size in staged]
- # Keep the scoped claim stable while this repo warms so a second pick adopts the
- # running @diffusion job instead of 409ing on a smaller file list. Cached names are
- # no-op Hub calls; only ``staged`` contributes to bytes/preflight.
+ # Keep the scoped claim stable while this repo warms so a second pick adopts the running @diffusion job
+ # instead of 409ing on a smaller file list. Cached names are no-op Hub calls; only ``staged`` contributes to
+ # bytes/preflight.
names = list(where)
gguf = labels[repo]["gguf"]
entries[repo] = {
@@ -3030,16 +2977,16 @@ class VideoBackend:
"files": names,
"bytes": sum(size for _name, size in staged),
"gguf_filename": gguf,
- # Describes the ENTRY, not the pick: a 2.3 repo whose checkpoint is already cached
- # stages companion files only, and calling that "Model file" misdescribes what is
- # downloading. A pipeline pick has no single file, so the repo itself is the model.
+ # Describes the ENTRY, not the pick: a 2.3 repo whose checkpoint is already cached stages companion
+ # files only, and calling that "Model file" misdescribes what is downloading. A pipeline pick has no
+ # single file, so the repo itself is the model.
"checkpoint": labels[repo]["checkpoint"]
and (gguf is None or gguf in missing_names),
}
return {
"entries": list(entries.values()),
- # Remaining vs the full footprint: cache state changes what is left to fetch, never
- # what the load needs on disk.
+ # Remaining vs the full footprint: cache state changes what is left to fetch, never what the load needs on
+ # disk
"total_bytes": sum(entry["bytes"] for entry in entries.values()),
"required_bytes": total,
"checkpoint_bytes": int(checkpoint_bytes),
@@ -3116,8 +3063,8 @@ class VideoBackend:
validate_h3_transformer_filename(gguf_filename)
qwen_filename = h3_text_encoder_filename(gguf_filename)
- # The same builder the load uses, so the plan cannot promise a download from one repo and
- # then fetch from the other.
+ # The same builder the load uses, so the plan cannot promise a download from one repo and then fetch from the
+ # other
wanted = VideoBackend._h3_native_requests(repo_id, gguf_filename, qwen_filename)
grouped: dict[str, dict[str, Any]] = {}
missing_files: dict[str, set[str]] = {}
@@ -3129,11 +3076,10 @@ class VideoBackend:
for repo, filename in wanted:
if Path(repo).expanduser().exists():
continue
- # Sizes come from the repo we control even when the bytes will be read from the
- # repack it was mirrored from. The repack may have been taken down since -- the
- # failure this move exists to survive -- and one raising model_info here fails the
- # WHOLE plan, so a load its own cache can still satisfy is refused by every
- # locality-dependent caller.
+ # Sizes come from the repo we control even when the bytes will be read from the repack it was mirrored
+ # from. The repack may have been taken down since -- the failure this move exists to survive -- and one
+ # raising model_info here fails the WHOLE plan, so a load its own cache can still satisfy is refused by
+ # every locality-dependent caller.
meta_repo = h3_component_metadata_repo(repo)
info = api.model_info(meta_repo, files_metadata = True)
match = next((s for s in (info.siblings or []) if s.rfilename == filename), None)
@@ -3157,9 +3103,8 @@ class VideoBackend:
)
if filename not in entry["files"]:
entry["files"].append(filename)
- # Only when the metadata came from the repo the bytes come from: the mirror's
- # head says nothing about which snapshot the repack's cache holds, and a
- # foreign sha would only ever be a pinned miss.
+ # Only when the metadata came from the repo the bytes come from: the mirror's head says nothing
+ # about which snapshot the repack's cache holds, and a foreign sha would only ever be a pinned miss.
revision = getattr(info, "sha", None) if meta_repo == repo else None
if not DiffusionBackend._hub_file_is_loadable(repo, filename, revision, size):
missing_files.setdefault(repo, set()).add(filename)
@@ -3169,8 +3114,8 @@ class VideoBackend:
entry["gguf_filename"] = filename
except Exception as exc: # noqa: BLE001 -- inline loading remains the fallback
logger.warning("video.h3_native_download_plan_failed: %s", exc)
- # Flagged like the outer planner's failure: zero here means "unknown", and a caller
- # that must not download (media auto-switch) has to tell the two apart.
+ # Flagged like the outer planner's failure: zero here means "unknown", and a caller that must not download
+ # (media auto-switch) has to tell the two apart.
return {
"entries": [],
"total_bytes": 0,
@@ -3235,7 +3180,8 @@ class VideoBackend:
fetched: list[str] = []
for component, source in sources.items():
- # A local path override is validated by the injection itself; nothing to fetch, and no basis for skipping the dense download.
+ # A local path override is validated by the injection itself; nothing to fetch, and no basis for skipping
+ # the dense download.
if getattr(source, "kind", None) != "repo" or not getattr(source, "filename", None):
continue
try:
@@ -3244,7 +3190,6 @@ class VideoBackend:
source.filename,
hf_token,
cancel_event = cancel,
- # Pre-quant encoders are planned with the same both-roots probe.
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
@@ -3315,7 +3260,8 @@ class VideoBackend:
snapshot_root: Optional[str] = None
roots: set[str] = set()
for name, _ in files:
- # Explicit check: a cached file returns without consulting the event, so a warm-cache sweep would run to completion after a cancel.
+ # Explicit check: a cached file returns without consulting the event, so a warm-cache sweep would run to
+ # completion after a cancel.
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
local = Path(
@@ -3324,12 +3270,11 @@ class VideoBackend:
name,
hf_token,
cancel_event = cancel,
- # Same reason as the checkpoint: the plan already treats either root as cached.
reuse_other_cache_root = True,
)
)
- # The resolved path minus the file's own relative parts, so a subfolder entry yields
- # the same root as a top-level one. Not resolve()d: that follows the link into blobs/.
+ # The resolved path minus the file's own relative parts, so a subfolder entry yields the same root as a
+ # top-level one. Not resolve()d: that follows the link into blobs/.
try:
root = str(local.parents[len(Path(name).parts) - 1])
except (IndexError, ValueError, OSError):
@@ -3337,10 +3282,9 @@ class VideoBackend:
roots.add(root)
if name == "model_index.json":
snapshot_root = root or None
- # Only hand back a snapshot the WHOLE set lives in. reuse_other_cache_root resolves per
- # file, so a moved cache can serve the manifest from the old root while the rest land
- # in the live one, pointing from_pretrained at a tree missing the others. Mirrors the
- # image prefetch path.
+ # Only hand back a snapshot the WHOLE set lives in. reuse_other_cache_root resolves per file, so a moved
+ # cache can serve the manifest from the old root while the rest land in the live one, pointing
+ # from_pretrained at a tree missing the others. Mirrors the image prefetch path.
if snapshot_root is not None and roots != {snapshot_root}:
return None
return snapshot_root
@@ -3392,12 +3336,14 @@ class VideoBackend:
expected = loading.expected_bytes
phase = "downloading"
if not expected:
- # No size estimate (metadata failure, or a fully cached load). ``downloaded`` counts what is PRESENT in the cache,
- # not what this load fetched, so it would show a multi-GB figure for a load that fetches nothing. Report the phase alone.
+ # No size estimate (metadata failure, or a fully cached load). ``downloaded`` counts what is PRESENT in the
+ # cache, not what this load fetched, so it would show a multi-GB figure for a load that fetches nothing.
+ # Report the phase alone.
return _progress(phase)
if downloaded >= expected:
phase = "finalizing"
- # The cache scan counts every blob, so the raw counter can exceed the scoped estimate; clamp to what the bar reports.
+ # The cache scan counts every blob, so the raw counter can exceed the scoped estimate; clamp to what the bar
+ # reports.
downloaded = expected
return _progress(
phase,
@@ -3461,7 +3407,6 @@ class VideoBackend:
text_encoder_quant: Optional[str] = None,
model_kind: Optional[str] = None,
h3_task: Optional[str] = None,
- # The torch ordinal begin_load resolved for this load, carried rather than re-derived.
gpu_ordinal: Optional[int] = None,
_load_token: Optional[int] = None,
_base_local_dir: Optional[str] = None,
@@ -3490,8 +3435,8 @@ class VideoBackend:
gguf_filename = gguf_filename,
hf_token = hf_token,
memory_mode = memory_mode,
- # Carried, not defaulted: load_pipeline is also reached directly (no _run_load),
- # and dropping it here would let an offline load fetch the four-file bundle.
+ # Carried, not defaulted: load_pipeline is also reached directly (no _run_load), and dropping it here
+ # would let an offline load fetch the four-file bundle.
local_files_only = local_files_only,
)
return self.status()
@@ -3499,9 +3444,8 @@ class VideoBackend:
import diffusers
import torch
- # Same reason as diffusion.py: diffusers hard-codes _tqdm_active = True at
- # import and honours no env var, so setup_logging cannot reach it and its
- # "Loading pipeline components..." bar lands on the structured stream.
+ # Same reason as diffusion.py: diffusers hard-codes _tqdm_active = True at import and honours no env var, so
+ # setup_logging cannot reach it and its "Loading pipeline components..." bar lands on the structured stream.
try:
from loggers.config import quiet_third_party_progress_bars
quiet_third_party_progress_bars()
@@ -3513,31 +3457,34 @@ class VideoBackend:
with self._lock:
if _load_token is not None and _load_token != self._load_token:
raise RuntimeError("Video load was cancelled or superseded.")
- # Signal a generation from the PREVIOUS model (the token check above bailed a superseded worker).
+ # Signal a generation from the PREVIOUS model (the token check above bailed a superseded worker)
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
- # Same fence unload() takes, raised BEFORE the barrier: a queued generation holds no cancel event, so the signal
- # above cannot reach it and it would slip through the moment the barrier released _generate_lock.
+ # Same fence unload() takes, raised BEFORE the barrier: a queued generation holds no cancel event, so the
+ # signal above cannot reach it and it would slip through the moment the barrier released _generate_lock.
self._teardown_waiters += 1
- # Barrier: wait for the signalled generation to exit before teardown, or two models coexist in VRAM.
+ # Barrier: wait for the signalled generation to exit before freeing the pipeline, else we report the VRAM free
+ # while the clip still holds it. Teardown runs INSIDE the barrier: releasing first would hand out one last clip.
with self._generate_lock:
with self._lock:
try:
- # The barrier wait can outlive this load (superseded by a newer load/unload); recheck before touching shared state.
+ # The barrier wait can outlive this load (superseded by a newer load/unload); recheck before
+ # touching shared state.
if _load_token is not None and _load_token != self._load_token:
raise RuntimeError("Video load was cancelled or superseded.")
self._teardown_state_locked()
finally:
- # Released here, not at the end of the load: the old pipe is gone (or this load bailed), and a raising teardown must not leave the fence up for the life of the process.
+ # Released here, not at the end of the load: the old pipe is gone (or this load bailed), and a
+ # raising teardown must not leave the fence up for the life of the process.
self._teardown_waiters -= 1
target = self._device_target(gpu_ordinal)
device = target.device
- # Video DiTs are bf16-native; fp16 overflows, so a resolved fp16 promotes to float32. CPU stays float32.
+ # Video DiTs are bf16-native; fp16 overflows, so a resolved fp16 promotes to float32.
dtype = target.dtype
if fam.fp16_incompatible and dtype is torch.float16:
dtype = torch.float32
- # Size tables below are bf16 (2-byte), so scale dense estimates when the promotion lands fp32 on an accelerator.
+ # Size tables below are bf16 (2-byte), so scale dense estimates when the promotion lands fp32 on an accelerator
dtype_scale = 2.0 if device != "cpu" and dtype is torch.float32 else 1.0
if fam.modular_workflow:
@@ -3554,16 +3501,10 @@ class VideoBackend:
device = device,
hf_token = hf_token,
memory_mode = memory_mode,
- # RAW, not normalised. Both normalisers fold "none"/"off" into the same None an
- # omitted request produces, and for a modular workflow those are opposite answers:
- # unset means "pick the hosted quantized components", "none" means "keep the
- # released bfloat16 ones". validate_load_request above already rejected malformed
- # values with both normalisers; the modular loader normalises after reading the
- # tri-state.
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
- # The speed layer lives BELOW this dispatch, which the modular branch never reached:
- # no channels_last VAE, no cudnn.benchmark, no attention backend, no compile.
+ # The speed layer lives BELOW this dispatch, which the modular branch never reached: no channels_last
+ # VAE, no cudnn.benchmark, no attention backend, no compile.
speed_mode = speed_mode,
attention_backend = attention_backend,
h3_task = h3_task,
@@ -3573,33 +3514,29 @@ class VideoBackend:
_h3_auto_denoiser_planned = _h3_auto_denoiser_planned,
)
- # What the CALLER asked for, before the rewrite below. The record has to report this, not
- # the internal value: an omitted precision under speed_mode="off" becomes "off" here, and
- # reporting that as the request makes the resolved record say the user pinned bf16. The
- # frontend then hides the Auto badge and reseeds the Precision select to none, so after
- # the user changes Speed and reloads, quantisation stays pinned off for no reason they
- # can see.
transformer_quant_requested = transformer_quant
- # Precision tri-state: unset/"auto" -> hardware ladder; "none"/"off" pins dense bf16; an explicit scheme pins it. Pipeline-kind only.
+ # Precision tri-state: unset/"auto" -> hardware ladder; "none"/"off" pins dense bf16; an explicit scheme pins
+ # it. Pipeline-kind only.
if transformer_quant is None or str(transformer_quant).strip().lower() in (
"",
"auto",
):
- # An explicit Speed="off" (bit-exact) load must stay dense bf16: auto-quant would engage int8/fp8 + compile and break the request.
+ # An explicit Speed="off" (bit-exact) load must stay dense bf16: auto-quant would engage int8/fp8 + compile
+ # and break the request.
speed_off = speed_mode is not None and str(speed_mode).strip().lower() == SPEED_OFF
transformer_quant = "off" if speed_off else TQ_AUTO
+ # ── memory plan: family-table resident estimate + frames-aware headroom. Settled, not raw: this plan is the one
+ # the unified-memory refusal below judges, so the previous pipeline's cached-but-free allocator buffers must not
+ # read as occupied.
# ── memory plan: family-table resident estimate + frames-aware headroom.
- # Settled, not raw: this plan is the one the unified-memory refusal below judges, so the
- # previous pipeline's cached-but-free allocator buffers must not read as occupied.
device_memory = settled_snapshot_device_memory(target)
components = fam.bf16_components_gb
mib_per_gb = 1000.0**3 / (1024.0 * 1024.0)
- # bf16_components_gb is (transformer, text_encoder, vae) at bf16. When this pick takes its
- # encoder PRE-CAST from a hosted fp8 checkpoint, the bf16 entry over-states what is loaded
- # by ~11 GB for ltx-2's Gemma3-12B, which drags every budget below with it. The VAE is
- # never quantised (quantize_text_encoders touches text encoders only, and vae_force_fp32
- # families widen it), so only components[1] is scaled.
+ # bf16_components_gb is (transformer, text_encoder, vae) at bf16. When this pick takes its encoder PRE-CAST from
+ # a hosted fp8 checkpoint, the bf16 entry over-states what is loaded by ~11 GB for ltx-2's Gemma3-12B, which
+ # drags every budget below with it. The VAE is never quantised (quantize_text_encoders touches text encoders
+ # only, and vae_force_fp32 families widen it), so only components[1] is scaled.
from .diffusion_te_prequant import te_prequant_budget_scale
te_scale = te_prequant_budget_scale(
@@ -3631,10 +3568,10 @@ class VideoBackend:
its bf16 size. Pure and cheap (``plan_diffusion_memory`` is arithmetic), so the
dense-encoder plan can be rebuilt below if the pre-cast injection does not land."""
text_encoder_gb = components[1] * scale if components is not None else 0.0
- # dtype_scale doubles bf16 terms when the fp16 promotion lands fp32 on an accelerator.
- # A vae_force_fp32 family is the one component it must NOT touch: its table term is
- # already recorded at fp32 and assembly pins the VAE to fp32 either way, so scaling it
- # counts those bytes twice (2.8 GB on Wan TI2V-5B) against a hard refusal.
+ # dtype_scale doubles bf16 terms when the fp16 promotion lands fp32 on an accelerator. A vae_force_fp32
+ # family is the one component it must NOT touch: its table term is already recorded at fp32 and assembly
+ # pins the VAE to fp32 either way, so scaling it counts those bytes twice (2.8 GB on Wan TI2V-5B) against a
+ # hard refusal.
vae_gb = components[2] if components is not None else 0.0
vae_scale = 1.0 if getattr(fam, "vae_force_fp32", False) else dtype_scale
scaled_te_gb = text_encoder_gb * dtype_scale
@@ -3661,7 +3598,8 @@ class VideoBackend:
companion_mib = (
int(scaled_companions_gb * mib_per_gb) if components is not None else None
)
- # Budget ALL weights: companions stay resident, so budgeting the transformer alone lets auto pick OFFLOAD_NONE and OOM.
+ # Budget ALL weights: companions stay resident, so budgeting the transformer alone lets auto pick
+ # OFFLOAD_NONE and OOM.
model_dense_mib = (
transformer_mib + (companion_mib or 0) if transformer_mib is not None else None
)
@@ -3673,17 +3611,17 @@ class VideoBackend:
companion_dense_mib = companion_mib,
requested_mode = normalize_memory_mode(memory_mode),
)
- # Parity with the image dense-quant path: the bf16-table plan can force offload a quantised DiT would not need, so re-plan with the scheme factor and keep resident if it fits.
+ # Parity with the image dense-quant path: the bf16-table plan can force offload a quantised DiT would not
+ # need, so re-plan with the scheme factor and keep resident if it fits.
dense_plan = planned
replanned_for_quant = False
- # NOT re-triggered by a unified-memory shortfall, deliberately. This re-plan prices the
- # quantised DiT's STEADY size, but the video path has no pre-quantised artifact: the DiT
- # is always materialised dense (from_pretrained / from_single_file) and
- # quantize_transformer rewrites it in place afterwards, so the build PEAK is the bf16
- # figure whatever scheme is picked. On discrete VRAM the steady state is the right thing
- # to plan placement against -- an oversized peak raises a catchable torch OOM and offload
- # is still there -- but on unified memory the peak is what the OS kills for, with no
- # exception to catch, so the refusal below has to keep reading the dense plan.
+ # NOT re-triggered by a unified-memory shortfall, deliberately. This re-plan prices the quantised DiT's
+ # STEADY size, but the video path has no pre-quantised artifact: the DiT is always materialised dense
+ # (from_pretrained / from_single_file) and quantize_transformer rewrites it in place afterwards, so the
+ # build PEAK is the bf16 figure whatever scheme is picked. On discrete VRAM the steady state is the right
+ # thing to plan placement against -- an oversized peak raises a catchable torch OOM and offload is still
+ # there -- but on unified memory the peak is what the OS kills for, with no exception to catch, so the
+ # refusal below has to keep reading the dense plan.
if (
kind == "pipeline"
and planned.offload_policy != "none"
@@ -3720,27 +3658,30 @@ class VideoBackend:
plan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = True)
- # On unified memory the plan's 'none' policy is a placement, not a fit: there is no
- # offload tier left to fall back to, so an oversized load is killed by the OS with no
- # torch OOM to catch. Refuse now, after the eviction above and before any weight is
- # materialised -- on the dense plan, which is what the DiT build peaks at (see above).
+ # On unified memory the plan's 'none' policy is a placement, not a fit: there is no offload tier left to fall
+ # back to, so an oversized load is killed by the OS with no torch OOM to catch. Refuse now, after the eviction
+ # above and before any weight is materialised -- on the dense plan, which is what the DiT build peaks at (see
+ # above).
raise_on_unified_memory_shortfall(plan, family = getattr(fam, "name", None), logger = logger)
# ── build the pipeline.
pipeline_cls = getattr(diffusers, fam.pipeline_class)
- # cache_dir pins every loader call to the live cache root, so a mid-session change cannot split one model across roots.
+ # cache_dir pins every loader call to the live cache root, so a mid-session change cannot split one model across
+ # roots.
pipe_kwargs: dict[str, Any] = {
"torch_dtype": dtype,
"cache_dir": hub_cache_dir(),
"local_files_only": local_files_only,
}
if getattr(fam, "vae_force_fp32", False):
- # Wan VAE must decode in float32: a scalar torch_dtype truncates its fp32 weights to bf16 and a later widen only
- # restores lossy values (banding / black frames). "default" MUST be set or unlisted components fall back to fp32.
+ # Wan VAE must decode in float32: a scalar torch_dtype truncates its fp32 weights to bf16 and a later widen
+ # only restores lossy values (banding / black frames). "default" MUST be set or unlisted components fall
+ # back to fp32.
pipe_kwargs["torch_dtype"] = {"vae": torch.float32, "default": dtype}
if hf_token:
pipe_kwargs["token"] = hf_token
- # A hosted pre-cast fp8 text encoder skips the dense TE download (for LTX Gemma3-27B, ~50 GB). The cast re-applies idempotently.
+ # A hosted pre-cast fp8 text encoder skips the dense TE download (for LTX Gemma3-27B, ~50 GB). The cast
+ # re-applies idempotently.
from .diffusion_te_prequant import te_prequant_pipe_kwargs
te_injected = te_prequant_pipe_kwargs(
@@ -3754,10 +3695,10 @@ class VideoBackend:
local_files_only = local_files_only,
)
pipe_kwargs.update(te_injected)
- # The fp8-sized budget is valid only once the pre-cast encoder is actually in hand. Injection is best-effort
- # (an unreachable / corrupt / base-mismatched checkpoint falls back to the dense encoder), and a dense encoder
- # under an fp8 budget under-states the resident requirement by ~11 GB for ltx-2, which is the direction that
- # OOMs. Placement is applied post-assembly (apply_memory_plan below), so re-plan at bf16 here, exactly as the
+ # The fp8-sized budget is valid only once the pre-cast encoder is in hand. Injection is best-effort (an
+ # unreachable / corrupt / base-mismatched checkpoint falls back to the dense encoder), and a dense encoder under
+ # an fp8 budget under-states the resident requirement by ~11 GB for ltx-2, which is the direction that OOMs.
+ # Placement is applied post-assembly (apply_memory_plan below), so re-plan at bf16 here, exactly as the
# dense-quant fallback below restores bf16_plan.
if te_scale != 1.0 and not te_injected:
logger.warning(
@@ -3765,8 +3706,9 @@ class VideoBackend:
"memory at the dense bf16 text-encoder size"
)
plan, bf16_plan, quant_replanned = _plan_for_te_scale(1.0, log = False)
- # Injection is best-effort (a missing checkpoint falls back to the dense encoder), but the scoped pre-download already dropped those
- # shards and from_pretrained cannot re-fetch from a local snapshot dir, so top them up here. ltx23 never reaches this: it gets the hub id.
+ # Injection is best-effort (a missing checkpoint falls back to the dense encoder), but the scoped pre-download
+ # already dropped those shards and from_pretrained cannot re-fetch from a local snapshot dir, so top them up
+ # here. ltx23 never reaches this: it gets the hub id.
missing_dense = [c for c in (_te_prequant_skipped or ()) if c not in pipe_kwargs]
if missing_dense and _base_local_dir:
logger.warning(
@@ -3780,9 +3722,9 @@ class VideoBackend:
hf_token,
kind,
ltx23 = False,
- # An offline load never had a scoped snapshot to top up (the prefetch returns
- # None), so this is a no-op there -- but the flag is carried rather than
- # defaulted so the top-up can never become the one call that reaches the Hub.
+ # An offline load never had a scoped snapshot to top up (the prefetch returns None), so this is a
+ # no-op there -- but the flag is carried rather than defaulted so the top-up can never become the
+ # one call that reaches the Hub.
local_files_only = local_files_only,
)
or _base_local_dir
@@ -3799,9 +3741,9 @@ class VideoBackend:
"subfolder": "transformer",
"token": hf_token,
"cache_dir": hub_cache_dir(),
- # base is a REPO ID that diffusers resolves through load_config(), so the flag has
- # to ride along or the non-LTX single-file path fetches the config after eviction.
- # The LTX 2.3 branch below takes local_files_only through its own assembler.
+ # base is a REPO ID that diffusers resolves through load_config(), so the flag has to ride along or the
+ # non-LTX single-file path fetches the config after eviction. The LTX 2.3 branch below takes
+ # local_files_only through its own assembler.
"local_files_only": local_files_only,
}
if kind == "gguf":
@@ -3811,19 +3753,20 @@ class VideoBackend:
from .video_ltx2 import is_ltx23_checkpoint, load_ltx23_pipeline
if fam.name == "ltx-2" and is_ltx23_checkpoint(checkpoint_path):
- # 2.3 checkpoints need the full assembly: new config flags, key renames the stock converter lacks, and the 2.3 connectors/VAEs/vocoder.
+ # 2.3 checkpoints need the full assembly: new config flags, key renames the stock converter lacks, and
+ # the 2.3 connectors/VAEs/vocoder.
pipe = load_ltx23_pipeline(
checkpoint_path,
base_repo = base,
torch_dtype = dtype,
is_gguf = kind == "gguf",
hf_token = hf_token,
- # The 2.3 assembly builds every component itself and never sees pipe_kwargs, so hand the pre-cast encoder over explicitly.
+ # The 2.3 assembly builds every component itself and never sees pipe_kwargs, so hand the pre-cast
+ # encoder over explicitly.
text_encoder = pipe_kwargs.get("text_encoder"),
- # And the no-download promise with it, for the same reason: it is in
- # pipe_kwargs, and this branch is the one path that does not read them. There is
- # no staged snapshot to fall back on either -- _base_local_dir is None for 2.3
- # by design -- so every component below resolves the hub id.
+ # And the no-download promise with it, for the same reason: it is in pipe_kwargs, and this branch is
+ # the one path that does not read them. There is no staged snapshot to fall back on either --
+ # _base_local_dir is None for 2.3 by design -- so every component below resolves the hub id.
local_files_only = local_files_only,
)
else:
@@ -3832,7 +3775,8 @@ class VideoBackend:
_base_local_dir or base, transformer = transformer, **pipe_kwargs
)
- # The dtype dict already loads the Wan VAE at float32; belt-and-suspenders for any path that bypassed it (e.g. a passed-in vae=).
+ # The dtype dict already loads the Wan VAE at float32; belt-and-suspenders for any path that bypassed it (e.g. a
+ # passed-in vae=).
if getattr(fam, "vae_force_fp32", False):
vae = getattr(pipe, "vae", None)
if vae is not None and getattr(vae, "dtype", None) is not torch.float32:
@@ -3843,16 +3787,17 @@ class VideoBackend:
clear_gpu_cache()
raise RuntimeError("Video load was cancelled or superseded.")
- # For a dual-DiT MoE every optimisation site below covers BOTH experts: ``views`` is (pipe, _SecondDiTView(pipe)); single-DiT resolves to (pipe,).
+ # For a dual-DiT MoE every optimisation site below covers BOTH experts: ``views`` is (pipe,
+ # _SecondDiTView(pipe)); single-DiT resolves to (pipe,).
views = _views_for(pipe, fam)
# dense transformer quant (opt-in, pipeline-kind only): torchao-quantise the dense bf16 DiT in place onto the
# low-precision tensor cores. CUDA + bf16 only, best-effort. Quant must precede compile (eager is ~30x slower).
transformer_quant_engaged: Optional[str] = None
quant_skipped_for_offload = False
- # An EXPLICIT scheme (not auto, not off) is a contract, so a decline below must be reported
- # and -- unless the escape hatch is set -- must stop the load rather than quietly running
- # the DiT at bf16 while the Precision dropdown still reads FP8.
+ # An EXPLICIT scheme (not auto, not off) is a contract, so a decline below must be reported and -- unless the
+ # escape hatch is set -- must stop the load rather than quietly running the DiT at bf16 while the Precision
+ # dropdown still reads FP8.
transformer_quant_pinned = normalize_transformer_quant(transformer_quant)
if transformer_quant_pinned == TQ_AUTO:
transformer_quant_pinned = None
@@ -3876,7 +3821,8 @@ class VideoBackend:
and dense_transformer_supported(target)
and plan.offload_policy != "none"
):
- # Offload hooks move modules with Module.to(), which torchao quantized tensors reject. Skip quant (dense-under-offload beats a crash); forceable via a resident mode.
+ # Offload hooks move modules with Module.to(), which torchao quantized tensors reject. Skip quant
+ # (dense-under-offload beats a crash); forceable via a resident mode.
logger.info(
"video.transformer_quant: skipped (offload policy '%s' moves the "
"DiT via Module.to(), unsupported for torchao quantized tensors); "
@@ -3896,7 +3842,8 @@ class VideoBackend:
):
engaged = []
for view in views:
- # Pass each expert view so both DiTs quantise with the same scheme. The family name drives _FAMILY_SCHEME_DENY.
+ # Pass each expert view so both DiTs quantise with the same scheme. The family name drives
+ # _FAMILY_SCHEME_DENY.
scheme = quantize_transformer(
view,
target,
@@ -3923,12 +3870,16 @@ class VideoBackend:
"rules it out); see the server log"
)
transformer_quant_decline_status = RESOLVED_UNSUPPORTED
- # The quant-sized plan is valid only when quant engaged; a dense fallback keeps bf16 placement.
+ # The quant-sized plan is valid only when quant engaged; a dense fallback keeps bf16 placement
if quant_replanned and transformer_quant_engaged is None:
plan = bf16_plan
- # Fail closed on a declined EXPLICIT precision, mirroring the image loader: a clip rendered
- # at bf16 under a Precision dropdown still reading FP8 is not evidence the request ran.
+ # Fail closed on a declined EXPLICIT precision, mirroring the image loader: a clip rendered at bf16 under a
+ # Precision dropdown still reading FP8 is not evidence the request ran.
+ # Same contract for the other half of "the requested precision": an explicit encoder mode that engaged NOTHING
+ # leaves a dense bf16 encoder the caller did not ask for, and a PARTIAL cast leaves conditioning split across
+ # quantised and dense towers. A mode that engaged something ELSE (int8 -> fp8) is reported by the badge instead:
+ # the encoder is quantised, just not the way asked, and the reason says so.
if (
transformer_quant_engaged is None
and transformer_quant_pinned is not None
@@ -3946,7 +3897,8 @@ class VideoBackend:
)
)
- # dense text-encoder quant (opt-in): the companion encoder is often the largest resident, so quantise it in place before placement, for every kind. Best-effort.
+ # dense text-encoder quant (opt-in): the companion encoder is often the largest resident, so quantise it in
+ # place before placement, for every kind. Best-effort.
text_encoder_outcome = quantize_text_encoders(
pipe,
target,
@@ -3956,11 +3908,6 @@ class VideoBackend:
logger = logger,
)
text_encoder_quant_engaged = text_encoder_outcome.mode
- # Same contract for the other half of "the requested precision": an explicit encoder mode
- # that engaged NOTHING leaves a dense bf16 encoder the caller did not ask for, and a
- # PARTIAL cast leaves conditioning split across quantised and dense towers. A mode that
- # engaged something ELSE (int8 -> fp8) is reported by the badge instead: the encoder is
- # quantised, just not the way asked, and the reason says so.
if (
(text_encoder_quant_engaged is None or text_encoder_outcome.partial)
and normalize_te_quant(text_encoder_quant) is not None
@@ -3983,7 +3930,8 @@ class VideoBackend:
effective_speed = resolve_speed_mode(
speed_mode, is_gguf = kind == "gguf", dense_default = SPEED_DEFAULT
)
- # A torchao-quantised DiT must be compiled (eager is ~30x slower), so force the regional profile when quant engaged but speed was off.
+ # A torchao-quantised DiT must be compiled (eager is ~30x slower), so force the regional profile when quant
+ # engaged but speed was off.
if transformer_quant_engaged is not None and effective_speed == SPEED_OFF:
logger.info(
"video.transformer_quant: forcing speed_mode=default "
@@ -3991,12 +3939,14 @@ class VideoBackend:
)
effective_speed = SPEED_DEFAULT
backend_flags = snapshot_backend_flags()
- # Until the state commit hands ownership to _teardown_state_locked, a failure must restore these globals itself. Registered BEFORE the first mutation.
+ # Until the state commit hands ownership to _teardown_state_locked, a failure has to restore these process-wide
+ # flags itself. Registered BEFORE the first mutation, as above.
self._precommit_globals = (_load_token, backend_flags)
- # Step cache tri-state: unset/"auto" -> FBCACHE_MIN_STEPS policy (re-checked per generation); "off"/"fbcache" pinned. Run per expert.
+ # Step cache tri-state: unset/"auto" -> FBCACHE_MIN_STEPS policy (re-checked per generation); "off"/"fbcache"
+ # pinned. Run per expert.
cache_request = normalize_transformer_cache(transformer_cache)
cache_auto = transformer_cache is None or cache_request == TC_AUTO
- # GGUF and torchao-quantised DiTs need the higher threshold to trigger over quant noise.
+ # GGUF and torchao-quantised DiTs need the higher threshold to trigger over quant noise
cache_quant_active = kind == "gguf" or transformer_quant_engaged is not None
default_cache_steps: Optional[int] = None
if cache_auto:
@@ -4008,7 +3958,8 @@ class VideoBackend:
view,
mode = cache_request,
threshold = transformer_cache_threshold,
- # A quantized transformer's residuals are larger, so both engaged quant and GGUF need the higher FBCache threshold.
+ # A quantized transformer's residuals are larger, so both engaged quant and GGUF need the higher FBCache
+ # threshold.
quant_active = cache_quant_active,
logger = logger,
)
@@ -4034,22 +3985,22 @@ class VideoBackend:
else:
cache_reason = "requested"
attention_engaged = None
- # HunyuanVideo-1.5 only, and once for the whole pipe (the installer fans out over every
- # denoiser DiT itself). Before apply_attention_backend below, so the requested kernel pins
- # onto the new processors. Held off on SPEED_OFF (which must stay bit-identical) and on
- # SPEED_MAX (its blocks compile with dynamic=False, and the trimmed text length varies per
- # prompt, so every prompt would be a fresh graph).
+ # HunyuanVideo-1.5 only, and once for the whole pipe (the installer fans out over every denoiser DiT itself).
+ # Before apply_attention_backend below, so the requested kernel pins onto the new processors. Held off on
+ # SPEED_OFF (which must stay bit-identical) and on SPEED_MAX (its blocks compile with dynamic=False, and the
+ # trimmed text length varies per prompt, so every prompt would be a fresh graph).
attention_trim_engaged = (
install_hunyuan_attention_trim(pipe, fam, logger = logger)
if effective_speed not in (SPEED_OFF, SPEED_MAX)
else False
)
- # Sets a flag the pipelines read when they first build their frequency tables, so it need
- # only happen before generation, not before placement.
+ # Sets a flag the pipelines read when they first build their frequency tables, so it need only happen before
+ # generation, not before placement.
force_float32_rope(pipe, target, logger = logger)
speed_optims: tuple = ()
for view in views:
- # Both helpers act on ``view.transformer``; call once per view (engaged values match, so record the first). is_gguf needs kind==gguf AND no quant engaged.
+ # Both helpers act on ``view.transformer``; call once per view (engaged values match, so record the first).
+ # is_gguf needs kind==gguf AND no quant engaged.
gguf_transformer = kind == "gguf" and transformer_quant_engaged is None
engaged = apply_attention_backend(
view,
@@ -4057,10 +4008,10 @@ class VideoBackend:
target, attention_backend, speed_active = effective_speed != SPEED_OFF
),
logger = logger,
- # The probe behind this must see the dtype the pipeline actually RUNS in. Every
- # video family promotes a resolved fp16 to fp32 above, and the fused SDPA kernels
- # are half-precision only, so probing the pre-promotion fp16 would report a healthy
- # device for a generation that is in fact dispatching to the quadratic math backend.
+ # The probe behind this must see the dtype the pipeline actually RUNS in. Every video family promotes a
+ # resolved fp16 to fp32 above, and the fused SDPA kernels are half-precision only, so probing the
+ # pre-promotion fp16 would report a healthy device for a generation that is in fact dispatching to the
+ # quadratic math backend.
target = types.SimpleNamespace(device = target.device, dtype = dtype),
)
applied = apply_speed_optims(
@@ -4069,7 +4020,8 @@ class VideoBackend:
is_gguf = gguf_transformer,
family = fam,
speed_mode = effective_speed,
- # An auto cache that could still engage also drops fullgraph (FBCache under a fullgraph-compiled DiT crashes).
+ # An auto cache that could still engage also drops fullgraph (FBCache under a fullgraph-compiled DiT
+ # crashes)
cache_active = cache_engaged is not None or cache_may_toggle,
offload_active = plan.offload_policy != "none",
)
@@ -4079,7 +4031,8 @@ class VideoBackend:
if attention_trim_engaged:
speed_optims += ("hunyuan_attn_trim",)
with self._generate_lock:
- # A cancelled/superseded load must not place weights on a GPU the arbiter may have reassigned; recheck before placement.
+ # A cancelled/superseded load must not place weights on a GPU the arbiter may have reassigned; recheck
+ # before placement.
if _load_token is not None and _load_token != self._load_token:
del pipe
clear_gpu_cache()
@@ -4088,14 +4041,14 @@ class VideoBackend:
pipe,
plan,
device = device,
- # Same reason as the image path: a bare "cuda" sends the CPU-offload hooks to
- # ordinal 0 whatever was selected.
+ # Same reason as the image path: a bare "cuda" sends the CPU-offload hooks to ordinal 0 whatever was
+ # selected
placement_device = target.torch_device,
logger = logger,
)
- # A dual-DiT MoE needs no extra per-expert pass: apply_memory_plan covers every DiT; a second pass would duplicate-hook.
+ # A dual-DiT MoE needs no extra per-expert pass: apply_memory_plan covers every DiT; a second pass would
+ # duplicate-hook.
if not vae_tiling:
- # Whole-clip decode is the video memory peak; tiling is near-free, so always on.
try:
pipe.vae.enable_tiling()
vae_tiling = True
@@ -4133,11 +4086,11 @@ class VideoBackend:
"transformer_quant": (
transformer_quant_requested,
transformer_quant_engaged or "off",
- # Honest framing: the shipped torchao schemes cut load time and resident memory ~2x, but per-step GEMMs are at best bf16 parity.
+ # Honest framing: the shipped torchao schemes cut load time and resident memory ~2x, but
+ # per-step GEMMs are at best bf16 parity.
"DiT(s) quantised (halves resident weights; hosted checkpoints cut "
"load time; per-step speed is roughly bf16 parity)"
if transformer_quant_engaged is not None
- # A recorded decline names WHY; the generic strings are the fallback.
else (
transformer_quant_decline
or (
@@ -4147,8 +4100,8 @@ class VideoBackend:
else "not engaged (dense bf16 DiT loaded)"
)
),
- # Honored when the quant engaged AND when the ask was "off" (bf16 is what
- # that asks for). Only reached with the escape hatch set for a pinned ask.
+ # Honored when the quant engaged AND when the ask was "off" (bf16 is what that asks for). Only
+ # reached with the escape hatch set for a pinned ask.
RESOLVED_APPLIED
if transformer_quant_engaged is not None or transformer_quant_pinned is None
else transformer_quant_decline_status,
@@ -4187,7 +4140,6 @@ class VideoBackend:
vae_tiling = vae_tiling,
memory_mode = plan.requested_mode,
speed_mode = effective_speed,
- # Already filtered to the engaged optimisations (True names only).
speed_optims = speed_optims,
backend_flags = backend_flags,
attention_backend = attention_engaged,
@@ -4199,7 +4151,6 @@ class VideoBackend:
text_encoder_quant = text_encoder_quant_engaged,
resolved = resolved,
)
- # Ownership of the globals transferred to _state / _teardown_state_locked.
self._precommit_globals = None
logger.info(
"video.loaded: %s (%s, %s, offload=%s, speed=%s, quant=%s)",
@@ -4239,17 +4190,14 @@ class VideoBackend:
return
import torch
- # Same rule as the conventional path: the table is bf16, so an fp32 promotion on an
- # accelerator doubles it.
+ # Same rule as the conventional path: the table is bf16, so an fp32 promotion on an accelerator doubles it.
dtype_scale = 2.0 if device != "cpu" and dtype is torch.float32 else 1.0
transformer_gb, text_encoder_gb, vae_gb = components
- # Only a scheme with a hosted checkpoint replaces the dense denoiser; anything else (auto
- # included) keeps the released bfloat16 components, which is what the modular loader does.
- #
- # Its MEASURED size when the family publishes one, not the generic steady factor: H3's
- # hosted denoisers are quantized AND structurally pruned (the curve-form adaLN), so 0.55 x
- # 66.3 GB reads 36.5 GB against a real ~20.3 GB, and the 16 GB gap is enough to refuse a
- # supported prequant load that fits on a 128 GB Mac.
+ # Only a scheme with a hosted checkpoint replaces the dense denoiser; anything else (auto included) keeps the
+ # released bfloat16 components, which is what the modular loader does. Its MEASURED size when the family
+ # publishes one, not the generic steady factor: H3's hosted denoisers are quantized AND structurally pruned (the
+ # curve-form adaLN), so 0.55 x 66.3 GB reads 36.5 GB against a real ~20.3 GB, and the 16 GB gap is enough to
+ # refuse a supported prequant load that fits on a 128 GB Mac.
measured = getattr(fam, "prequant_resident_gb", None) if scheme else None
factor = _QUANT_STEADY_FACTOR.get(scheme) if scheme else None
if measured:
@@ -4303,23 +4251,23 @@ class VideoBackend:
quantized CONDITIONER and the released bfloat16 DENOISER; ``"none"`` pins the released
component either way; an explicit scheme is honored or falls back with the reason
recorded."""
- # Defence in depth: validate_load_request now refuses a single_file modular pick outright,
- # so this is unreachable from the routes. It stays because this method is also reachable
- # directly, and by the time it runs the resident pipeline has already been torn down.
+ # Defence in depth: validate_load_request now refuses a single_file modular pick outright, so this is
+ # unreachable from the routes. It stays because this method is also reachable directly, and by the time it runs
+ # the resident pipeline has already been torn down.
if kind != "pipeline":
raise ValueError("MiniMax-H3 Diffusers loading requires the pipeline artifact.")
umem_target = target if target is not None else self._device_target()
- # What the CALLER asked for, kept for the resolved record, before the tri-state below
- # rewrites it. An unset request must not read back as a pin.
+ # What the CALLER asked for, kept for the resolved record, before the tri-state below rewrites it. An unset
+ # request must not read back as a pin.
transformer_quant_requested = transformer_quant
text_encoder_quant_requested = text_encoder_quant
transformer_quant_is_auto = _h3_precision_unset(transformer_quant)
text_encoder_quant_is_auto = _h3_precision_unset(text_encoder_quant)
# Load one denoiser partition. fl2va also covers text-only generation.
workflow = h3_task or fam.modular_workflow
- # Which component this workflow's denoise step reads. One repo, two partitions: seeding
- # the wrong attribute leaves the block with no denoiser AND has load_components fetch the
- # dense 66.28 GB partition the seed was meant to replace.
+ # Which component this workflow's denoise step reads. One repo, two partitions: seeding the wrong attribute
+ # leaves the block with no denoiser AND has load_components fetch the dense 66.28 GB partition the seed was
+ # meant to replace.
denoiser_component = h3_denoiser_component(workflow)
manager = diffusers.ComponentsManager()
load_kwargs: dict[str, Any] = {
@@ -4330,25 +4278,19 @@ class VideoBackend:
if hf_token:
load_kwargs["token"] = hf_token
pipe = diffusers.ModularPipeline.from_pretrained(_base_local_dir or repo_id, **load_kwargs)
- # ── hosted pre-quantized denoiser, BEFORE load_components builds a dense one.
transformer_quant_engaged: Optional[str] = None
transformer_quant_reason = "released bfloat16 components"
- # "auto" asks the backend to choose. The auto LADDER is still not it -- that quantises a
- # dense module on device, which this workflow never materialises -- so the choice is
- # between the released denoiser and a hosted checkpoint, and it turns on whether the
- # released one can stay resident.
- #
- # Where it fits, auto keeps it: same weights, same picture, and the pin plus regional
- # compile below make it fast without touching a single value. Where it does not, keeping it
- # is not a smaller win but a cliff -- it rides the CPU-offload rotation, a module that moves
- # cannot be compiled, and the same 8-step job goes 23.7 s to 194 s. The hosted checkpoint is
- # 20.3 GB resident, so it pins, so the compile comes back.
- #
- # The trade is that the hosted denoisers RE-ROLL the sample: mean SSIM 0.49 (int8) / 0.43
- # (fp8) against the released weights, where that config against itself scores 0.99. No NaN,
- # no black frames, no visible degradation at the model's own 30-step schedule, but not the
- # same picture. Which is why this is a fallback and not a blanket default: it is taken only
- # against a configuration nobody would choose on purpose.
+ # "auto" asks the backend to choose. The auto LADDER is still not it -- that quantises a dense module on device,
+ # which this workflow never materialises -- so the choice is between the released denoiser and a hosted
+ # checkpoint, and it turns on whether the released one can stay resident. Where it fits, auto keeps it: same
+ # weights, same picture, and the pin plus regional compile below make it fast without touching a single value.
+ # Where it does not, keeping it is not a smaller win but a cliff -- it rides the CPU-offload rotation, a module
+ # that moves cannot be compiled, and the same 8-step job goes 23.7 s to 194 s. The hosted checkpoint is 20.3 GB
+ # resident, so it pins, so the compile comes back. The trade is that the hosted denoisers RE-ROLL the sample:
+ # mean SSIM 0.49 (int8) / 0.43 (fp8) against the released weights, where that config against itself scores 0.99.
+ # No NaN, no black frames, no visible degradation at the model's own 30-step schedule, but not the same picture.
+ # Which is why this is a fallback and not a blanket default: it is taken only against a configuration nobody
+ # would choose.
scheme = (
None if transformer_quant_is_auto else normalize_transformer_quant(transformer_quant)
)
@@ -4356,18 +4298,17 @@ class VideoBackend:
scheme = None
auto_fallback_scheme = None
if transformer_quant_is_auto:
- # Already settled if the download planner acted on it -- the dense denoiser shards are
- # not on disk, so re-deciding here against a reading that has moved could ask for a
- # component this load can no longer open. Otherwise decide now, against live free
- # memory, which is the reading that describes the card once the previous pipeline is
- # gone (the plan only had the card's capacity to go on).
+ # Already settled if the download planner acted on it -- the dense denoiser shards are not on disk, so
+ # re-deciding here against a reading that has moved could ask for a component this load can no longer open.
+ # Otherwise decide now, against live free memory, which is the reading that describes the card once the
+ # previous pipeline is gone (the plan only had the card's capacity to go on).
auto_fallback_scheme = _h3_auto_denoiser_planned or _h3_auto_denoiser_scheme(
fam,
target = umem_target,
dtype = dtype,
device = device,
- # The conditioner precision this load WILL engage, predicted the same way the
- # encoder resolves it below, because it is part of what has to fit beside it.
+ # The conditioner precision this load WILL engage, predicted the same way the encoder resolves it below,
+ # because it is part of what has to fit beside it.
te_scheme = self._h3_te_quant_scheme(fam, text_encoder_quant, base, umem_target),
task = workflow,
base_repo = base,
@@ -4382,13 +4323,11 @@ class VideoBackend:
"%s checkpoint (ask for transformer_quant='none' to keep the released weights)",
auto_fallback_scheme,
)
- # Per (scheme, PARTITION), not per scheme: each workflow denoises against its own
- # checkpoint partition and they are indistinguishable once loaded, so a scheme that has an
- # artifact for the other one has none for this. It used to be enough to ask "is this the
- # reference workflow", because only the keyframe partition had hosted checkpoints; both do
- # now. validate_load_request refuses an unavailable pairing on the route; a direct call
- # lands here and drops to the released components instead of silently generating from the
- # wrong partition.
+ # Per (scheme, PARTITION), not per scheme: each workflow denoises against its own checkpoint partition and they
+ # are indistinguishable once loaded, so a scheme that has an artifact for the other one has none for this. It
+ # used to be enough to ask "is this the reference workflow", because only the keyframe partition had hosted
+ # checkpoints; both do now. validate_load_request refuses an unavailable pairing on the route; a direct call
+ # lands here and drops to the released components instead of silently generating from the wrong partition.
if scheme is not None and not video_family_prequant_available(
fam, scheme, task = workflow, base_repo = base
):
@@ -4396,13 +4335,12 @@ class VideoBackend:
f"no hosted pre-quantized {scheme} checkpoint for {fam.name} {workflow}"
)
scheme = None
- # The conventional loader refuses an oversized unified-memory load before any weight is
- # materialised; this workflow returns above that check, so it runs its own here. It has to:
- # load_components builds every component dense, and the ComponentsManager's CPU offload
- # frees nothing on unified memory (host and device are one pool), so H3's 144.2 GB
- # component set is an OS kill with no torch OOM to catch. Placed after `scheme` settles and
- # before ANY weight is opened -- the hosted pre-quantized denoiser below is materialised on
- # the CPU, which is the same memory.
+ # The conventional loader refuses an oversized unified-memory load before any weight is materialised; this
+ # workflow returns above that check, so it runs its own here. It has to: load_components builds every component
+ # dense, and the ComponentsManager's CPU offload frees nothing on unified memory (host and device are one pool),
+ # so H3's 144.2 GB component set is an OS kill with no torch OOM to catch. Placed after `scheme` settles and
+ # before ANY weight is opened -- the hosted pre-quantized denoiser below is materialised on the CPU, which is
+ # the same memory.
self._raise_on_modular_unified_shortfall(
fam,
target = umem_target,
@@ -4418,7 +4356,6 @@ class VideoBackend:
source = resolve_prequant_source(fam, scheme, base_repo = base, task = workflow)
if source is None:
- # validate_load_request already refused this on the route; a direct call lands here.
transformer_quant_reason = (
f"no hosted pre-quantized {scheme} checkpoint for {fam.name}"
)
@@ -4427,62 +4364,54 @@ class VideoBackend:
getattr(diffusers, fam.transformer_class),
base,
source,
- # CPU: enable_auto_cpu_offload below owns placement for every component, so a
- # pre-placed denoiser would just be moved again (and a 66 GB pin on a device the
- # manager wanted to offload from is how this OOMs). Same as the pre-cast text
- # encoder path, which also hands the pipeline a CPU-resident module.
+ # CPU: enable_auto_cpu_offload below owns placement for every component, so a pre-placed denoiser
+ # would just be moved again (and a 66 GB pin on a device the manager wanted to offload from is how
+ # this OOMs). Same as the pre-cast text encoder path, which also hands the pipeline a CPU-resident
+ # module.
device = "cpu",
dtype = dtype,
hf_token = hf_token,
scheme = scheme,
- # Reject a checkpoint baked under a different Linear filter, exactly as the
- # image path does, so prequant and runtime-quant stay the same model.
+ # Reject a checkpoint baked under a different Linear filter, exactly as the image path does, so
+ # prequant and runtime-quant stay the same model.
min_features = DEFAULT_MIN_LINEAR_FEATURES,
- # A load nobody asked for may not fetch this checkpoint either, exactly as the
- # image twin refuses it. The switch verified only what the PLAN listed, and
- # `auto` is re-decided here against live free memory rather than the card's
- # capacity, so this branch can ask for a hosted checkpoint the plan never
- # cleared and never staged. A cache miss returns None and the load continues
- # dense, which load_components refuses under the same flag a few lines below.
+ # A load nobody asked for may not fetch this checkpoint either, exactly as the image twin refuses
+ # it. The switch verified only what the PLAN listed, and `auto` is re-decided here against live free
+ # memory rather than the card's capacity, so this branch can ask for a hosted checkpoint the plan
+ # never cleared and never staged. A cache miss returns None and the load continues dense, which
+ # load_components refuses under the same flag a few lines below.
local_files_only = local_files_only,
- # Pin the live cache root, as every other loader call does: unset, the fetch
- # lands under huggingface_hub's import-time constant and a later cache change
- # re-downloads multiple GB into a root Unsloth no longer reads.
+ # Pin the live cache root, as every other loader call does: unset, the fetch lands under
+ # huggingface_hub's import-time constant and a later cache change re-downloads multiple GB into a
+ # root Unsloth no longer reads.
cache_dir = hub_cache_dir(),
- # The hosted H3 denoisers carry the PRUNED (curve-form) adaLN: the modulation is
- # a rank-8 factorization of the time-embedding curve plus a shared table, which
- # is where ~40% of the released model's parameters went. Built from the base
- # repo's dense config, the model is 4 keys short, 1 over and 51 shapes wrong, so
- # without this reshape the strict load fails and every hosted checkpoint falls
- # back to the dense download it exists to avoid.
+ # The hosted H3 denoisers carry the PRUNED (curve-form) adaLN: the modulation is a rank-8
+ # factorization of the time-embedding curve plus a shared table, which is where ~40% of the released
+ # model's parameters went. Built from the base repo's dense config, the model is 4 keys short, 1
+ # over and 51 shapes wrong, so without this reshape the strict load fails and every hosted
+ # checkpoint falls back to the dense download it exists to avoid.
prepare_model = h3_prepare_prequant_model(logger),
- # Read the DENOISER CONFIG from this workflow's own partition. The two H3
- # partitions ship byte-identical configs today, so this changes no shape; it
- # changes which file has to exist. A reference load stages transformer_ref/
- # and nothing under transformer/, so defaulting to "transformer" would send a
- # fully staged, offline-ready load back to the Hub for a config it never
+ # Read the DENOISER CONFIG from this workflow's own partition. The two H3 partitions ship
+ # byte-identical configs today, so this changes no shape; it changes which file has to exist. A
+ # reference load stages transformer_ref/ and nothing under transformer/, so defaulting to
+ # "transformer" would send a fully staged, offline-ready load back to the Hub for a config it never
# planned for -- and on failure straight to the dense fallback it cannot pull.
config_subfolder = denoiser_component,
logger = logger,
)
if transformer is None:
- # Best-effort by contract (missing / mismatched / unreadable checkpoint), so the
- # load continues dense rather than failing after the teardown.
+ # Best-effort by contract (missing / mismatched / unreadable checkpoint), so the load continues
+ # dense rather than failing after the teardown.
transformer_quant_reason = (
f"hosted pre-quantized {scheme} checkpoint unusable; "
f"loaded the released bfloat16 denoiser instead"
)
else:
- # Seeding is what actually saves the download: load_components(names=None) skips
- # every component whose attribute is already set, so the dense 66.3 GB
- # transformer is never fetched. It must happen BEFORE load_components -- after,
- # the dense one has already been pulled and built.
+ # Seeding is what actually saves the download: load_components(names=None) skips every component
+ # whose attribute is already set, so the dense 66.3 GB transformer is never fetched. It must happen
+ # BEFORE load_components -- after, the dense one has already been pulled and built.
pipe.update_components(**{denoiser_component: transformer})
transformer_quant_engaged = scheme
- # Says WHY when the caller did not ask for this. A record that reads the same
- # for a requested int8 and an automatic one leaves a user who never chose a
- # scheme with no way to see that the picture changed, or how to get the
- # released weights back.
transformer_quant_reason = (
f"hosted pre-quantized {scheme} denoiser checkpoint ({source.location})"
if auto_fallback_scheme is None
@@ -4497,50 +4426,47 @@ class VideoBackend:
scheme,
fam.name,
)
- # ── hosted quantized conditioner, for the same reason and in the same place.
- #
- # Under enable_auto_cpu_offload the VRAM floor is the LARGEST resident component, so the
- # 66.7 GB Qwen3-VL conditioner -- not the denoiser -- is what has been pinning it. Seeding
- # here rather than casting after load_components is the entire point: a runtime cast has to
- # materialise the dense encoder first, so its PEAK is the bfloat16 size it was meant to
+ # ── hosted quantized conditioner, for the same reason and in the same place. Under enable_auto_cpu_offload the
+ # VRAM floor is the LARGEST resident component, so the 66.7 GB Qwen3-VL conditioner -- not the denoiser -- is
+ # what has been pinning it. Seeding here rather than casting after load_components is the entire point: a
+ # runtime cast has to materialise the dense encoder first, so its PEAK is the bfloat16 size it was meant to
# avoid, and on this workflow it would also have to download the 62 GB dense component.
+ # ── hosted quantized conditioner, for the same reason and in the same place.
text_encoder_quant_engaged: Optional[str] = None
text_encoder_quant_reason = "released bfloat16 components"
- # Base-gated, through the same resolver the plan and the pre-fetch use, so a derivative
- # base that keeps its own conditioner can never be handed this one.
+ # Base-gated, through the same resolver the plan and the pre-fetch use, so a derivative base that keeps its own
+ # conditioner can never be handed this one.
te_scheme = self._h3_te_quant_scheme(fam, text_encoder_quant, base, umem_target)
- # From here down the request is the NORMALISED scheme (None once "none" has pinned dense),
- # which is what the reason strings and the fail-closed check below were written against.
+ # From here down the request is the NORMALISED scheme (None once "none" has pinned dense), which is what the
+ # reason strings and the fail-closed check below were written against.
text_encoder_quant = (
None if text_encoder_quant_is_auto else normalize_te_quant(text_encoder_quant)
)
if text_encoder_quant_is_auto and te_scheme is not None:
text_encoder_quant_reason = f"auto: hosted quantized {te_scheme} conditioner"
- # And gated a second time, on the identity the base NAME cannot establish. That resolver
- # compares repo ids, so a derivative stored under a directory (or repo) whose last segment
- # is also MiniMax-H3 passes it. This pipeline's own modular index names the repo its
- # conditioner would have come from, which is the actual question: a derivative that
- # retrained the encoder ships it under its own id and says so here, while one that reuses
- # the released encoder still points at it, and quantizing exactly those weights is what
- # the hosted artifact IS. Free, and it cannot disagree with the load, because it reads what
- # diffusers already parsed for it. Unanswerable is a refusal, not a pass.
+ # And gated a second time, on the identity the base NAME cannot establish. That resolver compares repo ids, so a
+ # derivative stored under a directory (or repo) whose last segment is also MiniMax-H3 passes it. This pipeline's
+ # own modular index names the repo its conditioner would have come from, which is the actual question: a
+ # derivative that retrained the encoder ships it under its own id and says so here, while one that reuses the
+ # released encoder still points at it, and quantizing exactly those weights is what the hosted artifact IS.
+ # Free, and it cannot disagree with the load, because it reads what diffusers already parsed for it.
+ # Unanswerable is a refusal, not a pass.
te_index_source = self._h3_te_index_source(pipe) if te_scheme is not None else None
te_index_declined = False
if te_scheme is not None:
- # EXACT here, unlike the plan-side comparison. te_base_equivalent falls through to
- # _same_base_model, which accepts a matching final path segment, and that tolerance is
- # the hole this gate exists to close: someone/MiniMax-H3 is a different repo with
- # different weights and would have passed it. canonical_base still folds a known mirror
- # onto the id it copies, so mirrors are not collateral. A pipeline re-saved locally
- # names local paths here and is refused, which costs the optimisation, not correctness.
+ # EXACT here, unlike the plan-side comparison. te_base_equivalent falls through to _same_base_model, which
+ # accepts a matching final path segment, and that tolerance is the hole this gate exists to close:
+ # someone/MiniMax-H3 is a different repo with different weights and would have passed it. canonical_base
+ # still folds a known mirror onto the id it copies, so mirrors are not collateral. A pipeline re-saved
+ # locally names local paths here and is refused, which costs the optimisation, not correctness.
if te_index_source is None or _h3_te_canonical(te_index_source) != _h3_te_canonical(
fam.base_repo
):
te_scheme = None
te_index_declined = True
if text_encoder_quant is not None and te_scheme is None:
- # A valid request this workflow has no hosted artifact for, or one it has but not for
- # THIS pipeline's conditioner. Say which; do NOT erase the request.
+ # A valid request this workflow has no hosted artifact for, or one it has but not for THIS pipeline's
+ # conditioner. Say which; do NOT erase the request.
if te_index_declined:
text_encoder_quant_reason = (
f"this pipeline's conditioner comes from "
@@ -4560,8 +4486,8 @@ class VideoBackend:
f"loaded the released bfloat16 encoder instead"
)
elif text_encoder_quant_is_auto and te_scheme is None:
- # An UNSET request never fails a load, but it must still say why the fast default did
- # not engage, or a 194-second generation looks like the intended behaviour.
+ # An UNSET request never fails a load, but it must still say why the fast default did not engage, or a
+ # 194-second generation looks like the intended behaviour.
text_encoder_quant_reason = (
f"auto declined: this pipeline's conditioner comes from "
f"{te_index_source or 'a source that could not be read'}, not {fam.base_repo}"
@@ -4576,22 +4502,20 @@ class VideoBackend:
te_scheme,
dtype = dtype,
hf_token = hf_token,
- # Pin the live cache root, exactly as the denoiser fetch above does.
cache_dir = hub_cache_dir(),
- # The scoped pre-download keeps every component config, so the staged snapshot
- # already holds text_encoder/config.json: read it from there rather than sending
- # the config resolution back to the hub.
+ # The scoped pre-download keeps every component config, so the staged snapshot already holds
+ # text_encoder/config.json: read it from there rather than sending the config resolution back to the
+ # hub.
local_base = _base_local_dir,
- # And the same refusal the denoiser and load_components take: this artifact is
- # ~27 GB, the flag protected only the calls either side of it, and the staging
- # fetch that cleared the skip accepted a copy under EITHER cache root while this
- # lookup searched only the live one.
+ # And the same refusal the denoiser and load_components take: this artifact is ~27 GB, the flag
+ # protected only the calls either side of it, and the staging fetch that cleared the skip accepted a
+ # copy under EITHER cache root while this lookup searched only the live one.
local_files_only = local_files_only,
logger = logger,
)
if text_encoder is None:
- # Best-effort by contract, like the denoiser: a missing / unreadable / mismatched
- # artifact continues dense rather than failing after the teardown.
+ # Best-effort by contract, like the denoiser: a missing / unreadable / mismatched artifact continues
+ # dense rather than failing after the teardown.
text_encoder_quant_reason = (
f"hosted quantized {te_scheme} conditioner unusable; "
f"loaded the released bfloat16 encoder instead"
@@ -4607,16 +4531,14 @@ class VideoBackend:
te_scheme,
fam.name,
)
- # Fail closed, exactly as the conventional path does for the same request: an explicit
- # encoder precision that engaged NOTHING leaves a dense bf16 encoder the caller did not ask
- # for, and a render that succeeds at an unrequested precision quietly invalidates whatever
- # it was measuring. Covers all three ways this can end up dense -- no hosted artifact for
- # the scheme, an artifact that is not this pipeline's conditioner, and one that failed to
- # load -- because the caller cannot tell them apart from the result.
- #
- # Raised HERE, before load_components, so the refusal costs nothing rather than arriving
- # after every component has been built. precision_fallback_allowed() is the documented
- # escape hatch, and it lands the dense encoder with the reason recorded as before.
+ # Fail closed, exactly as the conventional path does for the same request: an explicit encoder precision that
+ # engaged NOTHING leaves a dense bf16 encoder the caller did not ask for, and a render that succeeds at an
+ # unrequested precision quietly invalidates whatever it was measuring. Covers all three ways this can end up
+ # dense -- no hosted artifact for the scheme, an artifact that is not this pipeline's conditioner, and one that
+ # failed to load -- because the caller cannot tell them apart from the result. Raised HERE, before
+ # load_components, so the refusal costs nothing rather than arriving after every component has been built.
+ # precision_fallback_allowed() is the documented escape hatch, and it lands the dense encoder with the reason
+ # recorded as before.
if (
text_encoder_quant is not None
and text_encoder_quant_engaged is None
@@ -4631,20 +4553,18 @@ class VideoBackend:
auto_available = False,
)
)
- # The token above only opens the modular index. Every component's own from_pretrained runs
- # here, against the repos that index names, so it has to be passed again or a gated/private
- # component load goes out anonymously (load_components only WARNS on a failed component).
- # Restrict component loading to what this workflow's blocks expect, WITHOUT pruning the
- # block graph itself (the blocks are what route each request between t2va and the
- # keyframe/reference paths). workflow= only narrows the name list load_components already
- # built, and that list skips every component whose attribute is set, so the seeding above
- # still keeps the dense 66.3 GB transformer from being fetched.
+ # The token above only opens the modular index. Every component's own from_pretrained runs here, against the
+ # repos that index names, so it has to be passed again or a gated/private component load goes out anonymously
+ # (load_components only WARNS on a failed component). Restrict component loading to what this workflow's blocks
+ # expect, WITHOUT pruning the block graph itself (the blocks are what route each request between t2va and the
+ # keyframe/reference paths). workflow= only narrows the name list load_components already built, and that list
+ # skips every component whose attribute is set, so the seeding above still keeps the dense 66.3 GB transformer
+ # from being fetched.
if scheme is not None and transformer_quant_engaged is None:
- # The seeding above is best-effort by contract: a missing, corrupt, stale or
- # base-mismatched checkpoint drops to the released bfloat16 denoiser, and that is the
- # 66.3 GB the check before it did not size. load_components would then build it with
- # no refusal left to stop it, which on unified memory is the OS kill this whole guard
- # exists to prevent. Re-run it on the dense set, still before any weight is opened.
+ # The seeding above is best-effort by contract: a missing, corrupt, stale or base-mismatched checkpoint
+ # drops to the released bfloat16 denoiser, and that is the 66.3 GB the check before it did not size.
+ # load_components would then build it with no refusal left to stop it, which on unified memory is the OS
+ # kill this whole guard exists to prevent. Re-run it on the dense set, still before any weight is opened.
self._raise_on_modular_unified_shortfall(
fam,
target = umem_target,
@@ -4653,10 +4573,9 @@ class VideoBackend:
memory_mode = memory_mode,
scheme = None,
)
- # cache_dir for the same reason as the token: load_components forwards extra kwargs through
- # ComponentSpec.load into each from_pretrained, and without it those ~145 GB of Hub-pinned
- # components resolve against the import-time HF_HUB_CACHE snapshot rather than Unsloth's
- # live cache folder, which the user can move.
+ # cache_dir for the same reason as the token: load_components forwards extra kwargs through ComponentSpec.load
+ # into each from_pretrained, and without it those ~145 GB of Hub-pinned components resolve against the
+ # import-time HF_HUB_CACHE snapshot rather than Unsloth's live cache folder, which the user can move.
pipe.load_components(
workflow = workflow,
dtype = dtype,
@@ -4664,10 +4583,9 @@ class VideoBackend:
local_files_only = local_files_only,
**({"token": hf_token} if hf_token else {}),
)
- # The video VAE loads at float32 and the decode runs under float16 autocast, so both
- # copies are resident for the whole decode. Pre-casting the decoder removes the pair
- # without changing a single output value. The encoder stays resident because the
- # keyframe and reference paths both encode their conditioning frames through it.
+ # The video VAE loads at float32 and the decode runs under float16 autocast, so both copies are resident for the
+ # whole decode. Pre-casting the decoder removes the pair without changing a single output value. The encoder
+ # stays resident because the keyframe and reference paths both encode their conditioning frames through it.
from .video_minimax_h3 import trim_h3_video_vae
@@ -4685,67 +4603,60 @@ class VideoBackend:
)
offload_policy = "none"
denoiser_pinned = False
- # load_components just spent minutes building ~145 GB of components, and everything below
- # this line either moves weights onto the card or mutates process-wide backend flags. The
- # conventional placement path fences on the token for exactly that reason; without the
- # same fence here a cancelled or superseded worker resumes into a GPU a replacement load
- # already owns and puts a 66.3 GB denoiser next to it. The next check was the state commit,
+ # load_components just spent minutes building ~145 GB of components, and everything below this line either moves
+ # weights onto the card or mutates process-wide backend flags. The conventional placement path fences on the
+ # token for exactly that reason; without the same fence here a cancelled or superseded worker resumes into a GPU
+ # a replacement load already owns and puts a 66.3 GB denoiser next to it. The next check was the state commit,
# which is after the placement it is meant to prevent.
if _load_token is not None and _load_token != self._load_token:
del pipe
clear_gpu_cache()
raise RuntimeError("Video load was cancelled or superseded.")
- # Resolved BEFORE the placement below, not after it: the dense pin is a speed optimisation
- # by its own reasoning, so "off" has to be known in time to decline it.
+ # Resolved BEFORE the placement below, not after it: the dense pin is a speed optimisation by its own reasoning,
+ # so "off" has to be known in time to decline it.
effective_speed = resolve_speed_mode(speed_mode, is_gguf = False, dense_default = SPEED_DEFAULT)
if effective_speed == SPEED_MAX:
- # SPEED_MAX compiles with dynamic=False. H3's packed sequence length carries the
- # caption's token rows, so a static graph recompiles per prompt: measured 0.957-1.000
- # s/step static against 1.000-1.040 dynamic, and two recompiles paid for it.
+ # SPEED_MAX compiles with dynamic=False. H3's packed sequence length carries the caption's token rows, so a
+ # static graph recompiles per prompt: measured 0.957-1.000 s/step static against 1.000-1.040 dynamic, and
+ # two recompiles paid for it.
logger.info(
"video.speed_mode: MiniMax-H3 runs the 'default' regional profile under max "
"(a static graph retraces on the caption's contribution to the packed length)"
)
effective_speed = SPEED_DEFAULT
if device != "cpu":
- # The partition this workflow opened, not a hardcoded "transformer": a reference run
- # denoises out of transformer_ref, so pinning and sizing both have to name it or they
- # act on a component this load never built.
+ # The partition this workflow opened, not a hardcoded "transformer": a reference run denoises out of
+ # transformer_ref, so pinning and sizing both have to name it or they act on a component this load never
+ # built.
denoiser = getattr(pipe, denoiser_component, None)
- # Asked BEFORE the offload is installed, because the answer decides whether to install
- # it at all. ``_h3_dense_denoiser_resident_bytes`` already returns the denoiser plus
- # EVERYTHING else -- conditioner at its engaged precision, VAEs, activation headroom --
- # so "the denoiser fits" and "the whole set fits" are the same question, asked once.
- #
- # Reading free memory here rather than after the offload reads the same number: this
- # workflow builds every component on the CPU (load_components does, and the hosted
- # denoiser is seeded with device="cpu" for exactly this reason), so there is nothing
- # resident yet for the offload to have freed. And if that ever stopped being true the
- # reading would be SMALLER, which makes this check stricter, never looser.
+ # Asked BEFORE the offload is installed, because the answer decides whether to install it at all.
+ # ``_h3_dense_denoiser_resident_bytes`` already returns the denoiser plus EVERYTHING else -- conditioner at
+ # its engaged precision, VAEs, activation headroom -- so "the denoiser fits" and "the whole set fits" are
+ # the same question, asked once. Reading free memory here rather than after the offload reads the same
+ # number: this workflow builds every component on the CPU (load_components does, and the hosted denoiser is
+ # seeded with device="cpu" for exactly this reason), so there is nothing resident yet for the offload to
+ # have freed. And if that ever stopped being true the reading would be SMALLER, which makes this check
+ # stricter, never looser.
whole_set_sizes = _h3_dense_denoiser_resident_bytes(
fam, denoiser = denoiser, te_scheme = text_encoder_quant_engaged, dtype = dtype
)
free_before_placement = _h3_free_device_bytes(device)
- # The same speed_mode="off" gate the released-denoiser pin below carries, and for the
- # same reason: the headroom in the sizing above is for the family's DEFAULT frame
- # count, so a resident set trades the rotation's ability to absorb a much longer clip
- # for throughput. An explicit "off" is the one request that says do not make that
- # trade, and it keeps exactly the behaviour it has today.
+ # The same speed_mode="off" gate the released-denoiser pin below carries, and for the same reason: the
+ # headroom in the sizing above is for the family's DEFAULT frame count, so a resident set trades the
+ # rotation's ability to absorb a much longer clip for throughput. An explicit "off" is the one request that
+ # says do not make that trade, and it keeps exactly the behaviour it has today.
whole_set_resident = effective_speed != SPEED_OFF and _h3_dense_denoiser_fits(
whole_set_sizes, free_before_placement
)
if whole_set_resident:
- # Nothing to rotate. enable_auto_cpu_offload parks EVERY component in host RAM and
- # moves each one back inside its own pre_forward, so installing it on a card that
- # can hold the whole set buys nothing and costs twice: tens of GB of host RAM held
- # for the life of the load, and the conditioner and VAEs crossing the bus on every
- # generation. Measured 42.2 GB peak host RSS on a 183 GB card with 103 GB of it in
- # use, purely to service a rotation that never needed to happen.
- #
- # pipe.to() rather than the manager: with no hooks installed there is nothing to
- # unpin, and a torchao denoiser moves safely here because this is load time, not
- # mid-block (the move that breaks it is the one a pre_forward makes -- see
- # pin_prequantized_module).
+ # Nothing to rotate. enable_auto_cpu_offload parks EVERY component in host RAM and moves each one back
+ # inside its own pre_forward, so installing it on a card that can hold the whole set buys nothing and
+ # costs twice: tens of GB of host RAM held for the life of the load, and the conditioner and VAEs
+ # crossing the bus on every generation. Measured 42.2 GB peak host RSS on a 183 GB card with 103 GB of
+ # it in use, purely to service a rotation that never needed to happen. pipe.to() rather than the
+ # manager: with no hooks installed there is nothing to unpin, and a torchao denoiser moves safely here
+ # because this is load time, not mid-block (the move that breaks it is the one a pre_forward makes --
+ # see pin_prequantized_module).
pipe.to(device)
offload_policy = "none"
# Resident and hookless, which is the condition the regional compile needs.
@@ -4759,37 +4670,33 @@ class VideoBackend:
)
else:
manager.enable_auto_cpu_offload(
- # Measured H3 activations need substantially more than the
- # official 12 GB example at 10-15 seconds. Forty GB also keeps
- # very large GPUs from retaining both 66 GB components and OOMing
- # while smaller caps succeed by offloading one of them.
+ # Measured H3 activations need substantially more than the official 12 GB example at 10-15 seconds.
+ # Forty GB also keeps very large GPUs from retaining both 66 GB components and OOMing while smaller
+ # caps succeed by offloading one of them.
device = device,
memory_reserve_margin = "40GB",
)
offload_policy = "model"
if transformer_quant_engaged:
- # enable_auto_cpu_offload just parked every component on the CPU and will move
- # each one back inside its own pre_forward. A torchao pre-quantized denoiser does
- # not survive that mid-block move (see pin_prequantized_module), so place it now
- # and take it out of the rotation. Nothing else changes: the encoder and the VAEs
- # keep their hooks and still offload around it.
+ # enable_auto_cpu_offload just parked every component on the CPU and will move each one back inside
+ # its own pre_forward. A torchao pre-quantized denoiser does not survive that mid-block move (see
+ # pin_prequantized_module), so place it now and take it out of the rotation. Nothing else changes:
+ # the encoder and the VAEs keep their hooks and still offload around it.
from .diffusion_prequant import pin_prequantized_module
denoiser_pinned = pin_prequantized_module(
manager, denoiser, device, logger = logger
)
elif denoiser is not None and effective_speed != SPEED_OFF:
- # The RELEASED bfloat16 denoiser, for a different reason: not correctness, speed.
- # Which is why speed=off skips this branch and keeps the rotation: taking a module
- # out of the offload rotation is not free, it trades the ability to budget the
- # denoiser against the requested frame count for throughput, and an explicit "off"
- # is the one request that says do not make that trade. The pre-quantized pin above
- # is NOT gated the same way: there it is correctness, not speed.
- # Every denoise step moves the same 66.3 GB module in and out, and a module that
- # moves cannot be compiled either (the onload hooks fight the graph). Quantizing
- # the conditioner is what makes pinning affordable -- 66.3 GB denoiser + 27.2 GB
- # encoder resident rather than two 66 GB components. Sized against LIVE free
- # memory, after every component is built and parked: the only reading that
- # describes the card this load actually got.
+ # The RELEASED bfloat16 denoiser, for a different reason: not correctness, speed. Which is why
+ # speed=off skips this branch and keeps the rotation: taking a module out of the offload rotation is
+ # not free, it trades the ability to budget the denoiser against the requested frame count for
+ # throughput, and an explicit "off" is the one request that says do not make that trade. The
+ # pre-quantized pin above is NOT gated the same way: there it is correctness, not speed. Every
+ # denoise step moves the same 66.3 GB module in and out, and a module that moves cannot be compiled
+ # either (the onload hooks fight the graph). Quantizing the conditioner is what makes pinning
+ # affordable -- 66.3 GB denoiser + 27.2 GB encoder resident rather than two 66 GB components. Sized
+ # against LIVE free memory, after every component is built and parked: the only reading that
+ # describes the card this load got.
sizes = _h3_dense_denoiser_resident_bytes(
fam, denoiser = denoiser, te_scheme = text_encoder_quant_engaged, dtype = dtype
)
@@ -4819,17 +4726,16 @@ class VideoBackend:
(denoiser_bytes + others_bytes) / 1e9,
)
+ # ── the speed layer this workflow used to skip entirely. apply_speed_optims / select_attention_backend live
+ # below the modular dispatch in load_pipeline, which returns first, so every H3 load reported speed_optims []
+ # and attention_backend null. Called here rather than by moving the dispatch, because the compile decision below
+ # needs ``denoiser_pinned``, which only exists after the offload. ``effective_speed`` itself was resolved above
+ # the offload, because the pin reads it.
# ── the speed layer this workflow used to skip entirely.
- # apply_speed_optims / select_attention_backend live below the modular dispatch in
- # load_pipeline, which returns first, so every H3 load reported speed_optims [] and
- # attention_backend null. Called here rather than by moving the dispatch, because the
- # compile decision below needs ``denoiser_pinned``, which only exists after the offload.
- # ``effective_speed`` itself was resolved above the offload, because the pin reads it.
if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and not denoiser_pinned:
- # Compile ONLY over a resident denoiser: inside the rotation the compiled graph fights
- # the onload hooks, measured 69-85 s against 30-115 s eager on the same job at a
- # 135.7 GB peak. The rest of the tier (channels_last VAE, cudnn.benchmark, attention
- # backend) is kept.
+ # Compile ONLY over a resident denoiser: inside the rotation the compiled graph fights the onload hooks,
+ # measured 69-85 s against 30-115 s eager on the same job at a 135.7 GB peak. The rest of the tier
+ # (channels_last VAE, cudnn.benchmark, attention backend) is kept.
logger.info(
"video.speed_mode: MiniMax-H3 denoiser is in the CPU-offload rotation, so the "
"regional compile is held off (measured slower than eager there); keeping the "
@@ -4837,16 +4743,15 @@ class VideoBackend:
)
effective_speed = SPEED_EAGER
backend_flags = snapshot_backend_flags()
- # Until the state commit hands ownership to _teardown_state_locked, a failure has to restore
- # these process-wide flags itself. Registered BEFORE the first mutation, as above.
+ # Until the state commit hands ownership to _teardown_state_locked, a failure has to restore these process-wide
+ # flags itself.
self._precommit_globals = (_load_token, backend_flags)
attention_engaged = None
speed_optims: tuple = ()
- # Both helpers reach the denoiser through ``pipe.transformer``, and a reference load's DiT
- # is ``transformer_ref``. Handed the bare pipe they would enumerate no denoiser at all, so
- # the partition this run denoises with would keep native attention and stay uncompiled
- # while the resolved record below still reported the requested profile -- and the pin
- # above is what removes the eager downgrade that used to hide it.
+ # Both helpers reach the denoiser through ``pipe.transformer``, and a reference load's DiT is
+ # ``transformer_ref``. Handed the bare pipe they would enumerate no denoiser at all, so the partition this run
+ # denoises with would keep native attention and stay uncompiled while the resolved record below still reported
+ # the requested profile -- and the pin above is what removes the eager downgrade that used to hide it.
speed_view = _denoiser_view(pipe, denoiser_component)
try:
attention_engaged = apply_attention_backend(
@@ -4871,8 +4776,8 @@ class VideoBackend:
family = fam,
speed_mode = effective_speed,
cache_active = False,
- # The conditioner and the VAEs stay in the rotation even when the denoiser is
- # pinned, so the onload hooks are live and fullgraph has to drop.
+ # The conditioner and the VAEs stay in the rotation even when the denoiser is pinned, so the onload
+ # hooks are live and fullgraph has to drop.
offload_active = offload_policy != "none",
logger = logger,
)
@@ -4906,9 +4811,8 @@ class VideoBackend:
transformer_quant_engaged or "off",
transformer_quant_reason,
),
- # The REQUEST on the left, not None: a declined or failed request has to read as a
- # fallback in the resolved record, never disappear into a bare "off" that looks
- # like the caller never asked.
+ # The REQUEST on the left, not None: a declined or failed request has to read as a fallback in the
+ # resolved record, never disappear into a bare "off" that looks like the caller never asked.
"text_encoder_quant": (
text_encoder_quant_requested,
text_encoder_quant_engaged or "off",
@@ -4937,22 +4841,19 @@ class VideoBackend:
vae_tiling = True,
memory_mode = normalize_memory_mode(memory_mode),
speed_mode = effective_speed,
- # Already filtered to the engaged optimisations (True names only).
speed_optims = speed_optims,
backend_flags = backend_flags,
attention_backend = attention_engaged,
- # The ENGAGED scheme, not the requested one, exactly as the conventional path
- # records it: status() reports this field, and a dense fallback must not claim a
- # quant the resident denoiser does not have.
+ # The ENGAGED scheme, not the requested one, exactly as the conventional path records it: status()
+ # reports this field, and a dense fallback must not claim a quant the resident denoiser does not have.
transformer_quant = transformer_quant_engaged,
- # Ditto for the conditioner: the memory preflight reads this field back to size the
- # resident encoder, so a dense fallback recorded as int8 would under-state the floor
- # by 39 GB and let a doomed generation start.
+ # Ditto for the conditioner: the memory preflight reads this field back to size the resident encoder, so
+ # a dense fallback recorded as int8 would under-state the floor by 39 GB and let a doomed generation
+ # start.
text_encoder_quant = text_encoder_quant_engaged,
h3_denoiser_pinned = denoiser_pinned,
resolved = resolved,
)
- # Ownership of the globals transferred to _state / _teardown_state_locked.
self._precommit_globals = None
logger.info(
"video.loaded: %s (%s, modular diffusers, speed=%s%s, transformer_quant=%s, "
@@ -4992,7 +4893,6 @@ class VideoBackend:
repo_id,
gguf_filename or "",
hf_token,
- # Matches the planner's both-roots cache probe, as the diffusion fetches already do.
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
@@ -5107,7 +5007,6 @@ class VideoBackend:
cancel = threading.Event()
job_token = object() # this reservation's identity; only its own worker may finalise it
while True:
- # Resolve outside the lock, then retry if the resident state changed.
with self._lock:
state = self._state
if expected_state is not None and state is not expected_state:
@@ -5171,7 +5070,6 @@ class VideoBackend:
)
self._generate_job_active = True
self._generate_job_token = job_token
- # Register before the worker starts so cancellation covers the spawn window.
self._active_generate_cancel = cancel
self._gen_video_id = video_id
self._gen = {
@@ -5183,8 +5081,8 @@ class VideoBackend:
}
break
worker = threading.Thread(
- # The token and the /v1/videos job id ride on the target rather than in kwargs:
- # those kwargs are also a valid generate() call, and callers replay them as one.
+ # The token and the /v1/videos job id ride on the target rather than in kwargs: those kwargs are also a
+ # valid generate() call, and callers replay them as one.
target = functools.partial(self._run_generate, job_token = job_token, video_id = video_id),
kwargs = dict(
prompt = prompt,
@@ -5205,25 +5103,22 @@ class VideoBackend:
try:
worker.start()
except RuntimeError:
- # No thread, so nothing would ever release the slot reserved above: generate stays
- # refused for the session, and liveness reports a rendering backend to a watchdog
- # that answers busy by waiting longer rather than restarting.
- # RuntimeError, not BaseException: start() raises it only before the thread exists,
- # but it then waits on the child, and a signal in that wait unwinds with the worker
- # live. Rolling back there would retire a running render's token and drop its cancel
- # handle. A worker that exists finalises its own reservation.
+ # No thread, so nothing would ever release the slot reserved above: generate stays refused for the session,
+ # and liveness reports a rendering backend to a watchdog that answers busy by waiting longer rather than
+ # restarting. RuntimeError, not BaseException: start() raises it only before the thread exists, but it then
+ # waits on the child, and a signal in that wait unwinds with the worker live. Rolling back there would
+ # retire a running render's token and drop its cancel handle. A worker that exists finalises its own
+ # reservation.
self._finish_generate_job(
job_token = job_token,
cancel_event = cancel,
error = "Video generation could not start.",
)
raise
- # What this run actually reserved, read off the same state the lock committed.
- # A caller that describes the job from an earlier status() read can be wrong on
- # every one of these: a load committing in between swaps the family, so the
- # frame count it computed belongs to the old fps and the model it reports is
- # already gone. The canvas is here for the same reason -- with a keyframe it
- # follows the source aspect, not the family's first preset.
+ # What this run reserved, read off the same state the lock committed. A caller that describes the job from an
+ # earlier status() read can be wrong on every one of these: a load committing in between swaps the family, so
+ # the frame count it computed belongs to the old fps and the model it reports is already gone. The canvas is
+ # here for the same reason -- with a keyframe it follows the source aspect, not the family's first preset.
state = resolved_inputs.state
fam = getattr(state, "family", None)
return {
@@ -5260,9 +5155,6 @@ class VideoBackend:
)
finally:
if job_token is not None:
- # Only begin_generate makes a reservation, so only it can leave one dangling.
- # Firing for a direct call would match the same unreserved token the body just
- # finalised with, replacing its outcome with the generic failure below.
self._finish_generate_job(
job_token = job_token,
cancel_event = cancel_event,
@@ -5328,10 +5220,9 @@ class VideoBackend:
result["mp4_bytes"],
{
"prompt": gen_kwargs["prompt"],
- # From the RESULT, not the request, exactly like guidance below: a
- # guidance-distilled family consumes no negative prompt on either engine, and
- # persisting the caller's string made the restored recipe claim conditioning
- # that never reached a sampler.
+ # From the RESULT, not the request, exactly like guidance below: a guidance-distilled family
+ # consumes no negative prompt on either engine, and persisting the caller's string made the restored
+ # recipe claim conditioning that never reached a sampler.
"negative_prompt": result.get("negative_prompt"),
"width": result["width"],
"height": result["height"],
@@ -5347,9 +5238,9 @@ class VideoBackend:
"flow_shift": result.get("flow_shift"),
"audio_flow_shift": result.get("audio_flow_shift"),
"model": result["repo_id"],
- # The load-time build, so a clip's recipe still names the precision it ran at
- # once the model is unloaded. All engaged values; absent keys read back as null
- # on sidecars written before this existed (they are not in _REQUIRED_META).
+ # The load-time build, so a clip's recipe still names the precision it ran at once the model is
+ # unloaded. All engaged values; absent keys read back as null on sidecars written before this
+ # existed (they are not in _REQUIRED_META).
"model_kind": result.get("model_kind"),
"gguf_filename": result.get("gguf_filename"),
"transformer_quant": result.get("transformer_quant"),
@@ -5403,7 +5294,8 @@ class VideoBackend:
self._generate_job_token = None
self._generate_job_active = False
if cancel_event is not None and self._active_generate_cancel is cancel_event:
- # Covers a worker that failed before reaching generate()'s finally; identity-guarded so a direct generate() keeps its handle.
+ # Covers a worker that failed before reaching generate()'s finally; identity-guarded so a direct
+ # generate() keeps its handle.
self._active_generate_cancel = None
if error is not None:
self._gen = {
@@ -5452,7 +5344,8 @@ class VideoBackend:
cancel = cancel_event if cancel_event is not None else threading.Event()
with self._generate_lock:
with self._lock:
- # A teardown is waiting for this lock and Python locks are not FIFO, so refuse rather than denoise against a pipeline that is already being torn down.
+ # A teardown is waiting for this lock and Python locks are not FIFO, so refuse rather than denoise
+ # against a pipeline that is already being torn down.
if self._teardown_waiters:
raise RuntimeError(VIDEO_CANCELLED_MSG)
state = self._state
@@ -5461,14 +5354,13 @@ class VideoBackend:
if _resolved_inputs is not None and _resolved_inputs.state is not state:
raise RuntimeError(VIDEO_CANCELLED_MSG)
self._active_generate_cancel = cancel
- # Bound below, once the request is resolved. None means the failure beat the
- # resolution, and there is nothing truthful to report.
+ # Bound below, once the request is resolved. None means the failure beat the resolution, and there is
+ # nothing truthful to report.
request_shape: Optional[dict[str, Any]] = None
try:
- # FIRST, before any device object exists. begin_generate runs this on a fresh
- # daemon thread, so until it is pinned the un-indexed state.device below -- the H3
- # memory probe and every torch.Generator -- resolves to its own default card while
- # the pipeline sits on the selected one.
+ # FIRST, before any device object exists. begin_generate runs this on a fresh daemon thread, so until it
+ # is pinned the un-indexed state.device below -- the H3 memory probe and every torch.Generator --
+ # resolves to its own default card while the pipeline sits on the selected one.
self._state_device_target(state)
fam = state.family
if _resolved_inputs is None:
@@ -5512,25 +5404,20 @@ class VideoBackend:
)
steps = int(steps or default_steps)
guidance = float(default_guidance if guidance is None else guidance)
- # A guidance-free family (supports_cfg=False, MiniMax-H3) forwards no CFG control
- # to either engine: the diffusers branch below skips the kwarg entirely and the
- # native branch pins --cfg-scale 1.0. The requested number therefore never reached
- # a sampler, so recording it would label the clip and its gallery sidecar with a
- # parameter that did nothing. Normalise to the family default instead of refusing:
- # the value is inert, and a 422 would break every caller that sends the generic
- # default. negative_prompt goes the same way and for the same reason: a negative
- # prompt IS the unconditional branch, so a family without one consumes it on
- # neither engine (the diffusers call adds the kwarg only when the pipeline
- # signature has it, which a guidance-distilled workflow does not, and
- # SdCppVideoGenParams has no field for it at all). Left as sent, it reached the
- # gallery sidecar and the restored recipe claimed conditioning that never touched
- # a sampler.
- # fam.default_guidance, NOT the identifier-derived one:
- # default_video_generation_params matches on the repo id or path, so a local
- # H3 file whose path happens to contain another family's keyword (say
- # /models/wan/minimax_h3_fl2va-Q4.gguf) picks up that family's 5.0. Recording
- # it would write back the inaccurate recipe this normalisation exists to
- # prevent, and with a number no H3 sampler can ever have seen.
+ # A guidance-free family (supports_cfg=False, MiniMax-H3) forwards no CFG control to either engine: the
+ # diffusers branch below skips the kwarg entirely and the native branch pins --cfg-scale 1.0. The
+ # requested number therefore never reached a sampler, so recording it would label the clip and its
+ # gallery sidecar with a parameter that did nothing. Normalise to the family default instead of
+ # refusing: the value is inert, and a 422 would break every caller that sends the generic default.
+ # negative_prompt goes the same way and for the same reason: a negative prompt IS the unconditional
+ # branch, so a family without one consumes it on neither engine (the diffusers call adds the kwarg only
+ # when the pipeline signature has it, which a guidance-distilled workflow does not, and
+ # SdCppVideoGenParams has no field for it at all). Left as sent, it reached the gallery sidecar and the
+ # restored recipe claimed conditioning that never touched a sampler. fam.default_guidance, NOT the
+ # identifier-derived one: default_video_generation_params matches on the repo id or path, so a local H3
+ # file whose path happens to contain another family's keyword (say /models/wan/minimax_h3_fl2va-Q4.gguf)
+ # picks up that family's 5.0. Recording it would write back the inaccurate recipe this normalisation
+ # exists to prevent, and with a number no H3 sampler can ever have seen.
if not fam.supports_cfg:
guidance = float(fam.default_guidance)
negative_prompt = None
@@ -5565,8 +5452,8 @@ class VideoBackend:
device_obj = torch.device(state.device)
device_module = getattr(torch, device_obj.type, None)
if device_module is not None and hasattr(device_module, "mem_get_info"):
- # Windows ROCm's free half is an over-report and this is a
- # hard refusal, so cap it against the allocator (#8403).
+ # Windows ROCm's free half is an over-report and this is a hard refusal, so cap it against the
+ # allocator (#8403).
from utils.hardware import trusted_mem_get_info
free_bytes, _ = trusted_mem_get_info(device_obj, module = device_module)
@@ -5576,10 +5463,9 @@ class VideoBackend:
else 0
)
available_vram_gb = (free_bytes + reserved_bytes) / 1_000_000_000
- # Size the floor from the components this load ACTUALLY holds, not from
- # the released bfloat16 pair. Both fields are the ENGAGED schemes (a
- # declined or failed request is recorded as None at load), so a dense
- # fallback keeps the dense floor.
+ # Size the floor from the components this load ACTUALLY holds, not from the released bfloat16
+ # pair. Both fields are the ENGAGED schemes (a declined or failed request is recorded as None at
+ # load), so a dense fallback keeps the dense floor.
required_vram_gb = estimate_h3_diffusers_vram_gb(
width,
height,
@@ -5604,9 +5490,9 @@ class VideoBackend:
host_capacity_gb = (
psutil.virtual_memory().available + process_rss
) / 1_000_000_000
- # Same engaged components as the VRAM floor above. Sizing one from what
- # the load holds and the other from the released pair refuses exactly the
- # configuration the quantized components exist for.
+ # Same engaged components as the VRAM floor above. Sizing one from what the load holds and the
+ # other from the released pair refuses exactly the configuration the quantized components exist
+ # for.
required_host_gb = estimate_h3_diffusers_host_ram_gb(
available_vram_gb,
text_encoder_gb = h3_te_resident_gb(
@@ -5621,8 +5507,8 @@ class VideoBackend:
"available. Load the GGUF artifact instead."
)
- # MPS seeds from the CPU generator too: diffusers reproduces a Metal seed only
- # that way, and the pipelines move the noise to the device themselves.
+ # MPS seeds from the CPU generator too: diffusers reproduces a Metal seed only that way, and the
+ # pipelines move the noise to the device themselves.
generator_device = (
"cpu" if fam.modular_workflow or str(state.device) == "mps" else state.device
)
@@ -5631,12 +5517,11 @@ class VideoBackend:
seed = int(generator.seed()) % (2**53)
generator = generator.manual_seed(int(seed))
- # The RESOLVED request, snapshotted the moment it is known, so a failure logs what
- # actually ran rather than the caller's Nones. Without it the whole server-side
- # record of a failed video is the exception string: #8225 reported an OOM whose
- # resolution and frame count had to be reverse-engineered out of the allocation
- # size by hand. No prompt text -- a length is enough to tell an empty prompt from
- # a long one, and the log is not the place for user content.
+ # The RESOLVED request, snapshotted the moment it is known, so a failure logs what ran rather than the
+ # caller's Nones. Without it the whole server-side record of a failed video is the exception string:
+ # #8225 reported an OOM whose resolution and frame count had to be reverse-engineered out of the
+ # allocation size by hand. No prompt text -- a length is enough to tell an empty prompt from a long one,
+ # and the log is not the place for user content.
request_shape = {
"family": fam.name,
"repo_id": state.repo_id,
@@ -5667,8 +5552,9 @@ class VideoBackend:
"num_frames": frames,
"generator": generator,
}
- # The 2.3 distilled DiT was trained on a fixed 8-step sigma curve (DISTILLED_SIGMA_VALUES); at the distilled default
- # step count pass it verbatim with the scheduler re-shaping neutralised. Any other count keeps the scheduler's spacing.
+ # The 2.3 distilled DiT was trained on a fixed 8-step sigma curve (DISTILLED_SIGMA_VALUES); at the
+ # distilled default step count pass it verbatim with the scheduler re-shaping neutralised. Any other
+ # count keeps the scheduler's spacing.
sigma_ctx: Any = contextlib.nullcontext()
if fam.name == "ltx-2" and "sigmas" in call_params:
from .video_ltx2 import (
@@ -5690,10 +5576,11 @@ class VideoBackend:
kwargs[fam.cfg_kwarg] = guidance
if negative_prompt and "negative_prompt" in call_params:
kwargs["negative_prompt"] = negative_prompt
- # LTX-2 takes frame_rate (shapes audio length); others fix their rate, fps only at export.
+ # LTX-2 takes frame_rate (shapes audio length); others fix their rate, fps only at export
if "frame_rate" in call_params:
kwargs["frame_rate"] = float(out_fps)
- # Dual-DiT MoE: pass the low-noise expert guidance only when the family declares it AND the signature accepts it (WanPipeline raises with boundary_ratio=None).
+ # Dual-DiT MoE: pass the low-noise expert guidance only when the family declares it AND the signature
+ # accepts it (WanPipeline raises with boundary_ratio=None)
if fam.cfg2_kwarg and fam.cfg2_kwarg in call_params and guidance_2 is not None:
kwargs[fam.cfg2_kwarg] = float(guidance_2)
if fam.modular_workflow:
@@ -5741,7 +5628,7 @@ class VideoBackend:
return callback_kwargs
def _on_scheduler_step(done: int) -> None:
- # No cooperative _interrupt here, so cancellation must unwind the denoise loop via an exception.
+ # No cooperative _interrupt here, so cancellation must unwind the denoise loop via an exception
if cancel.is_set():
raise _VideoGenerationCancelled()
_tick(done)
@@ -5750,13 +5637,14 @@ class VideoBackend:
kwargs["callback_on_step_end"] = _on_step
progress_ctx = contextlib.nullcontext()
else:
- # HunyuanVideo-1.5 has no step callback, so wrap scheduler.step for progress + cancel and restore afterwards.
- # Same for H3's modular workflow: ModularPipeline takes no callback (an unknown input is only warned
- # about), but MiniMaxH3LoopSchedulerStep calls components.scheduler.step -- pipe.scheduler -- once per
- # denoise step, so the wrapper ticks and can unwind a multi-minute run on Cancel.
+ # HunyuanVideo-1.5 has no step callback, so wrap scheduler.step for progress + cancel and restore
+ # afterwards. Same for H3's modular workflow: ModularPipeline takes no callback (an unknown input is
+ # only warned about), but MiniMaxH3LoopSchedulerStep calls components.scheduler.step --
+ # pipe.scheduler -- once per denoise step, so the wrapper ticks and can unwind a multi-minute run on
+ # Cancel.
progress_ctx = _scheduler_step_progress(pipe, _on_scheduler_step)
- # Re-check an AUTO cache decision against the ACTUAL step count; explicit choices never toggle.
+ # Re-check an AUTO cache decision against the ACTUAL step count; explicit choices never toggle
if state.cache_auto:
toggled = state.transformer_cache
for view in _views_for(pipe, fam):
@@ -5784,7 +5672,8 @@ class VideoBackend:
with torch.inference_mode(), progress_ctx, sigma_ctx:
output = pipe(**kwargs)
except _VideoGenerationCancelled:
- # Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules would stay on the GPU.
+ # Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules would
+ # stay on the GPU.
free_hooks = getattr(pipe, "maybe_free_model_hooks", None)
if callable(free_hooks):
try:
@@ -5836,15 +5725,15 @@ class VideoBackend:
"conditioning": conditioning,
"steps": steps,
"guidance": guidance,
- # The EFFECTIVE negative prompt, like guidance beside it: a guidance-distilled
- # family normalises it away above, so the sidecar records what conditioned the
- # clip instead of what the caller happened to send.
+ # The EFFECTIVE negative prompt, like guidance beside it: a guidance-distilled family normalises it
+ # away above, so the sidecar records what conditioned the clip instead of what the caller happened
+ # to send.
"negative_prompt": negative_prompt,
"flow_shift": shift,
"audio_flow_shift": audio_shift,
- # The BUILD this clip came off, read from the ENGAGED state and never from the
- # request, so a saved MP4 can never be labelled with a precision that did not
- # run. Mirrors what the image gallery already records.
+ # The BUILD this clip came off, read from the ENGAGED state and never from the request, so a saved
+ # MP4 can never be labelled with a precision that did not run. Mirrors what the image gallery
+ # already records.
"model_kind": state.kind,
"gguf_filename": state.gguf_filename,
"transformer_quant": state.transformer_quant,
@@ -5886,10 +5775,10 @@ class VideoBackend:
if (width is None or height is None) and (first is not None or last is not None):
width, height = h3_canvas_for_aspect(*(first if first is not None else last).size)
else:
- # begin_generate runs this for EVERY family, keyframes or not, so it cannot assume a
- # family declares presets: an unusual/custom one with an empty tuple used to die here
- # with an IndexError (a 500) before the request reached the worker. Fall back to the
- # generic 768x512 the rest of the video path already treats as the default canvas.
+ # begin_generate runs this for EVERY family, keyframes or not, so it cannot assume a family declares
+ # presets: an unusual/custom one with an empty tuple used to die here with an IndexError (a 500) before the
+ # request reached the worker. Fall back to the generic 768x512 the rest of the video path already treats as
+ # the default canvas.
default_w, default_h = (
fam.resolution_presets[0] if fam.resolution_presets else (768, 512)
)
@@ -5994,10 +5883,9 @@ class VideoBackend:
decode_audio = not bool(override),
)
if override:
- # A replacement soundtrack is a separate upload on its own timeline: it starts
- # at its own zero and runs the clip's length. The video's coordinates dropped
- # its first trim_start seconds and refused anything shorter. Only the embedded
- # track shares the video's timeline.
+ # A replacement soundtrack is a separate upload on its own timeline: it starts at its own zero and runs
+ # the clip's length. The video's coordinates dropped its first trim_start seconds and refused anything
+ # shorter. Only the embedded track shares the video's timeline.
clip = validate_h3_reference_trim(trim_start, trim_end)
waveform, sample_rate = decode_h3_reference_audio(
_decode_b64_media(override),
@@ -6011,7 +5899,6 @@ class VideoBackend:
if waveform is None:
soundtrack_gap = True
elif soundtrack_gap:
- # sd-cli pairs soundtrack flags by position, so gaps would misattach audio.
raise ValueError(
"stable-diffusion.cpp can pair reference-video soundtracks only from "
f"Video 1 onward without gaps, but Video {index} has audio after a "
@@ -6071,8 +5958,8 @@ class VideoBackend:
"eta_seconds": None,
"error": None,
}
- # sd.cpp reuses an unlabeled bar for loading, denoising, and VAE work. Count only
- # iteration-rated bars inside its denoise window.
+ # sd.cpp reuses an unlabeled bar for loading, denoising, and VAE work. Count only iteration-rated bars inside
+ # its denoise window.
denoise_open = re.compile(r"generate_video\s+\d+x\d+x\d+")
denoise_close = re.compile(r"sampling completed", re.IGNORECASE)
step_bar = re.compile(r"\|\s*(\d+)\s*/\s*(\d+)\s*-\s*[\d.]+\s*(?:it/s|s/it)")
@@ -6085,8 +5972,8 @@ class VideoBackend:
return
if denoise_close.search(line):
denoising = False
- # sd.cpp still has to decode both latent streams and write its
- # intermediate video. Do not leave the UI at a completed denoise bar.
+ # sd.cpp still has to decode both latent streams and write its intermediate video. Do not leave the UI
+ # at a completed denoise bar.
self._gen.update(phase = "decode", eta_seconds = None)
return
match = step_bar.search(line) if denoising else None
@@ -6116,37 +6003,34 @@ class VideoBackend:
init_img = stage(first_frame, "first")
end_img = stage(last_frame, "last")
staged = stage_h3_references(references, Path(scratch))
- # An H3 native run is an sd-cli run out of the managed tree, exactly like a one-shot
- # image generation, so it takes the same reader claim. Without it an image-engine
- # request sees no readers, starts an install, and the sweep unlinks the CLI this
- # runtime resolved at load: on Linux this process survives on the open inode but the
- # next video request launches a path that is gone, and on Windows the unlink fails and
- # leaves a mixed tree. A claim held here also makes the install stand down instead.
+ # An H3 native run is an sd-cli run out of the managed tree, exactly like a one-shot image generation, so it
+ # takes the same reader claim. Without it an image-engine request sees no readers, starts an install, and
+ # the sweep unlinks the CLI this runtime resolved at load: on Linux this process survives on the open inode
+ # but the next video request launches a path that is gone, and on Windows the unlink fails and leaves a
+ # mixed tree. A claim held here also makes the install stand down instead.
from .sd_cpp_backend import _tree_reader
binary = getattr(runtime.engine, "binary", None)
- # Identity, not existence, and the identity RECORDED AT LOAD. An install can replace
- # the binary in place, leaving a file that still exists but is not the build
- # ensure_h3_sd_cpp_binary vetted: a different accelerator, or one predating the H3
- # options, which aborts partway through a render nobody wants to repeat. Comparing two
- # reads taken now would miss that entirely whenever the install landed before this
+ # Identity, not existence, and the identity RECORDED AT LOAD. An install can replace the binary in place,
+ # leaving a file that still exists but is not the build ensure_h3_sd_cpp_binary vetted: a different
+ # accelerator, or one predating the H3 options, which aborts partway through a render nobody wants to
+ # repeat. Comparing two reads taken now would miss that entirely whenever the install landed before this
# generation started, since both would see the replacement and agree.
binary_identity = getattr(runtime, "binary_identity", None)
try:
try:
with _tree_reader(binary, cancel, VIDEO_CANCELLED_MSG):
current_identity = _sd_cli_identity(binary)
- # Unreadable now (swept), or readable but not what the load vetted. A
- # runtime carrying no recorded identity cannot answer the second question,
- # so it is held to the first rather than refused outright.
+ # Unreadable now (swept), or readable but not what the load vetted. A runtime carrying no
+ # recorded identity cannot answer the second question, so it is held to the first rather than
+ # refused outright.
if binary and (
current_identity is None
or (binary_identity is not None and current_identity != binary_identity)
):
- # The engine is committed at load, so this cannot be re-resolved
- # underneath a live generation the way the image path can. Say so
- # plainly instead of running an unvetted build or surfacing an ENOENT
- # from the subprocess.
+ # The engine is committed at load, so this cannot be re-resolved underneath a live
+ # generation the way the image path can. Say so plainly instead of running an unvetted build
+ # or surfacing an ENOENT from the subprocess.
raise RuntimeError(
"The stable-diffusion.cpp binary this model was loaded with was "
"replaced by an install, so it is no longer the build this model "
@@ -6199,16 +6083,15 @@ class VideoBackend:
"conditioning": conditioning,
"steps": steps,
"guidance": guidance,
- # sd-cli's vid_gen mode has no negative-prompt input at all
- # (SdCppVideoGenParams carries no field for one), and this path serves only
- # MiniMax-H3, which is guidance-distilled. Recorded as the None it ran with.
+ # sd-cli's vid_gen mode has no negative-prompt input at all (SdCppVideoGenParams carries no field
+ # for one), and this path serves only MiniMax-H3, which is guidance-distilled. Recorded as the None
+ # it ran with.
"negative_prompt": None,
"flow_shift": flow_shift,
# sd.cpp pins the audio schedule, so the recipe records what it actually ran.
"audio_flow_shift": state.family.default_audio_flow_shift,
- # The BUILD this clip came off, from the ENGAGED state and never the request,
- # as the diffusers path records it. Without these six every native GGUF clip
- # saves a blank recipe.
+ # The BUILD this clip came off, from the ENGAGED state and never the request, as the diffusers path
+ # records it. Without these six every native GGUF clip saves a blank recipe.
"model_kind": state.kind,
"gguf_filename": state.gguf_filename,
"transformer_quant": state.transformer_quant,
@@ -6257,13 +6140,15 @@ class VideoBackend:
def generate_progress(self) -> dict[str, Any]:
with self._lock:
gen = dict(self._gen)
- # generate() swaps in a bare {"active": False} before the worker records the terminal dict; report active across that gap.
+ # generate() swaps in a bare {"active": False} before the worker records the terminal dict; report active
+ # across that gap.
if self._generate_job_active:
gen["active"] = True
if self._gen_video_id is not None:
gen["video_id"] = self._gen_video_id
gen.setdefault("active", False)
- # Mirror the image endpoint field names (total_steps / fraction) alongside the native "total": the two generate-progress APIs used to disagree.
+ # Mirror the image endpoint field names (total_steps / fraction) alongside the native "total": the two
+ # generate-progress APIs used to disagree.
total = int(gen.get("total") or 0)
step = int(gen.get("step") or 0)
gen["total_steps"] = total
@@ -6311,7 +6196,8 @@ class VideoBackend:
state, self._state = self._state, None
if state is not None:
restore_backend_flags(state.backend_flags)
- # A GGUF load may have installed the compiled GGUF dequantizer; restore the stock kernels so a later speed=off load is bit-identical.
+ # A GGUF load may have installed the compiled GGUF dequantizer; restore the stock kernels so a later
+ # speed=off load is bit-identical.
from . import diffusion_gguf_compile
diffusion_gguf_compile.uninstall_all()
@@ -6325,16 +6211,18 @@ class VideoBackend:
self._loading = None
if self._active_generate_cancel is not None:
self._active_generate_cancel.set()
- # Fence generations queued behind the active one too: they hold no cancel event, so the signal cannot reach them.
+ # Fence generations queued behind the active one too: they hold no cancel event, so the signal cannot reach
+ # them.
self._teardown_waiters += 1
- # Barrier: wait for the signalled generation to exit before freeing the pipeline, else we report the VRAM free while
- # the clip still holds it. Teardown runs INSIDE the barrier: releasing first would hand out one last clip.
+ # Barrier: wait for the signalled generation to exit before freeing the pipeline, else we report the VRAM free
+ # while the clip still holds it.
with self._generate_lock:
with self._lock:
try:
self._teardown_state_locked()
finally:
- # Released in a finally: _teardown_state_locked ends in clear_gpu_cache(), which raises on a sticky CUDA fault, and an un-drained fence refuses every later generation.
+ # Released in a finally: _teardown_state_locked ends in clear_gpu_cache(), which raises on a sticky
+ # CUDA fault, and an un-drained fence refuses every later generation.
self._teardown_waiters -= 1
logger.info("video.unloaded")
return self.status()
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index 8d8cc691cc8..942f72df4ac 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -21,9 +21,10 @@ import re
from dataclasses import dataclass, field
from typing import Optional
-# The request model's ceiling on num_frames, declared HERE so the shape gate and the bound cannot
-# drift: the gate's refusal names the lattice point above the request, and suggesting one the
-# request model would itself reject is a dead end. VideoGenerateRequest imports this for its `le`.
+# declared HERE so the shape gate and the bound cannot drift
+# The request model's ceiling on num_frames, declared HERE so the shape gate and the bound cannot drift: the gate's
+# refusal names the lattice point above the request, and suggesting one the request model would itself reject is a dead
+# end. VideoGenerateRequest imports this for its `le`.
MAX_VIDEO_NUM_FRAMES = 1024
# Runtime->route contract: routes match these EXACTLY for a 409 instead of a 500.
@@ -45,13 +46,16 @@ class VideoFamily:
denoiser_attr: str = "transformer"
# Extra lowercased substrings (besides ``name``) that map a repo id here.
aliases: tuple[str, ...] = field(default_factory = tuple)
- # True when the pipeline returns synchronized audio (LTX-2): export muxes the track and size estimates count the audio VAE + vocoder.
+ # True when the pipeline returns synchronized audio (LTX-2): export muxes the track and size estimates count the
+ # audio VAE + vocoder.
has_audio: bool = False
- # Wan2.2-A14B dual-expert MoE: a second DiT (transformer_2) handles the low-noise steps with its own guidance kwarg.
+ # Wan2.2-A14B dual-expert MoE: a second DiT handles the low-noise steps with its own guidance kwarg
transformer2_class: Optional[str] = None
is_moe: bool = False
cfg2_kwarg: Optional[str] = None
- # HunyuanVideo-1.5 guidance: __call__ takes NO guidance kwarg; CFG lives on a ``guider`` whose scale is set per request.
+ # HunyuanVideo-1.5: __call__ takes NO guidance kwarg
+ # HunyuanVideo-1.5 guidance: __call__ takes NO guidance kwarg; CFG lives on a ``guider`` whose scale is set per
+ # request.
guidance_via_guider: bool = False
# Generation defaults + shape. A valid frame count is k*frame_step + frame_offset.
default_steps: int = 40
@@ -70,53 +74,52 @@ class VideoFamily:
resolution_presets: tuple[tuple[int, int], ...] = ((768, 512),)
# Clip lengths offered by the UI. Most families use short previews; H3 is trained for 5-15 seconds.
duration_presets: tuple[float, ...] = (1.0, 2.0, 3.0, 5.0)
- # Component bf16-RESIDENT sizes in decimal GB (denoiser(s), text encoder, VAE + audio): what sits on device after the cast.
+ # Component bf16-RESIDENT sizes in decimal GB (denoiser(s), text encoder, VAE + audio): what sits on device after
+ # the cast.
bf16_components_gb: Optional[tuple[float, float, float]] = None
- # True when the DiT compiles cleanly with regional torch.compile (declares _repeated_blocks).
+ # true when the DiT compiles cleanly with regional torch.compile (declares _repeated_blocks)
supports_torch_compile: bool = True
# Video DiTs are bf16-native, so fp16 promotes to float32; defaults True.
fp16_incompatible: bool = True
- # Wan VAE decodes in float32 (bf16 causes banding / black frames), so the loader pins it back. Its size term is already fp32.
+ # Wan VAE decodes in float32 (bf16 causes banding / black frames), so the loader pins it back. Its size term is
+ # already fp32.
vae_force_fp32: bool = False
# Curated GGUF repo for the picker (the DiT as single-file GGUF quants).
gguf_repo: Optional[str] = None
- # Hosted PRE-CAST text-encoder checkpoints as (scheme, component, repo_id); same semantics as DiffusionFamily.te_prequant_repos.
+ # Hosted PRE-CAST text-encoder checkpoints as (scheme, component, repo_id); same semantics as
+ # DiffusionFamily.te_prequant_repos.
te_prequant_repos: tuple[tuple[str, str, str], ...] = field(default_factory = tuple)
- # Hosted PRE-QUANTIZED DENOISER checkpoints as (scheme, repo_id). The DiT, NOT the text
- # encoder that te_prequant_repos above covers: the two are separate artifacts because a load
- # can take one without the other. Same semantics as DiffusionFamily.prequant_repos, so the
- # shared diffusion_prequant resolver reads this table through plain attribute access.
+ # Hosted PRE-QUANTIZED DENOISER checkpoints as (scheme, repo_id). The DiT, NOT the text encoder that
+ # te_prequant_repos above covers: the two are separate artifacts because a load can take one without the other. Same
+ # semantics as DiffusionFamily.prequant_repos, so the shared diffusion_prequant resolver reads this table through
+ # plain attribute access.
prequant_repos: tuple[tuple[str, str], ...] = field(default_factory = tuple)
- # RESIDENT size of one of those hosted denoisers, in decimal GB, when the generic
- # _QUANT_STEADY_FACTOR does not describe it. MiniMax-H3's is both quantized AND structurally
- # pruned (the curve-form adaLN, ~40% of the released parameters), so 0.55 x 66.3 GB over-states
- # it by 16 GB and a hard refusal turns away a load that fits. Measured from Hub file metadata
- # (2026-08-09): MiniMax-H3-FP8.pt 20,260,192,855 bytes, MiniMax-H3-INT8.pt 20,253,894,865.
+ # RESIDENT size of one of those hosted denoisers, in decimal GB, when the generic _QUANT_STEADY_FACTOR does not
+ # describe it. MiniMax-H3's is both quantized AND structurally pruned (the curve-form adaLN, ~40% of the released
+ # parameters), so 0.55 x 66.3 GB over-states it by 16 GB and a hard refusal turns away a load that fits. Measured
+ # from Hub file metadata (2026-08-09): MiniMax-H3-FP8.pt 20,260,192,855 bytes, MiniMax-H3-INT8.pt 20,253,894,865.
prequant_resident_gb: Optional[float] = None
- # Per-variant overrides as (base_repo, scheme, repo_id), keyed on the LOWERCASED upstream base
- # id. A pre-quantized checkpoint is baked from ONE base's weights and the loader refuses it for
- # any other base, so a variant that ships its own denoiser needs its own entry; a variant
- # without one falls through to prequant_repos and, if that checkpoint was baked elsewhere, the
- # loader's base_model_id check sends the load back to the dense path.
+ # Per-variant overrides as (base_repo, scheme, repo_id), keyed on the LOWERCASED upstream base id. A pre-quantized
+ # checkpoint is baked from ONE base's weights and the loader refuses it for any other base, so a variant that ships
+ # its own denoiser needs its own entry; a variant without one falls through to prequant_repos and, if that
+ # checkpoint was baked elsewhere, the loader's base_model_id check sends the load back to the dense path.
prequant_variant_repos: tuple[tuple[str, str, str], ...] = field(default_factory = tuple)
- # Preferred checkpoint FILENAME for a scheme, as (scheme, filename), overriding the
- # ``<Model>-<SCHEME>.pt`` name ``prequant_repo_filename`` derives. The derived name stays on as
- # the fallback, so a repo hosting BOTH an old and a new artifact serves the new one to a build
- # that asks for it by name and the old one to every build that does not. That is what lets a
- # rotated (v2) checkpoint ship without regressing an already-installed Unsloth, which would
- # otherwise refuse the v2 tag and fall all the way back to the dense download.
- # A row may also be (scheme, task, filename), naming the artifact for ONE task; it beats the
- # task-agnostic row and, unlike it, gets no filename fallback (see resolve_prequant_source).
+ # Preferred checkpoint FILENAME for a scheme, as (scheme, filename), overriding the ``<Model>-<SCHEME>.pt`` name
+ # ``prequant_repo_filename`` derives. The derived name stays on as the fallback, so a repo hosting BOTH an old and a
+ # new artifact serves the new one to a build that asks for it by name and the old one to every build that does not.
+ # That is what lets a rotated (v2) checkpoint ship without regressing an already-installed Unsloth, which would
+ # otherwise refuse the v2 tag and fall all the way back to the dense download. A row may also be (scheme, task,
+ # filename), naming the artifact for ONE task; it beats the task-agnostic row and, unlike it, gets no filename
+ # fallback (see resolve_prequant_source).
prequant_filenames: tuple[tuple[str, ...], ...] = field(default_factory = tuple)
- # Tasks whose denoiser is a DIFFERENT checkpoint partition from the one the task-agnostic
- # ``prequant_filenames`` / ``prequant_repos`` rows describe. Such a task is served ONLY by its
- # own (scheme, task, filename) row: the partitions share a base, a class, a config and a key
- # set, so nothing downstream can tell them apart and an unnamed artifact would load cleanly
- # and generate from the wrong partition. Empty for every family with a single denoiser, which
- # is what makes this field free to ignore.
+ # Tasks whose denoiser is a DIFFERENT checkpoint partition from the one the task-agnostic ``prequant_filenames`` /
+ # ``prequant_repos`` rows describe. Such a task is served ONLY by its own (scheme, task, filename) row: the
+ # partitions share a base, a class, a config and a key set, so nothing downstream can tell them apart and an unnamed
+ # artifact would load cleanly and generate from the wrong partition. Empty for every family with a single denoiser,
+ # which is what makes this field free to ignore.
prequant_partition_tasks: tuple[str, ...] = field(default_factory = tuple)
- # Modular Diffusers workflow to load instead of a conventional DiffusionPipeline. Its
- # components are loaded without pruning the workflow's routing blocks.
+ # Modular Diffusers workflow to load instead of a conventional DiffusionPipeline. Its components are loaded without
+ # pruning the workflow's routing blocks.
modular_workflow: Optional[str] = None
# Released video and audio sigma shifts, when configurable.
default_flow_shift: Optional[float] = None
@@ -130,7 +133,6 @@ class VideoFamily:
_FAMILIES: tuple[VideoFamily, ...] = (
- # FL2VA covers text-only and keyframe generation. Ref2VA is a separate partition.
VideoFamily(
name = "minimax-h3",
pipeline_class = "ModularPipeline",
@@ -148,60 +150,59 @@ _FAMILIES: tuple[VideoFamily, ...] = (
max_num_frames = 345,
snap_frames_up = True,
resolution_multiple = 32,
- # Model-card ratios use H3's canvas rule. Keep the legacy 1024 square and cheaper
- # 16:9 tiers for compatibility.
+ # model-card ratios use H3's canvas rule; the legacy 1024 square and cheaper 16:9 tiers are kept for
+ # compatibility
+ # Model-card ratios use H3's canvas rule. Keep the legacy 1024 square and cheaper 16:9 tiers for compatibility.
resolution_presets = (
- (1344, 768), # 16:9
- (1536, 672), # 21:9
- (1024, 768), # 4:3
- (1024, 1024), # 1:1
- (768, 1024), # 3:4
- (768, 1344), # 9:16
- (960, 544), # 16:9, faster
- (544, 960), # 9:16, faster
+ (1344, 768),
+ (1536, 672),
+ (1024, 768),
+ (1024, 1024),
+ (768, 1024),
+ (768, 1344),
+ (960, 544), # faster
+ (544, 960), # faster
),
duration_presets = (5.0, 10.0, 14.4),
# Decimal GB resident estimates: transformer, Qwen3-VL conditioner, video+audio VAEs.
bf16_components_gb = (66.3, 66.8, 11.1),
+ # regionally compilable: every block sees (1, S, 5376) plus an (S,) index tensor
# Regionally compilable. The DiT declares _repeated_blocks (MiniMaxH3TransformerBlock +
- # MiniMaxH3TokenRefinerBlock); every block sees (1, S, 5376) plus an (S,) index tensor,
- # where S is the PACKED length (18,870 video + 207 audio rows + the caption's text rows at
- # 960x544x124). The caption moves S by ~2% (19,096 at 19 tokens vs 19,479 at 402) and S
- # cannot change mid-denoise, so dynamic=True traces once and holds: measured 1.298-1.342
- # s/step eager vs 1.000-1.040 compiled (1.30x), first forward 10.2 s, zero recompiles
- # across captions of 19/19/37/128/402 tokens. The loader engages this only when the
- # denoiser is RESIDENT; compiling inside a full CPU-offload rotation measured slower than
- # eager, so that case stays on the no-compile tier.
+ # MiniMaxH3TokenRefinerBlock); every block sees (1, S, 5376) plus an (S,) index tensor, where S is the PACKED
+ # length (18,870 video + 207 audio rows + the caption's text rows at 960x544x124). The caption moves S by ~2%
+ # (19,096 at 19 tokens vs 19,479 at 402) and S cannot change mid-denoise, so dynamic=True traces once and holds:
+ # measured 1.298-1.342 s/step eager vs 1.000-1.040 compiled (1.30x), first forward 10.2 s, zero recompiles
+ # across captions of 19/19/37/128/402 tokens. The loader engages this only when the denoiser is RESIDENT;
+ # compiling inside a full CPU-offload rotation measured slower than eager, so that case stays on the no-compile
+ # tier.
supports_torch_compile = True,
gguf_repo = "unsloth/MiniMax-H3-GGUF",
- # Hosted pre-quantized FL2VA denoisers. The modular workflow builds each component through
- # its own from_pretrained, so there is no dense module to quantise in place: these are the
- # ONLY way to run the 66.3 GB transformer quantized, and seeding one also stops that
- # download. Both schemes live in ONE repo, at the root, named <Model>-<SCHEME>.pt, which is
- # the layout every image-side prequant repo already uses and the one prequant_repo_filename
- # builds without help.
+ # the modular workflow builds each component through its own from_pretrained
+ # Hosted pre-quantized FL2VA denoisers. The modular workflow builds each component through its own
+ # from_pretrained, so there is no dense module to quantise in place: these are the ONLY way to run the 66.3 GB
+ # transformer quantized, and seeding one also stops that download. Both schemes live in ONE repo, at the root,
+ # named <Model>-<SCHEME>.pt, which is the layout every image-side prequant repo already uses and the one
+ # prequant_repo_filename builds without help.
prequant_repos = (("int8", "unsloth/MiniMax-H3-FP8"), ("fp8", "unsloth/MiniMax-H3-FP8")),
- # The INT8 denoiser is ConvRot-rotated (see diffusion_convrot): its weights live in a
- # Hadamard-rotated basis and are wrong unless the loader rotates the activations to match,
- # so it carries the v2 format tag an Unsloth predating that code refuses. Shipping it under
- # its own name rather than over MiniMax-H3-INT8.pt keeps both true at once: this build
- # gets the rotated artifact, and an older install still resolves the plain one instead of
- # refusing the v2 tag and falling back to the 66.3 GB dense download.
- # The reference (ref2va) denoiser is a SECOND partition in the same base repo, so it gets
- # its own artifact under its own name for both schemes. The two partitions are otherwise
- # indistinguishable to the loader -- same class, same config, same 635 keys, same
- # base_model_id -- so the task, not any later check, is the only thing that keeps a
- # reference load off the keyframe weights. Keyframe (fl2va, which also covers text-only)
- # keeps resolving exactly what it resolved before: the rotated INT8 by name, FP8 by the
- # derived MiniMax-H3-FP8.pt.
+ # The INT8 denoiser is ConvRot-rotated (see diffusion_convrot): its weights live in a Hadamard-rotated basis and
+ # are wrong unless the loader rotates the activations to match, so it carries the v2 format tag an Unsloth
+ # predating that code refuses. Shipping it under its own name rather than over MiniMax-H3-INT8.pt keeps both
+ # true at once: this build gets the rotated artifact, and an older install still resolves the plain one instead
+ # of refusing the v2 tag and falling back to the 66.3 GB dense download. The reference (ref2va) denoiser is a
+ # SECOND partition in the same base repo, so it gets its own artifact under its own name for both schemes. The
+ # two partitions are otherwise indistinguishable to the loader -- same class, same config, same 635 keys, same
+ # base_model_id -- so the task, not any later check, is the only thing that keeps a reference load off the
+ # keyframe weights. Keyframe (fl2va, which also covers text-only) keeps resolving exactly what it resolved
+ # before: the rotated INT8 by name, FP8 by the derived MiniMax-H3-FP8.pt.
prequant_filenames = (
("int8", "MiniMax-H3-INT8-ConvRot.pt"),
("int8", "ref2va", "MiniMax-H3-Ref2VA-INT8-ConvRot.pt"),
("fp8", "ref2va", "MiniMax-H3-Ref2VA-FP8.pt"),
),
- # Keeps the two partitions honest: without a ref2va row above, a ref2va prequant pick is
- # refused rather than served the keyframe checkpoint. Equal to H3_TASK_REFERENCES; the
- # literal avoids importing the H3 helper module into the registry.
+ # without a ref2va row above, a ref2va prequant pick is refused rather than served the keyframe checkpoint;
+ # Keeps the two partitions honest: without a ref2va row above, a ref2va prequant pick is refused rather than
+ # served the keyframe checkpoint. Equal to H3_TASK_REFERENCES; the literal avoids importing the H3 helper module
+ # into the registry.
prequant_partition_tasks = ("ref2va",),
# Both schemes are ~20.3 GB resident against the 66.3 GB dense denoiser; see the field.
prequant_resident_gb = 20.3,
@@ -212,8 +213,9 @@ _FAMILIES: tuple[VideoFamily, ...] = (
supports_references = True,
supports_cfg = False,
),
- # LTX-2 (diffusers >= 0.39): ~19B single-stream video DiT generating synchronized audio + video in one pass. The Gemma3-12B
- # encoder is stored fp32 on the hub (~49 GB download, ~24 GB resident bf16). The base repo carries the dev config (40 steps, CFG 4).
+ # LTX-2 (diffusers >= 0.39): ~19B single-stream video DiT generating synchronized audio + video in one pass. The
+ # Gemma3-12B encoder is stored fp32 on the hub (~49 GB download, ~24 GB resident bf16). The base repo carries the
+ # dev config (40 steps, CFG 4).
VideoFamily(
name = "ltx-2",
pipeline_class = "LTX2Pipeline",
@@ -229,41 +231,43 @@ _FAMILIES: tuple[VideoFamily, ...] = (
resolution_multiple = 32,
# 768x512 native default; 1216x704 the card's quality target; 704x1216 vertical.
resolution_presets = ((768, 512), (1216, 704), (704, 1216), (512, 768)),
- # transformer 37.8 bf16; Gemma3-12B TE ~24.4 RESIDENT (the hub stores it fp32 but the pipeline loads bf16); VAE 2.4 + connectors 2.9 + audio 0.2. The old 50.4 figure double-counted the fp32 store.
+ # transformer 37.8 bf16; Gemma3-12B TE ~24.4 RESIDENT (the hub stores it fp32 but the pipeline loads bf16); VAE
+ # 2.4 + connectors 2.9 + audio 0.2. The old 50.4 figure double-counted the fp32 store.
bf16_components_gb = (37.8, 24.4, 5.5),
gguf_repo = "unsloth/LTX-2.3-GGUF",
- # Pre-cast Gemma3-12B TE (fp32 ~49 GB on the hub, pre-cast ~13.2 GB): the biggest download win.
+ # pre-cast Gemma3-12B TE (fp32 ~49 GB on the hub, pre-cast ~13.2 GB): the biggest download win
te_prequant_repos = (("fp8", "text_encoder", "unsloth/LTX-2-FP8"),),
),
- # Wan2.2-TI2V-5B (diffusers >= 0.35, verified on 0.39): ~5B single-stream DiT (UMT5 encoder), no audio. Its VAE's temporal compression 4 gives valid frame counts 4k+1. Defaults 50 steps / CFG 5.
+ # Wan2.2-TI2V-5B: ~5B single-stream DiT (UMT5 encoder), no audio
+ # Wan2.2-TI2V-5B (diffusers >= 0.35, verified on 0.39): ~5B single-stream DiT (UMT5 encoder), no audio. Its VAE's
+ # temporal compression 4 gives valid frame counts 4k+1. Defaults 50 steps / CFG 5.
VideoFamily(
name = "wan2.2-ti2v-5b",
pipeline_class = "WanPipeline",
transformer_class = "WanTransformer3DModel",
base_repo = "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
- # "wan2.2-5b"/"wan-ti2v" are the picker/GGUF short ids; "wan2.2-ti2v" catches the repo stem.
+ # "wan2.2-5b"/"wan-ti2v" are the picker/GGUF short ids; "wan2.2-ti2v" catches the repo stem
aliases = ("wan2.2-5b", "wan-ti2v", "wan2.2-ti2v", "wan-ti2v-5b"),
has_audio = False,
default_steps = 50,
default_guidance = 5.0,
- # 121 frames at 24 fps ~5s; on the 4k+1 lattice (121 = 4*30 + 1) it needs no snapping.
default_num_frames = 121,
default_fps = 24,
- # Wan VAE temporal factor 4, so valid counts are 4k+1.
frame_step = 4,
- # TI2V-5B's VAE is 16x spatial + patch 2, so WanPipeline floors H/W to 32; snap to 32 so the recorded size matches the clip.
resolution_multiple = 32,
- # TI2V-5B is a 720P-only checkpoint: upstream SUPPORTED_SIZES is exactly
- # ('704*1280', '1280*704') and generate.py asserts membership, so nothing else is offered.
- # First is the default the loader plans against.
+ # TI2V-5B is a 720P-only checkpoint: upstream SUPPORTED_SIZES is exactly ('704*1280', '1280*704') and
+ # generate.py asserts membership, so nothing else is offered. First is the default the loader plans against.
resolution_presets = ((1280, 704), (704, 1280)),
- # bf16-RESIDENT. transformer + VAE ship FP32 on disk (20.0 GB = 5B x 4), so bf16 transformer ~10.0; UMT5 TE bf16 (11.4); VAE fp32 (2.8).
+ # bf16-RESIDENT. transformer + VAE ship FP32 on disk (20.0 GB = 5B x 4), so bf16 transformer ~10.0; UMT5 TE bf16
+ # (11.4); VAE fp32 (2.8).
bf16_components_gb = (10.0, 11.4, 2.8),
vae_force_fp32 = True,
# Byte-identical mirror of QuantStack/Wan2.2-TI2V-5B-GGUF (13 quants + companion VAE).
gguf_repo = "unsloth/Wan2.2-TI2V-5B-GGUF",
),
- # Wan2.2-T2V-A14B (diffusers >= 0.35, verified on 0.39): the dual-expert MoE. Both transformers are WanTransformer3DModel with boundary_ratio 0.875; high-noise steps route through transformer, low-noise through transformer_2, so cfg2_kwarg is threaded only here.
+ # Wan2.2-T2V-A14B (diffusers >= 0.35, verified on 0.39): the dual-expert MoE. Both transformers are
+ # WanTransformer3DModel with boundary_ratio 0.875; high-noise steps route through transformer, low-noise through
+ # transformer_2, so cfg2_kwarg is threaded only here.
VideoFamily(
name = "wan2.2-t2v-a14b",
pipeline_class = "WanPipeline",
@@ -271,7 +275,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (
base_repo = "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
aliases = ("wan2.2-14b", "wan-t2v", "wan2.2-t2v", "wan-t2v-a14b", "wan-a14b"),
has_audio = False,
- # is_moe drives the dual-DiT optimisation layers; cfg2_kwarg names the pipeline kwarg for transformer_2's guidance.
+ # is_moe drives the dual-DiT optimisation layers; cfg2_kwarg names the pipeline kwarg for transformer_2's
+ # guidance.
transformer2_class = "WanTransformer3DModel",
is_moe = True,
cfg2_kwarg = "guidance_scale_2",
@@ -284,12 +289,16 @@ _FAMILIES: tuple[VideoFamily, ...] = (
resolution_multiple = 16,
# 480p + 720p presets. A14B's VAE is 8x so multiple 16 renders 720 exactly (TI2V-5B's 16x VAE floors it to 704).
resolution_presets = ((1280, 720), (832, 480), (480, 832), (720, 1280)),
- # bf16-RESIDENT. Each expert ships FP32 (57.15 GB = 14.3B x 4), so ~28.6 bf16 each and ~57.2 for BOTH, not the 114.3 fp32 sum. UMT5 TE bf16 (11.4); VAE fp32 (0.5).
+ # bf16-RESIDENT: each expert ships FP32, so ~28.6 bf16 each and ~57.2 for BOTH
+ # bf16-RESIDENT. Each expert ships FP32 (57.15 GB = 14.3B x 4), so ~28.6 bf16 each and ~57.2 for BOTH, not the
+ # 114.3 fp32 sum. UMT5 TE bf16 (11.4); VAE fp32 (0.5).
bf16_components_gb = (57.2, 11.4, 0.5),
vae_force_fp32 = True,
- # No gguf_repo: community GGUFs split the experts, and a single-file load covers only one.
+ # no gguf_repo: community GGUFs split the experts, and a single-file load covers only one
),
- # HunyuanVideo-1.5 (diffusers >= 0.39): 8.3B DiT, Qwen2.5-VL + ByT5 encoders. Three quirks: no guidance kwarg (CFG on the ``guider``), no callback_on_step_end (generate() wraps scheduler.step), and no upstream model_index.json, so only the community repacks load.
+ # HunyuanVideo-1.5 (diffusers >= 0.39): 8.3B DiT, Qwen2.5-VL + ByT5 encoders. Three quirks: no guidance kwarg (CFG
+ # on the ``guider``), no callback_on_step_end (generate() wraps scheduler.step), and no upstream model_index.json,
+ # so only the community repacks load.
VideoFamily(
name = "hunyuanvideo-1.5",
pipeline_class = "HunyuanVideo15Pipeline",
@@ -301,21 +310,20 @@ _FAMILIES: tuple[VideoFamily, ...] = (
guidance_via_guider = True,
default_steps = 50,
default_guidance = 6.0,
- # 121 frames at 24 fps ~5s, the pipeline's own default.
default_num_frames = 121,
default_fps = 24,
- # HV15 VAE compresses 16x spatial / 4x temporal, patch-1, so sizes snap /16, frames 4k+1.
+ # HV15 VAE compresses 16x spatial / 4x temporal, patch-1, so sizes snap /16, frames 4k+1
frame_step = 4,
resolution_multiple = 16,
- # 480p-class presets (the base is the 480p variant): landscape, vertical, square. Every
- # entry is a real bucket of this tier (generate_crop_size_list(base_size=640)); 624x624
- # was not one, so the square option snapped off-tier at 1521 tokens against the trained
- # 1600.
+ # 480p-class presets (the base is the 480p variant): landscape, vertical, square. Every entry is a real bucket
+ # of this tier (generate_crop_size_list(base_size=640)); 624x624 was not one, so the square option snapped
+ # off-tier at 1521 tokens against the trained 1600.
resolution_presets = ((832, 480), (480, 832), (640, 640)),
- # DiT fp32 on disk (32.0 to 16.6 bf16); VAE (4.7 to 2.4); Qwen2.5-VL TE bf16 14.0 + ByT5 0.8.
+ # DiT fp32 on disk (32.0 to 16.6 bf16); VAE 4.7 to 2.4; Qwen2.5-VL TE bf16 14.0 + ByT5 0.8
bf16_components_gb = (16.6, 14.8, 2.4),
),
- # The 720p t2v repack: same architecture and footprint as the 480p entry, only the trained resolution differs. Its own family so a 720p load defaults to 720p sizes; the full-path alias outranks the generic token.
+ # The 720p t2v repack: same architecture and footprint as the 480p entry, only the trained resolution differs. Its
+ # own family so a 720p load defaults to 720p sizes; the full-path alias outranks the generic token.
VideoFamily(
name = "hunyuanvideo-1.5-720p",
pipeline_class = "HunyuanVideo15Pipeline",
@@ -365,11 +373,11 @@ def detect_video_family(repo_id: str, override: Optional[str] = None) -> Optiona
if best is None:
return None
fam = best[0]
- # HunyuanVideo-1.5's resolution tier is baked into the weights: the 480p and 720p repacks ship
- # transformer target_size 640 vs 960 and scheduler shift 5.0 vs 9.0, and their bucket lists are
- # disjoint. Only the literal 720p_t2v path was aliased, so 720p_i2v and every GGUF repack fell
- # through to the generic "hunyuanvideo-1.5" token and inherited the 480p base repo -- which is
- # also where the VAE and text encoder come from. Re-route on the tier marker instead.
+ # HunyuanVideo-1.5's resolution tier is baked into the weights: the 480p and 720p repacks ship transformer
+ # target_size 640 vs 960 and scheduler shift 5.0 vs 9.0, and their bucket lists are disjoint. Only the literal
+ # 720p_t2v path was aliased, so 720p_i2v and every GGUF repack fell through to the generic "hunyuanvideo-1.5" token
+ # and inherited the 480p base repo -- which is also where the VAE and text encoder come from. Re-route on the tier
+ # marker instead.
if fam.name == "hunyuanvideo-1.5" and re.search(r"(?:^|[-_./\\])720p", needle):
for candidate in _FAMILIES:
if candidate.name == "hunyuanvideo-1.5-720p":
@@ -556,17 +564,16 @@ def validate_video_request_shape(
deliberately NOT part of that escape hatch: every family declares a ``frame_step``
whether or not it declares presets, so an off-lattice count is always refused.
"""
- # Normalised to int pairs so membership holds however a family spelled its presets (the status payload
- # hands them out as lists, and a round-trip through it must not silently stop matching).
+ # Normalised to int pairs so membership holds however a family spelled its presets (the status payload hands them
+ # out as lists, and a round-trip through it must not silently stop matching).
presets = tuple((int(w), int(h)) for w, h in fam.resolution_presets)
- # No declared presets: no table to judge a SIZE against, so leave that to the snap (unusual/custom
- # families). The frame check below still runs either way -- frame_step is always declared.
+ # No declared presets: no table to judge a SIZE against, so leave that to the snap (unusual/custom families). The
+ # frame check below still runs either way -- frame_step is always declared.
if presets and (width is not None or height is not None):
# Resolve a half-specified request against the default preset first, as generate() does, then judge the pair.
- # Keyed on None, where generate() keys on falsiness: a 0 is judged as 0 here rather than
- # replaced by the default. The route cannot deliver one (the request model bounds it at 32),
- # and refusing an explicit 0 beats silently rendering something else, so the two agree on
- # every value that can actually arrive.
+ # Keyed on None, where generate() keys on falsiness: a 0 is judged as 0 here rather than replaced by the
+ # default. The route cannot deliver one (the request model bounds it at 32), and refusing an explicit 0 beats
+ # silently rendering something else, so the two agree on every value that can arrive.
want_w = presets[0][0] if width is None else int(width)
want_h = presets[0][1] if height is None else int(height)
if (want_w, want_h) not in presets:
@@ -575,29 +582,29 @@ def validate_video_request_shape(
f"Supported resolutions: {format_video_resolution_presets(fam)}."
)
if num_frames is not None:
- # The lattice is k * frame_step + frame_offset, NOT a hardcoded k * step + 1: most families
- # do sit at offset 1, but MiniMax-H3 is 17k + 5, and judging it against 17k + 1 would refuse
- # every count its own duration select offers, starting with its default of 124. Read the two
- # fields snap_num_frames reads, so the gate and the snap can never disagree about what is valid.
+ # The lattice is k * frame_step + frame_offset, NOT a hardcoded k * step + 1: most families do sit at offset 1,
+ # but MiniMax-H3 is 17k + 5, and judging it against 17k + 1 would refuse every count its own duration select
+ # offers, starting with its default of 124. Read the two fields snap_num_frames reads, so the gate and the snap
+ # can never disagree about what is valid.
step = max(1, fam.frame_step)
offset = max(1, fam.frame_offset)
count = int(num_frames)
- # The window the family declares it was trained for. Hoisted out of the lattice branch
- # because it is also enforced below: it used to exist only to WORD the lattice error
- # ("supported counts run from 124 to 345") while a request outside it was accepted and
- # silently snapped, so num_frames=5 rendered 124 frames and num_frames=872 rendered 345.
+ # The window the family declares it was trained for. Hoisted out of the lattice branch because it is also
+ # enforced below: it used to exist only to WORD the lattice error ("supported counts run from 124 to 345") while
+ # a request outside it was accepted and silently snapped, so num_frames=5 rendered 124 frames and num_frames=872
+ # rendered 345.
ceiling = MAX_VIDEO_NUM_FRAMES
if fam.max_num_frames is not None:
ceiling = min(ceiling, int(fam.max_num_frames))
floor = max(offset, int(fam.min_num_frames))
if count < offset or (count - offset) % step != 0:
- # The two lattice points straddling the request say more than a prefix of the lattice would,
- # and stay short. Computed from the lattice rather than via snap_num_frames, which floors for
- # some families and CEILS for others (snap_frames_up) and so cannot be relied on for "below".
+ # The two lattice points straddling the request say more than a prefix of the lattice would, and stay short.
+ # Computed from the lattice rather than via snap_num_frames, which floors for some families and CEILS for
+ # others (snap_frames_up) and so cannot be relied on for "below".
below = offset + max(0, (count - offset) // step) * step
above = below + step
- # Only name a point the caller could actually load: past the request model's own `le`, or
- # outside this family's declared range, it answers with a second, differently-shaped 422.
+ # Only name a point the caller could actually load: past the request model's own `le`, or outside this
+ # family's declared range, it answers with a second, differently-shaped 422.
loadable = [n for n in (below, above) if floor <= n <= ceiling]
if len(loadable) == 2:
nearest = f"the nearest supported counts are {loadable[0]} and {loadable[1]}"
@@ -611,9 +618,9 @@ def validate_video_request_shape(
f"{step}, so a frame count must be k * {step} + {offset}; {nearest} "
f"(the default is {fam.default_num_frames})."
)
- # On the lattice but outside the trained window. The request model bounds num_frames at
- # 1..1024, so a count well under the floor or well over the ceiling arrives here and used
- # to be snapped in silence, which on the native path is a 25x compute surprise.
+ # On the lattice but outside the trained window. The request model bounds num_frames at 1..1024, so a count well
+ # under the floor or well over the ceiling arrives here and used to be snapped in silence, which on the native
+ # path is a 25x compute surprise.
if count < floor or count > ceiling:
raise VideoShapeError(
f"{count} is not a supported frame count for {fam.name}. "
@@ -726,7 +733,8 @@ def validate_video_reference_conditioning(
)
-# Default (steps, guidance) per checkpoint variant, matched by substring (picked id then base repo), most specific first.
+# Default (steps, guidance) per checkpoint variant, matched by substring (picked id then base repo), most specific
+# first.
_VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
("distilled", 8, 1.0),
("ltx", 40, 4.0),
@@ -748,7 +756,8 @@ def default_video_generation_params(
for identifier in identifiers:
needle = (identifier or "").lower()
for key, steps, guidance in _VIDEO_GENERATION_DEFAULTS:
- # Match the key as a name segment: reject a preceding ASCII letter so "swan-video" does not false-match "wan".
+ # Match the key as a name segment: reject a preceding ASCII letter so "swan-video" does not false-match
+ # "wan".
if re.search(r"(?<![a-z])" + re.escape(key), needle):
return steps, guidance
return fallback
diff --git a/studio/backend/core/inference/video_gallery.py b/studio/backend/core/inference/video_gallery.py
index 0c3f2de95f3..a5b81839ea4 100644
--- a/studio/backend/core/inference/video_gallery.py
+++ b/studio/backend/core/inference/video_gallery.py
@@ -141,8 +141,6 @@ def save(
mp4_tmp = directory / f".{video_id}.mp4.tmp"
sidecar = directory / f"{video_id}.json"
sidecar_tmp = directory / f".{video_id}.json.tmp"
- # Stage both files, rename the MP4 in, then the sidecar (the pair's commit marker: list_videos skips an mp4 without a
- # readable sidecar). On any failure remove every artifact, else an invisible, undeletable orphan MP4 is left behind.
try:
mp4_tmp.write_bytes(mp4_bytes)
sidecar_tmp.write_text(json.dumps(meta), encoding = "utf-8")
@@ -163,7 +161,7 @@ def _record(
meta: dict[str, Any],
flags: Optional[dict[str, dict[str, Any]]] = None,
) -> dict[str, Any]:
- # Flags are library state, not recipe: they come from the .flags.json store, never the sidecar.
+ # flags are library state, not recipe: they come from the .flags.json store, never the sidecar
return {
**meta,
"id": video_id,
@@ -179,7 +177,6 @@ def video_path(video_id: str) -> Optional[Path]:
if not _ID_RE.match(video_id):
return None
path = gallery_dir() / f"{video_id}.mp4"
- # Defence in depth: confirm the resolved path is still inside the gallery.
try:
path.resolve().relative_to(gallery_dir().resolve())
except ValueError:
@@ -226,7 +223,7 @@ def transcode_to_file(video_id: str, fmt: str) -> Optional[Path]:
export of a clip that size runs to hundreds of MB, and holding it as one ``bytes`` (then again
in the response) let a couple of concurrent export clicks exhaust the process. The MP4 route
already streams from disk; this makes the transcodes behave the same way."""
- # Ownership-gate like /file: only transcode an Unsloth-owned clip (readable sidecar), so a guessed stem for a foreign MP4 cannot be re-encoded out either.
+ # only transcode an Unsloth-owned clip, so a guessed stem for a foreign MP4 cannot be re-encoded out
path = owned_video_path(video_id)
if path is None:
return None
@@ -278,10 +275,13 @@ def _transcode_webm(path: Path, dest: Path) -> None:
out_v.width = in_v.codec_context.width
out_v.height = in_v.codec_context.height
out_v.pix_fmt = "yuv420p"
- # Realtime settings: VP9's default "good" profile is slow; cpu-used 8 + row-mt is much faster at a small quality cost.
+ # VP9's default "good" profile is slow; cpu-used 8 + row-mt is much faster at a small quality cost
+ # Realtime settings: VP9's default "good" profile is slow; cpu-used 8 + row-mt is much faster at a small
+ # quality cost.
out_v.options = {"deadline": "realtime", "cpu-used": "8", "row-mt": "1"}
- # An LTX-2 clip carries a synchronized audio track and WebM is the web-embed format, so dropping it would hand back half the
- # result. Opus is WebM's audio codec: resample to its 48 kHz grid and feed whole frames through a FIFO (960 samples per frame).
+ # An LTX-2 clip carries a synchronized audio track and WebM is the web-embed format, so dropping it would
+ # hand back half the result. Opus is WebM's audio codec: resample to its 48 kHz grid and feed whole frames
+ # through a FIFO (960 samples per frame).
in_a = src.streams.audio[0] if src.streams.audio else None
out_a = fifo = resampler = None
if in_a is not None:
@@ -306,9 +306,9 @@ def _transcode_webm(path: Path, dest: Path) -> None:
for packet in out_a.encode(frame):
dst.mux(packet)
- # Demux both streams together so the muxer sees them interleaved rather than buffering every video packet.
+ # demux both streams together so the muxer sees them interleaved rather than buffering every video packet
for packet in src.demux(*([in_v] + ([in_a] if out_a is not None else []))):
- if packet.dts is None: # flush packet from the demuxer
+ if packet.dts is None:
continue
if packet.stream is in_v:
for frame in packet.decode():
@@ -317,7 +317,8 @@ def _transcode_webm(path: Path, dest: Path) -> None:
continue
for frame in packet.decode():
for resampled in resampler.resample(frame):
- # Let the FIFO time the output: the resampler's frames do not line up with Opus' fixed frame size.
+ # let the FIFO time the output: the resampler's frames do not line up with Opus' fixed frame
+ # size
resampled.pts = None
fifo.write(resampled)
_drain_audio()
@@ -333,8 +334,9 @@ def _transcode_webm(path: Path, dest: Path) -> None:
raise RuntimeError(f"WebM export failed (libvpx-vp9 unavailable?): {exc}") from exc
-# Ceilings for a GIF export, which must hold every kept frame in memory before encoding. 720 px and 300 frames (25s at the
-# 12 fps target) bound that at roughly 150 MB for the widest clip a generate request allows.
+# a GIF export holds every kept frame in memory
+# Ceilings for a GIF export, which must hold every kept frame in memory before encoding. 720 px and 300 frames (25s at
+# the 12 fps target) bound that at roughly 150 MB for the widest clip a generate request allows.
_GIF_MAX_EDGE = 720
_GIF_MAX_FRAMES = 300
@@ -356,8 +358,9 @@ def _transcode_gif(path: Path) -> bytes:
rate = float(in_v.average_rate or 24)
# Full-rate GIFs are huge and stutter; ~12 fps (skipping source frames) is the sweet spot.
step = max(1, round(rate / 12))
- # Every kept frame is held as a paletted image until the encoder runs, so an unbounded walk is a memory bomb (a 2048x2048 clip of 1024
- # frames is >4 GB). Bound both axes: downscale past _GIF_MAX_EDGE and widen the step to at most _GIF_MAX_FRAMES. MP4 keeps the full clip.
+ # Every kept frame is held as a paletted image until the encoder runs, so an unbounded walk is a memory bomb
+ # (a 2048x2048 clip of 1024 frames is >4 GB). Bound both axes: downscale past _GIF_MAX_EDGE and widen the
+ # step to at most _GIF_MAX_FRAMES. MP4 keeps the full clip.
total = in_v.frames or 0
kept = (total + step - 1) // step if total else 0
if kept > _GIF_MAX_FRAMES:
@@ -366,7 +369,6 @@ def _transcode_gif(path: Path) -> bytes:
if i % step:
continue
if len(frames) >= _GIF_MAX_FRAMES:
- # Frame count unknown up front (no stream metadata): stop at the cap.
break
image = frame.to_image()
if max(image.size) > _GIF_MAX_EDGE:
@@ -399,8 +401,9 @@ def _sidecar_path(video_id: str) -> Path:
return gallery_dir() / f"{video_id}.json"
-# Sidecar keys every genuine Unsloth record carries. delete()/clear() own a pair only when its sidecar has all of these, so a
-# hand-dropped MP4 with a partial sidecar is neither counted as ours nor destroyed. Key-presence only.
+# key-presence only: delete()/clear() own a pair only when its sidecar has all of these
+# Sidecar keys every genuine Unsloth record carries. delete()/clear() own a pair only when its sidecar has all of these,
+# so a hand-dropped MP4 with a partial sidecar is neither counted as ours nor destroyed. Key-presence only.
_REQUIRED_META = (
"prompt",
"width",
@@ -419,13 +422,14 @@ def _read_meta(sidecar: Path) -> Optional[dict[str, Any]]:
try:
raw = sidecar.read_text(encoding = "utf-8")
except (OSError, UnicodeError):
- # Invalid UTF-8 is a corrupt sidecar, not a listing failure.
return None
try:
meta = json.loads(raw)
except (ValueError, TypeError):
return None
- # A parseable dict is not enough: a foreign ("{}") or different-schema sidecar lacks these keys, and delete()/clear() must never destroy a clip the gallery never surfaced.
+ # a parseable dict is not enough: delete()/clear() must never destroy a clip the gallery never surfaced
+ # A parseable dict is not enough: a foreign ("{}") or different-schema sidecar lacks these keys, and
+ # delete()/clear() must never destroy a clip the gallery never surfaced.
if not isinstance(meta, dict) or any(k not in meta for k in _REQUIRED_META):
return None
return meta
@@ -483,19 +487,23 @@ def list_videos(
except OSError:
return []
flags = gallery_flags.read(gallery_dir())
- # Shelf split and pin sort run on file stems, BEFORE any sidecar is read, so they cost one
- # dict lookup per file and leave the early break below intact.
+ # both run on file stems BEFORE any sidecar is read
+ # Shelf split and pin sort run on file stems, BEFORE any sidecar is read, so they cost one dict lookup per file and
+ # leave the early break below intact.
paths = [p for p in paths if gallery_flags.is_archived(flags, p.stem) == archived]
paths.sort(key = lambda p: (gallery_flags.pin_rank(flags, p.stem), _mtime(p)), reverse = True)
- # Page over READABLE records, not raw files: filtering an orphan MP4 out of an already-sliced window would drop valid videos and make has_more wrong.
+ # page over READABLE records: filtering an orphan MP4 out of an already-sliced window would drop valid videos and
+ # make has_more wrong
+ # Page over READABLE records, not raw files: filtering an orphan MP4 out of an already-sliced window would drop
+ # valid videos and make has_more wrong.
want = None if limit is None else offset + limit
records = []
for path in paths:
meta = _read_meta(_sidecar_path(path.stem))
- if meta is None: # orphan mp4 (no readable sidecar)
+ if meta is None:
continue
record = _record(path.stem, meta, flags)
- if valid is not None and not valid(record): # parses but schema-invalid
+ if valid is not None and not valid(record):
continue
records.append(record)
if want is not None and len(records) >= want:
@@ -512,8 +520,8 @@ def set_flags(
"""Patch one clip's pin/archive flags and return its updated record, or None when the id is
not an Unsloth-owned clip. Ownership-gated like delete: a guessed stem for a hand-dropped or
orphan MP4 must not become flaggable."""
- # Ownership check and write under one lock, so a concurrent clear cannot delete the pair
- # between them and leave this reporting success for a clip that is already gone.
+ # Ownership check and write under one lock, so a concurrent clear cannot delete the pair between them and leave this
+ # reporting success for a clip that is already gone.
with gallery_flags.exclusive(gallery_dir()):
if owned_video_path(video_id) is None:
return None
@@ -529,17 +537,18 @@ def delete(video_id: str) -> bool:
path = video_path(video_id)
if path is None:
return False
- # Only delete a pair we own (a readable sidecar); a foreign/orphan MP4 is invisible to list_videos, so a guessed id must not destroy it.
+ # a foreign/orphan MP4 is invisible to list_videos, so a guessed id must not destroy it
+ # Only delete a pair we own (a readable sidecar); a foreign/orphan MP4 is invisible to list_videos, so a guessed id
+ # must not destroy it.
if _read_meta(_sidecar_path(video_id)) is None:
return False
- # Delete the MP4 FIRST: dropping the sidecar first and failing to unlink the mp4 would leave a clip that vanished from the
- # gallery with no retry. mp4-first leaves at worst an orphan sidecar, which list_videos ignores.
+ # delete the MP4 FIRST: sidecar-first plus a failed unlink leaves a clip that vanished from the gallery with no
+ # retry
try:
path.unlink()
except OSError as exc:
logger.warning("video_gallery.delete_failed: %s", exc)
return False
- # Best-effort sidecar unlink: a leftover json is skipped by list_videos anyway.
try:
_sidecar_path(video_id).unlink()
except OSError:
@@ -563,11 +572,10 @@ def clear(include_archived: bool = False, *, return_ids: bool = False) -> int |
Foreign/orphan MP4s are preserved: list_videos already hides them, so clear must not destroy them."""
removed = 0
directory = gallery_dir()
- # Hold the flag lock across the whole read-then-delete: an archive landing mid-loop would
- # otherwise be judged active from the stale snapshot and deleted, after its PATCH had already
- # reported success.
+ # Hold the flag lock across the whole read-then-delete: an archive landing mid-loop would otherwise be judged active
+ # from the stale snapshot and deleted, after its PATCH had already reported success.
with gallery_flags.exclusive(directory):
- # Read flags BEFORE listing: nothing is unlinked if the store turns out to be untrusted.
+ # read flags BEFORE listing: nothing is unlinked if the store turns out to be untrusted
flags = {} if include_archived else gallery_flags.read_trusted(directory)
try:
paths = list(directory.glob("*.mp4"))
@@ -575,11 +583,10 @@ def clear(include_archived: bool = False, *, return_ids: bool = False) -> int |
return [] if return_ids else 0
cleared: list[str] = []
for path in paths:
- if _read_meta(_sidecar_path(path.stem)) is None: # orphan / not ours
+ if _read_meta(_sidecar_path(path.stem)) is None:
continue
if not include_archived and gallery_flags.is_archived(flags, path.stem):
continue
- # mp4 first; if it can't be unlinked, leave the sidecar so the video stays listable.
try:
path.unlink()
except OSError:
@@ -590,8 +597,9 @@ def clear(include_archived: bool = False, *, return_ids: bool = False) -> int |
_sidecar_path(path.stem).unlink()
except OSError:
pass
- # Nothing left for an unreadable store to protect once every clip we own is gone, so this is
- # where the escape hatch escapes: replace it, or every later default clear still refuses.
+ # once every clip we own is gone an unreadable store protects nothing
+ # Nothing left for an unreadable store to protect once every clip we own is gone, so this is where the escape
+ # hatch escapes: replace it, or every later default clear still refuses.
if include_archived and not gallery_flags.is_trusted(directory):
gallery_flags.reset_locked(directory)
else:
diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.py
index 01af8a9d720..0671b110881 100644
--- a/studio/backend/core/inference/video_ltx2.py
+++ b/studio/backend/core/inference/video_ltx2.py
@@ -30,7 +30,8 @@ from loggers import get_logger
logger = get_logger(__name__)
-# Companion files (text projections, VAEs incl. vocoder) beside the quants in unsloth's GGUF repo: the official Lightricks weights split out of the combined checkpoint. Keyed by variant.
+# Companion files (text projections, VAEs incl. vocoder) beside the quants in unsloth's GGUF repo: the official
+# Lightricks weights split out of the combined checkpoint. Keyed by variant.
LTX23_EXTRAS_REPO = "unsloth/LTX-2.3-GGUF"
@@ -45,6 +46,7 @@ _EXTRAS_TEXT_PROJ = "text_encoders/ltx-2.3-22b-{variant}_embeddings_connectors.s
_EXTRAS_VIDEO_VAE = "vae/ltx-2.3-22b-{variant}_video_vae.safetensors"
_EXTRAS_AUDIO_VAE = "vae/ltx-2.3-22b-{variant}_audio_vae.safetensors"
+
# ── configs + rename tables, verbatim from scripts/convert_ltx2_to_diffusers.py ──
# from_single_file config overrides on top of the base 2.0 transformer config.
@@ -110,7 +112,6 @@ _CONNECTORS_CONFIG: dict[str, Any] = {
}
_VIDEO_VAE_RENAME = {
- # Encoder
"down_blocks.0": "down_blocks.0",
"down_blocks.1": "down_blocks.0.downsamplers.0",
"down_blocks.2": "down_blocks.1",
@@ -132,7 +133,6 @@ _VIDEO_VAE_RENAME = {
"up_blocks.8": "up_blocks.3",
"last_time_embedder": "time_embedder",
"last_scale_shift_table": "scale_shift_table",
- # Common
"res_blocks": "resnets",
"per_channel_statistics.mean-of-means": "latents_mean",
"per_channel_statistics.std-of-means": "latents_std",
@@ -355,11 +355,10 @@ def _load_extras_file(
LTX23_EXTRAS_REPO,
filename,
hf_token,
- # The plan counts an extras file cached under EITHER root and stages neither, so this has to
- # resolve both or it re-pulls what the planner skipped, inline and outside the manager.
+ # The plan counts an extras file cached under EITHER root and stages neither, so this has to resolve both or it
+ # re-pulls what the planner skipped, inline and outside the manager.
reuse_other_cache_root = True,
- # And a load nobody asked for takes the cached copy or fails: the switch's locality gate
- # cleared these three artifacts by name, so a miss here is a promise it cannot keep.
+ # the switch's locality gate cleared these three artifacts by name
local_files_only = local_files_only,
)
return load_file(path)
@@ -384,7 +383,8 @@ def ltx23_extras_files(checkpoint_path: Path | str) -> tuple[str, ...]:
# Upstream ltx_core's DISTILLED_SIGMA_VALUES: the fixed 8-step curve the 22B distilled DiT was trained against (the
-# scheduler appends the terminal 0). The base scheduler's shifted spacing never lands near it, so 8 steps pass this verbatim.
+# scheduler appends the terminal 0). The base scheduler's shifted spacing never lands near it, so 8 steps pass this
+# verbatim.
LTX23_DISTILLED_SIGMAS: tuple[float, ...] = (
1.0,
0.99375,
@@ -467,7 +467,8 @@ def load_ltx23_transformer(
import diffusers
from diffusers import LTX2VideoTransformer3DModel
- # Pre-rename the 2.3-only keys the converter does not know; from_single_file then merges the config overrides into the base 2.0 config and runs the stock conversion.
+ # Pre-rename the 2.3-only keys the converter does not know; from_single_file then merges the config overrides into
+ # the base 2.0 config and runs the stock conversion.
for old, new in _TRANSFORMER_PRERENAME:
for key in [k for k in dit_state if k.startswith(old)]:
dit_state[new + key[len(old) :]] = dit_state.pop(key)
@@ -495,7 +496,8 @@ def load_ltx23_connectors(
) -> Any:
from diffusers.pipelines.ltx2.connectors import LTX2TextConnectors
- # Transformer-only checkpoints carry the connector stacks but not the per-modality text projections, so fetch those from the companion file.
+ # Transformer-only checkpoints carry the connector stacks but not the per-modality text projections, so fetch those
+ # from the companion file.
if not any(k.startswith("text_embedding_projection") for k in connector_state):
connector_state = dict(connector_state)
connector_state.update(
@@ -562,7 +564,8 @@ def load_ltx23_audio_vae_and_vocoder(
_AUDIO_VAE_RENAME,
torch_dtype,
)
- # The 2.3 vocoder is a composite (base + bandwidth-extension stack + mel STFT buffers); keys line up module-for-module after the renames.
+ # The 2.3 vocoder is a composite (base + bandwidth-extension stack + mel STFT buffers); keys line up
+ # module-for-module after the renames.
vocoder_state = _apply_rename(_to_plain_dtype(vocoder_state, torch_dtype), _VOCODER_RENAME)
for key in [k for k in vocoder_state if ".ups." in k]:
vocoder_state[key.replace(".ups.", ".upsamplers.")] = vocoder_state.pop(key)
@@ -616,8 +619,9 @@ def load_ltx23_pipeline(
groups = _split_checkpoint(state)
del state
- # The Lightricks fp8 single files store SCALED float8 weights (.weight_scale/.input_scale companions), and casting without
- # the scales corrupts every quantized layer, so refuse loudly and point at the GGUF quants (Q8_0 for highest fidelity).
+ # The Lightricks fp8 single files store SCALED float8 weights (.weight_scale/.input_scale companions), and casting
+ # without the scales corrupts every quantized layer, so refuse loudly and point at the GGUF quants (Q8_0 for highest
+ # fidelity).
if any(k.endswith((".weight_scale", ".input_scale")) for k in groups["dit"]):
raise ValueError(
"This LTX checkpoint stores scaled fp8 weights, which this loader does "
@@ -656,11 +660,11 @@ def load_ltx23_pipeline(
local_files_only = local_files_only,
)
- # Shared 2.0/2.3 components from the base repo via model_index, so upstream class renames break loudly here rather than drift.
- # Pinned to the LIVE hub root, not huggingface_hub's import-time constant: Unsloth's cache
- # folder is a setting, and the locality gate that cleared this switch reads the live root. An
- # unpinned lookup after a mid-session change searches the OTHER root, so under
- # local_files_only it raises for a base that is fully downloaded, after eviction.
+ # Shared 2.0/2.3 components from the base repo via model_index, so upstream class renames break loudly here rather
+ # than drift. Pinned to the LIVE hub root, not huggingface_hub's import-time constant: Unsloth's cache folder is a
+ # setting, and the locality gate that cleared this switch reads the live root. An unpinned lookup after a
+ # mid-session change searches the OTHER root, so under local_files_only it raises for a base that is fully
+ # downloaded, after eviction.
cache_dir = _live_cache_dir()
index = LTX2Pipeline.load_config(
base_repo, token = hf_token, local_files_only = local_files_only, cache_dir = cache_dir
@@ -673,8 +677,6 @@ def load_ltx23_pipeline(
base_repo,
subfolder = name,
token = hf_token,
- # The dense Gemma3 encoder below is the largest of these by far, and every one of them
- # resolves the hub id: the flag is what keeps each a cache read.
local_files_only = local_files_only,
cache_dir = cache_dir,
**extra,
diff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.py
index 4255a2316e8..b86d7b5f33f 100644
--- a/studio/backend/core/inference/video_minimax_h3.py
+++ b/studio/backend/core/inference/video_minimax_h3.py
@@ -11,71 +11,60 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional
-# Must stay equal to the minimax-h3 family's `gguf_repo`. They are the same one-click pick, and
-# main's test_curated_gguf_repos_are_unsloth_mirrors only checks the family field, so a divergence
-# here would let that test pass while the actual download still came from a community repack.
-# tests/test_video_backend.py::test_the_h3_native_repo_matches_the_family_gguf_repo pins the pair.
-# The mirror carries the Qwen3-VL encoder quants as well as the denoisers, so this repo alone
-# satisfies h3_native_hub_files' first two entries.
+# Must stay equal to the minimax-h3 family's `gguf_repo`. They are the same one-click pick, and main's
+# test_curated_gguf_repos_are_unsloth_mirrors only checks the family field, so a divergence here would let that test
+# pass while the actual download still came from a community repack.
+# tests/test_video_backend.py::test_the_h3_native_repo_matches_the_family_gguf_repo pins the pair. The mirror carries
+# the Qwen3-VL encoder quants as well as the denoisers, so this repo alone satisfies h3_native_hub_files' first two
+# entries.
H3_GGUF_REPO = "unsloth/MiniMax-H3-GGUF"
-# The VAEs live beside the denoisers, so the native pick is one repo we control end to end. It was
-# Comfy-Org/MiniMax-H3, which put a community repack in the download path of BOTH H3 paths; an
-# install that already holds those bytes keeps using them through h3_component_source below,
-# because the HF cache is keyed by repo id and repointing alone re-downloads ~6 GB.
+# The VAEs live beside the denoisers, so the native pick is one repo we control end to end. It was Comfy-Org/MiniMax-H3,
+# which put a community repack in the download path of BOTH H3 paths; an install that already holds those bytes keeps
+# using them through h3_component_source below, because the HF cache is keyed by repo id and repointing alone
+# re-downloads ~6 GB.
H3_COMPONENT_REPO = "unsloth/MiniMax-H3-GGUF"
-# Where the component files came from originally. Only for reusing an existing cache entry: a
-# fresh install never reads it. The pairing itself lives in diffusion_families'
-# _SD_CPP_LEGACY_SOURCES, which owns this decision for every mirrored asset; this name is what the
-# delete-cached claims read, and a test pins the two together.
+# only for reusing an existing cache entry; the pairing itself lives in diffusion_families' _SD_CPP_LEGACY_SOURCES
+# Where the component files came from originally. Only for reusing an existing cache entry: a fresh install never reads
+# it. The pairing itself lives in diffusion_families' _SD_CPP_LEGACY_SOURCES, which owns this decision for every
+# mirrored asset; this name is what the delete-cached claims read, and a test pins the two together.
H3_LEGACY_COMPONENT_REPO = "Comfy-Org/MiniMax-H3"
H3_VIDEO_VAE = "vae/minimax_h3_video_vae_fp16.safetensors"
H3_AUDIO_VAE = "vae/minimax_h3_audio_vae_fp32.safetensors"
H3_QWEN_Q2 = "qwen3vl_32b_minimax_h3-Q2_K_M.gguf"
H3_QWEN_Q4 = "qwen3vl_32b_minimax_h3-Q4_K_M.gguf"
-# Measured with the merged Diffusers T2VA workflow and component-level CPU
-# offload. The base is the largest component plus runtime overhead; activation
-# memory scales with spatiotemporal volume across the tested 960x544 and
-# 1344x768, 124-345 frame matrix. The guard covers allocator variation around
-# the measured success and OOM boundaries.
+# Measured with the merged Diffusers T2VA workflow and component-level CPU offload. The base is the largest component
+# plus runtime overhead; activation memory scales with spatiotemporal volume across the tested 960x544 and 1344x768,
+# 124-345 frame matrix. The guard covers allocator variation around the measured success and OOM boundaries.
H3_DIFFUSERS_VRAM_BASE_GB = 68.5
H3_DIFFUSERS_VRAM_GB_PER_MPIXEL_FRAME = 0.08
-# The terms H3_DIFFUSERS_VRAM_BASE_GB is built from, so a load that shrinks one of the big
-# components can rebuild the floor from what it actually holds instead of the released sizes.
-#
-# With everything under enable_auto_cpu_offload the base is the LARGEST SINGLE RESIDENT COMPONENT
-# plus runtime overhead, not the sum: at any instant one component is on the device and the rest
-# are parked on the host. That is exactly why seeding a 20 GB pre-quantized denoiser moved this
-# number by nothing -- the 66.7 GB conditioner was already the larger of the two and simply took
-# over as the maximum. Both have to shrink before the floor does.
-#
-# ONE case breaks the max, and it is the case a pre-quantized denoiser creates. A torchao module
-# does not survive being moved mid-block, so _load_h3_modular_pipeline PINS it to the device and
-# takes it out of the offload rotation (see pin_prequantized_module). It is then resident for the
-# whole generation and the floor becomes additive: denoiser + whichever offloaded component is
-# largest. Measured at 960x544x124 with the int8 denoiser pinned, torch.cuda.max_memory_allocated:
-#
-# bfloat16 conditioner 94.62 GB = 20.3 + 66.7 + 5.18 activations + 2.44
-# int8 conditioner 55.20 GB = 20.3 + 27.1 + 5.18 activations + 2.58
-#
-# so 2.6 covers the pinned overhead on the conservative side of both. Note what the first row says
-# about the shipped constant: a pinned denoiser and a dense conditioner really need ~95 GB, and the
-# flat 68.5 under-states that by 26 GB. Rebuilding the floor from the resident components fixes
-# that under-estimate in the same stroke as crediting the saving.
+# The terms H3_DIFFUSERS_VRAM_BASE_GB is built from, so a load that shrinks one of the big components can rebuild the
+# floor from what it holds instead of the released sizes. With everything under enable_auto_cpu_offload the base is the
+# LARGEST SINGLE RESIDENT COMPONENT plus runtime overhead, not the sum: at any instant one component is on the device
+# and the rest are parked on the host. That is exactly why seeding a 20 GB pre-quantized denoiser moved this number by
+# nothing -- the 66.7 GB conditioner was already the larger of the two and took over as the maximum. Both have to shrink
+# before the floor does. ONE case breaks the max, and it is the case a pre-quantized denoiser creates. A torchao module
+# does not survive being moved mid-block, so _load_h3_modular_pipeline PINS it to the device and takes it out of the
+# offload rotation (see pin_prequantized_module). It is then resident for the whole generation and the floor becomes
+# additive: denoiser + whichever offloaded component is largest. Measured at 960x544x124 with the int8 denoiser pinned,
+# torch.cuda.max_memory_allocated: bfloat16 conditioner 94.62 GB = 20.3 + 66.7 + 5.18 activations + 2.44 int8
+# conditioner 55.20 GB = 20.3 + 27.1 + 5.18 activations + 2.58 so 2.6 covers the pinned overhead on the conservative
+# side of both. Note what the first row says about the shipped constant: a pinned denoiser and a dense conditioner need
+# ~95 GB, and the flat 68.5 under-states that by 26 GB. Rebuilding the floor from the resident components fixes that
+# under-estimate in the same stroke as crediting the saving.
H3_DIFFUSERS_VRAM_OVERHEAD_GB = 1.8
H3_DIFFUSERS_VRAM_PINNED_OVERHEAD_GB = 2.6
H3_TEXT_ENCODER_BF16_GB = 66.7
H3_TRANSFORMER_BF16_GB = 66.3
-# Video + audio VAE, from the family's bf16_components_gb. Only a floor for the offloaded term: it
-# stops a very small conditioner from claiming a base no component rotation could actually fit in.
+# Video + audio VAE, from the family's bf16_components_gb. Only a floor for the offloaded term: it stops a very small
+# conditioner from claiming a base no component rotation could actually fit in.
H3_VAE_RESIDENT_GB = 11.1
-# Resident decimal GB of each hosted pre-quantized denoiser, from the artifact sizes in
-# unsloth/MiniMax-H3-FP8 (MiniMax-H3-INT8.pt 18.86 GiB, MiniMax-H3-FP8.pt 18.87 GiB). Both are the
-# PRUNED (curve-form adaLN) partition, which is why they are so far under half the 66.3 GB dense
-# denoiser rather than at it.
+# Resident decimal GB of each hosted pre-quantized denoiser, from the artifact sizes in unsloth/MiniMax-H3-FP8
+# (MiniMax-H3-INT8.pt 18.86 GiB, MiniMax-H3-FP8.pt 18.87 GiB). Both are the PRUNED (curve-form adaLN) partition, which
+# is why they are so far under half the 66.3 GB dense denoiser rather than at it.
H3_TRANSFORMER_PREQUANT_GB: dict[str, float] = {"int8": 20.3, "fp8": 20.3}
@@ -130,16 +119,15 @@ def estimate_h3_diffusers_vram_gb(
return base + (H3_DIFFUSERS_VRAM_GB_PER_MPIXEL_FRAME * volume_mpixel_frames)
-# The VRAM at which the offload tier changes, and the host floor of the tier above it. Both are
-# the shipped values, unchanged: that tier is only reachable on a >= 132 GB device, where the
-# component sizes below are not what stands between a load and a generation, and there is no
-# measurement here to justify moving it.
+# both are the shipped values, unchanged: that tier is only reachable on a >= 132 GB device
+# The VRAM at which the offload tier changes, and the host floor of the tier above it. Both are the shipped values,
+# unchanged: that tier is only reachable on a >= 132 GB device, where the component sizes below are not what stands
+# between a load and a generation, and there is no measurement here to justify moving it.
H3_DIFFUSERS_HOST_RAM_TIER_VRAM_GB = 132.0
H3_DIFFUSERS_HOST_RAM_HIGH_VRAM_GB = 85.0
-# The offload tier parks every component on the host, so its floor is their SUM (unlike the VRAM
-# floor, which is the largest resident one). Derived, not newly measured: it is the shipped 150.0
-# minus the released component sum, so the released configuration still asks for exactly 150.0 and
-# only a load holding smaller components asks for less.
+# The offload tier parks every component on the host, so its floor is their SUM (unlike the VRAM floor, which is the
+# largest resident one). Derived, not newly measured: it is the shipped 150.0 minus the released component sum, so the
+# released configuration still asks for exactly 150.0 and only a load holding smaller components asks for less.
H3_DIFFUSERS_HOST_RAM_HEADROOM_GB = 5.9
@@ -170,9 +158,9 @@ def estimate_h3_diffusers_host_ram_gb(
return text_encoder + transformer + H3_VAE_RESIDENT_GB + H3_DIFFUSERS_HOST_RAM_HEADROOM_GB
-# torch.autocast casts the weight and bias of these module types to the autocast dtype on
-# entry. Norms sit on autocast's float32 promote list and bare parameters are read directly,
-# so both must keep their source precision.
+# torch.autocast casts these module types' weight and bias to the autocast dtype on entry
+# torch.autocast casts the weight and bias of these module types to the autocast dtype on entry. Norms sit on autocast's
+# float32 promote list and bare parameters are read directly, so both must keep their source precision.
_AUTOCAST_WEIGHT_MODULE_NAMES = ("Linear", "Conv1d", "Conv2d", "Conv3d")
@@ -211,8 +199,8 @@ def trim_h3_video_vae(vae: Any, *, workflow: str) -> dict[str, int]:
"""
import torch
- # Every lookup below is a getattr with a default, so a None vae and a vae whose
- # attributes moved both fall through to a zero report without a separate guard.
+ # Every lookup below is a getattr with a default, so a None vae and a vae whose attributes moved both fall through
+ # to a zero report without a separate guard.
report = {"encoder_freed": 0, "decoder_freed": 0}
if workflow == "t2va":
@@ -244,9 +232,9 @@ def trim_h3_video_vae(vae: Any, *, workflow: str) -> dict[str, int]:
return report
+# ── canvas geometry ────────────────────────────────────────────────────────── MiniMax-H3's upstream canvas rule,
+# shared by both engines.
# ── canvas geometry ──────────────────────────────────────────────────────────
-#
-# MiniMax-H3's upstream canvas rule, shared by both engines.
H3_CANVAS_SHORT_EDGE = 768
H3_CANVAS_MAX_PIXELS = 768 * 1344
H3_CANVAS_MULTIPLE = 32
@@ -306,13 +294,12 @@ def fit_h3_keyframe(image: Any, width: int, height: int, *, anchor: str) -> Any:
return image.resize(target, Image.LANCZOS, box = (left, top, left + crop_w, top + crop_h))
+# ── omni references (Ref2VA) ───────────────────────────────────────────────── Ref2VA uses a separate transformer
+# partition selected at load time.
# ── omni references (Ref2VA) ─────────────────────────────────────────────────
-#
-# Ref2VA uses a separate transformer partition selected at load time.
H3_TASK_KEYFRAMES = "fl2va"
H3_TASK_REFERENCES = "ref2va"
-# Upstream request limits.
H3_MAX_REF_IMAGES = 9
H3_MAX_REF_VIDEOS = 3
H3_MAX_REF_AUDIOS = 3
@@ -320,10 +307,11 @@ H3_MAX_REFERENCES = 12
# A reference video's trained window, in seconds.
H3_REF_VIDEO_MIN_SECONDS = 2.0
H3_REF_VIDEO_MAX_SECONDS = 15.0
-# How far a trim may reach past the video track before it is refused. A container reports its
-# longest track, so a file whose audio outruns its video reads as longer than it can show, and
-# a client picking an interval from that duration (HTMLMediaElement.duration, in Unsloth's case)
-# asks for slightly more video than exists. Within this margin the last frame is held instead.
+# a container reports its longest track, so a file whose audio outruns its video reads as longer than it can show
+# How far a trim may reach past the video track before it is refused. A container reports its longest track, so a file
+# whose audio outruns its video reads as longer than it can show, and a client picking an interval from that duration
+# (HTMLMediaElement.duration, in Unsloth's case) asks for slightly more video than exists. Within this margin the last
+# frame is held instead.
H3_REF_TRIM_COVERAGE_SLACK_SECONDS = 0.5
H3_FPS = 24
@@ -331,7 +319,7 @@ H3_FPS = 24
H3_REF_SIZE_MATCH = "match"
H3_REF_SIZE_MAX = "max"
H3_REF_IMAGE_SHORT_EDGE = 2048
-# H3 downscales references immediately, so this path can accept larger bounded sources.
+# H3 downscales references immediately, so this path can accept larger bounded sources
H3_REF_IMAGE_SOURCE_MAX_SIDE = 8192
H3_REF_IMAGE_SOURCE_MAX_PIXELS = 32_000_000
@@ -442,8 +430,8 @@ def decode_h3_reference_video(
)
expected_frames = int(round(duration * H3_FPS))
if trim is not None and len(frames) < expected_frames:
- # Hold the last frame across a shortfall the slack allows, as the trim decoders do at
- # their own endpoint; a larger gap means the range really was not there.
+ # Hold the last frame across a shortfall the slack allows, as the trim decoders do at their own endpoint; a
+ # larger gap means the range really was not there.
if len(frames) + math.ceil(H3_REF_TRIM_COVERAGE_SLACK_SECONDS * H3_FPS) < expected_frames:
raise ValueError("That reference video did not cover the selected range.")
frames.extend([frames[-1]] * (expected_frames - len(frames)))
@@ -633,9 +621,9 @@ def _decode_h3_video_trim_by_ordinal(
source_fps = float(stream.average_rate or stream.guessed_rate or H3_FPS)
if source_fps <= 0:
source_fps = float(H3_FPS)
- # The frame on screen at t is the last one starting at or before it, so the start
- # floors where the exclusive end ceils. Ceiling both skips the frame straddling a
- # fractional start, drifting this fallback ahead of the timestamp path.
+ # The frame on screen at t is the last one starting at or before it, so the start floors where the exclusive end
+ # ceils. Ceiling both skips the frame straddling a fractional start, drifting this fallback ahead of the
+ # timestamp path.
start_source_frame = math.floor(trim[0] * source_fps + 1e-6)
end_source_frame = math.ceil(trim[1] * source_fps - 1e-6)
target_count = int(round((trim[1] - trim[0]) * H3_FPS))
@@ -738,7 +726,6 @@ def _decode_audio_stream(
"Reference audio timestamps are unavailable and its stream cannot be reset."
) from exc
- # One resampler pass gives interleaved float32 whatever the source layout/format was.
resampler = av.AudioResampler(format = "flt", layout = stream.layout.name, rate = sample_rate)
channels = len(stream.layout.channels)
max_samples = math.floor(H3_REF_VIDEO_MAX_SECONDS * sample_rate + 1e-6)
@@ -767,9 +754,8 @@ def _decode_audio_stream(
take_end = min(block_end, end_sample) if end_sample is not None else block_end
if take_start < take_end:
chunks.append(block[take_start - block_start : take_end - block_start])
- # A track running past its video is clamped, not refused: encoder padding overshoots
- # routinely, and longer tracks decoded fine before trimming existed. Stopping here
- # also skips a tail no engine receives.
+ # A track running past its video is clamped, not refused: encoder padding overshoots routinely, and longer
+ # tracks decoded fine before trimming existed. Stopping here also skips a tail no engine receives.
return end_sample is not None and block_end >= end_sample
stopped = False
@@ -848,10 +834,9 @@ def _decode_audio_trim_by_timestamp(
for resampled in resampler.resample(None):
if _take(resampled):
break
- # Nothing copied means no soundtrack here, whether the track ended before the interval
- # or starts after it. Silence instead would be a fabricated track, and would hide the
- # gap from stage_h3_references' positional pairing. A track that merely runs out partway
- # did copy something, so it keeps its silent tail rather than failing.
+ # Nothing copied means no soundtrack here, whether the track ended before the interval or starts after it. Silence
+ # instead would be a fabricated track, and would hide the gap from stage_h3_references' positional pairing. A track
+ # that merely runs out partway did copy something, so it keeps its silent tail rather than failing.
if not copied_any:
return None, None
return output, sample_rate
@@ -896,7 +881,7 @@ class MiniMaxH3StagedReferences:
"""``MiniMaxH3References`` written to disk the way sd-cli reads them back."""
images: tuple[str, ...] = ()
- # Frame DIRECTORIES: sd-cli reads a reference video as images sorted lexicographically.
+ # frame DIRECTORIES: sd-cli reads a reference video as images sorted lexicographically
videos: tuple[str, ...] = ()
video_audios: tuple[str, ...] = ()
audios: tuple[str, ...] = ()
@@ -965,7 +950,7 @@ def h3_diffusers_references(references: MiniMaxH3References) -> list:
)
def waveform_tensor(waveform: Any) -> Any:
- # The blocks take a (channels, samples) tensor; the decoder produces (samples, channels).
+ # the blocks take a (channels, samples) tensor; the decoder produces (samples, channels)
return torch.from_numpy(waveform).transpose(0, 1).contiguous()
built: list = []
@@ -1126,11 +1111,10 @@ class MiniMaxH3NativeRuntime:
engine: Any
files: Any
offload_flags: tuple[str, ...]
- # (size, mtime_ns) of the sd-cli this runtime was built on, taken at load, right after
- # ensure_h3_sd_cpp_binary vetted it for H3 support and accelerator. Every generation compares
- # against THIS, not against whatever the path holds when it starts: an install that lands
- # between the load and a generation replaces the binary in place, and two reads taken after it
- # agree with each other while agreeing with nothing that was ever checked. None means the
+ # (size, mtime_ns) of the sd-cli this runtime was built on, taken at load, right after ensure_h3_sd_cpp_binary
+ # vetted it for H3 support and accelerator. Every generation compares against THIS, not against whatever the path
+ # holds when it starts: an install that lands between the load and a generation replaces the binary in place, and
+ # two reads taken after it agree with each other while agreeing with nothing that was ever checked. None means the
# identity could not be taken, which reads as "cannot vouch" rather than "unchanged".
binary_identity: Optional[tuple[int, int]] = None
diff --git a/studio/backend/core/inference/video_minimax_h3_adaln.py b/studio/backend/core/inference/video_minimax_h3_adaln.py
index 1052defeff9..5f9c889603d 100644
--- a/studio/backend/core/inference/video_minimax_h3_adaln.py
+++ b/studio/backend/core/inference/video_minimax_h3_adaln.py
@@ -42,9 +42,10 @@ from __future__ import annotations
import types
from typing import Any, Optional
-# Video, text and audio rows each get their own modulation row, so one projection emits three
-# blocks of six chunks. Mirrors diffusers' MINIMAX_H3_MODALITY_NUM; hardcoded rather than imported
-# so this module stays importable (and unit-testable) without diffusers.
+# video, text and audio rows each get a modulation row
+# Video, text and audio rows each get their own modulation row, so one projection emits three blocks of six chunks.
+# Mirrors diffusers' MINIMAX_H3_MODALITY_NUM; hardcoded rather than imported so this module stays importable (and
+# unit-testable) without diffusers.
MINIMAX_H3_MODALITY_NUM = 3
# Metadata keys the offline prequant builder bakes into a curve-form checkpoint.
@@ -52,8 +53,9 @@ ADALN_FORM_KEY = "adaln_form"
ADALN_CURVE_FORM = "curve"
CURVE_DIM_KEY = "curve_dim"
CURVE_GRID_KEY = "curve_grid"
-# The dtype of the block stack the modulation feeds, recorded by the builder. See
-# `_curve_modulation_forward` for why the chunks have to be cast to it.
+# dtype of the block stack the modulation feeds
+# The dtype of the block stack the modulation feeds, recorded by the builder. See `_curve_modulation_forward` for why
+# the chunks have to be cast to it.
ADALN_OUT_DTYPE_KEY = "adaln_out_dtype"
@@ -135,14 +137,14 @@ def _build_curve_time_embedder(curve_grid: int, curve_dim: int) -> Any:
class _MiniMaxH3CurveTimeEmbedder(nn.Module):
def __init__(self) -> None:
super().__init__()
- # Persistent: this IS the checkpoint's `time_embedder.table` entry, assigned by
- # load_state_dict. float32 like the reference; the curve is a smooth low-amplitude
- # signal whose differences drive every block's modulation.
+ # Persistent: this IS the checkpoint's `time_embedder.table` entry, assigned by load_state_dict. float32
+ # like the reference; the curve is a smooth low-amplitude signal whose differences drive every block's
+ # modulation.
self.register_buffer("table", torch.empty(curve_grid, curve_dim, dtype = torch.float32))
- # The model's forward reads `self.time_embedder.linear_1.weight.dtype` to decide what to
- # cast the timestep to. The dense module has that Linear; this one does not, so expose a
- # NON-PERSISTENT stand-in carrying only the dtype. Non-persistent keeps it out of the
- # state dict, so `strict = True` still matches the checkpoint exactly.
+ # The model's forward reads `self.time_embedder.linear_1.weight.dtype` to decide what to cast the timestep
+ # to. The dense module has that Linear; this one does not, so expose a NON-PERSISTENT stand-in carrying only
+ # the dtype. Non-persistent keeps it out of the state dict, so `strict = True` still matches the checkpoint
+ # exactly.
self.linear_1 = nn.Module()
self.linear_1.register_buffer(
"weight", torch.zeros(1, dtype = torch.float32), persistent = False
@@ -206,10 +208,12 @@ def apply_h3_adaln_curve(
)
def _reshape(linear: Any, where: str) -> Any:
- # Rebuild rather than resize: the replacement is a plain float32 Linear (the hosted
- # checkpoints store the pruned adaLN in float32) whose weights load_state_dict then
- # overwrites via assign=True. Built on the real device, never meta, so the module is
- # well-formed even if the checkpoint were to omit it and strict=True caught that instead.
+ # rebuild rather than resize: a plain float32 Linear (as the hosted checkpoints store it) that load_state_dict
+ # overwrites via assign=True
+ # Rebuild rather than resize: the replacement is a plain float32 Linear (the hosted checkpoints store the pruned
+ # adaLN in float32) whose weights load_state_dict then overwrites via assign=True. Built on the real device,
+ # never meta, so the module is well-formed even if the checkpoint were to omit it and strict=True caught that
+ # instead.
if not isinstance(linear, nn.Linear):
raise ValueError(f"MiniMax-H3 curve conversion expected a Linear at {where}.")
import torch
@@ -225,11 +229,13 @@ def apply_h3_adaln_curve(
if proj is None:
raise ValueError(f"MiniMax-H3 curve conversion: block {index} has no `adaln_proj`.")
proj.linear = _reshape(proj.linear, f"transformer_blocks.{index}.adaln_proj.linear")
- # Plain attribute, not a buffer: it must not reach the state dict, and `.to(device)` on the
- # module must not try to move it.
+ # plain attribute, not a buffer: it must stay out of the state dict and out of `.to(device)`
+ # Plain attribute, not a buffer: it must not reach the state dict, and `.to(device)` on the module must not try
+ # to move it.
proj._unsloth_adaln_out_dtype = out_dtype
- # Bind the SiLU-free forward per instance: the dense class is shared with dense loads in the
- # same process, so patching the CLASS would corrupt them.
+ # bind per instance: the dense class is shared with dense loads in the same process
+ # Bind the SiLU-free forward per instance: the dense class is shared with dense loads in the same process, so
+ # patching the CLASS would corrupt them.
proj.forward = types.MethodType(_curve_modulation_forward, proj)
converted += 1
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index 20e645e3a89..7aa4f993034 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -50,68 +50,65 @@ from __future__ import annotations
from functools import lru_cache
from typing import Any, Optional
-# The ConvRot primitives, shared with the denoiser's hosted INT8 checkpoint. Re-exported below
-# under the names this module has always used.
+# The ConvRot primitives, shared with the denoiser's hosted INT8 checkpoint. Re-exported below under the names this
+# module has always used.
from .diffusion_convrot import ( # noqa: F401 (re-export: callers and tests name them here)
build_convrot_hadamard,
rotate_convrot_activation,
)
-# The repo the prequantized denoisers already come from, so this adds no new dependency. It is no
-# longer H3_COMPONENT_REPO: the VAEs moved to the GGUF mirror and the conditioner did not, so the
-# two are now separate repos and aliasing them would send this file to the wrong one.
+# no longer H3_COMPONENT_REPO: the VAEs moved to the GGUF mirror and the conditioner did not
+# The repo the prequantized denoisers already come from, so this adds no new dependency. It is no longer
+# H3_COMPONENT_REPO: the VAEs moved to the GGUF mirror and the conditioner did not, so the two are now separate repos
+# and aliasing them would send this file to the wrong one.
H3_TE_QUANT_REPO = "unsloth/MiniMax-H3-FP8"
-# The community repack these quants were mirrored from, for an install whose cache predates the
-# move. Same reasoning as H3_LEGACY_COMPONENT_REPO, and the same owner: the pairing itself lives
-# in diffusion_families' _SD_CPP_LEGACY_SOURCES and is read through h3_te_quant_source below.
+# The community repack these quants were mirrored from, for an install whose cache predates the move. Same reasoning as
+# H3_LEGACY_COMPONENT_REPO, and the same owner: the pairing itself lives in diffusion_families' _SD_CPP_LEGACY_SOURCES
+# and is read through h3_te_quant_source below.
H3_LEGACY_TE_QUANT_REPO = "Comfy-Org/MiniMax-H3"
-# Hosted quantized conditioners, by ``text_encoder_quant`` scheme.
-#
-# nvfp4 (``qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors``, 14.61 GiB) is deliberately absent: it is
-# an AWQ NVFP4 layout with two levels of scale and a per-tensor global scale, a different loader
-# and a different kernel, and shipping it needs its own numerical verification. The table is the
-# one place to add it.
+# nvfp4 is deliberately absent: an AWQ NVFP4 layout with two levels of scale
+# Hosted quantized conditioners, by ``text_encoder_quant`` scheme. nvfp4
+# (``qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors``, 14.61 GiB) is deliberately absent: it is an AWQ NVFP4 layout with
+# two levels of scale and a per-tensor global scale, a different loader and a different kernel, and shipping it needs
+# its own numerical verification. The table is the one place to add it.
H3_TE_QUANT_FILES: dict[str, str] = {
"int8": "text_encoders/qwen3vl_32b_minimax_h3_int8_convrot.safetensors",
}
-# The scheme an UNSET ``text_encoder_quant`` resolves to on a device that supports it.
-#
-# The released conditioner is a 66.7 GB dense bfloat16 Qwen3-VL of 64 decoder layers and H3 reads
-# ``hidden_states[50]``; transformers has no early exit, so the default runs all 64 layers and
-# streams 66.7 GB across the CPU-offload boundary every generation. The hosted artifact is 27.1 GB
-# over 50 layers with the same read, which is why it is the default rather than an opt-in. NOT an
-# INT8 GEMM: the ConvRot forward dequantizes to the compute dtype and runs an ordinary
-# ``F.linear``, so the win is bytes moved and layers executed, not faster math.
+# the released conditioner is a 66.7 GB dense bfloat16 Qwen3-VL of 64 layers and H3 reads hidden_states[50]
+# The scheme an UNSET ``text_encoder_quant`` resolves to on a device that supports it. The released conditioner is a
+# 66.7 GB dense bfloat16 Qwen3-VL of 64 decoder layers and H3 reads ``hidden_states[50]``; transformers has no early
+# exit, so the default runs all 64 layers and streams 66.7 GB across the CPU-offload boundary every generation. The
+# hosted artifact is 27.1 GB over 50 layers with the same read, which is why it is the default rather than an opt-in.
+# NOT an INT8 GEMM: the ConvRot forward dequantizes to the compute dtype and runs an ordinary ``F.linear``, so the win
+# is bytes moved and layers executed, not faster math.
H3_TE_QUANT_DEFAULT = "int8"
-# Resident bytes of each conditioner, decimal GB, measured from the safetensors headers on the Hub
-# (2026-08-09) as the end of the last tensor's data. The quantized load is storage-faithful -- INT8
-# stays INT8, the per-output-channel scales stay float32, the vision tower and the embedding table
-# stay bfloat16 -- so the file size IS the resident size, unlike a cast-on-load path whose peak is
-# the dense encoder.
-#
-# qwen3vl_32b_minimax_h3_bf16.safetensors 51.51 GB (47.97 GiB)
-# qwen3vl_32b_minimax_h3_int8_convrot.safetensors 27.14 GB (25.28 GiB)
-#
-# The bfloat16 number here is the HOSTED 50-layer file. The budget default elsewhere is the
-# RELEASED 64-layer encoder (66.7 GB), which is what a load without this path actually materialises.
+# Resident decimal GB, measured from the safetensors headers (2026-08-09): bf16 51.51 GB, int8_convrot 27.14 GB.
+# Resident bytes of each conditioner, decimal GB, measured from the safetensors headers on the Hub (2026-08-09) as the
+# end of the last tensor's data. The quantized load is storage-faithful -- INT8 stays INT8, the per-output-channel
+# scales stay float32, the vision tower and the embedding table stay bfloat16 -- so the file size IS the resident size,
+# unlike a cast-on-load path whose peak is the dense encoder. qwen3vl_32b_minimax_h3_bf16.safetensors 51.51 GB (47.97
+# GiB) qwen3vl_32b_minimax_h3_int8_convrot.safetensors 27.14 GB (25.28 GiB) The bfloat16 number here is the HOSTED
+# 50-layer file. The budget default elsewhere is the RELEASED 64-layer encoder (66.7 GB), which is what a load without
+# this path materialises.
H3_TE_QUANT_RESIDENT_GB: dict[str, float] = {
"int8": 27.2,
}
+# must stay equal to MiniMaxH3ModularPipeline.text_encoder_layer
# Which Qwen3-VL hidden state conditions the transformer. Must stay equal to
# ``MiniMaxH3ModularPipeline.text_encoder_layer``; ``h3_te_layer_budget`` below is the only reader.
H3_TE_READ_LAYER = 50
-# ConvRot group size baked into the hosted checkpoint's per-tensor ``comfy_quant`` blob. Validated
-# per tensor at load rather than assumed, so a re-upload under a different group is refused instead
-# of silently decoding to noise.
+# ConvRot group size baked into the hosted checkpoint's per-tensor ``comfy_quant`` blob. Validated per tensor at load
+# rather than assumed, so a re-upload under a different group is refused instead of silently decoding to noise.
H3_TE_CONVROT_GROUP = 256
-# The quantization format the loader below implements. Any other value in a tensor's ``comfy_quant``
-# blob is a format this code has not been verified against, and is refused.
+# any other value in a tensor's `comfy_quant` blob is a format this code has not been verified against
+# The quantization format the loader below implements. Any other value in a tensor's ``comfy_quant`` blob is a format
+# this code has not been verified against, and is refused.
_H3_TE_EXPECTED_QUANT = {"format": "int8_tensorwise", "convrot": True}
# Tensor-name suffixes that make up one quantized linear in the hosted layout.
@@ -166,24 +163,22 @@ def h3_te_resident_gb(scheme: Optional[str], *, bf16_gb: float) -> float:
return H3_TE_QUANT_RESIDENT_GB[resolved] if resolved else bf16_gb
-# ── ConvRot INT8 ──────────────────────────────────────────────────────────────
-# W_rot = W @ H_block^T offline, x_rot = x @ H_block online, and H is a NORMALIZED REGULAR
-# Hadamard, which is symmetric and orthogonal, so H @ H == I exactly and
-# x_rot @ W_rot^T == x @ H @ H @ W^T == x @ W^T. The rotation is what lets INT8 survive Qwen3-VL's
-# per-channel activation outliers; dequantizing WITHOUT it is not an approximation, it is noise
-# (measured against the hosted bfloat16 file for one projection: 0.9% relative error with the
-# rotation, 137% without).
-#
-# This mirrors comfy-kitchen's ``_build_hadamard`` / ``_rotate_activation`` / ``_rotate_weight``
-# exactly, in a few lines of torch, rather than taking a dependency on a wheel Unsloth does not ship.
-#
-# ``build_convrot_hadamard`` and ``rotate_convrot_activation`` are imported at the top of this
-# module from ``diffusion_convrot``, where they now live. The DENOISER runs the same ConvRot on its
-# own hosted INT8 checkpoint, and the two rotations have to agree with the same comfy-kitchen
-# definition down to the normalizer -- two copies of a matrix nobody re-derives at review time is
-# exactly how they would stop agreeing. Both stay importable from here.
+# W_rot = W @ H_block^T offline, x_rot = x @ H_block online, with H a NORMALIZED REGULAR Hadamard (symmetric and
+# orthogonal, so H @ H == I exactly).
+# ── ConvRot INT8 ────────────────────────────────────────────────────────────── W_rot = W @ H_block^T offline, x_rot =
+# x @ H_block online, and H is a NORMALIZED REGULAR Hadamard, which is symmetric and orthogonal, so H @ H == I exactly
+# and x_rot @ W_rot^T == x @ H @ H @ W^T == x @ W^T. The rotation is what lets INT8 survive Qwen3-VL's per-channel
+# activation outliers; dequantizing WITHOUT it is not an approximation, it is noise (measured against the hosted
+# bfloat16 file for one projection: 0.9% relative error with the rotation, 137% without). This mirrors comfy-kitchen's
+# ``_build_hadamard`` / ``_rotate_activation`` / ``_rotate_weight`` exactly, in a few lines of torch, rather than taking
+# a dependency on a wheel Unsloth does not ship. ``build_convrot_hadamard`` and ``rotate_convrot_activation`` are
+# imported at the top of this module from ``diffusion_convrot``, where they now live. The DENOISER runs the same ConvRot
+# on its own hosted INT8 checkpoint, and the two rotations have to agree with the same comfy-kitchen definition down to
+# the normalizer -- two copies of a matrix nobody re-derives at review time is exactly how they would stop agreeing.
+# Both stay importable from here.
+# ── ConvRot INT8 ──────────────────────────────────────────────────────────────
@lru_cache(maxsize = None)
def _int8_convrot_linear_class() -> Any:
"""The ConvRot INT8 ``nn.Linear`` stand-in, built lazily so importing this module never imports
@@ -278,13 +273,14 @@ def _terminator_layer_class() -> Any:
return H3TextEncoderTerminatorLayer
-# ── name mapping ──────────────────────────────────────────────────────────────
-# The hosted checkpoint uses the ComfyUI flattening of the Qwen3-VL tree; transformers nests the
-# language model and the vision tower under ``model.``. Verified exhaustive on 2026-08-09: all 902
-# hosted names map into ``MiniMaxAI/MiniMax-H3`` ``text_encoder/model.safetensors.index.json``, with
-# nothing left over on this side.
+# the hosted checkpoint uses the ComfyUI flattening of the Qwen3-VL tree while transformers nests under `model.`;
+# ── name mapping ────────────────────────────────────────────────────────────── The hosted checkpoint uses the ComfyUI
+# flattening of the Qwen3-VL tree; transformers nests the language model and the vision tower under ``model.``. Verified
+# exhaustive on 2026-08-09: all 902 hosted names map into ``MiniMaxAI/MiniMax-H3``
+# ``text_encoder/model.safetensors.index.json``, with nothing left over on this side.
+# ── name mapping ──────────────────────────────────────────────────────────────
def h3_te_remap_key(key: str) -> str:
"""A hosted checkpoint tensor name in transformers' ``Qwen3VLForConditionalGeneration`` naming."""
if key.startswith("visual."):
@@ -358,17 +354,18 @@ def load_h3_quantized_text_encoder(
from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback
- # The repack when this install pulled the artifact before the move, else the mirror. The
- # stager resolved the same way, so a load that was cleared on a cached copy reads it from
- # the id it is actually cached under rather than 401ing or re-pulling 27 GB.
+ # The repack when this install pulled the artifact before the move, else the mirror. The stager resolved the
+ # same way, so a load that was cleared on a cached copy reads it from the id it is actually cached under rather
+ # than 401ing or re-pulling 27 GB.
source_repo = h3_te_quant_source(scheme)
path = hf_hub_download_with_xet_fallback(
source_repo,
filename,
hf_token,
cache_dir = cache_dir,
- # Resolve the artifact through whichever root holds it, exactly as the stager that
- # cleared this load did; pinned to cache_dir alone a moved cache folder re-pulls 27 GB.
+ # resolve through whichever root holds it, exactly as the stager did
+ # Resolve the artifact through whichever root holds it, exactly as the stager that cleared this load did;
+ # pinned to cache_dir alone a moved cache folder re-pulls 27 GB.
reuse_other_cache_root = True,
local_files_only = local_files_only,
)
@@ -378,39 +375,40 @@ def load_h3_quantized_text_encoder(
subfolder = "text_encoder",
token = hf_token,
cache_dir = cache_dir,
- # ``local_base`` is None on an offline load (the scoped base predownload stands down),
- # so this reads the hub id and would go out for the config without the flag.
+ # `local_base` is None on an offline load, so this reads the hub id and would go out for the config without
+ # the flag
local_files_only = local_files_only,
)
text_config = getattr(config, "text_config", config)
released_layers = int(getattr(text_config, "num_hidden_layers", 0))
if released_layers <= H3_TE_READ_LAYER:
- # A base repo whose conditioner is already at or below the read layer is not the model
- # this artifact was cut from; refuse rather than build something that reads a post-norm
- # state. Cannot happen for MiniMaxAI/MiniMax-H3 (64), but the pairing is not enforced
- # anywhere else.
+ # A base repo whose conditioner is already at or below the read layer is not the model this artifact was cut
+ # from; refuse rather than build something that reads a post-norm state. Cannot happen for
+ # MiniMaxAI/MiniMax-H3 (64), but the pairing is not enforced anywhere else.
raise ValueError(
f"{base} text_encoder has {released_layers} layers; MiniMax-H3 reads "
f"hidden_states[{H3_TE_READ_LAYER}] and needs more than that"
)
- # See the module docstring: one slot past the read layer, so the read lands on the raw
- # state rather than the post-norm one.
+ # one slot past the read layer, so the read lands on the raw state rather than the post-norm one
+ # See the module docstring: one slot past the read layer, so the read lands on the raw state rather than the
+ # post-norm one.
text_config.num_hidden_layers = H3_TE_READ_LAYER + 1
- # include_buffers=False on purpose (it is also accelerate's default, but the whole load
- # turns on it): PARAMETERS go to meta and are replaced wholesale by ``assign=True`` below,
- # while BUFFERS -- the rotary inverse frequencies, which no state dict carries because they
- # are non-persistent -- are built for real on CPU. They are a few KB. Putting them on meta
- # instead would leave the encoder with meta tensors after the load and force a dense CPU
- # rebuild, i.e. the 51 GB allocation this path exists to avoid.
+ # Include_buffers=False: PARAMETERS go to meta and are replaced wholesale by assign=True below
+ # include_buffers=False (it is also accelerate's default, but the whole load turns on it): PARAMETERS go to meta
+ # and are replaced wholesale by ``assign=True`` below, while BUFFERS -- the rotary inverse frequencies, which no
+ # state dict carries because they are non-persistent -- are built for real on CPU. They are a few KB. Putting
+ # them on meta instead would leave the encoder with meta tensors after the load and force a dense CPU rebuild,
+ # i.e. the 51 GB allocation this path exists to avoid.
with init_empty_weights(include_buffers = False):
encoder = transformers.Qwen3VLForConditionalGeneration(config)
language_model = encoder.model.language_model
language_model.layers[H3_TE_READ_LAYER] = _terminator_layer_class()()
- # Neither is in the artifact and neither can reach the conditioning: ``norm`` is only ever
- # applied to the terminator's output, and the head is never called (the pipeline invokes
- # ``text_encoder.model``, not ``text_encoder``). Dropping the head alone saves 1.56 GB.
+ # neither can reach the conditioning: `norm` is only applied to the terminator's output
+ # Neither is in the artifact and neither can reach the conditioning: ``norm`` is only ever applied to the
+ # terminator's output, and the head is never called (the pipeline invokes ``text_encoder.model``, not
+ # ``text_encoder``). Dropping the head alone saves 1.56 GB.
language_model.norm = torch.nn.Identity()
encoder.lm_head = torch.nn.Identity()
@@ -420,8 +418,8 @@ def load_h3_quantized_text_encoder(
quantized = {
name[: -len(_SCALE_SUFFIX)] for name in names if name.endswith(_SCALE_SUFFIX)
}
- # Swap every quantized projection for the INT8 module BEFORE loading, so the state dict
- # lands on buffers of the right dtype instead of being cast into bfloat16 Linears.
+ # Swap every quantized projection for the INT8 module BEFORE loading, so the state dict lands on buffers of
+ # the right dtype instead of being cast into bfloat16 Linears.
for prefix in sorted(quantized):
quant_key = prefix + _QUANT_SUFFIX
if quant_key not in names:
@@ -449,10 +447,10 @@ def load_h3_quantized_text_encoder(
),
)
- # The INT8 payload and its float32 scales are STORAGE and keep their dtypes; everything
- # still dense (the vision tower, the embedding table, the norms) follows the pipeline's
- # compute dtype, exactly as ``load_components(dtype=...)`` would have set it had this
- # component been loaded the ordinary way. A no-op for the bfloat16 H3 runs.
+ # The INT8 payload and its float32 scales are STORAGE and keep their dtypes; everything still dense (the
+ # vision tower, the embedding table, the norms) follows the pipeline's compute dtype, exactly as
+ # ``load_components(dtype=...)`` would have set it had this component been loaded the ordinary way. A no-op
+ # for the bfloat16 H3 runs.
state_dict = {}
for name in names:
if name.endswith(_QUANT_SUFFIX):
@@ -466,18 +464,15 @@ def load_h3_quantized_text_encoder(
tensor = tensor.to(dtype)
state_dict[h3_te_remap_key(name)] = tensor
- # strict=True proves the artifact and the meta-initialised skeleton describe the same 902
- # tensors, so a re-upload that renames, adds or drops one fails the load instead of
- # silently conditioning on partly random weights. It does NOT prove they are quantized:
- # a projection re-uploaded as a plain dense ``weight`` with its scale and metadata removed
- # simply drops out of the set above, leaves the original bfloat16 Linear in place, and
- # loads cleanly under strict. The load would then be recorded as engaged int8 while the
- # resident encoder crept back toward the dense 51 GB, and the VRAM preflight -- which
- # sizes the floor from the ENGAGED scheme -- would clear a generation that cannot fit.
- #
- # Checked structurally rather than against a count, so it stays true if the layer or
- # projection set ever changes: no ordinary Linear may survive inside the decoder stack.
- # The vision tower keeps its own dense Linears and is not in scope.
+ # Strict=True proves the artifact and the meta-initialised skeleton describe the same 902 tensors, so a
+ # re-upload that renames, adds or drops one fails the load instead of silently conditioning on partly random
+ # weights. It does NOT prove they are quantized: a projection re-uploaded as a plain dense ``weight`` with its
+ # scale and metadata removed drops out of the set above, leaves the original bfloat16 Linear in place, and loads
+ # cleanly under strict. The load would then be recorded as engaged int8 while the resident encoder crept back
+ # toward the dense 51 GB, and the VRAM preflight -- which sizes the floor from the ENGAGED scheme -- would clear
+ # a generation that cannot fit. Checked structurally rather than against a count, so it stays true if the layer
+ # or projection set ever changes: no ordinary Linear may survive inside the decoder stack. The vision tower
+ # keeps its own dense Linears and is not in scope.
dense = [
name
for name, module in language_model.layers.named_modules()
@@ -490,9 +485,10 @@ def load_h3_quantized_text_encoder(
f"path budgets for"
)
encoder.load_state_dict(state_dict, strict = True, assign = True)
- # Nothing may be left on meta. A dense CPU rebuild is NOT an acceptable repair here (it is
- # the 51 GB allocation this path exists to avoid), so a leftover is a refusal and the caller
- # falls back to the released encoder, which at least loads correctly.
+ # a dense CPU rebuild is NOT an acceptable repair here (it is the 51 GB allocation this path avoids)
+ # Nothing may be left on meta. A dense CPU rebuild is NOT an acceptable repair here (it is the 51 GB allocation
+ # this path exists to avoid), so a leftover is a refusal and the caller falls back to the released encoder,
+ # which at least loads correctly.
stranded = _meta_tensor_names(encoder)
if stranded:
raise ValueError(
diff --git a/studio/backend/core/inference/web_access_policy.py b/studio/backend/core/inference/web_access_policy.py
index 3c28622df8d..0b90e09d7b9 100644
--- a/studio/backend/core/inference/web_access_policy.py
+++ b/studio/backend/core/inference/web_access_policy.py
@@ -14,8 +14,8 @@ from urllib.parse import urlsplit
_DOMAIN_LABEL = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
_MAX_DOMAINS_PER_LIST = 100
-# A cache key holds the caller's raw strings for the life of the process, so only an entry
-# short enough to be a real domain is eligible; see normalize_website_policy.
+# A cache key holds the caller's raw strings for the life of the process, so only an entry short enough to be a real
+# domain is eligible; see normalize_website_policy.
_MAX_CACHEABLE_DOMAIN_LEN = 253
# Most search engines stop honouring site: past a handful of OR terms.
_SITE_FILTER_LIMIT = 8
@@ -92,10 +92,10 @@ def normalize_website_policy(value: Any) -> dict[str, list[str]]:
raise ValueError(f"{key} must be a list")
if len(raw_domains) > _MAX_DOMAINS_PER_LIST:
raise ValueError(f"{key} supports at most {_MAX_DOMAINS_PER_LIST} domains")
- # Exact ``str`` only: anything else can be unhashable and its error message carries
- # the original repr. Over-long entries stay off the cached path too -- nameprep
- # deletes characters such as U+00AD, so an arbitrarily long raw string can still
- # normalise, and caching would pin a caller-sized string. Both paths agree.
+ # exact str only (others can be unhashable); over-long raws stay uncached since nameprep can still shrink them
+ # Exact ``str`` only: anything else can be unhashable and its error message carries the original repr. Over-long
+ # entries stay off the cached path too -- nameprep deletes characters such as U+00AD, so an arbitrarily long raw
+ # string can still normalise, and caching would pin a caller-sized string. Both paths agree.
if all(
type(raw_domain) is str and len(raw_domain) <= _MAX_CACHEABLE_DOMAIN_LEN
for raw_domain in raw_domains
@@ -175,11 +175,11 @@ def scope_search_query(query: str, policy: dict[str, Any] | None) -> str:
allowed = normalize_website_policy(policy)["allowedDomains"]
if not allowed:
return query
- # Cap the site: filter (search engines limit OR operators) instead of dropping scoping for
- # large allow lists, which returned unrelated results that all got filtered out. Rotate the
- # window by query so every allowed domain stays reachable across a multi-step run (a fixed
- # head made domains past the cap permanently undiscoverable) and one query always scopes
- # the same way.
+ # cap the site: filter and rotate the window by query, else domains past the cap are permanently undiscoverable
+ # Cap the site: filter (search engines limit OR operators) instead of dropping scoping for large allow lists, which
+ # returned unrelated results that all got filtered out. Rotate the window by query so every allowed domain stays
+ # reachable across a multi-step run (a fixed head made domains past the cap permanently undiscoverable) and one
+ # query always scopes the same way.
window = allowed
if len(allowed) > _SITE_FILTER_LIMIT:
offset = zlib.crc32(query.encode("utf-8")) % len(allowed)