Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
pre-commit: cut the kwarg formatter's CPU 6.5x and stop the ruff fork bomb (#14174) (#14249)
Daniel Hand122ab5d9da3
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 33d69d88139..513242d4f13 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -37,6 +37,12 @@ repos:exclude: '(chat_templates|ollama_template_mappers|_auto_install|mapper)\.py$|^studio/backend/vendor/|^unsloth/_vendor/'additional_dependencies:- ruff==0.6.9+ # Serial: pre-commit used to start a partition per CPU, each with its own+ # multithreaded ruff, nproc x nproc threads on pre-commit.ci (#14174). A+ # long file list is still split into calls that run one after another;+ # each runs one ruff, and the script spreads the Python passes around it+ # over a bounded pool.+ require_serial: true# Re-pins allowScripts entries after dependency bumps. pre-commit.ci# pushes the fix to PR branches, Dependabot's included, so stale pins# heal without a human in the loop.diff --git a/scripts/enforce_kwargs_spacing.py b/scripts/enforce_kwargs_spacing.pyindex 3c1e597c2ce..754c7b9d106 100755--- a/scripts/enforce_kwargs_spacing.py+++ b/scripts/enforce_kwargs_spacing.py@@ -4,19 +4,30 @@ drop the blank line after a short indented import block, merge adjacent same-linstring literals, normalize def-signature magic commas (pre-ruff) so a def with>= 3 params and a default goes one-per-line while everything else stayscollapsible, and collapse a short multi-line assert onto one line (pre-ruff) by-stripping the magic trailing comma that holds it open."""+stripping the magic trailing comma that holds it open.++The fast engine shares one AST per file stage and tokenizes only headers,+short assert spans, and f-strings. A sparse lexer handles the rest. Compatibility+routines preserve the reference's behavior on malformed or unusual source.+There is no dependency on ruff's canonical layout and no persistent cache.+"""from __future__ import annotationsimport astimport argparse+import bisect+import functools+import gcimport ioimport os+import reimport statimport sysimport tempfile+import textwrapimport tokenize-from collections import defaultdict+from collections import defaultdict, dequefrom pathlib import Path@@ -57,7 +68,604 @@ def _atomic_write_text(path: Path, data: str, encoding: str) -> None:raise+# The entry points below use a stage-local analysis, never a persistent file or+# string cache. The _reference_* paths at the end of the file cover malformed+# input, unusual line separators, and syntax the sparse lexer does not know.+++def _without_gc(fn):+ """TokenInfo/AST objects have no cycles; do not scan them during this call.++ Restore the caller's GC setting, including on exceptions. No import-time+ global GC change (in particular, no effect on a differential reference).+ """++ @functools.wraps(fn)+ def run(*args, **kwargs):+ enabled = gc.isenabled()+ if enabled:+ gc.disable()+ try:+ return fn(*args, **kwargs)+ finally:+ if enabled:+ gc.enable()++ return run+++_ODD_LINES = re.compile(r"[\r\v\f\x1c-\x1e\x85\u2028\u2029]")+# Anything that could open a 3.14 template string (t"", tr"", rt""), anywhere in the+# text, strings and comments included: the sparse lexer below knows no t-strings.+_TSTRING_PREFIX = re.compile(r"(?<![\w])(?:[tT][rR]?|[rR][tT])['\"]")+++def _ordinary(text):+ # splitlines() and Python's physical-line numbering disagree on these.+ # Retain even the reference's erroneous indexing for that input.+ if _ODD_LINES.search(text):+ return False+ return sys.version_info < (3, 14) or not _TSTRING_PREFIX.search(text)+++# match_case is 3.10+; on 3.9 there is no match statement to reach.+_SUITE_NODES = (ast.stmt, ast.ExceptHandler) + (+ (ast.match_case,) if hasattr(ast, "match_case") else ()+)+++def _statements(tree):+ """Breadth-first statement/suite traversal, without visiting expressions.++ No expression can contain a statement. The order of statement ancestors and+ siblings agrees with ast.walk, including except handlers and match cases.+ """+ todo = deque([tree])+ while todo:+ node = todo.popleft()+ yield node+ for field in node._fields:+ value = getattr(node, field, None)+ if isinstance(value, list):+ todo.extend(x for x in value if isinstance(x, _SUITE_NODES))+++def _tree(text):+ try:+ return ast.parse(text)+ except SyntaxError:+ return None+++def _def_edits(lines, nodes):+ edits = []+ for node in nodes:+ if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):+ continue+ count, has_default = _def_spec(node)+ force = count >= _DEF_MIN_PARAMS_FOR_MULTILINE and has_default+ # Read only the header; the generator is stopped at its closing paren.+ # Including the first body line supports a one-line suite.+ lo = node.lineno - 1+ source = "".join(lines[lo : node.body[0].lineno])+ depth = 0+ last = None+ seen_def = False+ for tok in tokenize.generate_tokens(io.StringIO(source).readline):+ if not seen_def:+ seen_def = tok.type == tokenize.NAME and tok.string == "def"+ if seen_def and tok.start[0] != 1:+ # An explicit continuation between async and def gives+ # different keyword rows; the reference misses that def.+ break+ continue+ if not depth:+ if tok.type == tokenize.OP and tok.string == "[":+ # PEP 695: the reference searches the first '(', even if it+ # belongs to a type parameter bound. Keep its AST guard.+ return None+ if tok.type != tokenize.OP or tok.string != "(":+ continue+ depth = 1+ last = tok+ continue+ if tok.type == tokenize.OP:+ if tok.string == "(":+ depth += 1+ elif tok.string == ")":+ depth -= 1+ if depth == 0:+ comma = last.type == tokenize.OP and last.string == ","+ empty = last.type == tokenize.OP and last.string == "("+ if force and not comma and not empty:+ row, col = last.end+ edits.append((row + lo, col, col, ","))+ elif not force and comma:+ row, col = last.start+ edits.append((row + lo, col, col + 1, ""))+ break+ if tok.type not in _STRING_TRIVIA:+ last = tok+ return edits+++def _fits_one_line(segment):+ """The over-estimate the reference uses: lines joined with one space, plus the indent."""+ indent = len(segment[0]) - len(segment[0].lstrip())+ return indent + len(" ".join(s.strip() for s in segment)) <= _LINE_LENGTH+++def _apply(lines, edits):+ for row, c0, c1, replacement in sorted(edits, key=lambda e: (e[0], e[1]), reverse=True):+ line = lines[row - 1]+ lines[row - 1] = line[:c0] + replacement + line[c1:]+++def _assert_edits(lines, nodes):+ edits = []+ occupied = set()+ for node in nodes:+ if not isinstance(node, ast.Assert) or node.end_lineno == node.lineno:+ continue+ lo, hi = node.lineno, node.end_lineno+ segment = lines[lo - 1 : hi]+ if not _fits_one_line(segment):+ continue+ first = segment[0]+ if node.col_offset != len(first) - len(first.lstrip()) or (+ lo > 1 and lines[lo - 2].rstrip("\r\n").endswith("\\")+ ):+ # Another statement shares the first row, or the row continues the line+ # above: the rows alone would parse as something else (`obj\` + `[1,]`+ # is a subscript in the file but a list here). The whole-file check decides.+ return None+ # Shared physical rows require the reference's sequential whole-file+ # checks. They are uncommon (semicolon suites spanning multiple lines).+ if occupied.intersection(range(lo, hi + 1)):+ return None+ occupied.update(range(lo, hi + 1))+ source = "".join(segment)+ try:+ tokens = list(tokenize.generate_tokens(io.StringIO(source).readline))+ if any(t.type == tokenize.COMMENT for t in tokens):+ continue+ if any(+ t.type in (tokenize.STRING, getattr(tokenize, "FSTRING_MIDDLE", None))+ and t.start[0] != t.end[0]+ for t in tokens+ ):+ # Dedenting an isolated statement can alter a multiline string+ # value. Leave this unusual case to the whole-file guard.+ return None+ significant = [t for t in tokens if t.type not in _STRING_TRIVIA]+ positions = [+ t.start+ for t, nxt in zip(significant, significant[1:])+ if t.type == tokenize.OP+ and t.string == ","+ and nxt.type == tokenize.OP+ and nxt.string in (")", "]", "}")+ ]+ if not positions:+ continue+ trial = segment[:]+ _apply(trial, [(r, c, c + 1, "") for r, c in positions])+ before = ast.parse(textwrap.dedent(source))+ after = ast.parse(textwrap.dedent("".join(trial)))+ except (SyntaxError, tokenize.TokenError):+ # An assert on an elif/except/etc. header cannot be parsed alone.+ return None+ if ast.dump(before) == ast.dump(after):+ edits.extend((r + lo - 1, c, c + 1, "") for r, c in positions)+ return edits+++def _pre(text, definitions=True, assertions=True):+ if not _ordinary(text):+ return _reference_pre(text, definitions, assertions)+ if not ((definitions and "def" in text) or (assertions and "assert" in text)):+ return text, False+ tree = _tree(text)+ if tree is None:+ return text, False+ nodes = list(_statements(tree))+ lines = text.splitlines(True)+ changed = False+ if definitions:+ edits = _def_edits(lines, nodes)+ if edits is None:+ return _reference_pre(text, definitions, assertions)+ _apply(lines, edits)+ changed = bool(edits)+ if assertions:+ edits = _assert_edits(lines, nodes)+ if edits is None:+ updated, collapsed = _reference_collapse_short_asserts("".join(lines))+ return updated, changed or collapsed+ _apply(lines, edits)+ changed |= bool(edits)+ return ("".join(lines), True) if changed else (text, False)+++def _reference_pre(text, definitions, assertions):+ changed = False+ if definitions:+ text, changed = _reference_normalize_def_trailing_comma(text)+ if assertions:+ text, other = _reference_collapse_short_asserts(text)+ changed |= other+ return text, changed+++# A sparse lexer for VALID Python. The AST parser has already validated the+# input. We only need comments, string literals, and standalone '=' operators.+# F-strings use the running interpreter's tokenizer, up to their matching END;+# this deliberately preserves the Python 3.10/3.11 versus 3.12+ distinction.+# Unrolled (`a*(?:b a*)*`, not `(?:a|b)*`): the alternation kept one backtracking+# frame per character, about 120 bytes each, so a 10 MB literal took 1.2 GB.+_QUOTE = (+ r'''(?:"""[^"\\]*(?:(?:\\[\s\S]|"(?!""))[^"\\]*)*"""'''+ r"""|\'\'\'[^\'\\]*(?:(?:\\[\s\S]|\'(?!\'\'))[^\'\\]*)*\'\'\'"""+ r'''|"[^"\\\n]*(?:\\[\s\S][^"\\\n]*)*"'''+ r"""|\'[^\'\\\n]*(?:\\[\s\S][^\'\\\n]*)*\')"""+)+_MARKER = re.compile("[#'\"=]")+_LITERAL = re.compile(_QUOTE)+++def _literal_start(text, quote):+ def boundary(i):+ return i == 0 or not (text[i - 1].isalnum() or text[i - 1] == "_")++ if (+ quote >= 2+ and text[quote - 2 : quote].lower() in ("br", "rb", "fr", "rf")+ and boundary(quote - 2)+ ):+ return quote - 2+ if quote and text[quote - 1].lower() in "rubf" and boundary(quote - 1):+ return quote - 1+ return quote+++def _lex(text):+ """Return absolute '=' offsets and (kind, start, end) literal pieces."""+ equals, strings = [], []+ position = 0+ while True:+ match = _MARKER.search(text, position)+ if match is None:+ break+ start, end = match.span()+ marker = text[start]+ if marker == "#":+ end = text.find("\n", end)+ if end < 0:+ break+ elif marker == "=":+ if (start == 0 or text[start - 1] not in "!<>=:+-*/%@&|^") and (+ end == len(text) or text[end] != "="+ ):+ equals.append(start)+ else:+ quote = start+ start = _literal_start(text, quote)+ if "f" not in text[start:quote].lower():+ end = _LITERAL.match(text, quote).end()+ strings.append(("str", start, end))+ position = end+ continue+ piece = _fstring(text, start, equals)+ if piece is not None:+ end = piece[2]+ strings.append(piece)+ position = end+ return equals, strings+++def _fstring(text, start, equals):+ """Tokenize one f-string from ``start`` with the running interpreter, so its '=' tokens+ and its extent are whatever ``tokenize`` says on this Python (one STRING before 3.12,+ FSTRING_START..END from 3.12). Appends those '=' offsets to ``equals``; returns the piece."""+ cursor = start+ starts = [start]++ def readline():+ nonlocal cursor+ if cursor == len(text):+ return ""+ stop = text.find("\n", cursor)+ stop = len(text) if stop < 0 else stop + 1+ line = text[cursor:stop]+ cursor = stop+ starts.append(stop)+ return line++ depth = 0+ for tok in tokenize.generate_tokens(readline):+ if tok.type == tokenize.OP and tok.string == "=":+ equals.append(starts[tok.start[0] - 1] + tok.start[1])+ if tok.type == _FSTRING_START:+ depth += 1+ elif tok.type == _FSTRING_END:+ depth -= 1+ if depth == 0:+ return ("f", start, starts[tok.end[0] - 1] + tok.end[1])+ elif tok.type == tokenize.STRING and _FSTRING_START is None:+ return ("str", start, starts[tok.end[0] - 1] + tok.end[1])+ return None+++def _line_starts(lines):+ starts = [0]+ for line in lines:+ starts.append(starts[-1] + len(line))+ return starts+++def _space(lines, starts, equals):+ insertions = defaultdict(list)+ for offset in equals:+ row = bisect.bisect_right(starts, offset) - 1+ col = offset - starts[row]+ line = lines[row]+ if col and line[col - 1] not in " \t":+ insertions[row].append(col)+ if col + 1 < len(line) and line[col + 1] not in " \t\n\r":+ insertions[row].append(col + 1)+ for row, columns in insertions.items():+ line = lines[row]+ pieces = []+ previous = 0+ for col in columns:+ pieces.extend((line[previous:col], " "))+ previous = col+ pieces.append(line[previous:])+ lines[row] = "".join(pieces)+ return insertions+++def _blank_rows(lines, nodes):+ drop = set()+ for node in nodes:+ if isinstance(node, ast.Module):+ continue+ for attr in ("body", "orelse", "finalbody"):+ suite = getattr(node, attr, None)+ if not isinstance(suite, list) or not 1 < len(suite) <= 3:+ continue+ for previous, following in zip(suite, suite[1:]):+ if isinstance(previous, (ast.Import, ast.ImportFrom)) and not isinstance(+ following, (ast.Import, ast.ImportFrom)+ ):+ rows = range(previous.end_lineno + 1, following.lineno)+ if rows and all(not lines[r - 1].strip() for r in rows):+ drop.update(rows)+ return drop+++def _pass_nodes(tree):+ redundant = []+ todo = [tree]+ while todo:+ node = todo.pop()+ for attr in ("body", "orelse", "finalbody"):+ suite = getattr(node, attr, None)+ # Deliberately preserve the reference's single-statement-suite+ # recursion bug and its omission of match-case bodies.+ if not isinstance(suite, list) or len(suite) <= 1:+ continue+ redundant.extend(s for s in suite if isinstance(s, ast.Pass))+ todo.extend(child for child in suite if isinstance(child, ast.AST))+ todo.extend(getattr(node, "handlers", ()))+ return redundant+++def _merge_edits(text, lines, starts, strings, insertions, nodes):+ runs = []+ current = []+ previous_end = None+ previous_row = None+ for kind, start, end in strings:+ row = bisect.bisect_right(starts, start) - 1+ lastrow = bisect.bisect_right(starts, end - 1) - 1+ adjacent = current and previous_row == row and not text[previous_end:start].strip(" \t")+ if not adjacent or row != lastrow:+ if len(current) > 1:+ runs.append(current)+ current = []+ if row == lastrow:+ c0, c1 = start - starts[row], end - starts[row]+ cols = insertions.get(row, ())+ c0 += bisect.bisect_right(cols, c0)+ c1 += bisect.bisect_left(cols, c1)+ current.append((kind, row + 1, c0, c1, lines[row][c0:c1]))+ previous_end, previous_row = end, lastrow+ if len(current) > 1:+ runs.append(current)+ # A u prefix anywhere in a concatenation decides the `kind` of the constants an+ # f-fold produces, and the rest of that concatenation can sit outside the run+ # (`(f"" u""` + newline + `"a")`), so no check on the run alone can see it.+ # `u` combines with no other prefix, so a piece's first character tells.+ has_u = any(text[start] in "uU" for _kind, start, _end in strings)+ edits = []+ for run in runs:+ merged = _merge_string_run([(p[0], p[4]) for p in run])+ if merged is None:+ continue+ row, c0, c1 = run[0][1], run[0][2], run[-1][3]+ if any(p[0] == "f" for p in run):+ if has_u:+ return None+ enclosing = [+ n for n in nodes if isinstance(n, ast.stmt) and n.lineno <= row <= n.end_lineno+ ]+ stmt = min(enclosing, key=lambda n: n.end_lineno - n.lineno, default=None)+ if isinstance(stmt, ast.Assert) and stmt.lineno != stmt.end_lineno:+ segment = lines[stmt.lineno - 1 : stmt.end_lineno]+ segment[row - stmt.lineno] = lines[row - 1][:c0] + merged + lines[row - 1][c1:]+ if not _fits_one_line(segment):+ continue+ original = lines[row - 1][c0:c1]+ try:+ # Include parentheses to handle implicit concatenation and use AST+ # dumps only for these small expressions. With u prefixes ruled out+ # above, the pieces outside the run concatenate to the same values+ # either way, so local equality is a sufficient proof; local+ # inequality is NOT proof of file-level inequality (two changes in a+ # larger concatenation may cancel). Defer that case to the+ # reference's all-or-nothing global guard.+ before = ast.parse("(" + original + ")", mode="eval")+ after = ast.parse("(" + merged + ")", mode="eval")+ if ast.dump(before) != ast.dump(after):+ return None+ except SyntaxError:+ return None+ if merged[:1] != original[:1] and c0 and ("_" + lines[row - 1][c0 - 1]).isidentifier():+ # The fold adds a prefix, and the run is glued to a name or keyword+ # (`return"a" f"{x}"`): the result would read `returnf"..."`.+ return None+ edits.append((row, c0, c1, merged))+ return edits+++def _post(text, spacing=True, blanks=True, merging=True, passes=True):+ if not _ordinary(text):+ return _reference_post(text, spacing, blanks, merging, passes)+ tree = _tree(text)+ if tree is None:+ # enforce_spacing historically tokenizes even invalid syntax and can+ # raise TokenError/IndentationError; the other passes return unchanged.+ return _reference_post(text, spacing, blanks, merging, passes)+ lines = text.splitlines(True)+ if not lines:+ return text, False+ starts = _line_starts(lines)+ nodes = list(_statements(tree)) if blanks or merging else ()+ try:+ equals, strings = _lex(text) if spacing or merging else ((), ())+ except (tokenize.TokenError, SyntaxError):+ # The f-string scan runs the real tokenizer from mid-file; should it ever+ # disagree with the parser, the reference path decides, as for bad syntax.+ return _reference_post(text, spacing, blanks, merging, passes)+ original_lines = lines[:]+ insertions = _space(lines, starts, equals) if spacing else {}+ drop = _blank_rows(lines, nodes) if blanks and "import" in text else set()+ edits = _merge_edits(text, lines, starts, strings, insertions, nodes) if merging else []+ if edits is None:+ return _reference_post(text, spacing, blanks, merging, passes)+ redundant = _pass_nodes(tree) if passes and "pass" in text else []+ # Convert insertions/replacements to UTF-8 byte coordinates ONLY for pass+ # nodes. Preserve the reference's byte-column-as-character-column behavior.+ shifts = defaultdict(list)+ for node in redundant:+ row = node.lineno - 1+ if row not in shifts:+ shifts[row] = [+ (len(original_lines[row][:c].encode("utf-8")), 1) for c in insertions.get(row, ())+ ]+ pass_positions = []+ for node in redundant:+ row = node.lineno - 1+ start, end = node.col_offset, node.end_col_offset+ start += sum(delta for pos, delta in shifts[row] if pos <= node.col_offset)+ end += sum(delta for pos, delta in shifts[row] if pos < node.end_col_offset)+ merge_shift = 0+ for erow, c0, c1, replacement in edits:+ if erow == row + 1:+ end_byte = len(lines[row][:c1].encode("utf-8"))+ delta = len(replacement.encode("utf-8")) - len(lines[row][c0:c1].encode("utf-8"))+ if end_byte <= start:+ merge_shift += delta+ start += merge_shift+ end += merge_shift+ pass_positions.append((row, start, end))+ _apply(lines, edits)+ for row, start, end in sorted(pass_positions, reverse=True):+ line = lines[row]+ segment = line[:start] + line[end:]+ lines[row] = segment if segment.strip() else ""+ changed = bool(insertions or drop or edits or redundant)+ if not changed:+ return text, False+ kept = [line for row, line in enumerate(lines, 1) if row not in drop]+ if redundant:+ kept = [+ line if i == len(kept) - 1 or line.endswith("\n") else line + "\n"+ for i, line in enumerate(kept)+ if line+ ]+ return "".join(kept), True+++def _reference_post(text, spacing, blanks, merging, passes):+ changed = False+ for enabled, fn in (+ (spacing, _reference_enforce_spacing),+ (blanks, _reference_remove_blank_after_short_import),+ (merging, _reference_merge_adjacent_string_literals),+ (passes, _reference_remove_redundant_passes),+ ):+ if enabled:+ text, other = fn(text)+ changed |= other+ return text, changed+++@_without_gc+def normalize_def_trailing_comma(text: str) -> tuple[str, bool]:+ return _pre(text, assertions=False)+++@_without_gc+def collapse_short_asserts(text: str) -> tuple[str, bool]:+ return _pre(text, definitions=False)+++@_without_gcdef enforce_spacing(text: str) -> tuple[str, bool]:+ return _post(text, blanks=False, merging=False, passes=False)+++@_without_gc+def remove_blank_after_short_import(text: str) -> tuple[str, bool]:+ if "import" not in text:+ return text, False+ return _post(text, spacing=False, merging=False, passes=False)+++@_without_gc+def merge_adjacent_string_literals(text: str) -> tuple[str, bool]:+ return _post(text, spacing=False, blanks=False, passes=False)+++@_without_gc+def remove_redundant_passes(text: str) -> tuple[str, bool]:+ if "pass" not in text:+ return text, False+ return _post(text, spacing=False, blanks=False, merging=False)+++@_without_gc+def process_file(path: Path, pre: bool = False) -> bool:+ try:+ with tokenize.open(path) as handle:+ original = handle.read()+ encoding = handle.encoding+ except (OSError, SyntaxError) as exc:+ print(f"Failed to read {path}: {exc}", file=sys.stderr)+ return False+ updated, changed = _pre(original) if pre else _post(original)+ if changed:+ _atomic_write_text(path, updated, encoding)+ return changed+++# Shared literal policy and exact compatibility paths. These are deliberately+# retained so exceptional source keeps the reference behavior, including bugs.+++def _reference_enforce_spacing(text: str) -> tuple[str, bool]:"""Return updated text with keyword '=' padded by spaces, plus change flag."""lines = text.splitlines(keepends=True)if not lines:@@ -106,7 +714,7 @@ def enforce_spacing(text: str) -> tuple[str, bool]:return "".join(lines), True-def remove_redundant_passes(text: str) -> tuple[str, bool]:+def _reference_remove_redundant_passes(text: str) -> tuple[str, bool]:"""Drop pass statements that share a block with other executable code."""try:@@ -174,7 +782,7 @@ def remove_redundant_passes(text: str) -> tuple[str, bool]:return "".join(result_lines), changed-def remove_blank_after_short_import(text: str) -> tuple[str, bool]:+def _reference_remove_blank_after_short_import(text: str) -> tuple[str, bool]:"""Drop blank line(s) after an import block in a small nested suite. In an indented suite of <= 3 statements (never module level), when consecutive imports are followed across blank lines (nothing else) by another statement, remove those blanks. A comment in the gap blocks the rule. Removing blank lines never changes the AST."""try:tree = ast.parse(text)@@ -227,25 +835,29 @@ _STRING_TRIVIA = (tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT, tokenize.INDE_DEF_MIN_PARAMS_FOR_MULTILINE = 3 # signatures with < this many params stay one line+def _def_spec(node: ast.FunctionDef | ast.AsyncFunctionDef) -> tuple[int, bool]:+ """(param count, has-any-default). ``*`` and ``/`` markers are not counted. A default exists if any positional default is present or any keyword-only default is not ``None`` (``None`` in ``kw_defaults`` means a required keyword-only arg)."""+ a = node.args+ count = (+ len(a.posonlyargs)+ + len(a.args)+ + len(a.kwonlyargs)+ + (1 if a.vararg else 0)+ + (1 if a.kwarg else 0)+ )+ return count, bool(a.defaults) or any(d is not None for d in a.kw_defaults)++def _def_specs_by_line(tree: ast.AST) -> dict[int, tuple[int, bool]]:- """Map each def keyword line to (param count, has-any-default). ``*`` and ``/`` markers are not counted. A default exists if any positional default is present or any keyword-only default is not ``None`` (``None`` in ``kw_defaults`` means a required keyword-only arg)."""+ """Map each def keyword line to ``_def_spec`` of that def."""out: dict[int, tuple[int, bool]] = {}for node in ast.walk(tree):if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):- a = node.args- count = (- len(a.posonlyargs)- + len(a.args)- + len(a.kwonlyargs)- + (1 if a.vararg else 0)- + (1 if a.kwarg else 0)- )- has_default = bool(a.defaults) or any(d is not None for d in a.kw_defaults)- out[node.lineno] = (count, has_default)+ out[node.lineno] = _def_spec(node)return out-def normalize_def_trailing_comma(text: str) -> tuple[str, bool]:+def _reference_normalize_def_trailing_comma(text: str) -> tuple[str, bool]:"""Force a def signature one-per-line iff >= 3 params AND a default, else collapsible. A qualifying signature gets a magic trailing comma added (ruff wraps it one-per-line); every other signature has its trailing comma stripped so ruff collapses it when it fits. Def parameter lists only, never call sites or collection literals. Run BEFORE ruff format. Never changes the AST (re-checked)."""try:tree = ast.parse(text)@@ -441,7 +1053,7 @@ def _fold_collapses(return indent + len(joined) <= _LINE_LENGTH-def merge_adjacent_string_literals(text: str) -> tuple[str, bool]:+def _reference_merge_adjacent_string_literals(text: str) -> tuple[str, bool]:"""Merge adjacent string literals on ONE physical line into a single literal. Plain/raw/unicode runs merge by concatenation; an f-string + plain string folds into one f-string (plain braces escaped) only while the statement still fits one line. Runs of only f-strings, and bytes, are left side-by-side. The file AST is re-checked and a differing change dropped."""try:toks = list(tokenize.generate_tokens(io.StringIO(text).readline))@@ -497,7 +1109,7 @@ def merge_adjacent_string_literals(text: str) -> tuple[str, bool]:return out, True-def collapse_short_asserts(text: str) -> tuple[str, bool]:+def _reference_collapse_short_asserts(text: str) -> tuple[str, bool]:"""Collapse a multi-line ``assert`` onto one line when the estimated one-line length fits, by stripping the magic trailing commas (a comma before a closer) holding it open so ruff rejoins it. Run BEFORE ruff format. Skips asserts with a comment (would oscillate). Stripping is non-semantic except for a one-element tuple; the AST is re-checked and changing asserts left alone."""try:tree = ast.parse(text)@@ -559,34 +1171,6 @@ def collapse_short_asserts(text: str) -> tuple[str, bool]:return ("".join(working), True) if changed else (text, False)-def process_file(path: Path, pre: bool = False) -> bool:- try:- with tokenize.open(path) as handle:- original = handle.read()- encoding = handle.encoding- except (OSError, SyntaxError) as exc: # SyntaxError from tokenize on invalid python- print(f"Failed to read {path}: {exc}", file=sys.stderr)- return False-- if pre:- # Pre-ruff: normalize def-signature magic commas (>=3 params + a default add so ruff forces one-per-line, everything else strips so ruff collapses), and strip the magic trailing comma from a short multi-line assert so ruff joins it. Everything else runs post-ruff.- updated, normalized = normalize_def_trailing_comma(original)- updated, collapsed = collapse_short_asserts(updated)- if normalized or collapsed:- _atomic_write_text(path, updated, encoding)- return True- return False-- updated, changed = enforce_spacing(original)- updated, blanked = remove_blank_after_short_import(updated)- updated, merged = merge_adjacent_string_literals(updated)- updated, removed = remove_redundant_passes(updated)- if changed or blanked or merged or removed:- _atomic_write_text(path, updated, encoding)- return True- return False--def main(argv: list[str]) -> int:parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("files", nargs="+", help="Python files to fix")diff --git a/scripts/run_ruff_format.py b/scripts/run_ruff_format.pyindex 28065134ae7..9bf0dd57b09 100755--- a/scripts/run_ruff_format.py+++ b/scripts/run_ruff_format.py@@ -9,6 +9,7 @@ import osimport reimport subprocessimport sys+import tempfilefrom pathlib import PathHERE = Path(__file__).resolve().parent@@ -17,6 +18,12 @@ CONFIG = HERE.parent / ".pre-commit-config.yaml"# made under it will be reformatted by the hook and fail pre-commit.ANY_VERSION_ENV = "UNSLOTH_RUFF_FORMAT_ANY_VERSION"USAGE = "usage: run_ruff_format.py FILE [FILE ...] (formats in place; no options)"+# Worker count for the Python passes either side of ruff. The hook is require_serial+# (#14174), so these chunks are the only parallelism; ruff itself runs once.+JOBS_ENV = "UNSLOTH_RUFF_FORMAT_JOBS"+_MAX_JOBS = 8+# Below this many files per worker an extra interpreter costs more than it saves.+_FILES_PER_JOB = 32# `- ruff==0.6.9` under the hook's additional_dependencies. Read out of the config# rather than copied here, because a second copy of the pin is a second thing to@@ -111,6 +118,106 @@ def parse_files(argv: list[str]) -> tuple[list[str], str | None]:return list(argv), None+def job_count(n_files: int, environ: dict[str, str] | None = None) -> int:+ """Workers for the Python passes: bounded by CPUs, _MAX_JOBS and the file count."""+ env = os.environ if environ is None else environ+ override = env.get(JOBS_ENV, "").strip()+ if override:+ try:+ return max(1, int(override))+ except ValueError:+ pass+ return max(1, min(usable_cpus(), _MAX_JOBS, -(-n_files // _FILES_PER_JOB)))+++def usable_cpus() -> int:+ """CPUs this process may run on, which on a shared runner is fewer than the host has."""+ if hasattr(os, "process_cpu_count"): # 3.13++ return os.process_cpu_count() or 1+ try:+ return len(os.sched_getaffinity(0)) or 1+ except (AttributeError, OSError):+ return os.cpu_count() or 1+++def chunk_files(files: list[str], jobs: int) -> list[list[str]]:+ """Split ``files`` into at most ``jobs`` chunks of similar total size.++ Every name that reaches one file (repeated, respelled, symlinked) lands in the+ same chunk, in the caller's order, so no two workers ever rewrite one file and+ each worker sees exactly what the single serial pass saw. Largest first into+ the lightest chunk keeps a 50k-line file from landing beside a pile of others;+ each chunk keeps the caller's order, and chunks are ordered by their first file.+ """+ groups: dict[object, list[int]] = {}+ for index, name in enumerate(files):+ try:+ # Device and inode: the one identity case-insensitive filesystems and+ # hard links cannot fool. realpath is the fallback for a missing file.+ info = os.stat(name)+ key: object = (info.st_dev, info.st_ino)+ except OSError:+ key = os.path.normcase(os.path.realpath(name))+ groups.setdefault(key, []).append(index)++ def size(indices: list[int]) -> int:+ try:+ return os.path.getsize(files[indices[0]])+ except OSError:+ return 0++ bins: list[list[int]] = [[] for _ in range(max(1, min(jobs, len(groups))))]+ loads = [0] * len(bins)+ for indices in sorted(groups.values(), key = lambda group: (-size(group), group[0])):+ lightest = loads.index(min(loads))+ bins[lightest].extend(indices)+ loads[lightest] += size(indices)+ chunks = [sorted(chunk) for chunk in bins if chunk]+ chunks.sort(key = lambda chunk: chunk[0])+ return [[files[index] for index in chunk] for chunk in chunks]+++def run_chunks(head: list[str], chunks: list[list[str]]) -> int:+ """Run ``head + chunk`` for every chunk side by side; the first failure's code wins.++ One chunk runs as before, output straight through. Several write to temporary+ files that are replayed in chunk order, so the log reads the same from run to+ run. Anything that stops this process early (Ctrl+C, a timeout raising in this+ thread) kills the chunks still running instead of waiting for them.+ """+ if not chunks:+ return 0+ if len(chunks) == 1:+ return subprocess.run([*head, *chunks[0]]).returncode+ running: list[tuple[subprocess.Popen, object, object]] = []+ try:+ for chunk in chunks:+ out, err = tempfile.TemporaryFile(), tempfile.TemporaryFile()+ running.append((subprocess.Popen([*head, *chunk], stdout = out, stderr = err), out, err))+ codes = [proc.wait() for proc, _, _ in running]+ finally:+ for proc, _, _ in running:+ if proc.poll() is None:+ proc.kill()+ proc.wait()+ sys.stdout.flush()+ sys.stderr.flush()+ for _, out, err in running:+ for sink, stream in ((out, sys.stdout), (err, sys.stderr)):+ sink.seek(0)+ data = sink.read()+ sink.close()+ # Bytes where the stream has them: a path the locale codec cannot decode+ # would otherwise crash the replay after every file was formatted.+ if hasattr(stream, "buffer"):+ stream.buffer.write(data)+ stream.buffer.flush()+ else:+ stream.write(data.decode("utf-8", "replace"))+ stream.flush()+ return next((code for code in codes if code != 0), 0)++def main(argv: list[str]) -> int:files, error = parse_files(argv)if error is not None:@@ -156,22 +263,22 @@ def main(argv: list[str]) -> int:return 1spacing_script = HERE / "enforce_kwargs_spacing.py"+ chunks = chunk_files(files, job_count(len(files)))# Pre-ruff: normalize def-signature magic commas and strip the magic comma# from short multi-line asserts so ruff wraps/joins accordingly.- pre_cmd = [sys.executable, str(spacing_script), "--pre", *files]- pre_proc = subprocess.run(pre_cmd)- if pre_proc.returncode != 0:- return pre_proc.returncode+ pre_code = run_chunks([sys.executable, str(spacing_script), "--pre"], chunks)+ if pre_code != 0:+ return pre_code+ # One ruff over every file: it parallelises itself, and one per chunk would+ # stack its thread pools the way pre-commit's partitions did (#14174).ruff_cmd = [sys.executable, "-m", "ruff", "format", *files]ruff_proc = subprocess.run(ruff_cmd)if ruff_proc.returncode != 0:return ruff_proc.returncode- spacing_cmd = [sys.executable, str(spacing_script), *files]- spacing_proc = subprocess.run(spacing_cmd)- return spacing_proc.returncode+ return run_chunks([sys.executable, str(spacing_script)], chunks)if __name__ == "__main__":diff --git a/tests/test_formatter_fixed_point.py b/tests/test_formatter_fixed_point.pyindex f1cc0d90732..f62b10aa430 100644--- a/tests/test_formatter_fixed_point.py+++ b/tests/test_formatter_fixed_point.py@@ -25,6 +25,7 @@ if _SCRIPTS not in sys.path:import enforce_kwargs_spacing # noqa: E402from run_ruff_format import ( # noqa: E402CONFIG,+ JOBS_ENV,installed_ruff_version,pinned_ruff_version,version_mismatch,@@ -300,6 +301,8 @@ def _run_side_by_side(argvs: list[list[str]], log_dir: Path) -> list[tuple[int,"""Runs on the calling thread, not a pool, so a pytest-timeout raise reaches the kill loop."""log_dir.mkdir(parents = True, exist_ok = True)limit = max(1, min(len(argvs), os.cpu_count() or 1))+ # The batches are already one per CPU; a worker pool inside each would multiply that.+ env = {**os.environ, JOBS_ENV: "1"}results: list[tuple[int, str] | None] = [None] * len(argvs)running: dict[int, tuple[subprocess.Popen, Path]] = {}queued = list(enumerate(argvs))@@ -311,7 +314,9 @@ def _run_side_by_side(argvs: list[list[str]], log_dir: Path) -> list[tuple[int,# SIGALRM held across the launch so a timeout cannot land before the process is recorded.with _timeout_signal_held(), open(log, "wb") as sink:running[index] = (- subprocess.Popen(argv, stdout = sink, stderr = subprocess.STDOUT, **_OWN_GROUP),+ subprocess.Popen(+ argv, stdout = sink, stderr = subprocess.STDOUT, env = env, **_OWN_GROUP+ ),log,)for index, (proc, log) in list(running.items()):diff --git a/tests/test_run_ruff_format_jobs.py b/tests/test_run_ruff_format_jobs.pynew file mode 100644index 00000000000..43e4cd5c5bc--- /dev/null+++ b/tests/test_run_ruff_format_jobs.py@@ -0,0 +1,187 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+"""The Python passes run in bounded chunks around one ruff; chunking must lose, repeat or race nothing."""++from __future__ import annotations++import os+import re+import shutil+import subprocess+import sys+from pathlib import Path++import pytest++_SCRIPTS = str(Path(__file__).resolve().parent.parent / "scripts")+if _SCRIPTS not in sys.path:+ sys.path.insert(0, _SCRIPTS)++import run_ruff_format # noqa: E402+from run_ruff_format import JOBS_ENV, chunk_files, job_count, run_chunks # noqa: E402+++def _files(tmp_path: Path, sizes: list[int]) -> list[str]:+ names = []+ for index, size in enumerate(sizes):+ path = tmp_path / f"f{index}.py"+ path.write_text("x" * size, encoding = "utf-8")+ names.append(str(path))+ return names+++class TestJobCount:+ def test_few_files_stay_in_one_process(self):+ assert job_count(1, {}) == 1+ assert job_count(32, {}) == 1++ def test_bounded_by_the_cap_however_many_cpus(self, monkeypatch):+ monkeypatch.setattr(run_ruff_format, "usable_cpus", lambda: 192)+ assert job_count(100_000, {}) == run_ruff_format._MAX_JOBS++ def test_bounded_by_cpus(self, monkeypatch):+ monkeypatch.setattr(run_ruff_format, "usable_cpus", lambda: 2)+ assert job_count(100_000, {}) == 2++ def test_the_override_wins_and_a_bad_one_is_ignored(self):+ assert job_count(100_000, {JOBS_ENV: "1"}) == 1+ assert job_count(5, {JOBS_ENV: "3"}) == 3+ assert job_count(5, {JOBS_ENV: "0"}) == 1+ assert job_count(5, {JOBS_ENV: "many"}) == 1+++class TestChunkFiles:+ def test_every_file_once_in_caller_order_within_a_chunk(self, tmp_path):+ names = _files(tmp_path, [5, 1, 9, 3, 7, 2, 8])+ chunks = chunk_files(names, 3)+ flat = [name for chunk in chunks for name in chunk]+ assert sorted(flat) == sorted(names)+ for chunk in chunks:+ assert chunk == sorted(chunk, key = names.index)++ def test_every_name_for_one_file_shares_a_chunk_in_order(self, tmp_path):+ names = _files(tmp_path, [3, 4])+ # A string, not a Path: pathlib would fold the "." away and make this the same name.+ alias = os.path.join(os.path.dirname(names[0]), ".", os.path.basename(names[0]))+ assert alias != names[0]+ argv = [names[0], names[1], alias, names[0]]+ chunks = chunk_files(argv, 4)+ # Kept, not dropped: the serial pass rewrote the file once per name.+ assert sorted(name for chunk in chunks for name in chunk) == sorted(argv)+ assert [names[0], alias, names[0]] in chunks++ @pytest.mark.skipif(not hasattr(os, "symlink") or os.name == "nt", reason = "POSIX symlinks")+ def test_a_symlink_shares_its_targets_chunk(self, tmp_path):+ names = _files(tmp_path, [3, 4])+ link = tmp_path / "link.py"+ link.symlink_to(names[0])+ chunks = chunk_files([str(link), names[1], names[0]], 4)+ assert [str(link), names[0]] in chunks++ def test_the_largest_file_gets_a_chunk_to_itself(self, tmp_path):+ names = _files(tmp_path, [1000, 10, 10, 10, 10])+ chunks = chunk_files(names, 2)+ assert [names[0]] in chunks++ def test_deterministic_and_never_more_chunks_than_files(self, tmp_path):+ names = _files(tmp_path, [4, 4, 4])+ assert chunk_files(names, 8) == chunk_files(names, 8)+ assert len(chunk_files(names, 8)) == 3+ assert chunk_files(names, 1) == [names]+++class TestRunChunks:+ def test_output_in_chunk_order_and_the_first_failure_wins(self, capfd):+ head = [+ sys.executable,+ "-c",+ "import sys; print(sys.argv[1]); sys.exit(int(sys.argv[1]) % 3)",+ ]+ code = run_chunks(head, [["3"], ["4"], ["5"]])+ assert code == 1+ assert capfd.readouterr().out.split() == ["3", "4", "5"]++ def test_all_clean_is_zero(self):+ assert run_chunks([sys.executable, "-c", "pass"], [["a"], ["b"]]) == 0+++_ROOT = Path(__file__).resolve().parent.parent+_PINNED = run_ruff_format.pinned_ruff_version(run_ruff_format.CONFIG.read_text(encoding = "utf-8"))+++ run_ruff_format.installed_ruff_version() != _PINNED,+ reason = f"needs the pinned ruff {_PINNED}",+)+def test_chunked_and_serial_runs_write_the_same_bytes(tmp_path):+ """The hook takes the chunked path whenever it gets more than _FILES_PER_JOB files,+ which the fixed-point guard (pinned to one job) never exercises."""+ sources = sorted((_ROOT / "tests").glob("test_*.py"))[:70]+ assert len(sources) > 2 * run_ruff_format._FILES_PER_JOB+ outputs = {}+ for jobs in ("1", "3"):+ root = tmp_path / f"jobs{jobs}"+ root.mkdir()+ shutil.copy2(_ROOT / "pyproject.toml", root / "pyproject.toml")+ files, stripped = [], []+ for source in sources:+ target = root / source.name+ # Strip the kwarg spacing the post-pass has to put back.+ target.write_text(+ re.sub(r"(\w) = (\w)", r"\1=\2", source.read_text(encoding = "utf-8")),+ encoding = "utf-8",+ )+ files.append(str(target))+ stripped.append(target.read_bytes())+ proc = subprocess.run(+ [sys.executable, str(_ROOT / "scripts" / "run_ruff_format.py"), *files],+ env = {**os.environ, JOBS_ENV: jobs},+ capture_output = True,+ text = True,+ timeout = 300,+ )+ assert proc.returncode == 0, proc.stderr+ outputs[jobs] = [Path(name).read_bytes() for name in files]+ assert outputs[jobs] != stripped, "the formatter rewrote nothing, so this proves nothing"+ assert outputs["1"] == outputs["3"]+++class TestRunChunksStops:+ def test_an_interrupt_kills_the_chunks_still_running(self, tmp_path, monkeypatch):+ pid_file = tmp_path / "pids"+ head = [+ sys.executable,+ "-c",+ "import os, sys, time\n"+ f"open({str(pid_file)!r}, 'a').write(str(os.getpid()) + '\\n')\n"+ "time.sleep(120)\n",+ ]+ real_popen = subprocess.Popen+ started = []++ def popen(*args, **kwargs):+ started.append(real_popen(*args, **kwargs))+ if len(started) == 2:+ raise KeyboardInterrupt # as if Ctrl+C landed between two launches+ return started[-1]++ monkeypatch.setattr(run_ruff_format.subprocess, "Popen", popen)+ with pytest.raises(KeyboardInterrupt):+ run_chunks(head, [["a"], ["b"], ["c"]])+ # The first is in `running` and must have been killed; the second was started+ # by Popen but never recorded, so it is reaped here to keep the test clean.+ assert started[0].poll() is not None, "a chunk outlived the interrupt"+ started[1].kill()+ started[1].wait()++ def test_no_chunks_is_a_clean_no_op(self):+ assert run_chunks([sys.executable, "-c", "pass"], []) == 0++ def test_replay_works_without_a_byte_buffer(self, monkeypatch):+ import io++ out = io.StringIO()+ monkeypatch.setattr(sys, "stdout", out)+ head = [sys.executable, "-c", "import sys; print(sys.argv[1])"]+ assert run_chunks(head, [["x"], ["y"]]) == 0+ assert out.getvalue().split() == ["x", "y"]