Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Speed up the three slowest tests without changing what they check (#10867)

Daniel Han7bdace6155ac
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
index d80bb2d152e..3f60ab9a965 100644
--- a/.github/workflows/studio-backend-ci.yml
+++ b/.github/workflows/studio-backend-ci.yml
@@ -488,13 +488,15 @@ jobs:
# mirror the backend pytest job for the reasons written out there; the numbers are
# this suite's own.
#
- # 330s per test comes off this suite's measured tail, not off taste. Under -n 4 over
- # the whole discovered set the slowest is test_source_read_encoding.py::
- # test_checked_in_file_reads_name_an_encoding at 63.6s, then 48.3s, then a run of
- # ~31s docker-notebook durability cases. This step runs ~2.2x slower on a 4-vCPU
- # runner than on the box those came off, so call the worst legitimate test ~140s:
- # the cap is ~2.4x that. It is also the same 330 the backend pytest job settled on
- # against its own tail. Nothing here is near it, and a deadlock is named in minutes.
+ # 330s per test comes off this suite's measured tail, not off taste. The tail used
+ # to start at test_source_read_encoding.py::test_checked_in_file_reads_name_an_encoding
+ # at 63.6s, then 48.3s, then a run of ~31s docker-notebook durability cases. Those
+ # first two have since been made much cheaper and that scan now runs as 16 batches,
+ # so the docker-notebook cases are the tail: ~50s for the worst. This step runs
+ # ~2.2x slower on a 4-vCPU runner than on the box those came off, so call the worst
+ # legitimate test ~110s and the cap ~3x that. The headroom only grew, so 330 stands,
+ # and it is the same 330 the backend pytest job settled on against its own tail.
+ # Nothing here is near it, and a deadlock is named in minutes.
#
# --timeout arms inside each xdist worker, so a wedged CONTROLLER escapes it. 2100s
# bounds the session from outside, where it cannot: 1.25x the worst of the 40 runs
diff --git a/tests/python/test_docker_nb_refresh_state_durability.py b/tests/python/test_docker_nb_refresh_state_durability.py
index 369fbd91de5..efc2ff59277 100644
--- a/tests/python/test_docker_nb_refresh_state_durability.py
+++ b/tests/python/test_docker_nb_refresh_state_durability.py
@@ -17,22 +17,44 @@ write is never reached.
from __future__ import annotations
+import atexit
import os
import shutil
import subprocess
+import tempfile
from pathlib import Path
import pytest
-def _shared_setup_1(tmp_path):
- tpl, dest, up = _template(tmp_path), tmp_path / "dest", _upstream(tmp_path)
+def _build_shared_setup_1(root: Path):
+ tpl, dest, up = _template(root), root / "dest", _upstream(root)
dest.mkdir()
_run(tpl, dest, up, refresh = False)
_run(tpl, dest, up, refresh = True)
return dest, tpl, up
+# Five tests start from the same two sync runs over 500 notebooks, which is
+# ~7.4s each and ~37s of the file. Build it once per process and copy it in,
+# which is ~0.04s: nothing it writes embeds its own absolute path, so the copy
+# is the same tree at a different place. Each test still gets a private
+# directory it is free to mutate, so the isolation is unchanged.
+_SHARED_1: Path | None = None
+
+
+def _shared_setup_1(tmp_path):
+ global _SHARED_1
+ if _SHARED_1 is None:
+ prototype = Path(tempfile.mkdtemp(prefix = "unsloth-nbsync-"))
+ atexit.register(shutil.rmtree, prototype, True)
+ _build_shared_setup_1(prototype)
+ _SHARED_1 = prototype
+ root = tmp_path / "s1"
+ shutil.copytree(_SHARED_1, root, symlinks = True)
+ return root / "dest", root / "tpl", root / "up"
+
+
REPO_ROOT = Path(__file__).resolve().parents[2]
SYNC = REPO_ROOT / "docker" / "unsloth_sync_notebooks.sh"
diff --git a/tests/test_callback_signature_drift.py b/tests/test_callback_signature_drift.py
index a55660c696b..216e1085336 100644
--- a/tests/test_callback_signature_drift.py
+++ b/tests/test_callback_signature_drift.py
@@ -57,22 +57,41 @@ def _safe_parse(path: pathlib.Path):
key = path.resolve()
if key in _PARSE_CACHE:
return _PARSE_CACHE[key]
+ try:
+ text = path.read_text(encoding = "utf-8")
+ except (OSError, UnicodeDecodeError):
+ _PARSE_CACHE[key] = None
+ return None
+ # Both halves of the rule need this substring spelled out in the source: a
+ # producer holds `self._<name>_callbacks`, a consumer calls
+ # `add_<name>_callback(...)` or `register_<name>_callback(...)`, and the AST
+ # side matches those attribute names literally. So a file without it cannot
+ # contribute a producer or a registration, and parsing it only to walk it
+ # and find nothing is most of this test's runtime: 3137 files parsed where
+ # 102 can matter.
+ if "_callback" not in text:
+ _PARSE_CACHE[key] = None
+ return None
try:
import warnings as _w
with _w.catch_warnings():
# Suppress SyntaxWarning from third-party files with invalid escape sequences.
_w.simplefilter("ignore", SyntaxWarning)
- tree = ast.parse(path.read_text(encoding = "utf-8"))
+ tree = ast.parse(text)
except (SyntaxError, UnicodeDecodeError):
tree = None
_PARSE_CACHE[key] = tree
return tree
-def _callback_list_attrs_in_class(cls: ast.ClassDef) -> set[str]:
- """Find self._<name>_callbacks attributes assigned or appended-to inside cls."""
+def _callback_list_attrs_in_nodes(nodes) -> set[str]:
+ """self._<name>_callbacks attributes assigned or appended-to in a class.
+
+ Takes the already-walked nodes rather than the class, so the caller's walk
+ is shared instead of repeated.
+ """
found = set()
- for node in ast.walk(cls):
+ for node in nodes:
if isinstance(node, ast.Assign):
for t in node.targets:
if (
@@ -101,19 +120,26 @@ def _producer_arities(tree: ast.AST) -> dict[str, int]:
"""Return {cb_list_attr: max_arity} over all ``for cb in self._x_callbacks: cb(...)`` sites."""
out: dict[str, int] = {}
for cls in [n for n in ast.walk(tree) if isinstance(n, ast.ClassDef)]:
- cb_lists = _callback_list_attrs_in_class(cls)
+ # One walk per class, shared by both questions asked of it. Which
+ # `for cb in self.<x>:` loops a class contains does not depend on the
+ # name being asked about, and re-deriving that per name is what made
+ # this quadratic in classes declaring several lists.
+ nodes = list(ast.walk(cls))
+ cb_lists = _callback_list_attrs_in_nodes(nodes)
+ if not cb_lists:
+ continue
+ dispatch_loops = [
+ node
+ for node in nodes
+ if isinstance(node, ast.For)
+ and isinstance(node.iter, ast.Attribute)
+ and isinstance(node.iter.value, ast.Name)
+ and node.iter.value.id == "self"
+ and isinstance(node.target, ast.Name)
+ ]
for cb_list in cb_lists:
- for node in ast.walk(cls):
- if not isinstance(node, ast.For):
- continue
- if not (
- isinstance(node.iter, ast.Attribute)
- and isinstance(node.iter.value, ast.Name)
- and node.iter.value.id == "self"
- and node.iter.attr == cb_list
- ):
- continue
- if not isinstance(node.target, ast.Name):
+ for node in dispatch_loops:
+ if node.iter.attr != cb_list:
continue
cb_name = node.target.id
for inner in ast.walk(node):
@@ -184,22 +210,24 @@ def check_registrations(
tree = _safe_parse(src)
if tree is None:
continue
+ # One walk, collecting both. The definitions still have to be
+ # complete before any call is judged, so the calls are held and
+ # processed after, exactly as the second walk used to do.
defs_by_name: dict[str, ast.AST] = {}
+ registrations: list[ast.Call] = []
for node in ast.walk(tree):
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
defs_by_name[node.name] = node
- if isinstance(node, ast.Assign):
+ elif isinstance(node, ast.Assign):
if (
isinstance(node.value, ast.Lambda)
and len(node.targets) == 1
and isinstance(node.targets[0], ast.Name)
):
defs_by_name[node.targets[0].id] = node.value
- for call in ast.walk(tree):
- if not isinstance(call, ast.Call):
- continue
- if not isinstance(call.func, ast.Attribute):
- continue
+ elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ registrations.append(node)
+ for call in registrations:
cb_list = _registration_attr_to_list(call.func.attr)
if cb_list is None:
continue
diff --git a/tests/test_source_read_encoding.py b/tests/test_source_read_encoding.py
index 6426d82cfbe..9135e39ac61 100644
--- a/tests/test_source_read_encoding.py
+++ b/tests/test_source_read_encoding.py
@@ -61,6 +61,8 @@ import os
import subprocess
from pathlib import Path
+import pytest
+
TESTS = Path(__file__).resolve().parent
REPO = TESTS.parent
# Both trees ship to Windows contributors, and separate CI jobs collect them (repo-cpu-tests and the studio-backend
@@ -109,6 +111,36 @@ def _tracked_test_files(repo: Path):
SOURCES = _tracked_test_files(REPO)
if SOURCES is None:
SOURCES = _walked_test_files(REPO)
+
+# The rules below re-walk the same subtrees many times over: a fixed-point pass
+# repeats until it stops learning, and each pass walks every function again.
+# That is ~170M iter_child_nodes calls across the tree, and the traversal, not
+# the rules, is most of the runtime. The AST is never mutated while a file is
+# scanned, so a walk of a node is the same tuple every time it is asked for.
+#
+# Keyed on id(), which is only sound while the node is alive, so each entry
+# keeps a reference to the node it was keyed on and _scan clears both caches
+# when it is done with a file. The results are identical either way; this only
+# stops the work being repeated.
+_WALKS: dict = {}
+_KIDS: dict = {}
+_CONSUMED: dict = {}
+
+
+def _walk(node):
+ cached = _WALKS.get(id(node))
+ if cached is None:
+ cached = _WALKS[id(node)] = (node, tuple(ast.walk(node)))
+ return cached[1]
+
+
+def _kids(node):
+ cached = _KIDS.get(id(node))
+ if cached is None:
+ cached = _KIDS[id(node)] = (node, tuple(ast.iter_child_nodes(node)))
+ return cached[1]
+
+
GUARDED_METHODS = {"read_text", "write_text"}
# Openers that are somebody else's are recognised by the file's own imports rather than a fixed list, so `import tarfile
# as tf` and `from PIL import Image` are both covered without naming either. These wrap their stream in a TextIOWrapper
@@ -329,10 +361,18 @@ def _import_time_calls(tree: ast.Module):
body = list(helper.body)
_collect(body) # a def nested here is now callable
frontier.append(body)
- stack.extend(ast.iter_child_nodes(node))
+ stack.extend(_kids(node))
def _eagerly_consumed(tree: ast.Module) -> set:
+ """Both rules ask for this, so compute it once per file. See _walk."""
+ cached = _CONSUMED.get(id(tree))
+ if cached is None:
+ cached = _CONSUMED[id(tree)] = (tree, _eagerly_consumed_uncached(tree))
+ return cached[1]
+
+
+def _eagerly_consumed_uncached(tree: ast.Module) -> set:
"""Nodes whose lazy value is drained right where it is written.
Covers both things that defer: a generator expression, and a call to a
@@ -342,7 +382,7 @@ def _eagerly_consumed(tree: ast.Module) -> set:
# `texts = (p.read_text() for p in ...)` then `list(texts)` consumes the generator through a name, so the name has
# to lead back to it.
named: dict = {}
- for node in ast.walk(tree):
+ for node in _walk(tree):
if isinstance(node, ast.Assign) and len(node.targets) == 1:
target = node.targets[0]
if isinstance(target, ast.Name) and isinstance(node.value, ast.GeneratorExp):
@@ -354,14 +394,14 @@ def _eagerly_consumed(tree: ast.Module) -> set:
return node
consumed = set()
- for node in ast.walk(tree):
+ for node in _walk(tree):
if isinstance(node, ast.Call) and _is_eager_consumer(node.func):
consumed.update(id(_resolve(a)) for a in node.args)
consumed.update(id(_resolve(k.value)) for k in node.keywords)
elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)):
consumed.add(id(_resolve(node.iter))) # the loop pulls every item
# `list(enumerate(_paths()))` drains _paths() as well, one wrapper down.
- by_id = {id(n): n for n in ast.walk(tree)}
+ by_id = {id(n): n for n in _walk(tree)}
queue = [by_id[i] for i in list(consumed) if i in by_id]
while queue:
node = queue.pop()
@@ -382,8 +422,7 @@ def _temp_rooted_names(tree: ast.Module) -> set:
if value is None:
continue
if any(
- isinstance(n, ast.Call) and _callee_name(n.func) in TEMP_FACTORIES
- for n in ast.walk(value)
+ isinstance(n, ast.Call) and _callee_name(n.func) in TEMP_FACTORIES for n in _walk(value)
):
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
names.update(t.id for t in targets if isinstance(t, ast.Name))
@@ -419,7 +458,7 @@ def _is_generator(func) -> bool:
return True
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
continue
- stack.extend(ast.iter_child_nodes(node))
+ stack.extend(_kids(node))
return False
@@ -438,8 +477,7 @@ def _module_level_names(tree: ast.Module) -> set:
def _is_temp(value) -> bool:
return value is not None and any(
- isinstance(n, ast.Call) and _callee_name(n.func) in TEMP_FACTORIES
- for n in ast.walk(value)
+ isinstance(n, ast.Call) and _callee_name(n.func) in TEMP_FACTORIES for n in _walk(value)
)
names = set()
@@ -471,7 +509,7 @@ def _local_names(func) -> set:
for extra in (args.vararg, args.kwarg):
if extra is not None:
names.add(extra.arg)
- stack = list(ast.iter_child_nodes(func))
+ stack = list(_kids(func))
while stack:
node = stack.pop()
if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)):
@@ -486,7 +524,7 @@ def _local_names(func) -> set:
names.add(node.id)
elif isinstance(node, (ast.Import, ast.ImportFrom)):
names.update((a.asname or a.name).split(".")[0] for a in node.names)
- stack.extend(ast.iter_child_nodes(node))
+ stack.extend(_kids(node))
return names
@@ -503,7 +541,7 @@ def _imported_names(node) -> dict:
local `from PIL.Image import open` turn off the builtin check everywhere.
"""
bound = {}
- stack = list(ast.iter_child_nodes(node))
+ stack = list(_kids(node))
while stack:
item = stack.pop()
if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
@@ -511,7 +549,7 @@ def _imported_names(node) -> dict:
if isinstance(item, (ast.Import, ast.ImportFrom)):
bound.update(_import_bindings(item))
else:
- stack.extend(ast.iter_child_nodes(item))
+ stack.extend(_kids(item))
return bound
@@ -553,7 +591,7 @@ def _imports_at_each_call(tree: ast.Module) -> dict:
for child in arm:
walk(child, inner)
return
- for child in ast.iter_child_nodes(node):
+ for child in _kids(node):
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
walk(child, dict(visible)) # its own scope, so its own copy
else:
@@ -743,7 +781,7 @@ def _class_path_attrs(tree: ast.Module, module_names: set) -> set:
reads seven real source files exactly that way.
"""
attrs, mixed = set(), set()
- for node in ast.walk(tree):
+ for node in _walk(tree):
if not isinstance(node, ast.ClassDef):
continue
for stmt in node.body:
@@ -811,7 +849,7 @@ def _checked_in_locals(
assignments = []
targets = set()
bad = set()
- for node in ast.walk(func):
+ for node in _walk(func):
paired = False
if isinstance(node, ast.Assign) and len(node.targets) == 1:
target, value = node.targets[0], node.value
@@ -826,7 +864,7 @@ def _checked_in_locals(
targets.add(id(name_node))
if not _reads_itself(name_node.id, bound):
assignments.append((name_node.id, bound))
- for node in ast.walk(func):
+ for node in _walk(func):
# A with-as or an augassign says nothing about the value it binds.
if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
if id(node) not in targets:
@@ -910,7 +948,7 @@ def _checked_in_params(tree: ast.Module, module_names: set) -> set:
defs_in: dict = {}
def _index(node, scope):
- for child in ast.iter_child_nodes(node):
+ for child in _kids(node):
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
defs_in.setdefault(id(scope), {}).setdefault(child.name, child)
scope_of[id(child)] = scope
@@ -937,7 +975,7 @@ def _checked_in_params(tree: ast.Module, module_names: set) -> set:
def _mark(node, owning):
if isinstance(node, ast.Call):
owner[id(node)] = owning
- for child in ast.iter_child_nodes(node):
+ for child in _kids(node):
nested = isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef))
_mark(child, child if nested else owning)
@@ -949,7 +987,7 @@ def _checked_in_params(tree: ast.Module, module_names: set) -> set:
def _mark_class(node, cls):
if isinstance(node, ast.Call):
in_class[id(node)] = cls
- for child in ast.iter_child_nodes(node):
+ for child in _kids(node):
_mark_class(child, child if isinstance(child, ast.ClassDef) else cls)
_mark_class(tree, None)
@@ -972,7 +1010,7 @@ def _checked_in_params(tree: ast.Module, module_names: set) -> set:
# What the calling function itself can prove, recomputed each pass so a parameter resolved last time can feed a
# local this time.
scope: dict = {}
- for call in ast.walk(tree):
+ for call in _walk(tree):
if not isinstance(call, ast.Call):
continue
caller = owner.get(id(call))
@@ -1069,7 +1107,7 @@ def _checked_in_path_calls(
expr, module_names, shadowed, derived, attrs
):
yield node
- for child in ast.iter_child_nodes(node):
+ for child in _kids(node):
yield from visit(child, shadowed, derived)
yield from visit(tree, frozenset())
@@ -1203,6 +1241,17 @@ def _offender(call: ast.Call, modules = NO_MODULES) -> str | None:
def _scan(tree: ast.Module, rel: str):
"""Offenders from both rules, reported once each and in source order."""
+ try:
+ yield from _scan_one(tree, rel)
+ finally:
+ # This file's nodes are about to become unreachable, and a later file's
+ # node could then land on one of their ids.
+ _WALKS.clear()
+ _KIDS.clear()
+ _CONSUMED.clear()
+
+
+def _scan_one(tree: ast.Module, rel: str):
modules = _imported_names(tree)
visible_at = _imports_at_each_call(tree)
calls = {id(c): c for c in _import_time_calls(tree)}
@@ -1227,9 +1276,22 @@ def _scan(tree: ast.Module, rel: str):
yield f"{rel}:{call.lineno}: {name}"
-def test_checked_in_file_reads_name_an_encoding():
+# Scanning every file was one test, and a single test is one xdist worker, so
+# it set the floor for the whole suite however many workers were free. The
+# files are independent, so the same scan splits into batches that run in
+# parallel. Every file is still scanned exactly once, by exactly one batch.
+_BATCHES = 16
+
+
+def _batches():
+ ordered = sorted(SOURCES)
+ return [ordered[i::_BATCHES] for i in range(_BATCHES)]
+
+
[email protected]("batch", range(_BATCHES))
+def test_checked_in_file_reads_name_an_encoding(batch: int):
offenders = []
- for path in sorted(SOURCES):
+ for path in _batches()[batch]:
tree = ast.parse(path.read_text(encoding = "utf-8"), filename = str(path))
offenders.extend(_scan(tree, path.relative_to(REPO).as_posix()))
assert offenders == [], (