Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pin tokenizers with transformers so Apple Silicon keeps Train and Export (#10431)

Daniel Han9735ec096ec8
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
index 23b7cc1d0da..5f9d86202c0 100644
--- a/.github/workflows/studio-backend-ci.yml
+++ b/.github/workflows/studio-backend-ci.yml
@@ -439,11 +439,11 @@ jobs:
# --deselect:
# - test_model_registration / test_all_model_registration:
# hit huggingface_hub for live model existence checks.
- # - test_autoconfig_works_with_no_torch_runtime / test_autoconfig_succeeds:
- # fail because no-torch-runtime.txt does not pin tokenizers
- # and the latest tokenizers (0.23.1) is incompatible with the
- # transformers it resolves to. Tracked separately; this is a
- # real bug in the no-torch install path, not a CI issue.
+ # test_autoconfig_works_with_no_torch_runtime / test_autoconfig_succeeds
+ # used to be deselected here too, for a tokenizers pin no-torch-runtime.txt
+ # was missing. #5359 added that pin and both now pass, so the lines are gone
+ # -- they were redundant anyway, since the -m filter above already excludes
+ # their `e2e` class marker, and the reason they gave outlived the bug.
#
# -n 4: 806.1s -> 219.7s locally on the same 4-worker shape as the runner, with an
# identical 2-failed / 8066-passed / 178-skipped / 40-subtest result. The three pytest
@@ -465,9 +465,7 @@ jobs:
--ignore=tests/version_compat \
-m 'not server and not e2e' \
--deselect tests/test_model_registry.py::test_model_registration \
- --deselect tests/test_model_registry.py::test_all_model_registration \
- --deselect 'tests/python/test_tokenizers_and_torch_constraint.py::TestE2ETokenizersFix::test_autoconfig_works_with_no_torch_runtime' \
- --deselect 'tests/python/test_tokenizers_and_torch_constraint.py::TestE2EFullNoTorchSandbox::test_autoconfig_succeeds'
+ --deselect tests/test_model_registry.py::test_all_model_registration
- name: Hardware-spoof tests (state-sensitive, run in isolation)
env:
diff --git a/studio/backend/requirements/extras-no-deps.txt b/studio/backend/requirements/extras-no-deps.txt
index 9790357c539..fcc4152981b 100644
--- a/studio/backend/requirements/extras-no-deps.txt
+++ b/studio/backend/requirements/extras-no-deps.txt
@@ -17,6 +17,16 @@ torch-c-dlpack-ext==0.1.5
# unconditional pin here would reject 3.9 outright and undo that file's fallback.
sentence_transformers==5.2.0; python_version >= "3.10"
sentence_transformers==5.1.2; python_version < "3.10"
+# Pinned with transformers, not after it: --no-deps skips transformers' own
+# tokenizers requirement, so naming only one half of the pair leaves whatever the
+# earlier with-deps resolve happened to pick. transformers ENFORCES this window at
+# import, so a stranded tokenizers is not a latent mismatch -- every
+# `import transformers`, and so every `import mlx_lm`, raises, and Apple Silicon
+# drops to chat-only with Train and Export off. 5.5.0 and 4.57.6 declare the same
+# window, so one unconditional line covers both pins below; bump all three together.
+# 0.23.0 was never published, so this resolves to 0.22.2, whose cp39-abi3 wheels
+# cover every platform and interpreter this file reaches.
+tokenizers>=0.22.0,<=0.23.0
transformers==5.5.0; python_version >= "3.10"
transformers==4.57.6; python_version < "3.10"
# No macOS x86_64 wheel at any version, so uv falls back to an sdist that shells out to
diff --git a/studio/backend/requirements/no-torch-runtime.txt b/studio/backend/requirements/no-torch-runtime.txt
index 3e72ae2d873..190a7c1c0f8 100644
--- a/studio/backend/requirements/no-torch-runtime.txt
+++ b/studio/backend/requirements/no-torch-runtime.txt
@@ -75,8 +75,11 @@ anyio>=3.0,<4.14.0 # 4.14 asyncio cancel-scope RuntimeError on Py3
sniffio==1.3.1
h11==0.16.0
-# Transformers 5.5 supports tokenizers 0.22.x through 0.23.0.
-tokenizers<=0.23.0
+# Transformers 5.5 supports tokenizers 0.22.x through 0.23.0. Spelled as the full
+# window, matching extras-no-deps.txt and constraints.txt, so one canonical pair
+# exists repo-wide (#5359 added the cap here; the torch path lacked it until the
+# same break reached Apple Silicon).
+tokenizers>=0.22.0,<=0.23.0
transformers==5.5.0; python_version >= "3.10"
transformers==4.57.6; python_version < "3.10"
trl>=0.18.2,!=0.19.0,<=0.24.0
diff --git a/studio/backend/requirements/single-env/constraints.txt b/studio/backend/requirements/single-env/constraints.txt
index 9d35f4b212e..f3b4a7dff6b 100644
--- a/studio/backend/requirements/single-env/constraints.txt
+++ b/studio/backend/requirements/single-env/constraints.txt
@@ -2,6 +2,11 @@
# MiniMax-H3's merged Diffusers workflow needs the current Hugging Face stack.
transformers==5.5.0; python_version >= "3.10"
transformers==4.57.6; python_version < "3.10"
+# The window both transformers pins above declare and enforce at import. Redundant
+# while they are pinned, and kept for the same reason as the anyio cap below: a
+# global cap is what stops a later with-deps step, or a future relaxation of the
+# transformers pin, from pairing them with a tokenizers transformers rejects.
+tokenizers>=0.22.0,<=0.23.0
trl==0.23.1
huggingface-hub>=1.23.0,<2.0; python_version >= "3.10"
huggingface-hub==0.36.2; python_version < "3.10"
diff --git a/studio/backend/requirements/single-env/overrides-darwin-arm64.txt b/studio/backend/requirements/single-env/overrides-darwin-arm64.txt
index 7ecc7e9f4f8..90e5ce8d608 100644
--- a/studio/backend/requirements/single-env/overrides-darwin-arm64.txt
+++ b/studio/backend/requirements/single-env/overrides-darwin-arm64.txt
@@ -1,7 +1,25 @@
-# Keep mlx-vlm / mlx-lm on the same Transformers floor as the main Unsloth
+# Keep mlx-vlm / mlx-lm on the same Transformers version as the main Unsloth
# environment when uv applies their dependency metadata.
-transformers>=5.5.0; python_version >= "3.10"
-transformers>=4.57.6; python_version < "3.10"
+#
+# A window, not a floor. An override replaces EVERY requirement on the package,
+# including unsloth's own transformers<=5.5.0 cap in pyproject, and install.sh's
+# core install is the one uv command that does not also pass constraints.txt. A
+# bare floor therefore let that resolve pick transformers 5.16.1 (the first
+# release wanting tokenizers>=0.23.1) with tokenizers 0.23.2 on macOS arm64;
+# extras-no-deps.txt then pinned transformers back to 5.5.0 --no-deps and left
+# tokenizers stranded, so every `import transformers`, and so every
+# `import mlx_lm`, raised and Apple Silicon came up chat-only.
+#
+# Unlike the huggingface_hub case below, this cannot go unsatisfiable: that one
+# replaced other packages' requirements ON hub while a direct requirement still
+# had to be met, whereas here no requirement on transformers survives to
+# contradict the window. A future mlx-vlm needing a newer transformers gets this
+# one installed instead, which is what an override is for.
+#
+# Keep in lockstep with constraints.txt; tests/studio/install/
+# test_transformers_tokenizers_pair.py fails if the two disagree.
+transformers>=5.5.0,<=5.5.0; python_version >= "3.10"
+transformers>=4.57.6,<=4.57.6; python_version < "3.10"
# mlx-vlm / mlx-lm pull anyio>=4.14, which fights the constraints.txt cap (needed
# for the 4.14 Python-3.13 streaming cancel-scope RuntimeError, #6483). The -c
diff --git a/studio/setup.ps1 b/studio/setup.ps1
index 9accb5fe264..5b6c31359c5 100644
--- a/studio/setup.ps1
+++ b/studio/setup.ps1
@@ -4629,6 +4629,36 @@ sys.exit(0 if (major, minor) >= (4, 14) else 1)
substep "anyio >=4.14 found (#6483) -- forcing dependency pass to repair..." "Cyan"
$SkipPythonDeps = $false
}
+ # Same shape, same reason, and the sibling of the probe setup.sh runs: a venv
+ # installed before the tokenizers pin can hold a tokenizers the installed
+ # transformers rejects at import, which takes down every `import transformers`
+ # and so the whole model stack, while $_PkgName itself is current. Without this
+ # the fast path reports "up to date" and repairs nothing. Ask the metadata, not
+ # an import: the import is what is broken. Any unreadable half exits 1 and
+ # changes nothing.
+ $_tokenizersBad = $false
+ try {
+ & python -c "
+import sys
+from importlib.metadata import PackageNotFoundError, requires, version
+try:
+ from packaging.requirements import Requirement
+ installed = version('tokenizers')
+ windows = [
+ req.specifier
+ for req in (Requirement(raw) for raw in (requires('transformers') or []))
+ if req.name == 'tokenizers' and req.marker is None
+ ]
+except (PackageNotFoundError, ImportError, ValueError, IndexError):
+ sys.exit(1)
+sys.exit(0 if windows and installed not in windows[0] else 1)
+" 2>$null
+ if ($LASTEXITCODE -eq 0) { $_tokenizersBad = $true }
+ } catch {}
+ if ($_tokenizersBad) {
+ substep "installed transformers rejects the installed tokenizers -- forcing dependency pass to repair..." "Cyan"
+ $SkipPythonDeps = $false
+ }
# An interrupted install leaves $_PkgName current while studio.txt
# never finished, so the compare above says "up to date" and update --
# plus the desktop Repair button -- no-ops on a venv that cannot boot.
diff --git a/studio/setup.sh b/studio/setup.sh
index 09325c75267..5b054b382c6 100755
--- a/studio/setup.sh
+++ b/studio/setup.sh
@@ -1888,6 +1888,30 @@ sys.exit(0 if (major, minor) >= (4, 14) else 1)
substep "anyio >=4.14 found (#6483) -- forcing dependency pass to repair..."
_SKIP_PYTHON_DEPS=false
fi
+ # Same shape, same reason: a venv installed before the tokenizers pin can
+ # hold a tokenizers the installed transformers rejects at import, which
+ # takes down every `import transformers` and so the whole MLX stack, while
+ # $_PKG_NAME itself is current. Without this the fast path reports "up to
+ # date" and repairs nothing. Ask the metadata, not an import: the import is
+ # what is broken. Any unreadable half exits 1 and changes nothing.
+ if "$VENV_DIR/bin/python" -c "
+import sys
+from importlib.metadata import PackageNotFoundError, requires, version
+try:
+ from packaging.requirements import Requirement
+ installed = version('tokenizers')
+ windows = [
+ req.specifier
+ for req in (Requirement(raw) for raw in (requires('transformers') or []))
+ if req.name == 'tokenizers' and req.marker is None
+ ]
+except (PackageNotFoundError, ImportError, ValueError, IndexError):
+ sys.exit(1)
+sys.exit(0 if windows and installed not in windows[0] else 1)
+" 2>/dev/null; then
+ substep "installed transformers rejects the installed tokenizers -- forcing dependency pass to repair..."
+ _SKIP_PYTHON_DEPS=false
+ fi
# An interrupted install leaves $_PKG_NAME current while studio.txt
# never finished, so the compare above says "up to date" and update --
# plus the desktop Repair button -- no-ops on a venv that cannot boot.
diff --git a/tests/python/test_tokenizers_and_torch_constraint.py b/tests/python/test_tokenizers_and_torch_constraint.py
index 6c884bc2f3a..d09b8c15182 100644
--- a/tests/python/test_tokenizers_and_torch_constraint.py
+++ b/tests/python/test_tokenizers_and_torch_constraint.py
@@ -566,12 +566,21 @@ class TestE2ETokenizersFix:
assert result.returncode != 0, "torch should NOT be importable"
def test_negative_control_no_tokenizers(self, tmp_path):
- """Without the tokenizers line, AutoConfig must fail (negative control)."""
+ """Without the tokenizers line, AutoConfig must fail (negative control).
+
+ Dropped by package name, not by exact text. This filter used to compare the whole
+ line against "tokenizers", which matched the bare entry #4748 added and matched
+ nothing once #5359 gave it a version bound: the control then installed tokenizers
+ and asserted a failure that could not happen, so it failed on every tree from that
+ commit onward. A name-based match survives the next bound too.
+ """
venv = self._create_venv(tmp_path, "neg-ctrl", "3.12")
req_no_tokenizers = tmp_path / "no-tokenizers.txt"
req_no_tokenizers.write_text(
"\n".join(
- line for line in _read(_NO_TORCH_RT).splitlines() if line.strip() != "tokenizers"
+ line
+ for line in _read(_NO_TORCH_RT).splitlines()
+ if not re.match(r"tokenizers([^A-Za-z0-9._-]|$)", line.strip())
),
encoding = "utf-8",
)
diff --git a/tests/studio/install/test_transformers_tokenizers_pair.py b/tests/studio/install/test_transformers_tokenizers_pair.py
new file mode 100644
index 00000000000..8a6b5823ce9
--- /dev/null
+++ b/tests/studio/install/test_transformers_tokenizers_pair.py
@@ -0,0 +1,341 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""transformers and tokenizers have to be pinned together, or Apple Silicon loses Train.
+
+transformers declares a tokenizers window and ENFORCES it at import, so a venv holding
+a transformers that rejects its tokenizers is not subtly wrong: every
+`import transformers` raises, `import mlx_lm` with it, mlx_repair.mlx_stack_blockers()
+reports a blocker, and Studio comes up chat-only with Train and Export disabled.
+
+Two ways that happened, both covered here:
+
+* extras-no-deps.txt pinned `transformers==5.5.0` under --no-deps, which skips
+ transformers' own tokenizers requirement, so whatever the earlier with-deps resolve
+ had picked stayed. Half a pair is worse than no pin.
+* install.sh's core phase runs with UV_OVERRIDE=overrides-darwin-arm64.txt and without
+ single-env/constraints.txt. An override REPLACES every requirement on a package, so
+ its unbounded `transformers>=5.5.0` also replaced pyproject's `<=5.5.0` cap. From
+ transformers 5.16.0 (2026-08-26), the first release wanting `tokenizers>=0.23.1`, that
+ resolve began landing tokenizers 0.23.2 on macOS arm64, and step 3b then downgraded
+ transformers alone.
+
+No unsloth commit caused the second one: a third-party release walked into an unbounded
+override. So the resolver test below checks the pair a fresh macOS arm64 install would
+END with, rather than any version number a diff would show.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import pathlib
+import shutil
+import subprocess
+import urllib.error
+import urllib.request
+
+import pytest
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.utils import canonicalize_name
+from packaging.version import Version
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
+REQ_ROOT = REPO_ROOT / "studio" / "backend" / "requirements"
+SINGLE_ENV = REQ_ROOT / "single-env"
+EXTRAS_NO_DEPS = REQ_ROOT / "extras-no-deps.txt"
+CONSTRAINTS = SINGLE_ENV / "constraints.txt"
+DARWIN_OVERRIDES = SINGLE_ENV / "overrides-darwin-arm64.txt"
+
+# Files uv never installs FROM: constraints only narrow a resolve, overrides only replace
+# requirements. They carry no pairing duty, which is why they are exempt from the "pin the
+# pair" rule -- and why the override is the one file that can silently widen it.
+NON_INSTALLING = {"constraints.txt", "overrides-darwin-arm64.txt", "overrides.txt"}
+
+# The window each pinned transformers release declares. Keyed by version so that bumping a
+# pin fails here until someone confirms the new release's window, rather than silently
+# widening the assertion. Both current pins declare the same one.
+DECLARED_TOKENIZERS_WINDOW = {
+ "5.5.0": ">=0.22.0,<=0.23.0",
+ "4.57.6": ">=0.22.0,<=0.23.0",
+}
+
+# The first tokenizers release outside that window, i.e. the version a stranded venv ends
+# up on today. Named rather than computed so the negative control cannot drift with PyPI.
+STRANDED_TOKENIZERS = "0.23.2"
+
+
+def _requirements(path: pathlib.Path) -> list[Requirement]:
+ """Parsed requirement lines; comments, flags and unparseable lines dropped."""
+ out: list[Requirement] = []
+ for line in path.read_text(encoding = "utf-8").splitlines():
+ text = line.split("#", 1)[0].strip()
+ if not text or text.startswith("-"):
+ continue
+ try:
+ out.append(Requirement(text))
+ except InvalidRequirement:
+ continue
+ return out
+
+
+def _named(reqs: list[Requirement], name: str) -> list[Requirement]:
+ """Canonicalized match, so sentence-transformers never counts as transformers."""
+ return [req for req in reqs if canonicalize_name(req.name) == canonicalize_name(name)]
+
+
+def _pinned_versions(reqs: list[Requirement], name: str) -> list[str]:
+ return [
+ spec.version
+ for req in _named(reqs, name)
+ for spec in req.specifier
+ if spec.operator == "=="
+ ]
+
+
+def _installing_files() -> list[pathlib.Path]:
+ return [
+ path
+ for path in sorted(REQ_ROOT.rglob("*.txt"))
+ if path.name not in NON_INSTALLING and not path.name.startswith(".")
+ ]
+
+
+def test_every_installing_file_that_pins_transformers_also_bounds_tokenizers():
+ """The pair has to be named in the same file, because the file is the unit that gets
+ installed. extras-no-deps.txt is installed with --no-deps, so nothing else in the run
+ will supply the other half."""
+ offenders = []
+ for path in _installing_files():
+ reqs = _requirements(path)
+ if not _named(reqs, "transformers"):
+ continue
+ if not _named(reqs, "tokenizers"):
+ offenders.append(str(path.relative_to(REPO_ROOT)))
+ assert not offenders, (
+ f"these files pin transformers without bounding tokenizers: {offenders}. "
+ f"transformers enforces its tokenizers window at import, so pinning one half "
+ f"leaves the venv one `import transformers` away from chat-only."
+ )
+
+
+def test_the_tokenizers_bound_matches_the_pinned_transformers_release():
+ for path in _installing_files():
+ reqs = _requirements(path)
+ pins = _pinned_versions(reqs, "transformers")
+ if not pins:
+ continue
+ bounds = _named(reqs, "tokenizers")
+ assert len(bounds) == 1, f"{path.name}: expected one tokenizers line, got {bounds}"
+ window = bounds[0].specifier
+ for pinned in pins:
+ assert pinned in DECLARED_TOKENIZERS_WINDOW, (
+ f"{path.name} pins transformers=={pinned}, which this test does not know. "
+ f"Add its declared tokenizers window to DECLARED_TOKENIZERS_WINDOW and "
+ f"move the pins together."
+ )
+ declared = SpecifierSet(DECLARED_TOKENIZERS_WINDOW[pinned])
+ assert window == declared, (
+ f"{path.name}: tokenizers{window} does not match the window "
+ f"transformers=={pinned} declares ({declared})"
+ )
+
+
+def test_the_tokenizers_line_covers_both_python_branches():
+ """The transformers pins split on python_version; the tokenizers line must not, unless
+ it splits the same way. An unconditional line is correct only while both pinned
+ releases declare the same window, which the test above is what enforces."""
+ for path in _installing_files():
+ reqs = _requirements(path)
+ if not _pinned_versions(reqs, "transformers"):
+ continue
+ markers = [req.marker for req in _named(reqs, "tokenizers")]
+ assert markers and all(marker is None for marker in markers), (
+ f"{path.name}: the tokenizers bound carries a marker ({markers}) while the "
+ f"transformers pins split on python_version. Either branch left uncovered "
+ f"installs transformers without its tokenizers."
+ )
+
+
+def test_no_requirements_file_admits_a_tokenizers_outside_the_window():
+ """Including the non-installing files: a constraint or override that admits 0.23.1+
+ lets a later resolve walk tokenizers past what the pinned transformers accepts."""
+ offenders = {}
+ for path in sorted(REQ_ROOT.rglob("*.txt")):
+ if path.name.startswith("."):
+ continue
+ for req in _named(_requirements(path), "tokenizers"):
+ if STRANDED_TOKENIZERS in req.specifier:
+ offenders[str(path.relative_to(REPO_ROOT))] = str(req)
+ assert not offenders, (
+ f"these files admit tokenizers {STRANDED_TOKENIZERS}, which the pinned "
+ f"transformers rejects at import: {offenders}"
+ )
+
+
+def test_the_darwin_override_admits_only_the_pinned_transformers():
+ """The override is the one file that can widen what the core install resolves, because
+ install.sh runs that one uv command without constraints.txt and an override replaces
+ every requirement on the package -- including pyproject's own cap. Bounded to exactly
+ what constraints.txt pins, it cannot drift into a fourth independent pin."""
+ pinned = {
+ str(req.marker): version
+ for req in _named(_requirements(CONSTRAINTS), "transformers")
+ for version in ([spec.version for spec in req.specifier if spec.operator == "=="] or [None])
+ }
+ assert pinned, "constraints.txt no longer pins transformers"
+ for req in _named(_requirements(DARWIN_OVERRIDES), "transformers"):
+ version = pinned.get(str(req.marker))
+ assert version is not None, (
+ f"the override's transformers line for marker {req.marker} has no counterpart "
+ f"in constraints.txt; the two must move together"
+ )
+ assert version in req.specifier, (
+ f"the override ({req.specifier}) excludes the version constraints.txt pins "
+ f"({version})"
+ )
+ # Derived from the pin rather than listed, so this keeps asking the question after
+ # the pin moves. A fixed pair of releases goes vacuous the moment the pin passes
+ # them: with 5.16.1 pinned, neither 5.16.1 nor 5.15.1 is above it and nothing is
+ # tested. The two named releases are still probed, since they are the ones that
+ # actually shipped the tokenizers window change.
+ pin = Version(version)
+ higher = {f"{pin.major}.{pin.minor + 1}.0", f"{pin.major + 1}.0.0", "5.16.1", "5.15.1"}
+ excess = sorted(
+ candidate
+ for candidate in higher
+ if Version(candidate) > pin and candidate in req.specifier
+ )
+ assert not excess, (
+ f"the override admits transformers {excess}, above the pinned {version}. On "
+ f"macOS arm64 that is what install.sh's core phase installs, and a later "
+ f"--no-deps pin then strands tokenizers where the newer release put it."
+ )
+
+
+def test_the_checker_rejects_the_pair_that_broke_apple_silicon():
+ """Negative control. Every assertion above is a "nothing found" shape, which is also
+ what a checker that has quietly stopped checking reports."""
+ window = SpecifierSet(DECLARED_TOKENIZERS_WINDOW["5.5.0"])
+ assert STRANDED_TOKENIZERS not in window
+ assert "0.22.2" in window
+
+
+# ---------------------------------------------------------------------------
+# The resolver check: what a fresh macOS arm64 install actually ends with.
+# ---------------------------------------------------------------------------
+
+
+def _uv() -> str:
+ uv = shutil.which("uv")
+ if uv is None:
+ pytest.skip("uv is not installed")
+ return uv
+
+
+# uv's wording when the failure is a resolution verdict rather than a trip to the index.
+# Everything else -- a 429, a proxy, a DNS failure, an index 500 -- is not evidence about
+# this repo's pins, so it skips. Fail closed on the resolver's own answer, open on the
+# network: this test runs in the ordinary CPU lane, and a red X there has to mean the pins
+# are wrong, never that PyPI was busy.
+_RESOLVER_VERDICT_MARKERS = (
+ "no solution found",
+ "unsatisfiable",
+ "because",
+ "conflict",
+)
+# Long enough for a cold cache on a loaded runner, short enough that a hung index cannot
+# hold a CI job for ten minutes.
+_COMPILE_TIMEOUT_S = 180
+
+
+def _compile(args: list[str], stdin: str | None = None) -> dict[str, str]:
+ """Resolved name -> version for one `uv pip compile`, or skip when the index rather
+ than the requirements is what failed."""
+ try:
+ proc = subprocess.run(
+ [_uv(), "pip", "compile", *args],
+ input = stdin,
+ capture_output = True,
+ text = True,
+ timeout = _COMPILE_TIMEOUT_S,
+ cwd = REPO_ROOT,
+ # uv resolves aarch64-apple-darwin against macOS 12 by default, and mlx ships
+ # macosx_14_0 wheels only, so the resolve would fail for a reason that has nothing
+ # to do with the pair under test. Pin the deployment target instead of inheriting
+ # whatever the host happens to export.
+ env = {**os.environ, "MACOSX_DEPLOYMENT_TARGET": "15.0"},
+ )
+ except subprocess.TimeoutExpired:
+ pytest.skip(f"uv pip compile exceeded {_COMPILE_TIMEOUT_S}s; treating as index trouble")
+ except OSError as exc: # uv vanished mid-run, no fd, no memory
+ pytest.skip(f"uv pip compile could not run: {exc}")
+ if proc.returncode != 0:
+ stderr = proc.stderr or ""
+ if not any(marker in stderr.lower() for marker in _RESOLVER_VERDICT_MARKERS):
+ pytest.skip(f"uv pip compile failed without a resolver verdict: {stderr.strip()[:300]}")
+ pytest.fail(f"uv pip compile failed:\n{stderr[-3000:]}")
+ resolved = {}
+ for line in proc.stdout.splitlines():
+ text = line.split("#", 1)[0].strip()
+ if "==" in text:
+ name, _, version = text.partition("==")
+ resolved[canonicalize_name(name.strip())] = version.strip()
+ return resolved
+
+
+def _declared_window(version: str) -> SpecifierSet:
+ """transformers' own tokenizers requirement, read from PyPI rather than this file's
+ table, so the check keeps being true after a pin moves."""
+ url = f"https://pypi.org/pypi/transformers/{version}/json"
+ try:
+ with urllib.request.urlopen(url, timeout = 30) as response:
+ metadata = json.load(response)
+ except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
+ # Same rule as _compile: an index that is slow, throttling or serving something
+ # that is not JSON says nothing about this repo's pins.
+ pytest.skip(f"PyPI unreachable or unreadable: {exc}")
+ for raw in metadata["info"].get("requires_dist") or []:
+ try:
+ req = Requirement(raw)
+ except InvalidRequirement:
+ continue
+ if canonicalize_name(req.name) == "tokenizers" and req.marker is None:
+ return req.specifier
+ pytest.fail(f"transformers {version} declares no unconditional tokenizers requirement")
+
+
+def test_a_fresh_macos_arm64_install_ends_with_a_consistent_pair():
+ """Model the installer IN ORDER. Compiling everything at once would find a consistent
+ environment the installer never produces: install.sh's core phase resolves first, with
+ the override and without constraints.txt, and step 3b then replaces the packages
+ extras-no-deps.txt names, dependencies skipped.
+
+ Fails on the tree that shipped the defect (core phase -> transformers 5.16.1 +
+ tokenizers 0.23.2; step 3b -> transformers 5.5.0, tokenizers untouched).
+ """
+ platform_args = [
+ "--python-platform",
+ "aarch64-apple-darwin",
+ "--python-version",
+ "3.13",
+ ]
+ core = _compile(
+ ["-", *platform_args, "--override", str(DARWIN_OVERRIDES)],
+ stdin = "unsloth\nunsloth-zoo\n",
+ )
+ overlay = _compile([str(EXTRAS_NO_DEPS), "--no-deps", *platform_args])
+
+ final = dict(core)
+ final.update(overlay) # --no-deps: only the named packages move
+ transformers, tokenizers = final.get("transformers"), final.get("tokenizers")
+ assert transformers and tokenizers, f"resolve produced neither half of the pair: {final}"
+
+ window = _declared_window(transformers)
+ assert tokenizers in window, (
+ f"a fresh macOS arm64 install would end with transformers=={transformers} beside "
+ f"tokenizers=={tokenizers}, outside the declared window {window}. That venv cannot "
+ f"`import transformers`, so Train and Export stay off. Core phase resolved "
+ f"tokenizers=={core.get('tokenizers')}; step 3b left it at {tokenizers}."
+ )