Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Let the formatter fixed-point guard start on Windows (#11145)

Daniel Hane87d64c3bbd6
diff --git a/scripts/enforce_kwargs_spacing.py b/scripts/enforce_kwargs_spacing.py
index 6ccf77a0774..3c1e597c2ce 100755
--- a/scripts/enforce_kwargs_spacing.py
+++ b/scripts/enforce_kwargs_spacing.py
@@ -35,7 +35,14 @@ def _atomic_write_text(path: Path, data: str, encoding: str) -> None:
mode = None
fd, tmp_path = tempfile.mkstemp(prefix=".kwargs_fix.", dir=dirpath)
try:
- with os.fdopen(fd, "w", encoding=encoding) as handle:
+ # newline = "\n", not the default None. Default translates every "\n" to os.linesep,
+ # which on Windows is "\r\n", and the read side (tokenize.open, below) has already
+ # normalised CRLF to LF in memory -- so on Windows this rewrote EVERY file it touched to
+ # CRLF, against .gitattributes' own `*.py text eol=lf`. A Windows contributor running the
+ # pre-commit hook got a whole-file diff on everything they edited. "\n" rather than ""
+ # because it states the intended ending rather than "do not translate", and it holds even
+ # if the read side stops normalising.
+ with os.fdopen(fd, "w", encoding=encoding, newline="\n") as handle:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
diff --git a/scripts/scan_packages.py b/scripts/scan_packages.py
index b3982b30f48..59484aeb675 100644
--- a/scripts/scan_packages.py
+++ b/scripts/scan_packages.py
@@ -2380,7 +2380,12 @@ def update_req_line(raw_line: str, safe_ver: str, old_ver: str | None) -> str:
def update_req_file(filepath: str, updates: dict[int, str]) -> None:
"""Apply line-level updates ({1-indexed line_num: new_line_text}) to a requirements file. Writes atomically (sibling tmp file, fsync, os.replace) so a crash mid-write never leaves a half-written file that re-introduces a malicious pin."""
- with open(filepath) as f:
+ # encoding is explicit on both halves: open() without it takes the locale codec, so the same
+ # requirements file round-trips differently on a runner with LANG=C than on one with a UTF-8
+ # locale, and a non-ASCII comment is mangled or raises. studio/backend has a guard for exactly
+ # this (tests/test_text_io_encoding.py) but it scans BACKEND_ROOT only, so scripts/ was never
+ # covered by it.
+ with open(filepath, encoding = "utf-8") as f:
lines = f.readlines()
for line_num, new_text in updates.items():
@@ -2395,7 +2400,12 @@ def update_req_file(filepath: str, updates: dict[int, str]) -> None:
dir = dirpath,
)
try:
- with os.fdopen(fd, "w") as f:
+ # newline = "\n" so the line endings the loop above went to the trouble of preserving
+ # survive the write. readlines() above is universal-newline, so a CRLF file arrives as LF
+ # in memory, and the default newline then translates it back to os.linesep -- which on
+ # Windows rewrites every line of a tracked requirements file and makes the "preserve line
+ # ending" above a no-op.
+ with os.fdopen(fd, "w", encoding = "utf-8", newline = "\n") as f:
f.writelines(lines)
f.flush()
os.fsync(f.fileno())
@@ -2657,7 +2667,11 @@ def _write_baseline(
"version": 1,
"entries": entries,
}
- with open(path, "w", encoding = "utf-8") as fh:
+ # newline = "\n" for the same reason as update_req_file above: the default translates every
+ # "\n" json.dump emits to os.linesep, and --write-baseline rewrites the tracked
+ # scripts/scan_packages_baseline.json in place, so on Windows a one-entry review turned into a
+ # whole-file CRLF diff over several thousand lines.
+ with open(path, "w", encoding = "utf-8", newline = "\n") as fh:
json.dump(doc, fh, indent = 2, sort_keys = False)
fh.write("\n")
print(f" Wrote {len(entries)} baseline entr(y/ies) to {path}")
diff --git a/scripts/scan_packages_baseline.json b/scripts/scan_packages_baseline.json
index 869934f24ab..281ab3a2414 100644
--- a/scripts/scan_packages_baseline.json
+++ b/scripts/scan_packages_baseline.json
@@ -1895,6 +1895,32 @@
"evidence": "Obfusc: L4662: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L157: mx.eval(model.parameters()) | L191: mx.eval([v for k, v in tree_flatten(casted) if is_mlx_norm_parameter_path(k)]) | L1170: mx.eval(model.parameters()) | L1171: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L1238: mx.eval(model.parameters()) | L1241: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L2576: mx.eval(altup.correct(predictions, activated)) | L3017: mx.eval(baseline, perturbed) | L4310: model.eval() | L5049: mx.eval(model.parameters()) | L5164: mx.eval(module.weight) | L8557: lambda: mx.eval(model.parameters()), | L8617: lambda: mx.eval(model.parameters()), | L8770: mx.eval(model.parameters())",
"evidence_hash": "cf9c8ed7bf9c33f83b6280d7d2d60b072262a04ce3e886a1f38c4003fbef01f4",
"file_sha256": "d18e7eb087b09c360d02d6e274e92cca153768e8a8b61bc978e2a40f1981043d"
+ },
+ {
+ "package": "unsloth-zoo",
+ "file": "unsloth_zoo/compiler.py",
+ "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",
+ "severity": "HIGH",
+ "evidence": "Obfusc: L1513: _mod = __import__(model_location, fromlist=items) | L5865: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL5866: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL5867: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL5868: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L5866: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL5867: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL5868: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L5865: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL5866: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL5867: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nExec: L1442: _modeling_file = eval(model_location) | L2148: f = eval(f\"{model_location}.{module}\") | L2478: exec(f\"def raise_{j}(*args, **kwargs): print('{function}')\", globals(), locals()) | L2479: try: exec(f\"EMPTY_LOGITS.{function} = raise_{j}\", globals(), locals()) | L3910: exec(f\"import {parent}\", locals(), globals()) | L4041: dir(eval(parent)), | L4045: exec(f\"{parent}.{child}.forward = forward\", globals(), locals()) | L4119: module = eval(f\"modeling_file.{module}\") | L4146: inner_class = eval(f\"modeling_file.{inner_class}\") | L4276: exec(f\"from timm.layers.norm_act import {norm}\") | L4284: forward = eval(norm).forward | L4290: exec(f\"timm.layers.norm_act.{norm}.forward = forward\") | L4307: exec(f\"from timm.models._efficientnet_blocks import {block}\") | L4315: forward = eval(block).forward | L4321: exec(f\"timm.models._efficientnet_blocks.{block}.forward = forward\") | L4561: exec(\nL4562: f\"{model_location}.torch.nn.{module}.forward = forward\",\nL4563: globals(),\nL4564: locals(),\nL4565: ) | L4567: exec(f\"{model_location}.nn.{module}.forward = forward\", globals(), locals()) | L4571: exec(\nL4572: f\"combined_module.torch.nn.{module}.forward = forward\",\nL4573: globals(),\nL4574: locals(),\nL4575: ) | L4577: exec(f\"combined_module.nn.{module}.forward = forward\", globals(), locals()) | L4654: source = eval(f\"{model_location}.torch\") | L4661: function = eval(f\"source.nn.{module}\") | L4855: exec(\"import transformers\", globals()) | L4868: exec(\nL4869: \"model_logger.addFilter(HideLoggingMessage('`use_cache`'))\", globals(), locals()\nL4870: ) | L4872: exec(\nL4873: \"model_logger.addFilter(HideLoggingMessage('compile_config'))\",\nL4874: globals(),\nL4875: locals(),\nL4876: ) | L5110: source = eval(f\"modeling_file.{module}\") | L5124: source = eval(f\"modeling_file.{module}\") | L5225: source = eval(f\"modeling_file.{module}\") | L5263: source = eval(f\"{model_location}.{module}\") | L5343: source = eval(f\"{model_location}.{module}\") | L5391: source = eval(f\"{model_location}.{module}\") | L5628: source = eval(f\"{model_location}.{module}\") | L5639: exec(\nL5640: f\"{model_location}.{module}._update_causal_mask = no_update_causal_mask\",\nL5641: globals(),\nL5642: ) | L5705: source = eval(f\"{model_location}.{module}\") | L5746: module_cls = eval(f\"{model_location}.{module}\") | L5783: module_cls = eval(f\"{model_location}.{module}\") | L5850: exec(\nL5851: \"from transformers.trainer import (\" + \", \".join(x for x in good_items) + \")\",\nL5852: globals(),\nL5853: ) | L5915: exec(inner_training_loop, globals()) | L5924: function = eval(f\"{model_location}.{module}\") | L6011: function = eval(f\"{model_location}.{module}\") | L6165: exec(\nL6166: f\"{model_location}.{module} = combined_module.{module}\",\nL6167: globals(),\nL6168: locals(),\nL6169: ) | L6179: check_dicts = dir(eval(f\"{model_location}\")) | L6181: item = eval(f\"{model_location}.{check}\") | L6191: exec(\nL6192: f\"{model_location}.{check}['{key}'] = combined_module.{replaced_class}\",\nL6193: globals(),\nL6194: locals(),\nL6195: )",
+ "evidence_hash": "bdfae4c24572c2a86fa6a2dede720ebb0723746963e43c2cd42701de5fc48fdb",
+ "file_sha256": "47da60730becd8e548e9d486812cf0491ee92256f65109ebfef7e8b5412fa7e6"
+ },
+ {
+ "package": "unsloth-zoo",
+ "file": "unsloth_zoo/saving_utils.py",
+ "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",
+ "severity": "HIGH",
+ "evidence": "Obfusc: L4256: module = __import__(\"transformers\", fromlist = [model_class_name]) | L4280: module = __import__('transformers', fromlist=[model_class_name])\nExec: L4111: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L4157: exec(save_pretrained, globals(), functions)",
+ "evidence_hash": "2ab08222867e458035a053336fa2efcee824befc382e47eef02b5b78fe08c115"
+ },
+ {
+ "package": "unsloth-zoo",
+ "file": "unsloth_zoo/mlx/loader.py",
+ "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",
+ "severity": "HIGH",
+ "evidence": "Obfusc: L4855: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L158: mx.eval(model.parameters()) | L192: mx.eval([v for k, v in tree_flatten(casted) if is_mlx_norm_parameter_path(k)]) | L1254: mx.eval(model.parameters()) | L1255: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L1322: mx.eval(model.parameters()) | L1325: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L2711: mx.eval(altup.correct(predictions, activated)) | L3210: mx.eval(baseline, perturbed) | L4503: model.eval() | L5242: mx.eval(model.parameters()) | L5357: mx.eval(module.weight) | L8769: lambda: mx.eval(model.parameters()), | L8829: lambda: mx.eval(model.parameters()), | L8985: mx.eval(model.parameters())",
+ "evidence_hash": "cf9c8ed7bf9c33f83b6280d7d2d60b072262a04ce3e886a1f38c4003fbef01f4",
+ "file_sha256": "7a69b550899a2d782bb460e3d882a0840b1e2f8e7b50f43d6a9468ae808aad59"
}
]
}
diff --git a/scripts/stamp_studio_release.py b/scripts/stamp_studio_release.py
index 739f6d10636..0cc8820a812 100644
--- a/scripts/stamp_studio_release.py
+++ b/scripts/stamp_studio_release.py
@@ -28,7 +28,11 @@ def _atomic_write_text(
path.parent.mkdir(parents = True, exist_ok = True)
fd, tmp_path = tempfile.mkstemp(prefix = ".stamp_studio.", dir = dirpath)
try:
- with os.fdopen(fd, "w", encoding = encoding) as handle:
+ # newline = "\n": the target is studio/backend/utils/_studio_release_build.py, a TRACKED
+ # .py file that .gitattributes pins to `text eol=lf`, and `data` is a Python string
+ # literal carrying LF. The default newline translates each "\n" to os.linesep, so a
+ # release stamped on Windows landed CRLF and showed as a whole-file diff.
+ with os.fdopen(fd, "w", encoding = encoding, newline = "\n") as handle:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
diff --git a/scripts/sync_allow_scripts_pins.py b/scripts/sync_allow_scripts_pins.py
index 1d9a075ab43..d180f74b847 100644
--- a/scripts/sync_allow_scripts_pins.py
+++ b/scripts/sync_allow_scripts_pins.py
@@ -132,7 +132,13 @@ def main(argv: list[str] | None = None) -> int:
return 1
pkg["allowScripts"] = {renames.get(k, k): v for k, v in policy.items()}
- pkg_path.write_text(json.dumps(pkg, indent = 2, ensure_ascii = False) + "\n", encoding = "utf-8")
+ # newline = "\n", not the default None, which translates every "\n" to os.linesep. This is a
+ # pre-commit hook running --fix on studio/frontend/package.json, and .gitattributes pins that
+ # file to LF (`studio/frontend/** text=auto eol=lf`), so on Windows the default turned a
+ # two-line pin bump into a whole-file CRLF rewrite. open() rather than write_text(newline =):
+ # write_text only grew `newline` in 3.10 and pyproject still says requires-python >= 3.9.
+ with open(pkg_path, "w", encoding = "utf-8", newline = "\n") as handle:
+ handle.write(json.dumps(pkg, indent = 2, ensure_ascii = False) + "\n")
print(
f"sync-allow-scripts: re-pinned {len(renames)} entr{'y' if len(renames) == 1 else 'ies'} in {pkg_path}"
)
diff --git a/studio/backend/tests/test_startup_defers_torch.py b/studio/backend/tests/test_startup_defers_torch.py
index 07fede369e4..e588dec550e 100644
--- a/studio/backend/tests/test_startup_defers_torch.py
+++ b/studio/backend/tests/test_startup_defers_torch.py
@@ -426,12 +426,28 @@ def test_a_failed_detection_degrades_instead_of_raising():
hw.DEVICE, hw.CHAT_ONLY, hw.CHAT_ONLY_REASON = saved_device, saved_chat, saved_reason
+def _confine_the_torch_import_error(monkeypatch, hw):
+ """Undo hw.TORCH_IMPORT_ERROR at teardown, for a test that watches an import fail.
+
+ The three tests below restore ``builtins.__import__``, ``sys.modules`` and the detection
+ verdict, but _has_torch() also writes that module global as a SIDE EFFECT, and nothing
+ put it back: the worker carried ``OSError(...)`` into every later test in the same
+ process. classify_torch_build(), _torch_reports_an_xpu_runtime() and
+ _torch_reports_a_hip_runtime() all branch on it and read the wheel off DISK when it is
+ set, ignoring the fake torch a later test installed in sys.modules -- so
+ test_torch_cpu_build_on_nvidia_host.py failed only in the xdist worker that happened to
+ inherit the leak. Snapshot-restore rather than force None: nothing here owns the value.
+ """
+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", hw.TORCH_IMPORT_ERROR)
+
+
def test_a_broken_torch_counts_as_no_torch(monkeypatch):
"""_has_torch() must not let a non-ImportError escape into detection."""
import builtins
from utils.hardware import hardware as hw
+ _confine_the_torch_import_error(monkeypatch, hw)
real_import = builtins.__import__
def fake_import(name, *args, **kwargs):
@@ -503,6 +519,7 @@ def test_a_broken_torch_purges_its_own_zombie(monkeypatch):
from utils.hardware import hardware as hw
+ _confine_the_torch_import_error(monkeypatch, hw)
real_import = builtins.__import__
saved = {name: mod for name, mod in sys.modules.items() if name.split(".")[0] == "torch"}
@@ -542,6 +559,8 @@ def test_one_detection_pass_probes_torch_once(monkeypatch):
saved = {n: m for n, m in sys.modules.items() if n.split(".")[0] == "torch"}
saved_device = hw.DEVICE
saved_chat, saved_reason = hw.CHAT_ONLY, hw.CHAT_ONLY_REASON
+ # Before the __import__ patch below, so the finally's monkeypatch.undo() puts it back last.
+ _confine_the_torch_import_error(monkeypatch, hw)
real_import = builtins.__import__
attempts = []
diff --git a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py
index aa1b6c68ade..5316dfba8a1 100644
--- a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py
+++ b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py
@@ -52,6 +52,13 @@ def _shared_setup_1(monkeypatch, tmp_path):
import sys
monkeypatch.setitem(sys.modules, "torch", _fake_torch("cpu"))
+ # The fake above is only READ while the import is known good: TORCH_IMPORT_ERROR is a
+ # module global _has_torch() writes as a side effect, and _torch_reports_an_xpu_runtime()
+ # / _torch_reports_a_hip_runtime() / classify_torch_build() read the wheel off DISK
+ # whenever it is set. A sibling that watched a broken import leaves it non-None for the
+ # rest of the xdist worker, so pin it here rather than inherit it, as
+ # cpu_torch_on_an_nvidia_host already does.
+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", None)
monkeypatch.setattr(hw.sys, "prefix", str(tmp_path))
return sys
@@ -1225,6 +1232,46 @@ def test_a_nameless_intel_card_counts_once_xpu_was_actually_chosen(monkeypatch,
assert hw._devices_that_can_establish_a_mismatch(nameless) == nameless
+def test_a_leaked_import_error_is_what_stops_the_fake_torch_being_read(monkeypatch, tmp_path):
+ """Why _shared_setup_1 pins TORCH_IMPORT_ERROR: the assertion above is not self-contained.
+
+ _torch_reports_an_xpu_runtime() consults sys.modules only while that global is None;
+ set, it reads the wheel off disk and the fake +xpu torch above is never looked at, so
+ the nameless Intel card stops counting and the last assertion of the previous test
+ reads ``[] == [{...}]``. Nothing in this file writes the global -- _has_torch() does,
+ as a side effect, from whichever test in the xdist worker last watched an import fail,
+ which is why it went red under ``-n 4`` on one shard and green in every serial run.
+ """
+ sys = _shared_setup_1(monkeypatch, tmp_path)
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_FAMILY", raising = False)
+ (tmp_path / "unsloth_install_manifest.json").write_text(
+ '{"schema": 1, "expected_torch_tag": "cpu"}', encoding = "utf-8"
+ )
+ # The on-disk arm is pinned to a CPU wheel, so this does not turn on what torch the
+ # host running the suite happens to have installed.
+ monkeypatch.setattr(hw, "_installed_torch_label_on_disk", lambda: "2.11.0+cpu")
+ monkeypatch.setattr(
+ hw,
+ "_installed_torch_markers_on_disk",
+ lambda: {"cuda": None, "hip": None, "xpu": None},
+ )
+
+ nameless = [{"vendor": "intel", "name": None, "index": 0}]
+ xpu_torch = _fake_torch("cpu")
+ xpu_torch.__version__ = "2.9.0+xpu"
+ monkeypatch.setitem(sys.modules, "torch", xpu_torch)
+
+ # Pinned, as _shared_setup_1 leaves it: the fake in sys.modules is what answers.
+ assert hw._torch_reports_an_xpu_runtime() is True
+ assert hw._devices_that_can_establish_a_mismatch(nameless) == nameless
+
+ # Exactly what a sibling's _has_torch() leaves behind, and the verdict inverts.
+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", "OSError('undefined symbol: cudaGetDeviceCount')")
+ assert hw._torch_reports_an_xpu_runtime() is False
+ assert hw._devices_that_can_establish_a_mismatch(nameless) == []
+
+
def test_an_accelerator_that_came_back_retires_the_cached_verdict(monkeypatch):
"""Only reason and detail are refreshed here; DEVICE and CHAT_ONLY are not.
@@ -1480,6 +1527,11 @@ def test_an_unimportable_torch_still_reports_the_cards(monkeypatch, tmp_path):
monkeypatch.delitem(sys.modules, "torch", raising = False)
monkeypatch.setattr(sys, "meta_path", [_Finder(), *sys.meta_path])
+ # _has_torch() below writes hw.TORCH_IMPORT_ERROR as a side effect and nothing puts it
+ # back, so snapshot-restore it: left set, it sends every later test in this worker down
+ # the on-disk arm of the same three readers. Snapshot rather than None -- nothing here
+ # owns the value.
+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", hw.TORCH_IMPORT_ERROR)
# _has_torch() is NOT forced here: it reports False for a wheel that will not import,
# and the early return on it used to keep this host from the on-disk fallback below.
assert hw._has_torch() is False, "the premise: an unimportable torch reads as absent"
diff --git a/tests/test_formatter_fixed_point.py b/tests/test_formatter_fixed_point.py
index 76a0ac31f6c..b55bcef9be3 100644
--- a/tests/test_formatter_fixed_point.py
+++ b/tests/test_formatter_fixed_point.py
@@ -39,6 +39,29 @@ from run_ruff_format import ( # noqa: E402
version_mismatch,
)
+#: What Windows' CreateProcess accepts for a whole command line, in characters. Documented by
+#: Microsoft as the lpCommandLine cap and unchanged since Windows XP. POSIX has no equivalent
+#: single limit (execve fails with E2BIG against ARG_MAX, ~2 MB on the runners), so Windows is
+#: the binding constraint and the only number worth pinning.
+_WINDOWS_COMMAND_LINE_LIMIT = 32767
+
+#: Characters a single formatter invocation may build up to. The whole tracked set is ~2650
+#: files whose copied paths under a pytest tmp_path come to ~325,000 characters -- 9.9x the cap
+#: above -- so the single-invocation form this replaced could never run on Windows at all: it
+#: died at CreateProcess with `[WinError 206] The filename or extension is too long` before ruff
+#: opened one file. Nobody had seen it fail because every job that schedules this file is
+#: ubuntu-24.04, where execve's ARG_MAX is ~2 MB and the same argv fits easily.
+#:
+#: A LENGTH budget rather than a file COUNT. A count has to be re-tuned every time the tracked
+#: set or the path depth moves, and silently loses its margin in between: 200 files measured
+#: 26,687 characters here, only 1.2x under the cap, so a fifth more files would have put it back
+#: over. Accumulating by length is self-adjusting and needs no number kept in step with the
+#: repo. 8000 leaves 4x headroom for the argv-quoting model below being wrong.
+#:
+#: Batching at all is safe because the hook is per-file: it formats each path independently, so
+#: N calls over disjoint batches do exactly what one call over the union does.
+_FORMAT_ARGV_BUDGET = 8000
+
_HOOK_ID = "ruff-format-with-kwargs"
# The spacing pass refuses to rewrite itself ("skip modifying this script to
# avoid self-edit loops"), so it is not a file the hook keeps at a fixed point
@@ -222,6 +245,95 @@ class TestTheGuardCannotGoGreenHavingCheckedNothing:
assert running_in_ci({"CI": ""}) is False
+def formatter_argvs(copies: list[str], head: list[str] | None = None) -> "list[list[str]]":
+ """Every command line the fixed-point guard will run, in order.
+
+ Batches are accumulated until adding the next path would take the command line past
+ `_FORMAT_ARGV_BUDGET`, so a single path longer than the budget still gets its own call
+ rather than being dropped -- the caller would rather run one over-long command and see the
+ OS refuse it than silently skip a file.
+
+ The guard and the Windows-limit test below both go through this, deliberately. A test that
+ only checked the budget constant would be measuring a number while the caller did something
+ else, and deleting the batching at the call site would leave it green. Here there is one
+ definition of what actually gets executed, so the limit test cannot drift away from the run.
+ """
+ head = head or [sys.executable, str(_ROOT / "scripts" / "run_ruff_format.py")]
+ base = _command_line_length(head)
+ argvs: list[list[str]] = []
+ batch: list[str] = []
+ used = base
+ for path in copies:
+ cost = _command_line_length([path])
+ if batch and used + cost > _FORMAT_ARGV_BUDGET:
+ argvs.append([*head, *batch])
+ batch, used = [], base
+ batch.append(path)
+ used += cost
+ if batch:
+ argvs.append([*head, *batch])
+ return argvs
+
+
+def _command_line_length(argv: list[str]) -> int:
+ """What Windows counts against its command-line cap for this argv.
+
+ CreateProcess is handed ONE string, so the cost is the arguments joined by the separating
+ spaces, plus a pair of quotes around every argument a runner path forces (the hosted image
+ checks out under `D:\\a\\unsloth\\unsloth`, no spaces, but `C:\\Users\\RUNNER~1\\AppData\\
+ Local\\Temp` is where tmp_path lands and a user name with a space is normal off CI). Counted
+ with the quotes always, because this is a headroom check and the cheap direction to be wrong
+ in is pessimistic.
+ """
+ return sum(len(arg) + 3 for arg in argv)
+
+
[email protected](_VERDICT == "skip", reason = _RUFF_REASON or "")
+def test_the_formatter_invocation_fits_in_a_windows_command_line():
+ """The guard below must be able to START on Windows, not only pass on Linux.
+
+ It used to pass every tracked file as one argv. That is ~2650 paths and, under a Windows
+ tmp_path, roughly 325,000 characters against CreateProcess's 32,767 -- 9.9x over, so the
+ call died with `[WinError 206] The filename or extension is too long` before ruff opened a
+ single file. It had never been caught because every job that schedules this file is
+ ubuntu-24.04, where execve's ARG_MAX is ~2 MB and the same argv fits with room to spare.
+
+ So the limit is asserted here rather than left to a Windows runner to discover: this runs
+ in the existing Linux job, needs no second platform, and goes red the moment someone
+ reverts the batching or raises _FORMAT_ARGV_BUDGET past what the cap allows. Computed from the
+ REAL file list and a realistic Windows tmp_path prefix, not from a remembered number, so
+ the file set growing is what moves it.
+ """
+ names = eligible_files(_ROOT)
+ assert len(names) > 1000, f"only {len(names)} files matched; the file list has gone vacuous"
+
+ # A hosted Windows runner's pytest tmp_path. Longer than the Linux equivalent, which is the
+ # point: the platform with the smallest cap also has the longest prefix.
+ prefix = "C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\pytest-of-runner\\pytest-999\\test_0"
+ copies = [prefix + "\\" + name for name in names]
+
+ argvs = formatter_argvs(copies)
+ # Every path is still formatted: batching may not silently drop the tail.
+ assert [arg for argv in argvs for arg in argv[2:]] == copies, "batching lost or reordered files"
+
+ worst = max(_command_line_length(argv) for argv in argvs)
+ assert worst < _WINDOWS_COMMAND_LINE_LIMIT, (
+ f"the widest of the {len(argvs)} formatter batches builds a {worst}-character command "
+ f"line against Windows' {_WINDOWS_COMMAND_LINE_LIMIT}-character CreateProcess limit, so "
+ f"this guard would die with WinError 206 on Windows before formatting anything. Lower "
+ f"_FORMAT_ARGV_BUDGET (currently {_FORMAT_ARGV_BUDGET} over {len(names)} tracked files)."
+ )
+ # Non-vacuous: the batching has to be doing something. One call with everything on it is the
+ # shape that was broken, and it must still be over the cap -- if it ever fits, this test is
+ # measuring nothing and the batching can go.
+ unbatched = _command_line_length([sys.executable, "run_ruff_format.py", *copies])
+ assert unbatched > _WINDOWS_COMMAND_LINE_LIMIT, (
+ f"the whole tracked set now builds a {unbatched}-character command line, under the "
+ f"{_WINDOWS_COMMAND_LINE_LIMIT} cap, so batching is no longer load-bearing and this "
+ "test no longer proves anything. Delete both, or say why they stay."
+ )
+
+
@pytest.mark.skipif(_VERDICT == "skip", reason = _RUFF_REASON or "")
def test_every_tracked_python_file_is_already_formatted(tmp_path):
"""Run the hook over copies of the whole tracked set and expect no rewrite."""
@@ -251,12 +363,11 @@ def test_every_tracked_python_file_is_already_formatted(tmp_path):
target.write_bytes(originals[name])
copies.append(str(target))
- run = subprocess.run(
- [sys.executable, str(_ROOT / "scripts" / "run_ruff_format.py"), *copies],
- capture_output = True,
- text = True,
- )
- assert run.returncode == 0, f"the formatter itself failed:\n{run.stdout}\n{run.stderr}"
+ # Batched, and through the same builder the Windows-limit test above measures. One call with
+ # all ~2650 paths on it is 9.9x over Windows' CreateProcess cap and dies with WinError 206.
+ for argv in formatter_argvs(copies):
+ run = subprocess.run(argv, capture_output = True, text = True)
+ assert run.returncode == 0, f"the formatter itself failed:\n{run.stdout}\n{run.stderr}"
drifted = [name for name in names if (tmp_path / name).read_bytes() != originals[name]]
assert not drifted, (
diff --git a/tests/test_scripts_write_lf.py b/tests/test_scripts_write_lf.py
new file mode 100644
index 00000000000..f54b8d5dd31
--- /dev/null
+++ b/tests/test_scripts_write_lf.py
@@ -0,0 +1,250 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+"""Scripts that rewrite tracked files in place must write LF, on every platform.
+
+`.gitattributes` opens with `*.py text eol=lf`, so every tracked Python file in this repo is
+LF on disk and is meant to stay that way. Three scripts rewrite tracked files through the same
+temp-file-then-os.replace shape, and all three opened the temp file in text mode with the
+DEFAULT newline, which translates each "\\n" to `os.linesep`:
+
+ scripts/enforce_kwargs_spacing.py every tracked .py the formatting hook touches
+ scripts/stamp_studio_release.py studio/backend/utils/_studio_release_build.py
+ scripts/scan_packages.py studio/backend/requirements/*.txt, under --fix
+
+On Linux `os.linesep` is "\\n" and the round trip is a no-op, which is why this survived. On
+Windows it rewrites the file to CRLF. For enforce_kwargs_spacing that is every file a
+contributor edits, because the read side is `tokenize.open`, which normalises CRLF to LF in
+memory, so the write is what decides the ending. The symptom is a whole-file diff on files the
+contributor did not change, from running the project's own pre-commit hook.
+
+Found by running tests/test_formatter_fixed_point.py on a real windows-latest runner (see the
+staging evidence on the PR): the guard reported ~1300 of ~2650 tracked files as drifted, none
+of which had drifted.
+
+Two halves, deliberately:
+
+ * the BEHAVIOURAL tests below prove the end-to-end intent -- a CRLF file in, LF bytes out --
+ and on Windows they exercise the translation for real. On Linux they cannot fail if the fix
+ is reverted, because `os.linesep` is already "\\n" there. They are the demonstration.
+ * `test_every_in_place_rewriter_names_its_newline` is the DURABLE guard. It reads the call
+ sites and fails on any platform the moment a `newline =` argument goes missing, which is
+ the only way to catch a revert in the Linux job that actually runs.
+
+Same split, and the same reason, as test_formatter_fixed_point.py's Windows command-line check:
+assert the property where the platform cannot demonstrate it.
+"""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_ROOT = Path(__file__).resolve().parent.parent
+_SCRIPTS = _ROOT / "scripts"
+
+#: The in-place rewriters this file covers, as (path, the function holding the write).
+#: Named on the real files so a rename fails here rather than silently dropping coverage. A pair,
+#: not a {script: func} mapping, because one script can hold more than one rewriter.
+#:
+#: The list is what it is because the rule is "rewrites a TRACKED file in place", not "uses
+#: os.fdopen": the last two write through plain open()/Path.write_text and have exactly the same
+#: default-newline defect. sync_allow_scripts_pins is the one that matters most -- it runs as a
+#: pre-commit hook with --fix over studio/frontend/package.json, which .gitattributes pins to
+#: eol=lf.
+_REWRITERS = (
+ ("enforce_kwargs_spacing.py", "_atomic_write_text"),
+ ("stamp_studio_release.py", "_atomic_write_text"),
+ ("scan_packages.py", "update_req_file"),
+ ("scan_packages.py", "_write_baseline"),
+ ("sync_allow_scripts_pins.py", "main"),
+)
+
+
+def _load(name: str):
+ """Import a scripts/ module by filename, without putting scripts/ on sys.path for good."""
+ path = _SCRIPTS / name
+ assert path.is_file(), f"{path} is gone; _REWRITERS is stale"
+ spec = importlib.util.spec_from_file_location(f"_scripts_{path.stem}", path)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+# ── the durable half: the call sites name a newline ───────────────────────────────────────────
+
+
+def _text_write_calls(tree: ast.AST, func_name: str) -> list[ast.Call]:
+ """Every `os.fdopen(...)`/`open(...)`/`Path.write_text(...)` text-mode WRITE in the function.
+
+ Binary mode is excluded: `newline` is meaningless there and passing it raises. A call with
+ no mode argument at all defaults to "r", so it is not a write and is skipped too.
+
+ `write_text` is here because it carries the identical default: `Path.write_text(data,
+ encoding = ...)` leaves `newline` at None and so translates to os.linesep exactly like
+ `open()` does. Leaving it out would let a rewriter swap one for the other and drop out of
+ this guard silently, which is how sync_allow_scripts_pins.py was writing CRLF in the first
+ place.
+ """
+ target = next(
+ (
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name
+ ),
+ None,
+ )
+ assert target is not None, f"{func_name} is gone; _REWRITERS is stale"
+
+ calls = []
+ for node in ast.walk(target):
+ if not isinstance(node, ast.Call):
+ continue
+ func = node.func
+ named = (
+ func.attr
+ if isinstance(func, ast.Attribute)
+ else func.id
+ if isinstance(func, ast.Name)
+ else ""
+ )
+ if named == "write_text":
+ # No mode argument to read: write_text is always a text-mode write.
+ calls.append(node)
+ continue
+ if named not in ("fdopen", "open"):
+ continue
+ # Mode is the second positional for both open() and os.fdopen().
+ mode = next(
+ (
+ arg.value
+ for arg in node.args[1:2]
+ if isinstance(arg, ast.Constant) and isinstance(arg.value, str)
+ ),
+ None,
+ ) or next(
+ (
+ kw.value.value
+ for kw in node.keywords
+ if kw.arg == "mode"
+ and isinstance(kw.value, ast.Constant)
+ and isinstance(kw.value.value, str)
+ ),
+ None,
+ )
+ if mode is None or "b" in mode or not any(ch in mode for ch in "wax+"):
+ continue
+ calls.append(node)
+ return calls
+
+
[email protected](("script", "func"), _REWRITERS)
+def test_every_in_place_rewriter_names_its_newline(script, func):
+ """A text-mode write in one of these must say what line ending it wants.
+
+ This is the half that can fail on Linux. `os.linesep` is "\\n" here, so no behavioural test
+ in this file can notice the default coming back -- only reading the call site can.
+ """
+ tree = ast.parse((_SCRIPTS / script).read_text(encoding = "utf-8"))
+ calls = _text_write_calls(tree, func)
+ assert calls, f"no text-mode write found in {script}:{func}; this guard has gone vacuous"
+ for call in calls:
+ kwargs = {kw.arg for kw in call.keywords}
+ assert "newline" in kwargs, (
+ f"{script}:{call.lineno} in {func}() opens a text file for writing without a "
+ "`newline =` argument, so Python translates every '\\n' to os.linesep and this "
+ "rewrites the whole tracked file to CRLF on Windows. .gitattributes pins these to "
+ 'LF (`*.py text eol=lf`). Pass newline = "\\n".'
+ )
+ assert "encoding" in kwargs, (
+ f"{script}:{call.lineno} in {func}() opens a text file for writing without an "
+ "`encoding =` argument, so it takes the locale codec and round-trips differently "
+ 'between runners. Pass encoding = "utf-8".'
+ )
+
+
+def test_the_repo_agrees_these_files_are_lf():
+ """The premise. If .gitattributes stops pinning LF, everything above is arguing for nothing."""
+ attributes = (_ROOT / ".gitattributes").read_text(encoding = "utf-8")
+ assert "*.py text eol=lf" in attributes, (
+ ".gitattributes no longer pins tracked Python files to LF, so the rewriters this file "
+ "guards have no ending to preserve and this whole file needs re-reading"
+ )
+
+
+# ── the demonstration half: a CRLF file in, LF bytes out ──────────────────────────────────────
+
+
+def test_the_spacing_pass_writes_lf_for_a_crlf_source(tmp_path):
+ """The headline path: what the pre-commit hook does to a contributor's file."""
+ module = _load("enforce_kwargs_spacing.py")
+ target = tmp_path / "sample.py"
+ # CRLF on disk, and something the pass will actually rewrite, so the write really happens.
+ target.write_bytes(b"def f(a=1, b=2):\r\n return a + b\r\n")
+
+ module.process_file(target)
+
+ written = target.read_bytes()
+ assert (
+ b"\r\n" not in written
+ ), f"the spacing pass wrote CRLF into a file .gitattributes pins to LF: {written!r}"
+ assert written.endswith(b"\n"), written
+ # Non-vacuous: it must actually have done its job, or "no CRLF" is just "no write".
+ assert b"a = 1" in written, written
+
+
+def test_the_release_stamp_writes_lf(tmp_path):
+ """A tracked .py generated from a Python string literal, so LF is the only correct ending."""
+ module = _load("stamp_studio_release.py")
+ target = tmp_path / "_studio_release_build.py"
+
+ module._atomic_write_text(target, 'VERSION = "1.2.3"\nBUILD = 7\n', encoding = "utf-8")
+
+ written = target.read_bytes()
+ assert b"\r\n" not in written, f"the release stamp wrote CRLF: {written!r}"
+ assert written == b'VERSION = "1.2.3"\nBUILD = 7\n', written
+
+
+def test_the_requirements_fixer_writes_lf_and_utf8(tmp_path):
+ """--fix rewrites tracked requirements files, and used to take the locale codec too."""
+ module = _load("scan_packages.py")
+ target = tmp_path / "reqs.txt"
+ # CRLF, plus a non-ASCII comment: the encoding half of the same call.
+ target.write_bytes("requests==2.0.0 # naïve pin\r\nurllib3==1.0.0\r\n".encode("utf-8"))
+
+ module.update_req_file(str(target), {2: "urllib3==2.0.0"})
+
+ written = target.read_bytes()
+ assert b"\r\n" not in written, f"the requirements fixer wrote CRLF: {written!r}"
+ assert "naïve".encode("utf-8") in written, "the non-ASCII comment did not survive as UTF-8"
+ assert b"urllib3==2.0.0" in written, written
+
+
+def test_the_allow_scripts_pin_sync_writes_lf(tmp_path):
+ """The other pre-commit hook that rewrites a tracked file, on a file pinned `eol=lf`.
+
+ `.gitattributes` carries `studio/frontend/** text=auto eol=lf`, and .pre-commit-config.yaml
+ runs this one with `--fix` on every package.json touch, so it is the same contributor-facing
+ surface as the spacing hook above.
+ """
+ module = _load("sync_allow_scripts_pins.py")
+ (tmp_path / "package.json").write_bytes(
+ json.dumps({"name": "x", "allowScripts": {"[email protected]": True}}).encode("utf-8")
+ )
+ (tmp_path / "package-lock.json").write_bytes(
+ json.dumps(
+ {"packages": {"node_modules/esbuild": {"version": "0.2.0", "hasInstallScript": True}}}
+ ).encode("utf-8")
+ )
+
+ assert module.main(["--fix", "--dir", str(tmp_path)]) == 0
+
+ written = (tmp_path / "package.json").read_bytes()
+ assert b"\r\n" not in written, f"the allowScripts pin sync wrote CRLF: {written!r}"
+ # Non-vacuous: the re-pin must actually have happened, or "no CRLF" is just "no write".
+ assert b'"[email protected]"' in written, written