Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Let the formatter fixed-point guard start on Windows (#11145)
Daniel Hane87d64c3bbd6
diff --git a/scripts/enforce_kwargs_spacing.py b/scripts/enforce_kwargs_spacing.pyindex 6ccf77a0774..3c1e597c2ce 100755--- a/scripts/enforce_kwargs_spacing.py+++ b/scripts/enforce_kwargs_spacing.py@@ -35,7 +35,14 @@ def _atomic_write_text(path: Path, data: str, encoding: str) -> None:mode = Nonefd, tmp_path = tempfile.mkstemp(prefix=".kwargs_fix.", dir=dirpath)try:- with os.fdopen(fd, "w", encoding=encoding) as handle:+ # newline = "\n", not the default None. Default translates every "\n" to os.linesep,+ # which on Windows is "\r\n", and the read side (tokenize.open, below) has already+ # normalised CRLF to LF in memory -- so on Windows this rewrote EVERY file it touched to+ # CRLF, against .gitattributes' own `*.py text eol=lf`. A Windows contributor running the+ # pre-commit hook got a whole-file diff on everything they edited. "\n" rather than ""+ # because it states the intended ending rather than "do not translate", and it holds even+ # if the read side stops normalising.+ with os.fdopen(fd, "w", encoding=encoding, newline="\n") as handle:handle.write(data)handle.flush()os.fsync(handle.fileno())diff --git a/scripts/scan_packages.py b/scripts/scan_packages.pyindex b3982b30f48..59484aeb675 100644--- a/scripts/scan_packages.py+++ b/scripts/scan_packages.py@@ -2380,7 +2380,12 @@ def update_req_line(raw_line: str, safe_ver: str, old_ver: str | None) -> str:def update_req_file(filepath: str, updates: dict[int, str]) -> None:"""Apply line-level updates ({1-indexed line_num: new_line_text}) to a requirements file. Writes atomically (sibling tmp file, fsync, os.replace) so a crash mid-write never leaves a half-written file that re-introduces a malicious pin."""- with open(filepath) as f:+ # encoding is explicit on both halves: open() without it takes the locale codec, so the same+ # requirements file round-trips differently on a runner with LANG=C than on one with a UTF-8+ # locale, and a non-ASCII comment is mangled or raises. studio/backend has a guard for exactly+ # this (tests/test_text_io_encoding.py) but it scans BACKEND_ROOT only, so scripts/ was never+ # covered by it.+ with open(filepath, encoding = "utf-8") as f:lines = f.readlines()for line_num, new_text in updates.items():@@ -2395,7 +2400,12 @@ def update_req_file(filepath: str, updates: dict[int, str]) -> None:dir = dirpath,)try:- with os.fdopen(fd, "w") as f:+ # newline = "\n" so the line endings the loop above went to the trouble of preserving+ # survive the write. readlines() above is universal-newline, so a CRLF file arrives as LF+ # in memory, and the default newline then translates it back to os.linesep -- which on+ # Windows rewrites every line of a tracked requirements file and makes the "preserve line+ # ending" above a no-op.+ with os.fdopen(fd, "w", encoding = "utf-8", newline = "\n") as f:f.writelines(lines)f.flush()os.fsync(f.fileno())@@ -2657,7 +2667,11 @@ def _write_baseline("version": 1,"entries": entries,}- with open(path, "w", encoding = "utf-8") as fh:+ # newline = "\n" for the same reason as update_req_file above: the default translates every+ # "\n" json.dump emits to os.linesep, and --write-baseline rewrites the tracked+ # scripts/scan_packages_baseline.json in place, so on Windows a one-entry review turned into a+ # whole-file CRLF diff over several thousand lines.+ with open(path, "w", encoding = "utf-8", newline = "\n") as fh:json.dump(doc, fh, indent = 2, sort_keys = False)fh.write("\n")print(f" Wrote {len(entries)} baseline entr(y/ies) to {path}")diff --git a/scripts/scan_packages_baseline.json b/scripts/scan_packages_baseline.jsonindex 869934f24ab..281ab3a2414 100644--- a/scripts/scan_packages_baseline.json+++ b/scripts/scan_packages_baseline.json@@ -1895,6 +1895,32 @@"evidence": "Obfusc: L4662: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L157: mx.eval(model.parameters()) | L191: mx.eval([v for k, v in tree_flatten(casted) if is_mlx_norm_parameter_path(k)]) | L1170: mx.eval(model.parameters()) | L1171: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L1238: mx.eval(model.parameters()) | L1241: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L2576: mx.eval(altup.correct(predictions, activated)) | L3017: mx.eval(baseline, perturbed) | L4310: model.eval() | L5049: mx.eval(model.parameters()) | L5164: mx.eval(module.weight) | L8557: lambda: mx.eval(model.parameters()), | L8617: lambda: mx.eval(model.parameters()), | L8770: mx.eval(model.parameters())","evidence_hash": "cf9c8ed7bf9c33f83b6280d7d2d60b072262a04ce3e886a1f38c4003fbef01f4","file_sha256": "d18e7eb087b09c360d02d6e274e92cca153768e8a8b61bc978e2a40f1981043d"+ },+ {+ "package": "unsloth-zoo",+ "file": "unsloth_zoo/compiler.py",+ "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",+ "severity": "HIGH",+ "evidence": "Obfusc: L1513: _mod = __import__(model_location, fromlist=items) | L5865: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL5866: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL5867: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL5868: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L5866: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL5867: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL5868: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L5865: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL5866: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL5867: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nExec: L1442: _modeling_file = eval(model_location) | L2148: f = eval(f\"{model_location}.{module}\") | L2478: exec(f\"def raise_{j}(*args, **kwargs): print('{function}')\", globals(), locals()) | L2479: try: exec(f\"EMPTY_LOGITS.{function} = raise_{j}\", globals(), locals()) | L3910: exec(f\"import {parent}\", locals(), globals()) | L4041: dir(eval(parent)), | L4045: exec(f\"{parent}.{child}.forward = forward\", globals(), locals()) | L4119: module = eval(f\"modeling_file.{module}\") | L4146: inner_class = eval(f\"modeling_file.{inner_class}\") | L4276: exec(f\"from timm.layers.norm_act import {norm}\") | L4284: forward = eval(norm).forward | L4290: exec(f\"timm.layers.norm_act.{norm}.forward = forward\") | L4307: exec(f\"from timm.models._efficientnet_blocks import {block}\") | L4315: forward = eval(block).forward | L4321: exec(f\"timm.models._efficientnet_blocks.{block}.forward = forward\") | L4561: exec(\nL4562: f\"{model_location}.torch.nn.{module}.forward = forward\",\nL4563: globals(),\nL4564: locals(),\nL4565: ) | L4567: exec(f\"{model_location}.nn.{module}.forward = forward\", globals(), locals()) | L4571: exec(\nL4572: f\"combined_module.torch.nn.{module}.forward = forward\",\nL4573: globals(),\nL4574: locals(),\nL4575: ) | L4577: exec(f\"combined_module.nn.{module}.forward = forward\", globals(), locals()) | L4654: source = eval(f\"{model_location}.torch\") | L4661: function = eval(f\"source.nn.{module}\") | L4855: exec(\"import transformers\", globals()) | L4868: exec(\nL4869: \"model_logger.addFilter(HideLoggingMessage('`use_cache`'))\", globals(), locals()\nL4870: ) | L4872: exec(\nL4873: \"model_logger.addFilter(HideLoggingMessage('compile_config'))\",\nL4874: globals(),\nL4875: locals(),\nL4876: ) | L5110: source = eval(f\"modeling_file.{module}\") | L5124: source = eval(f\"modeling_file.{module}\") | L5225: source = eval(f\"modeling_file.{module}\") | L5263: source = eval(f\"{model_location}.{module}\") | L5343: source = eval(f\"{model_location}.{module}\") | L5391: source = eval(f\"{model_location}.{module}\") | L5628: source = eval(f\"{model_location}.{module}\") | L5639: exec(\nL5640: f\"{model_location}.{module}._update_causal_mask = no_update_causal_mask\",\nL5641: globals(),\nL5642: ) | L5705: source = eval(f\"{model_location}.{module}\") | L5746: module_cls = eval(f\"{model_location}.{module}\") | L5783: module_cls = eval(f\"{model_location}.{module}\") | L5850: exec(\nL5851: \"from transformers.trainer import (\" + \", \".join(x for x in good_items) + \")\",\nL5852: globals(),\nL5853: ) | L5915: exec(inner_training_loop, globals()) | L5924: function = eval(f\"{model_location}.{module}\") | L6011: function = eval(f\"{model_location}.{module}\") | L6165: exec(\nL6166: f\"{model_location}.{module} = combined_module.{module}\",\nL6167: globals(),\nL6168: locals(),\nL6169: ) | L6179: check_dicts = dir(eval(f\"{model_location}\")) | L6181: item = eval(f\"{model_location}.{check}\") | L6191: exec(\nL6192: f\"{model_location}.{check}['{key}'] = combined_module.{replaced_class}\",\nL6193: globals(),\nL6194: locals(),\nL6195: )",+ "evidence_hash": "bdfae4c24572c2a86fa6a2dede720ebb0723746963e43c2cd42701de5fc48fdb",+ "file_sha256": "47da60730becd8e548e9d486812cf0491ee92256f65109ebfef7e8b5412fa7e6"+ },+ {+ "package": "unsloth-zoo",+ "file": "unsloth_zoo/saving_utils.py",+ "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",+ "severity": "HIGH",+ "evidence": "Obfusc: L4256: module = __import__(\"transformers\", fromlist = [model_class_name]) | L4280: module = __import__('transformers', fromlist=[model_class_name])\nExec: L4111: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L4157: exec(save_pretrained, globals(), functions)",+ "evidence_hash": "2ab08222867e458035a053336fa2efcee824befc382e47eef02b5b78fe08c115"+ },+ {+ "package": "unsloth-zoo",+ "file": "unsloth_zoo/mlx/loader.py",+ "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",+ "severity": "HIGH",+ "evidence": "Obfusc: L4855: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L158: mx.eval(model.parameters()) | L192: mx.eval([v for k, v in tree_flatten(casted) if is_mlx_norm_parameter_path(k)]) | L1254: mx.eval(model.parameters()) | L1255: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L1322: mx.eval(model.parameters()) | L1325: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L2711: mx.eval(altup.correct(predictions, activated)) | L3210: mx.eval(baseline, perturbed) | L4503: model.eval() | L5242: mx.eval(model.parameters()) | L5357: mx.eval(module.weight) | L8769: lambda: mx.eval(model.parameters()), | L8829: lambda: mx.eval(model.parameters()), | L8985: mx.eval(model.parameters())",+ "evidence_hash": "cf9c8ed7bf9c33f83b6280d7d2d60b072262a04ce3e886a1f38c4003fbef01f4",+ "file_sha256": "7a69b550899a2d782bb460e3d882a0840b1e2f8e7b50f43d6a9468ae808aad59"}]}diff --git a/scripts/stamp_studio_release.py b/scripts/stamp_studio_release.pyindex 739f6d10636..0cc8820a812 100644--- a/scripts/stamp_studio_release.py+++ b/scripts/stamp_studio_release.py@@ -28,7 +28,11 @@ def _atomic_write_text(path.parent.mkdir(parents = True, exist_ok = True)fd, tmp_path = tempfile.mkstemp(prefix = ".stamp_studio.", dir = dirpath)try:- with os.fdopen(fd, "w", encoding = encoding) as handle:+ # newline = "\n": the target is studio/backend/utils/_studio_release_build.py, a TRACKED+ # .py file that .gitattributes pins to `text eol=lf`, and `data` is a Python string+ # literal carrying LF. The default newline translates each "\n" to os.linesep, so a+ # release stamped on Windows landed CRLF and showed as a whole-file diff.+ with os.fdopen(fd, "w", encoding = encoding, newline = "\n") as handle:handle.write(data)handle.flush()os.fsync(handle.fileno())diff --git a/scripts/sync_allow_scripts_pins.py b/scripts/sync_allow_scripts_pins.pyindex 1d9a075ab43..d180f74b847 100644--- a/scripts/sync_allow_scripts_pins.py+++ b/scripts/sync_allow_scripts_pins.py@@ -132,7 +132,13 @@ def main(argv: list[str] | None = None) -> int:return 1pkg["allowScripts"] = {renames.get(k, k): v for k, v in policy.items()}- pkg_path.write_text(json.dumps(pkg, indent = 2, ensure_ascii = False) + "\n", encoding = "utf-8")+ # newline = "\n", not the default None, which translates every "\n" to os.linesep. This is a+ # pre-commit hook running --fix on studio/frontend/package.json, and .gitattributes pins that+ # file to LF (`studio/frontend/** text=auto eol=lf`), so on Windows the default turned a+ # two-line pin bump into a whole-file CRLF rewrite. open() rather than write_text(newline =):+ # write_text only grew `newline` in 3.10 and pyproject still says requires-python >= 3.9.+ with open(pkg_path, "w", encoding = "utf-8", newline = "\n") as handle:+ handle.write(json.dumps(pkg, indent = 2, ensure_ascii = False) + "\n")print(f"sync-allow-scripts: re-pinned {len(renames)} entr{'y' if len(renames) == 1 else 'ies'} in {pkg_path}")diff --git a/studio/backend/tests/test_startup_defers_torch.py b/studio/backend/tests/test_startup_defers_torch.pyindex 07fede369e4..e588dec550e 100644--- a/studio/backend/tests/test_startup_defers_torch.py+++ b/studio/backend/tests/test_startup_defers_torch.py@@ -426,12 +426,28 @@ def test_a_failed_detection_degrades_instead_of_raising():hw.DEVICE, hw.CHAT_ONLY, hw.CHAT_ONLY_REASON = saved_device, saved_chat, saved_reason+def _confine_the_torch_import_error(monkeypatch, hw):+ """Undo hw.TORCH_IMPORT_ERROR at teardown, for a test that watches an import fail.++ The three tests below restore ``builtins.__import__``, ``sys.modules`` and the detection+ verdict, but _has_torch() also writes that module global as a SIDE EFFECT, and nothing+ put it back: the worker carried ``OSError(...)`` into every later test in the same+ process. classify_torch_build(), _torch_reports_an_xpu_runtime() and+ _torch_reports_a_hip_runtime() all branch on it and read the wheel off DISK when it is+ set, ignoring the fake torch a later test installed in sys.modules -- so+ test_torch_cpu_build_on_nvidia_host.py failed only in the xdist worker that happened to+ inherit the leak. Snapshot-restore rather than force None: nothing here owns the value.+ """+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", hw.TORCH_IMPORT_ERROR)++def test_a_broken_torch_counts_as_no_torch(monkeypatch):"""_has_torch() must not let a non-ImportError escape into detection."""import builtinsfrom utils.hardware import hardware as hw+ _confine_the_torch_import_error(monkeypatch, hw)real_import = builtins.__import__def fake_import(name, *args, **kwargs):@@ -503,6 +519,7 @@ def test_a_broken_torch_purges_its_own_zombie(monkeypatch):from utils.hardware import hardware as hw+ _confine_the_torch_import_error(monkeypatch, hw)real_import = builtins.__import__saved = {name: mod for name, mod in sys.modules.items() if name.split(".")[0] == "torch"}@@ -542,6 +559,8 @@ def test_one_detection_pass_probes_torch_once(monkeypatch):saved = {n: m for n, m in sys.modules.items() if n.split(".")[0] == "torch"}saved_device = hw.DEVICEsaved_chat, saved_reason = hw.CHAT_ONLY, hw.CHAT_ONLY_REASON+ # Before the __import__ patch below, so the finally's monkeypatch.undo() puts it back last.+ _confine_the_torch_import_error(monkeypatch, hw)real_import = builtins.__import__attempts = []diff --git a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.pyindex aa1b6c68ade..5316dfba8a1 100644--- a/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py+++ b/studio/backend/tests/test_torch_cpu_build_on_nvidia_host.py@@ -52,6 +52,13 @@ def _shared_setup_1(monkeypatch, tmp_path):import sysmonkeypatch.setitem(sys.modules, "torch", _fake_torch("cpu"))+ # The fake above is only READ while the import is known good: TORCH_IMPORT_ERROR is a+ # module global _has_torch() writes as a side effect, and _torch_reports_an_xpu_runtime()+ # / _torch_reports_a_hip_runtime() / classify_torch_build() read the wheel off DISK+ # whenever it is set. A sibling that watched a broken import leaves it non-None for the+ # rest of the xdist worker, so pin it here rather than inherit it, as+ # cpu_torch_on_an_nvidia_host already does.+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", None)monkeypatch.setattr(hw.sys, "prefix", str(tmp_path))return sys@@ -1225,6 +1232,46 @@ def test_a_nameless_intel_card_counts_once_xpu_was_actually_chosen(monkeypatch,assert hw._devices_that_can_establish_a_mismatch(nameless) == nameless+def test_a_leaked_import_error_is_what_stops_the_fake_torch_being_read(monkeypatch, tmp_path):+ """Why _shared_setup_1 pins TORCH_IMPORT_ERROR: the assertion above is not self-contained.++ _torch_reports_an_xpu_runtime() consults sys.modules only while that global is None;+ set, it reads the wheel off disk and the fake +xpu torch above is never looked at, so+ the nameless Intel card stops counting and the last assertion of the previous test+ reads ``[] == [{...}]``. Nothing in this file writes the global -- _has_torch() does,+ as a side effect, from whichever test in the xdist worker last watched an import fail,+ which is why it went red under ``-n 4`` on one shard and green in every serial run.+ """+ sys = _shared_setup_1(monkeypatch, tmp_path)+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)+ monkeypatch.delenv("UNSLOTH_TORCH_INDEX_FAMILY", raising = False)+ (tmp_path / "unsloth_install_manifest.json").write_text(+ '{"schema": 1, "expected_torch_tag": "cpu"}', encoding = "utf-8"+ )+ # The on-disk arm is pinned to a CPU wheel, so this does not turn on what torch the+ # host running the suite happens to have installed.+ monkeypatch.setattr(hw, "_installed_torch_label_on_disk", lambda: "2.11.0+cpu")+ monkeypatch.setattr(+ hw,+ "_installed_torch_markers_on_disk",+ lambda: {"cuda": None, "hip": None, "xpu": None},+ )++ nameless = [{"vendor": "intel", "name": None, "index": 0}]+ xpu_torch = _fake_torch("cpu")+ xpu_torch.__version__ = "2.9.0+xpu"+ monkeypatch.setitem(sys.modules, "torch", xpu_torch)++ # Pinned, as _shared_setup_1 leaves it: the fake in sys.modules is what answers.+ assert hw._torch_reports_an_xpu_runtime() is True+ assert hw._devices_that_can_establish_a_mismatch(nameless) == nameless++ # Exactly what a sibling's _has_torch() leaves behind, and the verdict inverts.+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", "OSError('undefined symbol: cudaGetDeviceCount')")+ assert hw._torch_reports_an_xpu_runtime() is False+ assert hw._devices_that_can_establish_a_mismatch(nameless) == []++def test_an_accelerator_that_came_back_retires_the_cached_verdict(monkeypatch):"""Only reason and detail are refreshed here; DEVICE and CHAT_ONLY are not.@@ -1480,6 +1527,11 @@ def test_an_unimportable_torch_still_reports_the_cards(monkeypatch, tmp_path):monkeypatch.delitem(sys.modules, "torch", raising = False)monkeypatch.setattr(sys, "meta_path", [_Finder(), *sys.meta_path])+ # _has_torch() below writes hw.TORCH_IMPORT_ERROR as a side effect and nothing puts it+ # back, so snapshot-restore it: left set, it sends every later test in this worker down+ # the on-disk arm of the same three readers. Snapshot rather than None -- nothing here+ # owns the value.+ monkeypatch.setattr(hw, "TORCH_IMPORT_ERROR", hw.TORCH_IMPORT_ERROR)# _has_torch() is NOT forced here: it reports False for a wheel that will not import,# and the early return on it used to keep this host from the on-disk fallback below.assert hw._has_torch() is False, "the premise: an unimportable torch reads as absent"diff --git a/tests/test_formatter_fixed_point.py b/tests/test_formatter_fixed_point.pyindex 76a0ac31f6c..b55bcef9be3 100644--- a/tests/test_formatter_fixed_point.py+++ b/tests/test_formatter_fixed_point.py@@ -39,6 +39,29 @@ from run_ruff_format import ( # noqa: E402version_mismatch,)+#: What Windows' CreateProcess accepts for a whole command line, in characters. Documented by+#: Microsoft as the lpCommandLine cap and unchanged since Windows XP. POSIX has no equivalent+#: single limit (execve fails with E2BIG against ARG_MAX, ~2 MB on the runners), so Windows is+#: the binding constraint and the only number worth pinning.+_WINDOWS_COMMAND_LINE_LIMIT = 32767++#: Characters a single formatter invocation may build up to. The whole tracked set is ~2650+#: files whose copied paths under a pytest tmp_path come to ~325,000 characters -- 9.9x the cap+#: above -- so the single-invocation form this replaced could never run on Windows at all: it+#: died at CreateProcess with `[WinError 206] The filename or extension is too long` before ruff+#: opened one file. Nobody had seen it fail because every job that schedules this file is+#: ubuntu-24.04, where execve's ARG_MAX is ~2 MB and the same argv fits easily.+#:+#: A LENGTH budget rather than a file COUNT. A count has to be re-tuned every time the tracked+#: set or the path depth moves, and silently loses its margin in between: 200 files measured+#: 26,687 characters here, only 1.2x under the cap, so a fifth more files would have put it back+#: over. Accumulating by length is self-adjusting and needs no number kept in step with the+#: repo. 8000 leaves 4x headroom for the argv-quoting model below being wrong.+#:+#: Batching at all is safe because the hook is per-file: it formats each path independently, so+#: N calls over disjoint batches do exactly what one call over the union does.+_FORMAT_ARGV_BUDGET = 8000+_HOOK_ID = "ruff-format-with-kwargs"# The spacing pass refuses to rewrite itself ("skip modifying this script to# avoid self-edit loops"), so it is not a file the hook keeps at a fixed point@@ -222,6 +245,95 @@ class TestTheGuardCannotGoGreenHavingCheckedNothing:assert running_in_ci({"CI": ""}) is False+def formatter_argvs(copies: list[str], head: list[str] | None = None) -> "list[list[str]]":+ """Every command line the fixed-point guard will run, in order.++ Batches are accumulated until adding the next path would take the command line past+ `_FORMAT_ARGV_BUDGET`, so a single path longer than the budget still gets its own call+ rather than being dropped -- the caller would rather run one over-long command and see the+ OS refuse it than silently skip a file.++ The guard and the Windows-limit test below both go through this, deliberately. A test that+ only checked the budget constant would be measuring a number while the caller did something+ else, and deleting the batching at the call site would leave it green. Here there is one+ definition of what actually gets executed, so the limit test cannot drift away from the run.+ """+ head = head or [sys.executable, str(_ROOT / "scripts" / "run_ruff_format.py")]+ base = _command_line_length(head)+ argvs: list[list[str]] = []+ batch: list[str] = []+ used = base+ for path in copies:+ cost = _command_line_length([path])+ if batch and used + cost > _FORMAT_ARGV_BUDGET:+ argvs.append([*head, *batch])+ batch, used = [], base+ batch.append(path)+ used += cost+ if batch:+ argvs.append([*head, *batch])+ return argvs+++def _command_line_length(argv: list[str]) -> int:+ """What Windows counts against its command-line cap for this argv.++ CreateProcess is handed ONE string, so the cost is the arguments joined by the separating+ spaces, plus a pair of quotes around every argument a runner path forces (the hosted image+ checks out under `D:\\a\\unsloth\\unsloth`, no spaces, but `C:\\Users\\RUNNER~1\\AppData\\+ Local\\Temp` is where tmp_path lands and a user name with a space is normal off CI). Counted+ with the quotes always, because this is a headroom check and the cheap direction to be wrong+ in is pessimistic.+ """+ return sum(len(arg) + 3 for arg in argv)++[email protected](_VERDICT == "skip", reason = _RUFF_REASON or "")+def test_the_formatter_invocation_fits_in_a_windows_command_line():+ """The guard below must be able to START on Windows, not only pass on Linux.++ It used to pass every tracked file as one argv. That is ~2650 paths and, under a Windows+ tmp_path, roughly 325,000 characters against CreateProcess's 32,767 -- 9.9x over, so the+ call died with `[WinError 206] The filename or extension is too long` before ruff opened a+ single file. It had never been caught because every job that schedules this file is+ ubuntu-24.04, where execve's ARG_MAX is ~2 MB and the same argv fits with room to spare.++ So the limit is asserted here rather than left to a Windows runner to discover: this runs+ in the existing Linux job, needs no second platform, and goes red the moment someone+ reverts the batching or raises _FORMAT_ARGV_BUDGET past what the cap allows. Computed from the+ REAL file list and a realistic Windows tmp_path prefix, not from a remembered number, so+ the file set growing is what moves it.+ """+ names = eligible_files(_ROOT)+ assert len(names) > 1000, f"only {len(names)} files matched; the file list has gone vacuous"++ # A hosted Windows runner's pytest tmp_path. Longer than the Linux equivalent, which is the+ # point: the platform with the smallest cap also has the longest prefix.+ prefix = "C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\pytest-of-runner\\pytest-999\\test_0"+ copies = [prefix + "\\" + name for name in names]++ argvs = formatter_argvs(copies)+ # Every path is still formatted: batching may not silently drop the tail.+ assert [arg for argv in argvs for arg in argv[2:]] == copies, "batching lost or reordered files"++ worst = max(_command_line_length(argv) for argv in argvs)+ assert worst < _WINDOWS_COMMAND_LINE_LIMIT, (+ f"the widest of the {len(argvs)} formatter batches builds a {worst}-character command "+ f"line against Windows' {_WINDOWS_COMMAND_LINE_LIMIT}-character CreateProcess limit, so "+ f"this guard would die with WinError 206 on Windows before formatting anything. Lower "+ f"_FORMAT_ARGV_BUDGET (currently {_FORMAT_ARGV_BUDGET} over {len(names)} tracked files)."+ )+ # Non-vacuous: the batching has to be doing something. One call with everything on it is the+ # shape that was broken, and it must still be over the cap -- if it ever fits, this test is+ # measuring nothing and the batching can go.+ unbatched = _command_line_length([sys.executable, "run_ruff_format.py", *copies])+ assert unbatched > _WINDOWS_COMMAND_LINE_LIMIT, (+ f"the whole tracked set now builds a {unbatched}-character command line, under the "+ f"{_WINDOWS_COMMAND_LINE_LIMIT} cap, so batching is no longer load-bearing and this "+ "test no longer proves anything. Delete both, or say why they stay."+ )++@pytest.mark.skipif(_VERDICT == "skip", reason = _RUFF_REASON or "")def test_every_tracked_python_file_is_already_formatted(tmp_path):"""Run the hook over copies of the whole tracked set and expect no rewrite."""@@ -251,12 +363,11 @@ def test_every_tracked_python_file_is_already_formatted(tmp_path):target.write_bytes(originals[name])copies.append(str(target))- run = subprocess.run(- [sys.executable, str(_ROOT / "scripts" / "run_ruff_format.py"), *copies],- capture_output = True,- text = True,- )- assert run.returncode == 0, f"the formatter itself failed:\n{run.stdout}\n{run.stderr}"+ # Batched, and through the same builder the Windows-limit test above measures. One call with+ # all ~2650 paths on it is 9.9x over Windows' CreateProcess cap and dies with WinError 206.+ for argv in formatter_argvs(copies):+ run = subprocess.run(argv, capture_output = True, text = True)+ assert run.returncode == 0, f"the formatter itself failed:\n{run.stdout}\n{run.stderr}"drifted = [name for name in names if (tmp_path / name).read_bytes() != originals[name]]assert not drifted, (diff --git a/tests/test_scripts_write_lf.py b/tests/test_scripts_write_lf.pynew file mode 100644index 00000000000..f54b8d5dd31--- /dev/null+++ b/tests/test_scripts_write_lf.py@@ -0,0 +1,250 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+"""Scripts that rewrite tracked files in place must write LF, on every platform.++`.gitattributes` opens with `*.py text eol=lf`, so every tracked Python file in this repo is+LF on disk and is meant to stay that way. Three scripts rewrite tracked files through the same+temp-file-then-os.replace shape, and all three opened the temp file in text mode with the+DEFAULT newline, which translates each "\\n" to `os.linesep`:++ scripts/enforce_kwargs_spacing.py every tracked .py the formatting hook touches+ scripts/stamp_studio_release.py studio/backend/utils/_studio_release_build.py+ scripts/scan_packages.py studio/backend/requirements/*.txt, under --fix++On Linux `os.linesep` is "\\n" and the round trip is a no-op, which is why this survived. On+Windows it rewrites the file to CRLF. For enforce_kwargs_spacing that is every file a+contributor edits, because the read side is `tokenize.open`, which normalises CRLF to LF in+memory, so the write is what decides the ending. The symptom is a whole-file diff on files the+contributor did not change, from running the project's own pre-commit hook.++Found by running tests/test_formatter_fixed_point.py on a real windows-latest runner (see the+staging evidence on the PR): the guard reported ~1300 of ~2650 tracked files as drifted, none+of which had drifted.++Two halves, deliberately:++ * the BEHAVIOURAL tests below prove the end-to-end intent -- a CRLF file in, LF bytes out --+ and on Windows they exercise the translation for real. On Linux they cannot fail if the fix+ is reverted, because `os.linesep` is already "\\n" there. They are the demonstration.+ * `test_every_in_place_rewriter_names_its_newline` is the DURABLE guard. It reads the call+ sites and fails on any platform the moment a `newline =` argument goes missing, which is+ the only way to catch a revert in the Linux job that actually runs.++Same split, and the same reason, as test_formatter_fixed_point.py's Windows command-line check:+assert the property where the platform cannot demonstrate it.+"""++from __future__ import annotations++import ast+import importlib.util+import json+import sys+from pathlib import Path++import pytest++_ROOT = Path(__file__).resolve().parent.parent+_SCRIPTS = _ROOT / "scripts"++#: The in-place rewriters this file covers, as (path, the function holding the write).+#: Named on the real files so a rename fails here rather than silently dropping coverage. A pair,+#: not a {script: func} mapping, because one script can hold more than one rewriter.+#:+#: The list is what it is because the rule is "rewrites a TRACKED file in place", not "uses+#: os.fdopen": the last two write through plain open()/Path.write_text and have exactly the same+#: default-newline defect. sync_allow_scripts_pins is the one that matters most -- it runs as a+#: pre-commit hook with --fix over studio/frontend/package.json, which .gitattributes pins to+#: eol=lf.+_REWRITERS = (+ ("enforce_kwargs_spacing.py", "_atomic_write_text"),+ ("stamp_studio_release.py", "_atomic_write_text"),+ ("scan_packages.py", "update_req_file"),+ ("scan_packages.py", "_write_baseline"),+ ("sync_allow_scripts_pins.py", "main"),+)+++def _load(name: str):+ """Import a scripts/ module by filename, without putting scripts/ on sys.path for good."""+ path = _SCRIPTS / name+ assert path.is_file(), f"{path} is gone; _REWRITERS is stale"+ spec = importlib.util.spec_from_file_location(f"_scripts_{path.stem}", path)+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++# ── the durable half: the call sites name a newline ───────────────────────────────────────────+++def _text_write_calls(tree: ast.AST, func_name: str) -> list[ast.Call]:+ """Every `os.fdopen(...)`/`open(...)`/`Path.write_text(...)` text-mode WRITE in the function.++ Binary mode is excluded: `newline` is meaningless there and passing it raises. A call with+ no mode argument at all defaults to "r", so it is not a write and is skipped too.++ `write_text` is here because it carries the identical default: `Path.write_text(data,+ encoding = ...)` leaves `newline` at None and so translates to os.linesep exactly like+ `open()` does. Leaving it out would let a rewriter swap one for the other and drop out of+ this guard silently, which is how sync_allow_scripts_pins.py was writing CRLF in the first+ place.+ """+ target = next(+ (+ node+ for node in ast.walk(tree)+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name+ ),+ None,+ )+ assert target is not None, f"{func_name} is gone; _REWRITERS is stale"++ calls = []+ for node in ast.walk(target):+ if not isinstance(node, ast.Call):+ continue+ func = node.func+ named = (+ func.attr+ if isinstance(func, ast.Attribute)+ else func.id+ if isinstance(func, ast.Name)+ else ""+ )+ if named == "write_text":+ # No mode argument to read: write_text is always a text-mode write.+ calls.append(node)+ continue+ if named not in ("fdopen", "open"):+ continue+ # Mode is the second positional for both open() and os.fdopen().+ mode = next(+ (+ arg.value+ for arg in node.args[1:2]+ if isinstance(arg, ast.Constant) and isinstance(arg.value, str)+ ),+ None,+ ) or next(+ (+ kw.value.value+ for kw in node.keywords+ if kw.arg == "mode"+ and isinstance(kw.value, ast.Constant)+ and isinstance(kw.value.value, str)+ ),+ None,+ )+ if mode is None or "b" in mode or not any(ch in mode for ch in "wax+"):+ continue+ calls.append(node)+ return calls++[email protected](("script", "func"), _REWRITERS)+def test_every_in_place_rewriter_names_its_newline(script, func):+ """A text-mode write in one of these must say what line ending it wants.++ This is the half that can fail on Linux. `os.linesep` is "\\n" here, so no behavioural test+ in this file can notice the default coming back -- only reading the call site can.+ """+ tree = ast.parse((_SCRIPTS / script).read_text(encoding = "utf-8"))+ calls = _text_write_calls(tree, func)+ assert calls, f"no text-mode write found in {script}:{func}; this guard has gone vacuous"+ for call in calls:+ kwargs = {kw.arg for kw in call.keywords}+ assert "newline" in kwargs, (+ f"{script}:{call.lineno} in {func}() opens a text file for writing without a "+ "`newline =` argument, so Python translates every '\\n' to os.linesep and this "+ "rewrites the whole tracked file to CRLF on Windows. .gitattributes pins these to "+ 'LF (`*.py text eol=lf`). Pass newline = "\\n".'+ )+ assert "encoding" in kwargs, (+ f"{script}:{call.lineno} in {func}() opens a text file for writing without an "+ "`encoding =` argument, so it takes the locale codec and round-trips differently "+ 'between runners. Pass encoding = "utf-8".'+ )+++def test_the_repo_agrees_these_files_are_lf():+ """The premise. If .gitattributes stops pinning LF, everything above is arguing for nothing."""+ attributes = (_ROOT / ".gitattributes").read_text(encoding = "utf-8")+ assert "*.py text eol=lf" in attributes, (+ ".gitattributes no longer pins tracked Python files to LF, so the rewriters this file "+ "guards have no ending to preserve and this whole file needs re-reading"+ )+++# ── the demonstration half: a CRLF file in, LF bytes out ──────────────────────────────────────+++def test_the_spacing_pass_writes_lf_for_a_crlf_source(tmp_path):+ """The headline path: what the pre-commit hook does to a contributor's file."""+ module = _load("enforce_kwargs_spacing.py")+ target = tmp_path / "sample.py"+ # CRLF on disk, and something the pass will actually rewrite, so the write really happens.+ target.write_bytes(b"def f(a=1, b=2):\r\n return a + b\r\n")++ module.process_file(target)++ written = target.read_bytes()+ assert (+ b"\r\n" not in written+ ), f"the spacing pass wrote CRLF into a file .gitattributes pins to LF: {written!r}"+ assert written.endswith(b"\n"), written+ # Non-vacuous: it must actually have done its job, or "no CRLF" is just "no write".+ assert b"a = 1" in written, written+++def test_the_release_stamp_writes_lf(tmp_path):+ """A tracked .py generated from a Python string literal, so LF is the only correct ending."""+ module = _load("stamp_studio_release.py")+ target = tmp_path / "_studio_release_build.py"++ module._atomic_write_text(target, 'VERSION = "1.2.3"\nBUILD = 7\n', encoding = "utf-8")++ written = target.read_bytes()+ assert b"\r\n" not in written, f"the release stamp wrote CRLF: {written!r}"+ assert written == b'VERSION = "1.2.3"\nBUILD = 7\n', written+++def test_the_requirements_fixer_writes_lf_and_utf8(tmp_path):+ """--fix rewrites tracked requirements files, and used to take the locale codec too."""+ module = _load("scan_packages.py")+ target = tmp_path / "reqs.txt"+ # CRLF, plus a non-ASCII comment: the encoding half of the same call.+ target.write_bytes("requests==2.0.0 # naïve pin\r\nurllib3==1.0.0\r\n".encode("utf-8"))++ module.update_req_file(str(target), {2: "urllib3==2.0.0"})++ written = target.read_bytes()+ assert b"\r\n" not in written, f"the requirements fixer wrote CRLF: {written!r}"+ assert "naïve".encode("utf-8") in written, "the non-ASCII comment did not survive as UTF-8"+ assert b"urllib3==2.0.0" in written, written+++def test_the_allow_scripts_pin_sync_writes_lf(tmp_path):+ """The other pre-commit hook that rewrites a tracked file, on a file pinned `eol=lf`.++ `.gitattributes` carries `studio/frontend/** text=auto eol=lf`, and .pre-commit-config.yaml+ runs this one with `--fix` on every package.json touch, so it is the same contributor-facing+ surface as the spacing hook above.+ """+ module = _load("sync_allow_scripts_pins.py")+ (tmp_path / "package.json").write_bytes(+ json.dumps({"name": "x", "allowScripts": {"[email protected]": True}}).encode("utf-8")+ )+ (tmp_path / "package-lock.json").write_bytes(+ json.dumps(+ {"packages": {"node_modules/esbuild": {"version": "0.2.0", "hasInstallScript": True}}}+ ).encode("utf-8")+ )++ assert module.main(["--fix", "--dir", str(tmp_path)]) == 0++ written = (tmp_path / "package.json").read_bytes()+ assert b"\r\n" not in written, f"the allowScripts pin sync wrote CRLF: {written!r}"+ # Non-vacuous: the re-pin must actually have happened, or "no CRLF" is just "no write".+ assert b'"[email protected]"' in written, written