Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# Unsloth - 2x faster, 60% less VRAM LLM training and finetuning
# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
"""Subprocess text calls must pin encoding utf-8; Windows defaults to cp1252 and aborts GGUF export."""
from __future__ import annotations
import ast
import subprocess
import sys
from pathlib import Path
import pytest
SAVE_PY = Path(__file__).resolve().parents[2] / "unsloth" / "save.py"
def _is_subprocess_call(node: ast.Call) -> bool:
"""True for ``subprocess.Popen(...)`` / ``subprocess.run(...)``."""
func = node.func
return (
isinstance(func, ast.Attribute)
and func.attr in {"Popen", "run"}
and isinstance(func.value, ast.Name)
and func.value.id == "subprocess"
)
def _kw(node: ast.Call, name: str):
for kw in node.keywords:
if kw.arg == name:
return kw.value
return None
def _is_true(value) -> bool:
return isinstance(value, ast.Constant) and value.value is True
def _is_text_mode(node: ast.Call) -> bool:
"""Text mode = ``text=True`` or ``universal_newlines=True``."""
return _is_true(_kw(node, "text")) or _is_true(_kw(node, "universal_newlines"))
def _collect_text_mode_subprocess_calls() -> list[ast.Call]:
tree = ast.parse(SAVE_PY.read_text(encoding = "utf-8"), filename = str(SAVE_PY))
return [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call) and _is_subprocess_call(node) and _is_text_mode(node)
]
def test_text_mode_subprocess_calls_exist():
"""Guard the guard: ensure save.py still has text-mode subprocess calls so
the drift test below isn't vacuously passing."""
calls = _collect_text_mode_subprocess_calls()
assert len(calls) >= 6, (
f"Expected several text-mode subprocess calls in {SAVE_PY.name}, "
f"found {len(calls)} -- has the file been restructured?"
)
def test_save_subprocess_text_calls_declare_utf8_encoding():
"""Every text-mode subprocess call in save.py must pin encoding='utf-8'
(else Windows cp1252 crashes reading child output; #2660 fix)."""
offenders = []
for node in _collect_text_mode_subprocess_calls():
enc = _kw(node, "encoding")
ok = isinstance(enc, ast.Constant) and enc.value == "utf-8"
if not ok:
offenders.append(node.lineno)
assert not offenders, (
"Text-mode subprocess call(s) in unsloth/save.py missing "
'encoding="utf-8" (UnicodeDecodeError on Windows, #2660) at line(s): '
+ ", ".join(map(str, sorted(offenders)))
)
def test_utf8_replace_decodes_non_cp1252_subprocess_output():
"""A real subprocess reproduces the bug: the child emits U+201D (UTF-8 E2 80 9D, where 0x9D is
undefined in cp1252), so cp1252 decoding raises while utf-8 with replace reads it."""
# All-ASCII argv; the child builds the non-ASCII char so it is locale-independent.
child = (
"import sys; "
"sys.stdout.buffer.write(('tensor ' + chr(0x201D) + ' x\\n').encode('utf-8'))"
)
raw = subprocess.run([sys.executable, "-c", child], capture_output = True).stdout
assert b"\x9d" in raw
# cp1252 (the Windows default) cannot decode this output.
with pytest.raises(UnicodeDecodeError):
raw.decode("cp1252")
result = subprocess.run(
[sys.executable, "-c", child],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
)
assert result.stdout.startswith("tensor ")
assert "”" in result.stdout