Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Make run_pwsh set the writing end of the pipe, not just the reading end
Daniel Han45469ede4c96
diff --git a/Microsoft/Windows/PowerShell/ModuleAnalysisCache b/Microsoft/Windows/PowerShell/ModuleAnalysisCachenew file mode 100644index 00000000000..73a2bdb0a85Binary files /dev/null and b/Microsoft/Windows/PowerShell/ModuleAnalysisCache differdiff --git a/tests/_shared/unsloth_pwsh_runner.py b/tests/_shared/unsloth_pwsh_runner.pyindex 28207b79a66..1f724cd0fbc 100644--- a/tests/_shared/unsloth_pwsh_runner.py+++ b/tests/_shared/unsloth_pwsh_runner.py@@ -140,6 +140,61 @@ def _crash_reason(proc: subprocess.CompletedProcess) -> str | None:return None+# Prepended to a -Command script so its stdout is UTF-8 whatever the host console is set to.+# UTF8Encoding($false), not [Text.Encoding]::UTF8: the latter emits a preamble, which lands in+# stdout as a BOM and breaks the first assertion of whatever reads it.+_UTF8_PROLOGUE = "[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false)\n"+++# Python's own aliases for UTF-8, after "_" is folded to "-". A caller that spells it any of+# these ways wants what we want and still needs the writing end set.+_UTF8_ALIASES = frozenset({"utf-8", "utf8", "u8", "utf", "u-8", "cp65001"})+++def _agree_on_utf8(argv: list[str], kwargs: dict) -> list[str]:+ """Make both ends of the pipe use UTF-8. Returns the argv to run; `kwargs` is updated.++ Nobody was setting the WRITING end, so the answer depended on the host's code pages.+ Windows PowerShell 5.1 writes a redirected pipe in the OEM code page (cp437 on a US box,+ where U+00E4 leaves as one 0x84 byte) while pwsh 7 writes UTF-8. `text = True` alone then+ decodes with the ANSI code page, which round-trips neither: 5.1 gives U+FFFD and pwsh+ gives mojibake. That is what test_a_non_ascii_marker_survives_the_rollback fails on in+ parity CI, on both shells.++ Naming `encoding = "utf-8"` at the call site, which is what that test already does, fixes+ pwsh 7 and makes 5.1 worse: 0x84 is not valid UTF-8, so the decode raises inside+ subprocess's reader thread, where the exception is swallowed and the attribute is left as+ None. The caller gets `returncode == 0` and `stdout is None`, so the run reads as a script+ that printed nothing rather than as a pipe nobody agreed on.++ Hence both halves, always together. The prologue makes the shell write UTF-8 whatever the+ console is set to, and the decode reads it back. The pair is exact for every code point on+ both shells, and a no-op where the output was already UTF-8 or pure ASCII.++ Left alone: byte-mode callers, who asked for bytes and can decode as they like, and a+ caller that named some OTHER encoding, who has chosen. `-File` has no script string to+ prepend to, so it gets the decode half only, which is the one available to it.++ The call site's own list is never written to: a caller that reuses its argv, or reads it+ after the call, sees exactly what it built.+ """+ if not (kwargs.get("text") or kwargs.get("universal_newlines")):+ return argv+ named = kwargs.get("encoding")+ if named is not None and named.lower().replace("_", "-") not in _UTF8_ALIASES:+ return argv+ kwargs["encoding"] = "utf-8"+ # -Command only. PowerShell accepts unambiguous prefixes, but every call site here spells+ # it in full, and prepending to the wrong element would run the prologue as a file path.+ try:+ script = argv.index("-Command") + 1+ except ValueError:+ return argv+ if script >= len(argv) or not isinstance(argv[script], str):+ return argv+ return argv[:script] + [_UTF8_PROLOGUE + argv[script]] + argv[script + 1 :]++def run_pwsh(argv: list[str],*,@@ -169,6 +224,8 @@ def run_pwsh(if attempts < 1:raise ValueError(f"attempts must be >= 1, got {attempts}")+ argv = _agree_on_utf8(argv, kwargs)+# Redirect only pwsh's own startup cache, leaving every other variable as the call site meant it: `env = None`# still means "inherit", and a hermetic env dict still gets exactly the keys it listed plus this one.env = kwargs.get("env")diff --git a/tests/python/test_pwsh_runner_encoding.py b/tests/python/test_pwsh_runner_encoding.pynew file mode 100644index 00000000000..47c1b48cfa1--- /dev/null+++ b/tests/python/test_pwsh_runner_encoding.py@@ -0,0 +1,169 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""run_pwsh has to round-trip text, and until now that depended on the host's code pages.++Nobody set the WRITING end. Windows PowerShell 5.1 writes a redirected pipe in the OEM code+page and pwsh 7 writes UTF-8, while `text = True` decodes with the ANSI one, so a non-ASCII+character survived only where those happened to agree. On a cp437/cp1252 box 5.1 gives+U+FFFD, and on a runner whose console is UTF-8 both shells give mojibake. That is what+test_a_non_ascii_marker_survives_the_rollback fails on in parity CI.++Naming `encoding = "utf-8"` at the call site, which that test already does, is the half-fix:+it corrects pwsh 7 and makes 5.1 worse, because 0x84 is not valid UTF-8 and the decode raises+inside subprocess's reader thread, where the exception is swallowed and stdout is left None.++The behavioural tests below are the point: they run a real shell and compare the string that+comes back with the string that went in. The rest pin the shape of the decision, most of+whose branches are about NOT interfering with a caller.+"""++from __future__ import annotations++import pathlib+import shutil+import subprocess+import sys++import pytest++sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "_shared"))++from unsloth_pwsh_runner import _UTF8_PROLOGUE, _agree_on_utf8, run_pwsh # noqa: E402++# Both, where both exist: the two disagree about what a redirected pipe is encoded in, which+# is the whole reason this module has anything to fix.+POWERSHELLS = [p for p in (shutil.which("pwsh"), shutil.which("powershell")) if p]++# One character per interesting width, because the failure mode differs by width: U+00E4 is a+# single byte in cp437 and two in UTF-8, U+4E2D has no code-page representation at all, and the+# emoji is a surrogate pair in PowerShell's UTF-16 strings.+SAMPLES = {+ "latin1": "käffee",+ "cjk": "中文",+ "astral": "\U0001f600",+ "mixed": "café 中 \U0001f600",+}++[email protected](not POWERSHELLS, reason = "PowerShell is unavailable")[email protected]("shell", POWERSHELLS)[email protected]("name", sorted(SAMPLES), ids = sorted(SAMPLES))+def test_a_non_ascii_string_survives_the_round_trip(shell: str, name: str):+ """The whole point. Built from code points inside PowerShell so the script itself stays+ ASCII: a script that carried the character would be testing how the -Command argument is+ encoded, which is a different pipe."""+ text = SAMPLES[name]+ literal = " + ".join(f"[char]::ConvertFromUtf32({ord(c)})" for c in text)+ done = run_pwsh(+ [+ shell,+ "-NoProfile",+ "-NonInteractive",+ "-Command",+ f"Write-Output ('<' + {literal} + '>')",+ ],+ check = True,+ capture_output = True,+ text = True,+ timeout = 60,+ )+ out = done.stdout+ assert out[out.index("<") + 1 : out.rindex(">")] == text++[email protected](not POWERSHELLS, reason = "PowerShell is unavailable")[email protected]("shell", POWERSHELLS)+def test_the_prologue_adds_no_bom(shell: str):+ """[Text.Encoding]::UTF8 carries a preamble; the one we set must not, or every caller's+ first assertion sees a U+FEFF it did not print."""+ done = run_pwsh(+ [shell, "-NoProfile", "-NonInteractive", "-Command", "Write-Output 'plain'"],+ check = True,+ capture_output = True,+ text = True,+ timeout = 60,+ )+ assert done.stdout.startswith("plain"), repr(done.stdout)++[email protected](not POWERSHELLS, reason = "PowerShell is unavailable")[email protected]("shell", POWERSHELLS)+def test_without_the_fix_the_same_string_does_not_survive(shell: str):+ """The control. Same script, run the old way, on this host. It is allowed to pass -- a+ box whose console is already UTF-8 was never broken -- so this asserts the two agree+ when it does, and that the fixed one is right when they do not."""+ script = "Write-Output ('<' + [char]::ConvertFromUtf32(0x00e4) + '>')"+ argv = [shell, "-NoProfile", "-NonInteractive", "-Command", script]+ old = subprocess.run(argv, capture_output = True, text = True, timeout = 60).stdout+ new = run_pwsh(argv, check = True, capture_output = True, text = True, timeout = 60).stdout+ assert new[new.index("<") + 1 : new.rindex(">")] == "ä"+ if "ä" in old:+ assert old.strip() == new.strip()+++def test_a_command_script_is_prefixed_once():+ kwargs = {"text": True}+ argv = _agree_on_utf8(["pwsh", "-Command", "Write-Output 1"], kwargs)+ assert argv[-1] == _UTF8_PROLOGUE + "Write-Output 1"+ assert kwargs["encoding"] == "utf-8"+++def test_the_call_sites_own_list_is_not_written_to():+ """A caller that reuses its argv, or reports it on failure, must see what it built."""+ original = ["pwsh", "-Command", "Write-Output 1"]+ argv = _agree_on_utf8(original, {"text": True})+ assert original == ["pwsh", "-Command", "Write-Output 1"]+ assert argv is not original+++ ("kwargs", "why"),+ [+ ({}, "bytes mode: the caller decodes, and a prologue would change the bytes"),+ ({"text": False}, "the same, spelled out"),+ ({"text": True, "encoding": "cp1252"}, "an encoding the caller chose is the answer"),+ ({"text": True, "encoding": "latin-1"}, "the same for one that cannot fail"),+ ],+ ids = ["bytes", "text-false", "explicit-cp1252", "explicit-latin1"],+)+def test_a_caller_that_already_decided_is_left_alone(kwargs, why):+ before = dict(kwargs)+ argv = _agree_on_utf8(["pwsh", "-Command", "Write-Output 1"], kwargs)+ assert argv[-1] == "Write-Output 1", why+ assert kwargs == before, why++[email protected]("spelling", ["utf-8", "utf8", "UTF-8", "utf_8", "U8", "cp65001"])+def test_a_caller_that_asked_for_utf8_still_gets_the_writing_end(spelling):+ """The half-fix this replaces. Naming the decode alone is what turns 5.1's OEM byte into+ a decode error inside subprocess's reader thread and a silent `stdout is None`, so the+ call sites that already spell it must pick up the prologue rather than be skipped."""+ kwargs = {"text": True, "encoding": spelling}+ argv = _agree_on_utf8(["pwsh", "-Command", "Write-Output 1"], kwargs)+ assert argv[-1] == _UTF8_PROLOGUE + "Write-Output 1"+ assert kwargs["encoding"] == "utf-8"+++def test_errors_alone_does_not_count_as_choosing_an_encoding():+ kwargs = {"text": True, "errors": "replace"}+ argv = _agree_on_utf8(["pwsh", "-Command", "Write-Output 1"], kwargs)+ assert argv[-1] == _UTF8_PROLOGUE + "Write-Output 1"+ assert kwargs == {"text": True, "errors": "replace", "encoding": "utf-8"}+++ "argv",+ [+ ["pwsh", "-NoProfile", "-File", "script.ps1"],+ ["pwsh", "-NoProfile"],+ ["pwsh", "-NoProfile", "-Command"],+ ],+ ids = ["-File", "no-command", "trailing-command"],+)+def test_an_invocation_with_no_script_string_still_gets_the_decode_half(argv):+ """-File names a path, so there is nothing to prepend to. Decoding UTF-8 is still right+ for pwsh 7, and is the only half available here."""+ kwargs = {"text": True}+ assert _agree_on_utf8(list(argv), kwargs) == argv+ assert kwargs["encoding"] == "utf-8"