Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Make the venv-hardening decode strict, and say which codepoints differed (#10432)

Daniel Han6dcfd4319604
diff --git a/tests/python/test_windows_python_venv_hardening.py b/tests/python/test_windows_python_venv_hardening.py
index 44071be2149..e97405a3bba 100644
--- a/tests/python/test_windows_python_venv_hardening.py
+++ b/tests/python/test_windows_python_venv_hardening.py
@@ -63,12 +63,65 @@ def _run_powershell(shell: str, script: str, env: dict[str, str]) -> str:
capture_output = True,
text = True,
encoding = "utf-8",
+ # strict, so the next mismatch raises here rather than turning into U+FFFD and
+ # failing an assertion somewhere downstream about a value that was written fine.
+ errors = "strict",
env = env,
timeout = 30,
)
return result.stdout.strip()
+def _assert_same_text(actual: str, expected: str, note: str) -> None:
+ """`actual == expected`, but a mismatch names the codepoints that differ.
+
+ A bare `==` between two paths that differ in one accented character is unreadable in
+ CI: pytest renders its diff through the same stdout that mangled the string, so on a
+ non-UTF-8 console one side comes back as U+FFFD and the report accuses the half that
+ was right. Codepoints survive any code page, so they are what gets printed.
+ """
+ if actual == expected:
+ return
+ head = len(os.path.commonprefix([actual, expected]))
+
+ def describe(text: str) -> str:
+ shown = text[:24]
+ codepoints = " ".join(f"U+{ord(ch):04X}" for ch in shown)
+ return f"{shown!r} [{codepoints}{' ...' if len(text) > len(shown) else ''}]"
+
+ lines = [
+ note,
+ f" first {head} characters match, then:",
+ f" actual {describe(actual[head:])}",
+ f" expected {describe(expected[head:])}",
+ ]
+ # Only claim a cause when the bytes show one, and only the cause that is still possible.
+ # _run_powershell decodes stdout as strict UTF-8, so a transport mis-decode raises there
+ # rather than arriving here: mojibake that reaches this point was produced inside the
+ # script, which is the product defect these cases exist to catch. Saying "the script did
+ # not lose the path" would be exactly backwards, and a wrong explanation on a real
+ # restore bug costs more than none.
+ if _looks_like_a_mis_decode(actual[head:], expected[head:]):
+ lines.append(
+ " actual is expected's UTF-8 bytes read back through a single-byte code page. "
+ "Stdout is decoded strictly as UTF-8 here, so the shell handed us these "
+ "characters: the mis-decode happened inside the script, not in transport. Check "
+ "the marker read/write path for a missing -Encoding utf8."
+ )
+ raise AssertionError("\n".join(lines))
+
+
+def _looks_like_a_mis_decode(actual: str, expected: str) -> bool:
+ """True when `actual` is `expected` encoded UTF-8 and decoded through a legacy code page."""
+ for codec in ("cp1252", "latin-1", "cp437", "cp850"):
+ try:
+ if expected.encode("utf-8").decode(codec) == actual:
+ return True
+ except (UnicodeDecodeError, UnicodeEncodeError):
+ continue
+ return False
+
+
def _uv_cache_functions(source: str) -> str:
return "".join(
_extract(rf" function {name} \{{.*?\n \}}\n", source)
@@ -862,7 +915,7 @@ Restore-StudioUvCacheMarker -StudioRoot $env:TEST_STUDIO_HOME
env["TEST_MARKER"] = str(marker)
result = json.loads(_run_powershell(shell, script, env).splitlines()[-1])
- assert result["After"] == expected, result
+ _assert_same_text(result["After"], expected, "the rollback did not restore the marker it saved")
@pytest.mark.skipif(not POWERSHELLS, reason = "PowerShell is unavailable")