Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Text I/O must pass encoding explicitly, since Windows falls back to the ANSI codepage, not UTF-8."""
from __future__ import annotations
import sys
import subprocess
import ast
import importlib.util
import json
import os
from pathlib import Path
from types import SimpleNamespace
import pytest
def _shared_setup_1(__file__):
import sys
backend = str(Path(__file__).resolve().parent.parent)
if backend not in sys.path:
sys.path.insert(0, backend)
BACKEND_ROOT = Path(__file__).resolve().parent.parent
# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped.
_SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__")
# Vendored packages are byte-identical to their wheel and get UTF-8 open via their loader.
_UTF8_BY_LOADER = {"vendor/laya": "core/systemone/laya_runtime.py"}
# Path.open()'s signature is what tells it apart from other libraries' open(),
# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...).
_FILE_MODE_CHARS = set("rwxabt+")
_PATH_OPEN_ARGS = ("mode", "buffering", "encoding", "errors", "newline")
_PATH_OPEN_KWARGS = set(_PATH_OPEN_ARGS)
_PATH_OPEN_ENCODING_ARG = _PATH_OPEN_ARGS.index("encoding")
_SUBPROCESS_CALLS = {"run", "Popen", "check_output", "check_call", "call"}
# open() takes encoding as its fourth argument, and os.fdopen forwards the same signature.
_OPEN_ENCODING_ARG = 3
def _studio_sources() -> list[Path]:
return [
path
for path in sorted(BACKEND_ROOT.rglob("*.py"))
if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts)
and not any(
path.relative_to(BACKEND_ROOT).as_posix().startswith(prefix + "/")
for prefix in _UTF8_BY_LOADER
)
]
def _has_keyword(node: ast.Call, name: str) -> bool:
return any(keyword.arg == name for keyword in node.keywords)
def _mode_is_binary(node: ast.Call) -> bool:
mode: str | None = None
if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant):
value = node.args[1].value
mode = value if isinstance(value, str) else None
for keyword in node.keywords:
if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
value = keyword.value.value
if isinstance(value, str):
mode = value
return bool(mode and "b" in mode)
def _open_has_encoding(node: ast.Call) -> bool:
"""open()/os.fdopen() also take encoding positionally: open(p, "w", 1, "utf-8")."""
return _has_keyword(node, "encoding") or len(node.args) > _OPEN_ENCODING_ARG
def _path_open_mode(node: ast.Call) -> str | None:
if node.args and isinstance(node.args[0], ast.Constant):
value = node.args[0].value
if isinstance(value, str):
return value
for keyword in node.keywords:
if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
value = keyword.value.value
if isinstance(value, str):
return value
return None
def _is_path_open(node: ast.Call) -> bool:
"""True only for calls matching ``Path.open``'s signature."""
if len(node.args) > len(_PATH_OPEN_ARGS):
return False
if any(k.arg not in _PATH_OPEN_KWARGS for k in node.keywords):
return False
mode = _path_open_mode(node)
if mode is not None:
return bool(mode) and set(mode) <= _FILE_MODE_CHARS
return not node.args
def _path_open_has_encoding(node: ast.Call) -> bool:
"""Path.open() also takes encoding positionally: open("w", 1, "utf-8")."""
return _has_keyword(node, "encoding") or len(node.args) > _PATH_OPEN_ENCODING_ARG
def _call_name(node: ast.Call) -> str | None:
func = node.func
if isinstance(func, ast.Name):
return func.id
if isinstance(func, ast.Attribute):
return func.attr
return None
def _subprocess_names(tree: ast.AST) -> set[str]:
"""Names subprocess is reachable under here, e.g. `import subprocess as _sp`."""
names = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
if alias.name == "subprocess":
names.add(alias.asname or alias.name)
return names
def _subprocess_aliases(tree: ast.AST, names: set[str]) -> set[str]:
"""Bare names bound to subprocess callables count too, e.g. install_wheel(run = subprocess.run)."""
def _is_bound(value: ast.expr | None) -> bool:
return (
isinstance(value, ast.Attribute)
and value.attr in _SUBPROCESS_CALLS
and isinstance(value.value, ast.Name)
and value.value.id in names
)
aliases: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module == "subprocess":
aliases.update(a.asname or a.name for a in node.names if a.name in _SUBPROCESS_CALLS)
elif isinstance(node, ast.Assign) and _is_bound(node.value):
aliases.update(t.id for t in node.targets if isinstance(t, ast.Name))
elif isinstance(node, ast.AnnAssign) and _is_bound(node.value):
if isinstance(node.target, ast.Name):
aliases.add(node.target.id)
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
args = node.args
positional = args.posonlyargs + args.args
padded = [None] * (len(positional) - len(args.defaults)) + list(args.defaults)
pairs = list(zip(positional, padded)) + list(zip(args.kwonlyargs, args.kw_defaults))
aliases.update(arg.arg for arg, default in pairs if _is_bound(default))
return aliases
def _is_subprocess_call(node: ast.Call, names: set[str], aliases: set[str]) -> bool:
func = node.func
if isinstance(func, ast.Name):
return func.id in aliases
if not isinstance(func, ast.Attribute) or func.attr not in _SUBPROCESS_CALLS:
return False
value = func.value
return isinstance(value, ast.Name) and value.id in names
def _text_mode_subprocess(node: ast.Call) -> bool:
for keyword in node.keywords:
if keyword.arg not in ("text", "universal_newlines"):
continue
if isinstance(keyword.value, ast.Constant) and keyword.value.value is True:
return True
return False
def _text_mode_dict(node: ast.Dict) -> bool:
"""A ``{"text": True, ...}`` literal with no "encoding" key."""
keys = [k.value for k in node.keys if isinstance(k, ast.Constant)]
if "encoding" in keys:
return False
for key, value in zip(node.keys, node.values):
if not isinstance(key, ast.Constant) or key.value not in (
"text",
"universal_newlines",
):
continue
if isinstance(value, ast.Constant) and value.value is True:
return True
return False
def _splatted_names(tree: ast.AST) -> set[str]:
"""Names handed to a call as ``**name``."""
names = set()
for node in ast.walk(tree):
if isinstance(node, ast.Call):
for keyword in node.keywords:
if keyword.arg is None and isinstance(keyword.value, ast.Name):
names.add(keyword.value.id)
return names
def _encoding_assigned_later(tree: ast.AST, name: str) -> bool:
"""``name["encoding"] = ...`` somewhere, so the literal need not carry it."""
for node in ast.walk(tree):
if not isinstance(node, ast.Subscript) or not isinstance(node.ctx, ast.Store):
continue
target, key = node.value, node.slice
if isinstance(target, ast.Name) and target.id == name:
if isinstance(key, ast.Constant) and key.value == "encoding":
return True
return False
def _splatted_kwargs_offenders(tree: ast.AST) -> list[ast.Dict]:
"""Only kwargs dicts splatted into a call are judged; a payload that merely holds text: True is not."""
found = []
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
for keyword in node.keywords:
if keyword.arg is None and isinstance(keyword.value, ast.Dict):
if _text_mode_dict(keyword.value):
found.append(keyword.value)
splatted = _splatted_names(tree)
if not splatted:
return found
for node in ast.walk(tree):
targets = []
if isinstance(node, ast.Assign):
targets = [t for t in node.targets if isinstance(t, ast.Name)]
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
targets = [node.target]
if not targets or not isinstance(node.value, ast.Dict):
continue
if not _text_mode_dict(node.value):
continue
for target in targets:
if target.id in splatted and not _encoding_assigned_later(tree, target.id):
found.append(node.value)
break
return found
def _is_locale_default(value: ast.expr) -> bool:
"""None and "locale" are the locale default spelled out, not a chosen encoding."""
return isinstance(value, ast.Constant) and value.value in (None, "locale")
_PATH_CLASSES = {"Path", "PosixPath", "PurePath", "WindowsPath"}
def _path_class_names(tree: ast.AST) -> set[str]:
"""The pathlib classes and every name they are imported under in this module.
`from pathlib import Path as P` still puts the path in front of an unbound `P.read_text(p)`,
as tests/test_source_read_encoding.py's `_is_path_class` already accounts for. Names bound by
assignment, defaults or subclassing are not followed: those need the binding visible at each
call site, and a module-wide guess flags valid calls on an instance that happens to share the
name in another scope."""
names = set(_PATH_CLASSES)
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module == "pathlib":
for alias in node.names:
if alias.name in _PATH_CLASSES:
names.add(alias.asname or alias.name)
return names
def _path_text_names_encoding(node: ast.Call, name: str, path_classes: set[str]) -> bool:
"""Encoding may be a keyword or positional; unbound Path.read_text(p) shifts it by one."""
for kw in node.keywords:
if kw.arg == "encoding":
return not _is_locale_default(kw.value)
receiver = node.func.value
unbound = (isinstance(receiver, ast.Name) and receiver.id in path_classes) or (
isinstance(receiver, ast.Attribute) and receiver.attr in _PATH_CLASSES
)
position = (0 if name == "read_text" else 1) + (1 if unbound else 0)
# A starred argument at or before the slot makes the runtime position unknowable statically,
# so only a keyword encoding counts there: conservative, and nothing in the tree spells it so.
if any(isinstance(arg, ast.Starred) for arg in node.args[: position + 1]):
return False
if len(node.args) <= position:
return False
return not _is_locale_default(node.args[position])
def _offenders(path: Path) -> list[str]:
source = path.read_text(encoding = "utf-8")
tree = ast.parse(source, filename = str(path))
subprocess_names = _subprocess_names(tree)
subprocess_aliases = _subprocess_aliases(tree, subprocess_names)
path_classes = _path_class_names(tree)
found: list[str] = []
for node in _splatted_kwargs_offenders(tree):
found.append(
f"{path.name}:{node.lineno}: subprocess kwargs with text = True and no encoding"
)
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
name = _call_name(node)
if _is_subprocess_call(node, subprocess_names, subprocess_aliases):
if _text_mode_subprocess(node) and not _has_keyword(node, "encoding"):
found.append(f"{path.name}:{node.lineno}: subprocess(text = True) without encoding")
continue
if name == "open" and isinstance(node.func, ast.Name):
if _mode_is_binary(node) or _open_has_encoding(node):
continue
found.append(f"{path.name}:{node.lineno}: open() without encoding")
continue
# os.fdopen is open() on a descriptor; mode defaults to text.
if name == "fdopen":
if _mode_is_binary(node) or _open_has_encoding(node):
continue
found.append(f"{path.name}:{node.lineno}: os.fdopen() without encoding")
continue
if name == "open" and isinstance(node.func, ast.Attribute):
if not _is_path_open(node) or _path_open_has_encoding(node):
continue
if _path_open_mode(node) and "b" in _path_open_mode(node):
continue
found.append(f"{path.name}:{node.lineno}: Path.open() without encoding")
continue
if name in ("read_text", "write_text") and isinstance(node.func, ast.Attribute):
if _path_text_names_encoding(node, name, path_classes):
continue
# importlib.metadata Distribution.read_text() takes no encoding kwarg.
if isinstance(node.func.value, ast.Name) and node.func.value.id == "dist":
continue
found.append(f"{path.name}:{node.lineno}: {name}() without encoding")
return found
@pytest.mark.parametrize("path", _studio_sources(), ids = lambda p: str(p.name))
def test_text_io_names_its_encoding(path: Path) -> None:
offenders = _offenders(path)
assert not offenders, (
"Text I/O without an explicit encoding falls back to locale.getencoding(), "
"which is the ANSI codepage on Windows (cp1252, cp932, cp1251, ... by system "
"locale) and ASCII under a C or POSIX locale on Linux, as containers and CI "
"runners routinely have. Either way non-ASCII (ä ö ü → 世) mojibakes or raises "
"UnicodeDecodeError.\n\n"
"A platform guard above the call does NOT make this safe: the Windows codepage "
"is only one of the two ways to get the wrong decoder, and a linux-only path "
'still meets the C-locale one. Pass encoding = "utf-8":\n ' + "\n ".join(offenders)
)
_STATE_STORE = (
BACKEND_ROOT
/ "plugins/data-designer-github-repo-seed/src"
/ "data_designer_github_repo_seed/scraper_impl/state_store.py"
)
def _load_state_store(codepage: str):
"""Load state_store with the writing machine's codepage pinned."""
spec = importlib.util.spec_from_file_location(f"state_store_{codepage}", _STATE_STORE)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
module.locale = SimpleNamespace(
getencoding = lambda: codepage,
getpreferredencoding = lambda _ = True: codepage,
)
return module
@pytest.mark.parametrize(
("codepage", "name"), [("cp1252", "Jürgen"), ("cp1251", "Юрий"), ("cp932", "田中")]
)
def test_resuming_a_legacy_jsonl_keeps_one_encoding(
tmp_path: Path, codepage: str, name: str
) -> None:
"""A scrape written before UTF-8 was explicit must resume, not duplicate."""
path = tmp_path / "out.jsonl"
records = [{"id": 1, "author": name}, {"id": 2, "author": name}]
body = "".join(json.dumps(r, ensure_ascii = False) + "\n" for r in records)
path.write_bytes(body.encode(codepage))
before = path.read_bytes()
writer = _load_state_store(codepage).JsonlWriter(path)
try:
assert writer.has("id:1") and writer.has("id:2")
assert writer.write(records[0]) is False
assert writer.write({"id": 3, "author": name}) is True
finally:
writer.close()
blob = path.read_bytes()
assert blob.startswith(before)
assert blob[len(before) :].isascii()
lines = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()]
assert len(lines) == 3
assert [line["author"] for line in lines] == [name] * 3
def test_a_coincidentally_utf8_legacy_line_is_left_alone(tmp_path: Path) -> None:
"""cp1251 `Р°` is D0 B0, which is also UTF-8 `а`, and nothing can tell them apart."""
path = tmp_path / "out.jsonl"
ambiguous = "Р°"
assert ambiguous.encode("cp1251").decode("utf-8") == "а" # the trap
authors = ["Привет", "Здравствуйте", "Москва", ambiguous]
path.write_bytes(
b"".join(
json.dumps({"id": i, "author": a}, ensure_ascii = False).encode("cp1251") + b"\n"
for i, a in enumerate(authors)
)
)
before = path.read_bytes()
_load_state_store("cp1251").JsonlWriter(path).close()
assert path.read_bytes() == before
rows = [json.loads(x) for x in path.read_text(encoding = "cp1251").splitlines() if x.strip()]
assert [row["author"] for row in rows] == authors
@pytest.mark.parametrize(
("codepage", "word"), [("cp1251", "Привет"), ("cp932", "こんにちは"), ("cp1252", "Jürgen")]
)
def test_a_moved_shard_is_not_rewritten_by_guesswork(
tmp_path: Path, codepage: str, word: str
) -> None:
"""Off the writing machine there is no codepage to attribute the file to."""
path = tmp_path / "out.jsonl"
# Two records: a lone non-UTF-8 line would count as damage, not legacy.
path.write_bytes(
b"".join(
json.dumps({"id": i, "author": word}, ensure_ascii = False).encode(codepage) + b"\n"
for i in (1, 4)
)
)
before = path.read_bytes()
# A UTF-8 host: latin-1 would read cp1251 `Привет` back as `Ïðèâåò`.
writer = _load_state_store("utf-8").JsonlWriter(path)
try:
assert writer.has("id:1")
assert writer.write({"id": 2, "author": "Grüße"}) is True
finally:
writer.close()
blob = path.read_bytes()
assert blob.startswith(before)
assert blob[len(before) :].isascii()
rows = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()]
assert [row["author"] for row in rows] == [word, word, "Grüße"]
def test_an_all_ambiguous_shard_still_gets_ascii_appends(tmp_path: Path) -> None:
"""Every line valid under both readings still means the append must not pick one."""
path = tmp_path / "out.jsonl"
ambiguous = "Р°" # cp1251 D0 B0, also valid UTF-8 for "а"
path.write_bytes(
b"".join(
json.dumps({"id": i, "a": ambiguous}, ensure_ascii = False).encode("cp1251") + b"\n"
for i in range(3)
)
)
before = path.read_bytes()
writer = _load_state_store("cp1251").JsonlWriter(path)
try:
assert writer.write({"id": 9, "a": "世界"}) is True
finally:
writer.close()
blob = path.read_bytes()
assert blob.startswith(before)
assert blob[len(before) :].isascii()
for codec in ("cp1251", "utf-8"):
rows = [json.loads(x) for x in blob.decode(codec).splitlines() if x.strip()]
assert rows[-1]["a"] == "世界"
def test_a_damaged_line_in_an_ascii_shard_does_not_block_its_retry(tmp_path: Path) -> None:
"""With no non-ASCII records to outvote it, one damaged line is still damage."""
path = tmp_path / "out.jsonl"
path.write_bytes(
b'{"id": 1, "author": "alice"}\n'
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
+ b'{"id": 2, "author": "bob"}\n'
)
writer = _load_state_store("cp1252").JsonlWriter(path)
try:
assert writer.has("id:1") and writer.has("id:2")
assert not writer.has("id:99")
assert writer.write({"id": 99, "author": "good byte"}) is True
finally:
writer.close()
def test_a_damaged_line_does_not_block_its_own_retry(tmp_path: Path) -> None:
"""Its key comes from the codepage reading, which a UTF-8 shard did not pick."""
path = tmp_path / "out.jsonl"
path.write_bytes(
json.dumps({"id": 1, "author": "Jürgen"}, ensure_ascii = False).encode()
+ b"\n"
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
)
writer = _load_state_store("cp1252").JsonlWriter(path)
try:
assert writer.has("id:1")
assert not writer.has("id:99")
assert writer.write({"id": 99, "author": "good byte"}) is True
finally:
writer.close()
def test_one_damaged_byte_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None:
"""A complete JSON line with a stray 0x96 parses as cp1252, but is only one vote."""
path = tmp_path / "out.jsonl"
healthy = ["Jürgen", "Grüße", "Björn"]
path.write_bytes(
json.dumps({"id": 0, "author": healthy[0]}, ensure_ascii = False).encode()
+ b"\n"
+ b'{"id": 99, "author": "bad \x96 byte"}\n'
+ b"".join(
json.dumps({"id": i, "author": a}, ensure_ascii = False).encode() + b"\n"
for i, a in enumerate(healthy[1:], start = 1)
)
)
before = path.read_bytes()
_load_state_store("cp1252").JsonlWriter(path).close()
assert path.read_bytes() == before
rows = []
for line in path.read_bytes().splitlines():
try:
rows.append(json.loads(line.decode()))
except (UnicodeDecodeError, ValueError):
continue
assert [row["author"] for row in rows] == healthy
def test_a_torn_line_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None:
"""One interrupted append must not get the whole shard read as cp1252."""
path = tmp_path / "out.jsonl"
good = [{"id": 1, "author": "Jürgen"}, {"id": 3, "author": "Grüße"}]
torn = '{"id": 2, "author": "Jürgen"}'.encode()[:-6] # cut mid-character
path.write_bytes(
json.dumps(good[0], ensure_ascii = False).encode()
+ b"\n"
+ torn
+ b"\n"
+ json.dumps(good[1], ensure_ascii = False).encode()
+ b"\n"
)
before = path.read_bytes()
writer = _load_state_store("cp1252").JsonlWriter(path)
try:
assert writer.has("id:1") and writer.has("id:3")
assert not writer.has("id:2")
finally:
writer.close()
after = path.read_bytes()
assert after.startswith(before)
assert "Jürgen".encode() in after
assert "Jürgen".encode("utf-8").decode("cp1252").encode() not in after
def test_an_undecodable_transport_marker_reads_as_unknown(tmp_path: Path) -> None:
"""An undecodable marker must read as unknown; a raised UnicodeDecodeError would abort the transfer."""
_shared_setup_1(__file__)
from hub.utils import download_registry as registry
marker = tmp_path / ".transport"
marker.write_bytes(b"\x80\xffnative\n")
assert registry._read_marker_value(marker) is None
marker.write_text("something-else\n", encoding = "utf-8")
assert registry._read_marker_value(marker) is None
def test_a_torn_cache_ref_reads_as_not_cached(tmp_path: Path, monkeypatch) -> None:
"""A refs/main with undecodable bytes must read as not cached, since a raise would turn into a 500."""
_shared_setup_1(__file__)
from utils import utils as backend_utils
good_root = tmp_path / "good"
torn_root = tmp_path / "torn"
for root, ref_bytes in ((torn_root, b"\x80\xff\n"), (good_root, b"abc123\n")):
repo = root / "models--Org--Model"
(repo / "refs").mkdir(parents = True)
(repo / "refs" / "main").write_bytes(ref_bytes)
(good_root / "models--Org--Model" / "snapshots" / "abc123").mkdir(parents = True)
monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root])
assert backend_utils.hf_cache_snapshot_dir("Org/Model") is None
monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root, good_root])
found = backend_utils.hf_cache_snapshot_dir("Org/Model")
assert found is not None and found.name == "abc123"
def test_a_corrupt_pid_file_does_not_abort_shutdown(tmp_path: Path, monkeypatch) -> None:
"""_remove_pid_file runs first in _graceful_shutdown, so a raise there leaves
the inference, export, training and tunnel children alive."""
_shared_setup_1(__file__)
import run as studio_run
pid_file = tmp_path / "studio.pid"
pid_file.write_bytes(b"\x80\xff")
monkeypatch.setattr(studio_run, "_PID_FILE", pid_file)
# _legacy_heir scans the real Unsloth root; a running server would change the outcome.
monkeypatch.setattr(studio_run, "_legacy_heir", lambda: None)
studio_run._remove_pid_file()
assert pid_file.exists()
pid_file.write_text(str(os.getpid()), encoding = "utf-8")
studio_run._remove_pid_file()
assert not pid_file.exists()
def test_the_legacy_pid_record_is_handed_to_a_live_sibling(tmp_path: Path, monkeypatch) -> None:
"""Only one server owns studio.pid, so deleting it while a sibling still serves would leave
that sibling unstoppable from an older CLI that reads no other file."""
_shared_setup_1(__file__)
import run as studio_run
pid_file = tmp_path / "studio.pid"
pid_file.write_text(str(os.getpid()), encoding = "utf-8")
monkeypatch.setattr(studio_run, "_PID_FILE", pid_file)
monkeypatch.setattr(studio_run, "_legacy_heir", lambda: 4242)
studio_run._remove_pid_file()
assert pid_file.read_text(encoding = "utf-8") == "4242"
def test_the_kwargs_guard_only_judges_dicts_that_reach_a_call(tmp_path: Path) -> None:
"""Only a dict splatted into a call is subprocess config; an unrelated payload with text=True is not."""
cases = {
"offender.py": 'kw = {"text": True}\nrun(cmd, **kw)\n',
"annotated.py": 'kw: dict = {"universal_newlines": True}\nrun(cmd, **kw)\n',
"payload.py": 'payload = {"text": True}\nrequests.post(url, json = payload)\n',
"inline.py": 'run(cmd, **{"text": True})\n',
"later.py": 'kw = {"text": True}\nkw["encoding"] = "utf-8"\nrun(cmd, **kw)\n',
"carried.py": 'kw = {"text": True, "encoding": "utf-8"}\nrun(cmd, **kw)\n',
}
flagged = set()
for name, source in cases.items():
path = tmp_path / name
path.write_text(source, encoding = "utf-8")
if any("subprocess kwargs" in line for line in _offenders(path)):
flagged.add(name)
assert flagged == {"offender.py", "annotated.py", "inline.py"}, flagged
def test_the_guard_follows_subprocess_through_an_alias(tmp_path: Path) -> None:
"""install_wheel() takes ``run = subprocess.run`` and calls it as a bare
name, so an attribute-only match let both of its installer calls drop their
encoding unnoticed. A name bound to something else is still not subprocess."""
cases = {
"param_default.py": (
"import subprocess\n"
"def install(*, run = subprocess.run):\n"
" run(cmd, text = True)\n"
),
"assigned.py": "import subprocess\n_run = subprocess.run\n_run(cmd, text = True)\n",
"imported.py": "from subprocess import check_output\ncheck_output(cmd, text = True)\n",
"renamed.py": "from subprocess import run as _r\n_r(cmd, universal_newlines = True)\n",
"encoded.py": (
"import subprocess\n"
"def install(*, run = subprocess.run):\n"
' run(cmd, text = True, encoding = "utf-8")\n'
),
"unrelated.py": "def run(cmd, text = False):\n pass\nrun(cmd, text = True)\n",
}
flagged = set()
for name, source in cases.items():
path = tmp_path / name
path.write_text(source, encoding = "utf-8")
if any("subprocess(text = True)" in line for line in _offenders(path)):
flagged.add(name)
assert flagged == {"param_default.py", "assigned.py", "imported.py", "renamed.py"}, flagged
def test_the_guard_sees_os_fdopen(tmp_path: Path) -> None:
"""os.fdopen shares open()'s locale default; missing it let a lock file write in the codepage."""
cases = {
"text.py": 'import os\nos.fdopen(fd, "w")\n',
"default_mode.py": "import os\nos.fdopen(fd)\n", # defaults to "r", still text
"binary.py": 'import os\nos.fdopen(fd, "wb")\n',
"keyword.py": 'import os\nos.fdopen(fd, "w", encoding = "utf-8")\n',
"positional.py": 'import os\nos.fdopen(fd, "w", 1, "utf-8")\n',
}
flagged = set()
for name, source in cases.items():
path = tmp_path / name
path.write_text(source, encoding = "utf-8")
if any("fdopen" in line for line in _offenders(path)):
flagged.add(name)
assert flagged == {"text.py", "default_mode.py"}, flagged
def test_the_guard_reads_a_positional_path_text_encoding(tmp_path: Path) -> None:
"""Path.read_text(encoding, errors) and Path.write_text(data, encoding, ...) take the
encoding positionally too. #13565's read_text("utf-8") is not a locale-default read."""
cases = {
"read_none.py": "p.read_text()\n",
"read_kw.py": 'p.read_text(encoding = "utf-8")\n',
"read_pos.py": 'p.read_text("utf-8")\n',
"write_none.py": 'p.write_text("x")\n',
"write_kw.py": 'p.write_text("x", encoding = "utf-8")\n',
"write_pos.py": 'p.write_text("x", "utf-8")\n',
# The locale default spelled out is still the locale default.
"read_none_pos.py": "p.read_text(None)\n",
"read_locale_pos.py": 'p.read_text("locale")\n',
"read_locale_kw.py": 'p.read_text(encoding = "locale")\n',
"write_none_pos.py": 'p.write_text("x", None)\n',
# Unbound calls carry the path first, so the encoding sits one slot later.
"unbound_read.py": "Path.read_text(p)\n",
"unbound_read_pos.py": 'Path.read_text(p, "utf-8")\n',
"unbound_write.py": 'pathlib.Path.write_text(p, "x")\n',
"unbound_write_pos.py": 'pathlib.Path.write_text(p, "x", "utf-8")\n',
# Under an alias, or a concrete pathlib class.
"unbound_alias.py": "from pathlib import Path as P\nP.read_text(p)\n",
"unbound_alias_pos.py": 'from pathlib import Path as P\nP.read_text(p, "utf-8")\n',
"unbound_posix.py": 'PosixPath.write_text(p, "x")\n',
# A same-named instance elsewhere is not an alias: no module-wide assignment guessing.
"instance_named_p.py": 'def a():\n P = Path\n\ndef b(P):\n P.read_text("utf-8")\n',
# A starred argument hides the runtime position: only a keyword encoding counts.
"starred_empty.py": "p.read_text(*())\n",
"unbound_starred.py": "Path.read_text(p, *())\n",
"starred_kw.py": 'p.read_text(*args, encoding = "utf-8")\n',
}
flagged = set()
for name, source in cases.items():
path = tmp_path / name
path.write_text(source, encoding = "utf-8")
if _offenders(path):
flagged.add(name)
assert flagged == {
"read_none.py",
"write_none.py",
"read_none_pos.py",
"read_locale_pos.py",
"read_locale_kw.py",
"write_none_pos.py",
"unbound_read.py",
"unbound_write.py",
"unbound_alias.py",
"unbound_posix.py",
"starred_empty.py",
"unbound_starred.py",
}, flagged
def test_an_undecodable_bootstrap_password_does_not_stop_startup(
tmp_path: Path, monkeypatch
) -> None:
"""Lifespan calls _load_bootstrap_password with no handler, so an unusable file must be ignored
rather than raised; a raise takes the backend down."""
_shared_setup_1(__file__)
from auth import storage
pw_file = tmp_path / ".bootstrap_password"
pw_file.write_bytes(b"\x80\xffnot-utf8\n")
monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", pw_file)
assert storage._load_bootstrap_password() is None
pw_file.write_text("correct horse battery staple\n", encoding = "utf-8")
assert storage._load_bootstrap_password() == "correct horse battery staple"
def test_a_damaged_checkpoint_resets_instead_of_resuming_on_a_broken_cursor(tmp_path: Path) -> None:
"""A mojibaked cursor makes GitHub report the stream done; reset the checkpoint, don't resume it."""
module = _load_state_store("cp1252")
cursor = "Y3Vyc29yOnYyOpK0MjAxMi0wMi0xNlQwNjo1Mzo0MVrOADGL_A=="
healthy = json.dumps({"issues_cursor": cursor, "issues_done": False}, indent = 2)
path = tmp_path / "octocat__Hello-World.json"
path.write_text(healthy, encoding = "utf-8")
assert module.StateStore(path).get("issues_cursor") == cursor
# Written in the operator's codepage by a pre-UTF-8 release.
path.write_bytes(healthy.encode("cp1252"))
assert module.StateStore(path).get("issues_cursor") == cursor
# Valid JSON under a single-byte codepage, so only refusing that reading resets it.
raw = healthy.encode()
at = raw.index(b"MjAxMi0wMi0xNlQ") + 3
path.write_bytes(raw[:at] + b"\x96" + raw[at + 1 :])
assert json.loads(path.read_bytes().decode("latin-1"))["issues_cursor"] != cursor
store = module.StateStore(path)
assert store.all() == {}
assert store.get("issues_cursor") is None
def test_a_utf8_record_is_not_parsed_a_second_time(tmp_path: Path) -> None:
"""These shards reach gigabytes and every resume reads all of one, so a
record that already read as UTF-8 must not be decoded and parsed again under
the codepage. The legacy reading exists only to recover keys UTF-8 could not."""
module = _load_state_store("cp1252")
calls: list[str] = []
real_parse = module._parse
def counting_parse(raw, encoding):
calls.append(encoding)
return real_parse(raw, encoding)
module._parse = counting_parse
try:
healthy = json.dumps({"id": 1, "author": "Jürgen"}).encode("utf-8")
reading = module._read_line(healthy, "cp1252")
assert reading.as_utf8 == {"id": 1, "author": "Jürgen"}
assert calls == ["utf-8"], calls
calls.clear()
legacy = json.dumps({"id": 2, "author": "Jürgen"}, ensure_ascii = False).encode("cp1252")
reading = module._read_line(legacy, "cp1252")
assert reading.as_utf8 is None
assert reading.as_legacy == {"id": 2, "author": "Jürgen"}
assert calls == ["utf-8", "cp1252"], calls
finally:
module._parse = real_parse
def _too_deeply_nested_json() -> str:
"""Probed: json.loads' nesting limit is the C recursion limit from 3.12, not setrecursionlimit."""
depth = 1
while depth <= 1 << 17:
document = "[" * depth + "]" * depth
try:
json.loads(document)
except RecursionError:
return document
depth *= 2
pytest.skip("this interpreter parses arbitrarily nested JSON")
def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path) -> None:
"""RecursionError is neither ValueError nor UnicodeDecodeError, so catch it or startup aborts."""
module = _load_state_store("cp1252")
nested = _too_deeply_nested_json()
checkpoint = tmp_path / "octocat__Hello-World.json"
checkpoint.write_text(nested, encoding = "utf-8")
assert module.StateStore(checkpoint).all() == {}
shard = tmp_path / "out.jsonl"
shard.write_text(
nested + "\n" + json.dumps({"id": 1}) + "\n" + json.dumps({"id": 2}) + "\n",
encoding = "utf-8",
)
writer = module.JsonlWriter(shard)
try:
assert writer.has("id:1") and writer.has("id:2")
finally:
writer.close()
def test_vendored_laya_reads_utf8_config_under_a_non_utf8_locale(tmp_path):
"""Under a non-UTF-8 locale bare open() misdecodes, so vendored laya must read its config as UTF-8."""
config = tmp_path / "tokenizer" / "tokenizer_config.json"
config.parent.mkdir()
config.write_text(
json.dumps(
{"tokenizer_class": None, "extra_special_tokens": ["ä", "世"]}, ensure_ascii = False
),
encoding = "utf-8",
)
# C locale without coercion decodes bare open() as ASCII (Linux/macOS) or ANSI (Windows).
script = (
"import codecs, locale, sys\n"
"if codecs.lookup(locale.getencoding()).name == 'utf-8': sys.exit(3)\n"
f"sys.path.insert(0, {str(BACKEND_ROOT)!r})\n"
"from core.systemone import laya_runtime\n"
f"laya_runtime._laya().agent._fix_tokenizer_config({str(tmp_path)!r})\n"
)
env = {**os.environ, "LC_ALL": "C", "LANG": "C", "PYTHONCOERCECLOCALE": "0", "PYTHONUTF8": "0"}
run = subprocess.run(
[sys.executable, "-c", script], env = env, capture_output = True, text = True, encoding = "utf-8"
)
if run.returncode == 3:
pytest.skip("this platform's locale is UTF-8 even under LC_ALL=C")
assert run.returncode == 0, run.stderr[-3000:]
repaired = json.loads(config.read_text(encoding = "utf-8"))
assert repaired["tokenizer_class"] == "PreTrainedTokenizerFast"
assert repaired["extra_special_tokens"] == {"extra_0": "ä", "extra_1": "世"}