Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

36.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Text I/O must pass encoding explicitly, since Windows falls back to the ANSI codepage, not UTF-8."""

from __future__ import annotations

import sys
import subprocess
import ast
import importlib.util
import json
import os
from pathlib import Path
from types import SimpleNamespace

import pytest


def _shared_setup_1(__file__):
    import sys
    backend = str(Path(__file__).resolve().parent.parent)
    if backend not in sys.path:
        sys.path.insert(0, backend)


BACKEND_ROOT = Path(__file__).resolve().parent.parent

# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped.
_SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__")

# Vendored packages are byte-identical to their wheel and get UTF-8 open via their loader.
_UTF8_BY_LOADER = {"vendor/laya": "core/systemone/laya_runtime.py"}

# Path.open()'s signature is what tells it apart from other libraries' open(),
# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...).
_FILE_MODE_CHARS = set("rwxabt+")
_PATH_OPEN_ARGS = ("mode", "buffering", "encoding", "errors", "newline")
_PATH_OPEN_KWARGS = set(_PATH_OPEN_ARGS)
_PATH_OPEN_ENCODING_ARG = _PATH_OPEN_ARGS.index("encoding")

_SUBPROCESS_CALLS = {"run", "Popen", "check_output", "check_call", "call"}

# open() takes encoding as its fourth argument, and os.fdopen forwards the same signature.
_OPEN_ENCODING_ARG = 3


def _studio_sources() -> list[Path]:
    return [
        path
        for path in sorted(BACKEND_ROOT.rglob("*.py"))
        if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts)
        and not any(
            path.relative_to(BACKEND_ROOT).as_posix().startswith(prefix + "/")
            for prefix in _UTF8_BY_LOADER
        )
    ]


def _has_keyword(node: ast.Call, name: str) -> bool:
    return any(keyword.arg == name for keyword in node.keywords)


def _mode_is_binary(node: ast.Call) -> bool:
    mode: str | None = None
    if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant):
        value = node.args[1].value
        mode = value if isinstance(value, str) else None
    for keyword in node.keywords:
        if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
            value = keyword.value.value
            if isinstance(value, str):
                mode = value
    return bool(mode and "b" in mode)


def _open_has_encoding(node: ast.Call) -> bool:
    """open()/os.fdopen() also take encoding positionally: open(p, "w", 1, "utf-8")."""
    return _has_keyword(node, "encoding") or len(node.args) > _OPEN_ENCODING_ARG


def _path_open_mode(node: ast.Call) -> str | None:
    if node.args and isinstance(node.args[0], ast.Constant):
        value = node.args[0].value
        if isinstance(value, str):
            return value
    for keyword in node.keywords:
        if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
            value = keyword.value.value
            if isinstance(value, str):
                return value
    return None


def _is_path_open(node: ast.Call) -> bool:
    """True only for calls matching ``Path.open``'s signature."""
    if len(node.args) > len(_PATH_OPEN_ARGS):
        return False
    if any(k.arg not in _PATH_OPEN_KWARGS for k in node.keywords):
        return False
    mode = _path_open_mode(node)
    if mode is not None:
        return bool(mode) and set(mode) <= _FILE_MODE_CHARS
    return not node.args


def _path_open_has_encoding(node: ast.Call) -> bool:
    """Path.open() also takes encoding positionally: open("w", 1, "utf-8")."""
    return _has_keyword(node, "encoding") or len(node.args) > _PATH_OPEN_ENCODING_ARG


def _call_name(node: ast.Call) -> str | None:
    func = node.func
    if isinstance(func, ast.Name):
        return func.id
    if isinstance(func, ast.Attribute):
        return func.attr
    return None


def _subprocess_names(tree: ast.AST) -> set[str]:
    """Names subprocess is reachable under here, e.g. `import subprocess as _sp`."""
    names = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            for alias in node.names:
                if alias.name == "subprocess":
                    names.add(alias.asname or alias.name)
    return names


def _subprocess_aliases(tree: ast.AST, names: set[str]) -> set[str]:
    """Bare names bound to subprocess callables count too, e.g. install_wheel(run = subprocess.run)."""

    def _is_bound(value: ast.expr | None) -> bool:
        return (
            isinstance(value, ast.Attribute)
            and value.attr in _SUBPROCESS_CALLS
            and isinstance(value.value, ast.Name)
            and value.value.id in names
        )

    aliases: set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.ImportFrom) and node.module == "subprocess":
            aliases.update(a.asname or a.name for a in node.names if a.name in _SUBPROCESS_CALLS)
        elif isinstance(node, ast.Assign) and _is_bound(node.value):
            aliases.update(t.id for t in node.targets if isinstance(t, ast.Name))
        elif isinstance(node, ast.AnnAssign) and _is_bound(node.value):
            if isinstance(node.target, ast.Name):
                aliases.add(node.target.id)
        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
            args = node.args
            positional = args.posonlyargs + args.args
            padded = [None] * (len(positional) - len(args.defaults)) + list(args.defaults)
            pairs = list(zip(positional, padded)) + list(zip(args.kwonlyargs, args.kw_defaults))
            aliases.update(arg.arg for arg, default in pairs if _is_bound(default))
    return aliases


def _is_subprocess_call(node: ast.Call, names: set[str], aliases: set[str]) -> bool:
    func = node.func
    if isinstance(func, ast.Name):
        return func.id in aliases
    if not isinstance(func, ast.Attribute) or func.attr not in _SUBPROCESS_CALLS:
        return False
    value = func.value
    return isinstance(value, ast.Name) and value.id in names


def _text_mode_subprocess(node: ast.Call) -> bool:
    for keyword in node.keywords:
        if keyword.arg not in ("text", "universal_newlines"):
            continue
        if isinstance(keyword.value, ast.Constant) and keyword.value.value is True:
            return True
    return False


def _text_mode_dict(node: ast.Dict) -> bool:
    """A ``{"text": True, ...}`` literal with no "encoding" key."""
    keys = [k.value for k in node.keys if isinstance(k, ast.Constant)]
    if "encoding" in keys:
        return False
    for key, value in zip(node.keys, node.values):
        if not isinstance(key, ast.Constant) or key.value not in (
            "text",
            "universal_newlines",
        ):
            continue
        if isinstance(value, ast.Constant) and value.value is True:
            return True
    return False


def _splatted_names(tree: ast.AST) -> set[str]:
    """Names handed to a call as ``**name``."""
    names = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Call):
            for keyword in node.keywords:
                if keyword.arg is None and isinstance(keyword.value, ast.Name):
                    names.add(keyword.value.id)
    return names


def _encoding_assigned_later(tree: ast.AST, name: str) -> bool:
    """``name["encoding"] = ...`` somewhere, so the literal need not carry it."""
    for node in ast.walk(tree):
        if not isinstance(node, ast.Subscript) or not isinstance(node.ctx, ast.Store):
            continue
        target, key = node.value, node.slice
        if isinstance(target, ast.Name) and target.id == name:
            if isinstance(key, ast.Constant) and key.value == "encoding":
                return True
    return False


def _splatted_kwargs_offenders(tree: ast.AST) -> list[ast.Dict]:
    """Only kwargs dicts splatted into a call are judged; a payload that merely holds text: True is not."""
    found = []
    for node in ast.walk(tree):
        if not isinstance(node, ast.Call):
            continue
        for keyword in node.keywords:
            if keyword.arg is None and isinstance(keyword.value, ast.Dict):
                if _text_mode_dict(keyword.value):
                    found.append(keyword.value)
    splatted = _splatted_names(tree)
    if not splatted:
        return found
    for node in ast.walk(tree):
        targets = []
        if isinstance(node, ast.Assign):
            targets = [t for t in node.targets if isinstance(t, ast.Name)]
        elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
            targets = [node.target]
        if not targets or not isinstance(node.value, ast.Dict):
            continue
        if not _text_mode_dict(node.value):
            continue
        for target in targets:
            if target.id in splatted and not _encoding_assigned_later(tree, target.id):
                found.append(node.value)
                break
    return found


def _is_locale_default(value: ast.expr) -> bool:
    """None and "locale" are the locale default spelled out, not a chosen encoding."""
    return isinstance(value, ast.Constant) and value.value in (None, "locale")


_PATH_CLASSES = {"Path", "PosixPath", "PurePath", "WindowsPath"}


def _path_class_names(tree: ast.AST) -> set[str]:
    """The pathlib classes and every name they are imported under in this module.

    `from pathlib import Path as P` still puts the path in front of an unbound `P.read_text(p)`,
    as tests/test_source_read_encoding.py's `_is_path_class` already accounts for. Names bound by
    assignment, defaults or subclassing are not followed: those need the binding visible at each
    call site, and a module-wide guess flags valid calls on an instance that happens to share the
    name in another scope."""
    names = set(_PATH_CLASSES)
    for node in ast.walk(tree):
        if isinstance(node, ast.ImportFrom) and node.module == "pathlib":
            for alias in node.names:
                if alias.name in _PATH_CLASSES:
                    names.add(alias.asname or alias.name)
    return names


def _path_text_names_encoding(node: ast.Call, name: str, path_classes: set[str]) -> bool:
    """Encoding may be a keyword or positional; unbound Path.read_text(p) shifts it by one."""
    for kw in node.keywords:
        if kw.arg == "encoding":
            return not _is_locale_default(kw.value)
    receiver = node.func.value
    unbound = (isinstance(receiver, ast.Name) and receiver.id in path_classes) or (
        isinstance(receiver, ast.Attribute) and receiver.attr in _PATH_CLASSES
    )
    position = (0 if name == "read_text" else 1) + (1 if unbound else 0)
    # A starred argument at or before the slot makes the runtime position unknowable statically,
    # so only a keyword encoding counts there: conservative, and nothing in the tree spells it so.
    if any(isinstance(arg, ast.Starred) for arg in node.args[: position + 1]):
        return False
    if len(node.args) <= position:
        return False
    return not _is_locale_default(node.args[position])


def _offenders(path: Path) -> list[str]:
    source = path.read_text(encoding = "utf-8")
    tree = ast.parse(source, filename = str(path))
    subprocess_names = _subprocess_names(tree)
    subprocess_aliases = _subprocess_aliases(tree, subprocess_names)
    path_classes = _path_class_names(tree)
    found: list[str] = []
    for node in _splatted_kwargs_offenders(tree):
        found.append(
            f"{path.name}:{node.lineno}: subprocess kwargs with text = True and no encoding"
        )
    for node in ast.walk(tree):
        if not isinstance(node, ast.Call):
            continue
        name = _call_name(node)

        if _is_subprocess_call(node, subprocess_names, subprocess_aliases):
            if _text_mode_subprocess(node) and not _has_keyword(node, "encoding"):
                found.append(f"{path.name}:{node.lineno}: subprocess(text = True) without encoding")
            continue

        if name == "open" and isinstance(node.func, ast.Name):
            if _mode_is_binary(node) or _open_has_encoding(node):
                continue
            found.append(f"{path.name}:{node.lineno}: open() without encoding")
            continue

        # os.fdopen is open() on a descriptor; mode defaults to text.
        if name == "fdopen":
            if _mode_is_binary(node) or _open_has_encoding(node):
                continue
            found.append(f"{path.name}:{node.lineno}: os.fdopen() without encoding")
            continue

        if name == "open" and isinstance(node.func, ast.Attribute):
            if not _is_path_open(node) or _path_open_has_encoding(node):
                continue
            if _path_open_mode(node) and "b" in _path_open_mode(node):
                continue
            found.append(f"{path.name}:{node.lineno}: Path.open() without encoding")
            continue

        if name in ("read_text", "write_text") and isinstance(node.func, ast.Attribute):
            if _path_text_names_encoding(node, name, path_classes):
                continue
            # importlib.metadata Distribution.read_text() takes no encoding kwarg.
            if isinstance(node.func.value, ast.Name) and node.func.value.id == "dist":
                continue
            found.append(f"{path.name}:{node.lineno}: {name}() without encoding")
    return found


@pytest.mark.parametrize("path", _studio_sources(), ids = lambda p: str(p.name))
def test_text_io_names_its_encoding(path: Path) -> None:
    offenders = _offenders(path)
    assert not offenders, (
        "Text I/O without an explicit encoding falls back to locale.getencoding(), "
        "which is the ANSI codepage on Windows (cp1252, cp932, cp1251, ... by system "
        "locale) and ASCII under a C or POSIX locale on Linux, as containers and CI "
        "runners routinely have. Either way non-ASCII (ä ö ü → 世) mojibakes or raises "
        "UnicodeDecodeError.\n\n"
        "A platform guard above the call does NOT make this safe: the Windows codepage "
        "is only one of the two ways to get the wrong decoder, and a linux-only path "
        'still meets the C-locale one. Pass encoding = "utf-8":\n  ' + "\n  ".join(offenders)
    )


_STATE_STORE = (
    BACKEND_ROOT
    / "plugins/data-designer-github-repo-seed/src"
    / "data_designer_github_repo_seed/scraper_impl/state_store.py"
)


def _load_state_store(codepage: str):
    """Load state_store with the writing machine's codepage pinned."""
    spec = importlib.util.spec_from_file_location(f"state_store_{codepage}", _STATE_STORE)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    module.locale = SimpleNamespace(
        getencoding = lambda: codepage,
        getpreferredencoding = lambda _ = True: codepage,
    )
    return module


@pytest.mark.parametrize(
    ("codepage", "name"), [("cp1252", "Jürgen"), ("cp1251", "Юрий"), ("cp932", "田中")]
)
def test_resuming_a_legacy_jsonl_keeps_one_encoding(
    tmp_path: Path, codepage: str, name: str
) -> None:
    """A scrape written before UTF-8 was explicit must resume, not duplicate."""
    path = tmp_path / "out.jsonl"
    records = [{"id": 1, "author": name}, {"id": 2, "author": name}]
    body = "".join(json.dumps(r, ensure_ascii = False) + "\n" for r in records)
    path.write_bytes(body.encode(codepage))
    before = path.read_bytes()

    writer = _load_state_store(codepage).JsonlWriter(path)
    try:
        assert writer.has("id:1") and writer.has("id:2")
        assert writer.write(records[0]) is False
        assert writer.write({"id": 3, "author": name}) is True
    finally:
        writer.close()

    blob = path.read_bytes()
    assert blob.startswith(before)
    assert blob[len(before) :].isascii()
    lines = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()]
    assert len(lines) == 3
    assert [line["author"] for line in lines] == [name] * 3


def test_a_coincidentally_utf8_legacy_line_is_left_alone(tmp_path: Path) -> None:
    """cp1251 `Р°` is D0 B0, which is also UTF-8 `а`, and nothing can tell them apart."""
    path = tmp_path / "out.jsonl"
    ambiguous = "Р°"
    assert ambiguous.encode("cp1251").decode("utf-8") == "а"  # the trap
    authors = ["Привет", "Здравствуйте", "Москва", ambiguous]
    path.write_bytes(
        b"".join(
            json.dumps({"id": i, "author": a}, ensure_ascii = False).encode("cp1251") + b"\n"
            for i, a in enumerate(authors)
        )
    )
    before = path.read_bytes()

    _load_state_store("cp1251").JsonlWriter(path).close()

    assert path.read_bytes() == before
    rows = [json.loads(x) for x in path.read_text(encoding = "cp1251").splitlines() if x.strip()]
    assert [row["author"] for row in rows] == authors


@pytest.mark.parametrize(
    ("codepage", "word"), [("cp1251", "Привет"), ("cp932", "こんにちは"), ("cp1252", "Jürgen")]
)
def test_a_moved_shard_is_not_rewritten_by_guesswork(
    tmp_path: Path, codepage: str, word: str
) -> None:
    """Off the writing machine there is no codepage to attribute the file to."""
    path = tmp_path / "out.jsonl"
    # Two records: a lone non-UTF-8 line would count as damage, not legacy.
    path.write_bytes(
        b"".join(
            json.dumps({"id": i, "author": word}, ensure_ascii = False).encode(codepage) + b"\n"
            for i in (1, 4)
        )
    )
    before = path.read_bytes()

    # A UTF-8 host: latin-1 would read cp1251 `Привет` back as `Ïðèâåò`.
    writer = _load_state_store("utf-8").JsonlWriter(path)
    try:
        assert writer.has("id:1")
        assert writer.write({"id": 2, "author": "Grüße"}) is True
    finally:
        writer.close()

    blob = path.read_bytes()
    assert blob.startswith(before)
    assert blob[len(before) :].isascii()
    rows = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()]
    assert [row["author"] for row in rows] == [word, word, "Grüße"]


def test_an_all_ambiguous_shard_still_gets_ascii_appends(tmp_path: Path) -> None:
    """Every line valid under both readings still means the append must not pick one."""
    path = tmp_path / "out.jsonl"
    ambiguous = "Р°"  # cp1251 D0 B0, also valid UTF-8 for "а"
    path.write_bytes(
        b"".join(
            json.dumps({"id": i, "a": ambiguous}, ensure_ascii = False).encode("cp1251") + b"\n"
            for i in range(3)
        )
    )
    before = path.read_bytes()

    writer = _load_state_store("cp1251").JsonlWriter(path)
    try:
        assert writer.write({"id": 9, "a": "世界"}) is True
    finally:
        writer.close()

    blob = path.read_bytes()
    assert blob.startswith(before)
    assert blob[len(before) :].isascii()
    for codec in ("cp1251", "utf-8"):
        rows = [json.loads(x) for x in blob.decode(codec).splitlines() if x.strip()]
        assert rows[-1]["a"] == "世界"


def test_a_damaged_line_in_an_ascii_shard_does_not_block_its_retry(tmp_path: Path) -> None:
    """With no non-ASCII records to outvote it, one damaged line is still damage."""
    path = tmp_path / "out.jsonl"
    path.write_bytes(
        b'{"id": 1, "author": "alice"}\n'
        + b'{"id": 99, "author": "bad \x96 byte"}\n'
        + b'{"id": 2, "author": "bob"}\n'
    )

    writer = _load_state_store("cp1252").JsonlWriter(path)
    try:
        assert writer.has("id:1") and writer.has("id:2")
        assert not writer.has("id:99")
        assert writer.write({"id": 99, "author": "good byte"}) is True
    finally:
        writer.close()


def test_a_damaged_line_does_not_block_its_own_retry(tmp_path: Path) -> None:
    """Its key comes from the codepage reading, which a UTF-8 shard did not pick."""
    path = tmp_path / "out.jsonl"
    path.write_bytes(
        json.dumps({"id": 1, "author": "Jürgen"}, ensure_ascii = False).encode()
        + b"\n"
        + b'{"id": 99, "author": "bad \x96 byte"}\n'
    )

    writer = _load_state_store("cp1252").JsonlWriter(path)
    try:
        assert writer.has("id:1")
        assert not writer.has("id:99")
        assert writer.write({"id": 99, "author": "good byte"}) is True
    finally:
        writer.close()


def test_one_damaged_byte_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None:
    """A complete JSON line with a stray 0x96 parses as cp1252, but is only one vote."""
    path = tmp_path / "out.jsonl"
    healthy = ["Jürgen", "Grüße", "Björn"]
    path.write_bytes(
        json.dumps({"id": 0, "author": healthy[0]}, ensure_ascii = False).encode()
        + b"\n"
        + b'{"id": 99, "author": "bad \x96 byte"}\n'
        + b"".join(
            json.dumps({"id": i, "author": a}, ensure_ascii = False).encode() + b"\n"
            for i, a in enumerate(healthy[1:], start = 1)
        )
    )
    before = path.read_bytes()

    _load_state_store("cp1252").JsonlWriter(path).close()

    assert path.read_bytes() == before
    rows = []
    for line in path.read_bytes().splitlines():
        try:
            rows.append(json.loads(line.decode()))
        except (UnicodeDecodeError, ValueError):
            continue
    assert [row["author"] for row in rows] == healthy


def test_a_torn_line_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None:
    """One interrupted append must not get the whole shard read as cp1252."""
    path = tmp_path / "out.jsonl"
    good = [{"id": 1, "author": "Jürgen"}, {"id": 3, "author": "Grüße"}]
    torn = '{"id": 2, "author": "Jürgen"}'.encode()[:-6]  # cut mid-character
    path.write_bytes(
        json.dumps(good[0], ensure_ascii = False).encode()
        + b"\n"
        + torn
        + b"\n"
        + json.dumps(good[1], ensure_ascii = False).encode()
        + b"\n"
    )
    before = path.read_bytes()

    writer = _load_state_store("cp1252").JsonlWriter(path)
    try:
        assert writer.has("id:1") and writer.has("id:3")
        assert not writer.has("id:2")
    finally:
        writer.close()

    after = path.read_bytes()
    assert after.startswith(before)
    assert "Jürgen".encode() in after
    assert "Jürgen".encode("utf-8").decode("cp1252").encode() not in after


def test_an_undecodable_transport_marker_reads_as_unknown(tmp_path: Path) -> None:
    """An undecodable marker must read as unknown; a raised UnicodeDecodeError would abort the transfer."""
    _shared_setup_1(__file__)
    from hub.utils import download_registry as registry

    marker = tmp_path / ".transport"
    marker.write_bytes(b"\x80\xffnative\n")
    assert registry._read_marker_value(marker) is None
    marker.write_text("something-else\n", encoding = "utf-8")
    assert registry._read_marker_value(marker) is None


def test_a_torn_cache_ref_reads_as_not_cached(tmp_path: Path, monkeypatch) -> None:
    """A refs/main with undecodable bytes must read as not cached, since a raise would turn into a 500."""
    _shared_setup_1(__file__)
    from utils import utils as backend_utils

    good_root = tmp_path / "good"
    torn_root = tmp_path / "torn"
    for root, ref_bytes in ((torn_root, b"\x80\xff\n"), (good_root, b"abc123\n")):
        repo = root / "models--Org--Model"
        (repo / "refs").mkdir(parents = True)
        (repo / "refs" / "main").write_bytes(ref_bytes)
    (good_root / "models--Org--Model" / "snapshots" / "abc123").mkdir(parents = True)

    monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root])
    assert backend_utils.hf_cache_snapshot_dir("Org/Model") is None
    monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root, good_root])
    found = backend_utils.hf_cache_snapshot_dir("Org/Model")
    assert found is not None and found.name == "abc123"


def test_a_corrupt_pid_file_does_not_abort_shutdown(tmp_path: Path, monkeypatch) -> None:
    """_remove_pid_file runs first in _graceful_shutdown, so a raise there leaves
    the inference, export, training and tunnel children alive."""
    _shared_setup_1(__file__)
    import run as studio_run

    pid_file = tmp_path / "studio.pid"
    pid_file.write_bytes(b"\x80\xff")
    monkeypatch.setattr(studio_run, "_PID_FILE", pid_file)
    # _legacy_heir scans the real Unsloth root; a running server would change the outcome.
    monkeypatch.setattr(studio_run, "_legacy_heir", lambda: None)
    studio_run._remove_pid_file()
    assert pid_file.exists()

    pid_file.write_text(str(os.getpid()), encoding = "utf-8")
    studio_run._remove_pid_file()
    assert not pid_file.exists()


def test_the_legacy_pid_record_is_handed_to_a_live_sibling(tmp_path: Path, monkeypatch) -> None:
    """Only one server owns studio.pid, so deleting it while a sibling still serves would leave
    that sibling unstoppable from an older CLI that reads no other file."""
    _shared_setup_1(__file__)
    import run as studio_run

    pid_file = tmp_path / "studio.pid"
    pid_file.write_text(str(os.getpid()), encoding = "utf-8")
    monkeypatch.setattr(studio_run, "_PID_FILE", pid_file)
    monkeypatch.setattr(studio_run, "_legacy_heir", lambda: 4242)

    studio_run._remove_pid_file()

    assert pid_file.read_text(encoding = "utf-8") == "4242"


def test_the_kwargs_guard_only_judges_dicts_that_reach_a_call(tmp_path: Path) -> None:
    """Only a dict splatted into a call is subprocess config; an unrelated payload with text=True is not."""
    cases = {
        "offender.py": 'kw = {"text": True}\nrun(cmd, **kw)\n',
        "annotated.py": 'kw: dict = {"universal_newlines": True}\nrun(cmd, **kw)\n',
        "payload.py": 'payload = {"text": True}\nrequests.post(url, json = payload)\n',
        "inline.py": 'run(cmd, **{"text": True})\n',
        "later.py": 'kw = {"text": True}\nkw["encoding"] = "utf-8"\nrun(cmd, **kw)\n',
        "carried.py": 'kw = {"text": True, "encoding": "utf-8"}\nrun(cmd, **kw)\n',
    }
    flagged = set()
    for name, source in cases.items():
        path = tmp_path / name
        path.write_text(source, encoding = "utf-8")
        if any("subprocess kwargs" in line for line in _offenders(path)):
            flagged.add(name)
    assert flagged == {"offender.py", "annotated.py", "inline.py"}, flagged


def test_the_guard_follows_subprocess_through_an_alias(tmp_path: Path) -> None:
    """install_wheel() takes ``run = subprocess.run`` and calls it as a bare
    name, so an attribute-only match let both of its installer calls drop their
    encoding unnoticed. A name bound to something else is still not subprocess."""
    cases = {
        "param_default.py": (
            "import subprocess\n"
            "def install(*, run = subprocess.run):\n"
            "    run(cmd, text = True)\n"
        ),
        "assigned.py": "import subprocess\n_run = subprocess.run\n_run(cmd, text = True)\n",
        "imported.py": "from subprocess import check_output\ncheck_output(cmd, text = True)\n",
        "renamed.py": "from subprocess import run as _r\n_r(cmd, universal_newlines = True)\n",
        "encoded.py": (
            "import subprocess\n"
            "def install(*, run = subprocess.run):\n"
            '    run(cmd, text = True, encoding = "utf-8")\n'
        ),
        "unrelated.py": "def run(cmd, text = False):\n    pass\nrun(cmd, text = True)\n",
    }
    flagged = set()
    for name, source in cases.items():
        path = tmp_path / name
        path.write_text(source, encoding = "utf-8")
        if any("subprocess(text = True)" in line for line in _offenders(path)):
            flagged.add(name)
    assert flagged == {"param_default.py", "assigned.py", "imported.py", "renamed.py"}, flagged


def test_the_guard_sees_os_fdopen(tmp_path: Path) -> None:
    """os.fdopen shares open()'s locale default; missing it let a lock file write in the codepage."""
    cases = {
        "text.py": 'import os\nos.fdopen(fd, "w")\n',
        "default_mode.py": "import os\nos.fdopen(fd)\n",  # defaults to "r", still text
        "binary.py": 'import os\nos.fdopen(fd, "wb")\n',
        "keyword.py": 'import os\nos.fdopen(fd, "w", encoding = "utf-8")\n',
        "positional.py": 'import os\nos.fdopen(fd, "w", 1, "utf-8")\n',
    }
    flagged = set()
    for name, source in cases.items():
        path = tmp_path / name
        path.write_text(source, encoding = "utf-8")
        if any("fdopen" in line for line in _offenders(path)):
            flagged.add(name)
    assert flagged == {"text.py", "default_mode.py"}, flagged


def test_the_guard_reads_a_positional_path_text_encoding(tmp_path: Path) -> None:
    """Path.read_text(encoding, errors) and Path.write_text(data, encoding, ...) take the
    encoding positionally too. #13565's read_text("utf-8") is not a locale-default read."""
    cases = {
        "read_none.py": "p.read_text()\n",
        "read_kw.py": 'p.read_text(encoding = "utf-8")\n',
        "read_pos.py": 'p.read_text("utf-8")\n',
        "write_none.py": 'p.write_text("x")\n',
        "write_kw.py": 'p.write_text("x", encoding = "utf-8")\n',
        "write_pos.py": 'p.write_text("x", "utf-8")\n',
        # The locale default spelled out is still the locale default.
        "read_none_pos.py": "p.read_text(None)\n",
        "read_locale_pos.py": 'p.read_text("locale")\n',
        "read_locale_kw.py": 'p.read_text(encoding = "locale")\n',
        "write_none_pos.py": 'p.write_text("x", None)\n',
        # Unbound calls carry the path first, so the encoding sits one slot later.
        "unbound_read.py": "Path.read_text(p)\n",
        "unbound_read_pos.py": 'Path.read_text(p, "utf-8")\n',
        "unbound_write.py": 'pathlib.Path.write_text(p, "x")\n',
        "unbound_write_pos.py": 'pathlib.Path.write_text(p, "x", "utf-8")\n',
        # Under an alias, or a concrete pathlib class.
        "unbound_alias.py": "from pathlib import Path as P\nP.read_text(p)\n",
        "unbound_alias_pos.py": 'from pathlib import Path as P\nP.read_text(p, "utf-8")\n',
        "unbound_posix.py": 'PosixPath.write_text(p, "x")\n',
        # A same-named instance elsewhere is not an alias: no module-wide assignment guessing.
        "instance_named_p.py": 'def a():\n    P = Path\n\ndef b(P):\n    P.read_text("utf-8")\n',
        # A starred argument hides the runtime position: only a keyword encoding counts.
        "starred_empty.py": "p.read_text(*())\n",
        "unbound_starred.py": "Path.read_text(p, *())\n",
        "starred_kw.py": 'p.read_text(*args, encoding = "utf-8")\n',
    }
    flagged = set()
    for name, source in cases.items():
        path = tmp_path / name
        path.write_text(source, encoding = "utf-8")
        if _offenders(path):
            flagged.add(name)
    assert flagged == {
        "read_none.py",
        "write_none.py",
        "read_none_pos.py",
        "read_locale_pos.py",
        "read_locale_kw.py",
        "write_none_pos.py",
        "unbound_read.py",
        "unbound_write.py",
        "unbound_alias.py",
        "unbound_posix.py",
        "starred_empty.py",
        "unbound_starred.py",
    }, flagged


def test_an_undecodable_bootstrap_password_does_not_stop_startup(
    tmp_path: Path, monkeypatch
) -> None:
    """Lifespan calls _load_bootstrap_password with no handler, so an unusable file must be ignored
    rather than raised; a raise takes the backend down."""
    _shared_setup_1(__file__)
    from auth import storage

    pw_file = tmp_path / ".bootstrap_password"
    pw_file.write_bytes(b"\x80\xffnot-utf8\n")
    monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", pw_file)
    assert storage._load_bootstrap_password() is None

    pw_file.write_text("correct horse battery staple\n", encoding = "utf-8")
    assert storage._load_bootstrap_password() == "correct horse battery staple"


def test_a_damaged_checkpoint_resets_instead_of_resuming_on_a_broken_cursor(tmp_path: Path) -> None:
    """A mojibaked cursor makes GitHub report the stream done; reset the checkpoint, don't resume it."""
    module = _load_state_store("cp1252")
    cursor = "Y3Vyc29yOnYyOpK0MjAxMi0wMi0xNlQwNjo1Mzo0MVrOADGL_A=="
    healthy = json.dumps({"issues_cursor": cursor, "issues_done": False}, indent = 2)
    path = tmp_path / "octocat__Hello-World.json"

    path.write_text(healthy, encoding = "utf-8")
    assert module.StateStore(path).get("issues_cursor") == cursor

    # Written in the operator's codepage by a pre-UTF-8 release.
    path.write_bytes(healthy.encode("cp1252"))
    assert module.StateStore(path).get("issues_cursor") == cursor

    # Valid JSON under a single-byte codepage, so only refusing that reading resets it.
    raw = healthy.encode()
    at = raw.index(b"MjAxMi0wMi0xNlQ") + 3
    path.write_bytes(raw[:at] + b"\x96" + raw[at + 1 :])
    assert json.loads(path.read_bytes().decode("latin-1"))["issues_cursor"] != cursor
    store = module.StateStore(path)
    assert store.all() == {}
    assert store.get("issues_cursor") is None


def test_a_utf8_record_is_not_parsed_a_second_time(tmp_path: Path) -> None:
    """These shards reach gigabytes and every resume reads all of one, so a
    record that already read as UTF-8 must not be decoded and parsed again under
    the codepage. The legacy reading exists only to recover keys UTF-8 could not."""
    module = _load_state_store("cp1252")
    calls: list[str] = []
    real_parse = module._parse

    def counting_parse(raw, encoding):
        calls.append(encoding)
        return real_parse(raw, encoding)

    module._parse = counting_parse
    try:
        healthy = json.dumps({"id": 1, "author": "Jürgen"}).encode("utf-8")
        reading = module._read_line(healthy, "cp1252")
        assert reading.as_utf8 == {"id": 1, "author": "Jürgen"}
        assert calls == ["utf-8"], calls

        calls.clear()
        legacy = json.dumps({"id": 2, "author": "Jürgen"}, ensure_ascii = False).encode("cp1252")
        reading = module._read_line(legacy, "cp1252")
        assert reading.as_utf8 is None
        assert reading.as_legacy == {"id": 2, "author": "Jürgen"}
        assert calls == ["utf-8", "cp1252"], calls
    finally:
        module._parse = real_parse


def _too_deeply_nested_json() -> str:
    """Probed: json.loads' nesting limit is the C recursion limit from 3.12, not setrecursionlimit."""
    depth = 1
    while depth <= 1 << 17:
        document = "[" * depth + "]" * depth
        try:
            json.loads(document)
        except RecursionError:
            return document
        depth *= 2
    pytest.skip("this interpreter parses arbitrarily nested JSON")


def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path) -> None:
    """RecursionError is neither ValueError nor UnicodeDecodeError, so catch it or startup aborts."""
    module = _load_state_store("cp1252")
    nested = _too_deeply_nested_json()

    checkpoint = tmp_path / "octocat__Hello-World.json"
    checkpoint.write_text(nested, encoding = "utf-8")
    assert module.StateStore(checkpoint).all() == {}

    shard = tmp_path / "out.jsonl"
    shard.write_text(
        nested + "\n" + json.dumps({"id": 1}) + "\n" + json.dumps({"id": 2}) + "\n",
        encoding = "utf-8",
    )
    writer = module.JsonlWriter(shard)
    try:
        assert writer.has("id:1") and writer.has("id:2")
    finally:
        writer.close()


def test_vendored_laya_reads_utf8_config_under_a_non_utf8_locale(tmp_path):
    """Under a non-UTF-8 locale bare open() misdecodes, so vendored laya must read its config as UTF-8."""
    config = tmp_path / "tokenizer" / "tokenizer_config.json"
    config.parent.mkdir()
    config.write_text(
        json.dumps(
            {"tokenizer_class": None, "extra_special_tokens": ["ä", "世"]}, ensure_ascii = False
        ),
        encoding = "utf-8",
    )
    # C locale without coercion decodes bare open() as ASCII (Linux/macOS) or ANSI (Windows).
    script = (
        "import codecs, locale, sys\n"
        "if codecs.lookup(locale.getencoding()).name == 'utf-8': sys.exit(3)\n"
        f"sys.path.insert(0, {str(BACKEND_ROOT)!r})\n"
        "from core.systemone import laya_runtime\n"
        f"laya_runtime._laya().agent._fix_tokenizer_config({str(tmp_path)!r})\n"
    )
    env = {**os.environ, "LC_ALL": "C", "LANG": "C", "PYTHONCOERCECLOCALE": "0", "PYTHONUTF8": "0"}
    run = subprocess.run(
        [sys.executable, "-c", script], env = env, capture_output = True, text = True, encoding = "utf-8"
    )
    if run.returncode == 3:
        pytest.skip("this platform's locale is UTF-8 even under LC_ALL=C")
    assert run.returncode == 0, run.stderr[-3000:]
    repaired = json.loads(config.read_text(encoding = "utf-8"))
    assert repaired["tokenizer_class"] == "PreTrainedTokenizerFast"
    assert repaired["extra_special_tokens"] == {"extra_0": "ä", "extra_1": "世"}