Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

31.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Units are frozen and all distinct, since Shiki caches output keyed on the source string."""

from __future__ import annotations

import hashlib
import json
import random
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Iterator, Optional

FROZEN_DIR = Path(__file__).resolve().parent / "corpus" / "frozen"
UNITS_JSONL = FROZEN_DIR / "units.jsonl"
MANIFEST_JSON = FROZEN_DIR / "manifest.json"

CORPUS_SEED = 20260819
# Bump whenever generated text changes: floor_table refuses to pool numbers across versions.
CORPUS_VERSION = 2

# Calibrated against the field capture; these are means of a jittered distribution.
PROSE_CHARS = 1250
FENCE_CHARS = 1800
PREAMBLE_FRACTION = 0.25

# Jitter decorrelates per-block from per-char cost; uses the per-unit rng so units stay stable.
BLOCK_JITTER = 0.55

UNIT_JITTER = 0.35

TOOL_CALLS_PER_UNIT = (0, 3)

#: The stored part shape that actually renders a tool block. VERIFIED, not assumed: a flat
#: `{"type": "tool-call"}` and an assistant-ui `{"type": "tool-invocation"}` both rendered
#: their sibling text and NO tool UI, while this shape produced a "Used tool" group.
TOOL_NAMES = ("web_search", "code_execution", "python", "terminal", "search_knowledge_base")

CYCLE_BASE = ((("reasoning", 10_000), ("code", 8_000)),)
# Cap doubling, else the 1M rung is a single 2.5M-char turn.
MAX_UNIT_CHARS = 320_000

# Units past this many chars are regenerated locally and checked against the manifest hash.
SHIPPED_CHARS_BUDGET = 460_000


_NOUNS = (
    "buffer",
    "scheduler",
    "fibre",
    "observer",
    "commit",
    "layout",
    "token",
    "shard",
    "lane",
    "chunk",
    "boundary",
    "descriptor",
    "checkpoint",
    "gradient",
    "kernel",
    "adapter",
    "window",
    "residual",
    "quantiser",
    "allocator",
    "transcript",
    "viewport",
    "sentinel",
    "digest",
)
_VERBS = (
    "reparses",
    "flushes",
    "clones",
    "invalidates",
    "schedules",
    "accumulates",
    "walks",
    "coalesces",
    "materialises",
    "spills",
    "reconciles",
    "hoists",
    "pins",
    "drains",
)
_ADJS = (
    "cumulative",
    "monotonic",
    "deferred",
    "synchronous",
    "transient",
    "bounded",
    "retained",
    "speculative",
    "interleaved",
    "hysteretic",
    "collinear",
    "saturating",
)
_CONNECTIVES = (
    "which means",
    "so in practice",
    "and therefore",
    "but only when",
    "except that",
    "as long as",
    "provided that",
    "which is why",
)
_LANGS = ("python", "typescript", "rust", "go", "c")
# Span density is per token, so long names dilute it.
_SHORT = (
    "a",
    "b",
    "c",
    "d",
    "e",
    "f",
    "g",
    "h",
    "i",
    "j",
    "k",
    "n",
    "p",
    "q",
    "s",
    "t",
    "u",
    "v",
    "w",
    "x",
    "y",
    "z",
    "acc",
    "buf",
    "idx",
    "key",
    "len",
    "map",
    "ptr",
    "res",
    "tmp",
)

# Both delimiter families: remark-math consumes $...$, preprocessLaTeX rewrites \(...\).
_MATH_OPS = ("+", "-", "\\cdot", "\\times", "\\oplus")
_MATH_RELS = ("=", "\\le", "\\ge", "\\approx", "\\equiv")
_MATH_FUNCS = ("\\log", "\\exp", "\\sin", "\\cos", "\\tanh")
_MATH_GREEK = (
    "\\alpha",
    "\\beta",
    "\\gamma",
    "\\delta",
    "\\theta",
    "\\lambda",
    "\\mu",
    "\\sigma",
    "\\phi",
    "\\omega",
)

# Math replaces a prose block with the same size distribution, so fence calibration is unchanged.
MATH_BLOCK_PROB = 0.16

# Inline math is spent from the prose block's own budget.
INLINE_MATH_PROB = 0.22

# The preamble stays pure prose so the onset of span cost is visible against it.


@dataclass(frozen = True)
class Unit:
    """One assistant turn's worth of content."""

    index: int
    kind: str
    reasoning: str
    content: str
    chars: int
    sha256: str
    # Not counted in `chars`, which must stay comparable with earlier measurements.
    tool_calls: tuple = ()

    @property
    def tool_call_count(self) -> int:
        return len(self.tool_calls)

    def as_row(self) -> dict:
        return {
            "index": self.index,
            "kind": self.kind,
            "chars": self.chars,
            "sha256": self.sha256,
            "reasoning": self.reasoning,
            "content": self.content,
            "tool_calls": list(self.tool_calls),
        }

    def clipped_to(self, chars: int) -> "Unit":
        """Clip on block boundaries: an unclosed fence would take a different render path."""
        if chars >= self.chars:
            return self

        def take(text: str, budget: int) -> str:
            blocks = text.split("\n\n")
            out: list[str] = []
            size = 0
            for b in blocks:
                if out and size + len(b) + 2 > budget:
                    break
                out.append(b)
                size += len(b) + 2
            return "\n\n".join(out)

        reasoning_budget = max(1, int(chars * (len(self.reasoning) / self.chars)))
        reasoning = take(self.reasoning, reasoning_budget)
        content = take(self.content, max(1, chars - len(reasoning)))
        text = reasoning + content
        return Unit(
            index = self.index,
            kind = self.kind,
            reasoning = reasoning,
            content = content,
            chars = len(text),
            # Different digest: a clipped unit must not be checked against the frozen unit's hash.
            sha256 = "clip:"
            + hashlib.sha256(f"{self.sha256}\x00{chars}".encode("utf-8")).hexdigest(),
            # Keep tool calls when clipping; they carry no chars and the smallest rung is the ratio base.
            tool_calls = self.tool_calls,
        )

    @classmethod
    def from_row(cls, row: dict) -> "Unit":
        return cls(
            index = row["index"],
            kind = row["kind"],
            reasoning = row["reasoning"],
            content = row["content"],
            chars = row["chars"],
            sha256 = row["sha256"],
            tool_calls = tuple(row.get("tool_calls") or ()),
        )


def _expression(rng: random.Random, salt: str, terms: int) -> str:
    """Salt sits in a subscript, not a comment, so no normaliser can strip it and restore the cache hit."""
    parts = [f"{rng.choice(_MATH_GREEK)}_{{{salt}}}", rng.choice(_MATH_RELS)]
    for i in range(terms):
        if i:
            parts.append(rng.choice(_MATH_OPS))
        shape = rng.random()
        if shape < 0.30:
            parts.append(
                f"\\frac{{{rng.choice(_MATH_GREEK)}^{{{rng.randint(2, 9)}}}}}"
                f"{{{rng.randint(1, 99)} {rng.choice(_MATH_GREEK)}}}"
            )
        elif shape < 0.50:
            parts.append(
                f"\\sum_{{{rng.choice(_SHORT)}={rng.randint(0, 3)}}}^{{{rng.randint(4, 99)}}} "
                f"{rng.choice(_MATH_GREEK)}_{{{rng.choice(_SHORT)}}}"
            )
        elif shape < 0.65:
            parts.append(f"\\sqrt{{{rng.choice(_MATH_GREEK)} {rng.randint(2, 99)}}}")
        elif shape < 0.80:
            parts.append(
                f"{rng.choice(_MATH_FUNCS)}\\left({rng.choice(_MATH_GREEK)}"
                f"^{{{rng.randint(2, 5)}}}\\right)"
            )
        else:
            parts.append(f"{rng.choice(_MATH_GREEK)}^{{{rng.randint(2, 9)}}}")
    return " ".join(parts)


def _inline_math(rng: random.Random, salt: str) -> str:
    """The backslash-paren form only reaches the renderer through preprocessLaTeX, not markdown syntax."""
    body = _expression(rng, salt, rng.randint(1, 2))
    return f"$ {body} $" if rng.random() < 0.65 else f"\\( {body} \\)"


def _sentence(
    rng: random.Random,
    salt: str,
    *,
    math: bool = False,
) -> str:
    tail = f" where {_inline_math(rng, salt)} holds" if math else ""
    return (
        f"The {rng.choice(_ADJS)} {rng.choice(_NOUNS)} {rng.choice(_VERBS)} the "
        f"{rng.choice(_ADJS)} {rng.choice(_NOUNS)}_{salt}, {rng.choice(_CONNECTIVES)} the "
        f"{rng.choice(_NOUNS)} stays {rng.choice(_ADJS)}{tail}."
    )


def _prose(
    rng: random.Random,
    target: int,
    salt: str,
    *,
    math: bool = False,
) -> str:
    """Fence-free prose; math is spent from target so block sizes keep their calibrated distribution."""
    out: list[str] = []
    size = 0
    while size < target:
        para: list[str] = []
        for _ in range(rng.randint(3, 6)):
            s = _sentence(rng, salt, math = math and rng.random() < INLINE_MATH_PROB)
            para.append(s)
            size += len(s) + 1
        out.append(" ".join(para))
        size += 2
        if size >= target:
            break
    return "\n\n".join(out)


def _math_block(rng: random.Random, target: int, salt: str) -> str:
    """Many display blocks, not one big equation, so the cost matches what accumulates in long threads."""
    out: list[str] = []
    size = 0
    i = 0
    while size < target:
        body = _expression(rng, f"{salt}m{i}", rng.randint(3, 6))
        block = f"$$\n{body}\n$$" if rng.random() < 0.65 else f"\\[\n{body}\n\\]"
        out.append(block)
        size += len(block) + 2
        i += 1
        if size >= target:
            break
        gloss = _prose(rng, _jitter(rng, 260, floor = 60), f"{salt}g{i}", math = True)
        out.append(gloss)
        size += len(gloss) + 2
    return "\n\n".join(out)


def _fence(
    rng: random.Random,
    target: int,
    salt: str,
    lang: Optional[str] = None,
) -> str:
    """Salt goes in identifiers, not comments, since a highlighter could normalise comments away."""
    lang = lang or rng.choice(_LANGS)
    lines = [f"```{lang}"]
    size = len(lines[0]) + 1
    i = 0
    # Short names: the density target is spans, not characters (field capture: 5.6 chars/span).
    r = rng.randint

    def v() -> str:
        # Mix short and long names to approximate the field's 5.6 chars/span.
        return rng.choice(_SHORT) if rng.random() < 0.55 else rng.choice(_NOUNS)

    while size < target:
        # One salted identifier per line suffices to defeat Shiki's source-keyed cache.
        u = f"{rng.choice(_SHORT)}{salt}{i}"
        if lang == "python":
            line = (
                f"    {u}={v()}[{r(0,9)}]*{r(2,99)}-{v()}({v()},{v()})+{v()}[{r(0,9)}:{r(1,9)}]"
                f";{v()}={v()}%{r(2,9)} if {v()}>{r(1,99)} else {v()}|{r(1,7)}"
            )
        elif lang == "typescript":
            line = (
                f"  const {u}={{a:{v()}[{r(0,9)}],b:{v()}({v()},{r(1,99)}),c:{v()}?{v()}:"
                f"{r(0,9)}}};{v()}={v()}&{r(1,7)}|{v()}<<{r(1,7)};"
            )
        elif lang == "rust":
            line = (
                f"    let {u}:u64={v()}[{r(0,9)}]^{v()}({v()},{r(1,99)})&{r(1,255)};"
                f"{v()}={v()}.iter().map(|x|x*{r(2,99)}).sum::<u64>();"
            )
        elif lang == "go":
            line = (
                f"\t{u}:={v()}[{r(0,9)}]+{v()}({v()},{r(1,99)})*{r(2,99)};"
                f"{v()},{v()}={v()}%{r(2,9)},{v()}>>{r(1,7)}"
            )
        else:
            line = (
                f"    uint64_t {u}={v()}[{r(0,9)}]|({v()}({v()},{r(1,99)})&0x{r(16,255):02x})"
                f";{v()}=*{v()}++^{r(1,255)};"
            )
        lines.append(line)
        size += len(line) + 1
        i += 1
    lines.append("```")
    return "\n".join(lines)


def _jitter(
    rng: random.Random,
    mean: int,
    spread: float = BLOCK_JITTER,
    floor: int = 80,
) -> int:
    """Uniform spread rather than lognormal, so the mean stays where the span-density calibration put it."""
    return max(floor, int(mean * (1.0 + rng.uniform(-spread, spread))))


def _body(rng: random.Random, target: int, salt: str, *, preamble: bool) -> str:
    """Preamble, then a jittered prose/fence alternation, to `target` characters."""
    parts: list[str] = []
    size = 0
    if preamble:
        want = int(target * PREAMBLE_FRACTION)
        head = _prose(rng, want, f"{salt}p")
        parts.append(head)
        size += len(head) + 2
    while size < target:
        want = _jitter(rng, PROSE_CHARS)
        if rng.random() < MATH_BLOCK_PROB:
            p = _math_block(rng, want, f"{salt}{len(parts)}")
        else:
            p = _prose(rng, want, f"{salt}{len(parts)}", math = True)
        parts.append(p)
        size += len(p) + 2
        if size >= target:
            break
        f = _fence(rng, _jitter(rng, FENCE_CHARS), f"{salt}{len(parts)}")
        parts.append(f)
        size += len(f) + 2
    return "\n\n".join(parts)


def _tool_calls(rng: random.Random, salt: str) -> list[dict]:
    """Zero to three tool calls for one turn, in the part shape the app actually renders."""
    low, high = TOOL_CALLS_PER_UNIT
    out: list[dict] = []
    for i in range(rng.randint(low, high)):
        name = rng.choice(TOOL_NAMES)
        query = _prose(rng, _jitter(rng, 220, floor = 40), f"{salt}t{i}q").replace("\n", " ")
        result = _prose(rng, _jitter(rng, 900, floor = 120), f"{salt}t{i}r")
        args = (
            {"query": query} if name in ("web_search", "search_knowledge_base") else {"code": query}
        )
        out.append(
            {
                "type": "tool-call",
                "toolCallId": f"call_{salt}_{i}",
                "toolName": name,
                "argsText": json.dumps(args),
                "args": args,
                "state": "result",
                "result": result,
            }
        )
    return out


def _unit_targets(index: int) -> tuple[str, int]:
    """(kind, target chars) for unit `index` of the escalating cycle."""
    cycle = index // 2
    slot = index % 2
    base = 10_000 if slot == 0 else 8_000
    kind = "reasoning" if slot == 0 else "code"
    nominal = min(MAX_UNIT_CHARS, base * (2**cycle))
    # Separate RNG seeded by index alone, so changing block jitter does not reshuffle unit sizes.
    rng = random.Random((index * 6_364_136_223_846_793_005) ^ 0x5DEECE66D)
    return kind, max(1_500, min(MAX_UNIT_CHARS, _jitter(rng, nominal, UNIT_JITTER, floor = 1_500)))


def generate_unit(index: int, seed: int = CORPUS_SEED) -> Unit:
    """Seeded per unit, not per corpus, so every rung shares the same prefix of units."""
    rng = random.Random((seed * 1_000_003) ^ (index * 2_654_435_761))
    kind, target = _unit_targets(index)
    salt = f"{index:04d}"
    if kind == "reasoning":
        reasoning = _body(rng, int(target * 0.8), f"r{salt}", preamble = True)
        content = _body(rng, int(target * 0.2), f"a{salt}", preamble = False)
    else:
        reasoning = _prose(rng, 1_200, f"r{salt}")
        content = _body(rng, target, f"a{salt}", preamble = False)
    tools = tuple(_tool_calls(rng, salt))
    chars = len(reasoning) + len(content)
    digest = hashlib.sha256(
        (
            f"{index}\x00{kind}\x00{reasoning}\x00{content}\x00" + json.dumps(tools, sort_keys = True)
        ).encode("utf-8")
    ).hexdigest()
    return Unit(
        index = index,
        kind = kind,
        reasoning = reasoning,
        content = content,
        chars = chars,
        sha256 = digest,
        tool_calls = tools,
    )


def units_for_chars(total_chars: int, seed: int = CORPUS_SEED) -> list[Unit]:
    """The shortest prefix of the cycle whose characters reach `total_chars`."""
    out: list[Unit] = []
    size = 0
    index = 0
    while size < total_chars:
        u = generate_unit(index, seed)
        out.append(u)
        size += u.chars
        index += 1
        if index > 4096:
            raise RuntimeError("corpus cycle did not reach the requested size")
    return out


def freeze(
    max_chars: int = SHIPPED_CHARS_BUDGET,
    seed: int = CORPUS_SEED,
    out_dir: Path = FROZEN_DIR,
) -> dict:
    """Write `units.jsonl` and `manifest.json`. Run deliberately; never on a benchmark run."""
    out_dir.mkdir(parents = True, exist_ok = True)
    shipped = units_for_chars(max_chars, seed)
    # Sized from the ladder, not a char budget: streamed turns and follow-ups live past the prefix.
    all_units = [generate_unit(i, seed) for i in range(manifest_unit_count(seed))]
    with (out_dir / "units.jsonl").open("w", encoding = "utf-8") as fh:
        for u in shipped:
            fh.write(json.dumps(u.as_row(), ensure_ascii = False) + "\n")
    manifest = {
        "corpus_version": CORPUS_VERSION,
        "seed": seed,
        "prose_chars": PROSE_CHARS,
        "fence_chars": FENCE_CHARS,
        "preamble_fraction": PREAMBLE_FRACTION,
        "math_block_prob": MATH_BLOCK_PROB,
        "inline_math_prob": INLINE_MATH_PROB,
        "max_unit_chars": MAX_UNIT_CHARS,
        "shipped_units": len(shipped),
        "shipped_chars": sum(u.chars for u in shipped),
        "units": [
            {"index": u.index, "kind": u.kind, "chars": u.chars, "sha256": u.sha256}
            for u in all_units
        ],
    }
    manifest["corpus_hash"] = corpus_hash(manifest)
    (out_dir / "manifest.json").write_text(
        json.dumps(manifest, indent = 2, ensure_ascii = False) + "\n", encoding = "utf-8"
    )
    return manifest


def corpus_hash(manifest: dict) -> str:
    """Includes parameters as well as unit digests, so different declared densities never hash equal."""
    h = hashlib.sha256()
    for key in (
        "corpus_version",
        "seed",
        "prose_chars",
        "fence_chars",
        "preamble_fraction",
        "math_block_prob",
        "inline_math_prob",
        "max_unit_chars",
    ):
        h.update(f"{key}={manifest[key]}\x00".encode("utf-8"))
    for u in manifest["units"]:
        h.update(f"{u['index']}:{u['sha256']}\x00".encode("utf-8"))
    return h.hexdigest()


class Corpus:
    """The frozen corpus, with every unit checked against the manifest before it is used."""

    def __init__(self, manifest: dict, shipped: dict[int, Unit], seed: int) -> None:
        self.manifest = manifest
        self.seed = seed
        self._shipped = shipped
        self._expected = {u["index"]: u["sha256"] for u in manifest["units"]}
        self.corpus_hash = manifest["corpus_hash"]
        self.regenerated: list[int] = []

    @classmethod
    def load(cls, frozen_dir: Optional[Path] = None) -> "Corpus":
        # Use the resource loader: inside studiobench.pyz the corpus is a zip member.
        from ..runtime import resources

        if frozen_dir is not None:
            manifest_path = Path(frozen_dir) / "manifest.json"
            if not manifest_path.exists():
                raise FileNotFoundError(f"no frozen manifest at {manifest_path}")
            raw_manifest = manifest_path.read_text(encoding = "utf-8")
            raw_units = ""
            units_path = Path(frozen_dir) / "units.jsonl"
            if units_path.exists():
                raw_units = units_path.read_text(encoding = "utf-8")
        else:
            try:
                raw_manifest = resources.read_text("fixture/corpus/frozen/manifest.json")
            except (FileNotFoundError, OSError) as exc:
                raise FileNotFoundError(
                    "the frozen corpus is not in this build. Run "
                    "`python -m tests.studio.studiobench.fixture.corpus --freeze` to build it."
                ) from exc
            try:
                raw_units = resources.read_text("fixture/corpus/frozen/units.jsonl")
            except (FileNotFoundError, OSError):
                raw_units = ""
        manifest = json.loads(raw_manifest)
        recomputed = corpus_hash(manifest)
        if recomputed != manifest.get("corpus_hash"):
            raise ValueError(
                f"the frozen manifest's own corpus_hash does not match its contents "
                f"({manifest.get('corpus_hash')} declared, {recomputed} recomputed). The corpus "
                "has been edited by hand; rebuild it with --freeze."
            )
        shipped: dict[int, Unit] = {}
        for line in raw_units.splitlines():
            line = line.strip()
            if line:
                u = Unit.from_row(json.loads(line))
                shipped[u.index] = u
        return cls(manifest, shipped, manifest["seed"])

    def unit(self, index: int) -> Unit:
        """Shipped or regenerated, a unit must match its manifest digest or the generator has drifted."""
        u = self._shipped.get(index)
        if u is None:
            u = generate_unit(index, self.seed)
            self.regenerated.append(index)
        expected = self._expected.get(index)
        if expected is None:
            raise KeyError(f"unit {index} is outside the frozen manifest")
        if u.sha256 != expected:
            raise ValueError(
                f"corpus unit {index} does not match the frozen manifest "
                f"(expected {expected[:16]}, got {u.sha256[:16]}). The generator has drifted; "
                "this tree cannot reproduce the shipped corpus."
            )
        return u

    def units_for_chars(self, total_chars: int) -> list[Unit]:
        out: list[Unit] = []
        size = 0
        index = 0
        while size < total_chars:
            u = self.unit(index)
            out.append(u)
            size += u.chars
            index += 1
        return out

    def iter_units(self) -> Iterator[Unit]:
        for entry in self.manifest["units"]:
            yield self.unit(entry["index"])


RUNGS: dict[str, int] = {
    "1K": 1_000,
    "10K": 10_000,
    "100K": 100_000,
    "500K": 500_000,
    "1M": 1_000_000,
}

# Only sizes the corpus; reported chars_per_token is always the measured one.
PROVISIONAL_CHARS_PER_TOKEN = 4.0

# Stream must finish within ~20 s, before the first after-generation slot at 22.5 s.
STREAM_TAIL_CHARS = 6_000

# Opening turn plus send_turn follow-ups; consecutive units alternate reasoning- and code-heavy.
STREAM_TURNS = 3

FOLLOW_UP_CHARS = 1_500

# Below this, three turns would drain the opening stream too early, so stream once.
MULTI_TURN_MIN_CHARS = 20_000

# Headroom over the provisional 4.0 ratio; above this plan_rung refuses.
MANIFEST_CHARS_PER_TOKEN = 5.0


def manifest_unit_count(
    seed: int = CORPUS_SEED, chars_per_token: float = MANIFEST_CHARS_PER_TOKEN
) -> int:
    """Sized from the ladder: a short manifest once clamped the top rung's streamed turns onto one unit."""
    prefix = len(units_for_chars(int(max(RUNGS.values()) * chars_per_token), seed))
    return prefix + STREAM_TURNS


@dataclass
class RungPlan:
    """What a rung is made of, and how much of it streams rather than being seeded."""

    rung: str
    target_tokens: int
    target_chars: int
    seeded_units: list[Unit] = field(default_factory = list)
    streamed_unit: Optional[Unit] = None
    follow_up_units: list[Unit] = field(default_factory = list)

    @property
    def seeded_chars(self) -> int:
        return sum(u.chars for u in self.seeded_units)

    @property
    def streamed_chars(self) -> int:
        return self.streamed_unit.chars if self.streamed_unit else 0

    @property
    def follow_up_chars(self) -> int:
        return sum(u.chars for u in self.follow_up_units)

    @property
    def total_chars(self) -> int:
        # Follow-ups are on screen, so they count toward the rung's mass.
        return self.seeded_chars + self.streamed_chars + self.follow_up_chars


def dollarise(text: str, salt: str) -> str:
    """Adds non-math dollars to the streamed unit only, not the frozen corpus; exercises the heuristics."""
    lines = text.split("\n")
    out: list[str] = []
    fenced = False
    for index, line in enumerate(lines):
        stripped = line.lstrip()
        if stripped.startswith("```") or stripped.startswith("~~~"):
            fenced = not fenced
            out.append(line)
            continue
        # Shell lines inside fences and prices in prose exercise the code-region and currency paths.
        if fenced and index % 11 == 0:
            out.append(f"{line}  # $HOME/{salt}{index} costs $1{index % 10}.99")
        elif not fenced and line and index % 17 == 0:
            out.append(f"{line} It costs ${index % 9}{index % 10}.99 or ${index % 7},200 a year.")
        else:
            out.append(line)
    return "\n".join(out)


def _too_small(rung: str, chars_per_token: float, last_index: int, what: str) -> ValueError:
    """Every rung outgrowing the frozen corpus raises this error, not a clamp that measures less quietly."""
    return ValueError(
        f"the frozen corpus is too small for the {rung} rung at {chars_per_token} chars per "
        f"token: {what} runs past the manifest, which ends at index {last_index}. Re-freeze with "
        "`python -m tests.studio.studiobench.fixture.corpus --freeze` after raising "
        "MANIFEST_CHARS_PER_TOKEN, and note that a re-freeze changes corpus_hash and so ends "
        "comparability with everything measured before it."
    )


def _tail_not_deliverable(
    rung: str,
    requested: int,
    source_index: int,
    deliverable: int,
    thread_chars: int,
    target_chars: int,
) -> ValueError:
    """Raises when the corpus cannot deliver the requested tail: a shortfall silently fails to vary it."""
    return ValueError(
        f"the frozen corpus cannot deliver a {requested:,} character streamed tail at the {rung} "
        f"rung: the streamed turn is unit {source_index}, which is {deliverable:,} characters, so "
        f"the reply would be {deliverable / requested:.1%} of the one asked for. Asking for more "
        f"tail makes this worse rather than better, because the seeded prefix shrinks to pay for "
        f"it and the streamed turn moves to a smaller unit -- here the thread would collapse to "
        f"{thread_chars:,} characters against the rung's {target_chars:,}, while still being "
        f"recorded, weighted and reported as {rung}. Ask for less, or move the same request to a "
        f"higher rung. NOT 'at most {deliverable:,}': that is this unit's size at THIS request, "
        f"and the ceiling RISES as the request falls, because a smaller tail leaves a longer "
        f"seeded prefix which draws the streamed turn from a larger unit. Lower the ask until "
        f"this stops firing rather than reading a maximum off this line."
    )


def unit_text(unit: Unit) -> str:
    """Everything a unit puts on screen, in the order it arrives."""
    return unit.reasoning + unit.content


def check_streamed_units_are_new(plan: RungPlan) -> None:
    """Streamed turns must be material the thread has not seen, compared as prefixes of clipped units."""
    streamed: list[tuple[str, str]] = []
    if plan.streamed_unit is not None:
        streamed.append(("streamed", unit_text(plan.streamed_unit)))
    for i, unit in enumerate(plan.follow_up_units):
        streamed.append((f"follow_up[{i}]", unit_text(unit)))

    def fail(what: str, a: str, b: str) -> None:
        raise ValueError(
            f"rung {plan.rung}: {a} and {b} {what}. A streamed turn that repeats material already "
            "on screen hits Shiki's source-keyed cache instead of missing it, so the rung reports "
            "less work per character than it does. The corpus is too small for this ladder; "
            "re-freeze it with `--freeze`."
        )

    seeded = [(f"seeded unit {u.index}", unit_text(u)) for u in plan.seeded_units]
    for i, (label, text) in enumerate(streamed):
        if not text:
            raise ValueError(f"rung {plan.rung}: {label} is empty")
        for other_label, other in streamed[i + 1 :] + seeded:
            if text.startswith(other) or other.startswith(text):
                fail("share a prefix", label, other_label)


def plan_rung(
    corpus: Corpus,
    rung: str,
    chars_per_token: float = PROVISIONAL_CHARS_PER_TOKEN,
    stream_tail_chars: Optional[int] = None,
    dollars: bool = False,
) -> RungPlan:
    """Only the last reply streams, since streaming a million tokens at field cadence would take hours."""
    tokens = RUNGS[rung]
    target_chars = int(tokens * chars_per_token)

    # The streamed tail is constant across rungs so film slots stay correctly labelled; only the
    # seeded prefix grows. So costs scaling with reply length read as a floor on this ladder.
    turns = STREAM_TURNS if target_chars >= MULTI_TURN_MIN_CHARS else 1
    tail_budget = STREAM_TAIL_CHARS if stream_tail_chars is None else max(1, stream_tail_chars)
    tail_target = min(tail_budget, target_chars) if stream_tail_chars is None else tail_budget
    follow_budget = FOLLOW_UP_CHARS * (turns - 1)
    seed_target = max(0, target_chars - tail_target - follow_budget)

    # Trim the prefix's last unit to land on target; whole units would overshoot by a full turn.
    last_index = max(e["index"] for e in corpus.manifest["units"])
    try:
        seeded = corpus.units_for_chars(seed_target) if seed_target > 0 else []
    except KeyError as exc:
        raise _too_small(rung, chars_per_token, last_index, "its seeded prefix alone") from exc
    if seeded:
        overshoot = sum(u.chars for u in seeded) - seed_target
        if overshoot > 0:
            seeded[-1] = seeded[-1].clipped_to(max(1, seeded[-1].chars - overshoot))

    # No clamp: clamping to last_index duplicates the final unit and hits Shiki's cache.
    needed = len(seeded) + turns - 1
    if needed > last_index:
        raise _too_small(
            rung,
            chars_per_token,
            last_index,
            f"its {len(seeded)}-unit seeded prefix plus {turns} streamed turns need units up to "
            f"index {needed}, which",
        )

    # The opening turn keeps the full tail budget; follow-ups are extra.
    source = corpus.unit(len(seeded))
    # An explicit tail must fit in the unit; the default is a ceiling small rungs are allowed under.
    if stream_tail_chars is not None and tail_target >= source.chars:
        raise _tail_not_deliverable(
            rung,
            tail_target,
            len(seeded),
            source.chars,
            sum(u.chars for u in seeded) + source.chars + FOLLOW_UP_CHARS * (turns - 1),
            target_chars,
        )
    streamed = source.clipped_to(tail_target)
    follow_ups = [corpus.unit(len(seeded) + i).clipped_to(FOLLOW_UP_CHARS) for i in range(1, turns)]
    if dollars:
        # Only streamed turns: the seeded prefix is never re-preprocessed.
        streamed = replace(
            streamed,
            reasoning = dollarise(streamed.reasoning, "r"),
            content = dollarise(streamed.content, "c"),
        )
        streamed = replace(streamed, chars = len(streamed.reasoning) + len(streamed.content))
        follow_ups = [
            replace(u, reasoning = dollarise(u.reasoning, "r"), content = dollarise(u.content, "c"))
            for u in follow_ups
        ]
        follow_ups = [replace(u, chars = len(u.reasoning) + len(u.content)) for u in follow_ups]

    plan = RungPlan(
        rung = rung,
        target_tokens = tokens,
        target_chars = target_chars,
        seeded_units = seeded,
        streamed_unit = streamed,
        follow_up_units = follow_ups,
    )
    check_streamed_units_are_new(plan)
    return plan


def _main(argv: list[str]) -> int:
    import argparse

    ap = argparse.ArgumentParser(description = "Build or verify the frozen studiobench corpus.")
    ap.add_argument("--freeze", action = "store_true", help = "regenerate frozen/*.jsonl")
    ap.add_argument("--verify", action = "store_true", help = "check the tree against the freeze")
    args = ap.parse_args(argv)
    if args.freeze:
        m = freeze()
        print(
            f"froze {m['shipped_units']} units, {m['shipped_chars']:,} chars, "
            f"{len(m['units'])} manifest entries"
        )
        print(f"corpus_hash {m['corpus_hash']}")
        return 0
    c = Corpus.load()
    n = 0
    for _ in c.iter_units():
        n += 1
    print(f"verified {n} units against corpus_hash {c.corpus_hash}")
    print(f"regenerated (not shipped as text): {len(c.regenerated)} units")
    for rung in RUNGS:
        p = plan_rung(c, rung)
        print(
            f"  {rung:>5}: {len(p.seeded_units):>4} seeded units, "
            f"{p.seeded_chars:>10,} seeded chars, {p.streamed_chars:>9,} streamed chars"
        )
    return 0


if __name__ == "__main__":
    import sys
    raise SystemExit(_main(sys.argv[1:]))