Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

14.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""No lone frame metric may be a headline, and excluded_cells is always rendered, even when empty."""

from __future__ import annotations

from typing import Any, Iterable, Mapping, Sequence

from ..scoring.ab import AbResult
from ..scoring.anchors import METRIC_BY_KEY
from ..scoring.frames import FrameStats
from ..scoring.schema import Measure
from ..scoring.score import LadderScore, RungScore
from .payload import excluded_totals

HEADLINE_FRAME_METRICS = ("time_in_jank_pct", "jank_index", "max_frame_ms")


class HeadlinePolicyError(AssertionError):
    """Raised when a caller tries to quote one frame summary as the headline."""


def assert_headline_pair(keys: Iterable[str]) -> None:
    """The frame metrics catch opposite failures, so a headline quoting only some of them is a selection."""

    chosen = {k for k in keys if k in HEADLINE_FRAME_METRICS}
    if chosen and chosen != set(HEADLINE_FRAME_METRICS):
        missing = sorted(set(HEADLINE_FRAME_METRICS) - chosen)
        raise HeadlinePolicyError(
            "a frame headline must quote "
            + ", ".join(HEADLINE_FRAME_METRICS)
            + f"; missing {', '.join(missing)}"
        )


def render_frame_health(stats: FrameStats, *, indent: str = "") -> str:
    """All three frame headlines plus the histogram pointer. Never one of them alone."""

    lines = [
        f"{indent}frames            {stats.frames_total} over {stats.window_ms:.0f} ms "
        f"(budget {stats.budget_ms:.2f} ms, {stats.refresh_source})",
        f"{indent}time in jank      {stats.time_in_jank_pct.display()}   "
        "(wall time inside frames over 100 ms)",
        f"{indent}jank index        {stats.jank_index.display()}   "
        "(sum of squared over-budget ms per ms of window)",
        f"{indent}worst frame       {stats.max_frame_ms.display()}",
        f"{indent}p50 / p95 / p99   {stats.p50_frame_ms.display()} / "
        f"{stats.p95_frame_ms.display()} / {stats.p99_frame_ms.display()}",
    ]
    if stats.no_frames_recorded:
        lines.append(
            f"{indent}NOTE: the frame recorder produced no frames. That is not zero jank, it is "
            "no measurement: an unscheduled rAF loop reads exactly like a perfectly smooth page."
        )
    lines.append(f"{indent}histogram         {len(stats.histogram)} buckets, in the payload")
    return "\n".join(lines)


def render_headline(ladder: LadderScore) -> str:
    """The onset rung, in words, followed by the machine-local scalar."""

    lines: list[str] = []
    if ladder.onset_rung_tokens is None:
        lines.append("ONSET RUNG: none. This build is not usable at any measured thread size.")
    else:
        lines.append(
            f"ONSET RUNG: usable up to {ladder.onset_rung_tokens:,} tokens of thread content."
        )
    lines.append(f"  {ladder.onset_reason}")
    if ladder.non_monotonic:
        lines.append(
            "  WARNING: usability is not monotone in thread size on this run. A smaller rung "
            "failed while a larger one passed, which thread size cannot cause. Something else "
            "moved (thermal throttling, another process, an unstable machine) and the onset rung "
            "should not be quoted from this run alone."
        )
    lines.append("")
    lines.append(
        f"  aggregate score {ladder.aggregate:.1f}/100 (machine-local; does not travel between "
        "machines, unlike the onset rung above)"
    )
    lines.append(f"  weights {ladder.weights_id}   ladder {ladder.rung_ladder_id}")
    return "\n".join(lines)


def render_rung_table(ladder: LadderScore) -> str:
    header = f"{'tokens':>10}  {'score':>6}  {'weight':>6}  {'usable':>6}  status"
    rows = [header, "-" * len(header)]
    for rung, weight in zip(ladder.rungs, ladder.rung_weights):
        status = "complete"
        if not rung.complete:
            status = f"INCOMPLETE: {rung.incomplete_reason}"
        elif rung.zeroed_by:
            status = "zeroed by " + ", ".join(rung.zeroed_by)
        rows.append(
            f"{rung.tokens:>10,}  {rung.score:>6.1f}  {weight:>6.3f}  "
            f"{'yes' if rung.usable else 'no':>6}  {status}"
        )
    rows.append("")
    rows.append(
        "An incomplete rung scores 0 and keeps its weight. It is not dropped: dropping it is how "
        "a build that crashes at 500K outscores one that limps through."
    )
    rows.append(
        "Weights are the trapezoid widths on the log(tokens) axis, so two rungs close together "
        "in log space SHARE the weight of that region rather than each getting a full share. "
        "That is why the top rung is not the whole score, and why 500K and 1M together weigh "
        "about what one decade-spaced rung does."
    )
    return "\n".join(rows)


def render_rung_metrics(rung: RungScore, *, indent: str = "  ") -> str:
    lines = [f"{indent}{rung.tokens:,} tokens -- score {rung.score:.1f}"]
    if not rung.complete:
        lines.append(f"{indent}  INCOMPLETE: {rung.incomplete_reason}")
        lines.append(
            f"{indent}  scores 0 and keeps its weight; no metric was measured at this rung"
        )
        return "\n".join(lines)
    for metric in rung.metric_scores:
        anchor = METRIC_BY_KEY.get(metric.key)
        anchor_text = f"[{anchor.good:g} -> 100, {anchor.bad:g} -> 0]" if anchor is not None else ""
        if metric.scored:
            lines.append(
                f"{indent}  {metric.key:<20} {metric.measure.display():>28}   "
                f"score {float(metric.score):5.1f}  {anchor_text}"
            )
        else:
            lines.append(f"{indent}  {metric.key:<20} {'NOT SCORED':>28}   {metric.reason}")
    return "\n".join(lines)


def render_excluded(payload: Mapping[str, Any]) -> str:
    """Always printed. An empty block is a claim, an absent block is a hole."""

    cells = payload.get("excluded_cells")
    if cells is None:
        raise AssertionError("excluded_cells is mandatory and must not be null")
    lines = ["EXCLUDED CELLS"]
    if not cells:
        lines.append("  none. Every cell that was measured is in the numbers above.")
        return "\n".join(lines)
    totals = excluded_totals(payload)
    for reason, count in sorted(totals.items(), key = lambda kv: (-kv[1], kv[0])):
        lines.append(f"  {count:>4}  {reason}")
    lines.append("")
    for cell in cells:
        detail = f" -- {cell['detail']}" if cell.get("detail") else ""
        lines.append(f"  {cell['cell_id']}: {cell['reason']} x{cell.get('count', 1)}{detail}")
    return "\n".join(lines)


def render_harness_bias(bias: Mapping[str, Any] | None) -> str:
    """The cost of the harness itself, printed at the top, never subtracted."""

    lines = ["HARNESS BIAS (dist-shipping vs dist-armed at control)"]
    if not bias:
        lines.append(
            "  not measured on this run. Every number below therefore includes an unknown "
            "amount of instrument cost."
        )
        return "\n".join(lines)
    for key, value in bias.items():
        rendered = value.display() if isinstance(value, Measure) else str(value)
        lines.append(f"  {key:<24} {rendered}")
    lines.append(
        "  This is NOT subtracted from anything below. Knowing the instrument costs 4% is "
        "information; removing 4% everywhere is a second unvalidated measurement."
    )
    return "\n".join(lines)


def render_ab_table(result: AbResult) -> str:
    """A void result prints its reason and no numbers; warnings above a table do not survive screenshots."""

    title = f"A/B: {result.label}"
    if result.is_null_control:
        title += "   (NULL-TREATMENT CONTROL, base vs base)"
    lines = [title, "=" * len(title)]

    if result.void:
        lines.append("")
        lines.append("VOID. No numbers are quotable from this comparison.")
        lines.append(f"  {result.void_reason}")
        if result.is_null_control:
            lines.append(
                "  The null-treatment control measures whether this machine can currently tell "
                "two identical builds apart. When it cannot, nothing measured alongside it can "
                "be believed, so nothing is printed."
            )
        else:
            lines.append(
                "  A comparison is all of its pairs. A cell that did not complete takes its "
                "healthy partner out of the table with it, so the pairs that remain are a "
                "selection and no verdict is printed over them."
            )
        return "\n".join(lines)

    lines.append(f"noise floor {result.noise_floor_pct:.2f}%  ({result.noise_floor_source})")
    lines.append("")
    header = f"{'metric':<20} {'pairs':>5}  {'ratio':>7}  {'range':>17}  {'ci95':>17}  verdict"
    lines.append(header)
    lines.append("-" * len(header))
    for metric in result.metrics:
        if metric.ratio_geomean is None:
            lines.append(
                f"{metric.metric_key:<20} {metric.n_pairs:>5}  "
                f"{'--':>7}  {'--':>17}  {'--':>17}  no reading"
            )
            continue
        rng = f"{metric.ratio_min:.3f}-{metric.ratio_max:.3f}"
        ci = (
            f"{metric.ci_low:.3f}-{metric.ci_high:.3f}"
            if metric.ci_low is not None
            else "too few pairs"
        )
        # An arm under its instrument floor gives only a bound, so mark the ratio cell itself.
        ratio_cell = (
            f">={metric.ratio_geomean:.3f}"
            if metric.bounded and metric.ratio_geomean >= 1.0
            else f"<={metric.ratio_geomean:.3f}"
            if metric.bounded
            else f"{metric.ratio_geomean:.3f}"
        )
        lines.append(
            f"{metric.metric_key:<20} {metric.n_pairs:>5}  {ratio_cell:>7}  "
            f"{rng:>17}  {ci:>17}  {metric.verdict}"
        )
    lines.append("")
    if result.headline_ratio is not None:
        direction = "faster" if result.headline_ratio < 1.0 else "slower"
        lines.append(
            f"headline ratio {result.headline_ratio:.3f} "
            f"({abs(1.0 - result.headline_ratio) * 100:.1f}% {direction}, weighted)"
        )
    lines.append(f"VERDICT: {result.verdict}")
    unresolved = [m for m in result.metrics if m.withheld]
    if unresolved:
        lines.append("")
        lines.append(
            "These metrics cleared the noise floor without a 95% CI that clears 1.0, either "
            "because the interval contains it or because there were too few pairs to compute "
            "one, so no direction is claimed for them. They are excluded from the headline "
            "ratio above, which would otherwise quote their size as a measured win:"
        )
        for metric in unresolved:
            why = (
                f"ci95 {metric.ci_low:.3f}-{metric.ci_high:.3f} contains 1.0"
                if metric.ci_low is not None and metric.ci_high is not None
                else f"no ci95 from {metric.n_pairs} usable pair(s)"
            )
            lines.append(f"  {metric.metric_key}: ratio {metric.ratio_geomean:.3f}, {why}")
    if result.regressions:
        lines.append("")
        lines.append(
            "FAIL. A per-metric regression beyond the noise floor is a fail regardless of the "
            "headline:"
        )
        for regression in result.regressions:
            lines.append(f"  {regression}")
    lines.append("")
    lines.append(
        "Ratios are paired within one session and interleaved. Ceiling shifts (the onset rung "
        "moving) are reported separately and are not in this table."
    )
    return "\n".join(lines)


def render_ceiling_shift(base: LadderScore, treatment: LadderScore) -> str:
    """Ceiling shift, reported on its own and never folded into the scalar."""

    lines = ["CEILING SHIFT (reported separately, never folded into the score)"]
    base_onset = base.onset_rung_tokens
    treat_onset = treatment.onset_rung_tokens
    lines.append(
        f"  base onset rung      {base_onset:,} tokens"
        if base_onset
        else "  base onset rung      none"
    )
    lines.append(
        f"  treatment onset rung {treat_onset:,} tokens"
        if treat_onset
        else "  treatment onset rung none"
    )
    if base_onset and treat_onset:
        if treat_onset > base_onset:
            lines.append(f"  the ceiling MOVED UP: {base_onset:,} -> {treat_onset:,}")
        elif treat_onset < base_onset:
            lines.append(f"  the ceiling MOVED DOWN: {base_onset:,} -> {treat_onset:,}")
        else:
            lines.append("  the ceiling did not move")
    return "\n".join(lines)


def render_summary(
    payload: Mapping[str, Any],
    ladder: LadderScore | None = None,
    *,
    harness_bias: Mapping[str, Any] | None = None,
    frame_stats_by_rung: Mapping[int, FrameStats] | None = None,
    extra_sections: Sequence[str] = (),
) -> str:
    """Assemble the human-facing summary in the order the policy above demands."""

    sections: list[str] = []
    sections.append("studiobench summary\n===================")

    if not payload.get("complete", False):
        sections.append(
            "RUN DID NOT FINISH. "
            + str(payload.get("incomplete_note", "no footer record was written."))
        )
        crashes = payload.get("crashes") or []
        for crash in crashes:
            sections.append(
                f"  crash: {crash.get('where', 'unknown')} "
                f"{crash.get('error_type', '')} {crash.get('error', '')}".rstrip()
            )

    sections.append(render_harness_bias(harness_bias))

    if ladder is not None:
        sections.append(render_headline(ladder))
        sections.append(render_rung_table(ladder))
        sections.append(
            "PER-RUNG METRICS\n" + "\n".join(render_rung_metrics(rung) for rung in ladder.rungs)
        )

    if frame_stats_by_rung:
        blocks = ["FRAME HEALTH"]
        for tokens in sorted(frame_stats_by_rung):
            blocks.append(f"  {tokens:,} tokens")
            blocks.append(render_frame_health(frame_stats_by_rung[tokens], indent = "    "))
        sections.append("\n".join(blocks))

    sections.extend(extra_sections)
    sections.append(render_excluded(payload))
    return "\n\n".join(sections) + "\n"