Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

83.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""Checks the T4 payloads' pass/fail rules on CPU, so no Kaggle session is needed to test them."""

from __future__ import annotations

import json
import re
import sys
import types
from pathlib import Path

import pytest


def _shared_setup_1():
    pytest.importorskip("safetensors")
    import torch
    from safetensors.torch import save_file

    from run_t4_smoke import saved_adapter_failures, verify_saved_adapter

    return save_file, saved_adapter_failures, torch, verify_saved_adapter


def _shared_setup_2(tmp_path):
    from run_t4_smoke import check_reference, reference_failures

    ref = tmp_path / "ref.json"
    _write_reference(ref, config = REFERENCE_CONFIG)
    return check_reference, ref, reference_failures


def _shared_setup_3(tmp_path):
    from run_t4_smoke import check_reference

    ref = tmp_path / "ref.json"
    _write_reference(ref, config = REFERENCE_CONFIG)
    return check_reference, ref


REPO_ROOT = Path(__file__).resolve().parents[2]
SMOKE_DIR = REPO_ROOT / "tests" / "kaggle" / "t4_smoke"
CI_DIR = REPO_ROOT / ".github" / "scripts" / "kaggle_t4_ci"

sys.path.insert(0, str(SMOKE_DIR))
sys.path.insert(0, str(CI_DIR))

NAN = float("nan")
INF = float("inf")


def test_a_field_logged_by_only_one_run_is_a_difference():
    """A field logged by only one run is a difference, not a skip, matching check_reference's shape rule."""
    from determinism import compare_metrics

    a = [{"step": 1, "loss": 1.0, "grad_norm": 2.0}]
    b = [{"step": 1, "loss": 1.0}]
    result = compare_metrics(a, b)
    assert result["identical"] is False
    assert result["first_diff_step"] == 1


def test_a_field_absent_from_both_runs_is_still_not_a_difference():
    from determinism import compare_metrics

    a = [{"step": 1, "loss": 1.0}]
    b = [{"step": 1, "loss": 1.0}]
    assert compare_metrics(a, b)["identical"] is True


def test_repeated_runs_must_agree_on_their_step_coordinates():
    """Same values in the same positions, against different step numbers."""
    from determinism import compare_metrics

    a = [{"step": 1, "loss": 1.0}, {"step": 2, "loss": 0.5}, {"step": 3, "loss": 0.25}]
    b = [{"step": 1, "loss": 1.0}, {"step": 2, "loss": 0.5}, {"step": 2, "loss": 0.25}]
    result = compare_metrics(a, b)
    assert result["identical"] is False
    assert result["step_mismatch"] == [{"index": 2, "a": 3, "b": 2}]


def test_matching_step_coordinates_are_not_reported_as_a_difference():
    from determinism import compare_metrics

    a = [{"step": 1, "loss": 1.0, "grad_norm": NAN}]
    b = [{"step": 1, "loss": 1.0, "grad_norm": NAN}]
    result = compare_metrics(a, b)
    assert result["identical"] is True
    assert result["step_mismatch"] == []


def test_two_runs_that_both_overflowed_on_the_same_step_agree():
    """fp16 overflow logs infinity and abs(inf-inf) is NaN, which is never 0.0, so identical
    overflowing traces were reported as nondeterministic."""
    from determinism import compare_metrics

    a = [{"step": 1, "loss": 10.0, "grad_norm": INF}, {"step": 2, "loss": 9.0, "grad_norm": 4.0}]
    b = [{"step": 1, "loss": 10.0, "grad_norm": INF}, {"step": 2, "loss": 9.0, "grad_norm": 4.0}]
    result = compare_metrics(a, b)
    assert result["identical"] is True
    assert result["first_diff_step"] is None
    assert result["max_abs_diff"] == {"loss": 0.0, "grad_norm": 0.0}


@pytest.mark.parametrize(
    ("norm_a", "norm_b"),
    [(INF, 4.0), (4.0, INF), (INF, -INF), (-INF, INF)],
)
def test_an_infinity_only_one_run_logged_is_still_a_difference(norm_a, norm_b):
    from determinism import compare_metrics

    a = [{"step": 1, "loss": 1.0, "grad_norm": norm_a}]
    b = [{"step": 1, "loss": 1.0, "grad_norm": norm_b}]
    assert compare_metrics(a, b)["identical"] is False


def test_the_canary_must_be_the_whole_answer():
    from run_t4_smoke import CANARY, canary_failures

    run = {"run_index": 0, "generated": CANARY + "<|im_start|>and on it went"}
    failures = canary_failures(run, require = True)
    assert failures and "exactly" in failures[0]


def test_the_canary_tolerates_surrounding_whitespace_only():
    from run_t4_smoke import CANARY, canary_failures
    assert canary_failures({"run_index": 0, "generated": CANARY}, require = True) == []
    assert canary_failures({"run_index": 0, "generated": "\n" + CANARY + " \n"}, require = True) == []


def test_a_missing_canary_is_still_a_failure():
    from run_t4_smoke import canary_failures
    failures = canary_failures({"run_index": 1, "generated": "def my_function():"}, require = True)
    assert failures and "did not emit" in failures[0]


def test_the_canary_can_be_downgraded_to_a_warning():
    from run_t4_smoke import canary_failures
    assert canary_failures({"run_index": 1, "generated": "nope"}, require = False) == []


def test_an_infinite_gradient_norm_is_not_an_applied_update():
    """Infinite grad_norm is a skipped step, not an applied update; a NaN-only check counted it applied."""
    from run_t4_smoke import optimisation_failures

    metrics = [{"step": s, "loss": 10.0 - s, "grad_norm": INF} for s in (1, 2, 3)]
    failures = optimisation_failures(metrics)
    assert failures and "no optimizer update was applied" in failures[0]


def test_one_finite_gradient_norm_is_enough():
    from run_t4_smoke import optimisation_failures
    metrics = [
        {"step": 1, "loss": 10.0, "grad_norm": NAN},
        {"step": 2, "loss": 5.0, "grad_norm": INF},
        {"step": 3, "loss": 1.0, "grad_norm": 11.2},
    ]
    assert optimisation_failures(metrics) == []


def _adapter_state(**over) -> dict:
    state = {
        "dir": "/kaggle/working/smoke0/lora_run0",
        "files": ["adapter_config.json", "adapter_model.safetensors"],
        "weight_file": "adapter_model.safetensors",
        "config_readable": True,
        "tensors": 224,
        "parameters": 4325376,
        "non_finite_tensors": [],
        "nonzero_tensors": 224,
        "b_tensors": 112,
        "nonzero_b_tensors": 112,
        "keys_checked": True,
        "keys_missing": [],
        "keys_unexpected": [],
        "keys_extra": [],
    }
    state.update(over)
    return state


def test_a_saved_adapter_that_reloads_with_weights_passes():
    from run_t4_smoke import saved_adapter_failures
    assert saved_adapter_failures(_adapter_state()) == []


def test_an_adapter_whose_weight_file_cannot_be_read_is_a_failure():
    from run_t4_smoke import saved_adapter_failures
    failures = saved_adapter_failures(
        _adapter_state(tensors = None, error = "SafetensorError: header too small")
    )
    assert failures and "could not be read back" in failures[0]


def test_an_adapter_with_no_tensors_is_a_failure():
    from run_t4_smoke import saved_adapter_failures
    failures = saved_adapter_failures(_adapter_state(tensors = 0, parameters = 0))
    assert failures and "no tensors" in failures[0]


def test_an_adapter_with_non_finite_weights_is_a_failure():
    from run_t4_smoke import saved_adapter_failures
    failures = saved_adapter_failures(_adapter_state(non_finite_tensors = ["...lora_B.weight"]))
    assert failures and "non-finite" in failures[0]


def test_an_all_zero_adapter_is_a_failure():
    """lora_B starts at zero, so an all-zero file is an untrained adapter."""
    from run_t4_smoke import saved_adapter_failures

    failures = saved_adapter_failures(_adapter_state(nonzero_tensors = 0, nonzero_b_tensors = 0))
    assert failures and "saved lora_B matrices is zero" in failures[0]


def test_an_adapter_whose_b_matrices_are_all_zero_is_a_failure():
    """Count nonzero lora_B matrices, not all tensors: lora_A is random and nonzero before any step."""
    from run_t4_smoke import saved_adapter_failures

    failures = saved_adapter_failures(
        _adapter_state(nonzero_tensors = 112, b_tensors = 112, nonzero_b_tensors = 0)
    )
    assert failures and "saved lora_B matrices is zero" in failures[0]


def test_an_adapter_with_no_b_matrices_at_all_is_unusable_rather_than_fine():
    """Dropped B matrices, or a peft naming change: either way, no reading."""
    from run_t4_smoke import saved_adapter_failures

    failures = saved_adapter_failures(
        _adapter_state(nonzero_tensors = 112, b_tensors = 0, nonzero_b_tensors = 0)
    )
    assert failures and "is a lora_B matrix" in failures[0]


def test_an_adapter_nobody_checked_the_names_of_is_not_a_pass():
    """No derivable expected key names means the save is unchecked, which must fail rather than pass."""
    from run_t4_smoke import saved_adapter_failures

    failures = saved_adapter_failures(
        _adapter_state(keys_checked = False, keys_error = "ImportError: no such name")
    )
    assert failures and "never checked" in failures[0]


def test_an_adapter_missing_tensors_peft_names_is_a_failure():
    from run_t4_smoke import saved_adapter_failures
    failures = saved_adapter_failures(
        _adapter_state(keys_missing = ["base_model.model.q_proj.lora_B.weight"])
    )
    assert failures and "does not carry" in failures[0]


def test_lora_tensors_under_names_peft_does_not_use_are_a_failure():
    from run_t4_smoke import saved_adapter_failures
    failures = saved_adapter_failures(_adapter_state(keys_unexpected = ["q_proj.lora_B.weight"]))
    assert failures and "ignores them silently" in failures[0]


def test_a_non_lora_tensor_beside_the_adapter_is_not_a_failure(tmp_path):
    """Non-LoRA tensors like an embedding copy are not failures; only names PEFT must match are checked."""
    save_file, saved_adapter_failures, torch, verify_saved_adapter = _shared_setup_1()

    lora = "base_model.model.layers.0.self_attn.q_proj.lora_B.weight"
    embedding = "base_model.model.model.embed_tokens.weight"
    (tmp_path / "adapter_config.json").write_text(json.dumps({"peft_type": "LORA", "r": 16}))
    save_file(
        {lora: torch.ones(8, 16) * 0.01, embedding: torch.ones(4, 4)},
        str(tmp_path / "adapter_model.safetensors"),
    )
    state = verify_saved_adapter(tmp_path, peft_keys = {"keys": [lora]})
    assert state["keys_extra"] == [embedding]
    assert state["keys_unexpected"] == []
    assert saved_adapter_failures(state) == []


def test_the_key_oracle_reports_a_failure_rather_than_raising():
    """It runs inside the payload, so a raise here would lose the whole leg."""
    from run_t4_smoke import peft_adapter_keys

    out = peft_adapter_keys(object())
    assert "keys" not in out and out["error"]


def test_an_adapter_peft_would_ignore_on_reload_does_not_pass(tmp_path):
    """An adapter whose tensor names PEFT ignores on reload must fail, though its weights look trained."""
    pytest.importorskip("peft")
    pytest.importorskip("transformers")
    pytest.importorskip("safetensors")
    import torch
    from peft import LoraConfig, PeftModel, get_peft_model
    from safetensors.torch import load_file, save_file
    from transformers import GPT2Config, GPT2LMHeadModel

    from run_t4_smoke import peft_adapter_keys, saved_adapter_failures, verify_saved_adapter

    def _base():
        torch.manual_seed(0)
        return GPT2LMHeadModel(GPT2Config(n_layer = 1, n_head = 2, n_embd = 32, vocab_size = 64))

    model = get_peft_model(
        _base(), LoraConfig(r = 4, target_modules = ["c_attn"], task_type = "CAUSAL_LM")
    )
    # lora_B starts at zero and only an applied optimizer step moves it.
    with torch.no_grad():
        for name, param in model.named_parameters():
            if "lora_B" in name:
                param.add_(0.25)
    model.save_pretrained(str(tmp_path))
    keys = peft_adapter_keys(model)
    assert keys["keys"], keys

    good = verify_saved_adapter(tmp_path, peft_keys = keys)
    assert good["keys_checked"] is True
    assert saved_adapter_failures(good) == []

    weights = tmp_path / "adapter_model.safetensors"
    tensors = load_file(str(weights))
    save_file(
        {name.removeprefix("base_model.model."): value for name, value in tensors.items()},
        str(weights),
    )
    broken = verify_saved_adapter(tmp_path, peft_keys = keys)

    assert broken["tensors"] == good["tensors"]
    assert broken["nonzero_b_tensors"] == good["nonzero_b_tensors"] > 0
    assert broken["non_finite_tensors"] == []
    assert broken["config_loadable"] is True

    failures = saved_adapter_failures(broken)
    assert failures, "the leg would have passed an adapter that reloads to the base model"
    assert "ignores them silently" in " ".join(failures)

    # Explicit: PEFT's infer_device() believes tests/_zoo_aggressive_cuda_spoof.py.
    reloaded = PeftModel.from_pretrained(_base(), str(tmp_path), torch_device = "cpu")
    b_matrices = [p for n, p in reloaded.named_parameters() if "lora_B" in n]
    assert b_matrices and all(float(p.abs().sum()) == 0.0 for p in b_matrices), (
        "peft loaded the renamed keys after all, so this test no longer describes "
        "the regression it was written for"
    )


def test_a_missing_adapter_config_is_a_failure():
    from run_t4_smoke import saved_adapter_failures
    failures = saved_adapter_failures(_adapter_state(config_readable = False))
    assert failures and "adapter_config.json" in failures[0]


def test_the_adapter_check_reads_a_real_file_it_just_wrote(tmp_path):
    """The check must reject a real file the old count accepted: random lora_A, lora_B never trained."""
    save_file, saved_adapter_failures, torch, verify_saved_adapter = _shared_setup_1()

    (tmp_path / "adapter_config.json").write_text(json.dumps({"peft_type": "LORA", "r": 16}))
    written = {
        "base_model.model.layers.0.self_attn.q_proj.lora_A.weight": torch.ones(16, 8),
        "base_model.model.layers.0.self_attn.q_proj.lora_B.weight": torch.zeros(8, 16),
    }
    peft_keys = {"keys": sorted(written)}
    save_file(written, str(tmp_path / "adapter_model.safetensors"))
    state = verify_saved_adapter(tmp_path, peft_keys = peft_keys)
    assert state["tensors"] == 2
    assert state["nonzero_tensors"] == 1
    assert state["b_tensors"] == 1
    assert state["nonzero_b_tensors"] == 0
    assert state["config_readable"] is True
    failures = saved_adapter_failures(state)
    assert failures and "saved lora_B matrices is zero" in failures[0]

    save_file(
        {
            "base_model.model.layers.0.self_attn.q_proj.lora_A.weight": torch.ones(16, 8),
            "base_model.model.layers.0.self_attn.q_proj.lora_B.weight": torch.ones(8, 16) * 0.01,
        },
        str(tmp_path / "adapter_model.safetensors"),
    )
    trained = verify_saved_adapter(tmp_path, peft_keys = peft_keys)
    assert trained["nonzero_b_tensors"] == 1
    assert saved_adapter_failures(trained) == []

    save_file({"a.lora_B.weight": torch.zeros(4, 4)}, str(tmp_path / "adapter_model.safetensors"))
    failures = saved_adapter_failures(
        verify_saved_adapter(tmp_path, peft_keys = {"keys": ["a.lora_B.weight"]})
    )
    assert failures and "saved lora_B matrices is zero" in failures[0]


def test_a_syntactically_valid_but_empty_adapter_config_is_not_a_pass(tmp_path):
    """An empty adapter_config.json is valid JSON but PEFT cannot load it; the check must ask PEFT."""
    pytest.importorskip("peft")
    save_file, saved_adapter_failures, torch, verify_saved_adapter = _shared_setup_1()

    save_file(
        {"q_proj.lora_B.weight": torch.ones(8, 16) * 0.01},
        str(tmp_path / "adapter_model.safetensors"),
    )
    for body in ("{}", "[]"):
        (tmp_path / "adapter_config.json").write_text(body)
        state = verify_saved_adapter(tmp_path, peft_keys = {"keys": ["q_proj.lora_B.weight"]})
        assert state["config_readable"] is True, "it IS readable JSON; that was never the question"
        assert state["config_loadable"] is False, body
        failures = saved_adapter_failures(state)
        assert failures and "PEFT cannot rebuild an adapter from it" in failures[0], body


def test_an_adapter_config_for_a_different_adapter_than_the_one_trained_fails(tmp_path):
    """A loadable config still fails if it does not match the get_peft_model arguments the run used."""
    pytest.importorskip("peft")
    save_file, saved_adapter_failures, torch, verify_saved_adapter = _shared_setup_1()

    save_file(
        {"q_proj.lora_B.weight": torch.ones(8, 16) * 0.01},
        str(tmp_path / "adapter_model.safetensors"),
    )
    requested = {"r": 16, "lora_alpha": 16, "target_modules": ["q_proj", "v_proj"]}
    (tmp_path / "adapter_config.json").write_text(
        json.dumps(
            {
                "peft_type": "LORA",
                "r": 8,
                "lora_alpha": 16,
                "target_modules": ["q_proj", "v_proj"],
            }
        )
    )
    keys = {"keys": ["q_proj.lora_B.weight"]}
    state = verify_saved_adapter(tmp_path, expected = requested, peft_keys = keys)
    assert state["config_loadable"] is True
    assert state["config_differences"], state
    failures = saved_adapter_failures(state)
    assert failures and "different adapter than the one that was trained" in failures[0]

    (tmp_path / "adapter_config.json").write_text(json.dumps({"peft_type": "LORA", **requested}))
    good = verify_saved_adapter(tmp_path, expected = requested, peft_keys = keys)
    assert good["config_differences"] == []
    assert saved_adapter_failures(good) == []


def _write_reference(
    path: Path,
    *,
    config: dict,
    model: str = "unsloth/Qwen2.5-0.5B-Instruct",
):
    path.write_text(
        json.dumps(
            {
                "config": config,
                "model": model,
                "environment": {},
                "metrics": [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)],
            }
        ),
        encoding = "utf-8",
    )


def test_a_band_check_against_a_reference_from_another_card_is_refused(tmp_path):
    """Refuse a band check against a reference from another GPU: a loss trace belongs to its hardware."""
    from run_t4_smoke import check_reference, reference_failures

    ref = tmp_path / "ref.json"
    ref.write_text(
        json.dumps(
            {
                "config": REFERENCE_CONFIG,
                "model": "unsloth/Qwen2.5-0.5B-Instruct",
                "environment": {"gpu_name": "Tesla T4", "gpu_capability": "sm_75"},
                "metrics": [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)],
            }
        ),
        encoding = "utf-8",
    )
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        config = REFERENCE_CONFIG,
        model = "unsloth/Qwen2.5-0.5B-Instruct",
        environment = {"gpu_name": "Tesla P100-PCIE-16GB", "gpu_capability": "sm_60"},
    )
    assert verdict["status"] == "hardware_mismatch", verdict
    # Metrics match the reference, so only the card check can refuse.
    assert verdict["deviations"] == []
    failures = reference_failures(verdict, 0.10)
    assert failures and "not for this run" in failures[0]
    assert "Tesla P100-PCIE-16GB" in failures[0]

    same = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        config = REFERENCE_CONFIG,
        model = "unsloth/Qwen2.5-0.5B-Instruct",
        environment = {"gpu_name": "Tesla T4", "gpu_capability": "sm_75"},
    )
    assert same["status"] == "ok", same
    assert reference_failures(same, 0.10) == []


def test_a_reference_that_records_no_hardware_is_unchecked_not_a_mismatch(tmp_path):
    """A reference that does not name its GPU is unchecked, not a mismatch, and the skip is recorded."""
    check_reference, ref, reference_failures = _shared_setup_2(tmp_path)
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        config = REFERENCE_CONFIG,
        environment = {"gpu_name": "Tesla T4", "gpu_capability": "sm_75"},
    )
    assert verdict["status"] == "ok", verdict
    assert "gpu_name" in verdict["config_unchecked"]
    assert reference_failures(verdict, 0.10) == []


@pytest.mark.parametrize(
    "environment",
    [
        pytest.param({"error": "RuntimeError: no CUDA driver"}, id = "probe_raised"),
        pytest.param({"python": "3.12.13", "torch": "2.10.0"}, id = "probe_saw_no_gpu"),
        pytest.param({}, id = "empty"),
        pytest.param(None, id = "not_supplied"),
        pytest.param({"gpu_name": "Tesla T4", "gpu_capability": None}, id = "one_key_missing"),
    ],
)
def test_a_run_that_cannot_name_its_card_is_refused_not_waved_through(tmp_path, environment):
    """A probe failure must not switch off the GPU check: a run that cannot name its card is refused."""
    from run_t4_smoke import check_reference, reference_failures

    ref = tmp_path / "ref.json"
    ref.write_text(
        json.dumps(
            {
                "config": REFERENCE_CONFIG,
                "model": "unsloth/Qwen2.5-0.5B-Instruct",
                "environment": {"gpu_name": "Tesla T4", "gpu_capability": "sm_75"},
                "metrics": [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)],
            }
        ),
        encoding = "utf-8",
    )
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        config = REFERENCE_CONFIG,
        model = "unsloth/Qwen2.5-0.5B-Instruct",
        environment = environment,
    )
    assert verdict["status"] == "hardware_unverified", verdict
    assert verdict["deviations"] == [], "refused before any number was compared"
    assert "gpu_name" not in verdict["config_unchecked"], verdict
    failures = reference_failures(verdict, 0.10)
    assert failures and "not for this run" in failures[0], failures


def test_the_committed_reference_names_the_card_the_gate_reads(tmp_path):
    """A recaptured reference without gpu_name would silently turn the card check into a skip."""
    reference = SMOKE_DIR / "references" / "t4_qwen2.5-0.5b.json"
    environment = json.loads(reference.read_text(encoding = "utf-8"))["environment"]
    assert environment["gpu_name"] == "Tesla T4"
    assert environment["gpu_capability"] == "sm_75"


REFERENCE_CONFIG = {
    "max_steps": 3,
    "dataset_digest": "d" * 64,
    "init_loss_scale": 0.0,
    "batch_size": 2,
    "grad_accum": 1,
    "max_seq_length": 512,
    "learning_rate": 0.001,
    "lora_r": 16,
    "lora_alpha": 32,
    "optim": "adamw_8bit",
    "gradient_checkpointing": "unsloth",
    "repeat": 2,
}


def test_a_reference_captured_with_another_learning_rate_is_refused(tmp_path):
    check_reference, ref, reference_failures = _shared_setup_2(tmp_path)
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        config = dict(REFERENCE_CONFIG, learning_rate = 0.005),
    )
    assert verdict["status"] == "config_mismatch"
    assert verdict["config_differences"] == [
        {"key": "learning_rate", "reference": 0.001, "observed": 0.005}
    ]
    assert reference_failures(verdict, 0.10)


def test_a_reference_captured_for_another_model_is_refused(tmp_path):
    check_reference, ref, reference_failures = _shared_setup_2(tmp_path)
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        config = REFERENCE_CONFIG,
        model = "unsloth/Llama-3.2-1B-Instruct",
    )
    assert verdict["status"] == "config_mismatch"
    assert reference_failures(verdict, 0.10)


def test_the_repeat_count_does_not_invalidate_a_reference(tmp_path):
    """Each cycle is a fresh process, so how many were run is not the run."""
    check_reference, ref = _shared_setup_3(tmp_path)
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(
        observed, ref, 0.10, 0.05, max_steps = 3, config = dict(REFERENCE_CONFIG, repeat = 5)
    )
    assert verdict["status"] == "ok"


def test_a_reference_that_predates_a_setting_does_not_refuse_on_it(tmp_path):
    """An older file simply does not carry the key; that is not a mismatch."""
    from run_t4_smoke import check_reference

    ref = tmp_path / "ref.json"
    older = {k: v for k, v in REFERENCE_CONFIG.items() if k != "gradient_checkpointing"}
    _write_reference(ref, config = older)
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(observed, ref, 0.10, 0.05, max_steps = 3, config = REFERENCE_CONFIG)
    assert verdict["status"] == "ok"
    # A pin present on one side only is recorded as unchecked.
    assert verdict["config_unchecked"] == ["gradient_checkpointing", "model"]


def test_the_reference_check_still_works_without_an_observed_config(tmp_path):
    """Backwards compatible: the step-count guard is what it always was."""
    check_reference, ref = _shared_setup_3(tmp_path)
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    assert check_reference(observed, ref, 0.10, 0.05, max_steps = 3)["status"] == "ok"
    assert (
        check_reference(observed, ref, 0.10, 0.05, max_steps = 10)["status"] == "step_count_mismatch"
    )


def test_a_shifted_step_coordinate_is_refused_by_the_band_check(tmp_path):
    check_reference, ref, reference_failures = _shared_setup_2(tmp_path)
    observed = [{"step": 1, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(observed, ref, 0.10, 0.05, max_steps = 3)
    assert verdict["status"] == "step_mismatch"
    assert verdict["step_differences"] == [
        {"index": 1, "reference": 2, "observed": 1},
        {"index": 2, "reference": 3, "observed": 1},
    ]
    assert reference_failures(verdict, 0.10)


def test_the_model_revision_travels_with_the_reference(tmp_path):
    check_reference, ref, reference_failures = _shared_setup_2(tmp_path)
    payload = json.loads(ref.read_text())
    payload["resolved_checkpoint"] = "unsloth/Qwen2.5-0.5B-Instruct-unsloth-bnb-4bit"
    payload["resolved_revision"] = "10413c288cb9629acdf60b3e0229f3ba75efe413"
    ref.write_text(json.dumps(payload))
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]

    same = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        resolved_checkpoint = "unsloth/Qwen2.5-0.5B-Instruct-unsloth-bnb-4bit",
        resolved_revision = "10413c288cb9629acdf60b3e0229f3ba75efe413",
    )
    assert same["status"] == "ok"

    moved = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        resolved_checkpoint = "unsloth/Qwen2.5-0.5B-Instruct-unsloth-bnb-4bit",
        resolved_revision = "0000000000000000000000000000000000000000",
    )
    assert moved["status"] == "config_mismatch"
    assert reference_failures(moved, 0.10)


def test_a_repo_id_that_differs_only_in_case_is_the_same_reference(tmp_path):
    """The Hub treats repo ids case-insensitively, and #8058 changed the loader from reporting
    the lower-case spelling to the canonical one. The committed reference still carries the
    old spelling; that is the same checkpoint, and the revision pin is what moves with weights."""
    check_reference, ref, reference_failures = _shared_setup_2(tmp_path)
    payload = json.loads(ref.read_text())
    payload["model"] = "unsloth/qwen2.5-0.5b-instruct"
    payload["resolved_checkpoint"] = "unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit"
    payload["resolved_revision"] = "10413c288cb9629acdf60b3e0229f3ba75efe413"
    ref.write_text(json.dumps(payload))
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]

    recased = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        model = "unsloth/Qwen2.5-0.5B-Instruct",
        resolved_checkpoint = "unsloth/Qwen2.5-0.5B-Instruct-unsloth-bnb-4bit",
        resolved_revision = "10413c288cb9629acdf60b3e0229f3ba75efe413",
    )
    assert recased["status"] == "ok", recased
    assert not reference_failures(recased, 0.10)

    # Only case is forgiven; a different revision is still a different experiment.
    other = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        resolved_checkpoint = "unsloth/Qwen2.5-1.5B-Instruct-unsloth-bnb-4bit",
        resolved_revision = "10413c288cb9629acdf60b3e0229f3ba75efe413",
    )
    assert other["status"] == "config_mismatch"
    revision = check_reference(
        observed,
        ref,
        0.10,
        0.05,
        max_steps = 3,
        resolved_checkpoint = "unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit",
        resolved_revision = "10413C288CB9629ACDF60B3E0229F3BA75EFE413",
    )
    assert revision["status"] == "config_mismatch"


def test_a_local_checkpoint_path_keeps_its_case(tmp_path):
    """Only Hub repo ids are case-insensitive. A local directory is a path, and on a
    case-sensitive filesystem another capitalisation can hold other weights."""
    check_reference, ref, reference_failures = _shared_setup_2(tmp_path)
    upper = tmp_path / "models" / "Foo"
    lower = tmp_path / "models" / "foo"
    upper.mkdir(parents = True)
    if lower.exists():
        pytest.skip("case-insensitive filesystem: the two paths are one directory here")
    lower.mkdir()
    payload = json.loads(ref.read_text())
    payload["resolved_checkpoint"] = str(upper)
    ref.write_text(json.dumps(payload))
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]

    verdict = check_reference(
        observed, ref, 0.10, 0.05, max_steps = 3, resolved_checkpoint = str(lower)
    )
    assert verdict["status"] == "config_mismatch", verdict
    assert reference_failures(verdict, 0.10)

    # A relative "owner/name" that exists locally is a directory, not a repo id.
    rel_upper = tmp_path / "Owner" / "Model"
    rel_lower = tmp_path / "owner" / "model"
    rel_upper.mkdir(parents = True)
    rel_lower.mkdir(parents = True)
    payload["resolved_checkpoint"] = "Owner/Model"
    ref.write_text(json.dumps(payload))
    import os

    cwd = os.getcwd()
    os.chdir(tmp_path)
    try:
        relative = check_reference(
            observed, ref, 0.10, 0.05, max_steps = 3, resolved_checkpoint = "owner/model"
        )
    finally:
        os.chdir(cwd)
    assert relative["status"] == "config_mismatch", relative


def test_a_reference_with_no_recorded_revision_does_not_refuse(tmp_path):
    """The committed file predates this; unknown is unknown, not a mismatch."""
    check_reference, ref = _shared_setup_3(tmp_path)
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(observed, ref, 0.10, 0.05, max_steps = 3, resolved_revision = "abc123")
    assert verdict["status"] == "ok"
    assert "resolved_revision" in verdict["config_unchecked"]


def test_a_pin_the_run_could_not_read_is_recorded_as_unchecked(tmp_path):
    """When the reference names a commit but the run cannot read one, the pin must be recorded as
    unchecked, not silently skipped."""
    check_reference, ref = _shared_setup_3(tmp_path)
    payload = json.loads(ref.read_text())
    payload["resolved_revision"] = "10413c288cb9629acdf60b3e0229f3ba75efe413"
    ref.write_text(json.dumps(payload))
    observed = [{"step": s, "loss": 1.0 / s, "grad_norm": 3.0} for s in (1, 2, 3)]
    verdict = check_reference(observed, ref, 0.10, 0.05, max_steps = 3)
    assert verdict["status"] == "ok"
    assert "resolved_revision" in verdict["config_unchecked"]
    assert "resolved_checkpoint" not in verdict["config_unchecked"]


def test_what_the_band_did_not_compare_reaches_the_summary():
    """A skip nobody can see reads as a comparison that passed."""
    import report as report_module

    lines = report_module.render(
        {
            "label": "control",
            "model": "unsloth/Qwen2.5-0.5B-Instruct",
            "passed": True,
            "metrics": [],
            "reference_check": {
                "status": "ok",
                "worst_rel": {"loss": 0.01},
                "reference_max_steps": 10,
                "config_unchecked": ["resolved_revision"],
            },
        }
    )
    assert any("resolved_revision" in line for line in lines), lines


def test_the_committed_reference_pins_the_model_it_was_captured_on():
    """The committed reference must pin DEFAULT_MODEL, since the control leg passes no --model."""
    from run_t4_smoke import DEFAULT_MODEL

    reference = json.loads(
        (SMOKE_DIR / "references" / "t4_qwen2.5-0.5b.json").read_text(encoding = "utf-8")
    )
    assert reference["model"] == DEFAULT_MODEL


def _batch_record(**over):
    """A healthy batched-generation record, with fields overridable per test."""
    base = {
        "prompt_token_lengths": [11, 14, 9, 17, 12, 20, 8, 15],
        "distinct_lengths": 8,
        "padding_side_observed": "left",
        "padding_side_after": "left",
        "singles": [f"out{i}" for i in range(8)],
        "batched": {
            "2": [f"out{i}" for i in range(8)],
            "4": [f"out{i}" for i in range(8)],
            "8": [f"out{i}" for i in range(8)],
        },
        "agrees": {"2": True, "4": True, "8": True},
        "empty_outputs": [],
    }
    base.update(over)
    return base


def _cycle(index: int, losses: list[float]) -> dict:
    return {
        "run_index": index,
        "metrics": [
            {"step": s, "loss": losses[s - 1], "grad_norm": 3.0} for s in range(1, len(losses) + 1)
        ],
        "generated": "__UNSLOTH__!!!",
        "canary_found": True,
        # A healthy simulated cycle needs this record: batched generation is checked by default.
        "batched_generation": _batch_record(),
        "adapter_files": ["adapter_config.json", "adapter_model.safetensors"],
        "saved_adapter": _adapter_state(),
        "determinism": {},
        "loss_scale": {},
        "timing_seconds": {},
        "peak_reserved_gb": 0.1,
    }


def _drive_main(monkeypatch, tmp_path, cycles: dict, argv: list[str]) -> tuple[int, dict]:
    import run_t4_smoke

    real_run = run_t4_smoke.subprocess.run

    def fake_run(cmd, *a, **kw):
        if "--cycle" not in cmd:
            return real_run(cmd, *a, **kw)
        index = int(cmd[cmd.index("--cycle") + 1])
        outdir = Path(cmd[cmd.index("--outdir") + 1])
        outdir.mkdir(parents = True, exist_ok = True)
        payload = cycles.get(index)
        if payload is None:
            return types.SimpleNamespace(returncode = 1)
        (outdir / "cycle_report.json").write_text(json.dumps(payload), encoding = "utf-8")
        return types.SimpleNamespace(returncode = 0)

    monkeypatch.setattr(run_t4_smoke.subprocess, "run", fake_run)
    monkeypatch.setattr(sys, "argv", ["run_t4_smoke.py", "--outdir", str(tmp_path)] + argv)
    code = run_t4_smoke.main()
    return code, json.loads((tmp_path / "t4_smoke_report.json").read_text(encoding = "utf-8"))


def test_every_requested_repeat_is_compared_against_the_baseline(monkeypatch, tmp_path):
    """--repeat 3 compared cycles 0 and 1 and threw cycle 2 away."""
    cycles = {
        0: _cycle(0, [3.0, 2.0, 1.0]),
        1: _cycle(1, [3.0, 2.0, 1.0]),
        2: _cycle(2, [3.0, 2.0, 99.0]),
    }
    code, report = _drive_main(monkeypatch, tmp_path, cycles, ["--repeat", "3", "--max-steps", "3"])
    assert code == 1
    assert report["passed"] is False
    assert any("cycle 2" in f for f in report["failures"])
    repro = report["reproducibility"]
    assert repro["compared_cycles"] == ["1", "2"]
    assert repro["cycles"]["1"]["identical"] is True
    assert repro["cycles"]["2"]["identical"] is False
    assert repro["identical"] is False
    assert repro["first_diff_step"] == 3
    assert repro["max_abs_diff"]["loss"] == 98.0


def test_two_agreeing_repeats_still_pass(monkeypatch, tmp_path):
    cycles = {0: _cycle(0, [3.0, 2.0, 1.0]), 1: _cycle(1, [3.0, 2.0, 1.0])}
    code, report = _drive_main(monkeypatch, tmp_path, cycles, ["--repeat", "2", "--max-steps", "3"])
    assert code == 0
    assert report["passed"] is True
    assert report["reproducibility"]["identical"] is True
    assert report["reproducibility"]["cycles"]["1"]["identical"] is True


def test_the_summary_renderer_still_reads_a_multi_repeat_report(monkeypatch, tmp_path):
    """report.py renders `identical` / `first_diff_step` / `max_abs_diff`
    straight off `reproducibility`; a shape change there is a silent
    "**DIFFERED**" on every healthy run."""
    from report import render

    cycles = {0: _cycle(0, [3.0, 2.0, 1.0]), 1: _cycle(1, [3.0, 2.0, 1.0])}
    _, report_dict = _drive_main(
        monkeypatch, tmp_path, cycles, ["--repeat", "2", "--max-steps", "3"]
    )
    rendered = "\n".join(render(report_dict))
    assert "agreed **bitwise**" in rendered
    assert "DIFFERED" not in rendered


def test_a_failed_cycle_still_reports_its_environment(monkeypatch, tmp_path):
    """The one case where knowing which library set died matters most."""
    code, report = _drive_main(monkeypatch, tmp_path, {}, ["--repeat", "2", "--max-steps", "3"])
    assert code == 1
    assert report["passed"] is False
    assert report["environment"]["resolved"]
    assert report["config"]["max_steps"] == 3
    assert report["label"] == "t4-smoke"


def _gptoss_args(**over):
    base = {"max_steps": 3, "require_compile": True}
    base.update(over)
    return types.SimpleNamespace(**base)


def _gptoss_result(**over) -> dict:
    result = {
        "metrics": [{"step": s, "loss": 3.0 - s, "grad_norm": 1.5} for s in (1, 2, 3)],
        "compile": {"available": True, "unique_graphs": 32, "unique_graphs_delta": 30},
        "generated": "hello there",
        "precision": {
            "fp16": False,
            "bf16": False,
            "force_float32_env": "1",
            "custom_dtype_env": "down_projs;mlp.router",
        },
        "environment": {"bf16_supported": False, "gpu_name": "Tesla T4"},
        "placement_after_load": {
            "parameters_by_device": {"cuda:0": 20_900_000_000},
            "hf_device_map_devices": None,
            "offloaded": False,
        },
    }
    result.update(over)
    return result


def test_a_healthy_gptoss_run_passes():
    from run_gptoss_t4 import failures_for
    assert failures_for(_gptoss_result(), _gptoss_args()) == []


def test_gptoss_fails_when_the_forced_float32_path_stopped_firing():
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        precision = {
            "fp16": True,
            "bf16": False,
            "force_float32_env": None,
            "custom_dtype_env": None,
        }
    )
    failures = failures_for(result, _gptoss_args())
    assert failures and any("float32" in f for f in failures)


def test_gptoss_does_not_demand_float32_on_a_card_with_bf16():
    """The force path exists for cards without bf16; elsewhere it is right
    that it does not fire, and a red there would be the harness's own bug."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        environment = {"bf16_supported": True, "gpu_name": "NVIDIA A100"},
        precision = {
            "fp16": False,
            "bf16": True,
            "force_float32_env": None,
            "custom_dtype_env": None,
        },
    )
    assert failures_for(result, _gptoss_args()) == []


def test_gptoss_fails_when_precision_was_never_recorded():
    from run_gptoss_t4 import failures_for

    result = _gptoss_result()
    result.pop("precision")
    failures = failures_for(result, _gptoss_args())
    assert failures and any("precision" in f for f in failures)


def test_gptoss_measures_compilation_across_training_only():
    """Loading compiled 32 graphs and training compiled none."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        compile = {"available": True, "unique_graphs": 32, "unique_graphs_delta": 0}
    )
    failures = failures_for(result, _gptoss_args())
    assert failures and any("eager" in f for f in failures)


@pytest.mark.parametrize("value", ["0", "", None, "true"])
def test_gptoss_requires_the_forcing_to_be_on_rather_than_merely_recorded(value):
    """UNSLOTH_FORCE_FLOAT32 must be exactly 1; its ordinary-branch value 0 is a truthy string."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        precision = {
            "fp16": False,
            "bf16": False,
            "force_float32_env": value,
            "custom_dtype_env": None,
        }
    )
    failures = failures_for(result, _gptoss_args())
    assert failures and any("UNSLOTH_FORCE_FLOAT32" in f for f in failures), failures


def test_gptoss_refuses_a_compile_check_that_has_no_baseline():
    """A post-training read with no pre-training one to subtract."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(compile = {"available": True, "unique_graphs": 32})
    failures = failures_for(result, _gptoss_args())
    assert failures and any("pre-training dynamo counters" in f for f in failures)


def test_gptoss_compile_counters_report_a_delta():
    from run_gptoss_t4 import compile_counters

    before = compile_counters()
    after = compile_counters(before = before)
    assert after["unique_graphs_delta"] == 0
    fake = compile_counters(before = {"available": True, "unique_graphs": 5, "calls_captured": 9})
    assert fake["unique_graphs_delta"] == fake["unique_graphs"] - 5


def test_gptoss_fails_when_no_optimizer_update_was_applied():
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        metrics = [{"step": s, "loss": 3.0 - s, "grad_norm": 0.0} for s in (1, 2, 3)]
    )
    failures = failures_for(result, _gptoss_args())
    assert failures and any("optimizer update" in f for f in failures)


def _moved(**over) -> dict:
    """An adapter reading from a run that trained."""
    reading = {
        "ok": True,
        "changed": True,
        "tensors": 168,
        "abs_sum_before": 100.0,
        "abs_sum_after": 101.5,
        "b_abs_sum_before": 0.0,
        "b_abs_sum_after": 1.5,
    }
    reading.update(over)
    return reading


def test_gptoss_does_not_infer_a_verdict_from_an_unlogged_grad_norm():
    """With no logged grad_norm the gpt-oss verdict comes from the adapter weights, not from a guess."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        metrics = [{"step": s, "loss": 3.0 - s} for s in (1, 2, 3)],
        adapter_update = _moved(),
    )
    assert failures_for(result, _gptoss_args()) == []


def test_gptoss_fails_when_nothing_at_all_can_say_the_adapter_moved():
    """With no grad_norm and no adapter reading, gpt-oss fails: other numbers come from the base model."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        metrics = [{"step": s, "loss": 3.0 - s} for s in (1, 2, 3)],
        adapter_update = {"ok": False, "error": "RuntimeError: meta tensor"},
    )
    failures = failures_for(result, _gptoss_args())
    assert any("could not be established" in f for f in failures), failures


def test_gptoss_fails_when_the_adapter_is_the_one_it_started_with():
    """Healthy grad norms do not count if the adapter weights did not move: the adapter reading decides."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        adapter_update = _moved(changed = False, abs_sum_after = 100.0, b_abs_sum_after = 0.0),
    )
    failures = failures_for(result, _gptoss_args())
    assert any("no optimizer update was applied" in f for f in failures), failures


@pytest.mark.parametrize(
    "placement, expected",
    [
        (
            {
                "parameters_by_device": {"cuda:0": 18_000_000_000, "cpu": 2_900_000_000},
                "hf_device_map_devices": ["0", "cpu"],
                "offloaded": True,
            },
            "off the GPU",
        ),
        (
            {
                "parameters_by_device": {"cuda:0": 18_000_000_000, "disk": 2_900_000_000},
                "hf_device_map_devices": ["0", "disk"],
                "offloaded": True,
            },
            "off the GPU",
        ),
        (
            {
                "parameters_by_device": {"cuda:0": 18_000_000_000, "meta": 2_900_000_000},
                "hf_device_map_devices": None,
                "offloaded": False,
            },
            "off the GPU",
        ),
        (
            {
                "parameters_by_device": {"error": "RuntimeError: no"},
                "hf_device_map_devices": None,
                "offloaded": False,
            },
            "could not be walked",
        ),
        (
            {"parameters_by_device": {}, "hf_device_map_devices": None, "offloaded": False},
            "recorded no devices",
        ),
        (None, "never recorded"),
    ],
)
def test_gptoss_fails_when_the_checkpoint_did_not_stay_on_the_gpu(placement, expected):
    """The leg's documented result is that 20B fits on one T4. Assert it.

    Every other number in the report survives an offload: the losses are finite,
    the adapter moves, compilation engages and generation returns text, just
    slower. So a loader or memory regression that spills to CPU, disk or meta
    reports green while the thin-memory condition this leg exists to hold has
    stopped holding. `placement_after_load` recorded that on every run and
    `failures_for` never read it.
    """
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(placement_after_load = placement)
    failures = failures_for(result, _gptoss_args())
    assert any(expected in f for f in failures), failures


def test_gptoss_accepts_the_placement_the_probe_measured():
    """A healthy run keeps every parameter on the one visible CUDA device with no accelerate dispatch."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        placement_after_load = {
            "parameters_by_device": {"cuda:0": 20_900_000_000},
            "hf_device_map_devices": None,
            "offloaded": False,
        },
    )
    assert failures_for(result, _gptoss_args()) == []


def test_gptoss_refuses_a_placement_record_it_cannot_read():
    """A non-boolean offload flag is a failure, not a pass: a check that can switch off never fires."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(
        placement_after_load = {"parameters_by_device": {"cuda:0": 20_900_000_000}},
    )
    failures = failures_for(result, _gptoss_args())
    assert any("whether the loader offloaded" in f for f in failures), failures


def test_grpo_fails_when_nothing_at_all_can_say_the_adapter_moved():
    """GRPO fails when nothing shows the adapter moved: reward and completions need no optimizer step."""
    from run_grpo_t4 import failures_for

    result = _grpo_result(
        metrics = [{"step": s, "loss": 0.0} for s in (1, 2, 3)],
    )
    failures = failures_for(result, _grpo_args())
    assert any("could not be established" in f for f in failures), failures


def test_grpo_reads_the_adapter_when_the_trainer_logged_no_norms():
    from run_grpo_t4 import failures_for
    result = _grpo_result(
        metrics = [{"step": s, "loss": 0.0} for s in (1, 2, 3)],
        adapter_update = _moved(),
    )
    assert failures_for(result, _grpo_args()) == []


class _Tensor:
    """The two calls adapter_fingerprint makes on a parameter, and no more."""

    def __init__(self, value):
        self._value = value

    def detach(self):
        return self

    def float(self):
        return self

    def abs(self):
        return self

    def sum(self):
        return self

    def item(self):
        return self._value


def test_an_empty_norm_list_is_not_the_same_answer_as_a_useless_one():
    """The distinction the `if norms and not applied` spelling collapsed."""
    from training_evidence import update_verdict

    useless = [{"step": 1, "loss": 1.0, "grad_norm": 0.0}]
    silent = [{"step": 1, "loss": 1.0}]
    assert update_verdict(useless)["verdict"] == "not_applied"
    assert update_verdict(silent)["verdict"] == "unverifiable"
    assert update_verdict(useless, _moved())["verdict"] == "not_applied"
    assert update_verdict(silent, _moved())["verdict"] == "applied"


def test_the_adapter_reading_beats_the_grad_norms_it_is_a_proxy_for():
    """Healthy norms over weights that did not move is still nothing trained."""
    from training_evidence import update_verdict

    healthy = [{"step": s, "loss": 1.0, "grad_norm": 2.0} for s in (1, 2)]
    frozen = _moved(changed = False, abs_sum_after = 100.0, b_abs_sum_after = 0.0)
    verdict = update_verdict(healthy, frozen)
    assert verdict["verdict"] == "not_applied"
    assert "bitwise identical" in verdict["detail"]


def test_a_fingerprint_of_a_model_with_no_lora_parameters_is_not_an_answer():
    """`ok: False` means the question could not be answered, never "no"."""
    from training_evidence import adapter_fingerprint, adapter_update

    class _Bare:
        def named_parameters(self):
            return iter([("model.layers.0.mlp.up_proj.weight", _Tensor(1.0))])

    from training_evidence import update_verdict

    reading = adapter_fingerprint(_Bare())
    assert reading["ok"] is False
    assert adapter_update(reading, reading)["ok"] is False
    assert update_verdict([{"step": 1, "loss": 1.0}], reading)["verdict"] == "unverifiable"


def test_a_fingerprint_that_raises_is_reported_rather_than_propagated():
    """A diagnostic that kills the payload it diagnoses is worse than the gap."""
    from training_evidence import adapter_fingerprint

    class _Broken:
        def named_parameters(self):
            raise RuntimeError("Cannot copy out of meta tensor")

    reading = adapter_fingerprint(_Broken())
    assert reading["ok"] is False
    assert "RuntimeError" in reading["error"]


def test_the_zero_initialised_b_matrices_are_what_make_the_comparison_safe():
    """Both sums are compared, so an exact float equality can be a red."""
    from training_evidence import adapter_fingerprint, adapter_update

    class _Adapter:
        def __init__(self, b):
            self._b = b

        def named_parameters(self):
            return iter(
                [
                    ("base.layers.0.q_proj.lora_A.default.weight", _Tensor(4.0)),
                    ("base.layers.0.q_proj.lora_B.default.weight", _Tensor(self._b)),
                ]
            )

    before = adapter_fingerprint(_Adapter(0.0))
    assert before == {"ok": True, "tensors": 2, "abs_sum": 4.0, "b_abs_sum": 0.0}
    after = adapter_fingerprint(_Adapter(0.0))
    after["b_abs_sum"] = 0.5
    assert adapter_update(before, after)["changed"] is True
    assert adapter_update(before, before)["changed"] is False


@pytest.mark.parametrize("bad", [float("nan"), float("inf"), float("-inf")])
def test_a_non_finite_lora_weight_is_not_read_as_a_successful_update(bad):
    """NaN and inf LoRA weights must not count as an applied update; != finite reads them as changed."""
    from training_evidence import adapter_fingerprint, adapter_update, update_verdict

    class _Corrupt:
        def __init__(self, b):
            self._b = b

        def named_parameters(self):
            return iter(
                [
                    ("base.layers.0.q_proj.lora_A.default.weight", _Tensor(4.0)),
                    ("base.layers.0.q_proj.lora_B.default.weight", _Tensor(self._b)),
                ]
            )

    before = adapter_fingerprint(_Corrupt(0.0))
    after = adapter_fingerprint(_Corrupt(bad))
    assert after["ok"] is False
    assert after["non_finite"] is True
    assert "lora_B" in after["error"]

    update = adapter_update(before, after)
    assert update["ok"] is False
    assert update["non_finite"] is True

    healthy = [{"step": s, "loss": 1.0, "grad_norm": 2.0} for s in (1, 2)]
    assert update_verdict(healthy, update)["verdict"] == "non_finite"
    assert update_verdict([], update)["verdict"] == "non_finite"


@pytest.mark.parametrize("field", ["abs_sum", "b_abs_sum"])
def test_a_non_finite_sum_is_refused_by_the_comparison_as_well(field):
    """Checked where the exact `!=` happens, not only where it was summed."""
    from training_evidence import adapter_update, update_verdict

    before = {"ok": True, "tensors": 4, "abs_sum": 1.0, "b_abs_sum": 0.0}
    after = dict(before, **{field: float("nan")})
    verdict = adapter_update(before, after)
    assert verdict["ok"] is False
    assert verdict["non_finite"] is True
    assert update_verdict([], verdict)["verdict"] == "non_finite"


def test_a_non_finite_adapter_turns_both_legs_red():
    """The verdict has to reach a failure string, or it is not a check."""
    from run_gptoss_t4 import failures_for as gptoss_failures
    from run_grpo_t4 import failures_for as grpo_failures

    corrupt = {
        "ok": False,
        "non_finite": True,
        "error": "1 of 4 LoRA tensors hold non-finite weights",
    }
    gptoss = gptoss_failures(_gptoss_result(adapter_update = corrupt), _gptoss_args())
    assert gptoss and any("non-finite weights" in f for f in gptoss)
    grpo = grpo_failures(_grpo_result(adapter_update = corrupt), _grpo_args())
    assert grpo and any("non-finite weights" in f for f in grpo)


def test_a_verdict_neither_leg_has_been_taught_about_is_still_a_failure():
    """A verdict the legs have not been taught fails: they test for applied, not unverifiable."""
    import run_gptoss_t4
    import run_grpo_t4

    unknown = {"verdict": "something_new", "detail": "d", "grad_norms": []}
    for module, args in ((run_gptoss_t4, _gptoss_args()), (run_grpo_t4, _grpo_args())):
        original = module.update_verdict
        module.update_verdict = lambda *a, **k: unknown
        try:
            result = _gptoss_result() if module is run_gptoss_t4 else _grpo_result()
            assert module.failures_for(result, args), module.__name__
        finally:
            module.update_verdict = original


def test_two_fingerprints_over_different_tensor_counts_are_not_compared():
    from training_evidence import adapter_update

    before = {"ok": True, "tensors": 168, "abs_sum": 1.0, "b_abs_sum": 0.0}
    after = {"ok": True, "tensors": 12, "abs_sum": 1.0, "b_abs_sum": 0.0}
    verdict = adapter_update(before, after)
    assert verdict["ok"] is False
    assert "not comparable" in verdict["error"]


def _grpo_args(**over):
    base = {"max_steps": 3}
    base.update(over)
    return types.SimpleNamespace(**base)


def _grpo_result(**over) -> dict:
    result = {
        "log_history": [
            {"step": s, "loss": 0.0, "reward": 0.5, "reward_std": 0.2} for s in (1, 2, 3)
        ],
        "metrics": [{"step": s, "loss": 0.0, "grad_norm": 1.25} for s in (1, 2, 3)],
        "completions": [["four score and seven"], ["a completion with 7 in it"]],
        "fast_generate": "some generated text",
        "fast_generate_lora": {"requested": True, "applied": True},
    }
    result.update(over)
    return result


def test_a_healthy_grpo_run_passes():
    from run_grpo_t4 import failures_for
    assert failures_for(_grpo_result(), _grpo_args()) == []


def test_grpo_fails_when_a_step_logged_no_reward():
    from run_grpo_t4 import failures_for

    history = [
        {"step": 1, "loss": 0.0, "reward": 0.5, "reward_std": 0.2},
        {"step": 2, "loss": 0.0},
        {"step": 3, "loss": 0.0, "reward": 0.5, "reward_std": 0.2},
    ]
    failures = failures_for(_grpo_result(log_history = history), _grpo_args())
    assert failures and any("every step" in f for f in failures)


def test_grpo_ignores_the_summary_entry_that_carries_no_loss():
    """`train()` appends a train_runtime row; it is not a training step."""
    from run_grpo_t4 import failures_for

    history = [{"step": s, "loss": 0.0, "reward": 0.5, "reward_std": 0.2} for s in (1, 2, 3)]
    history.append({"step": 3})
    assert failures_for(_grpo_result(log_history = history), _grpo_args()) == []


def test_grpo_fails_when_every_gradient_norm_is_nan():
    from run_grpo_t4 import failures_for

    result = _grpo_result(metrics = [{"step": s, "loss": 0.0, "grad_norm": NAN} for s in (1, 2, 3)])
    failures = failures_for(result, _grpo_args())
    assert failures and any("optimizer update" in f for f in failures)


def test_grpo_fails_when_the_final_generation_skipped_the_trained_adapter():
    from run_grpo_t4 import failures_for

    result = _grpo_result(
        fast_generate_lora = {"requested": True, "applied": False, "error": "AttributeError: x"}
    )
    failures = failures_for(result, _grpo_args())
    assert failures and any("adapter" in f for f in failures)


def test_grpo_records_the_engine_it_built_before_a_later_failure(monkeypatch, tmp_path):
    """Construction succeeded and training raised: the report must say so."""
    import run_grpo_t4

    def exploding_train(args, report = None):
        if report is not None:
            report["engine_built"] = True
        raise RuntimeError("CUDA error: an illegal memory access was encountered")

    monkeypatch.setattr(run_grpo_t4, "train", exploding_train)
    monkeypatch.setattr(run_grpo_t4, "make_libcuda_linkable", lambda: {"needed": False})
    monkeypatch.setattr(run_grpo_t4, "vllm_facts", lambda: {"version": "0.11.2"})
    monkeypatch.setattr(sys, "argv", ["run_grpo_t4.py", "--outdir", str(tmp_path)])
    code = run_grpo_t4.main()
    report = json.loads((tmp_path / "t4_smoke_report.json").read_text(encoding = "utf-8"))
    assert code == 1
    assert report["engine_built"] is True
    assert any("illegal memory access" in f for f in report["failures"])


def test_grpo_still_reports_an_engine_that_never_built(monkeypatch, tmp_path):
    import run_grpo_t4

    def exploding_train(args, report = None):
        raise RuntimeError("EngineCore failed to start")

    monkeypatch.setattr(run_grpo_t4, "train", exploding_train)
    monkeypatch.setattr(run_grpo_t4, "make_libcuda_linkable", lambda: {"needed": False})
    monkeypatch.setattr(run_grpo_t4, "vllm_facts", lambda: {"version": "0.11.2"})
    monkeypatch.setattr(sys, "argv", ["run_grpo_t4.py", "--outdir", str(tmp_path)])
    assert run_grpo_t4.main() == 1
    report = json.loads((tmp_path / "t4_smoke_report.json").read_text(encoding = "utf-8"))
    assert report["engine_built"] is False


def test_the_recapture_recipe_selects_the_control_report_by_label():
    """`reports[0]` is whichever kernel slug sorted first, not the control."""
    readme = (SMOKE_DIR / "references" / "README.md").read_text(encoding = "utf-8")
    assert 'reports"][0]' not in readme.replace(" ", "")
    assert '"control"' in readme
    recipe = readme[readme.index("## Capturing one") :]
    assert "label" in recipe


def _recapture_recipe() -> str:
    readme = (SMOKE_DIR / "references" / "README.md").read_text(encoding = "utf-8")
    body = readme.split("python - <<'PY'\n", 1)[1]
    return body.split("\nPY\n", 1)[0]


def test_the_recapture_recipe_records_the_kernel_it_came_from(tmp_path, monkeypatch):
    """The recapture recipe records source_kernel, the one field tying a band to its hardware run."""
    evidence = tmp_path / "kaggle_evidence"
    kernels = [
        ("danielhanchen/unsloth-t4-ci-9f0e1a2b", ("control", "canary")),
        ("danielhanchen/unsloth-t4-ci-0a1b2c3d", ("gptoss", "frontier")),
    ]
    reports = []
    for slug, leg_names in kernels:
        directory = evidence / slug.rsplit("/", 1)[-1]
        directory.mkdir(parents = True)
        for leg in leg_names:
            (directory / f"t4_{leg}_output.ipynb").write_text("{}", encoding = "utf-8")
            reports.append(
                {
                    "label": leg,
                    "passed": True,
                    "model": "unsloth/Qwen2.5-0.5B-Instruct",
                    "metrics": [{"step": 1, "loss": 1.0}],
                    "environment": {"gpu_name": "Tesla T4"},
                    "config": {"max_steps": 10},
                    "resolved_checkpoint": "unsloth/Qwen2.5-0.5B-Instruct-unsloth-bnb-4bit",
                    "resolved_revision": "0123456789abcdef",
                }
            )
    (evidence / "launch_result.json").write_text(
        json.dumps({"reports": reports, "kernels": [{"slug": s} for s, _ in kernels]}),
        encoding = "utf-8",
    )
    (tmp_path / "tests" / "kaggle" / "t4_smoke" / "references").mkdir(parents = True)

    monkeypatch.chdir(tmp_path)
    exec(compile(_recapture_recipe(), "<recapture-recipe>", "exec"), {"__name__": "__main__"})

    written = json.loads(
        (tmp_path / "tests/kaggle/t4_smoke/references/t4_qwen2.5-0.5b.json").read_text(
            encoding = "utf-8"
        )
    )
    assert written["source_kernel"] == "danielhanchen/unsloth-t4-ci-9f0e1a2b"
    assert written["model"] == "unsloth/Qwen2.5-0.5B-Instruct"
    assert written["resolved_revision"] == "0123456789abcdef"


def test_the_committed_reference_names_a_kernel_and_not_a_leg():
    reference = json.loads(
        (SMOKE_DIR / "references" / "t4_qwen2.5-0.5b.json").read_text(encoding = "utf-8")
    )
    assert "/" in reference["source_kernel"], reference["source_kernel"]


def test_reports_are_not_ordered_control_first(tmp_path):
    """The fact behind the item above, driven through the real extractor."""
    from launch import extract_reports

    def notebook(label: str) -> str:
        line = "T4_SMOKE_REPORT " + json.dumps(
            {"label": label, "model": "m", "metrics": [], "environment": {}, "config": {}}
        )
        return json.dumps({"cells": [{"outputs": [{"text": line + "\n"}]}]})

    for slug, legs in (
        ("unsloth-t4-ci-9f0e1a2b", ("control", "canary")),
        ("unsloth-t4-ci-0a1b2c3d", ("gptoss", "frontier")),
    ):
        kernel = tmp_path / slug
        kernel.mkdir(parents = True)
        for leg in legs:
            (kernel / f"t4_{leg}_output.ipynb").write_text(notebook(leg), encoding = "utf-8")

    labels = [r["label"] for r in extract_reports(tmp_path)]
    assert labels[0] != "control"
    assert sorted(labels) == ["canary", "control", "frontier", "gptoss"]


@pytest.mark.parametrize(
    "environment",
    [
        {"error": "RuntimeError: is_bf16_supported() failed"},
        {},
        {"gpu_name": "Tesla T4"},
        {"bf16_supported": None},
        {"bf16_supported": "false"},
        {"bf16_supported": 0},
    ],
)
def test_gptoss_fails_when_the_cards_bf16_support_is_unreadable(environment):
    """Anything other than a literal True or False for bf16 support is unverifiable, so gpt-oss fails."""
    from run_gptoss_t4 import failures_for

    result = _gptoss_result(environment = environment)
    failures = failures_for(result, _gptoss_args())
    assert failures and any("bf16" in f for f in failures), failures


def test_gptoss_still_reads_a_bf16_card_and_a_t4_the_way_it_did():
    """The two literal readings keep their meanings, so the widening above
    cannot be satisfied by turning every card into a failure."""
    from run_gptoss_t4 import failures_for

    assert failures_for(_gptoss_result(), _gptoss_args()) == []
    assert (
        failures_for(
            _gptoss_result(
                environment = {"bf16_supported": True, "gpu_name": "NVIDIA A100"},
                precision = {"fp16": False, "bf16": True, "force_float32_env": None},
            ),
            _gptoss_args(),
        )
        == []
    )


def test_batched_generation_runs_end_to_end_against_a_stub_model():
    """Runs the real batched_generation on a stub; dict-fed tests never ran it, so a missing import hid."""
    import torch

    from run_t4_smoke import batched_generation, batched_generation_failures

    class _Enc(dict):
        def to(self, _device):
            return self

    class _Tok:
        padding_side = "right"
        pad_token = "<pad>"
        eos_token = "<eos>"
        pad_token_id = 0

        def __call__(
            self,
            text,
            return_tensors = None,
            padding = False,
        ):
            texts = [text] if isinstance(text, str) else list(text)
            # One token per character, so prompt length spread is real padding.
            ids = [[ord(c) % 100 + 1 for c in t] for t in texts]
            if return_tensors is None:
                return {"input_ids": ids[0] if isinstance(text, str) else ids}
            width = max(len(i) for i in ids)
            # Left padding, which the padded-width slice depends on.
            padded = [[0] * (width - len(i)) + i for i in ids]
            return _Enc(
                input_ids = torch.tensor(padded),
                attention_mask = torch.tensor([[0] * (width - len(i)) + [1] * len(i) for i in ids]),
            )

        def decode(
            self,
            row,
            skip_special_tokens = False,
        ):
            return "".join(chr(int(v)) for v in row if int(v) != 0)

    class _Model:
        device = "cpu"

        def generate(
            self,
            input_ids = None,
            attention_mask = None,
            max_new_tokens = 8,
            **_kw,
        ):
            # The continuation depends only on the row's unpadded content.
            outs = []
            for row in input_ids:
                real = [int(v) for v in row if int(v) != 0]
                tail = [(sum(real) % 26) + 65] * max_new_tokens
                outs.append([int(v) for v in row] + tail)
            return torch.tensor(outs)

    # Eight prompts of lengths 1..8, matching the largest batch size, so every batch pads.
    prompts = ["abcdefgh"[:n] * 1 for n in range(1, 9)]
    record = batched_generation(_Model(), _Tok(), prompts, max_new_tokens = 4)

    assert record["distinct_lengths"] == 8, "the stub prompts must actually pad"
    assert record["padding_side_observed"] == "left"
    assert len(record["singles"]) == len(prompts)
    assert all(s for s in record["singles"]), "the padded-width slice dropped everything"
    assert batched_generation_failures(record) == []


def test_a_healthy_batched_generation_record_reports_no_failures():
    from run_t4_smoke import batched_generation_failures
    assert batched_generation_failures(_batch_record()) == []


def test_a_batch_whose_prompts_are_all_one_length_is_reported_as_proving_nothing():
    from run_t4_smoke import batched_generation_failures

    """The vacuity case, and the reason this check exists in this shape.

    Left padding only happens when the prompts in a batch differ in length. Feed
    eight identical-length prompts and every batch agrees with every single --
    perfectly, every time, having never padded once. That is a green
    left-padding assertion covering nothing, which is the failure mode this
    whole file keeps being caught by.
    """
    failures = batched_generation_failures(
        _batch_record(prompt_token_lengths = [12] * 8, distinct_lengths = 1)
    )
    assert any("nothing was ever padded" in f for f in failures), failures


def test_a_padding_side_silently_flipped_to_right_is_a_failure():
    from run_t4_smoke import batched_generation_failures

    """#2138: a release forced the tokenizer padding side to right in inference.

    Checked twice, before and after generating, because the override that
    caused it happened INSIDE the inference path -- so the value this payload
    set is not evidence of the value that was used.
    """
    assert any(
        "padding_side_after" in f
        for f in batched_generation_failures(_batch_record(padding_side_after = "right"))
    )
    assert any(
        "padding_side_observed" in f
        for f in batched_generation_failures(_batch_record(padding_side_observed = "right"))
    )


def test_a_batch_that_disagrees_with_one_at_a_time_is_a_failure():
    from run_t4_smoke import batched_generation_failures

    """#3699 / #1456: batched output diverging from sequential greedy output."""
    broken = _batch_record()
    broken["batched"]["4"] = ["different"] * 8
    broken["agrees"]["4"] = False
    failures = batched_generation_failures(broken)
    assert any("batch size 4 did not reproduce" in f for f in failures), failures


def test_empty_generations_are_a_failure_even_when_every_batch_agrees():
    from run_t4_smoke import batched_generation_failures

    """#1066: gibberish, of which "nothing at all" is the degenerate case.

    Agreement alone is not health: a model that emits an empty string for every
    prompt agrees with itself at every batch size.
    """
    failures = batched_generation_failures(
        _batch_record(
            singles = [""] * 8,
            empty_outputs = [0, 1, 2, 3, 4, 5, 6, 7],
            batched = {"2": [""] * 8, "4": [""] * 8, "8": [""] * 8},
        )
    )
    assert any("generated nothing at all" in f for f in failures), failures


def test_an_empty_row_inside_a_batch_is_a_failure_even_when_the_singles_are_fine():
    from run_t4_smoke import batched_generation_failures

    """The hole: `empty_outputs` was derived from the SINGLES only. A row that
    decodes to "" inside the batch of 8 while its one-at-a-time output is fine
    is not caught by that, and the disagreement it causes is excused for a
    model listed in KNOWN_BATCHED_GENERATION_BREAKAGE. That is #9848, a
    left-padded row attending to nothing, reported as a pass.
    """
    record = _batch_record(
        batched = {
            "2": [f"out{i}" for i in range(8)],
            "4": [f"out{i}" for i in range(8)],
            "8": [f"out{i}" if i != 2 else "" for i in range(8)],
        },
        agrees = {"2": True, "4": True, "8": False},
        empty_batched_outputs = {"8": [2]},
    )
    failures = batched_generation_failures(record, "unsloth/gemma-4-E2B-it")
    assert any("inside the batch" in f and "[2]" in f for f in failures), failures


def test_batched_generation_records_a_row_that_is_empty_only_inside_the_batch():
    """Row 2 is empty only inside a batch of 8; the record must list it under empty_batched_outputs."""
    import torch

    from run_t4_smoke import batched_generation, batched_generation_failures

    class _Enc(dict):
        def to(self, _device):
            return self

    class _Tok:
        padding_side = "right"
        pad_token = "<pad>"
        eos_token = "<eos>"
        pad_token_id = 0

        def __call__(
            self,
            text,
            return_tensors = None,
            padding = False,
        ):
            texts = [text] if isinstance(text, str) else list(text)
            ids = [[ord(c) % 100 + 1 for c in t] for t in texts]
            if return_tensors is None:
                return {"input_ids": ids[0] if isinstance(text, str) else ids}
            width = max(len(i) for i in ids)
            padded = [[0] * (width - len(i)) + i for i in ids]
            return _Enc(
                input_ids = torch.tensor(padded),
                attention_mask = torch.tensor([[0] * (width - len(i)) + [1] * len(i) for i in ids]),
            )

        def decode(
            self,
            row,
            skip_special_tokens = False,
        ):
            return "".join(chr(int(v)) for v in row if int(v) != 0)

    class _EmptyRowInsideBatch8:
        device = "cpu"

        def generate(
            self,
            input_ids = None,
            attention_mask = None,
            max_new_tokens = 8,
            **_kw,
        ):
            outs = []
            for index, row in enumerate(input_ids):
                real = [int(v) for v in row if int(v) != 0]
                tail = [(sum(real) % 26) + 65] * max_new_tokens
                if input_ids.shape[0] == 8 and index == 2:
                    tail = [0] * max_new_tokens  # pad only: decodes to ""
                outs.append([int(v) for v in row] + tail)
            return torch.tensor(outs)

    prompts = ["abcdefgh"[:n] for n in range(1, 9)]
    record = batched_generation(_EmptyRowInsideBatch8(), _Tok(), prompts, max_new_tokens = 4)

    assert record["empty_outputs"] == [], "the singles were fine; that is the point"
    assert record["empty_batched_outputs"] == {"8": [2]}
    failures = batched_generation_failures(record, "unsloth/gemma-4-E2B-it")
    assert any("inside the batch" in f and "[2]" in f for f in failures), failures


def test_too_few_prompts_to_fill_the_largest_batch_is_reported():
    from run_t4_smoke import batched_generation_failures

    """A batch size larger than the prompt list silently becomes one small batch."""
    failures = batched_generation_failures(
        _batch_record(singles = ["a", "b"], prompt_token_lengths = [5, 9], distinct_lengths = 2)
    )
    assert any("never actually formed" in f for f in failures), failures


def test_a_missing_batched_record_is_a_failure_not_a_pass():
    from run_t4_smoke import batched_generation_failures
    """A leg where the check never ran must not look like a leg where it passed."""
    assert batched_generation_failures(None) == ["batched generation was never run"]


def _completions(*lengths):
    """GRPO hands reward functions a list of message lists."""
    return [[{"role": "assistant", "content": "x" * n}] for n in lengths]


def test_the_length_reward_still_discriminates_at_the_lengths_the_model_emits():
    """reward_length saturated at 200 chars, under the 2534-3396 char completions seen; assert at those."""
    sys.path.insert(0, str(SMOKE_DIR))
    from run_grpo_t4 import reward_length

    for a, b in ((2534, 3396), (3013, 3328), (2738, 2633)):
        scores = reward_length(_completions(a, b))
        assert scores[0] != scores[1], (
            f"lengths {a} and {b} scored identically ({scores}); the reward has "
            f"saturated in the range the model actually produces"
        )


def test_the_length_reward_is_strictly_increasing_and_never_saturates():
    """Strict monotonicity must hold for lengths nobody has seen; a cap is a length where it stops
    being true, which is how the previous version passed review."""
    from run_grpo_t4 import reward_length

    lengths = [0, 1, 50, 200, 1000, 3000, 10000, 100000]
    scores = reward_length(_completions(*lengths))
    assert all(x < y for x, y in zip(scores, scores[1:])), scores
    assert all(0.0 <= s < 1.0 for s in scores), scores


def test_two_identical_completions_still_tie_because_that_is_real():
    """Identical completions should tie, since that is degenerate generation the leg should fail on;
    the bug was ties between different completions."""
    from run_grpo_t4 import reward_length

    scores = reward_length(_completions(2534, 2534))
    assert scores[0] == scores[1]


def _gguf_record(**over):
    """A healthy export, shaped like the one measured on
    unsloth-probe-gguf-q8-peft-920e3e: the GGUF in the SIBLING directory."""
    record = {
        "save_dir": "/tmp/q8p",
        "requested_quantization": "q8_0",
        "ok": True,
        "seconds": 40.6,
        "ggufs": [
            {
                "path": "/tmp/q8p_gguf/qwen3-0.6b.Q8_0.gguf",
                "mb": 609.8,
                "found_in": "/tmp/q8p_gguf",
                "suffix": "_gguf",
            }
        ],
    }
    record.update(over)
    return record


def test_a_healthy_gguf_export_reports_no_failures():
    from gguf_export import export_failures
    assert export_failures(_gguf_record(), accept_quantizations = ("q8_0",)) == []


def test_an_export_that_reported_ok_but_wrote_no_gguf_is_a_failure():
    """save_pretrained_gguf writes the GGUF to a sibling dir, so globbing the passed dir finds nothing."""
    from gguf_export import export_failures

    failures = export_failures(_gguf_record(ggufs = []), accept_quantizations = ("q8_0",))
    assert failures and "no .gguf" in failures[0]
    assert "_gguf sibling" in failures[0]


def test_a_gguf_search_finds_the_file_in_the_sibling_directory(tmp_path):
    """Executed against a real directory pair rather than a hand-written dict,
    because the whole point is WHERE the file is."""
    from gguf_export import find_ggufs

    save = tmp_path / "out"
    save.mkdir()
    (save / "model.safetensors").write_bytes(b"0" * 2048)
    sibling = tmp_path / "out_gguf"
    sibling.mkdir()
    (sibling / "qwen3-0.6b.Q8_0.gguf").write_bytes(b"0" * 4096)

    found = find_ggufs(str(save))
    assert len(found) == 1, found
    assert found[0]["suffix"] == "_gguf"
    assert found[0]["path"].endswith("qwen3-0.6b.Q8_0.gguf")


def test_a_gguf_that_is_only_a_header_is_not_an_export():
    from gguf_export import export_failures
    failures = export_failures(
        _gguf_record(
            ggufs = [
                {
                    "path": "/tmp/q8p_gguf/x.Q8_0.gguf",
                    "mb": 0.1,
                    "found_in": "/tmp/q8p_gguf",
                    "suffix": "_gguf",
                }
            ]
        ),
        accept_quantizations = ("q8_0",),
    )
    assert failures and "header and no weights" in failures[0]


def test_a_model_allowed_to_override_the_quantization_still_passes():
    """gpt-oss answers q8_0 with "GPT-OSS does not support GGUF quantization
    (requested: q8_0). Overriding to MXFP4 format." That is documented
    behaviour, so a leg that accepts MXFP4 must not fail on it."""
    from gguf_export import export_failures

    record = _gguf_record(
        ggufs = [
            {
                "path": "/tmp/g_gguf/gpt-oss-20b.MXFP4.gguf",
                "mb": 11800.0,
                "found_in": "/tmp/g_gguf",
                "suffix": "_gguf",
            }
        ]
    )
    assert export_failures(record, accept_quantizations = ("mxfp4",)) == []
    failures = export_failures(record, accept_quantizations = ("q8_0",))
    assert failures and "accepted quantization" in failures[0]


def test_a_gguf_that_no_runner_could_execute_is_a_failure():
    from gguf_export import run_failures
    failures = run_failures(
        {
            "gguf": "/tmp/q8p_gguf/x.gguf",
            "bench": {"seconds": 240.0, "error": "TimeoutExpired: ..."},
            "completion": {"seconds": 240.0, "returncode": 1, "stderr": "bad magic"},
        }
    )
    assert failures and "produced no output from any runner" in failures[0]


def test_one_successful_runner_is_enough():
    from gguf_export import run_failures
    assert (
        run_failures(
            {
                "gguf": "/tmp/q8p_gguf/x.gguf",
                "bench": {"seconds": 12.0, "returncode": 0, "stdout": "tg128 ... 41.2"},
            }
        )
        == []
    )


def test_a_bundle_with_no_runners_at_all_is_reported_as_that():
    """Distinct from "the file does not run": a missing binary is a bundle
    problem and blaming the model file for it would send the reader to the
    wrong place."""
    from gguf_export import run_failures

    failures = run_failures(
        {
            "gguf": "/tmp/x.gguf",
            "bench": {"skipped": "no llama-bench in the bundle"},
            "completion": {"skipped": "no llama-completion in the bundle"},
        }
    )
    assert failures and "missing from the llama.cpp bundle" in failures[0]


def test_the_llama_cpp_facts_read_a_tuple_not_a_directory(tmp_path):
    """install_llama_cpp returns (llama-quantize, convert_hf_to_gguf.py), not a bin directory."""
    from gguf_export import llama_cpp_facts

    quant = tmp_path / "llama-quantize"
    conv = tmp_path / "convert_hf_to_gguf.py"
    quant.write_text("")
    conv.write_text("")

    facts = llama_cpp_facts(
        "Unsloth: Installing prebuilt llama.cpp b10472-mix-4b653db "
        "(app-b10472-mix-4b653db-linux-x64-cpu.tar.gz) - skipping compilation.",
        (str(quant), str(conv)),
    )
    assert facts["all_exist"] is True
    assert facts["dir"] == str(tmp_path)
    assert facts["prebuilt"] is True
    assert facts["source_build_markers"] == []


def test_a_source_build_is_visible_even_though_it_succeeds():
    """The failure to catch is not "kernel missing" but "kernel built from
    source": silent, correct, and many minutes on 4 vCPUs."""
    from gguf_export import llama_cpp_facts

    facts = llama_cpp_facts("-- Configuring done\ncmake --build . -j 4\n", ())
    assert facts["prebuilt"] is False
    assert "cmake" in facts["source_build_markers"]


# Options only the parent acts on; each entry needs a stated reason.
PARENT_ONLY_DESTS = {
    "outdir",  # the parent gives each cycle its own subdirectory
    "cycle",  # set by the parent per child, never forwarded verbatim
    "repeat",  # how many children to launch
    "reference",  # band check runs in the parent, over collected cycles
    "rel_tol",  # ... and its tolerances
    "abs_floor",
    "require_canary",  # evaluated by the parent's failure collector
    "check_batched_generation",
    "export_gguf",  # forwarded as a bare flag, asserted separately below
    # The pin check runs in the parent over the cycles' reports.
    "pins",
    # Spawned and judged by the parent after the cycles.
    "compare_naive_trl",
    "control_oom_is_ok",
    # Collected in the child, but the flag goes via the bare-flag block; see test_kernel_provenance.py.
    "kernel_provenance",
    # Spawned by the parent in its own process: two 4bit models on a 14.56GB T4 would OOM.
    "vision_run",
}


def _smoke_source() -> str:
    return (SMOKE_DIR / "run_t4_smoke.py").read_text(encoding = "utf-8")


def _child_command_block() -> str:
    """The argv the parent builds for each cycle."""
    source = _smoke_source()
    start = source.index("cmd = [\n            sys.executable,")
    end = source.index("proc = subprocess.run(cmd)", start)
    return source[start:end]


def test_every_option_the_child_needs_actually_reaches_the_child():
    """A flag the parser accepts but the parent's child argv list omits is silently ignored by the child."""
    import argparse
    import importlib

    sys.path.insert(0, str(SMOKE_DIR))
    module = importlib.import_module("run_t4_smoke")

    # Build the parser as main() does, using a sentinel that makes it return.
    parser = argparse.ArgumentParser()
    source = _smoke_source()
    dests = set(re.findall(r'dest\s*=\s*"([a-z_0-9]+)"', source))
    dests |= {
        m.replace("-", "_") for m in re.findall(r'ap\.add_argument\(\s*"--([a-z0-9-]+)"', source)
    }
    assert "export_gguf" in dests, "the parser no longer defines --export-gguf"
    assert "model" in dests, "the dest scrape found nothing; fix the scrape"

    block = _child_command_block()
    forwarded = {m.replace("-", "_") for m in re.findall(r'"--([a-z0-9-]+)"', block)}

    missing = sorted(d for d in dests - forwarded - PARENT_ONLY_DESTS if not d.startswith("no_"))
    assert not missing, (
        f"these options are parsed but never forwarded to the cycle child, so "
        f"setting them does nothing: {missing}. Either forward them or list "
        f"them in PARENT_ONLY_DESTS with a reason."
    )
    del parser, module


def test_the_export_flag_is_forwarded_as_a_bare_flag():
    """store_true options cannot ride the (flag, value) loop, since --export-gguf True is not valid,
    so they need their own append line."""
    block = _child_command_block()
    assert 'cmd.append("--export-gguf")' in block
    assert "if args.export_gguf:" in block


def test_the_export_settings_ride_the_value_loop():
    block = _child_command_block()
    assert '("--gguf-quantization", args.gguf_quantization)' in block
    assert '("--gguf-accept", args.gguf_accept)' in block


def test_a_second_cycle_cannot_report_an_already_installed_llama_cpp_as_a_source_build():
    """Prebuilt banner prints only on install; a cycle that did not install reports None, not false."""
    from gguf_export import llama_cpp_facts

    quiet = llama_cpp_facts("", ())
    assert (
        quiet["prebuilt"] is None
    ), "an install that printed nothing must not claim a source build"
    assert quiet["source_build_markers"] == []

    assert (
        llama_cpp_facts(
            "Unsloth: Installing prebuilt llama.cpp b10472 - skipping compilation.", ()
        )["prebuilt"]
        is True
    )
    assert llama_cpp_facts("cmake --build . -j 4\n", ())["prebuilt"] is False


def test_the_text_leg_gguf_export_does_not_land_in_the_artifact_directory():
    """GGUF export must use a tempdir: the artifact dir ships back, and a big merge fills the disk."""
    src = _smoke_source()
    call = src[src.index("gguf_export_record = export_gguf(") :]
    call = call[: call.index(")")]
    assert "tempfile.mkdtemp" in call, "the export writes into the artifact directory"
    assert (
        "args.outdir" not in call
    ), "args.outdir is /kaggle/working, which is 21 GB and is collected"