Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

20.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Never leave a refreshed notebook on disk without its state record; the next refresh would keep it."""

from __future__ import annotations

import atexit
import os
import shutil
import subprocess
import tempfile
from pathlib import Path

import pytest


# Prototypes are built once and copytree'd in; no tree embeds its own absolute path.
_PROTOTYPES: dict[str, Path] = {}


def _prototype(key: str, build) -> Path:
    prototype = _PROTOTYPES.get(key)
    if prototype is None:
        prototype = Path(tempfile.mkdtemp(prefix = "unsloth-nbsync-"))
        atexit.register(shutil.rmtree, prototype, True)
        build(prototype)
        _PROTOTYPES[key] = prototype
    return prototype


def _copy_of(key: str, tmp_path: Path, build):
    root = tmp_path / key
    shutil.copytree(_prototype(key, build), root, symlinks = True)
    return root / "dest", root / "tpl", root / "up"


def _build_seed_synced(root: Path):
    """Seed-only template, 500 one-byte notebooks upstream, initial no-refresh sync."""
    tpl, dest, up = _template(root), root / "dest", _upstream(root)
    dest.mkdir()
    _run(tpl, dest, up, refresh = False)


def _build_refreshed(root: Path):
    """`_build_seed_synced` plus one uncapped refresh: everything published and recorded."""
    _build_seed_synced(root)
    _run(root / "tpl", root / "dest", root / "up", refresh = True)


def _build_big_synced(root: Path):
    """500-notebook baked template, same upstream, initial no-refresh sync."""
    tpl, dest, up = _big_template(root, NOTEBOOKS), root / "dest", _upstream(root)
    dest.mkdir()
    _run(tpl, dest, up, refresh = False)


# Two tests read the same capped run; neither mutates before reading.
_CAPPED_FIRST_REFRESH: subprocess.CompletedProcess | None = None


def _build_capped_first_refresh(root: Path):
    global _CAPPED_FIRST_REFRESH
    _build_seed_synced(root)
    _CAPPED_FIRST_REFRESH = _run(
        root / "tpl",
        root / "dest",
        root / "up",
        cap_kib = CAP_KIB,
        refresh = True,
    )


def _shared_capped_first_refresh(tmp_path):
    dest, tpl, up = _copy_of("capped", tmp_path, _build_capped_first_refresh)
    return dest, tpl, up, _CAPPED_FIRST_REFRESH


def _shared_setup_1(tmp_path):
    return _copy_of("s1", tmp_path, _build_refreshed)


def _shared_big_synced(tmp_path):
    return _copy_of("big", tmp_path, _build_big_synced)


REPO_ROOT = Path(__file__).resolve().parents[2]
SYNC = REPO_ROOT / "docker" / "unsloth_sync_notebooks.sh"

NOTEBOOKS = 500
CAP_KIB = 40  # between the clone's 40067-byte index and the 42000-byte state

needs_git = pytest.mark.skipif(
    shutil.which("git") is None or shutil.which("sha256sum") is None,
    reason = "the refresh path needs git and sha256sum",
)


def _upstream(tmp_path: Path) -> Path:
    up = tmp_path / "up"
    up.mkdir()
    for i in range(1, NOTEBOOKS + 1):
        (up / f"nb{i:09d}.ipynb").write_text("N", encoding = "utf-8")
    env = dict(os.environ, GIT_CONFIG_GLOBAL = "/dev/null", GIT_CONFIG_SYSTEM = "/dev/null")
    subprocess.run(["git", "init", "-q", "."], cwd = up, check = True, env = env)
    subprocess.run(["git", "add", "-A"], cwd = up, check = True, env = env)
    subprocess.run(
        ["git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "init"],
        cwd = up,
        check = True,
        env = env,
    )
    return up


def _template(tmp_path: Path) -> Path:
    tpl = tmp_path / "tpl"
    tpl.mkdir()
    (tpl / "seed.ipynb").write_text("T", encoding = "utf-8")
    (tpl / ".unsloth_template_commit").write_text("a" * 40 + "\n", encoding = "utf-8")
    return tpl


def _run(
    tpl: Path,
    dest: Path,
    up: Path,
    *,
    cap_kib: int | None = None,
    refresh: bool,
    keep_removed: bool = False,
):
    def _cap():
        import resource
        import signal

        # SIGXFSZ would kill the script; the shell has to see the write error.
        signal.signal(signal.SIGXFSZ, signal.SIG_IGN)
        n = cap_kib * 1024
        resource.setrlimit(resource.RLIMIT_FSIZE, (n, n))

    env = dict(
        os.environ,
        UNSLOTH_NOTEBOOKS_TEMPLATE = str(tpl),
        UNSLOTH_NOTEBOOKS_DIR = str(dest),
        UNSLOTH_NOTEBOOKS_REPO = str(up),
        UNSLOTH_SKIP_NOTEBOOK_VIEW = "1",
        UNSLOTH_KEEP_COLAB_INTRO = "1",
    )
    if keep_removed:
        env["UNSLOTH_KEEP_REMOVED_NOTEBOOKS"] = "1"
    if refresh:
        env["UNSLOTH_NB_REFRESH_CHILD"] = "1"
    else:
        env["UNSLOTH_SKIP_NOTEBOOK_REFRESH"] = "1"
    return subprocess.run(
        ["bash", str(SYNC)],
        capture_output = True,
        text = True,
        timeout = 600,
        env = env,
        preexec_fn = _cap if cap_kib else None,
    )


def _recorded(dest: Path) -> set[str]:
    state = dest / ".unsloth_sync_state"
    if not state.exists():
        return set()
    out = set()
    for line in state.read_text(encoding = "utf-8").splitlines():
        _, _, rel = line.partition("  ")
        if rel:
            out.add(rel)
    return out


def _published(dest: Path) -> set[str]:
    return {p.name for p in dest.glob("*.ipynb")}


@needs_git
def test_no_notebook_is_published_without_a_record(tmp_path: Path):
    """The invariant. A file we wrote but could not record is the unrecoverable
    state, so it must be rolled back rather than left behind."""
    dest, tpl, up, run = _shared_capped_first_refresh(tmp_path)
    assert "could not be written" in run.stdout, (
        "the cap did not bite; this test proves nothing unless some append failed\n"
        + run.stdout
        + run.stderr
    )

    orphans = _published(dest) - _recorded(dest) - {"seed.ipynb"}
    assert not orphans, (
        f"{len(orphans)} notebook(s) on disk with no state record; the next refresh "
        f"reads them as user edits and stops updating them: {sorted(orphans)[:5]}"
    )


@needs_git
def test_the_next_refresh_recovers_everything_that_was_rolled_back(tmp_path: Path):
    """Rollback is only correct if the retry actually restores them."""
    dest, tpl, up, first = _shared_capped_first_refresh(tmp_path)
    assert "could not be written" in first.stdout, first.stdout + first.stderr
    # Strictly fewer: seed.ipynb is dropped either way, so < NOTEBOOKS + 1 was vacuous.
    assert (
        len(_published(dest)) < NOTEBOOKS
    ), "nothing was rolled back, so the retry below proves nothing"

    second = _run(tpl, dest, up, refresh = True)
    assert "kept (your edits)" in second.stdout, second.stdout
    kept = int(second.stdout.split("updated, ")[1].split(" kept")[0])
    assert kept == 0, (
        f"{kept} notebook(s) became user-owned after a disk-full refresh; they would "
        f"never be updated again\n{second.stdout}"
    )
    assert len(_recorded(dest)) == NOTEBOOKS, sorted(_recorded(dest))[:5]


def _commit_upstream_change(up: Path):
    """A second commit so the next refresh does not exit early on remote == last."""
    (up / f"nb{NOTEBOOKS + 1:09d}.ipynb").write_text("N", encoding = "utf-8")
    env = dict(os.environ, GIT_CONFIG_GLOBAL = "/dev/null", GIT_CONFIG_SYSTEM = "/dev/null")
    subprocess.run(["git", "add", "-A"], cwd = up, check = True, env = env)
    subprocess.run(
        ["git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "more"],
        cwd = up,
        check = True,
        env = env,
    )


@needs_git
def test_a_user_edited_notebook_is_never_rolled_back(tmp_path: Path):
    """The rollback must never touch a user-edited notebook; every second file is edited since a
    sparse set can miss the failing range, and a pristine one must still be removed."""
    dest, tpl, up = _shared_setup_1(tmp_path)
    assert len(_published(dest)) == NOTEBOOKS

    edited = {f"nb{i:09d}.ipynb" for i in range(2, NOTEBOOKS + 1, 2)}
    for name in edited:
        (dest / name).write_text("USER EDIT", encoding = "utf-8")
    pristine = {f"nb{i:09d}.ipynb" for i in range(1, NOTEBOOKS + 1, 2)}
    _commit_upstream_change(up)

    run = _run(tpl, dest, up, cap_kib = CAP_KIB, refresh = True)
    assert "could not be written" in run.stdout, run.stdout + run.stderr

    survivors = _published(dest)
    assert pristine - survivors, (
        "no notebook was rolled back, so this test never asked the rollback about a "
        "user edit and proves nothing"
    )
    lost = edited - survivors
    assert not lost, f"user edits destroyed by the rollback: {sorted(lost)[:5]}"
    for name in sorted(edited):
        assert (dest / name).read_text(
            encoding = "utf-8"
        ) == "USER EDIT", f"{name} was overwritten while the disk was full"


def _json_upstream(tmp_path: Path, count: int) -> Path:
    """Writes valid notebook JSON so the body-aware comparison can report SAME and the unchanged
    branch is reachable; one-byte files never reach it."""
    import json

    up = tmp_path / "up"
    up.mkdir()
    body = json.dumps(
        {
            "cells": [
                {
                    "cell_type": "code",
                    "source": ["print(1)\n"],
                    "metadata": {},
                    "outputs": [],
                    "execution_count": None,
                }
            ],
            "metadata": {},
            "nbformat": 4,
            "nbformat_minor": 5,
        }
    )
    for i in range(1, count + 1):
        (up / f"nb{i:09d}.ipynb").write_text(body, encoding = "utf-8")
    env = dict(os.environ, GIT_CONFIG_GLOBAL = "/dev/null", GIT_CONFIG_SYSTEM = "/dev/null")
    subprocess.run(["git", "init", "-q", "."], cwd = up, check = True, env = env)
    subprocess.run(["git", "add", "-A"], cwd = up, check = True, env = env)
    subprocess.run(
        ["git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "init"],
        cwd = up,
        check = True,
        env = env,
    )
    return up


@needs_git
def test_the_refresh_child_never_republishes_an_unreadable_state_as_empty(tmp_path: Path):
    """No UNSLOTH_SKIP_NOTEBOOK_REFRESH here: that flag is what hid this."""
    dest, tpl, up = _shared_setup_1(tmp_path)
    before = _recorded(dest)
    assert len(before) == NOTEBOOKS

    state = dest / ".unsloth_sync_state"
    state.chmod(0o000)
    try:
        run = _run(tpl, dest, up, refresh = True)
    finally:
        state.chmod(0o644)

    assert run.returncode == 0, run.stdout + run.stderr
    assert state.stat().st_size > 0, (
        "the refresh child published an EMPTY state over a valid one; every notebook "
        "it described is now read as a user edit and frozen\n" + run.stdout
    )
    assert _recorded(dest) == before, run.stdout
    assert "0 updated" not in run.stdout, (
        "the refresh ran with an empty LAST, which is the defect itself\n" + run.stdout
    )


@needs_git
def test_a_failed_record_on_an_unchanged_notebook_rolls_it_back(tmp_path: Path):
    """Exercises drop_unrecordable's removal, reachable only through the unchanged branch; its other
    call sites decline user edits or return early on missing files."""
    # Own build: only JSON notebooks can reach the body-aware `unchanged` branch.
    tpl, dest = _template(tmp_path), tmp_path / "dest"
    up = _json_upstream(tmp_path, NOTEBOOKS)
    dest.mkdir()
    _run(tpl, dest, up, refresh = False)
    _run(tpl, dest, up, refresh = True)
    assert len(_published(dest)) == NOTEBOOKS

    _commit_upstream_change(up)
    run = _run(tpl, dest, up, cap_kib = CAP_KIB, refresh = True)
    assert "could not be written" in run.stdout, run.stdout + run.stderr
    assert "kept (only header/footer changed upstream)" in run.stdout, run.stdout

    survivors = _published(dest)
    rolled_back = NOTEBOOKS - len([n for n in survivors if n.startswith("nb")])
    assert rolled_back > 0, (
        "no unchanged notebook was rolled back, so drop_unrecordable's removal is "
        "still unexercised\n" + run.stdout
    )
    second = _run(tpl, dest, up, refresh = True)
    assert len(_published(dest)) == NOTEBOOKS + 1, second.stdout


@needs_git
def test_keeping_a_removed_notebook_keeps_its_record_too(tmp_path: Path):
    """UNSLOTH_KEEP_REMOVED_NOTEBOOKS kept the FILE and dropped its RECORD, so the
    next refresh read it as a user edit -- and turning the option back off never
    recovered it, because by then it is no longer in the state."""
    dest, tpl, up = _shared_setup_1(tmp_path)
    victim = f"nb{1:09d}.ipynb"
    assert victim in _recorded(dest)

    (up / victim).unlink()
    env = dict(os.environ, GIT_CONFIG_GLOBAL = "/dev/null", GIT_CONFIG_SYSTEM = "/dev/null")
    subprocess.run(["git", "add", "-A"], cwd = up, check = True, env = env)
    subprocess.run(
        ["git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "del"],
        cwd = up,
        check = True,
        env = env,
    )

    run = _run(tpl, dest, up, refresh = True, keep_removed = True)
    assert (dest / victim).exists(), "the opt-out did not keep the file: " + run.stdout
    assert victim in _recorded(dest), (
        "the file was kept but its record was dropped, so the next refresh reads a "
        "notebook the user never touched as a user edit\n" + run.stdout
    )


@needs_git
def test_an_unrecorded_notebook_identical_to_upstream_is_adopted(tmp_path: Path):
    """If the rollback cannot unlink a published file (EBUSY on a single-file bind mount), its bytes
    stay unrecorded, so a later refresh must adopt a file identical to the clone."""
    dest, tpl, up = _shared_setup_1(tmp_path)

    orphan = f"nb{7:09d}.ipynb"
    state = dest / ".unsloth_sync_state"
    surviving = [
        ln
        for ln in state.read_text(encoding = "utf-8").splitlines()
        if not ln.endswith("  " + orphan)
    ]
    state.write_text("\n".join(surviving) + "\n", encoding = "utf-8")
    (dest / ".unsloth_sync_commit").unlink(missing_ok = True)
    assert orphan not in _recorded(dest)
    assert (dest / orphan).exists()

    run = _run(tpl, dest, up, refresh = True)
    assert orphan in _recorded(dest), (
        "a notebook byte-identical to the clone but missing from the state was read "
        "as a user edit, so it stays unmanaged and stops tracking upstream for "
        "good\n" + run.stdout + run.stderr
    )


@needs_git
def test_a_genuinely_edited_unrecorded_notebook_is_still_left_alone(tmp_path: Path):
    """Adoption must key on the content matching the clone exactly; an unrecorded file whose bytes
    differ is a real user edit and must be left alone."""
    dest, tpl, up = _shared_setup_1(tmp_path)

    orphan = f"nb{7:09d}.ipynb"
    state = dest / ".unsloth_sync_state"
    surviving = [
        ln
        for ln in state.read_text(encoding = "utf-8").splitlines()
        if not ln.endswith("  " + orphan)
    ]
    state.write_text("\n".join(surviving) + "\n", encoding = "utf-8")
    (dest / ".unsloth_sync_commit").unlink(missing_ok = True)
    (dest / orphan).write_text("MINE, do not touch", encoding = "utf-8")

    run = _run(tpl, dest, up, refresh = True)
    assert (dest / orphan).read_text(encoding = "utf-8") == "MINE, do not touch", (
        "an unrecorded notebook the user had edited was overwritten\n" + run.stdout + run.stderr
    )
    assert orphan not in _recorded(dest), (
        "an unrecorded notebook the user had edited was adopted into the state\n" + run.stdout
    )


def _head(up: Path) -> str:
    env = dict(os.environ, GIT_CONFIG_GLOBAL = "/dev/null", GIT_CONFIG_SYSTEM = "/dev/null")
    out = subprocess.run(
        ["git", "rev-parse", "HEAD"],
        cwd = up,
        check = True,
        env = env,
        capture_output = True,
        text = True,
    )
    return out.stdout.strip()


def _delete_upstream(up: Path, victim: str) -> None:
    (up / victim).unlink()
    env = dict(os.environ, GIT_CONFIG_GLOBAL = "/dev/null", GIT_CONFIG_SYSTEM = "/dev/null")
    subprocess.run(["git", "add", "-A"], cwd = up, check = True, env = env)
    subprocess.run(
        ["git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "del"],
        cwd = up,
        check = True,
        env = env,
    )


@needs_git
def test_a_kept_removed_notebook_survives_a_failed_state_append(tmp_path: Path):
    """Every other caller may delete a notebook it could not record, because the clone
    still holds a copy to re-publish next start. This one may not: upstream DELETED
    the file, so the copy in $DEST is the last one in existence. Dropping it destroys
    exactly what UNSLOTH_KEEP_REMOVED_NOTEBOOKS was set to preserve, and no retry
    recovers it -- not with the option on, not with it off, because by then it is in
    neither the clone nor the state."""
    dest, tpl, up = _shared_setup_1(tmp_path)
    victim = f"nb{1:09d}.ipynb"
    assert victim in _recorded(dest)
    _delete_upstream(up, victim)

    # A record is 84 B, so 41916 <= cap < 42000; 41 KiB is the only fit, letting the
    # main loop finish and failing the removal loop's extra append.
    run = _run(tpl, dest, up, cap_kib = 41, refresh = True, keep_removed = True)

    assert (dest / victim).exists(), (
        "the last surviving copy of a notebook the user asked to KEEP was deleted "
        "because its record would not fit\n" + run.stdout + run.stderr
    )
    assert not (dest / ".unsloth_sync_commit").exists() or (
        dest / ".unsloth_sync_commit"
    ).read_text(encoding = "utf-8").strip() != _head(up), (
        "the record was lost but the run still counted as a success and stamped the "
        "marker, so the next start exits early instead of retrying\n" + run.stdout
    )


def _big_template(tmp_path: Path, count: int) -> Path:
    tpl = tmp_path / "tpl"
    tpl.mkdir()
    for i in range(1, count + 1):
        (tpl / f"nb{i:09d}.ipynb").write_text("N", encoding = "utf-8")
    (tpl / ".unsloth_template_commit").write_text("a" * 40 + "\n", encoding = "utf-8")
    return tpl


@needs_git
def test_an_abandoned_restore_puts_the_tree_back(tmp_path: Path):
    """When the state rewrite after the section 1b restores is abandoned, the old state describes
    the pre-restore tree, so restored files must go back too, or every one reads as a user edit."""
    dest, tpl, up = _shared_big_synced(tmp_path)
    before = _recorded(dest)
    assert len(before) == NOTEBOOKS

    # The last 50 records straddle where appends start failing; the script records find order
    # (directory order), so on ext4 the 50 highest names can all precede the cap.
    order = [
        line.partition("  ")[2]
        for line in (dest / ".unsloth_sync_state").read_text(encoding = "utf-8").splitlines()
    ]
    victims = order[-50:]
    for name in victims:
        (dest / name).unlink()

    run = _run(tpl, dest, up, cap_kib = CAP_KIB, refresh = False)
    assert run.returncode == 0, run.stdout + run.stderr
    assert "could not be rewritten" in run.stdout, (
        "the cap did not bite in section 1b; this test proves nothing\n" + run.stdout
    )

    assert _recorded(dest) == before, "the old state must be kept intact\n" + run.stdout
    still_there = [n for n in victims if (dest / n).exists()]
    assert not still_there, (
        f"{len(still_there)} notebook(s) were restored while the state that describes "
        f"them was abandoned, so the next refresh reads them as user edits: "
        f"{still_there[:5]}"
    )

    restored = (
        int(run.stdout.split("restored ")[1].split(" ")[0]) if "restored " in run.stdout else 0
    )
    assert restored > 0, (
        "no notebook was restored before an append failed, so the victims do not straddle "
        "the cap and the per-iteration guard is not exercised\n" + run.stdout
    )
    assert restored < len(victims), (
        f"section 1b restored all {restored} notebooks after deciding the state could "
        f"not be written; the guard is evaluated once instead of per iteration"
    )


@needs_git
def test_a_lost_merge_record_does_not_publish_the_short_state(tmp_path: Path):
    """A lost merge record must abandon the staged state, since nothing can re-derive upstream-only
    notebooks; a failed copy still publishes because the next boot re-walks the template."""
    dest, tpl, up = _shared_big_synced(tmp_path)

    state = dest / ".unsloth_sync_state"
    extra = [f"up_only_{i:04d}.ipynb" for i in range(100)]
    import hashlib

    with state.open("a", encoding = "utf-8") as f:
        for name in extra:
            (dest / name).write_text("U", encoding = "utf-8")
            f.write(hashlib.sha256(b"U").hexdigest() + "  " + name + "\n")
    (dest / ".unsloth_sync_partial").write_text("", encoding = "utf-8")

    run = _run(tpl, dest, up, cap_kib = CAP_KIB + 2, refresh = False)
    assert run.returncode == 0, run.stdout + run.stderr

    survived = _recorded(dest)
    lost = [n for n in extra if n not in survived]
    assert not lost, (
        f"{len(lost)} upstream-only record(s) were dropped and the short state was "
        f"published anyway; nothing can re-derive them: {lost[:5]}\n" + run.stdout
    )