Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

15.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""MLX context fitting: the orchestrator preflight and the tool loop's per-turn refit."""

from unittest.mock import Mock

import pytest

import core.inference.tools as tools_mod
from core.inference import llama_cpp
from core.inference.context_window import retrieval_budget
from core.inference.mcp_images import count_image_parts
from core.inference.orchestrator import InferenceOrchestrator

INSTRUCTION = "Standing instruction for the rest of this task: always answer in a markdown table."
SEARCH = {"type": "function", "function": {"name": "search_conversation"}}
WEB = {"type": "function", "function": {"name": "web_search"}}


def _count(messages, *_args, **_kwargs):
    return sum(max(1, len(str(m.get("content", ""))) // 4) for m in messages), "mlx-test"


def _backend():
    backend = InferenceOrchestrator()
    backend.active_model_name = "mlx-test"
    backend.models["mlx-test"] = {"is_mlx": True, "context_length": 1200}
    backend.count_chat_tokens = _count
    return backend


def _thread(latest = "continue"):
    messages = [{"role": "user", "content": INSTRUCTION}, {"role": "assistant", "content": "OK."}]
    for index in range(8):
        messages.append({"role": "user", "content": f"Section {index}. " + "x" * 600})
        messages.append({"role": "assistant", "content": f"Section {index} noted."})
    return messages + [{"role": "user", "content": latest}]


def _fit(backend, messages, **kwargs):
    kwargs.setdefault("context_overflow", "truncate_oldest")
    return backend.compact_chat_context(
        messages, system_prompt = "you are helpful", max_tokens = 200, **kwargs
    )


@pytest.fixture
def archive_calls(monkeypatch):
    """A saved thread with a healthy archive, so only the request's own gates decide."""
    calls = []
    idle = {"events": [], "counts": {}, "recalled": False, "anchored": []}

    def _archive(conversation, _before, **kwargs):
        calls.append({**kwargs, "branch_messages": list(kwargs["branch_messages"])})
        return {"conversation": conversation, **idle}

    def _can_reset(
        thread_id,
        supports_tools,
        *,
        tools_withheld = False,
    ):
        return bool(thread_id and supports_tools and not tools_withheld)

    monkeypatch.setattr(llama_cpp, "_archive_and_recall", _archive)
    monkeypatch.setattr(llama_cpp, "_can_reset_epoch", _can_reset)
    monkeypatch.setattr(llama_cpp, "_archive_is_degraded", lambda: False)
    return calls


# A 4200-character question is over the prompt budget even alone: a rescued refusal.
@pytest.mark.parametrize("question_chars, fits", [(8, True), (4200, False)])
def test_an_overflowing_prompt_drops_its_oldest_turns_and_reports_the_boundary(
    archive_calls, question_chars, fits
):
    messages = _thread("q" * question_chars)

    result = _fit(_backend(), messages, thread_id = "thread-1")

    [truncation] = result["events"]
    assert truncation["type"] == "context_truncated"
    assert truncation["fits"] is fits is (truncation["prompt_tokens_after"] <= 1000)
    assert truncation["boundary_messages"] == truncation["dropped_messages"] > 0
    assert archive_calls[-1]["recall_done"] is not fits
    assert result["system_prompt"] == ""
    assert result["messages"][0] == {"role": "system", "content": "you are helpful"}
    assert result["messages"][-1] is messages[-1]


def test_chat_stop_reaches_the_archive_embedding_pass(archive_calls):
    """Stop must be able to interrupt the archive's embedding before inference (#14519)."""
    import threading

    stop = threading.Event()
    _fit(_backend(), _thread(), thread_id = "thread-1", cancel_event = stop)

    cancel_check = archive_calls[-1]["cancel_check"]
    assert cancel_check() is False
    stop.set()
    assert cancel_check() is True


@pytest.mark.parametrize(
    "case", ["not_opted_in", "not_mlx", "media", "resumed_reply", "resumed_thought", "no_count"]
)
def test_a_prompt_the_fit_must_not_touch_is_returned_unchanged(case):
    backend, messages, kwargs = _backend(), _thread(), {}
    if case == "not_opted_in":
        kwargs["context_overflow"] = "error"
    elif case == "not_mlx":
        backend.models["mlx-test"]["is_mlx"] = False
    elif case == "media":
        messages[-1] = {"role": "user", "content": [{"type": "image_url", "image_url": {}}]}
    elif case.startswith("resumed"):
        partial = {"content": "so far"} if case == "resumed_reply" else {"reasoning_content": "hm"}
        messages.append({"role": "assistant", "content": "", **partial})
        kwargs["continue_final_message"] = True
    else:
        backend.count_chat_tokens = Mock(side_effect = RuntimeError("generation in progress"))

    result = _fit(backend, messages, **kwargs)

    assert result["messages"] == messages
    assert result["system_prompt"] == "you are helpful"
    assert result["events"] == [] and "boundary_applied" not in result


PICTURE = "cGl4ZWxz"


def _pictured(text):
    return {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": text}]}


def _count_pictures(
    messages,
    *_args,
    images = (),
    **_kwargs,
):
    assert len(images) == count_image_parts(messages, "image")
    return _count(messages)[0] + 300 * len(images), "mlx-test"


@pytest.mark.parametrize("owner, dropped", [(2, [0]), (-1, [])], ids = ["old_turn", "newest_turn"])
def test_a_picture_is_priced_into_the_fit_and_leaves_with_its_turn(archive_calls, owner, dropped):
    backend, messages = _backend(), _thread()
    backend.count_chat_tokens = _count_pictures
    messages[owner] = _pictured(messages[owner]["content"])

    result = _fit(backend, messages, thread_id = "thread-1", images = [PICTURE])

    [truncation] = result["events"]
    kept = count_image_parts(result["messages"], "image")
    assert truncation["fits"] and truncation["dropped_messages"] > 0
    assert result["dropped_images"] == dropped and kept == 1 - len(dropped)
    assert truncation["prompt_tokens_after"] == _count(result["messages"])[0] + 300 * kept


def test_a_picture_without_its_marker_leaves_the_prompt_unfitted():
    messages = _thread()

    result = _fit(_backend(), messages, images = [PICTURE])

    assert result["messages"] == messages and result["events"] == []


def test_the_tool_loop_stops_sending_a_picture_once_its_turn_is_dropped(monkeypatch, archive_calls):
    backend, sent = _backend(), []
    backend.models["mlx-test"]["is_vision"] = True
    backend.count_chat_tokens = _count_pictures

    def _generate(**call):
        sent.append((count_image_parts(call["messages"], "image"), call["images"]))
        yield CALL % len(sent) if len(sent) < 3 else "done"

    monkeypatch.setattr(backend, "generate_chat_response", _generate)
    monkeypatch.setattr(tools_mod, "execute_tool", lambda *_a, **_k: "r" * 2400)
    events = list(
        backend.generate_chat_completion_with_tools(
            messages = [_pictured("o" * 400), ANSWERED, QUESTION],
            tools = [WEB],
            images = [PICTURE],
            caller_image_indexes = (0,),
            max_tokens = 200,
            context_overflow = "truncate_oldest",
            thread_id = "thread-1",
        )
    )

    assert any(event.get("dropped_messages") for event in events)
    assert sent[0] == (1, [PICTURE]) and sent[-1] == (0, None)


def test_a_request_resets_only_where_search_can_follow_and_names_only_an_offered_tool(
    archive_calls, monkeypatch
):
    def fit(messages = None, **kwargs):
        result = _fit(
            _backend(),
            messages or _thread(),
            context_policy = "checkpoint",
            thread_id = "thread-1",
            **kwargs,
        )
        return result["events"][-1], result["messages"][0]["content"], archive_calls[-1]

    truncation, system, archived = fit(tools = [SEARCH])
    assert not truncation.get("checkpoint") and "carried_forward" not in system
    assert archived["style"] == "inline"

    # A plain request has no catalogue to miss the tool from: the route's answer alone decides.
    with monkeypatch.context() as patch:
        patch.setattr(llama_cpp, "_memory_tool_withheld", lambda thread_id, tools: True)
        truncation, system, archived = fit(recall_reachable = True)
        assert truncation["checkpoint_started"] is True and "carried_forward" in system
        assert "search_conversation tool" not in system and archived["style"] == "inline"
        assert not fit(tool_loop = True, tools = [WEB])[0].get("checkpoint")

    truncation, system, archived = fit(tool_loop = True, tools = [WEB])
    assert truncation["checkpoint_started"] is True and "carried_forward" in system
    assert "search_conversation tool" not in system
    assert archived["style"] == "inline"

    truncation, system, archived = fit(tool_loop = True, tools = [WEB, SEARCH])
    assert "search_conversation tool" in system
    assert archived["style"] == "tool"
    halved = retrieval_budget(1200, 200, truncation["prompt_tokens_after"], reply_returns = True)
    assert archived["recall_budget_tokens"] == halved
    assert halved < retrieval_budget(1200, 200, truncation["prompt_tokens_after"])

    turns = [{"role": "user", "content": "x" * size} for size in (3200, 800, 8)]
    truncation, _, archived = fit(messages = turns, tool_loop = True, recall_reachable = True)
    after = truncation["prompt_tokens_after"]
    whole = retrieval_budget(1200, 200, after)
    assert not truncation.get("checkpoint") and archived["recall_budget_tokens"] == whole
    assert whole > retrieval_budget(1200, 200, after, reply_returns = True)


QUESTION = {"role": "user", "content": "What changed in release 4?"}
ANSWERED = {"role": "assistant", "content": "OK."}
CALL = '<tool_call>{"name":"web_search","arguments":{"query":"release %d"}}</tool_call>'


def _run_loop(monkeypatch, messages, replies, result_chars, **kwargs):
    backend, turns, prompts, branches = _backend(), iter(replies), [], []

    def _generate(**call):
        prompts.append(list(call["messages"]))
        yield next(turns)

    def _execute(*_args, conversation_branch, **_kwargs):
        branches.append(list(conversation_branch))
        return f"result {len(branches)} " + "r" * result_chars

    monkeypatch.setattr(backend, "generate_chat_response", _generate)
    monkeypatch.setattr(tools_mod, "execute_tool", _execute)
    events = backend.generate_chat_completion_with_tools(
        messages = messages,
        tools = [WEB],
        max_tokens = 200,
        nudge_tool_calls = True,
        context_overflow = "truncate_oldest",
        **{"thread_id": "thread-1", **kwargs},
    )
    truncated = [event for event in events if event.get("type") == "context_truncated"]
    return truncated, prompts, branches


@pytest.mark.parametrize("policy", ["checkpoint", "rolling"])
@pytest.mark.parametrize("old_result_chars, fits", [(3900, 3), (0, 2)])
def test_the_tool_loop_refits_every_turn_and_keeps_the_question_past_a_reprompt(
    monkeypatch, archive_calls, policy, old_result_chars, fits
):
    old = {"role": "user", "content": "o" * 400}
    old_exchange = [
        {"role": "assistant", "content": "", "tool_calls": [{"id": "0"}]},
        {"role": "tool", "content": "h" * old_result_chars},
    ]
    truncated, prompts, branches = _run_loop(
        monkeypatch,
        [old, *(old_exchange if old_result_chars else []), ANSWERED, QUESTION],
        ["Let me search the web for that.", CALL % 1, CALL % 2, CALL % 3, "done"],
        2400,
        max_tool_iterations = 3,
        context_policy = policy,
    )

    final = prompts[-1]
    assert final[-1]["role"] == "user" and final[-1] is not QUESTION
    assert any(m is QUESTION for m in final) and not any(m is old for m in final)
    assert [m["content"][:8] for m in final if m.get("role") == "tool"] == ["result 3"]
    assert sum(m["content"].startswith("result") for m in branches[2] if m["role"] == "tool") == 2
    last_fit = archive_calls[-1]
    ran = [m for m in last_fit["branch_messages"] if m["role"] == "tool"][-3:]
    assert [m["content"][:8] for m in ran] == ["result 1", "result 2", "result 3"]
    assert [m for m in last_fit["branch_messages"] if m["role"] == "user"] == [old, QUESTION]
    # The recall query comes from the client's turns, never from a re-prompt the loop added (#16187).
    assert [m for m in last_fit["query_messages"] if m["role"] == "user"][-1] is QUESTION
    assert len(truncated) == fits and all(event["fits"] for event in truncated)
    assert bool(truncated[0].get("checkpoint_started")) is (policy == "checkpoint")
    assert last_fit["thread_id"] == "thread-1"


# A result overflowing by less than one old turn, against a boundary two turns deep; or neither.
@pytest.mark.parametrize("result_chars, saved", [(3300, 4), (8, 0)])
def test_a_turn_that_is_not_fitted_leaves_the_saved_boundary_to_the_next(
    monkeypatch, archive_calls, result_chars, saved
):
    asked = []

    def _saved_boundary(*_args, **_kwargs):
        asked.append(1)
        return saved, False

    monkeypatch.setattr(llama_cpp, "_sticky_compaction_state", _saved_boundary)
    first, second = ({"role": "user", "content": letter * 400} for letter in "ab")
    resumed = {"role": "assistant", "content": "Let me"}
    _, prompts, _ = _run_loop(
        monkeypatch,
        [first, ANSWERED, second, ANSWERED, QUESTION, resumed],
        [CALL % 1, CALL % 2, "done"],
        result_chars,
        continue_final_message = True,
        context_policy = "rolling",
    )

    assert any(m is first for m in prompts[0])
    assert any(m is second for m in prompts[1]) == (not saved)
    assert len(asked) == 1


# The cap's budget notice, then no-op notices: once, twice, and twice with model text between.
@pytest.mark.parametrize(
    "replies, chars, fits, dropped",
    [
        ([CALL % 1], 3900, False, 0),
        ([CALL % 1] * 2, 3900, False, 0),
        ([CALL % 1] * 3, 3550, True, 1),
        ([CALL % 1] * 2 + ["I'll check that again. " + CALL % 1], 3550, True, 2),
    ],
)
def test_the_only_tool_result_is_not_evicted_to_make_room_for_what_the_loop_says_next(
    monkeypatch, archive_calls, replies, chars, fits, dropped
):
    [truncation], prompts, _ = _run_loop(
        monkeypatch,
        [QUESTION],
        [*replies, "done"],
        chars,
        max_tool_iterations = 5 if len(replies) > 1 else 1,
    )

    # Dropping the result would fit but leave nothing to answer from.
    assert truncation["fits"] is fits and truncation["dropped_messages"] == dropped
    assert [m["role"] for m in prompts[-1]].count("tool") == 1


@pytest.fixture
def fresh_cut_memory():
    from core.inference import prefix_boundary

    prefix_boundary.clear()
    yield
    prefix_boundary.clear()


def test_a_threadless_tool_loop_keeps_its_cut_between_rounds(
    monkeypatch, archive_calls, fresh_cut_memory
):
    rounds = 16
    _, prompts, _ = _run_loop(
        monkeypatch,
        [QUESTION],
        [CALL % i for i in range(rounds)] + ["done"],
        400,
        max_tool_iterations = rounds,
        context_policy = "rolling",
        thread_id = None,
    )

    heads = [str(prompt[:3]) for prompt in prompts]
    moves = sum(before != after for before, after in zip(heads, heads[1:]))
    assert moves < len(prompts) // 2, moves