Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

49.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import asyncio
import json
import threading
import time
from datetime import date, datetime, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock

import pytest
from starlette.requests import Request

from core.inference import chat_generation_runs, llama_keepwarm
from core.inference.chat_generation_runs import (
    _EVENT_FLUSH_SECONDS,
    ChatGenerationSupervisor,
)
from models.inference import ChatCompletionRequest
from routes import chat_generation_runs as run_routes
from routes import inference
from state import active_generations
from storage import chat_generation_runs_db as runs_db
from storage import studio_db
from utils.current_date_prompt_settings import _request_local_date


@pytest.fixture
def durable_run(request):
    engine = getattr(request, "param", "gguf")
    model = "local.gguf" if engine == "gguf" else "local.safetensors"
    studio_db.upsert_chat_thread(
        {"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": model, "createdAt": 1}
    )
    studio_db.upsert_chat_message(
        {
            "id": "user-1",
            "threadId": "thread-1",
            "role": "user",
            "content": [{"type": "text", "text": "Hello"}],
            "createdAt": 2,
        }
    )
    run, _created = runs_db.create_run(
        run_id = "run-1",
        owner_subject = "alice",
        thread_id = "thread-1",
        user_message_id = "user-1",
        assistant_message_id = "assistant-1",
        request_payload = {
            "model": model,
            "messages": [{"role": "user", "content": "Hello"}],
            "stream": True,
            "cancel_id": "run-1",
            "thread_id": "thread-1",
            "generation_run_id": "run-1",
        },
    )
    active_generations.reset_for_tests()
    yield run
    active_generations.reset_for_tests()


def _create_payload(content = "Hello"):
    return run_routes.CreateChatGenerationRun(
        runId = "run-1",
        threadId = "thread-1",
        userMessageId = "user-1",
        assistantMessageId = "assistant-1",
        requestPayload = {"model": "local", "messages": [{"role": "user", "content": content}]},
    )


def _route_request(supervisor):
    return SimpleNamespace(
        app = SimpleNamespace(state = SimpleNamespace(chat_generation_supervisor = supervisor)),
        headers = {},
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("scope", ["off", "full"])
@pytest.mark.parametrize(
    "caller,expected",
    [("session", False), ("keyless", True), ("api-key", True), ("workflow", False)],
)
async def test_durable_producer_preserves_monitor_origin(monkeypatch, scope, caller, expected):
    from auth import policy, storage
    from utils import keyless_api_access as keyless

    studio_db.upsert_chat_thread(
        {"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
    )
    studio_db.upsert_chat_message(
        {"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
    )
    app = SimpleNamespace(state = SimpleNamespace(bind_host = "127.0.0.1"))
    headers = (
        []
        if caller == "keyless"
        else [
            (
                b"authorization",
                b"Bearer session-jwt" if caller == "session" else b"Bearer sk-unsloth-test",
            )
        ]
    )
    request = Request(
        {
            "type": "http",
            "method": "POST",
            "path": "/api/inference/chat-runs",
            "headers": headers,
            "app": app,
            "client": ("127.0.0.1", 5000),
            "server": ("127.0.0.1", 8000),
        }
    )
    keyless.mark_keyless_admission(request, caller == "keyless")
    monkeypatch.setattr(policy, "installation_has_managed_accounts", lambda: False)
    monkeypatch.setattr(storage, "is_internal_api_key", lambda _token: caller == "workflow")
    run = await run_routes.create_chat_generation_run(_create_payload(), request, "alice")
    assert run["requestPayload"][runs_db.API_MONITOR_ORIGIN_FIELD] is expected
    request.state.api_monitor_via_api_key = not expected
    retry = await run_routes.create_chat_generation_run(_create_payload(), request, "alice")
    assert retry["created"] is False
    assert retry["requestPayload"][runs_db.API_MONITOR_ORIGIN_FIELD] is expected
    monkeypatch.setattr(keyless, "get_keyless_api_access_scope", lambda: scope)
    observed = []

    async def body():
        yield "data: [DONE]\n\n"

    async def fake(payload, background, _subject, *, cancel_on_disconnect):
        observed.append(
            (
                inference._request_used_api_key(background),
                runs_db.API_MONITOR_ORIGIN_FIELD in payload.model_extra,
                "authorization" in background.headers,
            )
        )
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    await ChatGenerationSupervisor(app)._produce("run-1")
    assert observed == [(expected, False, False)]


def test_monitor_origin_is_not_part_of_retry_identity():
    args = dict(thread_id = "t", user_message_id = "u", assistant_message_id = "a")
    payload = {"model": "local"}
    _, original = runs_db.canonical_request(**args, request_payload = payload)
    for origin in (False, True):
        encoded, identity = runs_db.canonical_request(
            **args, request_payload = {**payload, runs_db.API_MONITOR_ORIGIN_FIELD: origin}
        )
        assert identity == original
        assert json.loads(encoded)[runs_db.API_MONITOR_ORIGIN_FIELD] is origin


def test_client_cannot_supply_monitor_origin():
    from fastapi import HTTPException

    payload = _create_payload()
    payload.requestPayload[runs_db.API_MONITOR_ORIGIN_FIELD] = False
    with pytest.raises(HTTPException) as exc:
        run_routes._sanitize_request(payload)
    assert exc.value.status_code == 400


@pytest.mark.asyncio
async def test_public_chat_wrapper_keeps_cancel_on_disconnect(monkeypatch):
    observed = []

    async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
        observed.append(cancel_on_disconnect)
        return "response"

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    payload = ChatCompletionRequest(
        model = "local",
        messages = [{"role": "user", "content": "Hello"}],
    )
    assert await inference.openai_chat_completions(payload, object(), "alice") == "response"
    assert observed == [True]


@pytest.mark.parametrize(
    "is_mlx,durable_run,generation_run_id,completion_tokens,expected",
    [
        (True, True, "run-1", 8, "length"),
        (True, True, "run-1", 7, "stop"),
        (True, False, "run-1", 8, "stop"),
        (False, True, "run-1", 8, "stop"),
    ],
)
def test_only_durable_mlx_normalizes_stop_at_token_cap(
    is_mlx, durable_run, generation_run_id, completion_tokens, expected
):
    payload = SimpleNamespace(
        generation_run_id = generation_run_id,
        max_tokens = 8,
        max_completion_tokens = None,
    )
    stats = {"usage": {"completion_tokens": completion_tokens}}
    assert (
        inference._safetensors_finish_reason(stats, payload, is_mlx = is_mlx, durable_run = durable_run)
        == expected
    )


@pytest.mark.asyncio
async def test_create_route_schedules_producer_on_request_loop(monkeypatch):
    started = []
    supervisor = SimpleNamespace(
        start = lambda run_id, **identity: started.append((run_id, identity))
    )
    request = _route_request(supervisor)

    def create_run(**_kwargs):
        assert started == []
        return (
            {
                "id": "run-1",
                "status": "queued",
                "threadId": "thread-1",
                "requestPayload": {"model": "local"},
            },
            True,
        )

    monkeypatch.setattr(run_routes.db, "create_run", create_run)
    response = await run_routes.create_chat_generation_run(_create_payload(), request, "alice")
    assert response["created"] is True
    assert started == [("run-1", {"thread_id": "thread-1", "model": "local", "images": {}})]


@pytest.mark.asyncio
async def test_terminal_idempotent_create_does_not_reserve_generation(monkeypatch):
    terminal = {
        "id": "run-1",
        "status": "completed",
        "threadId": "thread-1",
        "requestPayload": {"model": "local"},
    }
    supervisor = SimpleNamespace(
        start = lambda *_args, **_kwargs: pytest.fail("terminal run must not start"),
    )
    request = _route_request(supervisor)
    monkeypatch.setattr(run_routes.db, "create_run", lambda **_kwargs: (terminal, False))
    response = await run_routes.create_chat_generation_run(_create_payload("Hi"), request, "alice")
    assert response["created"] is False


@pytest.mark.asyncio
async def test_background_producer_persists_chunks_and_completes(durable_run, monkeypatch):
    observed = []
    leaked = []
    chunks = [
        {"choices": [{"delta": {"content": "Hello"}, "finish_reason": None}]},
        {"choices": [{"delta": {}, "finish_reason": "stop"}]},
    ]

    async def body():
        for chunk in chunks:
            yield f"data: {json.dumps(chunk)}\n\n"
        yield "data: [DONE]\n\n"

    async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
        observed.append(cancel_on_disconnect)
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    loop = asyncio.get_running_loop()
    previous_handler = loop.get_exception_handler()
    loop.set_exception_handler(lambda _loop, context: leaked.append(context))
    try:
        await supervisor._produce("run-1")
        await asyncio.sleep(0)
    finally:
        loop.set_exception_handler(previous_handler)
    run = runs_db.get_run("run-1", "alice")
    assert leaked == []
    assert observed == [False]
    assert (run["status"], run["finishReason"]) == ("completed", "stop")
    assert [
        event["payload"] for event in runs_db.list_events("run-1") if event["type"] == "chunk"
    ] == chunks


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "finish, cut, expected",
    [
        ("stop", True, {"reason": "quote_cut"}),
        ("stop", False, None),
        ("length", True, {"reason": "length"}),
    ],
    ids = ["cut", "clean-stop", "length-wins"],
)
async def test_producer_stamps_a_reported_quote_cut(
    durable_run, monkeypatch, finish, cut, expected
):
    """The producer must persist the reason before the client can settle with it."""
    chunks = [
        {"choices": [{"delta": {"reasoning_content": "The tokens are `"}, "finish_reason": None}]},
        *([{"choices": [], "quote_cut": True}] if cut else []),
        {"choices": [{"delta": {}, "finish_reason": finish}]},
    ]

    async def body():
        for chunk in chunks:
            yield f"data: {json.dumps(chunk)}\n\n"
        yield "data: [DONE]\n\n"

    async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    await supervisor._produce("run-1")

    run = runs_db.get_run("run-1", "alice")
    assert (run["status"], run["finishReason"]) == ("completed", finish)
    message = studio_db.get_chat_message("thread-1", "assistant-1")
    assert message["metadata"].get("incomplete") == expected


@pytest.mark.asyncio
async def test_producer_dates_the_prompt_in_the_browser_timezone(monkeypatch):
    studio_db.upsert_chat_thread(
        {"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
    )
    studio_db.upsert_chat_message(
        {"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
    )
    browser = Request(
        {
            "type": "http",
            "method": "POST",
            "path": "/api/inference/chat-runs",
            "query_string": b"",
            "headers": [
                (b"x-unsloth-timezone", b"Pacific/Kiritimati"),
                (b"x-unsloth-timezone-offset-minutes", b"-840"),
            ],
            "app": SimpleNamespace(state = SimpleNamespace()),
        }
    )
    await run_routes.create_chat_generation_run(_create_payload(), browser, "alice")
    instant = datetime(2026, 9, 24, 10, 5, tzinfo = timezone.utc)
    observed = []

    async def body():
        yield "data: [DONE]\n\n"

    async def fake(payload, request, _subject, *, cancel_on_disconnect):
        observed.append((sorted(payload.model_extra), _request_local_date(request, instant)))
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    await supervisor._produce("run-1")
    assert observed == [(["generation_run_id"], date(2026, 9, 25))]


@pytest.mark.asyncio
async def test_a_follow_up_reaches_the_model_with_an_image_the_run_never_stored(monkeypatch):
    studio_db.upsert_chat_thread(
        {"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
    )
    studio_db.upsert_chat_message(
        {"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
    )
    url = "data:image/png;base64,iVBORw0KGgo="
    image_turn = [
        {"type": "text", "text": "what?"},
        {"type": "image_url", "image_url": {"url": url}},
    ]
    payload = _create_payload("and the colour?")
    payload.requestPayload["messages"] = [
        {"role": "user", "content": image_turn},
        {"role": "assistant", "content": "A square."},
        *payload.requestPayload["messages"],
    ]
    observed = []

    async def body():
        yield "data: [DONE]\n\n"

    async def fake(payload, _request, _subject, *, cancel_on_disconnect):
        observed.append(payload.messages[0].content[1].image_url.url)
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    await run_routes.create_chat_generation_run(payload, _route_request(supervisor), "alice")
    await supervisor._tasks["run-1"]
    assert observed == [url]
    assert url not in json.dumps(runs_db.get_run("run-1")["requestPayload"])
    assert supervisor._detached_images == {}


@pytest.mark.asyncio
async def test_a_create_retried_across_a_dst_change_returns_the_committed_run():
    studio_db.upsert_chat_thread(
        {"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
    )
    studio_db.upsert_chat_message(
        {"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
    )

    def browser(offset):
        request = _route_request(None)
        request.headers = {
            "x-unsloth-timezone": "America/Los_Angeles",
            "x-unsloth-timezone-offset-minutes": offset,
        }
        return request

    first = await run_routes.create_chat_generation_run(_create_payload(), browser("420"), "alice")
    retry = await run_routes.create_chat_generation_run(_create_payload(), browser("480"), "alice")
    assert (first["created"], retry["created"]) == (True, False)
    assert retry["requestPayload"]["timezone_headers"]["x-unsloth-timezone-offset-minutes"] == "420"


@pytest.mark.asyncio
async def test_a_prefill_reporting_only_progress_renews_the_lease(durable_run, monkeypatch):
    """A 250K prefill outruns the 1200s lease before its first token and only progress writes renew
    it, so dropping content-less chunks would reap a healthy prefill."""
    released = asyncio.Event()
    sampled: dict = {}

    def _progress(processed):
        return {
            "choices": [{"delta": {"role": "assistant", "content": None}, "finish_reason": None}],
            "prompt_progress": {
                "total": 250000,
                "cache": 0,
                "processed": processed,
                "time_ms": processed,
            },
        }

    async def body():
        for processed in (1024, 8192, 65536):
            yield f"data: {json.dumps(_progress(processed))}\n\n"
        # Polled, not slept: a sleep sized against the flush timer flakes under load.
        _deadline = time.monotonic() + 10.0
        while time.monotonic() < _deadline:
            sampled["events"] = [
                e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"
            ]
            if len(sampled["events"]) >= 3:
                break
            await asyncio.sleep(0.01)
        sampled["progress"] = runs_db.get_progress("run-1")
        released.set()
        yield f"data: {json.dumps({'choices': [{'delta': {'content': 'Hi'}, 'finish_reason': 'stop'}]})}\n\n"
        yield "data: [DONE]\n\n"

    async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    await supervisor._produce("run-1")
    assert released.is_set()
    assert len(sampled["events"]) == 3, sampled["events"]
    assert all("prompt_progress" in e for e in sampled["events"])
    assert sampled["progress"][1] == 3, sampled["progress"]
    assert runs_db.get_run("run-1", "alice")["status"] == "completed"


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "comment, reaped",
    [
        ("_OPENAI_TOOL_HEARTBEAT_SSE", []),
        ("_OPENAI_PREFILL_PROGRESS_SSE", []),
        ("_OPENAI_PASSTHROUGH_SSE_KEEPALIVE", ["run-1"]),
    ],
)
async def test_progress_comments_hold_the_lease_but_a_stalled_stream_does_not(
    durable_run, monkeypatch, comment, reaped
):
    comment = getattr(inference, comment)
    now = {"ms": runs_db.now_ms()}
    monkeypatch.setattr(runs_db, "now_ms", lambda: now["ms"])
    sampled = {}

    async def body():
        now["ms"] += 21 * 60_000
        yield comment
        yield comment
        sampled["reaped"] = runs_db.reconcile_runs(stale_after_ms = 1_200_000)
        yield "data: [DONE]\n\n"

    async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
        monkeypatch.setattr(chat_generation_runs, "_renew_interval_seconds", lambda: 0.0)
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    await supervisor._produce("run-1")
    assert sampled["reaped"] == reaped


@pytest.mark.asyncio
@pytest.mark.parametrize("tools", [False, True], ids = ["chat", "tool-loop"])
@pytest.mark.parametrize("reports_progress, reaped", [(True, []), (False, ["run-1"])])
async def test_native_gguf_prefill_progress_holds_the_lease(
    durable_run, monkeypatch, tools, reports_progress, reaped
):
    """Native GGUF generators are silent until the first token; advancing progress must still renew the lease."""
    from core.inference.llama_cpp import _report_live_llama_timings

    entered, release = threading.Event(), threading.Event()

    def _prefill(perf_callback):
        entered.set()
        processed = 0
        while not release.wait(0.005):
            if reports_progress:
                processed += 512
                _report_live_llama_timings(
                    perf_callback,
                    {"choices": [], "prompt_progress": {"total": 10**6, "processed": processed}},
                )

    def generate(*, perf_callback = None, **_kwargs):
        _prefill(perf_callback)
        yield "Hi"
        yield {"type": "metadata", "finish_reason": "stop"}

    def generate_with_tools(*, perf_callback = None, **_kwargs):
        _prefill(perf_callback)
        yield {"type": "content", "text": "Hi"}
        yield {"type": "metadata", "finish_reason": "stop"}

    llama = SimpleNamespace(
        is_loaded = True,
        model_identifier = "local.gguf",
        base_url = "http://llama.test",
        effective_parallel_slots = 1,
        supports_tools = tools,
        supports_tool_passthrough = tools,
        is_vision = False,
        _is_audio = False,
        context_length = None,
        generate_chat_completion = generate,
        generate_chat_completion_with_tools = generate_with_tools,
        release_idle_chat_slot = lambda *_a, **_k: False,
        count_chat_tokens = lambda *_a, **_k: 16,
        _request_reasoning_kwargs = lambda *_a, **_k: None,
    )
    monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: llama)
    monkeypatch.setattr(inference, "_automatic_model_load_may_run", lambda: False)

    async def no_switch(*_a, **_k):
        return None

    monkeypatch.setattr(inference, "_maybe_auto_switch_model", no_switch)
    monkeypatch.setattr(inference, "_LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S", 0.02)
    if not tools:
        monkeypatch.setattr(inference, "_effective_enable_tools", lambda _payload: False)
    now = {"ms": runs_db.now_ms()}
    monkeypatch.setattr(runs_db, "now_ms", lambda: now["ms"])
    real_produce = inference.produce_openai_chat_completions
    sampled = {"comments": set()}

    async def wrapped(payload, request, subject, *, cancel_on_disconnect):
        if tools:
            payload.enable_tools = True
            payload.enabled_tools = ["web_search"]
        response = await real_produce(
            payload, request, subject, cancel_on_disconnect = cancel_on_disconnect
        )
        inner = response.body_iterator
        monkeypatch.setattr(chat_generation_runs, "_renew_interval_seconds", lambda: 0.0)

        async def body():
            comments = 0
            chunks = 0
            async for raw in inner:
                text = raw.decode() if isinstance(raw, bytes) else str(raw)
                chunks += text.count("data: {")
                if text.startswith(":") and entered.is_set():
                    comments += 1
                    sampled["comments"].add(text.strip())
                    if comments == 2:
                        # Commits run beside the stream now, so let the chunks read so far land before the clock
                        # jumps; a slow runner would otherwise stamp them as progress made after it.
                        await _await_chunk_payloads("run-1", chunks, 10.0)
                        now["ms"] += 21 * 60_000
                    elif comments == 5 and "reaped" not in sampled:
                        sampled["reaped"] = runs_db.reconcile_runs(stale_after_ms = 1_200_000)
                        release.set()
                yield raw

        response.body_iterator = body()
        return response

    monkeypatch.setattr(inference, "produce_openai_chat_completions", wrapped)
    try:
        supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
        await asyncio.wait_for(supervisor._produce("run-1"), 30)
    finally:
        release.set()
    assert sampled["reaped"] == reaped
    expected = ": prefill-progress" if reports_progress else ": keep-alive"
    assert expected in sampled["comments"], sampled["comments"]
    if not reports_progress:
        assert ": prefill-progress" not in sampled["comments"]


def test_prefill_signal_counts_only_moving_progress():
    from core.inference.llama_cpp import _perf_callback_wants_timings

    forwarded = []
    bare = inference._PrefillProgressSignal(None)
    assert _perf_callback_wants_timings(bare) is False
    assert bare.needs_phase is False
    assert bare.advanced() is False
    bare({"prompt_progress": {"processed": 10}})
    bare({"prompt_progress": {"processed": 10}})
    assert bare.advanced() is True
    assert bare.advanced() is False
    bare({"prompt_progress": {"processed": 2}})
    assert bare.advanced() is True
    bare({"timings": {"predicted_n": 1}})
    assert bare.advanced() is False

    def monitor(sample):
        forwarded.append(sample)

    monitor.needs_phase = False
    wrapped = inference._PrefillProgressSignal(monitor)
    assert _perf_callback_wants_timings(wrapped) is True
    assert wrapped.needs_phase is False
    wrapped({"prompt_progress": {"processed": 1}})
    assert forwarded == [{"prompt_progress": {"processed": 1}}]


async def _subscriber_sequences(after = 0):
    response = await run_routes.chat_generation_events(
        "run-1",
        SimpleNamespace(is_disconnected = AsyncMock(return_value = True)),
        after = after,
        last_event_id = None,
        current_subject = "alice",
    )
    raw = ""
    async for part in response.body_iterator:
        raw += part.decode() if isinstance(part, bytes) else part
    return [int(line[4:]) for line in raw.splitlines() if line.startswith("id: ")]


def _route_engine(monkeypatch, model, first, release):
    gguf = model.endswith(".gguf")

    def generate(*, stats_holder = None, **_kwargs):
        first.set()
        yield "A"
        assert release.wait(5)
        yield "AB"
        if stats_holder is None:
            yield {"type": "metadata", "finish_reason": "stop"}
        else:
            stats_holder["stats"] = {"usage": {"completion_tokens": 2}}

    llama = SimpleNamespace(
        is_loaded = gguf,
        model_identifier = model,
        base_url = "http://llama.test",
        effective_parallel_slots = 1,
        supports_tools = False,
        is_vision = False,
        _is_audio = False,
        context_length = None,
        generate_chat_completion = generate,
    )
    mlx = SimpleNamespace(
        active_model_name = model,
        models = {model: {"is_mlx": True, "chat_template_info": {"template": "chatml"}}},
        generate_chat_response = generate,
        reset_generation_state = lambda *_a, **_k: None,
    )
    monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: llama)
    monkeypatch.setattr(inference, "get_inference_backend", lambda: mlx)
    monkeypatch.setattr(inference, "_automatic_model_load_may_run", lambda: False)
    monkeypatch.setattr(
        inference, "_detect_safetensors_features", lambda *_a, **_k: {"supports_tools": False}
    )
    monkeypatch.setattr(inference, "_effective_enable_tools", lambda _payload: False)

    async def no_switch(*_args, **_kwargs):
        return None

    monkeypatch.setattr(inference, "_maybe_auto_switch_model", no_switch)


@pytest.mark.asyncio
@pytest.mark.parametrize("durable_run", ["gguf", "mlx"], indirect = True)
async def test_subscribers_detach_then_replay_the_same_engine_run(durable_run, monkeypatch):
    first_chunk, release = threading.Event(), threading.Event()
    _route_engine(monkeypatch, durable_run["requestPayload"]["model"], first_chunk, release)
    task = asyncio.create_task(
        ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))._produce("run-1")
    )
    if not await asyncio.to_thread(first_chunk.wait, 5):
        await task
        pytest.fail(str(runs_db.get_run("run-1", "alice")))
    while len(runs_db.list_events("run-1")) < 4:
        await asyncio.sleep(0.01)
    first, second = await asyncio.gather(_subscriber_sequences(), _subscriber_sequences())
    assert first == second == list(range(1, max(first) + 1))
    assert runs_db.get_run("run-1", "alice")["status"] == "running"
    release.set()
    await task
    tail = await _subscriber_sequences(after = max(first))
    run = runs_db.get_run("run-1", "alice")
    assert first + tail == list(range(1, run["lastEventSeq"] + 1))
    deltas = [
        event["payload"].get("choices", [{}])[0].get("delta", {}).get("content")
        for event in runs_db.list_events("run-1")
        if event["type"] == "chunk"
    ]
    assert [text for text in deltas if text] == ["A", "B"]


async def _await_chunk_payloads(run_id: str, count: int, deadline_s: float) -> list:
    """Returns chunk payloads once count are durable or the deadline passes; the caller asserts."""
    started = time.monotonic()
    while True:
        stored = [e["payload"] for e in runs_db.list_events(run_id) if e["type"] == "chunk"]
        if len(stored) >= count or time.monotonic() - started >= deadline_s:
            return stored
        await asyncio.sleep(0.005)


@pytest.mark.asyncio
async def test_event_batch_flushes_while_upstream_is_idle(durable_run, monkeypatch):
    release = asyncio.Event()
    chunks = [
        {"choices": [{"delta": {"role": "assistant"}}]},
        {"choices": [{"delta": {"content": "Hello"}}]},
    ]

    async def body():
        for chunk in chunks:
            yield f"data: {json.dumps(chunk)}\n\n"
        await release.wait()
        yield "data: [DONE]\n\n"

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    task = asyncio.create_task(supervisor._produce("run-1"))
    stored = await _await_chunk_payloads("run-1", len(chunks), 0.5)
    assert stored == chunks
    release.set()
    await task


def test_event_flush_interval_is_one_display_frame():
    assert _EVENT_FLUSH_SECONDS <= 1 / 60


@pytest.mark.asyncio
async def test_a_lone_chunk_is_appended_without_waiting_for_another(durable_run, monkeypatch):
    release = asyncio.Event()
    chunk = {"choices": [{"delta": {"content": "Hello"}}]}

    async def body():
        yield f"data: {json.dumps(chunk)}\n\n"
        await release.wait()
        yield "data: [DONE]\n\n"

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    task = asyncio.create_task(supervisor._produce("run-1"))
    stored = await _await_chunk_payloads("run-1", 1, 0.5)
    assert stored == [chunk]
    release.set()
    await task


def _numbered_chunks(count):
    return [{"choices": [{"delta": {"content": f"t{i} "}}]} for i in range(count)]


def _held_append(monkeypatch):
    """Patch append_events so every commit blocks until `gate` is set. Returns (entered, gate)."""
    entered = threading.Event()
    gate = threading.Event()
    original = runs_db.append_events

    def held(run_id, worker_token, events):
        entered.set()
        assert gate.wait(10)
        return original(run_id, worker_token, events)

    monkeypatch.setattr(runs_db, "append_events", held)
    return entered, gate


@pytest.mark.asyncio
async def test_a_slow_commit_does_not_stop_the_stream_being_read(durable_run, monkeypatch):
    chunks = _numbered_chunks(50)
    drained = asyncio.Event()
    release = asyncio.Event()

    async def body():
        yield f"data: {json.dumps(chunks[0])}\n\n"
        # The rest is only offered once the first commit is under way and held.
        assert await asyncio.to_thread(entered.wait, 2)
        for chunk in chunks[1:]:
            yield f"data: {json.dumps(chunk)}\n\n"
        drained.set()
        await release.wait()
        yield "data: [DONE]\n\n"

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    entered, gate = _held_append(monkeypatch)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    task = asyncio.create_task(supervisor._produce("run-1"))
    try:
        await asyncio.wait_for(drained.wait(), timeout = 2)
        assert not gate.is_set()
    finally:
        gate.set()
        release.set()
        await task
    stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
    assert stored == chunks
    assert runs_db.get_run("run-1", "alice")["status"] == "completed"


@pytest.mark.asyncio
async def test_slow_commits_batch_a_burst_instead_of_one_chunk_each(durable_run, monkeypatch):
    chunks = _numbered_chunks(400)
    calls = []
    original = runs_db.append_events

    def slow(run_id, worker_token, events):
        calls.append(len(events))
        time.sleep(0.04)
        return original(run_id, worker_token, events)

    async def body():
        for chunk in chunks:
            yield f"data: {json.dumps(chunk)}\n\n"
        yield "data: [DONE]\n\n"

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    monkeypatch.setattr(runs_db, "append_events", slow)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    await supervisor._produce("run-1")
    stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
    assert stored == chunks
    assert len(calls) <= 5, calls
    assert runs_db.get_run("run-1", "alice")["status"] == "completed"


@pytest.mark.asyncio
async def test_a_failed_commit_keeps_its_events_and_fails_the_run(durable_run, monkeypatch):
    chunks = _numbered_chunks(3)

    def broken(*_args, **_kwargs):
        raise RuntimeError("database is locked")

    async def body():
        for chunk in chunks:
            yield f"data: {json.dumps(chunk)}\n\n"
        yield "data: [DONE]\n\n"

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    monkeypatch.setattr(runs_db, "append_events", broken)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    await supervisor._produce("run-1")
    run = runs_db.get_run("run-1", "alice")
    assert (run["status"], run["finishReason"], run["error"]) == (
        "failed",
        "error",
        "database is locked",
    )
    stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
    assert stored and stored == chunks[: len(stored)]


@pytest.mark.asyncio
@pytest.mark.parametrize("cancels", [1, 2], ids = ["cancel", "cancel-again-while-settling"])
async def test_cancel_during_a_commit_waits_for_it_before_settling(
    durable_run, monkeypatch, cancels
):
    chunks = _numbered_chunks(20)
    leaked = []
    drained = asyncio.Event()

    async def body():
        for chunk in chunks:
            yield f"data: {json.dumps(chunk)}\n\n"
        drained.set()
        await asyncio.Event().wait()

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    entered, gate = _held_append(monkeypatch)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    loop = asyncio.get_running_loop()
    previous_handler = loop.get_exception_handler()
    loop.set_exception_handler(lambda _loop, context: leaked.append(context))
    cancel_event = threading.Event()
    task = asyncio.create_task(supervisor._produce("run-1", cancel_event))
    try:
        assert await asyncio.to_thread(entered.wait, 2)
        await asyncio.wait_for(drained.wait(), timeout = 2)
        cancel_event.set()
        task.cancel()
        # A second cancel (shutdown's forced one) lands while settlement waits on the commit; the run must still settle.
        for _ in range(cancels - 1):
            loop.call_later(0.05, task.cancel)
        # finish_run must not overtake the commit still holding its batch.
        loop.call_later(0.1, gate.set)
        with pytest.raises(asyncio.CancelledError):
            await task
        await asyncio.sleep(0)
    finally:
        gate.set()
        loop.set_exception_handler(previous_handler)
    assert leaked == []
    assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
    stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
    assert stored == chunks


@pytest.mark.asyncio
async def test_model_lifecycle_cancel_reaches_same_registered_event(durable_run, monkeypatch):
    registered = asyncio.Event()

    async def body(cancel_event):
        with active_generations.ActiveGeneration(
            cancel_event,
            thread_id = "thread-1",
            run_id = "run-1",
        ):
            registered.set()
            while not cancel_event.is_set():
                await asyncio.sleep(0.01)
        if False:
            yield ""

    async def fake(_payload, request, *_args, **_kwargs):
        return SimpleNamespace(
            status_code = 200,
            body_iterator = body(request.state.generation_cancel_event),
        )

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    task = asyncio.create_task(supervisor._produce("run-1"))
    await asyncio.wait_for(registered.wait(), timeout = 2)
    assert active_generations.cancel_all() == 1
    await asyncio.wait_for(task, timeout = 2)
    assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
    metadata = studio_db.get_chat_message("thread-1", "assistant-1")["metadata"]
    assert metadata["incomplete"] == {"reason": "cancelled"}
    assert active_generations.count() == 0


@pytest.mark.asyncio
async def test_cancel_before_registration_signals_load_event(durable_run, monkeypatch):
    entered = asyncio.Event()
    cancel_ids = []
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))

    async def fake(_payload, request, *_args, **_kwargs):
        event = request.state.generation_cancel_event
        entered.set()
        while not event.is_set():
            await asyncio.sleep(0.01)

        async def body():
            if False:
                yield ""

        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    monkeypatch.setattr(
        inference,
        "_cancel_by_cancel_id_or_stash",
        lambda run_id: cancel_ids.append(run_id) or 0,
    )
    supervisor.start("run-1")
    await asyncio.wait_for(entered.wait(), timeout = 2)
    supervisor.cancel("run-1")
    await asyncio.wait_for(supervisor._tasks["run-1"], timeout = 2)
    assert cancel_ids == ["run-1"]
    assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"


@pytest.mark.asyncio
async def test_start_reserves_slot_and_lifecycle_before_worker_runs(durable_run, monkeypatch):
    monkeypatch.setattr(llama_keepwarm, "_pending", 0)
    monkeypatch.setattr(llama_keepwarm, "_inflight", 0)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    supervisor.start("run-1", thread_id = "thread-1", model = "local")
    assert (llama_keepwarm._pending, active_generations.count()) == (1, 1)
    assert active_generations.snapshot()[0]["thread_id"] == "thread-1"
    supervisor.cancel("run-1")
    await asyncio.wait_for(supervisor._tasks["run-1"], timeout = 2)
    assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
    assert (llama_keepwarm._pending, llama_keepwarm._inflight) == (0, 0)


@pytest.mark.asyncio
async def test_cancelled_producer_error_is_cancelled(durable_run, monkeypatch):
    entered = asyncio.Event()

    async def fake(_payload, request, *_args, **_kwargs):
        entered.set()
        while not request.state.generation_cancel_event.is_set():
            await asyncio.sleep(0.01)
        raise RuntimeError("Generation cancelled")

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    supervisor.start("run-1")
    await asyncio.wait_for(entered.wait(), timeout = 2)
    assert active_generations.cancel_all() == 1
    await asyncio.wait_for(supervisor._tasks["run-1"], timeout = 2)
    assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"


@pytest.mark.asyncio
async def test_uncancelled_partial_eof_is_interrupted(durable_run, monkeypatch):
    async def body():
        yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    await ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))._produce("run-1")
    run = runs_db.get_run("run-1", "alice")
    assert (run["status"], run["finishReason"]) == ("failed", "interrupted")


@pytest.mark.asyncio
async def test_a_caught_up_reconnect_to_a_settled_run_does_not_block(durable_run, monkeypatch):
    """Nothing left to replay must return at once, not after the 15s event wait; otherwise a
    finished answer reads as still generating and holds an event-wait worker for the timeout."""

    async def body():
        yield 'data: {"choices":[{"delta":{"content":"done"},"finish_reason":"stop"}]}\n\n'
        yield "data: [DONE]\n\n"

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    await ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))._produce("run-1")
    settled = runs_db.get_run("run-1", "alice")
    assert settled["status"] == "completed"

    caught_up = await asyncio.wait_for(
        _subscriber_sequences(after = int(settled["lastEventSeq"])),
        timeout = 5,
    )
    assert caught_up == []
    assert await _subscriber_sequences() == list(range(1, int(settled["lastEventSeq"]) + 1))


@pytest.mark.asyncio
async def test_streamed_error_outranks_cleanup_cancellation(durable_run, monkeypatch):
    """A backend failure must not be recorded as if the user pressed Stop.

    ``gguf_stream_chunks`` emits the error in band, follows it with ``[DONE]`` and then
    sets this same ``cancel_event`` from its ``finally`` because the stream did not
    complete. Nobody asked to cancel, so the run has to settle as ``failed`` carrying
    the diagnostic the user needs to act on.
    """

    async def body(cancel_event):
        try:
            yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'
            yield 'data: {"error": {"message": "Out of memory"}}\n\ndata: [DONE]\n\n'
        finally:
            cancel_event.set()

    async def fake(_payload, request, *_args, **_kwargs):
        return SimpleNamespace(
            status_code = 200,
            body_iterator = body(request.state.generation_cancel_event),
        )

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    await ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))._produce("run-1")
    run = runs_db.get_run("run-1", "alice")
    assert run["status"] == "failed"
    assert run["error"] == "Out of memory"
    assert run["finishReason"] != "cancelled"
    metadata = studio_db.get_chat_message("thread-1", "assistant-1")["metadata"]
    assert metadata["incomplete"] != {"reason": "cancelled"}


@pytest.mark.asyncio
async def test_graceful_supervisor_shutdown_is_interrupted(durable_run, monkeypatch):
    entered = asyncio.Event()

    async def body(request):
        entered.set()
        while not request.state.generation_cancel_event.is_set():
            await asyncio.sleep(0.01)
        if False:
            yield ""

    async def fake(_payload, request, *_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body(request))

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    supervisor.start("run-1")
    await asyncio.wait_for(entered.wait(), timeout = 2)
    await supervisor.stop()
    run = runs_db.get_run("run-1", "alice")
    assert (run["status"], run["finishReason"]) == ("failed", "interrupted")
    assert run["error"] == "Unsloth shut down during generation"


def test_thread_delete_captures_durable_run_before_cascade(durable_run):
    research_ids, chat_ids = studio_db.delete_chat_threads_with_active_runs(["thread-1"])
    assert research_ids == []
    assert chat_ids == ["run-1"]
    assert runs_db.get_run("run-1", "alice") is None


def test_project_delete_captures_durable_run_before_cascade(durable_run):
    studio_db.upsert_chat_project(
        {"id": "project-1", "name": "Project", "createdAt": 1, "updatedAt": 1}
    )
    studio_db.update_chat_thread("thread-1", {"projectId": "project-1"})
    deleted = studio_db.delete_chat_project("project-1")
    assert deleted["activeChatGenerationRunIds"] == ["run-1"]


def test_clear_captures_durable_run_before_cascade(durable_run):
    removed, research_ids, chat_ids = studio_db.clear_chat_history(
        include_chat_generation_runs = True
    )
    assert (removed, research_ids, chat_ids) == (["thread-1"], [], ["run-1"])


def test_startup_reconcile_marks_stored_assistant_interrupted(durable_run):
    worker_token = runs_db.get_worker_token("run-1")
    assert runs_db.mark_running("run-1", worker_token)
    assert runs_db.reconcile_orphaned_runs() == 1
    message = studio_db.get_chat_message("thread-1", "assistant-1")
    assert message["metadata"]["generationStatus"] == "failed"
    assert message["metadata"]["incomplete"] == {"reason": "interrupted"}


@pytest.mark.asyncio
async def test_shutdown_returns_even_when_a_producer_will_not_unwind(durable_run, monkeypatch):
    """A generator whose teardown blocks must not block uvicorn's shutdown; the gather after
    task.cancel() must be bounded too, since an engine draining a subprocess in aclose never
    finishes cancelling and stop() would wait forever."""
    import core.inference.chat_generation_runs as chat_generation_runs

    monkeypatch.setattr(chat_generation_runs, "_SHUTDOWN_GRACE_SECONDS", 0.2)
    monkeypatch.setattr(chat_generation_runs, "_SHUTDOWN_CANCEL_SECONDS", 0.5)

    wedged = asyncio.Event()
    release = asyncio.Event()

    async def body():
        yield 'data: {"choices":[{"delta":{"content":"a"}}]}\n\n'
        wedged.set()
        try:
            await release.wait()
        except (asyncio.CancelledError, GeneratorExit):
            await release.wait()
            raise
        yield "data: [DONE]\n\n"

    async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    supervisor.start("run-1", thread_id = "thread-1", model = "local.gguf")
    await asyncio.wait_for(wedged.wait(), 10)

    try:
        await asyncio.wait_for(supervisor.stop(), timeout = 10)
    finally:
        release.set()
        await asyncio.sleep(0)


# Pins the line in _ensure_reservation that sets cancel.durable; other tests set it by hand.


def test_the_producers_cancel_event_carries_the_durable_marker_and_its_run_id():
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    try:
        assert supervisor._ensure_reservation("run-durable", thread_id = "thread-1") is True
        cancel_event = supervisor._cancel_events["run-durable"]
        assert getattr(cancel_event, "durable", False) is True, (
            "state.tool_approvals.wait_tool_decision reads this to park an approval instead of "
            "auto-denying it, so without it a tool turn that outlives its tab still loses the call"
        )
        assert getattr(cancel_event, "durable_run_id", None) == "run-durable", (
            "the gate resolves attendance by run id (state/run_subscribers.py); without it an "
            "attended approval cannot be told from an abandoned one and expires at the park ceiling"
        )
    finally:
        supervisor.cancel("run-durable")
        registration = supervisor._active_registrations.pop("run-durable", None)
        if registration is not None:
            registration.__exit__(None, None, None)
        supervisor._cancel_events.pop("run-durable", None)
        activity = supervisor._activities.pop("run-durable", None)
        if activity is not None:
            activity.finish()


@pytest.mark.asyncio
async def test_permitted_prefill_renews_lease_but_decode_silence_does_not(durable_run, monkeypatch):
    from utils.first_token_timeout import start_first_token_wait, finish_first_token_wait

    monkeypatch.setattr(chat_generation_runs, "_renew_interval_seconds", lambda: 0.01)
    supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
    touch = AsyncMock(wraps = supervisor._try_touch_progress)
    monkeypatch.setattr(supervisor, "_try_touch_progress", touch)
    observations = []

    async def body():
        # Sync reader runs in a copied context; the supervisor polls the shared wait object.
        await asyncio.to_thread(start_first_token_wait, float("inf"))
        before = touch.await_count
        await asyncio.sleep(0.08)
        observations.append(touch.await_count > before)
        await asyncio.to_thread(finish_first_token_wait)
        yield 'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
        before = touch.await_count
        await asyncio.sleep(0.08)
        observations.append(touch.await_count == before)
        yield "data: [DONE]\n\n"

    async def fake(*_args, **_kwargs):
        return SimpleNamespace(status_code = 200, body_iterator = body())

    monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
    await supervisor._produce("run-1")
    assert observations == [True, True]