Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

12.3 KB
Raw Blame
import importlib
import unittest

llm_server = importlib.import_module("services.llm_server")


class _FakeSlot:
    """Stands in for a loaded LlamaServerManager: alive, with a last-used stamp
    and a VRAM estimate."""

    def __init__(self, last_used: float, vram_mb: int, busy: int = 0, port: int = 0) -> None:
        self._last_used = last_used
        self.vram_mb = vram_mb
        self.port = port
        self.busy_count = busy
        self.unloaded = False

    def _alive(self) -> bool:
        return not self.unloaded

    def unload(self) -> None:
        self.unloaded = True


class EstimateVramTests(unittest.TestCase):
    def test_declared_estimate_wins(self):
        self.assertEqual(llm_server.estimate_vram_mb({"vram_estimate_mb": 4200}), 4200)

    def test_custom_model_is_derived_from_file_size(self):
        # 9 GB of weights: enough on its own to fill a 12 GB card.
        mb = llm_server.estimate_vram_mb({"size_bytes": 9 * 1024**3})
        self.assertGreater(mb, 11000)
        self.assertLess(mb, 12500)

    def test_unknown_size_yields_zero(self):
        self.assertEqual(llm_server.estimate_vram_mb({}), 0)


class PoolBudgetTests(unittest.TestCase):
    """Numbers are the ones measured on the reference machine: an RTX 5070
    reporting 11.9 GiB, so a 768 MiB reserve leaves a 11.4 GiB budget."""

    def setUp(self) -> None:
        self.pool = llm_server.LlamaPool()
        self._orig_max = llm_server.resolve_max_models
        self._orig_budget = llm_server.vram_budget_mb
        llm_server.resolve_max_models = lambda: 2
        llm_server.vram_budget_mb = lambda: int(11.9 * 1024) - 768  # 11417

    def tearDown(self) -> None:
        llm_server.resolve_max_models = self._orig_max
        llm_server.vram_budget_mb = self._orig_budget

    def _load(self, **slots) -> None:
        for i, (mid, vram) in enumerate(slots.items()):
            self.pool._slots[mid] = _FakeSlot(last_used=float(i), vram_mb=vram)

    def test_a_pair_that_fits_is_kept(self):
        # qwen3-4b (4200) already loaded, cadquery-coder-7b (6200) incoming.
        self._load(qwen4b=4200)
        self.pool._enforce_limit_locked(reserve=1, incoming_mb=6200)
        self.assertEqual(list(self.pool._slots), ["qwen4b"])

    def test_a_pair_that_does_not_fit_evicts_the_oldest(self):
        # qwen3-4b (4200) + qwen3-vl-8b (7800) = 12000 > 11417.
        self._load(qwen4b=4200)
        self.pool._enforce_limit_locked(reserve=1, incoming_mb=7800)
        self.assertEqual(list(self.pool._slots), [])

    def test_an_oversized_model_gets_the_card_to_itself(self):
        # The custom 14B alone exceeds the budget: it still loads, alone.
        self._load(qwen4b=4200, coder7b=6200)
        self.pool._enforce_limit_locked(reserve=1, incoming_mb=11700)
        self.assertEqual(list(self.pool._slots), [])

    def test_count_limit_still_applies_under_the_budget(self):
        # Three tiny models fit the VRAM budget but not `max_models: 2`.
        self._load(a=500, b=500)
        self.pool._enforce_limit_locked(reserve=1, incoming_mb=500)
        self.assertEqual(list(self.pool._slots), ["b"])

    def test_evicts_least_recently_used_first(self):
        self._load(oldest=4200, newest=4200)
        llm_server.resolve_max_models = lambda: 3
        self.pool._enforce_limit_locked(reserve=1, incoming_mb=7800)
        self.assertNotIn("oldest", self.pool._slots)

    def test_no_gpu_falls_back_to_the_count_rule(self):
        llm_server.vram_budget_mb = lambda: 0
        self._load(a=9000)
        self.pool._enforce_limit_locked(reserve=1, incoming_mb=9000)
        self.assertEqual(list(self.pool._slots), ["a"])

    def test_unknown_estimate_never_evicts(self):
        # A model with no size on disk must not push everything out.
        self._load(a=4200)
        self.pool._enforce_limit_locked(reserve=1, incoming_mb=0)
        self.assertEqual(list(self.pool._slots), ["a"])

    def test_a_concurrent_load_counts_against_the_budget(self):
        # ensure() reserves under the lock but loads outside it, so a slot being
        # started has no process yet. Counting only live slots let two callers
        # (agent turn + workflow LLM node) each load 7.8 GB on an 11.4 GB card.
        loading = _FakeSlot(last_used=0.0, vram_mb=7800, port=llm_server.SERVER_PORT)
        loading.unloaded = True  # reserved: not alive yet
        self.pool._slots["incoming-a"] = loading
        self.pool._loading_ports.add(llm_server.SERVER_PORT)
        self._load(alive_one=4200)

        self.pool._enforce_limit_locked(reserve=1, incoming_mb=7800)
        # 7800 (in flight) + 7800 (incoming) already blows the budget, so the
        # live 4.2 GB model goes.
        self.assertNotIn("alive_one", self.pool._slots)

    def test_a_concurrent_load_counts_against_the_model_limit(self):
        loading = _FakeSlot(last_used=0.0, vram_mb=500, port=llm_server.SERVER_PORT)
        loading.unloaded = True
        self.pool._slots["incoming-a"] = loading
        self.pool._loading_ports.add(llm_server.SERVER_PORT)
        self._load(a=500)  # 1 live + 1 loading + 1 incoming > max_models (2)

        self.pool._enforce_limit_locked(reserve=1, incoming_mb=500)
        self.assertNotIn("a", self.pool._slots)

    def test_a_slot_answering_a_request_is_never_evicted(self):
        # Evicting it would terminate llama-server mid-generation: the caller
        # gets a truncated stream and the node fails with no message.
        self.pool._slots["busy"] = _FakeSlot(last_used=0.0, vram_mb=7800, busy=1)
        self.pool._enforce_limit_locked(reserve=1, incoming_mb=7800)
        self.assertIn("busy", self.pool._slots)

    def test_a_slot_being_started_is_never_evicted(self):
        # From the moment Popen returns, a slot is _alive() while the loading
        # thread still holds its lock for the whole of _wait_for_health (up to
        # 180 s). unload() runs UNDER the pool lock, so evicting it there would
        # block every other pool operation: /llm/status, ensure(), and the
        # unload_all() the 3D pipeline calls to reclaim VRAM: for that long.
        starting = _FakeSlot(last_used=0.0, vram_mb=7800, port=llm_server.SERVER_PORT)
        self.pool._slots["starting"] = starting
        self.pool._loading_ports.add(llm_server.SERVER_PORT)

        self.pool._enforce_limit_locked(reserve=1, incoming_mb=7800)
        self.assertIn("starting", self.pool._slots)
        self.assertFalse(starting.unloaded)

    def test_over_capacity_counts_loads_in_flight(self):
        loading = _FakeSlot(last_used=0.0, vram_mb=7800, port=llm_server.SERVER_PORT)
        loading.unloaded = True  # reserved, no process yet
        self.pool._slots["incoming-a"] = loading
        self.pool._loading_ports.add(llm_server.SERVER_PORT)
        # Nothing is alive, yet the card is already spoken for.
        self.assertTrue(self.pool._over_capacity_locked(7800))
        self.pool._loading_ports.clear()
        self.assertFalse(self.pool._over_capacity_locked(7800))

    def test_only_loads_and_live_requests_are_worth_waiting_for(self):
        # ensure() waits for these to end instead of loading a second model on a
        # one-model card; a merely idle slot is evicted rather than waited on.
        idle = _FakeSlot(last_used=0.0, vram_mb=4200, port=llm_server.SERVER_PORT)
        self.pool._slots["idle"] = idle
        self.assertFalse(self.pool._transient_blockers_locked())

        idle.busy_count = 1
        self.assertTrue(self.pool._transient_blockers_locked())

        idle.busy_count = 0
        self.pool._loading_ports.add(llm_server.SERVER_PORT + 1)
        self.assertTrue(self.pool._transient_blockers_locked())

    def test_a_crashed_slot_is_restarted_on_a_fresh_port(self):
        # Its port was handed to another model meanwhile; respawning in place
        # killed that model's server through _kill_stale_server.
        dead = _FakeSlot(last_used=0.0, vram_mb=500, port=llm_server.SERVER_PORT)
        dead.unloaded = True
        self.pool._slots["x"] = dead
        self.pool._slots["y"] = _FakeSlot(last_used=1.0, vram_mb=500, port=llm_server.SERVER_PORT)
        orig = llm_server.LlamaServerManager.ensure
        llm_server.LlamaServerManager.ensure = lambda self, mid, spec: None
        try:
            slot = self.pool.ensure("x", {"vram_mb": 500})
        finally:
            llm_server.LlamaServerManager.ensure = orig
        self.assertIsNot(slot, dead)
        self.assertNotEqual(slot.port, llm_server.SERVER_PORT)
        self.assertFalse(self.pool._slots["y"].unloaded)

    def test_unload_all_spares_requests_and_cold_starts(self):
        idle = _FakeSlot(last_used=0.0, vram_mb=4200, port=llm_server.SERVER_PORT)
        busy = _FakeSlot(last_used=0.0, vram_mb=4200, busy=1, port=llm_server.SERVER_PORT + 1)
        starting = _FakeSlot(last_used=0.0, vram_mb=4200, port=llm_server.SERVER_PORT + 2)
        self.pool._slots.update(idle=idle, busy=busy, starting=starting)
        self.pool._loading_ports.add(starting.port)

        self.pool.unload_all()
        self.assertEqual(sorted(self.pool._slots), ["busy", "starting"])
        self.assertTrue(idle.unloaded)

        self.pool.unload_all(force=True)
        self.assertEqual(list(self.pool._slots), [])
        self.assertTrue(busy.unloaded and starting.unloaded)

    def test_no_free_port_raises_a_readable_error(self):
        for i in range(llm_server.MAX_SLOT_PORTS):
            self.pool._loading_ports.add(llm_server.SERVER_PORT + i)
        with self.assertRaises(RuntimeError) as ctx:
            self.pool.ensure("whatever", {"vram_mb": 500})
        self.assertIn("slots", str(ctx.exception))


class ReaperTests(unittest.TestCase):
    def setUp(self) -> None:
        self.pool = llm_server.LlamaPool()

    def test_an_idle_slot_is_unloaded(self):
        old = _FakeSlot(last_used=0.0, vram_mb=4200)
        self.pool._slots["old"] = old
        self.pool._reap_once(now=llm_server.IDLE_TTL_SECONDS + 1)
        self.assertEqual(list(self.pool._slots), [])
        self.assertTrue(old.unloaded)

    def test_a_slot_answering_a_long_request_survives(self):
        # A generation longer than the TTL (Text-to-CAD, a 14B on CPU) used to
        # be killed mid-answer: _last_used only moved once a call had finished.
        busy = _FakeSlot(last_used=0.0, vram_mb=4200, busy=1)
        self.pool._slots["busy"] = busy
        self.pool._reap_once(now=llm_server.IDLE_TTL_SECONDS * 10)
        self.assertEqual(list(self.pool._slots), ["busy"])
        self.assertFalse(busy.unloaded)

    def test_a_crashed_slot_is_dropped(self):
        dead = _FakeSlot(last_used=0.0, vram_mb=4200)
        dead.unloaded = True
        self.pool._slots["dead"] = dead
        self.pool._reap_once(now=1.0)
        self.assertEqual(list(self.pool._slots), [])


    def test_a_slot_being_started_is_not_idle(self):
        # Same reason as the eviction case: the reaper also unloads under the
        # pool lock, so a slot mid-cold-start must not be its victim.
        starting = _FakeSlot(last_used=0.0, vram_mb=4200, port=llm_server.SERVER_PORT)
        self.pool._slots["starting"] = starting
        self.pool._loading_ports.add(llm_server.SERVER_PORT)
        self.pool._reap_once(now=llm_server.IDLE_TTL_SECONDS * 10)
        self.assertEqual(list(self.pool._slots), ["starting"])
        self.assertFalse(starting.unloaded)


class BusyContextTests(unittest.TestCase):
    def test_busy_counts_nest_and_stamp_last_used(self):
        slot = llm_server.LlamaServerManager(port=1)
        self.assertEqual(slot.busy_count, 0)
        with slot.busy():
            self.assertEqual(slot.busy_count, 1)
            with slot.busy():
                self.assertEqual(slot.busy_count, 2)
            self.assertEqual(slot.busy_count, 1)
        self.assertEqual(slot.busy_count, 0)
        self.assertGreater(slot._last_used, 0.0)

    def test_hold_and_release_pair_up(self):
        # ensure(hold=True) claims the slot before returning it; the caller
        # releases it once the request is over.
        slot = llm_server.LlamaServerManager(port=1)
        slot.hold()
        self.assertEqual(slot.busy_count, 1)
        slot.release()
        self.assertEqual(slot.busy_count, 0)

    def test_busy_is_released_when_the_request_raises(self):
        slot = llm_server.LlamaServerManager(port=1)
        with self.assertRaises(ValueError):
            with slot.busy():
                raise ValueError("client disconnected")
        self.assertEqual(slot.busy_count, 0)


if __name__ == "__main__":
    unittest.main()