Skip to content

Make videos on the computer you already own. FreeVideo runs MiniMax H3 in as little as 8 GB of VRAM and 16 GB of RAM, and adapts its acceleration path to your hardware.

36.6 KB
Raw Blame
"""Private local IPC for a ComfyUI-owned, reusable engine worker.

The worker receives the active controller's real GPU lease by descriptor/handle
transfer. It closes that lease before going idle. The controller still measures
and can stop the physical worker, so caching does not bypass RAM guards, attempt
history, cancellation, or the Windows process tree supervisor. No public port.
"""
import atexit
from dataclasses import replace
import json
import os
from pathlib import Path
import secrets
import signal
import subprocess
import sys
import threading
import time

from .execution_paths import execution_environment
from . import processes
from .monitoring import save

ENV = 'FREEVIDEO_RESIDENT_SESSION'
GiB = 2**30
# A successful request settles individual caches inside the worker, then keeps
# it without an idle limit unless KEEP_IDLE_SECONDS sets one. It gives up the
# worker at once when another program's GPU use grows, host memory runs low, an
# image or prompt model starts (those release it themselves), or the agent asks
# for the card back. A failed request without a queued part releases it as
# before; a next part already on its way is held for HOLD_SECONDS.
HOLD_SECONDS = 90.
AGENT_HOLD_SECONDS = 30.
KEEP_IDLE_SECONDS = None
OTHER_GPU_GROWTH = 512 * 2**20
PRELOAD_CONTEXT_BYTES = 1536 * 2**20  # what the preload's process adds beside its weights (context, allocator, staging; measured 1.4 GiB)
HOST_FLOOR = 4 * GiB
# Below this much host memory (free plus the idle worker's own), a kept worker's
# base leaves the next part less room than a fresh one; measured on 16 GiB.
KEEP_MIN_HOST = 24 * GiB
RELEASE_LOG = 20


class SessionRestarted(RuntimeError):
    """No new GPU work started; the old worker was confirmed stopped."""


class SessionBusy(RuntimeError):
    """A running request owns this worker; its allocations are not idle credit."""


def send(connection, value):
    connection.send_bytes(json.dumps(value, allow_nan=False).encode('utf-8'))


def receive(connection, timeout=None):
    if timeout is not None and not connection.poll(timeout):
        raise TimeoutError('Resident engine did not respond')
    return json.loads(connection.recv_bytes(2**20))


def _send_lease(connection, descriptor, destination_pid):
    if os.name != 'nt':
        from multiprocessing.reduction import send_handle
        send_handle(connection, descriptor, destination_pid)
        return
    import _winapi
    import msvcrt
    # CPython 3.9/3.12 send_handle passes DUPLICATE_SAME_ACCESS as the desired
    # access mask, not DuplicateHandle's options. For files that becomes only
    # FILE_WRITE_DATA, losing the permissions needed to validate/use the lease.
    # Duplicate explicitly with unchanged access and transfer only its integer
    # value over our authenticated JSON channel. The receiver owns the copy;
    # RemoteProcess stops it on any failed/ambiguous handoff, closing that copy.
    target = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, destination_pid)
    try:
        handle = _winapi.DuplicateHandle(_winapi.GetCurrentProcess(),
            msvcrt.get_osfhandle(descriptor), target, 0, False, _winapi.DUPLICATE_SAME_ACCESS)
        send(connection, dict(command='lease', handle=handle))
    finally:
        _winapi.CloseHandle(target)


def _receive_lease(connection):
    if os.name != 'nt':
        from multiprocessing.reduction import recv_handle
        return recv_handle(connection)
    import _winapi
    import msvcrt
    message = receive(connection, 10.)
    handle = message.get('handle')
    if message.get('command') != 'lease' or type(handle) is not int or handle <= 0:
        raise ValueError('Invalid resident runtime lease transfer')
    try:
        return msvcrt.open_osfhandle(handle, os.O_RDWR | os.O_BINARY)
    except BaseException:
        _winapi.CloseHandle(handle)
        raise


def alive(pid, created):
    import psutil
    try:
        process = psutil.Process(pid)
        return process.create_time() == created and process.is_running() and process.status() != psutil.STATUS_ZOMBIE
    except (psutil.Error, ValueError):
        return False


def connect(path):
    from multiprocessing.connection import Client
    record = json.loads(Path(path).read_text(encoding='utf-8'))
    ready = json.loads(Path(path).with_name('ready.json').read_text(encoding='utf-8'))
    if not alive(ready['pid'], ready['created']):
        raise SessionRestarted('The previous resident worker has stopped; resume in a fresh worker')
    if ready.get('idle') is False:
        raise SessionBusy('Resident engine is executing another request')
    connection = Client(record['address'], family=record['family'], authkey=bytes.fromhex(record['key']))
    hello = receive(connection, 5.)
    if (hello['pid'] != ready['pid'] or hello['created'] != ready['created'] or
            not alive(hello['pid'], hello['created'])):
        connection.close()
        raise RuntimeError('Resident engine process changed')
    return connection, hello


def status(environment=None):
    path = (os.environ if environment is None else environment).get(ENV)
    if not path:
        return None
    try:
        connection, _ = connect(path)
        try:
            send(connection, {'command': 'status'})
            return receive(connection, 5.)
        finally:
            connection.close()
    except (OSError, ValueError, EOFError, TimeoutError, SessionRestarted, SessionBusy):
        return None


def credit(hardware, environment=None, *, reuse_encoder=False):
    """Plan whole-worker budgets using only authenticated idle allocations."""
    state = status(environment)
    if not state or not hardware.gpu_uuid or state.get('gpu_uuid') != hardware.gpu_uuid or not state.get('idle'):
        return hardware, None
    gpu = state.get('reclaimable_gpu_bytes')
    ram = state.get('reclaimable_ram_bytes', 0)
    if type(gpu) is not int or gpu < 0 or type(ram) is not int or ram < 0:
        return hardware, None
    # Nominal or explicit user capacities still cap choose(). Active allocations
    # from ComfyUI or unrelated applications are never included in this credit.
    available = min(hardware.ram_total, hardware.ram_available + ram)
    owned_encoding_ram = 0
    memory = state.get('memory') or {}
    if hardware.system == 'Windows':
        # Windows global availability excludes our resident mapped weights.
        # Reuse the worker guard's effective availability, capped by physical
        # RAM and commit headroom. Never add mapped credit to min(RAM, commit)
        # directly: that would manufacture commit capacity.
        keys = ('effective_available_bytes', 'system_physical_available_bytes',
                'system_commit_available_bytes', 'reclaimable_mapped_bytes', 'guard_bytes')
        if all(type(memory.get(k)) is int and memory[k] >= 0 for k in keys):
            # Pinned private storage of an idle model is memory this worker
            # already committed: reusing it charges nothing new and evicting it
            # returns the commit. Commit headroom alone counted it twice, so a
            # repeat was planned with less RAM than the model it already held.
            # Pinned read-only checkpoint maps are file-backed and earn none.
            owned = min(ram, memory['guard_bytes'])
            committed = state.get('reclaimable_commit_bytes', 0)
            committed = min(owned, committed) if type(committed) is int and committed > 0 else 0
            available = min(hardware.ram_total, memory['system_commit_available_bytes'] + committed,
                memory['effective_available_bytes'] + owned,
                memory['system_physical_available_bytes'] + memory['reclaimable_mapped_bytes'] + owned)
            if (reuse_encoder and memory.get('guard_metric') == 'tree private working set'
                    and any(model.get('role') == 'encoder' for model in state.get('models', []))):
                # The encoder is staying in this same worker. Its measured
                # private working set is already charged to both RAM and
                # Commit; a whole-worker budget must not charge it again as
                # newly allocated memory. Replace the pin credit, do not add
                # both. No RSS, mapped section or nonresident commit is added.
                owned_encoding_ram = memory['guard_bytes']
                available = min(hardware.ram_total,
                    memory['effective_available_bytes'] + owned_encoding_ram,
                    memory['system_commit_available_bytes'] + owned_encoding_ram,
                    memory['system_physical_available_bytes'] + memory['reclaimable_mapped_bytes'] + owned_encoding_ram)
        else:
            available = hardware.ram_available  # Unknown commit/mapped accounting earns no RAM credit.
    state = dict(state, ram_accounting=dict(raw_available_bytes=hardware.ram_available,
        credited_available_bytes=available, pinned_model_bytes=ram,
        reused_encoder_working_bytes=owned_encoding_ram,
        scope=('Windows whole-worker encoding allowance: effective physical/commit headroom plus '
               'already resident private working set, counted once; no new system capacity'
               if owned_encoding_ram else
               'Windows worker effective physical availability bounded by commit; Linux available RAM plus owned pins')))
    gpu_available = min(hardware.vram_total, hardware.vram_free + gpu)
    capacity = state.get('gpu_capacity')
    if hardware.system == 'Windows' and isinstance(capacity, dict):
        limit = capacity.get('allocator_limit_bytes')
        if type(limit) is int and 0 <= limit <= hardware.vram_total:
            gpu_available = min(gpu_available, limit)
    return replace(hardware, vram_free=gpu_available,
                   ram_available=available), state


class RemoteProcess:
    """Popen-shaped active request; pid is the measured GPU worker, not an RPC shim."""
    def __init__(self, endpoint, command, environment, descriptor, log, *, required_capabilities=(), operation='run'):
        import psutil
        if operation not in ('run', 'release'):
            raise ValueError('Invalid resident operation')
        self.connection, hello = connect(endpoint)
        self.pid, self.created = hello['pid'], hello['created']
        self.returncode = None
        self.settled = False
        self.args = command
        if not set(required_capabilities).issubset(hello.get('capabilities', [])):
            # A previous worker must not interpret a weights-only request as
            # ordinary prompt encoding. Complete a read-only exchange and keep
            # it idle, without transferring a lease or beginning CUDA work.
            try:
                send(self.connection, {'command': 'status'})
                receive(self.connection, 5.)
            finally:
                self.connection.close()
            raise RuntimeError('Resident worker does not support the requested idle preparation; restart the app')
        selected = environment.get('PYTORCH_ALLOC_CONF', environment.get('PYTORCH_CUDA_ALLOC_CONF', ''))
        if hello.get('allocator_config') is not None and hello['allocator_config'] != selected:
            self.kill()
            deadline = time.monotonic()+5.
            while alive(self.pid, self.created) and time.monotonic() < deadline:
                time.sleep(.025)
            self.connection.close()
            if alive(self.pid, self.created):
                raise RuntimeError('Could not stop the old resident allocator; refusing concurrent GPU work')
            raise SessionRestarted('Allocator settings changed; cleared idle cache and started a fresh worker')
        try:
            send(self.connection, dict(command=operation, argv=list(command), environment=dict(environment),
                log=str(Path(log).resolve()), owner_pid=os.getpid(), owner_created=psutil.Process().create_time()))
            _send_lease(self.connection, descriptor, self.pid)
            try:
                accepted = receive(self.connection, 10.)
            except (EOFError, OSError) as error:
                raise RuntimeError('Resident worker disconnected during runtime lease handoff; see ' +
                    str(Path(endpoint).with_name('session.log'))) from error
            if accepted.get('status') != 'accepted':
                raise RuntimeError('Resident worker rejected request during ' +
                    str(accepted.get('phase', 'handoff')) + ': ' + str(accepted.get('error')))
        except BaseException as error:
            # A failed handoff must not leave a worker waiting for a descriptor,
            # or executing work which the controller cannot monitor/cancel.
            try:
                self.stop_request(grace=.25)
            except BaseException as cleanup:
                error.worker_stop_confirmed = False
                error.worker_stop_error = repr(cleanup)
            self.connection.close()
            raise

    def poll(self):
        if self.returncode is not None:
            return self.returncode
        try:
            if self.connection.poll(0):
                result = receive(self.connection)
                self.returncode = int(result['returncode'])
                self.settled = True
                self.connection.close()
            elif not alive(self.pid, self.created):
                self.returncode = -1
                self.connection.close()
        except (EOFError, OSError):
            self.returncode = -1
            self.connection.close()
        return self.returncode

    def wait(self, timeout=None):
        started = time.monotonic()
        while self.poll() is None:
            if timeout is not None and time.monotonic() - started >= timeout:
                raise subprocess.TimeoutExpired(self.args, timeout)
            time.sleep(.025)
        return self.returncode

    def send_signal(self, value):
        if alive(self.pid, self.created):
            os.kill(self.pid, value)

    def kill(self):
        import psutil
        if alive(self.pid, self.created):
            process = psutil.Process(self.pid)
            for child in reversed(process.children(recursive=True)):
                try:
                    child.kill()
                except psutil.Error:
                    pass
            process.kill()

    def terminate(self):
        self.send_signal(signal.SIGTERM)

    def stop_request(self, grace=10):
        self.poll()
        if self.settled:
            return
        self.terminate()
        deadline = time.monotonic()+grace
        while alive(self.pid, self.created) and time.monotonic() < deadline:
            time.sleep(.025)
        if alive(self.pid, self.created):
            self.kill()
            deadline = time.monotonic()+5.
            while alive(self.pid, self.created) and time.monotonic() < deadline:
                time.sleep(.025)
        if alive(self.pid, self.created):
            raise subprocess.TimeoutExpired(self.args, grace+5)
        self.poll()
        self.connection.close()


def release_idle_cache(environment, descriptor):
    """Release only our authenticated idle process, under the runtime lease.

    Used once when cached models prevent admitting the next stage. Exiting the
    process releases encoder mappings, CUDA allocations and WDDM commit together.
    Conditioning already saved on disk is unaffected; no generation is repeated.
    """
    endpoint = environment[ENV]
    log = Path(endpoint).with_name('release-%d.log' % time.time_ns())
    worker = RemoteProcess(endpoint, ['release-idle-cache'], environment, descriptor, log,
        required_capabilities=('idle-cache-release',), operation='release')
    try:
        code = worker.wait(timeout=5.)
        if code:
            raise RuntimeError('Idle cache release failed with exit %s' % code)
        deadline = time.monotonic()+5.
        while alive(worker.pid, worker.created) and time.monotonic() < deadline:
            time.sleep(.025)
        if alive(worker.pid, worker.created):
            worker.kill()
            deadline = time.monotonic()+3.
            while alive(worker.pid, worker.created) and time.monotonic() < deadline:
                time.sleep(.025)
        if alive(worker.pid, worker.created):
            raise RuntimeError('Idle worker has not exited; memory release is not confirmed')
        return dict(released=True, pid=worker.pid, reason='Release idle models before replanning; saved conditioning retained')
    except BaseException:
        # The release was accepted under our lease. Do not leave a half-stopped
        # process or assume its memory is available after an IPC timeout.
        if alive(worker.pid, worker.created):
            worker.stop_request(grace=1.) if not worker.settled else worker.kill()
        raise
    finally:
        worker.connection.close()


class SessionOwner:
    """One persistent worker owned by the interactive ComfyUI process."""
    def __init__(self):
        self.process = None
        self.endpoint = None
        self.identity = None
        self.active = False
        self.prewarm = None
        self.gpu = self.baseline = self.own = None
        self.mutex = threading.RLock()
        self.generation = 0
        self.idle_since = None
        self.decisions = []
        atexit.register(self.close)

    def record(self, **row):
        row.setdefault('at', time.time())
        with self.mutex:
            self.decisions = (self.decisions + [row])[-RELEASE_LOG:]
        return row

    def take_decisions(self):
        """Keep or release decisions since the previous request, for its report."""
        with self.mutex:
            rows, self.decisions = self.decisions, []
        return rows

    def finished(self, *, next_part=None, agent=False, gpu=None, warm=False, decisions=None):
        """Release the idle worker now, or keep it for the next request.

        A successful request (warm) keeps its settled caches above HOST_FLOOR while the card's use can be read, so
        another program's growth is seen; a next part on its way keeps it for a short hold. Anything else
        gives the memory back now.
        """
        with self.mutex:
            if self.active or self.process is None or self.process.poll() is not None:
                return None
            hold = min(HOLD_SECONDS, AGENT_HOLD_SECONDS) if agent else HOLD_SECONDS
            if not next_part and warm and gpu_used(gpu) is not None:
                hold, next_part = KEEP_IDLE_SECONDS, 'warm'
            else:
                warm = False
            if not next_part or (hold is not None and hold <= 0):
                return self.release('finished' if not next_part else 'no_hold')
            # The worker settles individual caches against its live RAM budget.
            # A small host alone is no reason to discard everything it kept.
            reason = pressure(gpu, None) or self._tight()
            if reason:
                return self.release(reason)
            self.generation += 1
            self.idle_since = time.monotonic()
            baseline = gpu_used(gpu)
            if warm and baseline is None:
                return self.release('gpu_unreadable')
            self.gpu, self.baseline = gpu, baseline
            self.own = gpu_own(gpu) if warm else None
            threading.Thread(target=self._watch, args=(self.generation, self.idle_since, hold, gpu),
                             name='freevideo-resident-hold', daemon=True).start()
            return self.record(action='kept', reason=next_part, seconds=hold, pid=self.process.pid,
                               **(held_summary(decisions) if decisions else {}))

    def _tight(self):
        from .system import windows
        if windows():
            # Windows keeps its earlier behaviour: the available-memory reading
            # here has not been checked against its commit accounting.
            return None
        try:
            from .background_room import available_memory
            own = worker_memory(self.process.pid).get('worker_rss_bytes', 0)
            if available_memory() + own < KEEP_MIN_HOST:
                return 'host_memory_tight'
        except (OSError, RuntimeError, AttributeError, ValueError, ImportError):
            pass
        return None

    def preloading(self):
        return self.prewarm is not None and self.prewarm.thread is not None and self.prewarm.thread.is_alive()

    def warm_encoder(self, output, python, environment, *, busy=None, notify=None):
        """Place the encoder on the card while the kept worker idles, under the worker's own lease.

        Only the models the worker can hold beside what it keeps are placed (it
        sizes that from live free memory), and any new request, queued prompt or
        memory pressure stops it first.
        """
        with self.mutex:
            if (self.process is None or self.process.poll() is not None or self.active or
                    environment.get('FREEVIDEO_ENCODER_PREWARM', 'auto').lower() in ('0', 'off', 'false')):
                return False
            self.stop_prewarm()
            if self.process is None:
                return False
            from .idle_encoder import IdleEncoder
            self.prewarm = IdleEncoder()
            return self.prewarm.schedule(self.endpoint, output, python, environment, busy=busy, notify=notify)

    def release_idle(self, reason='requested'):
        """Give the card and host memory back on a caller's request; never touches a running request."""
        with self.mutex:
            if self.active:
                return dict(released=False, reason='busy')
            if self.process is None or self.process.poll() is not None:
                return dict(released=False, reason='nothing_held')
            row = self.release(reason)
        return dict(released=True, reason=reason, **{k: v for k, v in (row or {}).items()
                                                      if k in ('idle_seconds', 'worker_rss_bytes')})

    def placed_bytes(self):
        """Card bytes the last idle preload accounts for: its reported growth, else its budget, plus the context."""
        try:
            report = json.loads(Path(self.prewarm.report).read_text(encoding='utf-8'))
            before, after = report.get('gpu_bytes_before'), report.get('gpu_bytes_after')
            if type(before) is int and type(after) is int:
                known = max(0, after - before)
            else:
                known = max(0, int(report.get('gpu_budget_bytes') or 0))
        except (AttributeError, OSError, TypeError, ValueError):
            known = 0
        return known + PRELOAD_CONTEXT_BYTES

    def _watch(self, generation, since, hold, gpu):
        placing = False
        while True:
            time.sleep(.5)
            # Our own idle preload grows the card; only another program's growth counts,
            # so the reference is read again once placement has ended.
            unreadable = False
            if self.preloading():
                placing = True
            elif placing:
                placing = False
                used = gpu_used(gpu)
                own = gpu_own(gpu)
                if used is None:
                    unreadable = True  # Same as at the start of the hold: never keep without a reading.
                elif own is not None and self.own is not None and self.baseline is not None:
                    # Only the part this tree added moves the reference; another program's share stays visible.
                    self.baseline += max(0, own - self.own)
                    self.own = own
                elif self.baseline is not None:
                    # No per-process bytes (WDDM): only what the preload reports it placed, plus its
                    # context, moves the reference; the rest of the growth is another program's.
                    self.baseline += min(max(0, used - self.baseline), self.placed_bytes())
                else:
                    self.baseline = used
            reason = ('gpu_unreadable' if unreadable else
                      'idle' if hold is not None and time.monotonic() - since >= hold
                      else pressure(gpu, None if placing else self.baseline))
            with self.mutex:
                if generation != self.generation or self.active or self.process is None:
                    return
                if self.process.poll() is not None:
                    return  # The image or prompt model released it and recorded why.
                if reason:
                    self.release(reason)
                    return

    def release(self, reason):
        with self.mutex:
            if self.process is None or self.process.poll() is not None:
                self.close()
                return None
            row = dict(action='released', reason=reason, pid=self.process.pid,
                       idle_seconds=None if self.idle_since is None else round(time.monotonic() - self.idle_since, 1),
                       **worker_memory(self.process.pid))
            self.generation += 1
            self.idle_since = None
            self.close()
            return self.record(**row)

    def start(self, root, source, python, environment, busy=False):
        with self.mutex:
            self.generation += 1
            self.idle_since = None
            endpoint = self._start(root, source, python, environment)
            # Marked under the same lock, so a release cannot land between starting and using the worker.
            if busy:
                self.active = True
            return endpoint

    def _start(self, root, source, python, environment):
        import hashlib
        import psutil
        from .triton_compat import COMPILER_ENVIRONMENT_KEYS, environment as compiler_environment
        environment = compiler_environment(root, environment)
        self.stop_prewarm()
        source = Path(source).resolve()
        signature = hashlib.sha256()
        for file in sorted((source/'freevideo_engine').glob('*.py')):
            signature.update(file.name.encode())
            signature.update(file.read_bytes())
        compiler = tuple(environment.get(key) for key in COMPILER_ENVIRONMENT_KEYS)
        identity = (str(source), str(python), signature.hexdigest(), environment.get('CUDA_VISIBLE_DEVICES'), compiler)
        if self.process is not None and self.process.poll() is None and self.identity == identity:
            return str(self.endpoint)
        self.close()
        directory = Path(root)/'sessions'/('interactive-' + secrets.token_hex(8))
        directory.mkdir(parents=True, mode=0o700)
        self.endpoint = directory/'endpoint.json'
        family = 'AF_PIPE' if os.name == 'nt' else 'AF_UNIX'
        address = (r'\\.\pipe\FreeVideo-' + secrets.token_hex(16) if os.name == 'nt'
                   else str(directory/'worker.sock'))
        # Unix-domain paths have a short OS limit; use a private short directory.
        if os.name != 'nt' and len(address.encode()) >= 100:
            import tempfile
            socket_directory = Path(tempfile.mkdtemp(prefix='freevideo-session-'))
            address = str(socket_directory/'worker.sock')
        save(self.endpoint, dict(address=address, family=family, key=secrets.token_hex(32),
            owner_pid=os.getpid(), owner_created=psutil.Process().create_time(), source_sha256=identity[2]))
        if os.name != 'nt':
            self.endpoint.chmod(0o600)
        env = dict(environment, PYTHONPATH=str(source), PYTHONUNBUFFERED='1')
        env.pop(ENV, None)
        env.pop('FREEVIDEO_RUNTIME_LOCK_FD', None)
        env.pop(processes.HANDLE_ENV, None)
        with (directory/'session.log').open('wb') as log:
            self.process = processes.popen([str(python), '-B', '-m', 'freevideo_engine.resident_worker',
                '--endpoint', str(self.endpoint)], env=env, cwd=source, stdin=subprocess.DEVNULL,
                stdout=log, stderr=subprocess.STDOUT, start_new_session=True, supervise=True)
        self.identity = identity
        deadline = time.monotonic() + 15
        while time.monotonic() < deadline:
            if self.process.poll() is not None:
                raise RuntimeError('Resident worker could not start; see ' + str(directory/'session.log'))
            if (directory/'ready.json').exists():
                return str(self.endpoint)
            time.sleep(.05)
        self.close()
        raise TimeoutError('Resident worker startup timed out')

    def stop_prewarm(self):
        receipt = {}
        if self.prewarm is not None:
            stopped = self.prewarm.stop()
            from .encoder_diagnostics import prewarm_receipt
            receipt = prewarm_receipt(self.prewarm.report, gpu=True)
            receipt['worker_released'] = False
            self.prewarm = None
            # An interrupted native loader can be stuck in IO. Never grant
            # memory credit or start another request until it yielded/exited.
            if self.process is not None and self.process.poll() is None:
                try:
                    ready = json.loads(self.endpoint.with_name('ready.json').read_text(encoding='utf-8'))
                except (OSError, ValueError):
                    ready = {}
                if not stopped or ready.get('idle') is not True:
                    processes.stop(self.process, grace=1.)
                    self.process = None
                    receipt['worker_released'] = True
        return receipt

    def close(self):
        self.stop_prewarm()
        if self.process is not None:
            processes.stop(self.process, grace=2)
            self.process = None


def held_summary(decisions):
    """Roles the worker still holds after a request and why others were given back (fields for the kept event)."""
    held, evicted = {}, []
    for row in decisions if isinstance(decisions, list) else []:
        if not isinstance(row, dict) or not isinstance(row.get('role'), str):
            continue
        if row.get('action') in ('admit', 'park', 'reuse', 'unpark'):
            held[row['role']] = True
        elif row.get('action') == 'evict':
            held.pop(row['role'], None)
            evicted.append({'role': row['role'], 'reason': str(row.get('reason') or '')[:120]})
    return {'held': sorted(held), 'evicted': evicted}


def gpu_memory(gpu):
    """(used, total) bytes of the whole device, other programs included; None when unreadable."""
    try:
        from .background_room import Gpu
        device = Gpu(gpu)
        try:
            return device.memory() or None
        finally:
            device.close()
    except (OSError, RuntimeError, AttributeError, ValueError, ImportError):
        return None


def gpu_used(gpu):
    memory = gpu_memory(gpu)
    return memory[0] if memory else None


def gpu_own(gpu):
    """Card bytes held by this process tree (the kept worker and its leases), or None if not listed."""
    try:
        from .background_room import Gpu, family
        device = Gpu(gpu)
        try:
            rows = device.processes()
        finally:
            device.close()
        if rows is None or any(size is None for size in rows.values()):
            return None
        pids = family(os.getpid())
        return sum(size for pid, size in rows.items() if pid in pids)
    except (OSError, RuntimeError, AttributeError, ValueError, ImportError):
        return None


def pressure(gpu, baseline):
    """Why a kept idle worker should go now, or None."""
    try:
        from .background_room import available_memory
        if available_memory() < HOST_FLOOR:
            return 'host_memory_low'
    except (OSError, RuntimeError, AttributeError, ValueError, ImportError):
        pass
    if baseline is not None:
        # The idle worker does not allocate, so growth on the card is another program's.
        used = gpu_used(gpu)
        if used is None:
            return 'gpu_unreadable'
        if used - baseline >= OTHER_GPU_GROWTH:
            return 'other_program_gpu'
    return None


def worker_memory(pid):
    """Host memory the release returns, from the worker tree."""
    try:
        import psutil
        from .background_room import family
        rss = private = 0
        for member in family(pid):
            info = psutil.Process(member).memory_info()
            rss += info.rss
            private += getattr(info, 'private', 0) or getattr(info, 'rss', 0) - getattr(info, 'shared', 0)
        return dict(worker_rss_bytes=rss, worker_private_bytes=private)
    except Exception:
        return {}


OWNER = SessionOwner()


def serve(endpoint, runner, snapshot):
    """Shared transport with a CPU-injectable runner for lifecycle tests."""
    from multiprocessing.connection import Listener
    import psutil
    record = json.loads(Path(endpoint).read_text(encoding='utf-8'))
    hello = dict(pid=os.getpid(), created=psutil.Process().create_time())
    done = threading.Event()
    def owner_watch():
        while not done.wait(.5):
            if not alive(record['owner_pid'], record['owner_created']):
                os._exit(72)  # Owner crash must not leave a hidden CUDA worker.
    threading.Thread(target=owner_watch, daemon=True).start()
    try:
        with Listener(record['address'], family=record['family'], authkey=bytes.fromhex(record['key'])) as listener:
            ready_path = Path(endpoint).with_name('ready.json')
            save(ready_path, dict(hello, idle=True))
            while True:
                with listener.accept() as connection:
                    state = snapshot()
                    send(connection, dict(hello, allocator_config=state.get('allocator_config'),
                                          capabilities=list(state.get('capabilities', []))+['idle-cache-release']))
                    request = receive(connection, 10.)
                    if request.get('command') == 'status':
                        send(connection, dict(snapshot(), idle=True))
                        continue
                    if request.get('command') not in ('run', 'release'):
                        raise ValueError('Unknown resident worker command')
                    save(ready_path, dict(hello, idle=False))
                    descriptor = None
                    accepted = False
                    phase = 'runtime lease transfer'
                    stopped = threading.Event()
                    def request_watch():
                        while not stopped.wait(.25):
                            if not alive(request['owner_pid'], request['owner_created']):
                                os._exit(73)  # Preserve the unsettled attempt/lease evidence.
                    watcher = threading.Thread(target=request_watch, daemon=True)
                    previous = dict(os.environ)
                    try:
                        descriptor = _receive_lease(connection)
                        phase = 'runtime lease validation'
                        environment = execution_environment(request['environment'])
                        os.environ.clear()
                        os.environ.update(environment)
                        from .triton_compat import own_temporary_environment
                        own_temporary_environment()
                        os.environ.pop(processes.HANDLE_ENV, None)
                        os.environ['FREEVIDEO_RUNTIME_LOCK_FD'] = str(descriptor)
                        from .locking import runtime_lock
                        with runtime_lock():
                            send(connection, {'status': 'accepted'})
                            accepted = True
                            watcher.start()
                            code = runner(request) if request['command'] == 'run' else 0
                    except BaseException as error:
                        if not accepted:
                            failure = dict(status='rejected', phase=phase,
                                           error=type(error).__name__ + ': ' + str(error))
                            # Send the cause before closing the pipe. No argv,
                            # environment, IPC key or model data is logged.
                            try:
                                with Path(request['log']).open('a', encoding='utf-8') as log:
                                    log.write(json.dumps(failure, ensure_ascii=False) + '\n')
                            except OSError:
                                pass
                            try:
                                send(connection, failure)
                            except (EOFError, OSError):
                                pass
                        raise
                    finally:
                        stopped.set()
                        if watcher.is_alive():
                            watcher.join(timeout=1)
                        if descriptor is not None:
                            os.close(descriptor)
                        os.environ.clear()
                        os.environ.update(previous)
                    save(ready_path, dict(hello, idle=request['command'] != 'release'))
                    send(connection, {'returncode': code})
                    if request['command'] == 'release':
                        return
    finally:
        done.set()