Skip to content

armgit / PS5LM

Public Imported from GitHub
0

llama.cpp on a jailbroken PS5: run any GGUF model natively on the console, starting with Qwen 3.8

9.5 KB
Raw Blame
"""Host CLI: acquire one GGUF, estimate a split, and serve through real RPC.

No node enrollment, payload deployment or network listener changes are performed.
SPDX-License-Identifier: GPL-3.0-or-later
"""
from __future__ import annotations

import argparse
import copy
import hashlib
import json
import os
from pathlib import Path
import re
import signal
import subprocess
import sys

from .models import ModelError, fetch_model, inspect_file
from .planner import FleetError, MIB, inventory, plan

LLAMA_REVISION = '970b51ce7c9ba5fff0726c5e9994f1c2f0e4eedc'
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BINARY = ROOT / 'build/fleet-host/bin/llama-server'


def child_environment():
    # Environment CLI overrides could enable unrelated endpoints, routers, models
    # or caches. Do not silently let them invalidate the checked placement.
    env = {k: v for k, v in os.environ.items() if not k.startswith('LLAMA_ARG_')}
    env['GGML_RPC_NO_RDMA'] = '1'
    env['GGML_RPC_NO_COMM'] = '1'
    return env


def command(binary, model_path, placement, port=8081):
    nodes = placement['nodes']
    return [str(Path(binary).resolve()), '--model', str(Path(model_path).resolve()),
            '--rpc', ','.join(n['endpoint'] for n in nodes),
            '--device', ','.join(n['device'] for n in nodes),
            '--gpu-layers', 'all', '--split-mode', 'layer',
            '--tensor-split', ','.join(str(v) for v in placement['tensor_split']),
            '--fit', 'off', '--ctx-size', str(placement['context_tokens']), '--parallel', '1',
            '--cache-type-k', 'f16', '--cache-type-v', 'f16', '--flash-attn', 'off',
            '--load-mode', 'mmap', '--lazy-mode', 'off', '--no-op-offload',
            '--batch-size', '128', '--ubatch-size', '128', '--no-warmup',
            '--host', '127.0.0.1', '--port', str(port)]


def probe(binary, config, timeout=20):
    """Use the pinned client's own RPC handshake, never implement a second protocol.

    An offline node, incompatible server, unexpected device topology or timeout
    is an error. Free memory is a snapshot, not a reservation.
    """
    config = copy.deepcopy(inventory(config))
    args = [str(Path(binary).resolve()), '--rpc', ','.join(n['endpoint'] for n in config['nodes']), '--list-devices']
    try:
        result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, env=child_environment())
    except (OSError, subprocess.TimeoutExpired) as exc:
        raise FleetError(f'RPC preflight failed ({type(exc).__name__}); no model server started') from exc
    if result.returncode:
        raise FleetError(f'RPC preflight exited {result.returncode}; check worker/revision/network.\n' + result.stderr[-2000:])
    matches = re.findall(r'^\s*(RPC[0-9]+): ([0-9.]+:[0-9]+) \(([0-9]+) MiB, ([0-9]+) MiB free\)\s*$',
                         result.stdout, re.MULTILINE)
    if len(matches) != len(config['nodes']):
        raise FleetError('Expected exactly one RPC device per endpoint; refusing to guess topology')
    for index, (node, match) in enumerate(zip(config['nodes'], matches)):
        name, address, total_text, free_text = match
        if name != f'RPC{index}' or address != node['endpoint']:
            raise FleetError(f'{node["id"]}: RPC device order/endpoint differs from the requested inventory')
        total, free = int(total_text), int(free_text)
        if not 0 < free <= total:
            raise FleetError(f'{node["id"]}: worker did not report valid free memory')
        node['memory_mib'] = min(node['memory_mib'], free)
    return config


def checksum(path):
    digest = hashlib.sha256()
    with Path(path).open('rb') as stream:
        for chunk in iter(lambda: stream.read(MIB), b''):
            digest.update(chunk)
    return digest.hexdigest()


def read_json(path):
    if Path(path).stat().st_size > MIB:
        raise FleetError('JSON configuration exceeds 1 MiB')
    return json.loads(Path(path).read_text())


def verify_binary(binary):
    binary = Path(binary).resolve()
    if not binary.is_file() or not os.access(binary, os.X_OK):
        raise FleetError(f'No executable at {binary}; run scripts/build-fleet-host.sh first')
    manifest = binary.parent.parent / 'fleet-build.json'
    if not manifest.is_file():
        raise FleetError(f'Missing build manifest {manifest}; use scripts/build-fleet-host.sh')
    data = read_json(manifest)
    if data.get('llama_revision') != LLAMA_REVISION:
        raise FleetError('Host build revision differs from the pinned PS5 worker; rebuild both together')
    if data.get('server_sha256') != checksum(binary):
        raise FleetError('Host binary checksum differs from fleet-build.json; rebuild the host')


class _StopSignal(Exception):
    def __init__(self, signum):
        self.signum = signum


def run_foreground(argv):
    """Own the server until exit; wrapper SIGTERM must not orphan RPC users."""
    child = None

    def stop(signum, _frame):
        raise _StopSignal(signum)

    previous = signal.signal(signal.SIGTERM, stop)
    try:
        child = subprocess.Popen(argv, env=child_environment())
        return child.wait()
    except KeyboardInterrupt:
        return 130
    except _StopSignal as exc:
        return 128 + exc.signum
    finally:
        signal.signal(signal.SIGTERM, previous)
        if child is not None and child.poll() is None:
            child.terminate()
            try:
                child.wait(timeout=10)
            except subprocess.TimeoutExpired:
                child.kill()
                child.wait()


def main(argv=None):
    parser = argparse.ArgumentParser(description='Experimental PS5LM single-model layer fleet (not a load balancer)')
    parser.add_argument('action', choices=('plan', 'serve'))
    parser.add_argument('--model', required=True, help='single-file local GGUF or public HTTPS URL')
    parser.add_argument('--nodes', required=True, help='inventory JSON; see fleet/nodes.example.json')
    parser.add_argument('--context', type=int, default=4096, help='total context for one sequence, default 4096')
    parser.add_argument('--binary', default=str(DEFAULT_BINARY))
    parser.add_argument('--cache-dir', default=str(ROOT / 'models/fleet-cache'))
    parser.add_argument('--sha256', help='optional expected full-file SHA256')
    parser.add_argument('--max-download-gib', type=int, default=32)
    parser.add_argument('--port', type=int, default=8081, help='controller API port, localhost only')
    parser.add_argument('--plan-out', help='write reviewed JSON plan (not executable state)')
    parser.add_argument('--ack-insecure-rpc', action='store_true',
                        help='acknowledge RPC is unauthenticated and all endpoints are on an isolated trusted network')
    args = parser.parse_args(argv)
    try:
        if not 1 <= args.port <= 65535 or not 1 <= args.max_download_gib <= 128:
            raise FleetError('Port must be 1..65535 and download limit 1..128 GiB')
        if args.sha256 and not re.fullmatch(r'[0-9a-fA-F]{64}', args.sha256):
            raise FleetError('--sha256 must contain exactly 64 hexadecimal characters')
        config = inventory(read_json(args.nodes))
        if args.action == 'serve':
            if not args.ack_insecure_rpc:
                raise FleetError('Serving requires --ack-insecure-rpc; isolate and trust every RPC peer first')
            verify_binary(args.binary)
            # Fail early, before downloading a large model. Probe again immediately
            # before execution because downloading/hashing may take a long time.
            config = probe(args.binary, config)
        if args.model.startswith('https://'):
            path, digest = fetch_model(args.model, args.cache_dir, max_bytes=args.max_download_gib * 1024**3,
                                       max_cache_bytes=max(64, args.max_download_gib) * 1024**3, max_seconds=3600)
        elif '://' in args.model:
            raise FleetError('Only local paths and public HTTPS model URLs are accepted')
        else:
            path = Path(args.model).resolve()
            # Reject devices/directories before hashing, and inspect metadata first.
            inspect_file(path)
            digest = checksum(path)
        if args.sha256 and digest.lower() != args.sha256.lower():
            raise FleetError('Full model SHA256 does not match --sha256; no server started')
        model = inspect_file(path)
        if args.action == 'serve':
            config = probe(args.binary, config)
        placement = plan(model, config, args.context)
        placement.update(model_path=str(Path(path).resolve()), model_sha256=digest,
                         llama_revision=LLAMA_REVISION,
                         memory_source='min(operator_budget,live_RPC_snapshot)' if args.action == 'serve' else 'operator_budgets_only')
        placement['argv'] = command(args.binary, path, placement, args.port)
        encoded = json.dumps(placement, indent=2) + '\n'
        if args.plan_out:
            Path(args.plan_out).write_text(encoded)
        print(encoded, end='', flush=True)
        if args.action == 'plan':
            return 0
        print('Starting host llama-server; wait for its ready/health response. A printed plan is not successful inference.', file=sys.stderr)
        # Foreground process: no daemon, enrollment, automatic console deployment,
        # or insecure public API. Ctrl+C reaches both and stops this session.
        return run_foreground(placement['argv'])
    except (FleetError, ModelError, OSError, ValueError) as exc:
        print(f'ps5lm-fleet: {exc}', file=sys.stderr)
        return 2


if __name__ == '__main__':
    raise SystemExit(main())