Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
13.4 KB
Raw Blame
/**
 * Engine supervisor.
 *
 * Owns the inference process: resolves a binary, translates the tuning
 * configuration into real flags, spawns it, waits for health, keeps a log ring
 * buffer, and polls the engine's own Prometheus metrics so the Monitor shows
 * measured numbers rather than guesses.
 *
 * Supported backends, in preference order:
 *   1. llama.cpp `llama-server` (bundled under $STRATA_HOME/engine, or on PATH)
 *   2. any already-running OpenAI-compatible server on localhost
 */
import { CONTEXT_TOKENS } from './catalog.mjs';
import { spawn } from 'node:child_process';
import { existsSync, readdirSync, statSync } from 'node:fs';
import { join } from 'node:path';
import { MODELS_DIR } from './store.mjs';
import { which } from './util.mjs';

const LOG_LIMIT = 400;

const state = {
  status: 'stopped', // stopped | starting | running | stopping | error
  binary: null,
  binarySource: null,
  pid: null,
  port: 8080,
  modelId: null,
  modelPath: null,
  args: [],
  command: '',
  startedAt: null,
  error: null,
  logs: [],
  metrics: {
    source: 'none',
    promptTokens: 0,
    predictedTokens: 0,
    promptTps: 0,
    outputTps: 0,
    kvUsage: 0,
    kvTokens: 0,
    requestsProcessing: 0,
    requestsDeferred: 0,
    sampledAt: null,
  },
};

let child = null;
let pollTimer = null;
let hubRef = null;
let mcpAt=0,mcpStatus=null,mcpPending=false;

export function init(hub) {
  hubRef = hub;
}

/* ------------------------------------------------------------------ binary --- */

const BINARY_NAMES = ['llama-server'];

export function findBinary() {
  const candidates = [];

  if (process.env.STRATA_LLAMA_SERVER) candidates.push({ path: process.env.STRATA_LLAMA_SERVER, source: 'STRATA_LLAMA_SERVER' });

  // A build downloaded into the app's own folder wins over anything on PATH:
  // it is the version Strata was tested against.
  try {
    const engineDir = join(MODELS_DIR, '..', 'engine');
    if (existsSync(engineDir)) {
      for (const entry of readdirSync(engineDir)) {
        const dir = join(engineDir, entry);
        if (!statSync(dir).isDirectory()) continue;
        for (const name of BINARY_NAMES) {
          const full = join(dir, name);
          if (existsSync(full)) candidates.push({ path: full, source: 'bundled' });
        }
      }
    }
  } catch {
    /* ignore unreadable engine dir */
  }

  for (const name of BINARY_NAMES) {
    candidates.push({ path: `/opt/homebrew/bin/${name}`, source: 'homebrew' });
    candidates.push({ path: `/usr/local/bin/${name}`, source: 'homebrew' });
  }

  return candidates.find((c) => existsSync(c.path)) ?? null;
}

export async function resolveBinary() {
  const found = findBinary();
  if (found) {
    state.binary = found.path;
    state.binarySource = found.source;
    return found;
  }
  const onPath = await which('llama-server');
  if (onPath) {
    state.binary = onPath;
    state.binarySource = 'PATH';
    return { path: onPath, source: 'PATH' };
  }
  return null;
}

/* --------------------------------------------------------------- arguments --- */

const KV_TYPES = { INT8: 'q8_0', FP16: 'f16', Q8_0: 'q8_0', Q4_0: 'q4_0' };

/**
 * Turn the tuning configuration into a real llama-server command line.
 * Exported so the UI can show exactly what will run.
 */
export function buildArgs({ modelPath, config, hardware, port = state.port }) {
  const beginner = config.beginner ?? {};
  const params = new Map((config.expert ?? []).map((p) => [p.id, p.value]));
  const num = (id, fallback) => {
    const v = Number(params.get(id));
    return Number.isFinite(v) ? v : fallback;
  };
  const ctxLabel = beginner.contextLength === 'recommended' ? '128K' : String(beginner.contextLength).toUpperCase();
  const ctx = CONTEXT_TOKENS[ctxLabel.toLowerCase()] ?? 32768;

  const args = [
    '-m', modelPath,
    '--host', '127.0.0.1',
    '--port', String(port),
    '-c', String(ctx),
    '--metrics',
    '--slots',
    '--jinja',
    '--no-webui',
  ];

  // GPU offload — on unified memory "all layers" is the right default.
  const split = String(params.get('gpu.layers') ?? 'Automatic');
  const ngl = split === 'CPU only' ? 0 : split === 'Quarter' ? Math.max(1, Math.round(Number(config.layerCount ?? 48) / 4)) : split === 'Half' ? Math.max(1, Math.round(Number(config.layerCount ?? 48) / 2)) : 999;
  args.push('-ngl', String(ngl));

  const threads = num('perf.threads', Math.max(4, (hardware?.cpu?.cores ?? 8) - 2));
  args.push('-t', String(threads));

  const kv = KV_TYPES[String(params.get('ctx.kvMode') ?? 'INT8')] ?? 'q8_0';
  args.push('-ctk', kv, '-ctv', kv);

  const batch = num('perf.batch', 2048);
  args.push('-b', String(batch), '-ub', String(Math.max(128, Math.min(512, Math.round(batch / 4)))));

  if (params.get('perf.flashAttn') === false) args.push('-fa', 'off');
  else args.push('-fa', 'on');

  if (params.get('mem.mmap') === false) args.push('--no-mmap');
  if (params.get('mem.lock') === true) args.push('--mlock');

  const parallel = Math.max(1, num('server.parallel', 1));
  args.push('-np', String(parallel));

  // Sampling defaults applied to every request by the server.
  args.push(
    '--temp', String(params.get('sample.temp') ?? 0.6),
    '--top-p', String(params.get('sample.topP') ?? 0.95),
    '--top-k', String(params.get('sample.topK') ?? 40),
    '--repeat-penalty', String(params.get('sample.repeat') ?? 1.05),
  );
  const seed = Number(params.get('sample.seed') ?? -1);
  if (seed >= 0) args.push('--seed', String(seed));

  if (config.serving?.apiKey) args.push('--api-key', config.serving.apiKey);

  return args;
}

export function describeCommand() {
  return [state.binary, ...state.args].filter(Boolean).join(' ');
}

/* ---------------------------------------------------------------- lifecycle --- */

function log(stream, line) {
  const entry = { at: new Date().toISOString(), stream, line };
  state.logs.push(entry);
  if (state.logs.length > LOG_LIMIT) state.logs.shift();
  hubRef?.send('engine-log', entry);
}

export function logs(limit = 200) {
  return state.logs.slice(-limit);
}

async function waitForHealth(port, timeoutMs = 90_000) {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    try {
      const res = await fetch(`http://127.0.0.1:${port}/health`, { signal: AbortSignal.timeout(1500) });
      const body = await res.json().catch(() => ({}));
      if (res.ok && body.status === 'ok') return true;
      if (body.status) log('supervisor', `engine reports: ${body.status}`);
    } catch {
      /* not up yet */
    }
    if (child?.exitCode !== null && child?.exitCode !== undefined) {
      throw new Error(`Engine exited early with code ${child.exitCode}`);
    }
    await new Promise((r) => setTimeout(r, 700));
  }
  throw new Error('Engine did not become healthy in time');
}

export async function start({ modelId, modelPath, port = 8080, config, hardware, timeoutMs = 120_000 }) {
  if (state.status === 'running' && state.modelId === modelId && state.port === port) {
    return status();
  }
  if (child) await stop();

  const binary = await resolveBinary();
  if (!binary) {
    state.status = 'error';
    state.error = 'No llama-server binary found. Download the engine from Settings → Engine.';
    throw new Error(state.error);
  }
  if (!existsSync(modelPath)) {
    state.status = 'error';
    state.error = `Model file missing: ${modelPath}`;
    throw new Error(state.error);
  }

  state.status = 'starting';
  state.port = port;
  state.modelId = modelId;
  state.modelPath = modelPath;
  state.error = null;
  state.logs = [];
  state.args = buildArgs({ modelPath, config, hardware, port });
  state.command = describeCommand();
  hubRef?.send('engine-status', status());

  log('supervisor', `$ ${state.command}`);

  child = spawn(binary.path, state.args, {
    cwd: join(MODELS_DIR, '..'),
    env: { ...process.env, LLAMA_CACHE: join(MODELS_DIR, '..', 'cache') },
    stdio: ['ignore', 'pipe', 'pipe'],
  });

  state.pid = child.pid;
  state.startedAt = new Date().toISOString();

  const onData = (stream) => (chunk) => {
    for (const line of chunk.toString().split('\n')) {
      if (line.trim()) log(stream, line.replace(/\s+$/, ''));
    }
  };
  child.stdout.on('data', onData('out'));
  child.stderr.on('data', onData('err'));

  child.on('exit', (code, signal) => {
    log('supervisor', `engine exited (${signal ?? code})`);
    child = null;
    state.pid = null;
    if (state.status !== 'stopping') {
      state.status = code === 0 ? 'stopped' : 'error';
      if (code !== 0 && code !== null) state.error = `Engine exited with code ${code}`;
    } else {
      state.status = 'stopped';
    }
    stopPolling();
    hubRef?.send('engine-status', status());
  });

  try {
    await waitForHealth(port, timeoutMs);
  } catch (err) {
    state.status = 'error';
    state.error = err.message;
    hubRef?.send('engine-status', status());
    await stop();
    throw err;
  }

  state.status = 'running';
  state.startedAt = new Date().toISOString();
  startPolling();
  hubRef?.send('engine-status', status());
  log('supervisor', 'engine ready');
  return status();
}

export async function stop() {
  if (!child) {
    state.status = 'stopped';
    state.pid = null;
    stopPolling();
    hubRef?.send('engine-status', status());
    return status();
  }
  state.status = 'stopping';
  hubRef?.send('engine-status', status());
  const proc = child;
  proc.kill('SIGTERM');
  const exited = await Promise.race([
    new Promise((r) => proc.once('exit', () => r(true))),
    new Promise((r) => setTimeout(() => r(false), 4000)),
  ]);
  if (!exited) {
    log('supervisor', 'engine ignored SIGTERM — killing');
    proc.kill('SIGKILL');
  }
  child = null;
  state.status = 'stopped';
  state.pid = null;
  state.startedAt = null;
  stopPolling();
  hubRef?.send('engine-status', status());
  return status();
}

export async function restart(opts) {
  await stop();
  await new Promise((r) => setTimeout(r, 400));
  return start(opts);
}

/* ------------------------------------------------------------------ metrics --- */

function startPolling() {
  stopPolling();
  pollTimer = setInterval(() => void pollMetrics(), 1000);
  pollTimer.unref?.();
  void pollMetrics();
}

function stopPolling() {
  if (pollTimer) clearInterval(pollTimer);
  pollTimer = null;
}

function parsePrometheus(text) {
  const out = {};
  for (const line of text.split('\n')) {
    if (!line || line.startsWith('#')) continue;
    const match = /^([a-zA-Z0-9_:]+)(\{[^}]*\})?\s+([0-9.eE+-]+)$/.exec(line.trim());
    if (!match) continue;
    out[match[1]] = Number(match[3]);
  }
  return out;
}

/** Original Strata serves JSON; llama.cpp serves Prometheus. Preserve the original dashboard facts. */
export function parseEngineMetrics(text) {
  if(text.trim().startsWith('{')) {
    const dashboard=JSON.parse(text);
    if(!dashboard.engine||!dashboard.live)throw new Error('Unrecognized engine metrics');
    const live=dashboard.live;
    return {source:'engine',dashboard,promptTokens:live.prompt_tokens??0,predictedTokens:live.generated??0,promptTps:live.prefill_tok_s_mean??0,outputTps:live.tok_s??0,kvUsage:0,kvTokens:0,requestsProcessing:live.running??(['reading','generating'].includes(live.state)?1:0),requestsDeferred:live.queued??0};
  }
  const m=parsePrometheus(text);
  if(!Object.keys(m).some(key=>/prompt_tokens|tokens_predicted|predicted_tokens|kv_cache|requests_processing/.test(key)))throw new Error('Unrecognized engine metrics');
  const p=name=>m[`llamacpp:${name}`]??m[name]??0;
  return {source:'engine',promptTokens:p('prompt_tokens_total'),predictedTokens:p('tokens_predicted_total'),promptTps:p('prompt_tokens_seconds'),outputTps:p('predicted_tokens_seconds'),kvUsage:p('kv_cache_usage_ratio'),kvTokens:p('kv_cache_tokens'),requestsProcessing:p('requests_processing'),requestsDeferred:p('requests_deferred')};
}

export async function pollMetrics() {
  if (state.status !== 'running') return state.metrics;
  try {
    const res = await fetch(`http://127.0.0.1:${state.port}/metrics${state.metrics.dashboard?'?requests=all':''}`, {
      signal: AbortSignal.timeout(1500),
      headers: state.apiKey ? { authorization: `Bearer ${state.apiKey}` } : undefined,
    });
    if (!res.ok) throw new Error(String(res.status));
    state.metrics = { ...parseEngineMetrics(await res.text()), sampledAt: new Date().toISOString() };
    if(state.metrics.dashboard) {
      state.metrics.dashboard.mcp=mcpStatus;
      if(!mcpPending&&Date.now()-mcpAt>10000) {
        mcpPending=true;mcpAt=Date.now();
        void fetch(`http://127.0.0.1:${state.port}/mcp`,{signal:AbortSignal.timeout(1500),headers:state.apiKey?{authorization:`Bearer ${state.apiKey}`}:undefined})
          .then(async response=>{mcpStatus=response.ok?await response.json():null;})
          .catch(()=>{mcpStatus=null;}).finally(()=>{mcpPending=false;});
      }
    }
  } catch {
    state.metrics = { ...state.metrics, source: 'unavailable', sampledAt: new Date().toISOString() };
  }
  return state.metrics;
}

/* ------------------------------------------------------------------- status --- */

export function status() {
  const binary = state.binary ? { path: state.binary, source: state.binarySource } : null;
  return {
    status: state.status,
    running: state.status === 'running',
    binary,
    pid: state.pid,
    port: state.port,
    modelId: state.modelId,
    modelPath: state.modelPath,
    startedAt: state.startedAt,
    error: state.error,
    args: state.args,
    command: state.command,
    metrics: state.metrics,
    logCount: state.logs.length,
  };
}

export function currentPort() {
  return state.port;
}

export function isRunning() {
  return state.status === 'running';
}