Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
5.7 KB
Raw Blame
/** Serving surface: API keys, access policy, and the request log. */
import { randomBytes } from 'node:crypto';
import os from 'node:os';
import { get as getStore, update } from './store.mjs';
import { nowIso, round, uid } from './util.mjs';

const requests = [];
const MAX_REQUESTS = 60;
const cancellations = new Map();
let revision = Date.now();

let hubRef = null;
let tunnel = { status: 'idle', mode: null, url: null, error: null, installed: false, originUrl: null };
export function setTunnel(value) { tunnel = value; emit(); }
const tunnelActive = () => ['starting', 'connected', 'stopping'].includes(tunnel.status);
let sharing = { active: false, url: null };
export function setSharing(value) { sharing = value; emit(); }
let port = 8080;
let running = false;
let bindHost = '127.0.0.1';
const isLocal = () => ['127.0.0.1', 'localhost', '::1'].includes(bindHost);

export function init(hub, opts = {}) {
  hubRef = hub;
  port = opts.port ?? port;
  running = true;
  bindHost = opts.host ?? '127.0.0.1';
}

export function baseUrl() {
  const host = isLocal() ? '127.0.0.1' : localIp();
  return `http://${host}:${port}/v1`;
}

export function state() {
  const s = getStore();
  const active = requests.filter((r) => r.state === 'generating' || r.state === 'queued');
  return {
    revision,
    running,
    baseUrl: sharing.active ? sharing.url : baseUrl(),
    sharing,
    tunnel,
    port,
    endpoints: { openai: '/chat/completions', anthropic: '/messages', models: '/models' },
    access: {
      otherDevices: !isLocal() || sharing.active || tunnelActive(),
      apiKeySet: !!s.serving?.apiKey,
      apiKeyHint: s.serving?.apiKey ? `Ends in ${s.serving.apiKey.slice(-4)}` : null,
      allowOrigin: '*',
    },
    stats: {
      running: active.filter((r) => r.state === 'generating').length,
      waiting: active.filter((r) => r.state === 'queued').length,
      completed: requests.filter((r) => r.state === 'done').length,
      failed: requests.filter((r) => r.state === 'error').length,
      tokensOut: requests.reduce((sum, r) => sum + (r.tokensOut || 0), 0),
    },
    requests: [...requests].reverse(),
    pauseServing: !!s.serving?.pauseServing,
  };
}

function emit() {
  revision = Math.max(Date.now(), revision + 1);
  hubRef?.send('serving', state());
}

export function patch(body) {
  const s = getStore();
  const serving = { ...(s.serving ?? {}) };
  if (typeof body.otherDevices === 'boolean' && body.otherDevices !== !isLocal()) throw new Error('Change the host listen address and restart to update network access.');
  // The key is optional (owner decision, 0.2.2): removing it while sharing or the tunnel runs is allowed; the app warns in plain words (local-share.mjs, cloudflare.mjs).
  if (body.apiKey === null) serving.apiKey = null;
  else if (typeof body.apiKey === 'string' && body.apiKey.trim()) serving.apiKey = body.apiKey.trim();
  else if (body.generateKey) serving.apiKey = `sk-strata-${randomBytes(12).toString('hex')}`;
  if (typeof body.pauseServing === 'boolean') serving.pauseServing = body.pauseServing;
  update({ serving });
  emit();
  return state();
}

export function authorized(req) {
  const key = getStore().serving?.apiKey;
  if (!key) return true;
  const header = req.headers.authorization ?? '';
  const bearer = header.startsWith('Bearer ') ? header.slice(7) : '';
  const alt = req.headers['x-api-key'];
  return bearer === key || alt === key;
}

export function open({ client, endpoint, tokensIn = null, model, instanceId, gpuIds, cancel }) {
  const entry = {
    id: uid('req'),
    at: nowIso(),
    client,
    endpoint,
    state: 'queued',
    tokensIn,
    tokensOut: null,
    durationMs: 0,
    model: model ?? 'local',
    instanceId, gpuIds,
    cancellable: typeof cancel === 'function',
    cancelRequested: false,
    usageSource: 'unavailable',
  };
  if (entry.cancellable) cancellations.set(entry.id, cancel);
  requests.push(entry);
  if (requests.length > MAX_REQUESTS) {
    const oldestCompleted = requests.findIndex(r => !['generating', 'queued'].includes(r.state));
    if (oldestCompleted >= 0) requests.splice(oldestCompleted, 1);
  }
  emit();
  return entry;
}

export function mark(id, patchObj) {
  const entry = requests.find((r) => r.id === id);
  if (!entry) return;
  Object.assign(entry, patchObj);
  emit();
  return entry;
}

export function finish(id, { tokensOut, durationMs, error }) {
  const entry = requests.find((r) => r.id === id);
  if (!entry) return;
  entry.state = error ? 'error' : 'done';
  entry.tokensOut = tokensOut ?? entry.tokensOut;
  entry.durationMs = durationMs ?? entry.durationMs;
  if (error) entry.error = error;
  entry.cancellable = false;
  cancellations.delete(id);
  emit();
}

export function cancel(id) {
  const entry = requests.find((r) => r.id === id);
  if (!entry) throw new Error('This request is no longer retained.');
  if (!['generating', 'queued'].includes(entry.state)) return state();
  const abort = cancellations.get(id);
  if (!abort) throw new Error('This request cannot be cancelled by this host.');
  if (!entry.cancelRequested) { entry.cancelRequested = true; abort(); }
  emit();
  return state();
}

export function rateSample() {
  const done = requests.filter((r) => r.state === 'done' && r.tokensOut > 0);
  if (!done.length) return 0;
  const tps = done.filter((r) => r.tps > 0).map((r) => r.tps);     // answer speeds (decode time), not tokens over the whole request
  if (!tps.length) return 0;
  return round(tps.reduce((a, b) => a + b, 0) / tps.length, 1);
}

function localIp() {
  try {
    const nets = os.networkInterfaces();
    for (const list of Object.values(nets)) {
      for (const net of list ?? []) {
        if (net.family === 'IPv4' && !net.internal) return net.address;
      }
    }
  } catch {
    /* fall through */
  }
  return '127.0.0.1';
}