Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
17.1 KB
Raw Blame
/** Live monitor: one state machine for phase, throughput, memory and thermals. */
import { variant as catalogVariant, footprint } from './catalog.mjs';
import { detect, cachedHardware } from './hardware.mjs';
import { readGpuTelemetry, gpuMemoryTotals } from './device-probes.mjs';
import { createSystemSampler } from './system-telemetry.mjs';
import { sensorPace, WATCH_MS } from './sensor-pace.mjs';
import { readTemperatures } from './temperature-telemetry.mjs';
import { addEnergy, energyWh } from './energy-meter.mjs';
import { state as servingState } from './serving.mjs';
import { originalDashboard } from './monitor-dashboard.mjs';
import { get as getStore } from './store.mjs';
import * as supervisor from './supervisor.mjs';
import { clamp, round } from './util.mjs';

const HISTORY = 60;

const state = {
  phase: 'idle',
  phaseLabel: 'Waiting for a request',
  variantId: null,
  prompt: { done: 0, total: 0, pct: 0 },
  generation: { done:null,total:null },
  queue: { running: 0, waiting: 0 },
  context: { usedTokens: 0, maxTokens: 32768, pct: 0 },
  expertCache: { used: 48, total: 64 },
  hardware: {
    gpus: [],
    gpuLoad: 3,
    vramUsedGb: 2.1,
    vramTotalGb: 24,
    ramUsedGb: 14,
    ramTotalGb: 32,
    gpuTempC: 44,
    tempLimitC: 100,
  },
  history: [],
  totals: {
    tokensIn: 0,
    tokensOut: 0,
    requests: 0,
    avgTps: 0,
    since: new Date().toISOString(),
    /** Time to first token, averaged over the session. */
    ttftMs: 0,
    /** Prompt-cache accounting. Everything the engine read is either reused
     *  from the previous request's KV state or processed fresh. */
    cachedTokens: 0,
    uncachedTokens: 0,
    usageRequests: 0,
    latencyRequests: 0,
  },
  /** Where the context window actually went, so the breakdown can be shown. */
  contextParts: { system: 0, tools: 0, messages: 0 },
  target: { gpuLoad: 3, outputTps: 0, promptTps: 0 },
  /** Which numbers come from the engine and which Strata has to estimate. */
  telemetry: { throughput: 'estimated', thermals: 'estimated', memory: 'estimated', ttftMs: null },
};

let hubRef = null;
let hwCache = null;
let ticker = null;
let memoryWarningShown = false;
let sensorsPending = false, sensorsAt = 0;
/* The Monitor screen asks (GET /api/monitor) every few seconds while it is open; a model that is loaded counts as live (#4066). */
let watchedUntil = 0, liveProbe = () => false;
export function watch() {
  const wasIdle = Date.now() >= watchedUntil;
  watchedUntil = Date.now() + WATCH_MS;
  if (wasIdle) sensorsAt = 0;                                    // opening the Monitor after a quiet spell reads the card at once
}
export function setLiveProbe(probe) { liveProbe = typeof probe === 'function' ? probe : () => false; }
let systemReading=null;
/* CPU and SSD temperatures (null where this PC cannot give one) and the GPU energy used since the host started (#3628). */
let temperatures={cpuTempC:null,ssdTempC:null};
const sampleSystem=createSystemSampler();
const finite = value => typeof value==='number'&&Number.isFinite(value);
function aggregate(readings,key,mode='sum') {
  const values=readings.map(g=>g[key]);
  if(!values.length||values.some(v=>!finite(v)))return null;
  return mode==='max'?Math.max(...values):values.reduce((n,v)=>n+v,0)/(mode==='mean'?values.length:1);
}

function hardwareSnapshot() {
  const demo=process.env.STRATA_REAL_ENGINE!=='1',hw=state.hardware;
  const devices=cachedHardware()?.gpus??hwCache?.gpus??[];
  const memoryPools=Math.max(1,devices.filter(g=>!g.unified).length+Number(devices.some(g=>g.unified)));
  const readings=demo?devices.map((gpu,i)=>({id:gpu.id??`${gpu.vendor}:${gpu.index}`,source:'demo',sampledAt:new Date().toISOString(),utilizationPct:Math.round(clamp(hw.gpuLoad+i*2,0,100)),temperatureC:Math.round(hw.gpuTempC+i),memoryUsedGb:round(Math.min(gpu.vramGb,hw.vramUsedGb/memoryPools),2),powerW:gpu.unified?null:round(35+hw.gpuLoad*2,0),powerLimitW:gpu.unified?null:300,pcieGen:gpu.unified?null:4,pcieGenMax:gpu.unified?null:4,pcieWidth:gpu.unified?null:16,pcieRxMbs:gpu.unified?null:round(hw.gpuLoad*1.2,1),fanPct:gpu.unified?null:Math.round(30+hw.gpuLoad*0.5),clockMhz:gpu.unified?null:Math.round(1400+hw.gpuLoad*14)})):hw.gpus.map(g=>({...g}));
  const memory = gpuMemoryTotals(devices,readings);
  return {
    gpus:readings,cpus:systemReading?.cpus??[],cpuLoad:systemReading?.cpuLoad??null,
    gpuLoad:aggregate(readings,'utilizationPct','mean'),
    vramUsedGb:memory.used,
    vramTotalGb:memory.total,
    ramUsedGb:systemReading?.ramUsedGb??null,ramTotalGb:systemReading?.ramTotalGb??hw.ramTotalGb,
    gpuTempC:aggregate(readings,'temperatureC','max'),tempLimitC:hw.tempLimitC,
    powerW:aggregate(readings,'powerW'),powerLimitW:aggregate(readings,'powerLimitW'),
    pcieRxMbs:aggregate(readings,'pcieRxMbs'),
    cpuTempC:demo?54:temperatures.cpuTempC,ssdTempC:demo?41:temperatures.ssdTempC,energyWh:round(energyWh(),3),
    diskReadGbs:systemReading?.diskReadGbs??null,diskWriteMbs:systemReading?.diskWriteMbs??null,
  };
}

export function init(hub, hardware) {
  hubRef = hub;
  hwCache = hardware;
  supervisor.init(hub);
  const total = hardware.memory.totalGb;
  state.hardware.ramTotalGb = Math.round(total);
  state.hardware.ramUsedGb = Math.round(hardware.memory.usedGb);
  state.hardware.vramTotalGb = Math.round(hardware.accelerator.usableVramGb);
  state.hardware.tempLimitC = hardware.accelerator.unified ? 105 : 90;
  state.expertCache = { used: 0, total: 64 };
  state.history = [];
  state.totals.since = new Date().toISOString();

  if (ticker) clearInterval(ticker);
  ticker = setInterval(tick, 1000);
  ticker.unref?.();
}

function approach(current, target, factor = 0.22) {
  return current + (target - current) * factor;
}

function tick() {
  const pace=sensorPace({live:supervisor.status().running||!!liveProbe(),watchedUntil});
  if(!sensorsPending&&Date.now()-sensorsAt>=pace.intervalMs){
    sensorsPending=true;sensorsAt=Date.now();
    const hardware=cachedHardware()??hwCache;
    void Promise.all([readGpuTelemetry(hardware?.gpus??[],{transfer:pace.transfer}),sampleSystem(hardware?.processors??[]),(process.env.STRATA_REAL_ENGINE==='1'?readTemperatures():Promise.resolve(temperatures)).catch(()=>({cpuTempC:null,ssdTempC:null}))])
      .then(([readings,system,temps])=>{state.hardware.gpus=readings;systemReading=system;temperatures=temps;})
      .catch(()=>{state.hardware.gpus=[];systemReading=null;})
      .finally(()=>{sensorsPending=false;});
  }
  addEnergy(hardwareSnapshot().powerW,Date.now());
  const hw = state.hardware;
  const busy = state.phase === 'generating' || state.phase === 'reading' || state.phase === 'thinking';

  // Real numbers whenever a real engine is running.
  const engine = supervisor.status();
  const engineMetrics = engine.metrics;
  if(state.hardware.ramUsedGb > state.hardware.ramTotalGb && !memoryWarningShown){
    console.warn('Memory reading exceeded physical capacity; clamping the monitor snapshot.');
    memoryWarningShown = true;
  }
  if(state.hardware.ramUsedGb <= state.hardware.ramTotalGb) memoryWarningShown = false;
  if (engine.running && engineMetrics.source === 'engine') {
    state.telemetry.throughput = 'engine';
    state.telemetry.memory = 'estimated';
    if (state.phase === 'generating' && engineMetrics.outputTps > 0) state.target.outputTps = engineMetrics.outputTps;
    if (engineMetrics.promptTps > 0) state.target.promptTps = engineMetrics.promptTps;
    if (engineMetrics.kvUsage > 0) {
      state.context.usedTokens = Math.round(engineMetrics.kvUsage * state.context.maxTokens);
    }
    const v = engine.modelId ? catalogVariant(engine.modelId) : null;
    if (v) {
      const kvUsed = (state.context.usedTokens / 1024) * (v.kvKbPerToken / 1024);
      const estimate = footprint(v, state.context.usedTokens || 1024).weightsGb + kvUsed;
      state.hardware.vramUsedGb = clamp(estimate, 0.2, state.hardware.vramTotalGb);
    }
    state.queue.running = engineMetrics.requestsProcessing || (busy ? 1 : 0);
    state.queue.waiting = engineMetrics.requestsDeferred || 0;
  } else {
    state.telemetry.throughput = process.env.STRATA_REAL_ENGINE !== '1' ? 'demo' : 'none';
  }

  state.target.gpuLoad = busy ? (state.phase === 'reading' ? 88 : 82) : state.phase === 'loading' ? 64 : 3;
  hw.gpuLoad = clamp(approach(hw.gpuLoad, state.target.gpuLoad) + (Math.random() - 0.5) * 3, 1, 100);

  const modelVram = state.variantId ? 0 : 0;
  const targetVram = busy
    ? Math.min(hw.vramTotalGb, hw.vramTotalGb * 0.86)
    : state.phase === 'loading'
      ? hw.vramTotalGb * 0.55
      : 2.4 + modelVram;
  if (process.env.STRATA_REAL_ENGINE !== '1') hw.vramUsedGb = clamp(approach(hw.vramUsedGb, targetVram, 0.14), 0.4, hw.vramTotalGb);

  const ramTarget = busy ? hw.ramTotalGb * 0.66 : hw.ramTotalGb * 0.44;
  if (process.env.STRATA_REAL_ENGINE !== '1') hw.ramUsedGb = clamp(approach(hw.ramUsedGb, ramTarget, 0.08), 1, hw.ramTotalGb);

  const tempTarget = busy ? hw.tempLimitC - 18 : 44;
  hw.gpuTempC = clamp(approach(hw.gpuTempC, tempTarget, 0.05) + (Math.random() - 0.5), 32, hw.tempLimitC);

  const demo = process.env.STRATA_REAL_ENGINE !== '1';
  state.telemetry.thermals = demo ? 'demo' : 'none';
  state.telemetry.memory = demo ? 'demo' : state.variantId ? 'estimated' : 'none';
  const outNow = state.phase === 'generating' ? state.target.outputTps : 0;
  const promptNow = state.phase === 'reading' ? state.target.promptTps : 0;
  const measuredHardware=hardwareSnapshot();

  state.history = [
    ...state.history.slice(-(HISTORY - 1)),
    {
      t: Date.now(),
      outputTps: round(outNow * (demo ? 0.9 + Math.random() * 0.2 : 1), 1),
      promptTps: round(promptNow * (demo ? 0.9 + Math.random() * 0.2 : 1), 0),
      ...measuredHardware,
    },
  ];

  if (!engine.running) state.queue.running = busy ? 1 : 0;
  state.context.pct = clamp((state.context.usedTokens / Math.max(state.context.maxTokens, 1)) * 100, 0, 100);
  hubRef?.send('monitor', snapshot());
}

export function snapshot() {
  const last = state.history.at(-1);
  const engine = supervisor.status();
  const engineMetrics = engine.metrics;
  if(state.hardware.ramUsedGb > state.hardware.ramTotalGb && !memoryWarningShown){
    console.warn('Memory reading exceeded physical capacity; clamping the monitor snapshot.');
    memoryWarningShown = true;
  }
  if(state.hardware.ramUsedGb <= state.hardware.ramTotalGb) memoryWarningShown = false;
  const result = {
    phase: state.phase,
    phaseLabel: state.phaseLabel,
    variantId: state.variantId ?? getStore().loaded,
    prompt: { ...state.prompt },
    generation: { ...state.generation },
    outputTps: state.phase === 'generating' ? round(state.target.outputTps, 1) : (last?.outputTps ?? 0),
    promptTps: ['reading','thinking','generating'].includes(state.phase) ? Math.round(state.target.promptTps) : (last?.promptTps ?? 0),
    queue: { ...state.queue },
    context: { ...state.context, parts: { ...state.contextParts } },
    expertCache: { ...state.expertCache },
    hardware: hardwareSnapshot(),
    requests: servingState().requests,
    conversationCache: { source: 'unavailable' },
    history: state.history,
    totals: { ...state.totals },
    telemetry: { ...state.telemetry, system:systemReading?'sensor':'none' },
    engine: {
      status: engine.status,
      running: engine.running,
      pid: engine.pid,
      port: engine.port,
      modelId: engine.modelId,
      metricsSource: engineMetrics.source,
      command: engine.command,
      binary: engine.binary,
      startedAt: engine.startedAt,
      logCount: engine.logCount,
    },
  };
  return engine.running&&engineMetrics.source==='engine'&&engineMetrics.dashboard?originalDashboard(engineMetrics.dashboard,cachedHardware()??hwCache,result):result;
}

export function setContextMax(max) {
  state.context.maxTokens = max;
}

export function setRecordedContext({ tokensIn, tokensOut, usageSource, requestId, sessionId }) {
  const available = Number.isFinite(tokensIn) && Number.isFinite(tokensOut);
  state.context = { ...state.context, usedTokens: available ? tokensIn + tokensOut : 0, requestId, sessionId };
  state.context.pct = available ? clamp(state.context.usedTokens / Math.max(1, state.context.maxTokens) * 100, 0, 100) : 0;
  state.contextParts = {};
  state.telemetry.context = available ? usageSource : 'unavailable';
}

/**
 * Records how full the context window is *now*.
 *
 * This used to **accumulate** — `usedTokens += tokensIn + tokensOut` — while the
 * per-part split beside it was computed as an absolute size. The two disagreed,
 * and the accumulating one is simply the wrong quantity: a context window is a
 * sliding space, so what matters is what has to fit in it at this moment, not
 * the sum of everything ever sent through it.
 *
 * The visible result was a meter that climbed past its own maximum within one
 * conversation — "40k / 32K tokens, 100%", clamped, with no way back down — on
 * the one indicator that tells a reader why a local model's answers are getting
 * worse.
 */
export function setContext(parts) {
  state.contextParts = {
    system: parts.system ?? 0,
    tools: parts.tools ?? 0,
    messages: parts.messages ?? 0,
  };
  state.context.usedTokens =
    state.contextParts.system + state.contextParts.tools + state.contextParts.messages;
}

export function resetContext() {
  state.context.usedTokens = 0;
  state.contextParts = { system: 0, tools: 0, messages: 0 };
}

export function setExpertCache(used, total = 64) {
  state.expertCache = { used, total };
}

export async function setModel(id) {
  state.variantId = id;
  if (!id) {
    state.phase = 'idle';
    state.phaseLabel = 'Waiting for a request';
    state.expertCache = { used: 0, total: 64 };
    return;
  }
  const hw = hwCache ?? (await detect());
  const v = getStore();
  const total = 64;
  const demo=process.env.STRATA_REAL_ENGINE!=='1';
  const used = v.loaded&&demo ? Math.round(total * 0.75) : 0;
  state.expertCache = { used, total, source:demo?'demo':'none' };
  state.hardware.vramTotalGb = Math.round(hw.accelerator.usableVramGb);
  state.hardware.ramTotalGb = Math.round(hw.memory.totalGb);
}

export async function beginLoading(id) {
  state.variantId = id;
  state.phase = 'loading';
  state.phaseLabel = 'Loading model into memory';
  state.prompt = { done: 0, total: 0, pct: 0 };
  state.queue.waiting = 0;
  hubRef?.send('monitor', snapshot());
}

export function beginReading(tokens, id) {
  state.variantId = id;
  state.phase = 'reading';
  state.phaseLabel = 'Reading prompt';
  state.prompt = { done: 0, total: tokens, pct: 0 };
  state.generation={done:null,total:Number(getStore().expert?.['sample.maxTokens'])||null};
  state.target.promptTps = process.env.STRATA_REAL_ENGINE !== '1' ? 1400 + Math.random() * 500 : 0;
  hubRef?.send('monitor', snapshot());
}

export function progressReading(done) {
  state.prompt.done = Math.min(done, state.prompt.total);
  state.prompt.pct = state.prompt.total ? (state.prompt.done / state.prompt.total) * 100 : 100;
  if (state.prompt.pct >= 100 && state.phase === 'reading') {
    state.phase = 'thinking';
    state.phaseLabel = 'Thinking';
  }
  hubRef?.send('monitor', snapshot());
}

export function beginGenerating() {
  state.phase = 'generating';
  state.phaseLabel = 'Generating response';
  state.target.outputTps = process.env.STRATA_REAL_ENGINE !== '1' ? 34 + Math.random() * 14 : 0;
  hubRef?.send('monitor', snapshot());
}

export function progressOutput(tokens) {
  if(Number.isFinite(tokens)&&tokens>=0)state.generation.done=tokens;
}

export function beginThinking() {
  state.phase = 'thinking';
  state.phaseLabel = 'Thinking';
  hubRef?.send('monitor', snapshot());
}

export function idle(label = 'Waiting for a request') {
  state.phase = 'idle';
  state.phaseLabel = label;
  state.prompt = { done: 0, total: 0, pct: 0 };
  state.target.outputTps = 0;
  state.target.promptTps = 0;
  state.queue = { running: 0, waiting: state.queue.waiting };
  hubRef?.send('monitor', snapshot());
}

export function record({ tokensIn, tokensOut, durationMs = 0, ttftMs = null, cachedTokens = null, tps: answerTps = null }) {
  if (Number.isFinite(tokensIn)) state.totals.tokensIn += tokensIn;
  if (Number.isFinite(tokensOut)) state.totals.tokensOut += tokensOut;
  state.totals.requests += 1;
  // The average is of answer speeds (tokens over decode time). A request with no decode figure is left out: dividing by the whole request would
  // put the prompt reading into it and read lower than the engine and the chat do.
  if (Number.isFinite(answerTps) && answerTps > 0) {
    const tps = answerTps;
    const n = ++state.totals.usageRequests;
    state.totals.avgTps = round((state.totals.avgTps * (n - 1) + tps) / n, 1);
  }

  // Time to first token, averaged the same way as throughput.
  if (typeof ttftMs === 'number' && ttftMs >= 0) {
    const n = ++state.totals.latencyRequests;
    state.totals.ttftMs = Math.round(((state.totals.ttftMs * (n - 1)) + ttftMs) / n);
  }

  // Include cache reuse only when the inference server reports it.
  if (Number.isFinite(cachedTokens) && Number.isFinite(tokensIn)) {
    const cached = Math.min(cachedTokens, tokensIn);
    state.totals.cachedTokens += cached;
    state.totals.uncachedTokens += Math.max(0, tokensIn - cached);
  }
}

export function queueWaiting(n) {
  state.queue.waiting = n;
}

export function currentOutputTps() {
  return state.target.outputTps;
}