Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.3 KB
Raw Blame
/**
 * Measured local-model assessments — the durable store and the environment the
 * measurements are only valid for.
 *
 * Split out of `localModelAssessments.js` so the READ side has no path to a
 * provider. Two reasons that matters:
 *
 *   1. **AI Provider Usage Policy** (root AGENTS.md). Everything here touches
 *      disk (plus, on the run path, one cheap version probe). Nothing in this
 *      file can reach an LLM, so importing it from a status/catalog read is safe
 *      by construction rather than by review.
 *   2. **No import cycle.** `localModelAssessments.js` imports `localLlm.js`
 *      (for the installed-model list), so `localLlm.js` could not import the
 *      store back from there. It imports this module instead.
 *
 * ## Storage (docs/STORAGE.md)
 *
 * `file-primary`, **intentionally machine-local — never federated.** An
 * assessment is a statement about THIS machine's hardware; a peer's copy would
 * be actively misleading (a peer's 8 GB laptop must not inherit a 128 GB box's
 * "fits" verdict). No sync cursor, no tombstone, no schema-version entry.
 *
 * ## Privacy
 *
 * The recorded environment is deliberately coarse — platform, arch, CPU count,
 * total/available memory, backend version. It never captures a hostname, a
 * username, a path, or any other machine identity, because assessments are
 * user-visible and end up in bug reports.
 */

import { join } from 'path';
import { rename } from 'fs/promises';
import os from 'os';
import { PATHS, atomicWrite, ensureDir, tryReadFile, safeJSONParse } from '../lib/fileUtils.js';
import { getAvailableMemoryGb } from './localMemory.js';
import { compareEnvironments, describeStaleness, measuredFitVerdict } from '../lib/localModelAssessment.js';
import { ASSESSABLE_RUNTIMES } from '../lib/localProviderRuntime.js';
import { getVersion as getOllamaVersion } from './ollamaManager.js';

// Resolved lazily, not at import time: `PATHS.data` is patched by suites that
// re-root the data dir, and a module-level `join()` would capture (or crash on)
// whatever the value happened to be the moment this module was first imported.
const assessmentsDir = () => join(PATHS.data, 'local-llm');
const assessmentsFile = () => join(assessmentsDir(), 'assessments.json');

// Storage-layout version for the on-disk file. Bump only when the persisted
// shape changes in a way a reader must branch on.
const STORE_SCHEMA_VERSION = 1;

const GB = 2 ** 30;

// ---- environment ------------------------------------------------------------

/**
 * The machine facts that are free to read: no subprocess, no network. Used on
 * every READ path (catalog badges, status) to decide whether a stored
 * measurement still describes this machine.
 */
export function captureDurableEnvironment() {
  return {
    platform: os.platform(),
    arch: os.arch(),
    cpuCount: os.cpus()?.length ?? null,
    totalMemoryGb: Number((os.totalmem() / GB).toFixed(1)),
  };
}

/**
 * Coarse description of the machine a measurement was taken on. Deliberately
 * carries NO machine identity (see the privacy note at the top of this file).
 *
 * Costs a `vm_stat` subprocess (macOS) and, for Ollama, one loopback HTTP GET —
 * so this is the RUN-path capture. Read paths use `captureDurableEnvironment()`
 * plus, where they can afford it, `captureLiveEnvironments()`.
 *
 * @param {{ backend?: string }} [options]
 */
export async function captureEnvironment({ backend } = {}) {
  const totalGb = os.totalmem() / GB;
  // `null` (not `0`) when the probe fails — an unknown budget must not read as
  // "no memory available", which would make every model look like it fits
  // nothing.
  const availableGb = await getAvailableMemoryGb().catch(() => null);
  return {
    ...captureDurableEnvironment(),
    totalMemoryGb: Number(totalGb.toFixed(1)),
    availableMemoryGb: Number.isFinite(availableGb) ? Number(availableGb.toFixed(1)) : null,
    // The budget the fit verdict is scored against: never more than what is
    // actually free right now, never more than the box has.
    memoryBudgetGb: Number.isFinite(availableGb)
      ? Number(Math.min(totalGb, availableGb).toFixed(1))
      : null,
    // `null` = not recorded, which is the honest answer for LM Studio (it
    // exposes no version endpoint) and for a stopped Ollama. It must never
    // default to a string, or a later comparison would read a fabricated
    // "unchanged".
    backendVersion: backend === 'ollama' ? await getOllamaVersion().catch(() => null) : null,
  };
}

// A backend UPDATE is one of the things that invalidates a measurement, so the
// read paths have to see the live version too — but the catalog path runs on
// every debounced keystroke, and an unconditional probe there would be one
// loopback GET per keystroke. Cache it briefly instead.
//
// `fetched` is the sentinel that keeps "never probed" distinct from "probed and
// Ollama is down" (a legitimate `null`); without it a down backend would be
// re-probed on every single call.
const VERSION_CACHE_MS = 60000;
let versionCache = { value: null, at: 0, fetched: false };

async function liveBackendVersion(backend) {
  // Only Ollama reports a version; LM Studio exposes none, so `null` there is
  // "not recorded", and a comparison against it is skipped rather than failed.
  if (backend !== 'ollama') return null;
  const now = Date.now();
  if (versionCache.fetched && now - versionCache.at < VERSION_CACHE_MS) return versionCache.value;
  const value = await getOllamaVersion().catch(() => null);
  versionCache = { value, at: now, fetched: true };
  return value;
}

/** Test seam: drop the cached backend version so a suite can re-probe. */
export function __resetBackendVersionCache() {
  versionCache = { value: null, at: 0, fetched: false };
}

/**
 * Live environment for a STALENESS comparison, including the backend version.
 * Costs one loopback GET for Ollama, so it belongs on paths that already talk to
 * the backend (the assessments report) — not on the catalog badge path.
 *
 * @param {{ backends?: string[] }} [options]
 * @returns {Promise<Record<string, object>>} keyed by backend
 */
export async function captureLiveEnvironments({ backends = ASSESSABLE_RUNTIMES } = {}) {
  const durable = captureDurableEnvironment();
  const entries = await Promise.all(backends.map(async (backend) => [
    backend,
    { ...durable, backendVersion: await liveBackendVersion(backend) },
  ]));
  return Object.fromEntries(entries);
}

// ---- store ------------------------------------------------------------------

/**
 * Identity of one stored measurement.
 *
 * `tuningKey` is the stable signature from `lib/localModelTuning.js`. An UNTUNED
 * measurement passes `''` and keys exactly as it did before tuning existed — so
 * every record already on disk keeps resolving with no migration, and a tuned
 * run of the same model lands beside it instead of overwriting it.
 */
export const assessmentKey = (backend, modelId, tuningKey = '') =>
  (tuningKey ? `${backend}:${modelId}@${tuningKey}` : `${backend}:${modelId}`);

/** The key of a stored record, from whichever fields it carries. */
export const keyOfAssessment = (a) => assessmentKey(a?.backend, a?.modelId, a?.tuningKey || '');

// Move an unparseable store aside so a fresh one can be written without losing
// whatever the old file held. Best-effort: if the rename fails there is nothing
// further to preserve, and refusing the write outright would leave assessments
// permanently unusable.
async function quarantineStore(reason) {
  const parked = `${assessmentsFile()}.corrupt-${Date.now()}`;
  const moved = await rename(assessmentsFile(), parked).then(() => true).catch(() => false);
  console.error(`❌ Local LLM: ${reason} — ${moved ? `parked the old file as ${parked.split('/').pop()}` : 'could not park the old file'}`);
}

export async function loadStore() {
  const raw = await tryReadFile(assessmentsFile());
  // Never read = an empty store, not an error: the file simply doesn't exist
  // until the first assessment runs.
  if (raw == null) return { schemaVersion: STORE_SCHEMA_VERSION, assessments: [], readError: null };
  const parsed = safeJSONParse(raw, null);
  if (!parsed || !Array.isArray(parsed.assessments)) {
    // Present but unparseable is NOT an empty store — say so rather than
    // silently reporting "nothing assessed" and letting a re-run overwrite it.
    return { schemaVersion: STORE_SCHEMA_VERSION, assessments: [], readError: 'assessments file is unreadable or malformed' };
  }
  return { schemaVersion: parsed.schemaVersion ?? STORE_SCHEMA_VERSION, assessments: parsed.assessments, readError: null };
}

/**
 * Read the persisted assessments. Disk only — never calls a provider, so this is
 * safe from boot, a poll, or any read path.
 *
 * @returns {Promise<Array<object>>} `[]` means "no assessments recorded", which
 *   is a real, measured-empty answer — distinct from a read failure, which is
 *   surfaced by `loadStore().readError`.
 */
export async function loadAssessments() {
  return (await loadStore()).assessments;
}

export async function saveAssessment(assessment) {
  const { assessments, readError } = await loadStore();
  // An unreadable store reports zero assessments, and writing on top of that
  // would replace every prior measurement with this one record — a read failure
  // silently destroying data the user spent minutes of compute on. Quarantine
  // the unreadable file instead: nothing is lost, and the feature keeps working
  // rather than wedging on a file the user has no way to repair from the UI.
  if (readError) await quarantineStore(readError);
  const key = keyOfAssessment(assessment);
  // One record per (backend, model, TUNING): the newest measurement supersedes
  // the old one for that exact configuration. History within one tuning is not
  // kept — a stale reading from a different memory state is worse than no
  // reading, and the run is cheap to repeat. Two DIFFERENT tunings are two
  // different answers to two different questions, so they both survive.
  const next = assessments.filter((a) => keyOfAssessment(a) !== key);
  next.push(assessment);
  await ensureDir(assessmentsDir());
  await atomicWrite(assessmentsFile(), { schemaVersion: STORE_SCHEMA_VERSION, assessments: next });
  return assessment;
}

/**
 * Drop one recorded assessment. Returns whether a record was actually removed,
 * so the caller can 404 rather than reporting a phantom success.
 */
export async function deleteAssessment(backend, modelId, tuningKey = '') {
  const { assessments, readError } = await loadStore();
  // Same hazard as saveAssessment: rewriting from an empty in-memory list would
  // wipe the file. A delete against an unreadable store has nothing to remove.
  if (readError) return { deleted: false };
  const key = assessmentKey(backend, modelId, tuningKey);
  const next = assessments.filter((a) => keyOfAssessment(a) !== key);
  if (next.length === assessments.length) return { deleted: false };
  await ensureDir(assessmentsDir());
  await atomicWrite(assessmentsFile(), { schemaVersion: STORE_SCHEMA_VERSION, assessments: next });
  console.log(`🧹 Local LLM: dropped assessment for ${backend}/${modelId}${tuningKey ? ` (${tuningKey})` : ''}`);
  return { deleted: true };
}

// ---- measured evidence for the ESTIMATE-driven surfaces ---------------------

/**
 * Annotate one assessment with how well its recorded environment still matches
 * the machine as it is now.
 *
 * @param {object} assessment
 * @param {object|null} liveEnvironment
 */
export function withStaleness(assessment, liveEnvironment) {
  const staleness = compareEnvironments(assessment?.environment, liveEnvironment);
  return {
    ...assessment,
    staleness: { ...staleness, description: describeStaleness(staleness) },
  };
}

/**
 * Compact measured-fit records the catalog badge and the editorial
 * recommendation fold in, keyed by model id for one backend.
 *
 * Disk only apart from a 60s-cached backend-version probe: the memory/CPU facts
 * are free to read, and the version is what catches a backend update, which is
 * just as stale-making as a RAM change.
 *
 * @param {string} backend
 * @returns {Promise<Record<string, {fit:string|null, verdict:string, assessedAt:string|null, stale:boolean, staleReason:string|null, meanCharsPerSecond:number|null, residentGb:number|null}>>}
 */
export async function getMeasuredFits(backend) {
  const { assessments } = await loadStore();
  const live = { ...captureDurableEnvironment(), backendVersion: await liveBackendVersion(backend) };
  const out = {};
  // A model can now hold several measurements (one per tuning). The badge has
  // room for ONE, so the newest wins: it describes the configuration the user
  // most recently cared about, and every tuning is visible in full on the
  // Performance page. Records with no timestamp sort last rather than
  // masquerading as the newest.
  const newestFirst = [...assessments].sort(
    (a, b) => String(b?.assessedAt || '').localeCompare(String(a?.assessedAt || ''))
  );
  for (const assessment of newestFirst) {
    if (assessment?.backend !== backend || !assessment?.modelId) continue;
    if (out[assessment.modelId]) continue;
    const staleness = compareEnvironments(assessment.environment, live);
    out[assessment.modelId] = {
      fit: measuredFitVerdict(assessment),
      verdict: assessment.verdict || 'unknown',
      assessedAt: assessment.assessedAt || null,
      stale: staleness.stale,
      staleReason: describeStaleness(staleness),
      // `null` = never measured on both of these; a consumer rendering them must
      // say "not measured" rather than showing a zero.
      meanCharsPerSecond: Number.isFinite(assessment.performance?.meanCharsPerSecond)
        ? assessment.performance.meanCharsPerSecond
        : null,
      residentGb: Number.isFinite(assessment.residentGb) ? assessment.residentGb : null,
      // LM Studio model ids are repo-level and carry no quant, so the quant the
      // measurement actually ran has to travel separately or a Q4 reading would
      // decorate every quant of the repo. `null` = the record predates this
      // field (or the backend reported none), and a consumer must then decline
      // to match a quantized variant rather than guess.
      quantization: assessment.quantization ?? null,
      // Which launch configuration produced this reading. `null` = the record
      // predates tuning (or ran on backend defaults), which is a real answer —
      // not an unknown.
      tuningLabel: assessment.tuningLabel ?? null,
    };
  }
  return out;
}