Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Local-model assessment scoring — pure, dependency-free.
*
* PortOS already guesses whether a local model fits from catalog metadata alone
* (weight bytes × a fixed overhead multiplier, in `huggingFaceCatalog.js`). That
* guess never touches the machine: it can't tell a model that loads and streams
* fast from one that loads, thrashes, and crawls, and it says
* nothing at all about how a model behaves as the prompt gets long.
*
* This module turns *measured* evidence — one bounded generation per context
* length, recorded by `services/localModelAssessments.js` — into an explainable
* verdict and an intent-specific ranking.
*
* ## The sentinel contract (read before touching anything here)
*
* `null` means NOT MEASURED. It never means zero, never means failed, and never
* collapses into a fallback. `[]` / `0` mean measured-and-empty. Concretely:
*
* - `sample.charsPerSecond === null` → the sample never produced timings
* - `sample.ok === false` → the sample RAN and FAILED (has `error`)
* - `verdict === 'unknown'` → nothing usable was measured
* - `scores.speed === null` → speed was never observed; a ranking
* must not treat that as "slow"
*
* Anything that conflates those makes an unassessed model look worse than a
* measured-bad one, which is the exact failure this feature exists to remove.
*
* (Dependency-free apart from `localModelHeuristics.js`, which is itself pure —
* a sweep has to know which installed models are even *assessable*.)
*/
import { isEmbeddingModel } from './localModelHeuristics.js';
/**
* Fit verdicts. Deliberately four values, not a boolean:
* - `fits` — at least one context length generated successfully
* - `does-not-fit` — every attempted context length failed for a resource
* reason (OOM / load failure / timeout)
* - `incompatible` — the backend refused the model outright (wrong format,
* unsupported architecture) — retrying with less context
* or more RAM will not help
* - `unknown` — nothing was measured
*/
export const FIT_VERDICTS = ['fits', 'does-not-fit', 'incompatible', 'unknown'];
/** Ranking intents, mirroring how people actually pick a local model. */
export const ASSESSMENT_INTENTS = ['balanced', 'smartest', 'fastest', 'lightweight'];
// Error text that means "this model can never run here", as opposed to "this
// model ran out of room at this context length". Matched case-insensitively
// against the sample error. Kept narrow on purpose: a false `incompatible`
// permanently hides a model that a smaller context would have run.
const INCOMPATIBLE_RE =
/unsupported|unknown model architecture|unrecognized|not supported|invalid model file|no such file|unsupported model format|does not support/i;
// Error text that means "resource exhaustion at this size" — the model itself is
// fine, this machine could not hold it as configured.
const RESOURCE_RE =
/out of memory|oom|cannot allocate|insufficient memory|not enough memory|resource exhausted|model requires more system memory|timed out|context (?:length|window) exceed/i;
/**
* Classify one recorded sample's failure. Returns `null` for a sample that
* succeeded or that carries no error text (unknown cause — never guessed at).
*
* @param {{ ok?: boolean, error?: string }} sample
* @returns {'incompatible'|'resource'|null}
*/
export function classifySampleFailure(sample) {
if (!sample || sample.ok !== false) return null;
const error = typeof sample.error === 'string' ? sample.error : '';
if (!error) return null;
if (INCOMPATIBLE_RE.test(error)) return 'incompatible';
if (RESOURCE_RE.test(error)) return 'resource';
return null;
}
const isMeasured = (value) => typeof value === 'number' && Number.isFinite(value);
/** Samples that actually produced output. */
const okSamples = (samples) => (Array.isArray(samples) ? samples : []).filter((s) => s?.ok === true);
/**
* Derive the fit verdict from the recorded samples.
*
* One success anywhere is enough for `fits` — a model that runs at 4k but dies
* at 32k still fits this machine, and `maxWorkingContextTokens` (below) is what
* carries the nuance. `incompatible` wins over `does-not-fit` because it is the
* actionable one: no amount of freed memory changes it.
*
* @param {Array<{ ok?: boolean, error?: string }>} samples
* @returns {'fits'|'does-not-fit'|'incompatible'|'unknown'}
*/
export function classifyFitVerdict(samples) {
const list = Array.isArray(samples) ? samples : [];
if (list.length === 0) return 'unknown';
if (list.some((s) => s?.ok === true)) return 'fits';
const failures = list.map(classifySampleFailure);
if (failures.some((f) => f === 'incompatible')) return 'incompatible';
if (failures.some((f) => f === 'resource')) return 'does-not-fit';
// Ran, failed, and we can't say why (network blip, backend restart). That is
// NOT evidence the model doesn't fit — leave it unknown so a re-run decides.
return 'unknown';
}
/**
* Largest context length that produced a successful generation.
* `null` when nothing succeeded — distinct from `0`, which never occurs here.
*
* @param {Array<{ ok?: boolean, contextTokens?: number }>} samples
* @returns {number|null}
*/
export function maxWorkingContextTokens(samples) {
const working = okSamples(samples)
.map((s) => s.contextTokens)
.filter(isMeasured);
return working.length ? Math.max(...working) : null;
}
/**
* Mean of a numeric field across successful samples, or `null` when the field
* was never measured. Rounded to two decimals so persisted records stay stable.
*/
function meanOf(samples, field) {
const values = okSamples(samples).map((s) => s[field]).filter(isMeasured);
if (!values.length) return null;
const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
return Number(mean.toFixed(2));
}
/**
* Collapse the per-context samples into one performance summary.
*
* `contextDegradation` is the ratio of the throughput at the LARGEST working
* context to the peak throughput across all contexts (0..1]; 1 means the longest
* context ran as fast as the fastest one did, 0.25 means it ran at a quarter of
* that.
*
* It is deliberately anchored to the longest context rather than to the slowest
* sample. Results are not always monotonic — a mid-range context can dip below
* both its neighbours from an unrelated hiccup — and a min-based ratio would
* both contradict what this number is reported as ("held X% of peak throughput
* at its longest context") and penalize the ranking for a one-off intermediate
* outlier instead of for the long-context behavior the axis is about.
*
* `null` when fewer than two context lengths produced throughput: a single
* sample cannot describe a trend, and reporting `1` there would fake perfect
* scaling.
*
* @param {Array<object>} samples
*/
export function summarizePerformance(samples) {
const ok = okSamples(samples);
const measured = ok
.filter((s) => isMeasured(s.charsPerSecond) && s.charsPerSecond > 0 && isMeasured(s.contextTokens));
const fastest = measured.length ? Math.max(...measured.map((s) => s.charsPerSecond)) : null;
const atLongest = measured.length
? measured.reduce((longest, s) => (s.contextTokens > longest.contextTokens ? s : longest)).charsPerSecond
: null;
const tokenRates = ok.map((s) => s.tokensPerSecond).filter(isMeasured);
return {
samplesRun: Array.isArray(samples) ? samples.length : 0,
samplesOk: ok.length,
meanCharsPerSecond: meanOf(samples, 'charsPerSecond'),
meanTtftMs: meanOf(samples, 'ttftMs'),
peakCharsPerSecond: fastest,
// Token-denominated throughput, present only when the daemon reported token
// counts. `null` here means the runtime told us nothing — a reader must show
// "not measured", not fall back to chars/s under a tokens/s label.
meanTokensPerSecond: meanOf(samples, 'tokensPerSecond'),
peakTokensPerSecond: tokenRates.length ? Math.max(...tokenRates) : null,
meanPromptTokensPerSecond: meanOf(samples, 'promptTokensPerSecond'),
timingSource: (() => {
const sources = [...new Set(ok.map((s) => s.timingSource).filter(Boolean))];
return sources.length === 1 ? sources[0] : (sources.length ? 'mixed' : null);
})(),
// `true` when ANY contributing sample counted streamed frames instead of
// reading the daemon's own count. One estimated sample makes the mean an
// estimate, so this is deliberately pessimistic. `null` = no token evidence.
tokensEstimated: tokenRates.length
? ok.some((s) => isMeasured(s.tokensPerSecond) && s.tokensEstimated === true)
: null,
maxWorkingContextTokens: maxWorkingContextTokens(samples),
// Clamped: the longest context can measure marginally ABOVE the peak of the
// others only through noise, and a >1 "degradation" is meaningless.
contextDegradation:
measured.length >= 2 && fastest > 0 ? Number(Math.min(1, atLongest / fastest).toFixed(3)) : null,
};
}
/**
* Flatten every measurement into one tokens-per-second table, fastest first.
*
* This is the "how fast is each model, really" view: one row per measured
* model+tuning, carrying the per-context readings and elapsed test time so
* falloff as the prompt grows is visible rather than averaged away. It is a
* REPORT, not a ranking — no scoring, no intent, no exclusions — so a model that
* only ran at 512 tokens still appears, with one column filled and the rest
* honestly blank.
*
* Rows whose runtime reported no token counts are still included (with `null`
* rates) rather than dropped: "this runtime does not report tokens" is the
* answer to why a model is missing from a tokens/s table, and silently omitting
* it would make the table look complete when it isn't.
*
* @param {Array<object>} assessments stored assessment records
* @returns {{ rows: Array<object>, contexts: number[], modelsWithTokenRates: number }}
*/
export function buildThroughputReport(assessments) {
const list = Array.isArray(assessments) ? assessments : [];
// Every rate on a row is null-or-measured; nothing is defaulted to 0.
const num = (value) => (isMeasured(value) ? value : null);
const contexts = new Set();
const rows = [];
for (const assessment of list) {
const samples = Array.isArray(assessment?.samples) ? assessment.samples : [];
const perf = assessment?.performance || {};
const points = [];
for (const sample of samples) {
if (!isMeasured(sample?.contextTokens)) continue;
contexts.add(sample.contextTokens);
points.push({
contextTokens: sample.contextTokens,
ok: sample?.ok === true,
// A failed sample carries its error so the cell can say WHY it is blank
// instead of just being blank.
tokensPerSecond: num(sample.tokensPerSecond),
promptTokensPerSecond: num(sample.promptTokensPerSecond),
charsPerSecond: num(sample.charsPerSecond),
ttftMs: num(sample.ttftMs),
// Keep the end-to-end duration with the context column. Unlike the
// aggregate mean TTFT, this answers how long THIS individual test took,
// including prompt processing and generation.
totalMs: num(sample.totalMs),
completionTokens: num(sample.completionTokens),
timingSource: sample.timingSource || null,
error: sample?.error || null,
});
}
rows.push({
backend: assessment?.backend || null,
modelId: assessment?.modelId || null,
tuningKey: assessment?.tuningKey || '',
tuningLabel: assessment?.tuningLabel || null,
// A reading taken under a tuning that never reached the daemon describes
// some OTHER configuration. It stays in the table (the numbers are real)
// but has to carry the caveat, or the row credits the wrong config.
tuningApplied: assessment?.tuningApplied ?? null,
tuningNotApplied: assessment?.tuningNotApplied || null,
verdict: assessment?.verdict || 'unknown',
assessedAt: assessment?.assessedAt || null,
staleness: assessment?.staleness || null,
meanTokensPerSecond: num(perf.meanTokensPerSecond),
peakTokensPerSecond: num(perf.peakTokensPerSecond),
meanPromptTokensPerSecond: num(perf.meanPromptTokensPerSecond),
meanCharsPerSecond: num(perf.meanCharsPerSecond),
meanTtftMs: num(perf.meanTtftMs),
timingSource: perf.timingSource || null,
tokensEstimated: typeof perf.tokensEstimated === 'boolean' ? perf.tokensEstimated : null,
points: points.sort((a, b) => a.contextTokens - b.contextTokens),
});
}
// Fastest first, but a row with NO token rate sorts last rather than as zero —
// unmeasured is not slow. Ties fall back to model id so the order is stable.
rows.sort((a, b) => {
const left = a.meanTokensPerSecond;
const right = b.meanTokensPerSecond;
if (left === null && right === null) return String(a.modelId).localeCompare(String(b.modelId));
if (left === null) return 1;
if (right === null) return -1;
return right - left || String(a.modelId).localeCompare(String(b.modelId));
});
return {
rows,
contexts: [...contexts].sort((a, b) => a - b),
// How many rows actually carry a tokens/s figure, so the UI can explain an
// empty table ("no local runtime here reports token counts") instead of
// rendering a blank grid.
modelsWithTokenRates: rows.filter((r) => r.meanTokensPerSecond !== null).length,
};
}
// Speed normalization anchor, in CHARACTERS per second.
//
// Chars/s, not tokens/s, and deliberately so. PortOS now RECORDS tokens/s when
// the daemon reports token counts (`summarizeTimings`), and reports it — but it
// cannot SCORE on it: a runtime that reports no usage block would score `null`
// on speed and drop out of the ranking entirely, and records written before
// token counts existed carry none. Chars/s is the one figure every runtime and
// every stored record has, so the ranking stays comparable across both.
// Still no tokenizer in the repo: nothing here ever derives a token count from
// a character count.
//
// 240 chars/s (~60 tok/s at the conventional ~4 chars/token) is a comfortable
// interactive rate on consumer hardware; at or above it the axis scores 1, below
// it the score is linear so a crawling model reads as clearly worse rather than
// both bottoming out.
const SPEED_CEILING_CHARS_PER_SECOND = 240;
// Parameter-count anchor for the capability score. Local models above ~70B are
// rare on consumer hardware and already score at the top of the curve.
const PARAMS_CEILING_B = 70;
const clamp01 = (v) => Math.min(1, Math.max(0, v));
/**
* Parse a parameter count in billions out of a params string or model id
* (`"32B"`, `"qwen3:14b"`). `null` when the id carries no size marker — never
* defaulted, so an unsized model doesn't masquerade as a small one.
*
* @param {{ params?: string, modelId?: string }} model
* @returns {number|null}
*/
export function parseParamsBillions({ params, modelId } = {}) {
const match = `${params || ''} ${modelId || ''}`.match(/(\d+(?:\.\d+)?)\s*[bB]\b/);
return match ? Number(match[1]) : null;
}
/**
* Score one assessment on the four axes the ranking uses. Every axis is `null`
* when its evidence is missing — callers must handle that rather than
* substituting 0.
*
* - `capability` — parameter count, the only capability proxy available offline
* - `speed` — measured mean throughput against the interactive anchor
* - `fidelity` — how well throughput held up as context grew (needs ≥2 samples)
* - `memory` — measured headroom left after the model's resident footprint
*
* @param {object} assessment
* @returns {{capability:number|null, speed:number|null, fidelity:number|null, memory:number|null}}
*/
export function scoreAssessment(assessment) {
const perf = assessment?.performance || {};
const paramsB = parseParamsBillions({ params: assessment?.params, modelId: assessment?.modelId });
const residentGb = assessment?.residentGb;
const budgetGb = assessment?.environment?.memoryBudgetGb;
const memory =
isMeasured(residentGb) && isMeasured(budgetGb) && budgetGb > 0
? clamp01(1 - residentGb / budgetGb)
: null;
return {
capability: paramsB == null ? null : clamp01(Math.log10(1 + paramsB) / Math.log10(1 + PARAMS_CEILING_B)),
speed: isMeasured(perf.meanCharsPerSecond) ? clamp01(perf.meanCharsPerSecond / SPEED_CEILING_CHARS_PER_SECOND) : null,
fidelity: isMeasured(perf.contextDegradation) ? clamp01(perf.contextDegradation) : null,
memory,
};
}
// Per-intent axis weights. Weights of axes with no measurement are dropped and
// the rest renormalized, so a partly-measured model is scored on what IS known
// instead of being penalized for the gap. `coverage` reports how much of the
// intended weight was actually backed by evidence, and the caller surfaces it.
const INTENT_WEIGHTS = {
balanced: { capability: 0.3, speed: 0.3, fidelity: 0.2, memory: 0.2 },
smartest: { capability: 0.6, speed: 0.1, fidelity: 0.2, memory: 0.1 },
fastest: { capability: 0.1, speed: 0.6, fidelity: 0.2, memory: 0.1 },
lightweight: { capability: 0.1, speed: 0.25, fidelity: 0.15, memory: 0.5 },
};
/**
* Weighted score for one intent.
*
* @param {object} scores from `scoreAssessment`
* @param {string} intent one of ASSESSMENT_INTENTS
* @returns {{ score: number|null, coverage: number }} `score` is null when NO
* axis was measured — an unassessed model must not be rankable as "0".
*/
export function scoreForIntent(scores, intent) {
const weights = INTENT_WEIGHTS[intent] || INTENT_WEIGHTS.balanced;
let weighted = 0;
let used = 0;
for (const [axis, weight] of Object.entries(weights)) {
const value = scores?.[axis];
if (!isMeasured(value)) continue;
weighted += value * weight;
used += weight;
}
if (used === 0) return { score: null, coverage: 0 };
return { score: Number((weighted / used).toFixed(4)), coverage: Number(used.toFixed(2)) };
}
const INTENT_LABEL = {
balanced: 'balanced',
smartest: 'strongest',
fastest: 'fastest',
lightweight: 'lightest',
};
/**
* One-sentence, evidence-only explanation for a ranked candidate. Every clause
* names a measured number; unmeasured axes are simply omitted rather than
* described with a guess.
*/
export function explainAssessment(assessment, intent) {
const perf = assessment?.performance || {};
const parts = [];
// Tokens/s leads when the runtime reported token counts — it is the figure
// people actually compare local models on. Chars/s is the fallback, never both:
// two throughput numbers in one sentence read as a contradiction.
if (isMeasured(perf.meanTokensPerSecond)) {
parts.push(`${perf.meanTokensPerSecond} tok/s${perf.tokensEstimated ? ' (estimated)' : ''} measured`);
} else if (isMeasured(perf.meanCharsPerSecond)) {
parts.push(`${perf.meanCharsPerSecond} chars/s measured`);
}
if (isMeasured(perf.maxWorkingContextTokens)) {
parts.push(`ran at up to ${perf.maxWorkingContextTokens.toLocaleString('en-US')} tokens of context`);
}
if (isMeasured(perf.contextDegradation)) {
parts.push(`held ${Math.round(perf.contextDegradation * 100)}% of peak throughput at its longest context`);
}
if (isMeasured(assessment?.residentGb)) parts.push(`${assessment.residentGb} GB resident`);
if (!parts.length) return `No measurements recorded yet — run an assessment to rank it for ${INTENT_LABEL[intent] || intent} use.`;
return `${parts.join(', ')}.`;
}
/**
* Rank assessed models for one intent.
*
* Only `fits` candidates are ranked — a model that did not run is not a
* recommendation, however good its metadata looks. Everything else is returned
* under `excluded` with its verdict so the UI can explain the absence instead
* of silently dropping it.
*
* @param {Array<object>} assessments
* @param {string} intent
* @returns {{ intent: string, ranked: Array<object>, excluded: Array<object> }}
*/
export function rankByIntent(assessments, intent = 'balanced') {
const resolvedIntent = ASSESSMENT_INTENTS.includes(intent) ? intent : 'balanced';
const list = Array.isArray(assessments) ? assessments : [];
const ranked = [];
const excluded = [];
for (const assessment of list) {
const verdict = assessment?.verdict || 'unknown';
if (verdict !== 'fits') {
excluded.push({
backend: assessment?.backend || null,
modelId: assessment?.modelId || null,
tuningKey: assessment?.tuningKey || '',
tuningLabel: assessment?.tuningLabel || null,
verdict,
reason: assessment?.verdictReason || null,
});
continue;
}
const scores = scoreAssessment(assessment);
const { score, coverage } = scoreForIntent(scores, resolvedIntent);
if (score == null) {
excluded.push({
backend: assessment?.backend || null,
modelId: assessment?.modelId || null,
tuningKey: assessment?.tuningKey || '',
tuningLabel: assessment?.tuningLabel || null,
verdict,
reason: 'ran, but no axis of this intent was measured',
});
continue;
}
ranked.push({
backend: assessment?.backend || null,
modelId: assessment?.modelId || null,
verdict,
score,
coverage,
scores,
performance: assessment?.performance || null,
// The measured footprint feeds the memory axis, so it has to travel WITH
// the ranked entry — a consumer that renders it from the entry alone would
// otherwise show "not measured" for every model /api/ps actually measured.
residentGb: Number.isFinite(assessment?.residentGb) ? assessment.residentGb : null,
params: assessment?.params ?? null,
assessedAt: assessment?.assessedAt || null,
// Travels with the entry so the UI can mark a reading taken before a RAM
// upgrade / backend update WITHOUT re-deriving the comparison client-side.
// `null` = the caller did not annotate staleness, which is not the same as
// "compared and current".
staleness: assessment?.staleness || null,
// The tuning IS part of this row's identity, not decoration. A model can
// hold several measurements, one per launch configuration, and the
// consumer uses these to key the row, to delete THIS measurement rather
// than the backend-defaults one, and to pre-fill a re-measure with the
// configuration that produced it. Dropping them collapsed every variant
// onto the default record.
tuningKey: assessment?.tuningKey || '',
tuning: assessment?.tuning || {},
tuningLabel: assessment?.tuningLabel || null,
// `null` = nothing was settable (see runAssessment); `false` means these
// numbers describe some OTHER configuration, which the row has to say.
tuningApplied: assessment?.tuningApplied ?? null,
tuningNotApplied: assessment?.tuningNotApplied || null,
explanation: explainAssessment(assessment, resolvedIntent),
});
}
// FRESH evidence outranks stale evidence outright, before score is even
// considered: a reading taken before a RAM upgrade or a backend update
// describes a machine that no longer exists, so it must never be the top
// recommendation — the same rule `reconcileFit` and the editorial recommender
// apply. Stale rows stay in the list (labelled) rather than vanishing, because
// this panel is where the user goes to see and re-run them.
// Then: higher score first; ties broken by evidence coverage so a
// fully-measured model outranks one scored on a single axis, then by model id
// for stability.
const isStale = (entry) => (entry.staleness?.stale ? 1 : 0);
ranked.sort((a, b) =>
(isStale(a) - isStale(b))
|| (b.score - a.score)
|| (b.coverage - a.coverage)
|| String(a.modelId).localeCompare(String(b.modelId))
// Two tunings of ONE model tie on model id, so the signature is what makes
// their order stable across reloads.
|| String(a.tuningKey).localeCompare(String(b.tuningKey)));
return { intent: resolvedIntent, ranked, excluded };
}
// ---- batch measurement ------------------------------------------------------
/** What a "measure everything" run can cover. */
export const SWEEP_SCOPES = ['unmeasured', 'stale', 'all'];
const targetKey = (t) => `${t.backend}:${t.modelId}@${t.tuningKey || ''}`;
/**
* Expand a sweep scope into the exact list of measurements to run.
*
* Pure and deterministic so the consent modal can be told the real count before
* anything runs, and so the sweep and the count can never disagree about what
* "all" means — they call this same function.
*
* A re-measure reuses the tuning that produced the existing record, matching
* what the single-model Measure-again button does: "run it again" has to
* reproduce the same configuration, or the sweep would quietly replace every
* tuned reading with a backend-defaults one.
*
* ## The second dimension
*
* `tunings` swaps that rule out: instead of each model keeping the tuning it
* carries, every model is crossed with the supplied grid — one measurement per
* (model, tuning) pair. That is what makes a TUNING sweep a case of this
* function rather than a queue of its own, and it is why the dedupe below is
* keyed on model AND tuning: two variants sharing a key are one measurement, and
* running it twice would put the same configuration on both sides of the
* comparison table.
*
* @param {object} options
* @param {Array<object>} options.assessments stored records, already filtered to
* models that are STILL INSTALLED — a sweep must not queue a model the runtime
* can no longer load.
* @param {Array<{backend:string, modelId:string}>} options.unassessed models with
* no evidence at all
* @param {'unmeasured'|'stale'|'all'} options.scope
* @param {{backend:string, modelId:string}|null} [options.only] restrict to one
* model. Naming a model leaves the scope nothing to decide, so it implies
* `all` — otherwise "sweep THIS model" would silently do nothing whenever the
* model happened to fall outside the scope.
* @param {Array<{tuning:object, key:string, label:string|null}>|null} [options.tunings]
* knob sets to measure every selected model under. Entries are taken as
* already-normalized (they come from `localModelTuning.js#tuningGridFor`), so
* this module stays dependency-free.
* @returns {Array<{backend:string, modelId:string, tuning:object, tuningKey:string, tuningLabel:string|null, reason:string}>}
*/
export function selectSweepTargets({ assessments = [], unassessed = [], scope = 'unmeasured', only = null, tunings = null } = {}) {
const resolved = only ? 'all' : (SWEEP_SCOPES.includes(scope) ? scope : 'unmeasured');
const wanted = (m) => !only || (m?.backend === only.backend && m?.modelId === only.modelId);
const stored = Array.isArray(assessments) ? assessments : [];
const fresh = Array.isArray(unassessed) ? unassessed : [];
const targets = [];
const seen = new Set();
const add = (target) => {
const key = targetKey(target);
if (!target.backend || !target.modelId || seen.has(key)) return;
// An assessment measures a model by GENERATING with it, so an embedding-only
// model has no measurement to take: the daemon answers every sample with
// `400 "<model>" does not support chat`, the record lands as `incompatible`
// (which reads as "this machine can't run it" — it can, that is just not
// what it does), and each doomed sample raises an AI-provider investigation
// task. Excluded from BOTH branches, so a bogus record written before this
// existed is not re-queued forever by the `stale`/`all` scopes.
if (isEmbeddingModel(target.modelId)) return;
seen.add(key);
targets.push(target);
};
// Never-measured models come first in every scope that includes them: they are
// the ones with no answer at all, so if an overnight run is cut short they are
// the measurements worth having got.
if (resolved === 'unmeasured' || resolved === 'all') {
for (const model of fresh) {
if (!wanted(model)) continue;
add({ backend: model?.backend, modelId: model?.modelId, tuning: {}, tuningKey: '', tuningLabel: null, reason: 'never measured' });
}
}
if (resolved === 'stale' || resolved === 'all') {
for (const assessment of stored) {
if (resolved === 'stale' && !assessment?.staleness?.stale) continue;
if (!wanted(assessment)) continue;
add({
backend: assessment?.backend,
modelId: assessment?.modelId,
tuning: assessment?.tuning && typeof assessment.tuning === 'object' ? assessment.tuning : {},
tuningKey: assessment?.tuningKey || '',
tuningLabel: assessment?.tuningLabel || null,
reason: assessment?.staleness?.stale ? 'measured on a different machine state' : 're-measure',
});
}
}
if (!Array.isArray(tunings) || tunings.length === 0) return targets;
// Cross the selected models with the grid. Re-running `add` rather than
// pushing directly keeps ONE dedupe rule for both dimensions — a model that
// reached this point under two stored tunings collapses back to one entry per
// grid variant instead of measuring each variant twice.
const models = targets.splice(0, targets.length);
seen.clear();
for (const model of models) {
for (const variant of tunings) {
add({
backend: model.backend,
modelId: model.modelId,
tuning: variant?.tuning && typeof variant.tuning === 'object' ? variant.tuning : {},
tuningKey: variant?.key || '',
tuningLabel: variant?.label ?? null,
// `''` is the baseline — the reading every other variant's throughput is
// reported relative to, so it is named rather than left blank.
reason: variant?.key ? 'tuning variant' : 'backend defaults',
});
}
}
return targets;
}
/**
* How many measurements each scope would run, for the consent gate.
* Derived from `selectSweepTargets` so a displayed count is always the count
* that would actually execute.
*/
export function summarizeSweepScopes({ assessments, unassessed } = {}) {
return Object.fromEntries(
SWEEP_SCOPES.map((scope) => [scope, selectSweepTargets({ assessments, unassessed, scope }).length])
);
}
// ---- staleness ---------------------------------------------------------------
/**
* Machine facts a measurement is only valid FOR. These are the durable ones: a
* RAM upgrade, a CPU swap, an OS/arch move, or a backend update all invalidate a
* recorded reading, and none of them announce themselves to the user.
*
* Transient state is deliberately NOT here. `availableMemoryGb` swings by
* gigabytes minute to minute on any working machine, so comparing it would flag
* essentially every assessment as stale and the badge would mean nothing.
*/
const ENVIRONMENT_FIELDS = [
{ field: 'platform', label: 'operating system' },
{ field: 'arch', label: 'CPU architecture' },
{ field: 'cpuCount', label: 'CPU count' },
{ field: 'totalMemoryGb', label: 'installed memory', unit: ' GB', toleranceGb: 0.5 },
{ field: 'backendVersion', label: 'backend version' },
];
// Absent on EITHER side is not-comparable, never a change: an older record
// predates a captured field (`backendVersion` landed after the first
// assessments), and LM Studio reports no version at all. Reading either as
// "changed" would mark every legacy record stale on sight.
const present = (v) => v !== null && v !== undefined && v !== '';
/**
* Compare a recorded assessment environment against the live one.
*
* @param {object|null} recorded environment stored with the assessment
* @param {object|null} live environment captured now
* @returns {{ comparable: boolean, stale: boolean, changes: Array<{field:string,label:string,from:*,to:*}> }}
* `comparable: false` means no field could be compared — that is UNKNOWN, not
* fresh, and callers must not render it as a clean bill of health.
*/
export function compareEnvironments(recorded, live) {
const changes = [];
let compared = 0;
for (const { field, label, toleranceGb } of ENVIRONMENT_FIELDS) {
const from = recorded?.[field];
const to = live?.[field];
if (!present(from) || !present(to)) continue;
compared += 1;
const changed = typeof toleranceGb === 'number' && typeof from === 'number' && typeof to === 'number'
? Math.abs(from - to) > toleranceGb
: from !== to;
if (changed) changes.push({ field, label, from, to });
}
return { comparable: compared > 0, stale: changes.length > 0, changes };
}
/**
* One human sentence naming what changed since the measurement, or `null` when
* nothing did (or nothing could be compared).
*/
export function describeStaleness(staleness) {
const changes = staleness?.changes;
if (!Array.isArray(changes) || changes.length === 0) return null;
const parts = changes.map(({ label, from, to }) => `${label} ${from} → ${to}`);
return `Measured on a different machine state — ${parts.join('; ')}.`;
}
// ---- folding measured evidence into the estimated fit badge ------------------
// The catalog's estimate calls a model "tight" once its projected resident size
// passes 60% of the usable budget (`huggingFaceCatalog.js#classifyFit`). The
// measured verdict uses the SAME threshold against the measured footprint so the
// two vocabularies stay comparable — a badge that flips from `comfortable` to
// `tight` must mean the same thing whichever source produced it.
const TIGHT_FRACTION = 0.6;
/**
* Translate a stored assessment into the catalog's fit vocabulary.
*
* `null` means the measurement says nothing about fit (`unknown` verdict, or no
* assessment at all) — it must NOT collapse into `'unknown'`, which is a real
* badge value the estimate also produces.
*
* @param {object|null} assessment
* @returns {'comfortable'|'tight'|'too-large'|'incompatible'|null}
*/
export function measuredFitVerdict(assessment) {
switch (assessment?.verdict) {
case 'fits': {
const resident = assessment?.residentGb;
const budget = assessment?.environment?.memoryBudgetGb;
// Footprint unknown (LM Studio reports none) — it RAN, so `comfortable`
// is the honest read; we just can't say how close to the edge it was.
if (!Number.isFinite(resident) || !Number.isFinite(budget) || budget <= 0) return 'comfortable';
return resident > budget * TIGHT_FRACTION ? 'tight' : 'comfortable';
}
case 'does-not-fit': return 'too-large';
case 'incompatible': return 'incompatible';
default: return null;
}
}
/**
* Reconcile the size ESTIMATE with the MEASUREMENT for one model.
*
* Rules, in order:
* 1. No usable measurement → the estimate stands, unchanged.
* 2. A STALE measurement never overrides the estimate. It describes a machine
* that no longer exists (RAM changed, backend updated), so trusting it over
* a fresh estimate would be worse than the estimate alone — but it is still
* reported, because "we measured this before your upgrade" is information.
* 3. Otherwise the measurement wins. It ran the model; the estimate multiplied
* a file size by 1.2.
*
* `disagrees` is the point of the whole exercise: it marks the models where
* running the thing contradicted the arithmetic, which is the most useful thing
* this data can say.
*
* @param {string|null} estimatedFit the catalog's `classifyFit` result
* @param {object|null} measurement compact record from the assessments store:
* `{ fit, verdict, assessedAt, stale, staleReason }`
*/
export function reconcileFit(estimatedFit, measurement) {
const estimated = estimatedFit || null;
const measuredFit = measurement?.fit || null;
const base = {
fit: estimated,
fitSource: 'estimated',
estimatedFit: estimated,
measuredFit,
verdict: measurement?.verdict || null,
assessedAt: measurement?.assessedAt || null,
// `disagrees` is only meaningful when BOTH sides have an opinion; an
// unmeasured or unestimated model disagrees with nothing.
disagrees: Boolean(measuredFit && estimated && estimated !== 'unknown' && measuredFit !== estimated),
stale: Boolean(measurement?.stale),
staleReason: measurement?.staleReason || null,
};
if (!measuredFit || measurement?.stale) return base;
return { ...base, fit: measuredFit, fitSource: 'measured' };
}