Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Capability test results — the durable store.
*
* Split from `modelCapabilityTests.js` for the same two reasons the assessment
* store is split from its runner:
*
* 1. **AI Provider Usage Policy** (root AGENTS.md). Nothing in this file can
* reach a provider, so a read path that imports it is safe by construction
* rather than by review.
* 2. **No import cycle** — the report builder needs stored results, and the
* runner needs to write them.
*
* ## Storage (docs/STORAGE.md)
*
* `file-primary`, **machine-local — never federated.** A capability result
* describes what a model did on THIS machine, through THIS runtime, at the
* quantization installed here. A peer inheriting "passed" for a model it runs at
* a different quant would be told something untrue about its own install. No
* sync cursor, no tombstone, no `schemaVersions.js` entry — the same rule
* `localModelAssessmentStore.js` follows and for the same reason.
*
* ## Privacy
*
* A record holds the model's own output and, for the sandbox test, the agent
* transcript — because seeing the output IS the feature. Both are produced
* inside a throwaway sandbox from a fixed fixture, so neither carries user data,
* and neither ever leaves this machine. The recorded environment stays coarse
* (platform, arch, CPU count, total memory) exactly as assessments do: no
* hostname, no username, no paths.
*/
import { join } from 'path';
import { rename } from 'fs/promises';
import { PATHS, atomicWrite, ensureDir, tryReadFile, safeJSONParse } from '../lib/fileUtils.js';
import { captureDurableEnvironment } from './localModelAssessmentStore.js';
const storeDir = () => join(PATHS.data, 'local-llm');
const storeFile = () => join(storeDir(), 'capability-tests.json');
// Storage-layout version. Bump only when a reader must branch on the shape.
const STORE_SCHEMA_VERSION = 1;
/**
* Caps on what one record keeps.
*
* Generous enough that a real answer is never clipped — a twelve-beat outline
* runs ~5k characters and a repair transcript ~20k — but bounded, because the
* whole store is one JSON file rewritten on every save. A runaway model that
* emits a megabyte must not make the file unreadable for every other result.
*/
export const MAX_OUTPUT_CHARS = 24000;
export const MAX_TRANSCRIPT_CHARS = 80000;
/**
* Trim to a cap, keeping the end the reader needs.
*
* `head` keeps the beginning (a model's answer starts with its answer); `tail`
* keeps the end (a transcript's verdict is its last lines). Either way the
* elision is MARKED — silently returning a shortened string would let a
* truncated outline read as a model that stopped early.
*/
export function clampText(text, limit, keep = 'head') {
const value = String(text ?? '');
if (value.length <= limit) return value;
const dropped = value.length - limit;
const note = `\n\n…[${dropped.toLocaleString('en-US')} characters trimmed by PortOS]\n\n`;
return keep === 'tail'
? note.trimStart() + value.slice(-limit)
: value.slice(0, limit) + note.trimEnd();
}
/** Identity of one stored result: one record per model per test. */
export const capabilityResultKey = (backend, modelId, testId) => `${backend}:${modelId}:${testId}`;
export const keyOfResult = (r) => capabilityResultKey(r?.backend, r?.modelId, r?.testId);
async function quarantineStore(reason) {
const parked = `${storeFile()}.corrupt-${Date.now()}`;
const moved = await rename(storeFile(), parked).then(() => true).catch(() => false);
console.error(`❌ Capability tests: ${reason} — ${moved ? `parked the old file as ${parked.split('/').pop()}` : 'could not park the old file'}`);
}
export async function loadStore() {
const raw = await tryReadFile(storeFile());
// Never written = an empty store, not an error.
if (raw == null) return { schemaVersion: STORE_SCHEMA_VERSION, results: [], readError: null };
const parsed = safeJSONParse(raw, null);
if (!parsed || !Array.isArray(parsed.results)) {
// Present but unparseable is NOT "nothing has been tested" — say so, or a
// re-run would quietly overwrite results the user spent minutes earning.
return { schemaVersion: STORE_SCHEMA_VERSION, results: [], readError: 'capability test results file is unreadable or malformed' };
}
return { schemaVersion: parsed.schemaVersion ?? STORE_SCHEMA_VERSION, results: parsed.results, readError: null };
}
/** Every stored result. Disk only — safe from any read path. */
export async function loadResults() {
return (await loadStore()).results;
}
/**
* Persist one result, replacing any previous run of the same model+test.
*
* History is deliberately not kept: a capability verdict is a statement about
* the model as installed now, and an older run under a different quantization
* or runtime build is misleading rather than informative. Re-running is cheap
* enough that keeping a trail would cost more than it explains.
*/
export async function saveResult(result) {
const { results, readError } = await loadStore();
// Rewriting from an empty in-memory list after a failed read would replace
// every stored result with this one. Quarantine instead — nothing is lost and
// the feature keeps working.
if (readError) await quarantineStore(readError);
const record = {
...result,
output: clampText(result.output, MAX_OUTPUT_CHARS, 'head'),
transcript: clampText(result.transcript, MAX_TRANSCRIPT_CHARS, 'tail'),
environment: result.environment ?? captureDurableEnvironment(),
};
const key = keyOfResult(record);
const next = results.filter((r) => keyOfResult(r) !== key);
next.push(record);
await ensureDir(storeDir());
await atomicWrite(storeFile(), { schemaVersion: STORE_SCHEMA_VERSION, results: next });
return record;
}
/**
* Drop one recorded result. Reports whether anything was actually removed so the
* caller can 404 rather than claiming a phantom success.
*/
export async function deleteResult(backend, modelId, testId) {
const { results, readError } = await loadStore();
if (readError) return { deleted: false };
const key = capabilityResultKey(backend, modelId, testId);
const next = results.filter((r) => keyOfResult(r) !== key);
if (next.length === results.length) return { deleted: false };
await ensureDir(storeDir());
await atomicWrite(storeFile(), { schemaVersion: STORE_SCHEMA_VERSION, results: next });
console.log(`🧹 Capability tests: dropped ${testId} result for ${backend}/${modelId}`);
return { deleted: true };
}
/**
* Stored results indexed by `backend:modelId:testId`, for the report builder.
* `{}` is a real answer (nothing tested yet), distinct from `readError`.
*/
export async function indexResults() {
const { results, readError } = await loadStore();
return { index: Object.fromEntries(results.map((r) => [keyOfResult(r), r])), readError };
}