Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/** Shipped public reference data and machine-local PortOS benchmark observations. */
import { join } from 'path';
import { ServerError } from '../lib/errorHandler.js';
import { readFile } from 'fs/promises';
import { PATHS } from '../lib/paths.js';
import { atomicWrite } from '../lib/fileCore.js';
import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
import { modelComparisonCatalogSchema, modelComparisonImportSchema } from '../lib/validation.js';
const queueWrite = createFileWriteQueue();
const catalogPath = () => join(PATHS.data, 'model-comparison.json');
const isPortosBenchmarkObservation = row => row?.id?.startsWith('portos:')
|| /^PortOS Task Bench v\d+\b/.test(row?.benchmark || '');
const hasPortosRunSource = row => [
row?.quality, row?.costPerTask, row?.inputPerMillion, row?.outputPerMillion,
row?.reasoningPerMillion, row?.responseSeconds, row?.tokensPerSecond,
row?.tokensPerRun, row?.inputTokens, row?.outputTokens, row?.apiEquivalentCost, row?.quota,
].some(metric => typeof metric?.source?.url === 'string'
&& /^portos:\/\/model-comparison\/[0-9a-f-]{36}$/i.test(metric.source.url));
// This shipped label was clarified without changing the benchmark's dataset or
// scoring. Normalize the exact legacy name at read/import boundaries so upgrades
// retain stable observation ids without relaxing semantic identity checks.
const normalizeCatalog = catalog => ({
...catalog,
observations: catalog.observations.map(row => row.benchmark === 'LiveCodeBench (generation, pass@1, 2023-05-08 to 2025-04-07)'
? { ...row, benchmark: 'LiveCodeBench generation pass@1 (1,055 problems; 2023-05-08 to 2025-04-07)' }
: row),
});
export async function getModelComparison() {
const raw = await readFile(catalogPath(), 'utf8').catch(error => {
if (error.code !== 'ENOENT') throw error;
return readFile(join(PATHS.root, 'data.reference/model-comparison.json'), 'utf8');
});
// A malformed or future-version catalog must surface an error, never be
// replaced with an empty store by the next import.
const catalog = normalizeCatalog(modelComparisonCatalogSchema.parse(JSON.parse(raw)));
return catalog;
}
/** Public comparison data always comes from the shipped catalog, never local files. */
export async function getShippedModelComparison() {
const raw = await readFile(join(PATHS.root, 'data.reference/model-comparison.json'), 'utf8');
const catalog = normalizeCatalog(modelComparisonCatalogSchema.parse(JSON.parse(raw)));
return {
...catalog,
observations: catalog.observations.filter(row =>
!isPortosBenchmarkObservation(row) && !hasPortosRunSource(row)),
};
}
/** Local task-benchmark history is exposed only to Models → Performance. */
export async function getPortosModelBenchmarkObservations() {
const catalog = await getModelComparison();
return {
schemaVersion: catalog.schemaVersion,
observations: catalog.observations.filter(isPortosBenchmarkObservation),
};
}
async function mergeModelComparison(incoming) {
const [current, publicCatalog] = await Promise.all([getModelComparison(), getPublicModelComparison()]);
const rows = new Map([...current.observations, ...publicCatalog.observations].map(row => [row.id, row]));
for (const row of incoming.observations) {
const prior = rows.get(row.id);
if (prior) {
// Stable ids cannot silently change the meaning of existing evidence.
for (const key of ['provider', 'model', 'effort', 'configuration', 'billing', 'benchmark']) {
if (row[key] !== prior[key]) throw new ServerError(`Observation identity changed: ${row.id}`, { status: 409 });
}
for (const key of ['quality', 'costPerTask', 'apiEquivalentCost', 'inputPerMillion', 'outputPerMillion', 'reasoningPerMillion', 'responseSeconds', 'tokensPerSecond', 'tokensPerRun', 'inputTokens', 'outputTokens', 'quota']) {
const before = prior[key];
if (before && (!row[key] || Date.parse(row[key].source.retrievedAt) < Date.parse(before.source.retrievedAt))) row[key] = before;
}
}
rows.set(row.id, row);
}
const result = modelComparisonCatalogSchema.parse({ schemaVersion: 1, observations: [...rows.values()] });
await atomicWrite(catalogPath(), result);
return result;
}
export function importModelComparison(input) {
const incoming = normalizeCatalog(modelComparisonImportSchema.parse(input));
if (incoming.observations.some(row => isPortosBenchmarkObservation(row) || hasPortosRunSource(row))) {
throw new ServerError('PortOS benchmark observations can only be created by the explicit benchmark run.', { status: 400 });
}
return queueWrite(() => mergeModelComparison(incoming));
}
export function recordPortosModelBenchmark(observation) {
const incoming = modelComparisonImportSchema.parse({ schemaVersion: 1, observations: [observation] });
if (!isPortosBenchmarkObservation(observation) || !hasPortosRunSource(observation)) {
throw new ServerError('Invalid PortOS benchmark observation.', { status: 400 });
}
return queueWrite(() => mergeModelComparison(incoming));
}
/** Merge release evidence with researched local evidence without exposing run history. */
export async function getPublicModelComparison() {
const [shipped, local] = await Promise.all([getShippedModelComparison(), getModelComparison()]);
const rows = new Map(shipped.observations.map(row => [row.id, row]));
for (const row of local.observations) {
if (isPortosBenchmarkObservation(row) || hasPortosRunSource(row)) continue;
const prior = rows.get(row.id);
if (!prior) { rows.set(row.id, row); continue; }
for (const key of ['provider', 'model', 'effort', 'configuration', 'billing', 'benchmark']) {
if (row[key] !== prior[key]) throw new ServerError(`Observation identity changed: ${row.id}`, { status: 409 });
}
const merged = { ...prior };
for (const key of Object.keys(row)) {
if (row[key]?.source && (!prior[key]?.source || Date.parse(row[key].source.retrievedAt) >= Date.parse(prior[key].source.retrievedAt))) merged[key] = row[key];
}
if (Object.keys(row).some(key => merged[key] === row[key] && row[key]?.source && merged[key] !== prior[key])) merged.notes = row.notes;
rows.set(row.id, merged);
}
return { schemaVersion: 1, observations: [...rows.values()] };
}