Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { getCudaCapability, getCudaUtilization } from '../lib/cudaCapability.js';
import { summarizeSystemActivity } from '../lib/systemIdle.js';
import { runningAgentsByTaskId, unclaimedTaskIds } from '../lib/cosSpawnWindow.js';
import { listJobs, getRunningJob } from './mediaJobQueue/index.js';
import { sanitizeJob } from './mediaJobQueue/sanitizeJob.js';
import { listGeneratingModelSummaries } from './imageTo3d/models.js';
import { getLoadedModels, getLastLoadedModelsError } from './ollamaManager.js';
import { listActiveAppOperations } from './appOperations.js';
import { readPersistentMindStateForSafetyCheck } from './cosState.js';
import { isUpdateInProgress } from './updateChecker.js';
import { listActiveRenders } from './renderActivity.js';
import { getAgents } from './cosAgentLifecycle.js';
import { getPendingTaskIds } from './cosTaskStore.js';
import { getStatus } from './cos.js';
// Lazy — NOT a static top-level import. `services/runner.js` and
// `services/backup.js` both drag heavy subtrees (provider CLI spawning;
// `socket.js` → `instances.js` respectively) that this module's own callers
// (`routes/systemHealth.js`, the dashboard) must not eagerly reach just to
// read a count/flag. `routes/update.js` already defers this whole module the
// same way for the same reason — see "Import scoping" in server/AGENTS.md.
const importRunner = () => import('./runner.js');
const importBackup = () => import('./backup.js');
// Ollama has no lifecycle callback for external loads/evictions. Shared cache
// also coalesces simultaneous telemetry and initial inspector reads.
let ollamaSample = null;
let ollamaSampleAt = 0;
let ollamaRead = null;
async function getOllamaTelemetry() {
if (ollamaSample && Date.now() - ollamaSampleAt < 3000) return ollamaSample;
if (!ollamaRead) {
ollamaRead = getLoadedModels({ timeout: 1500 }).then((models) => ({
trusted: !getLastLoadedModelsError(),
models: models.slice(0, 50),
})).catch(() => ({ trusted: false, models: [] })).then((sample) => {
ollamaSample = sample;
ollamaSampleAt = Date.now();
return sample;
}).finally(() => { ollamaRead = null; });
}
return ollamaRead;
}
const LIVE_STATUSES = new Set(['queued', 'running']);
/**
* What the Persistent Mind is doing, WITHOUT any of what it is thinking about.
*
* Counts and lifecycle only: a queued message's text, its attachments, and the
* turn's own content never appear here. This rides the same `/api/system/
* processing` payload the dashboard polls every few seconds, and a mind's
* messages are the most personal records on the install.
*
* `trusted: false` (unreadable state) is deliberately NOT collapsed into "idle".
* `summarizeSystemActivity` treats it as a blocker, matching the update path's
* own `PERSISTENT_MIND_STATE_UNTRUSTED` refusal.
*/
function summarizeMind(snapshot) {
if (!snapshot?.trusted) return { trusted: false, status: 'unknown', thinking: false, queued: 0 };
const mind = snapshot.persistentMind;
const activeTurn = mind?.activeTurn || null;
return {
trusted: true,
enabled: mind?.enabled === true,
started: mind?.started === true,
status: typeof mind?.status === 'string' ? mind.status : 'disabled',
thinking: Boolean(activeTurn),
thinkingSince: activeTurn?.startedAt || null,
queued: Array.isArray(mind?.queuedMessages) ? mind.queuedMessages.length : 0,
};
}
/**
* In-flight LLM/pipeline run count, off the toolkit's own `activeRuns` +
* `externalRuns` tracking (`getActiveRunCount` in `services/runner.js`).
* Counts and lifecycle only — no prompt or response content ever reaches this
* slice. `trusted: false` (the count could not be read) is NOT collapsed into
* "idle", the same contract the agent and Persistent Mind slices keep: a
* failed read must never manufacture the zero that unlocks a restart.
*/
function llmRunState(count) {
return count === null ? { trusted: false, active: 0 } : { trusted: true, active: count };
}
/**
* Active vs queued CoS agents, from ONE read of the agent list.
*
* A task stays 'pending' until spawnAgentForTask flips it to 'in_progress',
* which happens AFTER its agent is registered as running — so a snapshot taken
* in between would count one task as queued AND active, and the widget read
* 'N active, N queued' for a queue of N. A task a live agent holds is active —
* the settlement in lib/cosSpawnWindow.js, shared with every other surface that
* pairs the two lists.
*
* When the agent list is readable, BOTH counts come off that one read: taking
* `active` from `getStatus()` and `queued` from here would let the two skew
* against each other, which is the same defect one layer up. When it is NOT
* readable (`agents === null`), there is no claim set to subtract — fall back to
* `getStatus()`'s own tally rather than reporting zero active agents while still
* counting their tasks as queued, which would understate BOTH numbers at once.
*
* And when BOTH reads failed, `active` is 0 because nothing could be counted —
* not because nothing is running. `trusted: false` says which, so the idle
* verdict can refuse instead of reading that zero as an empty install and
* restarting out from under a live agent (the contract the Persistent Mind
* slice already keeps). The queued count is unaffected: the pending-task list
* is a separate read, and with no claim set to subtract it over-reports rather
* than under-reports, which is the safe direction for a gate.
*/
function agentCounts(agents, cosStatus, pendingTaskIds) {
const runningAgents = agents === null ? null : agents.filter((agent) => agent.status === 'running');
const heldByRunningAgent = runningAgentsByTaskId(agents);
return {
// An unreadable pending-task list is its own untrusted reading: zero queued
// tasks is the value that unlocks a restart, so it may not come from a
// failed read any more than a zero agent count may.
trusted: (agents !== null || Boolean(cosStatus)) && pendingTaskIds !== null,
active: runningAgents ? runningAgents.length : (cosStatus?.activeAgents || 0),
queued: pendingTaskIds ? unclaimedTaskIds(pendingTaskIds, heldByRunningAgent).length : 0,
};
}
/**
* Just the slices `summarizeSystemActivity` reads, plus the verdict.
*
* Split out because two callers — `GET /api/update/auto` and the auto-update
* tick — want only `activity`, and the full snapshot also shells out to
* nvidia-smi and reads Ollama residency. Neither feeds the verdict.
* The HTTP inspector opts into cached residency; safety checks omit it.
*/
export async function getSystemActivity({ includeOllama = false } = {}) {
const [jobs, models, pendingTaskIds, agents, mindState, llmState, backupInProgress, ollama] = await Promise.all([
Promise.resolve(listJobs()).then((items) => items.filter((job) => LIVE_STATUSES.has(job.status))),
// `null` = the read FAILED, distinct from `[]` = read fine, nothing
// building. Both of these degrade to the value that unlocks a restart, so
// neither may be manufactured from a failed read.
listGeneratingModelSummaries().catch(() => null),
getPendingTaskIds().catch(() => null),
// `null` = the read FAILED, distinct from `[]` = read fine, no agents. The
// counts degrade differently for the two, so they must stay separable.
getAgents().catch(() => null),
// Same contract one layer down: the reader reports `trusted: false` rather
// than an empty mind, so an unreadable state cannot read as an idle one.
readPersistentMindStateForSafetyCheck().catch(() => ({ trusted: false, persistentMind: null })),
// Same contract again: a failed read degrades to `null`, not 0.
importRunner().then(async (m) => {
const [count, runs] = await Promise.all([m.getActiveRunCount(), m.getActiveRunSummaries().catch(() => null)]);
return { ...llmRunState(count), runs };
}).catch(() => ({ trusted: false, active: 0, runs: null })),
// The only way this rejects is the lazy import itself failing (the module
// is otherwise a synchronous, no-I/O flag read) — an anomaly rare enough
// that it is itself reason to refuse rather than read as "no backup
// running", the same fail-closed direction every other slice here takes.
importBackup().then((m) => m.isBackupInProgress()).catch(() => true),
includeOllama ? getOllamaTelemetry() : null,
]);
const cosStatus = agents === null ? await getStatus().catch(() => null) : null;
const slices = {
jobs: jobs.map(sanitizeJob),
// `null` survives to the verdict as a blocker; the widget's own
// `extras?.imageTo3d || []` already reads it as an empty list.
extras: { imageTo3d: models === null ? null : models.map((model) => ({ id: model.id, name: model.name || model.id })) },
agents: agentCounts(agents, cosStatus, pendingTaskIds),
mind: summarizeMind(mindState),
// A prompt/stage run holds a provider connection or a CLI/TUI child
// process — activity in exactly the sense the updater must not restart
// through, and distinct from a CoS agent (its own slice above, spawned
// through a different path entirely).
llm: llmState,
// An App Management update/standardize holds a checkout and restarts PM2
// processes — activity in exactly the sense that matters to a caller
// deciding whether it may restart the install.
appOperations: listActiveAppOperations(),
update: { inProgress: isUpdateInProgress() },
// Final renders, draft excerpts and shot renders run their own ffmpeg and
// browsers outside the media queue; an in-process count, no I/O.
renders: listActiveRenders(),
// No trusted contract needed: an in-process flag with no I/O once loaded.
backup: { inProgress: backupInProgress },
};
return { ...slices, ...(includeOllama ? { ollama } : {}), activity: summarizeSystemActivity(slices) };
}
/**
* GPU samples have no lifecycle event — nvidia-smi is a poll. Kept off
* `getSystemActivity()` so the shared activity read stays bounded, and so the
* dashboard inspector can poll this alone while it is on screen.
*/
export async function getGpuTelemetry() {
const capability = await getCudaCapability();
const utilization = capability.status === 'available' ? await getCudaUtilization() : { status: capability.status, gpus: [] };
const running = getRunningJob();
return {
updatedAt: new Date().toISOString(),
ollama: await getOllamaTelemetry(),
gpu: {
status: capability.status,
laneBusy: Boolean(running),
laneKind: running?.kind || null,
gpus: (utilization.gpus.length ? utilization.gpus : capability.gpus).map((gpu) => ({
name: gpu.name,
utilizationPercent: gpu.utilizationPercent ?? null,
memoryUsedMib: gpu.memoryUsedMib ?? null,
memoryTotalMib: gpu.memoryTotalMib ?? gpu.vramMib ?? null,
})),
},
};
}
export async function getActiveProcessing() {
const [telemetry, activity] = await Promise.all([
getGpuTelemetry(),
getSystemActivity(),
]);
return {
...activity,
updatedAt: telemetry.updatedAt,
gpu: telemetry.gpu,
extras: { ...activity.extras, ollama: telemetry.ollama.models },
};
}