Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/** Shared best-effort local-model memory reclamation and headroom reporting. */
import { execFile } from '../lib/childProcess.js';
import { platform, freemem, totalmem } from 'os';
import { promisify } from 'util';
import { getLoadedModels as ollamaLoadedModels, unloadModel as ollamaUnload, getBaseUrl as ollamaBaseUrl } from './ollamaManager.js';
import { getLoadedModels as lmStudioLoadedModels, unloadModel as lmStudioUnload, getBaseUrl as lmStudioBaseUrl } from './lmStudioManager.js';
import { getAllProviders } from './providers.js';
import { probeOpenAiModels } from '../lib/openAiModelsProbe.js';
import { getCudaCapability } from '../lib/cudaCapability.js';
import { localRuntimeForProvider, LOCAL_RUNTIMES } from '../lib/localProviderRuntime.js';
import { resolveVllmProjectDir } from '../lib/vllmQwenProject.js';
import { resolveSglangProjectDir } from '../lib/sglangQwenProject.js';
const execFileAsync = promisify(execFile);
const GB = 2 ** 30;
const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '0.0.0.0', '::1']);
export function isLocalBackendUrl(url) {
if (!url || !URL.canParse(url)) return false;
const host = new URL(url).hostname.replace(/^\[|\]$/g, '');
return LOOPBACK_HOSTS.has(host) || host.startsWith('127.');
}
export async function unloadResidentModels() {
const unloaded = [];
if (isLocalBackendUrl(ollamaBaseUrl())) {
const models = await ollamaLoadedModels().catch(() => []);
for (const model of models) {
const name = model?.name || model?.id;
const result = name ? await ollamaUnload(name).catch(() => null) : null;
if (result?.unloaded) unloaded.push(`ollama:${name}`);
}
}
if (isLocalBackendUrl(lmStudioBaseUrl())) {
const models = await lmStudioLoadedModels(true).catch(() => []);
for (const model of models) {
const result = model?.id ? await lmStudioUnload(model.id).catch(() => null) : null;
if (result?.success) unloaded.push(`lmstudio:${model.id}`);
}
}
return unloaded;
}
/**
* Short on purpose. This runs on the hot path of every GPU-heavy local job, and
* the only question it asks is whether something on loopback answers *now* — a
* container that needs longer than this to reply is not the thing about to lose
* the VRAM race.
*/
export const GPU_BLOCKER_PROBE_TIMEOUT_MS = 1_500;
/**
* The refusal prose for a serving vLLM container. Written the way
* `vllmStartBlockedReason` writes its refusals: it names the one command that
* fixes it, and says why PortOS will not run that command for the operator.
*/
const vllmBlockerReason = ({ endpoint, projectDir }) =>
`${LOCAL_RUNTIMES.vllm.label} is serving at ${endpoint} and holds nearly all of this GPU's VRAM, so this job would run out of memory partway through its model load. Stop the container with \`docker compose --profile single stop\` in ${projectDir}, then start this job again. PortOS will not stop it for you: nothing would restart it, an agent session attached to it dies with it, and a cold start takes roughly 5–7 minutes.`;
/**
* The same refusal for the SGLang container. The stop command deliberately has
* no `--profile`: the vLLM stack is upstream's compose project, whose services
* sit behind a `single` profile, while the SGLang compose file is generated by
* `lib/sglangQwenRecipe.js` and declares one unprofiled service. Copying vLLM's
* `--profile single` here would print a command that stops nothing.
*/
const sglangBlockerReason = ({ endpoint, projectDir }) =>
`${LOCAL_RUNTIMES.sglang.label} is serving at ${endpoint} and holds nearly all of this GPU's VRAM, so this job would run out of memory partway through its model load. Stop the container with \`docker compose stop\` in ${projectDir}, then start this job again. PortOS will not stop it for you: nothing would restart it, an agent session attached to it dies with it, and a cold start is measured in minutes.`;
/**
* One row per GPU-exclusive local container, keyed by its `*Backed` provider
* marker. Table-driven rather than a second copy of the probe: the two
* containers differ only in which marker selects them, which directory holds
* their compose file, and how the refusal is worded — everything else (the
* enabled-provider gate, the CUDA gate, the credential-preferring endpoint
* dedupe, the reachable-means-blocking rule) is identical, and a forked copy is
* how the second runtime would quietly miss a fix made to the first.
*/
const GPU_EXCLUSIVE_RUNTIMES = Object.freeze([
Object.freeze({ kind: 'vllm', marker: 'vllmBacked', resolveProjectDir: resolveVllmProjectDir, reason: vllmBlockerReason }),
Object.freeze({ kind: 'sglang', marker: 'sglangBacked', resolveProjectDir: resolveSglangProjectDir, reason: sglangBlockerReason }),
]);
/**
* GPU tenants that make a local media job unwinnable *before* it spends minutes
* loading a model into VRAM it cannot have.
*
* Today that is two things: the vLLM Qwen3.8-27B compose container (~23 GB of a
* 24 GB RTX 3090) and the SGLang Qwen3.8-27B container on a Hopper/Blackwell
* card. `unloadResidentModels()` above cannot help — it evicts loopback Ollama /
* LM Studio, and a container is invisible to both. The job therefore proceeded,
* allocated, and died inside the model load with an OOM naming neither the
* container nor a remedy.
*
* **Detect and refuse — never auto-stop** (#4766). An Ollama unload is
* transparent (the next request reloads); stopping one of these containers is
* not. Nothing restarts it, an attached CoS session dies with it, and a cold
* start is minutes. PortOS deliberately never *starts* them either — both
* directions stay operator decisions, and the refusal prose is what makes that
* actionable.
*
* **Sentinel discipline.** A probe that fails, times out, or throws means "the
* container is not serving" → **no blocker, proceed**. *Couldn't check* must
* never collapse into *is blocking*, or an unrelated network hiccup would refuse
* every media job on the box. Only an endpoint that actually answers blocks —
* including a 401/403, which is a container up behind an API key.
*
* Two cheap gates keep the common path free of latency, in cost order:
* 1. No **enabled** `vllmBacked` / `sglangBacked` provider → nothing to probe,
* no network call.
* 2. `getCudaCapability()` says `'absent'` → this host has no NVIDIA GPU to
* contend over. `'unknown'` (nvidia-smi wedged or too old) is NOT a
* negative and still probes — same sentinel rule as above, in the other
* direction.
*
* @param {{env?: NodeJS.ProcessEnv, providers?: Array<object>|null, timeoutMs?: number}} [opts]
* `providers` is injectable for tests; production reads the provider store.
* @returns {Promise<Array<{runtime: string, providerId: string|null, providerName: string|null,
* endpoint: string, reason: string}>>} empty when nothing is holding the GPU
*/
export async function detectGpuBlockers({ env = process.env, providers = null, timeoutMs = GPU_BLOCKER_PROBE_TIMEOUT_MS } = {}) {
const all = Array.isArray(providers) ? providers : await getAllProviders().catch(() => []);
const enabled = (Array.isArray(all) ? all : []).filter((p) => p?.enabled === true);
const rows = GPU_EXCLUSIVE_RUNTIMES
.map((row) => ({ row, candidates: enabled.filter((p) => p?.[row.marker] === true) }))
.filter(({ candidates }) => candidates.length > 0);
if (!rows.length) return [];
// No custom timeout: this shares the memo the image-to-3D lane gating already
// fills, and a shorter deadline here would write an 'unknown' into it that the
// other reader would have to treat as "couldn't detect".
const cuda = await getCudaCapability().catch(() => null);
if (cuda?.status === 'absent') return [];
// Two providers can point at one container with different credentials, and a
// remote endpoint is a different machine's GPU — `localRuntimeForProvider`
// drops the latter and returns null for it.
const targets = new Map();
for (const { row, candidates } of rows) {
for (const provider of candidates) {
const runtime = localRuntimeForProvider(provider);
if (runtime?.kind !== row.kind || !runtime.endpoint) continue;
const existing = targets.get(runtime.endpoint);
// Prefer the credentialed record: an unauthenticated probe still detects
// the container (401 ⇒ reachable), but a readable listing is the cleaner
// signal.
if (!existing || (!existing.apiKey && typeof provider.apiKey === 'string' && provider.apiKey !== '')) {
targets.set(runtime.endpoint, { row, provider, apiKey: typeof provider.apiKey === 'string' ? provider.apiKey : '' });
}
}
}
if (!targets.size) return [];
const probes = await Promise.all([...targets].map(async ([endpoint, { row, provider, apiKey }]) => {
const probe = await probeOpenAiModels(endpoint, { timeoutMs, apiKey }).catch(() => null);
if (!probe?.reachable) return null;
return {
runtime: row.kind,
providerId: provider?.id ?? null,
providerName: provider?.name ?? null,
endpoint,
reason: row.reason({ endpoint, projectDir: row.resolveProjectDir(env) }),
};
}));
return probes.filter(Boolean);
}
/**
* One refusal message for a `blockers` array, so all four GPU-heavy callers word
* the refusal identically instead of each joining the reasons their own way.
*
* @param {Array<{reason?: string}>|null|undefined} blockers
* @returns {string} `''` when nothing is blocking — callers gate on
* `blockers.length`, never on this string
*/
export const gpuBlockersMessage = (blockers) =>
(Array.isArray(blockers) ? blockers : []).map((b) => b?.reason).filter(Boolean).join(' ');
const parsePageSize = (out) => Number(out.match(/page size of (\d+) bytes/i)?.[1] || 4096);
async function darwinAvailableGb() {
const { stdout } = await execFileAsync('vm_stat');
const pageSize = parsePageSize(stdout);
const pages = (label) => Number(stdout.match(new RegExp(`${label}:\\s+(\\d+)\\.`))?.[1] || 0);
const available = pages('Pages free') + pages('Pages inactive') + pages('Pages speculative') + pages('Pages purgeable');
return available ? (available * pageSize) / GB : null;
}
export async function getAvailableMemoryGb() {
if (platform() === 'darwin') {
const available = await darwinAvailableGb().catch(() => null);
if (Number.isFinite(available) && available > 0) return available;
}
return freemem() / GB;
}
/**
* The preflight every GPU-heavy local job runs: refuse-or-reclaim, then report
* the headroom the caller sizes its run against.
*
* `blockers` is additive (#4766) — a non-empty array means the job cannot
* succeed no matter how much system RAM is free, and the caller must fail fast
* with `gpuBlockersMessage(...)` rather than start a model load.
*
* Nothing is unloaded while blocked: evicting the user's resident LLM is a real
* cost, and paying it for a run that is about to be refused helps nobody.
*/
export async function prepareLocalMemory() {
const blockers = await detectGpuBlockers().catch(() => []);
const unloaded = blockers.length ? [] : await unloadResidentModels().catch(() => []);
const availableGb = await getAvailableMemoryGb().catch(() => 0);
const totalGb = totalmem() / GB;
return { unloaded, availableGb, totalGb, budgetGb: Math.min(totalGb, availableGb), blockers };
}