Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

11.1 KB
Raw Blame
/** Shared best-effort local-model memory reclamation and headroom reporting. */

import { execFile } from '../lib/childProcess.js';
import { platform, freemem, totalmem } from 'os';
import { promisify } from 'util';
import { getLoadedModels as ollamaLoadedModels, unloadModel as ollamaUnload, getBaseUrl as ollamaBaseUrl } from './ollamaManager.js';
import { getLoadedModels as lmStudioLoadedModels, unloadModel as lmStudioUnload, getBaseUrl as lmStudioBaseUrl } from './lmStudioManager.js';
import { getAllProviders } from './providers.js';
import { probeOpenAiModels } from '../lib/openAiModelsProbe.js';
import { getCudaCapability } from '../lib/cudaCapability.js';
import { localRuntimeForProvider, LOCAL_RUNTIMES } from '../lib/localProviderRuntime.js';
import { resolveVllmProjectDir } from '../lib/vllmQwenProject.js';
import { resolveSglangProjectDir } from '../lib/sglangQwenProject.js';

const execFileAsync = promisify(execFile);
const GB = 2 ** 30;
const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '0.0.0.0', '::1']);

export function isLocalBackendUrl(url) {
  if (!url || !URL.canParse(url)) return false;
  const host = new URL(url).hostname.replace(/^\[|\]$/g, '');
  return LOOPBACK_HOSTS.has(host) || host.startsWith('127.');
}

export async function unloadResidentModels() {
  const unloaded = [];
  if (isLocalBackendUrl(ollamaBaseUrl())) {
    const models = await ollamaLoadedModels().catch(() => []);
    for (const model of models) {
      const name = model?.name || model?.id;
      const result = name ? await ollamaUnload(name).catch(() => null) : null;
      if (result?.unloaded) unloaded.push(`ollama:${name}`);
    }
  }
  if (isLocalBackendUrl(lmStudioBaseUrl())) {
    const models = await lmStudioLoadedModels(true).catch(() => []);
    for (const model of models) {
      const result = model?.id ? await lmStudioUnload(model.id).catch(() => null) : null;
      if (result?.success) unloaded.push(`lmstudio:${model.id}`);
    }
  }
  return unloaded;
}

/**
 * Short on purpose. This runs on the hot path of every GPU-heavy local job, and
 * the only question it asks is whether something on loopback answers *now* — a
 * container that needs longer than this to reply is not the thing about to lose
 * the VRAM race.
 */
export const GPU_BLOCKER_PROBE_TIMEOUT_MS = 1_500;

/**
 * The refusal prose for a serving vLLM container. Written the way
 * `vllmStartBlockedReason` writes its refusals: it names the one command that
 * fixes it, and says why PortOS will not run that command for the operator.
 */
const vllmBlockerReason = ({ endpoint, projectDir }) =>
  `${LOCAL_RUNTIMES.vllm.label} is serving at ${endpoint} and holds nearly all of this GPU's VRAM, so this job would run out of memory partway through its model load. Stop the container with \`docker compose --profile single stop\` in ${projectDir}, then start this job again. PortOS will not stop it for you: nothing would restart it, an agent session attached to it dies with it, and a cold start takes roughly 5–7 minutes.`;

/**
 * The same refusal for the SGLang container. The stop command deliberately has
 * no `--profile`: the vLLM stack is upstream's compose project, whose services
 * sit behind a `single` profile, while the SGLang compose file is generated by
 * `lib/sglangQwenRecipe.js` and declares one unprofiled service. Copying vLLM's
 * `--profile single` here would print a command that stops nothing.
 */
const sglangBlockerReason = ({ endpoint, projectDir }) =>
  `${LOCAL_RUNTIMES.sglang.label} is serving at ${endpoint} and holds nearly all of this GPU's VRAM, so this job would run out of memory partway through its model load. Stop the container with \`docker compose stop\` in ${projectDir}, then start this job again. PortOS will not stop it for you: nothing would restart it, an agent session attached to it dies with it, and a cold start is measured in minutes.`;

/**
 * One row per GPU-exclusive local container, keyed by its `*Backed` provider
 * marker. Table-driven rather than a second copy of the probe: the two
 * containers differ only in which marker selects them, which directory holds
 * their compose file, and how the refusal is worded — everything else (the
 * enabled-provider gate, the CUDA gate, the credential-preferring endpoint
 * dedupe, the reachable-means-blocking rule) is identical, and a forked copy is
 * how the second runtime would quietly miss a fix made to the first.
 */
const GPU_EXCLUSIVE_RUNTIMES = Object.freeze([
  Object.freeze({ kind: 'vllm', marker: 'vllmBacked', resolveProjectDir: resolveVllmProjectDir, reason: vllmBlockerReason }),
  Object.freeze({ kind: 'sglang', marker: 'sglangBacked', resolveProjectDir: resolveSglangProjectDir, reason: sglangBlockerReason }),
]);

/**
 * GPU tenants that make a local media job unwinnable *before* it spends minutes
 * loading a model into VRAM it cannot have.
 *
 * Today that is two things: the vLLM Qwen3.8-27B compose container (~23 GB of a
 * 24 GB RTX 3090) and the SGLang Qwen3.8-27B container on a Hopper/Blackwell
 * card. `unloadResidentModels()` above cannot help — it evicts loopback Ollama /
 * LM Studio, and a container is invisible to both. The job therefore proceeded,
 * allocated, and died inside the model load with an OOM naming neither the
 * container nor a remedy.
 *
 * **Detect and refuse — never auto-stop** (#4766). An Ollama unload is
 * transparent (the next request reloads); stopping one of these containers is
 * not. Nothing restarts it, an attached CoS session dies with it, and a cold
 * start is minutes. PortOS deliberately never *starts* them either — both
 * directions stay operator decisions, and the refusal prose is what makes that
 * actionable.
 *
 * **Sentinel discipline.** A probe that fails, times out, or throws means "the
 * container is not serving" → **no blocker, proceed**. *Couldn't check* must
 * never collapse into *is blocking*, or an unrelated network hiccup would refuse
 * every media job on the box. Only an endpoint that actually answers blocks —
 * including a 401/403, which is a container up behind an API key.
 *
 * Two cheap gates keep the common path free of latency, in cost order:
 *   1. No **enabled** `vllmBacked` / `sglangBacked` provider → nothing to probe,
 *      no network call.
 *   2. `getCudaCapability()` says `'absent'` → this host has no NVIDIA GPU to
 *      contend over. `'unknown'` (nvidia-smi wedged or too old) is NOT a
 *      negative and still probes — same sentinel rule as above, in the other
 *      direction.
 *
 * @param {{env?: NodeJS.ProcessEnv, providers?: Array<object>|null, timeoutMs?: number}} [opts]
 *   `providers` is injectable for tests; production reads the provider store.
 * @returns {Promise<Array<{runtime: string, providerId: string|null, providerName: string|null,
 *   endpoint: string, reason: string}>>} empty when nothing is holding the GPU
 */
export async function detectGpuBlockers({ env = process.env, providers = null, timeoutMs = GPU_BLOCKER_PROBE_TIMEOUT_MS } = {}) {
  const all = Array.isArray(providers) ? providers : await getAllProviders().catch(() => []);
  const enabled = (Array.isArray(all) ? all : []).filter((p) => p?.enabled === true);
  const rows = GPU_EXCLUSIVE_RUNTIMES
    .map((row) => ({ row, candidates: enabled.filter((p) => p?.[row.marker] === true) }))
    .filter(({ candidates }) => candidates.length > 0);
  if (!rows.length) return [];

  // No custom timeout: this shares the memo the image-to-3D lane gating already
  // fills, and a shorter deadline here would write an 'unknown' into it that the
  // other reader would have to treat as "couldn't detect".
  const cuda = await getCudaCapability().catch(() => null);
  if (cuda?.status === 'absent') return [];

  // Two providers can point at one container with different credentials, and a
  // remote endpoint is a different machine's GPU — `localRuntimeForProvider`
  // drops the latter and returns null for it.
  const targets = new Map();
  for (const { row, candidates } of rows) {
    for (const provider of candidates) {
      const runtime = localRuntimeForProvider(provider);
      if (runtime?.kind !== row.kind || !runtime.endpoint) continue;
      const existing = targets.get(runtime.endpoint);
      // Prefer the credentialed record: an unauthenticated probe still detects
      // the container (401 ⇒ reachable), but a readable listing is the cleaner
      // signal.
      if (!existing || (!existing.apiKey && typeof provider.apiKey === 'string' && provider.apiKey !== '')) {
        targets.set(runtime.endpoint, { row, provider, apiKey: typeof provider.apiKey === 'string' ? provider.apiKey : '' });
      }
    }
  }
  if (!targets.size) return [];

  const probes = await Promise.all([...targets].map(async ([endpoint, { row, provider, apiKey }]) => {
    const probe = await probeOpenAiModels(endpoint, { timeoutMs, apiKey }).catch(() => null);
    if (!probe?.reachable) return null;
    return {
      runtime: row.kind,
      providerId: provider?.id ?? null,
      providerName: provider?.name ?? null,
      endpoint,
      reason: row.reason({ endpoint, projectDir: row.resolveProjectDir(env) }),
    };
  }));
  return probes.filter(Boolean);
}

/**
 * One refusal message for a `blockers` array, so all four GPU-heavy callers word
 * the refusal identically instead of each joining the reasons their own way.
 *
 * @param {Array<{reason?: string}>|null|undefined} blockers
 * @returns {string} `''` when nothing is blocking — callers gate on
 *   `blockers.length`, never on this string
 */
export const gpuBlockersMessage = (blockers) =>
  (Array.isArray(blockers) ? blockers : []).map((b) => b?.reason).filter(Boolean).join(' ');

const parsePageSize = (out) => Number(out.match(/page size of (\d+) bytes/i)?.[1] || 4096);

async function darwinAvailableGb() {
  const { stdout } = await execFileAsync('vm_stat');
  const pageSize = parsePageSize(stdout);
  const pages = (label) => Number(stdout.match(new RegExp(`${label}:\\s+(\\d+)\\.`))?.[1] || 0);
  const available = pages('Pages free') + pages('Pages inactive') + pages('Pages speculative') + pages('Pages purgeable');
  return available ? (available * pageSize) / GB : null;
}

export async function getAvailableMemoryGb() {
  if (platform() === 'darwin') {
    const available = await darwinAvailableGb().catch(() => null);
    if (Number.isFinite(available) && available > 0) return available;
  }
  return freemem() / GB;
}

/**
 * The preflight every GPU-heavy local job runs: refuse-or-reclaim, then report
 * the headroom the caller sizes its run against.
 *
 * `blockers` is additive (#4766) — a non-empty array means the job cannot
 * succeed no matter how much system RAM is free, and the caller must fail fast
 * with `gpuBlockersMessage(...)` rather than start a model load.
 *
 * Nothing is unloaded while blocked: evicting the user's resident LLM is a real
 * cost, and paying it for a run that is about to be refused helps nobody.
 */
export async function prepareLocalMemory() {
  const blockers = await detectGpuBlockers().catch(() => []);
  const unloaded = blockers.length ? [] : await unloadResidentModels().catch(() => []);
  const availableGb = await getAvailableMemoryGb().catch(() => 0);
  const totalGb = totalmem() / GB;
  return { unloaded, availableGb, totalGb, budgetGb: Math.min(totalGb, availableGb), blockers };
}