Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

8.1 KB
Raw Blame
/**
 * Live-work probes for Data Manager category purges (issue #3342).
 *
 * A category-wide purge empties the whole directory in one action. Three
 * categories are correctly classified as reproducible scratch — the bytes do
 * come back — but only once nothing is USING them:
 *
 *   image-clean-tmp — the init/mask/original inputs a queued or running Image
 *     Cleaner render is about to read.
 *   training-runs   — the checkpoints, caches, and sample previews a LoRA
 *     trainer is writing right now (hours of GPU time).
 *   update-detached — the control files the detached self-update script is
 *     being polled through.
 *
 * Each probe resolves `{ busy, reason }`. `reason` is user-facing copy: it is
 * both the 409 `CATEGORY_BUSY` body and the text the Data Manager row shows in
 * place of the Purge button, so it must name what is running and imply when to
 * retry. Every probe takes its state as an optional argument so it is testable
 * without standing up the real queue, a real detached process, or the real
 * `data/` tree.
 *
 * Directories are joined off `PATHS.data` inside each probe (the category key
 * IS the directory name) rather than read from the `PATHS.imageCleanTmp` /
 * `PATHS.trainingRuns` aliases — those are captured at module load, so a test
 * that redirects the data root would not be honored by them.
 */

import { readdir } from 'fs/promises';
import { join } from 'path';
import { PATHS } from '../lib/fileUtils.js';
import { isDetachedRunning } from '../lib/detachedSpawn.js';
import { listJobs } from './mediaJobQueue/index.js';
import { collectActiveCleanBasenames } from './imageCleanTmpGc.js';
import { collectActiveFederatedAssetBasenames } from './federatedMedia/assetStore.js';

const IDLE = { busy: false, reason: null };

// Jobs the media queue still owes work to. A terminal job (completed / failed /
// canceled) has no claim on anything under `data/`.
const IN_FLIGHT_STATUSES = new Set(['queued', 'running']);

// A directory that isn't there has nothing left to lose, so ENOENT reads as
// empty. Any OTHER failure means we could not look — rethrow so
// `resolveCategoryBusy` fails closed rather than reporting a locked or
// unreadable directory as idle and clearing the way for the purge.
const listDirEntries = (dir) => readdir(dir, { withFileTypes: true }).catch((err) => {
  if (err?.code === 'ENOENT') return [];
  throw err;
});

/**
 * `data/image-clean-tmp` — the pinned set comes from the SAME predicate the
 * hourly GC sweep uses to spare an active job's files
 * (`collectActiveCleanBasenames`), so a one-click purge can never be more
 * permissive than the automatic sweep that deliberately leaves them alone.
 *
 * That predicate is intentionally broad: it pins `<jobId>-{mask,original}.png`
 * and friends for EVERY in-flight image job, because a job's clean side files
 * are keyed by job id. Left at that, an unrelated gallery render would report
 * the scratch dir busy. So the pinned names are intersected with what is
 * actually in the directory — the purge is only refused when a live job's file
 * is really sitting there to be destroyed.
 *
 * @param {{ jobs?: Array, entries?: string[] }} [state] - injected job list / directory listing
 * @returns {Promise<{ busy: boolean, reason: string|null }>}
 */
export async function imageCleanTmpBusy({ jobs = null, entries = null } = {}) {
  const pinned = collectActiveCleanBasenames(jobs || listJobs({ kind: 'image' }));
  if (pinned.size === 0) return IDLE;
  const present = entries
    || (await listDirEntries(join(PATHS.data, 'image-clean-tmp'))).filter(e => e.isFile()).map(e => e.name);
  const atRisk = present.filter(name => pinned.has(name));
  if (atRisk.length === 0) return IDLE;
  return {
    busy: true,
    reason: `An image job is queued or running and ${atRisk.length} working file(s) here belong to it — purge once it finishes.`
  };
}

/**
 * `data/federated-media-inbox` — conditioning images an allowlisted peer
 * uploaded for a federated render (#4348). Structurally the same case as
 * `image-clean-tmp`: ephemeral image inputs a queued or running media job is
 * about to read, reached through the same `initImagePath`-style params.
 *
 * The pinned set comes from the SAME predicate the TTL sweep uses
 * (`collectActiveFederatedAssetBasenames`), so a one-click purge can never be
 * more permissive than the automatic sweep that deliberately leaves them alone.
 * Destroying one of these is worse than the local case: the consumer already
 * committed and is waiting, and nothing re-uploads — the asset ids were resolved
 * at admission, and this side never tells the peer the bytes went away.
 *
 * Not scoped to one kind: image and video jobs both stage conditioning here.
 *
 * @param {{ jobs?: Array, entries?: string[] }} [state] - injected job list / directory listing
 * @returns {Promise<{ busy: boolean, reason: string|null }>}
 */
export async function federatedMediaInboxBusy({ jobs = null, entries = null } = {}) {
  const pinned = collectActiveFederatedAssetBasenames(jobs || listJobs());
  if (pinned.size === 0) return IDLE;
  const present = entries
    || (await listDirEntries(join(PATHS.data, 'federated-media-inbox'))).filter(e => e.isFile()).map(e => e.name);
  const atRisk = present.filter(name => pinned.has(name));
  if (atRisk.length === 0) return IDLE;
  return {
    busy: true,
    reason: `A federated render is queued or running and ${atRisk.length} source image(s) here belong to it — purge once it finishes.`
  };
}

/**
 * `data/training-runs`. Two sources, because neither alone covers a restart:
 *
 *   1. The media queue's GPU lane, where training runs as `kind: 'training'` —
 *      the live registry the cancel path and the boot reconcile both key on,
 *      and cheaper than a Postgres round-trip on every `GET /api/data`.
 *   2. Each run's `.detached` control dir. A trainer is a detached child that
 *      SURVIVES a pm2 restart, so between accepting requests and
 *      `initLoraTraining()` finishing its reconcile there is a window where a
 *      live trainer has no queue job. Asking the control dir answers "is a
 *      process writing here right now" independent of any in-memory state.
 *
 * @param {{ jobs?: Array, runsDir?: string }} [state] - injected job list / runs root
 * @returns {Promise<{ busy: boolean, reason: string|null }>}
 */
export async function trainingRunsBusy({ jobs = null, runsDir = null } = {}) {
  const all = jobs || listJobs({ kind: 'training' });
  const active = (Array.isArray(all) ? all : []).filter((job) => IN_FLIGHT_STATUSES.has(job?.status));
  if (active.length > 0) {
    return {
      busy: true,
      reason: `${active.length} LoRA training run(s) queued or running — purging now would delete checkpoints out from under a live trainer. Purge once training finishes.`
    };
  }

  const dir = runsDir || join(PATHS.data, 'training-runs');
  const runs = (await listDirEntries(dir)).filter(e => e.isDirectory()).map(e => e.name);
  // Not caught: a probe that cannot answer must reach `resolveCategoryBusy` and
  // fail closed, not read as "no trainer here".
  const surviving = await Promise.all(
    runs.map(name => isDetachedRunning(join(dir, name, '.detached')))
  );
  if (!surviving.some(Boolean)) return IDLE;
  return {
    busy: true,
    reason: 'A LoRA trainer that outlived a restart is still writing here — purge once it finishes or is cancelled.'
  };
}

/**
 * `data/update-detached` — the same `isDetachedRunning` probe `updateExecutor.js`
 * uses to refuse a second concurrent update. The directory name is duplicated
 * from that module's `controlDir`; keep the two in step.
 *
 * @param {{ controlDir?: string }} [state] - injected control dir; defaults to the real one
 * @returns {Promise<{ busy: boolean, reason: string|null }>}
 */
export async function updateDetachedBusy({ controlDir = null } = {}) {
  const dir = controlDir || join(PATHS.data, 'update-detached');
  const running = await isDetachedRunning(dir);
  if (!running) return IDLE;
  return {
    busy: true,
    reason: 'A self-update is running and PortOS polls its control files here — purging now loses the handle on it. Purge once the update finishes.'
  };
}