Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

6.9 KB
Raw Blame
/**
 * Importer Orphan-Shell GC (issue #727)
 *
 * `analyzeImport` creates a brand-new universe + series shell so the analyze
 * preview has stable ids the client can commit/retry against. When the user
 * abandons the import before committing (e.g. an issue-split failure on a
 * fresh universe/series), those shells linger on disk forever.
 *
 * The primary defense is marking analyze-created shells as import drafts
 * (`ephemeral` so they stay out of every sync transport, plus a dedicated
 * `importDraft` marker) and clearing both flags in `commitImport` once the
 * import lands. This sweep is the belt-and-suspenders second half: it deletes
 * shells that are STILL import drafts, hold no committed work (zero issues,
 * zero canon entities, no arc/seasons), and are older than a grace window —
 * so an abandoned analyze self-cleans instead of accreting orphan data.
 *
 * Conservative by construction:
 * - Only `importDraft` records are candidates. `ephemeral` alone is NOT enough
 *   — it's a general user-settable "don't sync / keep local" marker, so a
 *   user's deliberately-private empty universe/series is also `ephemeral` and
 *   must never be swept. `importDraft` is stamped ONLY by analyzeImport and
 *   cleared on commit, so it uniquely identifies an abandoned import shell.
 * - A universe is only swept once its ephemeral series are gone AND it has no
 *   OTHER live (non-deleted) series — so a shared universe is never removed as
 *   a side-effect. `deleteUniverse` also enforces this block-until-empty rule.
 * - The age gate is measured from `updatedAt` (the most recent touch), so a
 *   user mid-flight on a fresh analyze is never swept out from under them.
 *
 * Runs OUTSIDE the Express request lifecycle through eventScheduler, which
 * catches handler failures and records them in scheduler history.
 */

import { listUniverses, deleteUniverse } from './universeBuilder.js';
import { listSeries, deleteSeries } from './pipeline/series.js';
import { listAllIssues } from './pipeline/issues.js';
import { createSweepScheduler } from './sweepScheduler.js';

// Grace window before an abandoned, never-committed shell is eligible for GC.
// Measured against `updatedAt`. Generous on purpose — the cost of an orphan
// shell lingering an extra day is trivial next to deleting work a user is
// still actively shaping.
export const ORPHAN_SHELL_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days

// How often the sweep runs once started. The shells are cheap to leave around,
// so a slow cadence is fine — daily keeps `data/` tidy without churn.
const SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours

const ageMs = (record, now) => {
  const ts = Date.parse(record?.updatedAt || record?.createdAt || '');
  return Number.isFinite(ts) ? now - ts : NaN;
};

// A universe carries no committed canon when every entity registry is empty.
const universeHasCanon = (u) =>
  (Array.isArray(u?.characters) && u.characters.length > 0)
  || (Array.isArray(u?.places) && u.places.length > 0)
  || (Array.isArray(u?.objects) && u.objects.length > 0);

// A series carries committed story work when it has an arc or any seasons.
// (Issues are checked separately via listIssues since they live in their own
// store keyed by seriesId.)
const seriesHasStoryWork = (s) =>
  (s?.arc != null) || (Array.isArray(s?.seasons) && s.seasons.length > 0);

/**
 * One GC pass. Pure-ish: takes `now` so tests can pin the clock. Returns a
 * summary `{ deletedSeries, deletedUniverses }` of ids removed.
 *
 * Order matters: series are swept first because `deleteUniverse` refuses to
 * delete a universe that still has live series (hierarchy invariant). A
 * universe only becomes eligible once its own ephemeral shells are gone.
 */
export async function sweepOrphanShells({ now = Date.now(), maxAgeMs = ORPHAN_SHELL_MAX_AGE_MS } = {}) {
  const deletedSeries = [];
  const deletedUniverses = [];

  // --- Pass 1: ephemeral, empty, aged-out series ---
  const allSeries = await listSeries();
  // Track which series survive so the universe pass can tell whether a
  // universe still has live series — built from the in-memory list rather than
  // a second listSeries() after the deletes below, which would just re-read
  // the same data we already have in hand.
  const survivingSeriesByUniverse = new Map();
  const noteSurvivor = (s) => {
    const key = s.universeId || '';
    survivingSeriesByUniverse.set(key, (survivingSeriesByUniverse.get(key) || 0) + 1);
  };

  // Pre-load ALL issues once and group by seriesId to avoid an N+1 pattern
  // (one listIssues call per candidate series). listAllIssues is UNCAPPED —
  // listIssues({}) slices at 1000 total, and a series whose issues fall past
  // that cap would count as zero here and get DELETED while it still has
  // live issues.
  const allIssues = await listAllIssues();
  const issueCountBySeries = new Map();
  for (const issue of allIssues) {
    if (!issue.seriesId) continue;
    issueCountBySeries.set(issue.seriesId, (issueCountBySeries.get(issue.seriesId) || 0) + 1);
  }

  // True only for an import-draft, story-work-free, aged-out, zero-issue shell.
  // Each gate runs cheapest-first; the issue count lookup is now O(1) from the map.
  const seriesIsSweepable = (s) => {
    if (s.importDraft !== true || seriesHasStoryWork(s)) return false;
    const age = ageMs(s, now);
    if (!Number.isFinite(age) || age < maxAgeMs) return false;
    return (issueCountBySeries.get(s.id) || 0) === 0;
  };

  for (const s of allSeries) {
    if (seriesIsSweepable(s)) {
      await deleteSeries(s.id);
      deletedSeries.push(s.id);
    } else {
      noteSurvivor(s);
    }
  }

  // --- Pass 2: import-draft, empty, aged-out universes with no surviving series ---
  const allUniverses = await listUniverses();
  for (const u of allUniverses) {
    if (u.importDraft !== true) continue;
    if (universeHasCanon(u)) continue;
    const age = ageMs(u, now);
    if (!Number.isFinite(age) || age < maxAgeMs) continue;
    // A universe with any surviving (live, non-deleted) series — whether we
    // just spared an ephemeral one above or a committed one shares it — must
    // not be removed. deleteUniverse enforces this too, but checking here
    // keeps us from a noisy throw on the expected case.
    if ((survivingSeriesByUniverse.get(u.id) || 0) > 0) continue;
    await deleteUniverse(u.id);
    deletedUniverses.push(u.id);
  }

  return { deletedSeries, deletedUniverses };
}

/**
 * Log the domain summary for one scheduler-owned sweep.
 */
const runSweep = async () => {
  const { deletedSeries, deletedUniverses } = await sweepOrphanShells();
  if (deletedSeries.length > 0 || deletedUniverses.length > 0) {
    console.log(`🧹 Importer orphan GC: removed ${deletedSeries.length} series + ${deletedUniverses.length} universe shells`);
  }
};

export const {
  start: startOrphanShellGc,
  stop: stopOrphanShellGc,
} = createSweepScheduler({
  id: 'importer-orphan-gc',
  intervalMs: SWEEP_INTERVAL_MS,
  initialDelayMs: 5 * 60 * 1000,
  handler: runSweep,
  source: 'importerOrphanGc',
});