Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

24.4 KB
Raw Blame
import { withRetiredDataAccess } from '../lib/retiredLoraTraining.js';
/**
 * LoRA training datasets — collectionStore CRUD + image file management.
 *
 * One dataset per universe bible subject (find-or-create keyed on
 * `(universeId, entryKind, entryId)`), stored machine-locally at
 * `data/lora-datasets/<id>/index.json` with image bytes at
 * `data/lora-datasets/<id>/images/<imageId>.png`. Datasets never federate
 * — like `data/loras/` itself, they organize artifacts tied to this
 * machine's GPU output (see docs/STORAGE.md).
 *
 * Generation (batch renders via the media-job queue) lives in
 * `loraDatasetGenerate.js`; vision captioning in `loraDatasetCaption.js`;
 * pure helpers in `lib/loraDataset.js`.
 */

import { existsSync } from 'fs';
import { join, basename } from 'path';
import sharp from 'sharp';
import { PATHS, ensureDir, copyFileGuarded, unlinkGuarded } from '../lib/fileUtils.js';
import { assertGalleryFilename } from './imageGen/local.js';
import { createCollectionStore } from '../lib/collectionStore.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import {
  LORA_DATASET_SCHEMA_VERSION,
  LORA_DATASET_ENTRY_KINDS,
  analyzeCaptionInvariants,
  captionHasTriggerWord,
  computeDatasetReadiness,
  deriveTriggerWord,
  isValidTriggerWord,
  prefixCaption,
  sanitizeLoraDataset,
  stripSharedFragments,
} from '../lib/loraDataset.js';
import { ServerError } from '../lib/errorHandler.js';
import { v4 as uuidv4 } from '../lib/uuid.js';
import { getUniverse } from './universeBuilder.js';
import { getJob } from './mediaJobQueue/index.js';
import { trainingEvents } from './loraTraining/events.js';

export const loraDatasetStore = createCollectionStore({
  dir: PATHS.loraDatasets,
  type: 'lora-datasets',
  schemaVersion: LORA_DATASET_SCHEMA_VERSION,
  sanitizeRecord: sanitizeLoraDataset,
});

export const datasetImagesDir = (datasetId) => join(loraDatasetStore.recordDir(datasetId), 'images');
export const datasetImagePath = (datasetId, file) => join(datasetImagesDir(datasetId), file);

const nowIso = () => new Date().toISOString();
const normalizeEntryKind = (entryKind) => (
  LORA_DATASET_ENTRY_KINDS.includes(entryKind) ? entryKind : 'characters'
);
const subjectLabel = (entryKind) => {
  switch (normalizeEntryKind(entryKind)) {
    case 'objects': return 'object';
    case 'places': return 'place';
    default: return 'character';
  }
};

const requireDataset = async (id) => {
  const dataset = await loraDatasetStore.loadOne(id);
  if (!dataset) {
    throw new ServerError(`LoRA dataset not found: ${id}`, { status: 404, code: 'NOT_FOUND' });
  }
  return dataset;
};

/**
 * Read-modify-write a dataset inside its per-id write queue so concurrent
 * mutations (caption blur-save + render-completion hook) merge against the
 * freshest persisted record. `mutate` receives the sanitized record and
 * returns the next record (or null to abort without writing). Backup admission
 * begins before entering the queue; callers publishing bytes hold the outer
 * lease through this record commit.
 */
export async function updateDataset(id, mutate) {
  return withRetiredDataAccess(() => withBackupAssetPublication(() => loraDatasetStore.queueRecordWrite(id, async () => {
    const current = await loraDatasetStore.loadOne(id);
    if (!current) {
      throw new ServerError(`LoRA dataset not found: ${id}`, { status: 404, code: 'NOT_FOUND' });
    }
    const next = await mutate(current);
    if (!next) return current;
    next.updatedAt = nowIso();
    await loraDatasetStore.saveOneNow(id, next);
    trainingEvents.emit('dataset:changed', { datasetId: id });
    return next;
  })));
}

const summarize = (dataset) => ({
  id: dataset.id,
  character: dataset.character,
  triggerWord: dataset.triggerWord,
  status: dataset.status,
  readiness: computeDatasetReadiness(dataset),
  // First few image files for the list page's thumb strip.
  thumbnails: dataset.images
    .filter((img) => img.status === 'ready')
    .slice(0, 4)
    .map((img) => img.file),
  training: dataset.training,
  createdAt: dataset.createdAt,
  updatedAt: dataset.updatedAt,
});

export async function listDatasets({
  universeId = null, entryKind = null, entryId = null, ingredientId = null,
} = {}) {
  const normalizedKind = entryKind ? normalizeEntryKind(entryKind) : null;
  const all = await loraDatasetStore.loadAll();
  const filtered = all.filter((d) => {
    if (universeId && d.character.universeId !== universeId) return false;
    if (normalizedKind && normalizeEntryKind(d.character.entryKind) !== normalizedKind) return false;
    if (entryId && d.character.entryId !== entryId) return false;
    if (ingredientId && d.character.ingredientId !== ingredientId) return false;
    return true;
  });
  return filtered
    .map(summarize)
    .sort((a, b) => (b.updatedAt || '').localeCompare(a.updatedAt || ''));
}

export async function getDataset(id) {
  const dataset = await requireDataset(id);
  return { ...dataset, readiness: computeDatasetReadiness(dataset) };
}

/**
 * Resolve a universe bible subject and the identity snapshot a dataset stores
 * for it. Validates the subject exists in the universe (404 otherwise) and is
 * the single source of truth for the snapshot shape — both createDataset and
 * patchDataset's reassignment go through here so the stored `character` fields
 * can't drift between the two write paths. Returns `{ subject, snapshot }`
 * (the live canon entry plus the persisted shape).
 */
async function resolveDatasetSubjectSnapshot(universeId, entryId, entryKind = 'characters') {
  const normalizedKind = normalizeEntryKind(entryKind);
  const universe = await getUniverse(universeId);
  const subjects = Array.isArray(universe[normalizedKind]) ? universe[normalizedKind] : [];
  const subject = subjects.find((entry) => entry.id === entryId);
  if (!subject) {
    throw new ServerError(`${subjectLabel(normalizedKind)} ${entryId} not found in universe`, {
      status: 404, code: 'UNIVERSE_CANON_NOT_FOUND',
    });
  }
  return {
    universe,
    subject,
    snapshot: {
      entryId,
      entryKind: normalizedKind,
      ingredientId: subject.ingredientId || null,
      universeId,
      name: subject.name || subject.slugline || 'Unnamed',
    },
  };
}

/**
 * Find-or-create the dataset for one universe bible subject. Validates the
 * subject exists in the universe and snapshots its identity (entryId +
 * catalog ingredientId + name) into the record. Returns `{ dataset, created }`.
 */
export async function createDataset({
  universeId, entryId, entryKind = 'characters', triggerWord = null,
}) {
  const normalizedKind = normalizeEntryKind(entryKind);
  const { subject, snapshot } = await resolveDatasetSubjectSnapshot(universeId, entryId, normalizedKind);

  const existingAll = await loraDatasetStore.loadAll();
  const existing = existingAll.find(
    (d) => d.character.universeId === universeId
      && normalizeEntryKind(d.character.entryKind) === normalizedKind
      && d.character.entryId === entryId,
  );
  if (existing) return { dataset: { ...existing, readiness: computeDatasetReadiness(existing) }, created: false };

  if (triggerWord != null && !isValidTriggerWord(triggerWord)) {
    throw new ServerError('Trigger word must be 2-64 chars of [a-z0-9_]', {
      status: 400, code: 'VALIDATION_ERROR',
    });
  }
  const taken = existingAll.map((d) => d.triggerWord).filter(Boolean);
  const resolvedTrigger = triggerWord || deriveTriggerWord(snapshot.name, { taken });

  const id = uuidv4();
  const record = sanitizeLoraDataset({
    schemaVersion: LORA_DATASET_SCHEMA_VERSION,
    id,
    character: snapshot,
    triggerWord: resolvedTrigger,
    status: 'draft',
    images: [],
    training: {},
    createdAt: nowIso(),
    updatedAt: nowIso(),
  });
  await loraDatasetStore.saveOne(id, record);
  await ensureDir(datasetImagesDir(id));
  console.log(`🧬 Created LoRA dataset ${id} for ${subjectLabel(normalizedKind)} "${subject.name || subject.slugline || entryId}" (trigger=${resolvedTrigger})`);
  return { dataset: { ...record, readiness: computeDatasetReadiness(record) }, created: true };
}

/**
 * Patch a dataset's trigger word and/or reassign it to a different
 * universe bible subject. `universeId` + `entryId` must travel together — a
 * reassignment re-snapshots the subject identity (entryId, entryKind, universeId,
 * ingredientId, name) exactly the way createDataset does, and is refused
 * if it would collide with another dataset already keyed on that
 * (universeId, entryKind, entryId) tuple (the one-dataset-per-subject invariant).
 */
export async function patchDataset(id, {
  triggerWord, universeId, entryId, entryKind,
} = {}) {
  if (triggerWord !== undefined && !isValidTriggerWord(triggerWord)) {
    throw new ServerError('Trigger word must be 2-64 chars of [a-z0-9_]', {
      status: 400, code: 'VALIDATION_ERROR',
    });
  }

  // Reassignment is all-or-nothing: a half-specified target (universe but no
  // subject) can't resolve a subject snapshot, so reject it up front.
  const reassigning = universeId !== undefined || entryId !== undefined || entryKind !== undefined;
  let nextCharacter = null;
  if (reassigning) {
    if (!universeId || !entryId) {
      throw new ServerError('Reassignment requires both universeId and entryId', {
        status: 400, code: 'VALIDATION_ERROR',
      });
    }
    const normalizedKind = normalizeEntryKind(entryKind);
    // The subject validation and the collision scan are independent reads —
    // run them together. resolveDatasetSubjectSnapshot keeps the snapshot shape in
    // lockstep with createDataset.
    const [{ subject, snapshot }, all] = await Promise.all([
      resolveDatasetSubjectSnapshot(universeId, entryId, normalizedKind),
      loraDatasetStore.loadAll(),
    ]);
    // (The in-flight-run guard moved into the write lock below — a pre-write
    // status snapshot races a concurrent startTrainingRun stamping 'training'.)
    // Enforce the same find-or-create key createDataset uses: at most one
    // dataset per (universeId, entryKind, entryId). Reassigning onto a subject that
    // already owns a dataset would create a duplicate the list can't tell apart.
    const clash = all.find(
      (d) => d.id !== id
        && d.character.universeId === universeId
        && normalizeEntryKind(d.character.entryKind) === normalizedKind
        && d.character.entryId === entryId,
    );
    if (clash) {
      throw new ServerError(`A dataset already exists for "${subject.name || subject.slugline || entryId}" in that universe`, {
        status: 409, code: 'DATASET_EXISTS',
      });
    }
    nextCharacter = snapshot;
  }

  return updateDataset(id, (current) => {
    const triggerChanged = triggerWord !== undefined && triggerWord !== current.triggerWord;
    const characterChanged = nextCharacter && (
      nextCharacter.entryId !== current.character.entryId
      || nextCharacter.entryKind !== normalizeEntryKind(current.character.entryKind)
      || nextCharacter.universeId !== current.character.universeId
      || nextCharacter.ingredientId !== current.character.ingredientId
      || nextCharacter.name !== current.character.name);
    // Authoritative in-flight guard, under the per-record write lock: a
    // concurrent startTrainingRun may have stamped status:'training' since the
    // pre-write reads above. Reassigning then would move the dataset out from
    // under a queued run that captured its id + old character — and because
    // flipDatasetAfterRun skips character-mismatched runs, the dataset would be
    // left stuck in 'training'. Refuse here so the user cancels the run first.
    if (characterChanged && current.status === 'training') {
      throw new ServerError('Cannot reassign while a training run is in progress — cancel it first', {
        status: 409, code: 'DATASET_TRAINING',
      });
    }
    if (!triggerChanged && !characterChanged) return null;
    // Re-prefix every captioned image so the binding token follows a trigger
    // rename. Without this, computeDatasetReadiness (which gates `captioned`
    // on the caption containing the trigger word) silently drops every
    // previously-captioned image and any training binds the stale token.
    // Empty captions are left untouched — don't fabricate a caption.
    const prev = current.triggerWord;
    const images = triggerChanged
      ? current.images.map((img) => (img.caption
        ? { ...img, caption: prefixCaption(triggerWord, img.caption, { previousTriggerWord: prev }) }
        : img))
      : current.images;
    // A trained dataset's LoRA is registered against the OLD character
    // (its sidecar drives /loras/by-character + render auto-apply). Moving
    // the dataset to a new character must NOT carry the `trained` status and
    // `training.loraFilename` over — that would advertise the new character
    // as trained while the actual LoRA still resolves only for the old one.
    // Reset to the untrained baseline so the new character must retrain; the
    // old character keeps its registered LoRA.
    const trainingReset = characterChanged ? { status: 'draft', training: {} } : {};
    return {
      ...current,
      ...(characterChanged ? { character: nextCharacter } : {}),
      ...(triggerChanged ? { triggerWord } : {}),
      ...trainingReset,
      images,
    };
  });
}

export async function deleteDataset(id) {
  return withBackupAssetPublication(async () => {
    await requireDataset(id);
    // deleteOne removes the whole `lora-datasets/<id>/` subtree — record AND
    // the images/ sidecar dir go together.
    await loraDatasetStore.deleteOne(id);
    console.log(`🗑️ Deleted LoRA dataset ${id}`);
    return { ok: true, id };
  });
}

/** Canonical dataset-image entry. One source of truth for the shape so every
 *  image source (upload / gallery import / generation) stamps identical fields. */
const makeImageEntry = ({ imageId, file, source, info, sourceJobId = null }) => ({
  id: imageId,
  file,
  caption: '',
  captionSource: null,
  captionedAt: null,
  source,
  sourceJobId,
  variation: null,
  status: 'ready',
  width: info?.width || null,
  height: info?.height || null,
  createdAt: nowIso(),
});

/**
 * Normalize an uploaded image to PNG inside the dataset's images dir and
 * append its entry. `tmpPath` is the multipart parser's staged temp file —
 * always unlinked, success or failure. The route's mimetype `fileFilter` is
 * only an early reject of obvious non-images (the declared MIME is
 * client-controlled) — `sharp` below is the real gate: it transcodes any
 * format it can decode and 422s anything it can't, so a wrong-extension
 * upload is normalized rather than trusted by its claimed type.
 */
export async function addUploadedImage(id, { tmpPath, originalname = '' }) {
  return withBackupAssetPublication(async () => {
    await requireDataset(id);
    const imageId = uuidv4();
    const file = `${imageId}.png`;
    await ensureDir(datasetImagesDir(id));
    const destPath = datasetImagePath(id, file);
    const cleanup = () => unlinkGuarded(tmpPath).catch(() => {});
    const info = await sharp(tmpPath).rotate().png().toFile(destPath).catch(async (err) => {
      await cleanup();
      throw new ServerError(
        `"${originalname || 'upload'}" is not a decodable image: ${err?.message || err}`,
        { status: 422, code: 'INVALID_IMAGE' },
      );
    });
    await cleanup();

    const entry = makeImageEntry({ imageId, file, source: 'upload', info });
    await updateDataset(id, (current) => ({ ...current, images: [...current.images, entry] }));
    console.log(`📥 Dataset ${id} ← upload ${file} (${entry.width}×${entry.height})`);
    return entry;
  });
}

/**
 * Import existing gallery images (by basename, e.g. `<jobId>.png`) into a
 * dataset. Each is normalized through sharp into the dataset's images dir —
 * same gate as a fresh upload — so a gallery PNG with odd metadata can't smuggle
 * anything in, and the dataset owns an independent copy (deleting the gallery
 * image later won't strand the dataset). Each call appends a fresh copy (no
 * dedup — same as uploading the same file twice). Returns the appended entries.
 */
export async function importGalleryImages(id, { filenames = [] } = {}) {
  return withBackupAssetPublication(async () => {
    await requireDataset(id);
    if (!Array.isArray(filenames) || !filenames.length) {
      throw new ServerError('No gallery filenames provided', { status: 400, code: 'VALIDATION_ERROR' });
    }
    await ensureDir(datasetImagesDir(id));
    // sharp transcodes are independent CPU/disk work — run them concurrently
    // (single-user trust model; the per-id write queue still serializes the one
    // dataset mutation below). Use allSettled, NOT all: a rejection from all()
    // returns before sibling toFile() calls finish, so files written *after* the
    // first failure would escape cleanup and orphan. allSettled waits for every
    // transcode to land before we either commit them all or unlink them all.
    const written = [];
    const results = await Promise.allSettled(filenames.map(async (filename) => {
      // Reuse the gallery's own path-traversal/extension guard.
      assertGalleryFilename(filename);
      const sourcePath = join(PATHS.images, basename(filename));
      if (!existsSync(sourcePath)) {
        throw new ServerError(`Gallery image not found: ${filename}`, { status: 404, code: 'NOT_FOUND' });
      }
      const imageId = uuidv4();
      const file = `${imageId}.png`;
      const destPath = datasetImagePath(id, file);
      const info = await sharp(sourcePath).rotate().png().toFile(destPath)
        .then((i) => { written.push(destPath); return i; })
        .catch((err) => {
          throw new ServerError(
            `"${filename}" is not a decodable image: ${err?.message || err}`,
            { status: 422, code: 'INVALID_IMAGE' },
          );
        });
      return makeImageEntry({ imageId, file, source: 'gallery', info });
    }));
    const failure = results.find((r) => r.status === 'rejected');
    if (failure) {
      // All transcodes have settled now, so `written` is complete — no late write
      // can re-orphan a file after this cleanup.
      await Promise.all(written.map((p) => unlinkGuarded(p).catch(() => {})));
      throw failure.reason;
    }
    const entries = results.map((r) => r.value);
    await updateDataset(id, (current) => ({ ...current, images: [...current.images, ...entries] }));
    console.log(`🖼️ Dataset ${id} ← imported ${entries.length} gallery image(s)`);
    return entries;
  });
}

export async function updateImageCaption(id, imageId, caption) {
  const next = await updateDataset(id, (current) => {
    if (!current.images.some((img) => img.id === imageId)) {
      throw new ServerError(`Image ${imageId} not found in dataset`, { status: 404, code: 'NOT_FOUND' });
    }
    return {
      ...current,
      images: current.images.map((img) => (img.id === imageId
        ? { ...img, caption, captionSource: 'manual', captionedAt: nowIso() }
        : img)),
    };
  });
  return next.images.find((img) => img.id === imageId);
}

/**
 * Strip the invariant identity fragments shared across most captions (issue
 * #1320) — phrases repeated in nearly every caption bind the character to the
 * caption text instead of the trigger token. The analysis and the rewrite both
 * run inside the write lock against the SAME record, so the shared set is never
 * computed from a stale snapshot and applied to a newer one. Recomputes the
 * shared set server-side (never trusts a client-supplied fragment list) and
 * rewrites every captioned image, leaving each caption with the trigger word +
 * only what varies shot-to-shot. Returns `{ dataset, removedFragments,
 * updatedImages }`; a no-op (nothing shared, or too few captions to analyze)
 * leaves the record untouched (no write, no updatedAt bump) and returns an empty
 * `removedFragments`.
 */
export async function stripSharedCaptionFragments(id) {
  let removedFragments = [];
  let updatedImages = 0;
  const next = await updateDataset(id, (record) => {
    const { sharedFragments } = analyzeCaptionInvariants(record.images, record.triggerWord);
    removedFragments = sharedFragments;
    // Returning null skips the write entirely (see updateDataset) so a no-op
    // doesn't bump updatedAt.
    if (!sharedFragments.length) return null;
    const toStrip = sharedFragments.map((f) => f.fragment);
    return {
      ...record,
      images: record.images.map((img) => {
        // Only rewrite the captions the analysis actually looked at — ready AND
        // trigger-bearing. A ready caption that lacks the trigger was excluded
        // from analyzeCaptionInvariants; rewriting it would re-`prefixCaption`
        // the trigger onto it, silently making an unrelated caption count toward
        // readiness/the training manifest even when no shared fragment matched.
        if (img.status !== 'ready' || !captionHasTriggerWord(img.caption, record.triggerWord)) return img;
        const stripped = stripSharedFragments(img.caption, toStrip, record.triggerWord);
        if (stripped === img.caption) return img;
        updatedImages += 1;
        return { ...img, caption: stripped };
      }),
    };
  });
  if (removedFragments.length) {
    console.log(`🧹 Stripped ${removedFragments.length} shared caption fragment(s) across ${updatedImages} image(s) — dataset=${id}`);
  }
  return {
    dataset: { ...next, readiness: computeDatasetReadiness(next) },
    removedFragments,
    updatedImages,
  };
}

export async function deleteImage(id, imageId) {
  return withBackupAssetPublication(async () => {
    let removed = null;
    await updateDataset(id, (current) => {
      removed = current.images.find((img) => img.id === imageId) || null;
      if (!removed) {
        throw new ServerError(`Image ${imageId} not found in dataset`, { status: 404, code: 'NOT_FOUND' });
      }
      return { ...current, images: current.images.filter((img) => img.id !== imageId) };
    });
    await unlinkGuarded(datasetImagePath(id, removed.file)).catch(() => {});
    return { ok: true, imageId };
  });
}

/**
 * Read-time healer for images stuck in `rendering`. The generation hook
 * normally flips them on the media-job 'completed' event, but a server
 * restart drops that in-memory subscription (the queue marks interrupted
 * jobs failed on boot). For each rendering image, consult the job archive:
 * completed → copy the result into the dataset + mark ready; failed /
 * canceled / unknown → mark failed. Returns the (possibly updated) record.
 *
 * `jobLookup` is injectable for tests; defaults to the queue's getJob.
 */
export async function reconcileRenderingImages(id, { jobLookup = getJob } = {}) {
  return withBackupAssetPublication(async () => {
    const dataset = await requireDataset(id);
    const pending = dataset.images.filter((img) => img.status === 'rendering');
    if (!pending.length) return dataset;

    const resolutions = new Map(); // imageId → { status, sourceFilename? , dims? }
    for (const img of pending) {
      const job = img.sourceJobId ? jobLookup(img.sourceJobId) : null;
      if (!job || job.status === 'failed' || job.status === 'canceled') {
        resolutions.set(img.id, { status: 'failed' });
      } else if (job.status === 'completed' && job.result?.filename) {
        resolutions.set(img.id, { status: 'ready', sourceFilename: job.result.filename });
      }
      // queued/running jobs stay 'rendering' — the live hook will land them.
    }
    if (!resolutions.size) return dataset;

    for (const [imageId, res] of resolutions) {
      if (res.status !== 'ready') continue;
      const img = pending.find((p) => p.id === imageId);
      await ensureDir(datasetImagesDir(id));
      // basename() so a hand-edited media-jobs.json filename can't traverse
      // out of the gallery (mirrors onRenderComplete in loraDatasetGenerate).
      await copyFileGuarded(join(PATHS.images, basename(res.sourceFilename)), datasetImagePath(id, img.file))
        .catch((err) => {
          console.error(`❌ Dataset ${id} reconcile copy failed [${imageId}]: ${err?.message}`);
          resolutions.set(imageId, { status: 'failed' });
        });
    }

    const updated = await updateDataset(id, (current) => ({
      ...current,
      images: current.images.map((img) => {
        const res = resolutions.get(img.id);
        return res ? { ...img, status: res.status } : img;
      }),
    }));
    const resolved = [...resolutions.values()];
    const readyCount = resolved.filter((r) => r.status === 'ready').length;
    console.log(`🩹 Dataset ${id} reconciled ${resolved.length} rendering image(s) → ${readyCount} ready, ${resolved.length - readyCount} failed`);
    return updated;
  });
}