Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

19.9 KB
Raw Blame
/**
 * Storyboard panel maker — runs a board's recipe nodes for one panel.
 *
 * An LLM node (prompt writer / enhancer) runs inline: its input prompt and the
 * system prompt wired into it are sent as one visible text, and the answer
 * lands on the panel as that node's editable output. An image node queues one
 * media job per frame; storyboardPanelImageHook.js turns each finished frame
 * into a panel take (selection is never changed by a new take). Runs happen
 * only on the user's tap — nothing here is called at boot or in the background.
 *
 * Run state lives on the panel (`panel.make.nodes[nodeId]`) and is written
 * through `mutateStoryboard`, so a run racing a panel edit merges against the
 * freshest record. A run id guards each LLM result and each frame's
 * bookkeeping, so a slow earlier run can't overwrite a newer one.
 */

import { randomUUID } from 'node:crypto';
import { ServerError } from '../../lib/errorHandler.js';
import { resolveImageInputPath } from '../../lib/fileUtils.js';
import { promptAddition } from '../../lib/modelInput.js';
import { CREATIVE_LATITUDE_ADDITION, withCreativeLatitude } from '../../lib/creativeLatitude.js';
import {
  effectiveRecipe, nodeInputValue, panelNodeState, RECIPE_LIMITS, recipeRunOrder, sanitizeRecipe, STORYBOARD_RENDER_SIZES,
  upstreamNode,
} from '../../lib/storyboardRecipe.js';
import { newStoryboardEntityId, STORYBOARD_LIMITS } from '../../lib/storyboardVocabulary.js';
import { getStoryboard, mutateStoryboard } from './index.js';

export const STORYBOARD_PANEL_RENDER_TAG = 'storyboardPanelRender';

// Overridable for tests (provider runner, queue, settings and socket are process singletons).
const deps = {
  resolveProvider: async (args) => (await import('../promptRunner.js')).resolveProviderAndModel(args),
  runPrompt: async (args) => (await import('../promptRunner.js')).runPromptThroughProvider(args),
  getSettings: async () => (await import('../settings.js')).getSettings(),
  imageParams: async (settings, route, common) => (await import('../imageGen/queueableParams.js')).queueableImageParams(
    settings, route, common, { what: 'storyboard panels', code: 'STORYBOARD_RENDER_UNAVAILABLE' },
  ),
  enqueue: async (job) => (await import('../mediaJobQueue/index.js')).enqueueJob(job),
  linkedMedia: async (ids) => (await import('../catalogDB/media.js')).listMediaForIngredients(ids),
  ingredient: async (id) => (await import('../catalogDB/ingredients.js')).getIngredient(id),
  emit: (payload) => import('../socket.js')
    .then(({ getIo }) => getIo()?.emit('storyboard:make', payload))
    .catch((err) => console.error(`❌ storyboard:make emit failed: ${err.message}`)),
  emitBatch: (payload) => import('../socket.js')
    .then(({ getIo }) => getIo()?.emit('storyboard:make-batch', payload))
    .catch((err) => console.error(`❌ storyboard:make-batch emit failed: ${err.message}`)),
};
export const __setDepsForTests = (overrides) => Object.assign(deps, overrides);

const notFound = (what) => new ServerError(`${what} not found`, { status: 404, code: 'NOT_FOUND' });
const refuse = (message, code = 'STORYBOARD_MAKE_REFUSED', status = 400) => new ServerError(message, { status, code });

function locate(board, panelId) {
  for (const beat of board.beats) {
    const panel = beat.panels.find((p) => p.id === panelId);
    if (panel) return { beat, panel };
  }
  throw notFound('Panel');
}

function recipeNode(board, nodeId) {
  const recipe = effectiveRecipe(board);
  const node = recipe.nodes.find((n) => n.id === nodeId);
  if (!node) throw notFound('Recipe node');
  return { recipe, node };
}

/** Write one node's state on a panel; `patch` may be a function of the current state. */
function patchNodeState(draft, panelId, nodeId, patch) {
  const { panel } = locate(draft, panelId);
  const current = panelNodeState(panel, nodeId);
  const next = typeof patch === 'function' ? patch(current, panel) : { ...current, ...patch };
  if (!next) return false;
  panel.make = { nodes: { ...(panel.make?.nodes || {}), [nodeId]: next } };
  return true;
}

function announce(storyboardId, panelId, nodeId, status, error = '') {
  deps.emit({ storyboardId, panelId, nodeId, status, ...(error ? { error } : {}) });
}

/**
 * The panel's linked catalog cast, environments and objects as reference
 * images: each record's newest portrait, else its newest reference image.
 * A catalog that can't be read yields none (the run still has its own refs).
 */
async function resolveLinkedRefs(panel) {
  const ids = [...panel.castIds, ...panel.environmentIds, ...panel.objectIds];
  if (!ids.length) return [];
  const media = await deps.linkedMedia(ids).catch((err) => {
    console.warn(`⚠️ Storyboard references unavailable: ${err.message}`);
    return new Map();
  });
  const firsts = ids
    .map((id) => ({ id, row: (media.get(id) || []).find((m) => (m.kind === 'portrait' || m.kind === 'reference') && /\.(png|jpe?g|webp)$/i.test(m.mediaKey || '')) }))
    .filter(({ row }) => row);
  return Promise.all(firsts.map(async ({ id, row }) => {
    const record = await deps.ingredient(id).catch(() => null);
    return { refId: id, image: row.mediaKey, label: record?.name || '' };
  }));
}

/** What the Make tab shows beside the board record: the panel's linked reference images. */
export async function getPanelMake(storyboardId, panelId) {
  const board = await getStoryboard(storyboardId);
  const { panel } = locate(board, panelId);
  return { linkedRefs: await resolveLinkedRefs(panel) };
}

/** Replace a board's recipe; null restores the default for its art style. */
export async function updateRecipe(storyboardId, recipe) {
  const { board } = await mutateStoryboard(storyboardId, (draft) => {
    draft.recipe = recipe === null ? null : sanitizeRecipe(recipe);
  });
  return board;
}

const NODE_STATE_FIELDS = ['text', 'excludedRefs', 'extraRefs'];

/** Edit a node's per-panel values: an LLM output, removed or added references. */
export async function updatePanelNode(storyboardId, panelId, nodeId, patch) {
  const fields = Object.fromEntries(NODE_STATE_FIELDS.filter((k) => Object.hasOwn(patch, k)).map((k) => [k, patch[k]]));
  const { board } = await mutateStoryboard(storyboardId, (draft) => {
    recipeNode(draft, nodeId);
    patchNodeState(draft, panelId, nodeId, fields);
  });
  return board;
}

/** Run a node for one panel. Resolves to the board with the node's new state. */
export async function runPanelNode(storyboardId, panelId, nodeId) {
  const board = await getStoryboard(storyboardId);
  const { node } = recipeNode(board, nodeId);
  if (node.type === 'llm') return runLlmNode(board, panelId, node);
  if (node.type === 'image') return runImageNode(board, panelId, node);
  throw refuse(`A ${node.title} node has nothing to run; it passes its value along.`);
}

function inputContext(board, panelId, linkedRefs = []) {
  const { beat, panel } = locate(board, panelId);
  return { beat, panel, linkedRefs };
}

async function runLlmNode(board, panelId, node) {
  const recipe = effectiveRecipe(board);
  const context = inputContext(board, panelId);
  const prompt = nodeInputValue(recipe, node.id, 'prompt', context).trim();
  if (!prompt) throw refuse(`${node.title} has no prompt: write the shot in the panel, or wire a text node into its Prompt.`);
  const system = nodeInputValue(recipe, node.id, 'system', context).trim();
  const systemNode = upstreamNode(recipe, node.id, 'system');

  const { provider, selectedModel } = await deps.resolveProvider({ providerId: node.settings.providerId || null, model: node.settings.model || null });
  if (!provider) throw refuse('No AI provider is set up; pick one in the node settings.', 'NO_PROVIDER', 503);
  if (provider.enabled === false) throw refuse(`Provider "${provider.name || provider.id}" is disabled; enable it in Settings › Providers first.`, 'PROVIDER_DISABLED');

  // The system prompt leads the one text the model reads; both blocks are
  // listed beside the user's words so Show full prompt names each.
  const sentPrompt = withCreativeLatitude(system ? `${system}\n\n---\n\n${prompt}` : prompt);
  const promptAdditions = [
    CREATIVE_LATITUDE_ADDITION,
    system ? promptAddition('system-prompt', `System prompt (${systemNode?.title || 'node'})`, system) : null,
  ].filter(Boolean);
  const runId = randomUUID();
  await mutateStoryboard(board.id, (draft) => patchNodeState(draft, panelId, node.id, {
    status: 'running', error: '', runId, sentPrompt, promptAdditions, providerId: provider.id, model: selectedModel || '',
  }));
  announce(board.id, panelId, node.id, 'running');

  let text = '';
  let error = '';
  try {
    const result = await deps.runPrompt({
      provider, model: selectedModel, effort: node.settings.effort || undefined, prompt: sentPrompt, source: 'storyboard-panel-prompt',
    });
    text = String(result?.text || '').trim();
    if (!text) error = 'The model answered with nothing.';
  } catch (err) {
    error = err?.message || 'The prompt run failed';
  }
  const { board: next } = await mutateStoryboard(board.id, (draft) => patchNodeState(draft, panelId, node.id, (current) => (
    current.runId === runId
      ? { ...current, status: error ? 'failed' : 'done', error: error.slice(0, RECIPE_LIMITS.error), ...(error ? {} : { text: text.slice(0, RECIPE_LIMITS.text) }), runAt: new Date().toISOString() }
      : null
  )));
  announce(board.id, panelId, node.id, error ? 'failed' : 'done', error);
  if (error) console.log(`🚫 Storyboard ${board.id.slice(0, 11)} ${node.title} run failed: ${error}`);
  return next;
}

async function runImageNode(board, panelId, node) {
  const recipe = effectiveRecipe(board);
  const { panel } = locate(board, panelId);
  const context = inputContext(board, panelId, await resolveLinkedRefs(panel));
  const prompt = nodeInputValue(recipe, node.id, 'prompt', context).trim();
  if (!prompt) throw refuse(`${node.title} has no prompt: run the prompt writer first, or wire a text node into its Prompt.`);
  if (prompt.length > RECIPE_LIMITS.imagePrompt) {
    throw refuse(`The prompt is ${prompt.length.toLocaleString()} characters; image models take at most ${RECIPE_LIMITS.imagePrompt.toLocaleString()}. Shorten it in the node it comes from.`, 'PROMPT_TOO_LONG');
  }
  const count = node.settings.count;
  if (panel.takes.length + count > STORYBOARD_LIMITS.takes) {
    throw refuse(`This panel keeps at most ${STORYBOARD_LIMITS.takes} takes and has ${panel.takes.length}; remove some before making ${count} more.`, 'STORYBOARD_LIMIT', 409);
  }
  const refs = nodeInputValue(recipe, node.id, 'images', context);
  const referenceImagePaths = refs.map((ref) => resolveImageInputPath(ref.image)).filter(Boolean);
  const runId = randomUUID();
  const settings = await deps.getSettings();
  const common = {
    prompt,
    ...STORYBOARD_RENDER_SIZES[board.aspectRatio],
    ...(referenceImagePaths.length ? { referenceImagePaths, referenceImageStrengths: referenceImagePaths.map(() => 1) } : {}),
    [STORYBOARD_PANEL_RENDER_TAG]: { storyboardId: board.id, panelId, nodeId: node.id, runId },
  };
  const params = await deps.imageParams(settings, { mode: node.settings.imageMode, model: node.settings.imageModelId }, common);
  // The run id lands before any job is queued, so a frame that finishes or
  // fails at once is already counted against this run.
  await mutateStoryboard(board.id, (draft) => patchNodeState(draft, panelId, node.id, {
    status: 'queued', error: '', runId, frameCount: count, jobIds: [], failedJobIds: [], takeIds: [], sentPrompt: prompt,
    sentRefs: refs, promptAdditions: [], providerId: node.settings.imageMode || '', model: node.settings.imageModelId || '',
    runAt: new Date().toISOString(),
  }));
  const jobIds = [];
  let enqueueError = null;
  for (let i = 0; i < count && !enqueueError; i += 1) {
    const queued = await deps.enqueue({ kind: 'image', params, owner: `storyboard:${board.id}` }).catch((err) => { enqueueError = err; return null; });
    if (queued?.jobId) jobIds.push(queued.jobId);
  }
  if (enqueueError) console.log(`🚫 Storyboard ${board.id.slice(0, 11)} queued ${jobIds.length} of ${count} frames: ${enqueueError.message}`);
  const { board: next } = await mutateStoryboard(board.id, (draft) => patchNodeState(draft, panelId, node.id, (current) => {
    if (current.runId !== runId) return null;
    const recorded = { ...current, jobIds, frameCount: jobIds.length, ...(enqueueError ? { error: String(enqueueError.message).slice(0, RECIPE_LIMITS.error) } : {}) };
    return { ...recorded, status: settledStatus(recorded) };
  }));
  if (!jobIds.length) {
    announce(board.id, panelId, node.id, 'failed', enqueueError?.message || 'No frame was queued');
    throw enqueueError || refuse('No frame was queued.');
  }
  announce(board.id, panelId, node.id, 'queued');
  console.log(`🖼️ Storyboard ${board.id.slice(0, 11)} panel ${panelId.slice(0, 12)} → ${jobIds.length} image job(s)`);
  return next;
}

const settledStatus = (state) => (state.takeIds.length + state.failedJobIds.length >= state.frameCount
  ? (state.takeIds.length ? 'done' : 'failed')
  : 'queued');

/** Completion: the frame becomes a panel take (selection unchanged). Null when the panel is gone. */
export async function attachPanelFrame({ storyboardId, panelId, nodeId, runId, jobId, filename }) {
  let takeId = null;
  const { board } = await mutateStoryboard(storyboardId, (draft) => {
    const { panel } = locate(draft, panelId);
    // A full panel (a manual take or another run filled it meanwhile) drops
    // the frame, counted as failed so the node still settles.
    if (panel.takes.length >= STORYBOARD_LIMITS.takes) {
      patchNodeState(draft, panelId, nodeId, (current) => countFailed(current, runId, jobId, `The panel is full (${STORYBOARD_LIMITS.takes} takes); a frame was dropped`));
      return;
    }
    takeId = newStoryboardEntityId('take');
    panel.takes.push({ id: takeId, source: 'generated', image: filename, addedAt: new Date().toISOString() });
    patchNodeState(draft, panelId, nodeId, (current) => {
      if (current.runId !== runId) return null;
      const next = { ...current, takeIds: [...current.takeIds, takeId] };
      return { ...next, status: settledStatus(next) };
    });
  }).catch((err) => {
    if (err?.code === 'NOT_FOUND') return { board: null };
    throw err;
  });
  if (!board) return null;
  announce(storyboardId, panelId, nodeId, takeId ? 'frame' : 'failed');
  return takeId ? { takeId } : null;
}

function countFailed(current, runId, jobId, reason) {
  if (current.runId !== runId || current.failedJobIds.includes(jobId)) return null;
  const next = { ...current, failedJobIds: [...current.failedJobIds, jobId], error: String(reason).slice(0, RECIPE_LIMITS.error) };
  return { ...next, status: settledStatus(next) };
}

/** A failed or canceled frame is counted, and the node settles when its last frame does. */
export async function failPanelFrame({ storyboardId, panelId, nodeId, runId, jobId }, status, error) {
  const reason = status === 'canceled' ? 'A frame was canceled' : (error || 'A frame failed');
  await mutateStoryboard(storyboardId, (draft) => patchNodeState(draft, panelId, nodeId, (current) => countFailed(current, runId, jobId, reason)))
    .catch((err) => {
      if (err?.code !== 'NOT_FOUND') throw err;
    });
  announce(storyboardId, panelId, nodeId, 'failed', reason);
}

// Make empty panels: one batch per board at a time, in memory only (a restart
// ends it; nothing resumes on its own at boot).
const batches = new Map();
const RUNNABLE_TYPES = new Set(['llm', 'image']);
const IN_FLIGHT = new Set(['running', 'queued']);

const runnableSteps = (board) => recipeRunOrder(effectiveRecipe(board)).filter((node) => RUNNABLE_TYPES.has(node.type));
// A panel with no art and no frames on the way.
const isEmptyPanel = (panel, steps) => !panel.takes.length && !steps.some((node) => IN_FLIGHT.has(panelNodeState(panel, node.id).status));
const emptyPanelIds = (board, steps) => board.beats.flatMap((beat) => beat.panels).filter((panel) => isEmptyPanel(panel, steps)).map((p) => p.id);
const batchView = (storyboardId, batch) => (batch ? { storyboardId, ...batch } : null);

async function stepLabel(node) {
  if (node.type === 'image') {
    return { nodeId: node.id, title: node.title, type: node.type, model: node.settings.imageModelId || 'Default image model', frames: node.settings.count };
  }
  const { provider, selectedModel } = await deps.resolveProvider({ providerId: node.settings.providerId || null, model: node.settings.model || null })
    .catch(() => ({ provider: null }));
  const model = provider ? [provider.name || provider.id, selectedModel].filter(Boolean).join(' · ') : 'No AI provider set up';
  return { nodeId: node.id, title: node.title, type: node.type, model, frames: 0 };
}

/**
 * What "Make empty panels" would do, for the confirm that names it: how many
 * panels have no art, and each node it runs with the model it uses.
 */
export async function getMakeEmptyPlan(storyboardId) {
  const board = await getStoryboard(storyboardId);
  const steps = runnableSteps(board);
  return {
    batch: batchView(storyboardId, batches.get(storyboardId)),
    panelCount: emptyPanelIds(board, steps).length,
    steps: await Promise.all(steps.map(stepLabel)),
  };
}

/**
 * Run the recipe on every panel with no art, one panel after another. A
 * prompt already written for a panel is kept (it may be the user's edit); an
 * LLM failure skips that panel's image so no frame is made from no prompt.
 * Answers at once; `storyboard:make-batch` reports progress.
 */
export async function startMakeEmpty(storyboardId) {
  if (batches.has(storyboardId)) throw refuse('This board is already making its empty panels.', 'STORYBOARD_BATCH_RUNNING', 409);
  const board = await getStoryboard(storyboardId);
  const steps = runnableSteps(board);
  if (!steps.some((node) => node.type === 'image')) throw refuse('The recipe has no Image node, so there is nothing to make.');
  const panelIds = emptyPanelIds(board, steps);
  if (!panelIds.length) throw refuse('Every panel already has art.');
  // Re-checked after the read: two quick taps (or two tabs) both passed the first check.
  if (batches.has(storyboardId)) throw refuse('This board is already making its empty panels.', 'STORYBOARD_BATCH_RUNNING', 409);
  const batch = { status: 'running', total: panelIds.length, done: 0, failed: 0, startedAt: new Date().toISOString() };
  batches.set(storyboardId, batch);
  deps.emitBatch(batchView(storyboardId, batch));
  console.log(`🎬 Storyboard ${storyboardId.slice(0, 11)} making ${panelIds.length} empty panel(s)`);
  runBatch(storyboardId, panelIds, batch)
    .catch((err) => console.error(`❌ Storyboard ${storyboardId.slice(0, 11)} empty-panel batch stopped: ${err.message}`))
    .finally(() => {
      batches.delete(storyboardId);
      deps.emitBatch(batchView(storyboardId, { ...batch, status: 'done' }));
    });
  return { batch: batchView(storyboardId, batch) };
}

async function runBatch(storyboardId, panelIds, batch) {
  for (const panelId of panelIds) {
    const ok = await makePanel(storyboardId, panelId);
    batch.done += 1;
    if (!ok) batch.failed += 1;
    deps.emitBatch(batchView(storyboardId, batch));
  }
}

/** One panel's run; false when it was skipped or a node failed. A deleted board ends the batch. */
async function makePanel(storyboardId, panelId) {
  const board = await getStoryboard(storyboardId);
  const steps = runnableSteps(board);
  const panel = board.beats.flatMap((beat) => beat.panels).find((p) => p.id === panelId);
  // Removed, or given art since the batch started.
  if (!panel || !isEmptyPanel(panel, steps)) return false;
  for (const node of steps) {
    if (node.type === 'llm' && panelNodeState(panel, node.id).text.trim()) continue;
    try {
      const next = await runPanelNode(storyboardId, panelId, node.id);
      const state = panelNodeState(locate(next, panelId).panel, node.id);
      if (state.status === 'failed') return false;
    } catch (err) {
      if (err?.code === 'NOT_FOUND' && err.message.startsWith('Storyboard')) throw err;
      console.log(`🚫 Storyboard ${storyboardId.slice(0, 11)} panel ${panelId.slice(0, 12)} ${node.title} skipped: ${err.message}`);
      return false;
    }
  }
  return true;
}