Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
18.8 KB
Raw Blame
import { describeDiagnosticError } from "./diagnostics";
import { releaseRendered } from "./generatedVideo";
import { isTauri } from "./persistence";
import type { ProjectSession } from "./projectSession";
import { referenceImages, type ProjectReference } from "./project";
import { needsReferenceIcon, referenceIconPrompt, REFERENCE_ICON_RENDER_SIZE, REFERENCE_ICON_STEPS } from "./referenceIcons";
import { cancelSlopfabGeneration, enqueueSlopfabGeneration, saveReferenceIcon } from "./runtime";
import { engineProviderSetting, generationStepsWithLoras } from "./settings";
import { activeReferenceRefmods, referenceRefmodInputs } from "./project";
import { loadReferenceIconAutomation, referenceIconGenerator, saveReferenceIconAutomation, subscribeReferenceIconAutomation } from "./referenceIconSettings";
import { projectQueueKey, type WorkItem } from "./workQueue";
import { hasPresetIcon, REFERENCE_PRESETS, REFERENCE_TYPES, type ReferencePreset } from "./reference-presets";
import { saveBuiltinIcon } from "./builtinReferenceIcons";

type TaskStatus = "queued" | "generating" | "saving" | "completed" | "failed" | "cancelled" | "skipped";
interface IconTask {
  key: string;
  session: ProjectSession;
  preset?: ReferencePreset;
  replaceIcon?: boolean;
  yielded?: boolean;
  referenceId: string;
  name: string;
  prompt: string;
  status: TaskStatus;
  nativeId?: string;
  submitted: boolean;
  refmodsSnapshot?: string;
  cancelled: boolean;
  force?: boolean;
  approved?: boolean;
  error?: string;
  needsSave?: boolean;
  finish?: () => void;
}

/** One visible batch across projects. The owner runs only one icon at a time,
 * returning to its video-priority scheduler after each native result is freed. */
export class ReferenceIconWork {
  private tasks = new Map<string, IconTask>();
  private timers = new Map<ProjectSession, ReturnType<typeof setTimeout>>();
  private active?: IconTask;
  private progress = 0;
  private item?: WorkItem;
  private watches = new Map<ProjectSession, () => void>();
  private stopSettings?: () => void;

  constructor(private changed: (item: WorkItem) => void, private wake: () => void) {}

  watch(session: ProjectSession) {
    if (!isTauri() || this.watches.has(session)) return;
    this.stopSettings ??= subscribeReferenceIconAutomation(() => {
      this.watches.forEach((_stop, watched) => this.scan(watched));
      this.wake();
    });
    let references = session.getSnapshot().config.references;
    const schedule = () => {
      clearTimeout(this.timers.get(session));
      this.timers.set(session, setTimeout(() => {
        this.timers.delete(session);
        this.scan(session);
      }, 1000));
    };
    this.watches.set(session, session.subscribe(() => {
      const next = session.getSnapshot().config.references;
      if (next === references) return;
      references = next;
      schedule();
    }));
    schedule();
  }

  forget(session: ProjectSession) {
    clearTimeout(this.timers.get(session));
    this.timers.delete(session);
    this.watches.get(session)?.();
    this.watches.delete(session);
    for (const [key, task] of this.tasks) if (task.session === session && !task.preset) this.tasks.delete(key);
    this.publish();
  }

  stop() {
    this.stopSettings?.();
    this.stopSettings = undefined;
    this.timers.forEach(clearTimeout);
    this.timers.clear();
    this.watches.forEach((stop) => stop());
    this.watches.clear();
  }

  scan(session: ProjectSession) {
    const references = session.getSnapshot().config.references;
    for (const task of this.tasks.values()) {
      if (task.preset || task.session !== session || task.status !== "queued") continue;
      const reference = references.find((reference) => reference.id === task.referenceId);
      if (!reference || !this.shouldGenerate(reference, task) || (!task.force && loadReferenceIconAutomation() === "disabled")) task.status = "skipped";
    }
    let added = false;
    for (const reference of references) {
      const key = `${session.record.folderPath}::${session.record.config.id}::${reference.id}`;
      const prompt = referenceIconPrompt(reference);
      const previous = this.tasks.get(key);
      const force = previous?.force && previous.status === "skipped";
      if (!force && loadReferenceIconAutomation() === "disabled") continue;
      if (!this.shouldGenerate(reference, { force })) continue;
      if (previous && (previous === this.active || (previous.prompt === prompt && previous.status !== "skipped"))) continue;
      this.tasks.set(key, { key, session, referenceId: reference.id, name: reference.name, prompt, status: "queued", submitted: false, cancelled: false, force });
      added = true;
    }
    this.publish();
    if (added) this.wake();
  }

  enqueueBuiltins(session: ProjectSession, presets: readonly ReferencePreset[] = REFERENCE_PRESETS, replaceIcon = false) {
    for (const preset of presets) {
      const key = `builtin::${preset.id}`;
      const previous = this.tasks.get(key);
      if ((!replaceIcon && hasPresetIcon(preset)) || (previous && ["queued", "generating", "saving"].includes(previous.status))) continue;
      this.tasks.set(key, { key, session, preset, replaceIcon, referenceId: preset.id, name: preset.name, prompt: "", status: "queued", submitted: false, cancelled: false, force: true });
    }
    this.publish();
    this.wake();
  }

  regenerateBuiltin(session: ProjectSession, presetId: string) {
    const preset = REFERENCE_PRESETS.find(({ id }) => id === presetId);
    if (preset) this.enqueueBuiltins(session, [preset], true);
  }

  pendingBuiltinIds(): ReadonlySet<string> {
    return new Set([...this.tasks.values()].filter((task) => task.preset && ["queued", "generating", "saving"].includes(task.status)).map((task) => task.referenceId));
  }

  private referenceFor(task: IconTask): ProjectReference | undefined {
    if (!task.preset) return task.session.getSnapshot().config.references.find((reference) => reference.id === task.referenceId);
    const preset = task.preset;
    return { id: preset.id, kind: "text", name: preset.name, description: preset.prompt,
      intendedUse: [preset.type], subcategory: preset.subcategory, createdAt: task.session.record.config.createdAt };
  }

  /** Interrupt a built-in icon so normal video work gets the GPU immediately
   * after native cancellation. The interrupted icon remains queued to retry. */
  yieldToVideo() {
    const task = this.active;
    if (!task?.preset || task.status !== "generating" || task.cancelled || task.yielded) return;
    task.yielded = true;
    if (task.submitted) void this.cancelForVideo(task);
    this.publish();
  }

  private async cancelForVideo(task: IconTask) {
    try { await cancelSlopfabGeneration(task.nativeId!); }
    catch (reason) { task.yielded = false; task.error = describeDiagnosticError(reason); this.publish(); }
  }

  private shouldGenerate(reference: ProjectReference, task: { force?: boolean }) {
    if (reference.refmods?.length && !activeReferenceRefmods(reference).length) return false;
    return task.force ? Boolean(reference.description.trim() || activeReferenceRefmods(reference).length) && (referenceImages(reference).length === 0 || activeReferenceRefmods(reference).length > 0) : needsReferenceIcon(reference);
  }

  isPending(session: ProjectSession, referenceId: string) {
    return [...this.tasks.values()].some((task) => task.session === session && task.referenceId === referenceId && ["queued", "generating", "saving"].includes(task.status));
  }

  regenerate(session: ProjectSession, referenceId: string) {
    const reference = session.getSnapshot().config.references.find((reference) => reference.id === referenceId);
    if (!reference || !this.shouldGenerate(reference, { force: true }) || this.isPending(session, referenceId)) return;
    const key = `${session.record.folderPath}::${session.record.config.id}::${reference.id}`;
    this.tasks.set(key, { key, session, referenceId, name: reference.name, prompt: referenceIconPrompt(reference), status: "queued", submitted: false, cancelled: false, force: true });
    this.publish();
    this.wake();
  }

  hasQueued() { return [...this.tasks.values()].some((task) => task.status === "queued"); }
  private canRun(task: IconTask) {
    const automation = loadReferenceIconAutomation();
    return task.status === "queued" && (task.force || automation === "enabled" || (automation === "ask" && task.approved));
  }
  hasRunnable() { return [...this.tasks.values()].some((task) => this.canRun(task)); }
  private awaitingConfirmation() {
    return loadReferenceIconAutomation() === "ask"
      ? [...this.tasks.values()].filter((task) => task.status === "queued" && !task.force && !task.approved)
      : [];
  }
  confirmationCount() { return this.awaitingConfirmation().length; }
  answerConfirmation(confirmed: boolean, remember: boolean) {
    for (const task of this.awaitingConfirmation()) {
      if (confirmed) task.approved = true;
      else { task.status = "cancelled"; task.cancelled = true; }
    }
    if (remember) saveReferenceIconAutomation(confirmed ? "enabled" : "disabled");
    this.publish();
    this.wake();
  }
  hasActiveProject(session: ProjectSession) {
    return [...this.tasks.values()].some((task) => !task.preset && task.session === session && ["queued", "generating", "saving"].includes(task.status));
  }
  owns(id: string) { return this.item?.id === id; }
  clearFinished(id: string) { if (this.owns(id)) this.item = undefined; }

  private publish() {
    if (this.hasQueued() && !this.item) {
      this.item = {
        id: `icons-${crypto.randomUUID()}`, kind: "reference-icons", projectKey: "", folderPath: "", projectName: "", sceneId: "",
        title: "Reference icons", submittedAt: new Date().toISOString(), status: "queued", progress: 0,
        detail: "Waiting to generate icons", error: null, completionAt: null, cancelling: false, needsSave: false,
        settings: { frames: 1, steps: REFERENCE_ICON_STEPS, seed: -1, canvasWidth: REFERENCE_ICON_RENDER_SIZE, canvasHeight: REFERENCE_ICON_RENDER_SIZE },
      };
    }
    if (!this.item) return;
    const tasks = [...this.tasks.values()].filter((task) => task.status !== "skipped");
    const completed = tasks.filter((task) => task.status === "completed").length;
    const settled = tasks.filter((task) => ["completed", "failed", "cancelled"].includes(task.status)).length;
    const queued = this.hasQueued();
    const failed = tasks.filter((task) => task.status === "failed");
    const projects = new Set(tasks.map((task) => task.session));
    const target = this.active ?? tasks.find((task) => task.status === "queued") ?? tasks.at(-1);
    this.item = {
      ...this.item,
      ...(target ? { projectKey: projectQueueKey(target.session.record), folderPath: target.session.record.folderPath, sceneId: target.referenceId } : {}),
      projectName: tasks.every((task) => task.preset) ? "Built-in references" : projects.size === 1 ? [...projects][0].getSnapshot().config.name : `${projects.size} projects`,
      status: this.active ? (this.active.status === "saving" ? "encoding" : "generating") : queued ? "queued" : failed.length ? "failed" : tasks.some((task) => task.status === "cancelled") ? "cancelled" : "completed",
      progress: tasks.length ? Math.min(1, (settled + (this.active ? this.progress : 0)) / tasks.length) : 1,
      detail: this.active ? `${this.active.yielded ? "Pausing for video" : this.active.cancelled ? "Cancelling" : this.active.status === "saving" ? "Saving" : "Generating"} ${this.active.name} · ${completed}/${tasks.length} icons saved`
        : this.confirmationCount() > 0 ? `${this.confirmationCount()} icons awaiting confirmation`
        : queued ? `${completed}/${tasks.length} icons saved · waiting behind videos` : `${completed}/${tasks.length} icons saved`,
      error: failed.length ? failed.map((task) => `${task.name}: ${task.error}`).join("\n") : null,
      cancelling: this.active?.cancelled ?? false,
      needsSave: tasks.some((task) => task.needsSave),
    };
    this.changed(this.item);
  }

  async runNext(ready: Promise<void>, videoWaiting: () => boolean) {
    const runnable = [...this.tasks.values()].filter((task) => this.canRun(task));
    const categoryOrder = (task: IconTask) => REFERENCE_TYPES.findIndex(({ id }) => id === task.preset?.type);
    const task = runnable.find((task) => !task.preset) ?? runnable.sort((a, b) => categoryOrder(a) - categoryOrder(b))[0];
    if (!task) return;
    this.active = task;
    task.submitted = false;
    task.yielded = false;
    task.status = "generating";
    this.progress = 0;
    this.publish();
    try {
      await ready;
      if (task.cancelled) return;
      if (!task.preset) await task.session.save();
      if (task.cancelled) return;
      if (videoWaiting()) { task.status = "queued"; return; }
      if (!task.force && loadReferenceIconAutomation() === "disabled") { task.status = "skipped"; return; }
      if (!task.force && !task.approved && loadReferenceIconAutomation() === "ask") { task.status = "queued"; return; }
      if (task.preset && !task.replaceIcon && hasPresetIcon(task.preset)) { task.status = "skipped"; return; }
      const reference = this.referenceFor(task);
      if (!reference || !this.shouldGenerate(reference, task)) { task.status = "skipped"; return; }
      task.refmodsSnapshot = JSON.stringify(activeReferenceRefmods(reference));
      task.name = reference.name;
      const template = referenceIconGenerator();
      if (!template) throw new Error("Choose an available text-prompt generator for reference icons in Settings.");
      task.prompt = referenceIconPrompt(reference, template.modelType);
      const config = task.session.getSnapshot().config;
      const runtimeConfig = { ...config, providerSettings: { ...config.providerSettings,
        slopfab: engineProviderSetting(template.paths, config.providerSettings.slopfab, template.attention, template.loras ?? [], "prompt", template.additionalSafetensors ?? [], template.motionCache ?? false, template.sources, template),
      } };
      const steps = generationStepsWithLoras(template.defaultSteps, runtimeConfig);
      if (this.item) { this.item = { ...this.item, settings: { ...this.item.settings, steps } }; this.publish(); }
      task.nativeId = `icon-${crypto.randomUUID()}`;
      const done = new Promise<void>((resolve) => { task.finish = resolve; });
      await enqueueSlopfabGeneration({
        jobId: task.nativeId, prompt: task.prompt, stillImage: true, frames: 1,
        steps, seed: -1, canvasWidth: REFERENCE_ICON_RENDER_SIZE, canvasHeight: REFERENCE_ICON_RENDER_SIZE, referencePaths: [],
        ...(activeReferenceRefmods(reference).length ? { refmods: referenceRefmodInputs(task.session.record.folderPath, [reference]) } : {}),
      }, runtimeConfig, task.session.record.folderPath);
      task.submitted = true;
      if (task.cancelled) await this.cancelNative(task);
      else if (task.yielded) await this.cancelForVideo(task);
      await done;
    } catch (reason) {
      task.status = task.cancelled ? "cancelled" : "failed";
      task.error = describeDiagnosticError(reason);
    } finally {
      if (task.cancelled) task.status = "cancelled";
      else if (task.yielded && task.status === "cancelled") { task.status = "queued"; task.error = undefined; }
      task.yielded = false;
      // Stop a built-in batch on an engine/storage failure instead of sending
      // the same failing setup through the entire catalog.
      if (task.preset && task.status === "failed") {
        for (const waiting of this.tasks.values()) if (waiting.preset && waiting.status === "queued") waiting.status = "cancelled";
      }
      this.active = undefined;
      this.publish();
      if (!task.preset) this.scan(task.session);
    }
  }

  progressEvent(event: { jobId: string; step: number; totalSteps: number }): boolean {
    if (!this.active || this.active.nativeId !== event.jobId) return false;
    this.progress = Math.max(this.progress, event.totalSteps > 0 ? Math.min(0.9, 0.1 + event.step / event.totalSteps * 0.8) : 0.05);
    this.publish();
    return true;
  }

  event(event: { jobId: string; state: string; detail: string }): boolean {
    const task = this.active;
    if (!task || task.nativeId !== event.jobId) return false;
    if (task.status === "saving") return true;
    if (event.state === "framesReady") {
      task.status = "saving";
      this.publish();
      void this.save(task);
    } else if (event.state === "failed" || event.state === "cancelled") {
      task.status = task.cancelled || event.state === "cancelled" ? "cancelled" : "failed";
      task.error = event.detail;
      task.finish?.();
    }
    return true;
  }

  private async save(task: IconTask) {
    try {
      const reference = this.referenceFor(task);
      if (task.cancelled) { task.status = "cancelled"; return; }
      if (!reference || !this.shouldGenerate(reference, task) || referenceIconPrompt(reference) !== task.prompt || JSON.stringify(activeReferenceRefmods(reference)) !== task.refmodsSnapshot) { task.status = "skipped"; return; }
      if (task.preset) {
        await saveBuiltinIcon(task.preset.id, task.nativeId!);
        task.status = "completed";
        return;
      }
      const iconRelativePath = await saveReferenceIcon(task.session.record.folderPath, task.nativeId!);
      let attached = false;
      task.session.update((current) => ({ ...current, references: current.references.map((reference) => {
        if (reference.id !== task.referenceId || !this.shouldGenerate(reference, task) || referenceIconPrompt(reference) !== task.prompt || JSON.stringify(activeReferenceRefmods(reference)) !== task.refmodsSnapshot) return reference;
        attached = true;
        return { ...reference, iconRelativePath };
      }) }));
      if (!attached) { task.status = "skipped"; return; }
      task.needsSave = true;
      await task.session.save();
      task.needsSave = false;
      task.status = "completed";
    } catch (reason) {
      task.status = "failed";
      task.error = describeDiagnosticError(reason);
    } finally {
      try { await releaseRendered(task.nativeId!); }
      catch (reason) { task.status = "failed"; task.error = describeDiagnosticError(reason); }
      finally { task.finish?.(); }
    }
  }

  async cancel() {
    for (const task of this.tasks.values()) {
      if (task.status === "queued") { task.cancelled = true; task.status = "cancelled"; }
    }
    if (this.active && this.active.status !== "saving") {
      this.active.cancelled = true;
      if (this.active.submitted) await this.cancelNative(this.active);
    }
    this.publish();
  }
  private async cancelNative(task: IconTask) {
    try { await cancelSlopfabGeneration(task.nativeId!); }
    catch (reason) { task.cancelled = false; task.error = describeDiagnosticError(reason); }
  }
  async retrySave() {
    for (const task of this.tasks.values()) {
      if (!task.needsSave) continue;
      try { await task.session.save(); task.needsSave = false; task.status = "completed"; task.error = undefined; }
      catch (reason) { task.error = describeDiagnosticError(reason); }
    }
    this.publish();
  }
}