Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
22.0 KB
Raw Blame
import { invoke } from "@tauri-apps/api/core";
import { writeDiagnostic } from "./diagnostics";
import { bitrateFor, outputCodec } from "./export";
import { isTauri } from "./persistence";
import { TIMELINE_THUMBNAIL_INTERVAL_MS, writeTimelineThumbnail } from "./timelineThumbnails";
import { encodeGenerationPreview, type GenerationPreview } from "./generationPreview";

/* Turning a finished render into a file in the project folder.
 *
 * slopfab makes pictures, not files: it hands Rust raw frames and raw audio and
 * frees them. Encoding is this side's job — WebCodecs owns the machine's
 * hardware encoder and Slopus ships no FFmpeg by design (CLAUDE.md) — so a
 * saved scene is a round trip:
 *
 *   Rust holds the frames  →  this reads them one at a time  →  WebCodecs
 *   encodes  →  mp4-muxer muxes  →  Rust writes the .mp4 into the project.
 *
 * The frames are pulled ONE AT A TIME rather than in a single read, because a
 * whole render is hundreds of megabytes and the encoder only ever wants the
 * next one. What is held in memory here is one frame plus whatever the encoder
 * has not yet drained.
 *
 * Nothing here invents a fallback. A machine with no VideoEncoder cannot save a
 * scene, and this says so rather than writing something that is not a video. */

/** The shape of a render still waiting in Rust's memory. */
export interface RenderedSummary {
  jobId: string;
  width: number;
  height: number;
  frameCount: number;
  fps: number;
  audioChannels: number;
  audioSampleRate: number;
  /** Interleaved samples across every channel. Zero means it rendered silent. */
  audioSamples: number;
}

/** What was written, in the project's own terms. */
export interface SavedGeneration {
  /** The path to record on the job — Rust's own answer, never a guess here. */
  relativePath: string;
  bytes: number;
  /** Null when everything the render produced is in the file. A sentence when
   *  something was left out, so a silent video is never silently silent. */
  note: string | null;
  /** Whether the saved MP4 actually contains an audio stream. */
  hasAudio?: boolean;
  width?: number;
  height?: number;
  durationMs?: number;
}

/** How often a keyframe is written: every two seconds, so the file can be
 *  seeked and a long delta chain cannot quietly cost quality. */
const KEYFRAME_SECONDS = 2;
/** How many frames may be in flight before this stops handing the encoder more.
 *  Each one is a full picture, so an unbounded queue is an unbounded heap. */
const ENCODE_QUEUE_LIMIT = 8;
const AUDIO_BLOCK = 1024;
const AUDIO_CODEC = "mp4a.40.2";
const AUDIO_BITRATE = 192_000;
/** Generated footage is a MASTER, not a delivery file: it is what every later
 *  export is cut from, so it is encoded at the highest preset the app has. */
const GENERATED_QUALITY = "high" as const;
const TIMELINE_THUMBNAIL_WIDTH = 192;
const TIMELINE_THUMBNAIL_JPEG_QUALITY = 0.82;

const describe = (reason: unknown) => (reason instanceof Error ? reason.message : String(reason));
const tick = () => new Promise((resolve) => setTimeout(resolve, 0));

/** Downsamples one RGBA render frame directly into a small JPEG. Sampling the
 * pixels avoids allocating a second full-resolution canvas merely to shrink
 * it, and this runs only at the timeline cache interval. */
async function timelineThumbnailJpeg(pixels: Uint8Array, width: number, height: number): Promise<Uint8Array | null> {
  try {
    const outputWidth = Math.min(width, TIMELINE_THUMBNAIL_WIDTH);
    const outputHeight = Math.max(1, Math.round(height * outputWidth / width));
    const canvas = document.createElement("canvas");
    canvas.width = outputWidth;
    canvas.height = outputHeight;
    const context = canvas.getContext("2d");
    if (!context) return null;
    const image = context.createImageData(outputWidth, outputHeight);
    for (let y = 0; y < outputHeight; y += 1) {
      const sourceY = Math.min(height - 1, Math.floor(y * height / outputHeight));
      for (let x = 0; x < outputWidth; x += 1) {
        const sourceX = Math.min(width - 1, Math.floor(x * width / outputWidth));
        const source = (sourceY * width + sourceX) * 4;
        const target = (y * outputWidth + x) * 4;
        image.data[target] = pixels[source];
        image.data[target + 1] = pixels[source + 1];
        image.data[target + 2] = pixels[source + 2];
        image.data[target + 3] = pixels[source + 3];
      }
    }
    context.putImageData(image, 0, 0);
    const blob = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/jpeg", TIMELINE_THUMBNAIL_JPEG_QUALITY));
    return blob ? new Uint8Array(await blob.arrayBuffer()) : null;
  } catch {
    // A missing canvas encoder must not cost the user the rendered video. The
    // timeline can build this disposable cache later from the saved MP4.
    return null;
  }
}

/** What Rust still holds for this scene, or null if nothing does. */
export async function renderedSummary(jobId: string): Promise<RenderedSummary | null> {
  if (!isTauri()) return null;
  return invoke<RenderedSummary | null>("generated_summary", { jobId });
}

type IpcBytes = ArrayBuffer | ArrayBufferView | number[];

/** Tauri's raw response is an ArrayBuffer in the desktop webview, while its
 * test/mock transports and some older runtimes expose the same body as a byte
 * array. Normalize both without ever treating byte values as PCM samples. */
export function bytesFromIpc(raw: IpcBytes): Uint8Array {
  if (raw instanceof ArrayBuffer) return new Uint8Array(raw);
  if (ArrayBuffer.isView(raw)) return new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength);
  return Uint8Array.from(raw);
}

/** One frame's pixels, RGBA, straight out of the render. */
async function renderedFrame(jobId: string, index: number): Promise<Uint8Array> {
  return bytesFromIpc(await invoke<IpcBytes>("generated_frame", { jobId, index }));
}

/** Turns the raw IPC body into the interleaved samples slopfab produced. */
export function audioSamplesFromIpc(raw: IpcBytes, expectedSamples: number): Float32Array {
  const bytes = bytesFromIpc(raw);
  if (bytes.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0) {
    throw new Error(`The render's audio buffer has ${bytes.byteLength} bytes, which is not whole 32-bit PCM.`);
  }
  // Copy the exact view. Constructing Float32Array from Uint8Array would turn
  // each byte into a sample; constructing it from an offset view can also fail
  // alignment. slopfab and Rust both use native little-endian f32 PCM.
  const copy = new Uint8Array(bytes.byteLength);
  copy.set(bytes);
  const samples = new Float32Array(copy.buffer);
  if (samples.length !== expectedSamples) {
    throw new Error(`The render promised ${expectedSamples} audio samples, but ${samples.length} came back.`);
  }
  return samples;
}

/** The render's soundtrack, interleaved, as Rust holds it. */
async function renderedAudio(jobId: string, expectedSamples: number): Promise<Float32Array> {
  return audioSamplesFromIpc(await invoke<IpcBytes>("generated_audio", { jobId }), expectedSamples);
}

/** Gives the frames back. Called on the way out whether or not a file was
 *  written: a render nobody can encode is still hundreds of megabytes. */
export async function releaseRendered(jobId: string): Promise<boolean> {
  if (!isTauri()) return false;
  return invoke<boolean>("release_generated_frames", { jobId }).catch(() => false);
}

/** Writes the muxed bytes into the project folder. The path is Rust's to
 *  decide — this names the scene, not the file. Raw IPC body for the same
 *  reason an export uses one: a JSON array of numbers for a 30 MB file is
 *  hundreds of megabytes of text. */
async function writeGeneratedVideo(folderPath: string, jobId: string, bytes: Uint8Array): Promise<{ relativePath: string; bytes: number }> {
  return invoke<{ relativePath: string; bytes: number }>("write_generated_video", bytes, {
    headers: {
      "x-generated-folder": encodeURIComponent(folderPath),
      "x-generated-job": encodeURIComponent(jobId),
    },
  });
}

/** Promote a cancelled render's latest preview to a normal project video. */
export async function saveGenerationPreview(folderPath: string, jobId: string, preview: GenerationPreview): Promise<SavedGeneration> {
  if (!isTauri()) throw new Error("Saving a generation preview needs the desktop app.");
  const video = await encodeGenerationPreview(preview, new AbortController().signal);
  const written = await writeGeneratedVideo(folderPath, jobId, new Uint8Array(await video.arrayBuffer()));
  return { ...written, note: null, hasAudio: false, width: preview.width, height: preview.height,
    durationMs: Math.round(preview.frames.length / preview.fps * 1000) };
}

/** The codec string this machine actually accepts for this picture, asked of
 *  the encoder rather than assumed from the name. */
async function pickCodec(width: number, height: number, fps: number, bitrate: number): Promise<{ codec: string; tried: string[] }> {
  const tried: string[] = [];
  for (const candidate of outputCodec("h264").candidates(width, height, fps)) {
    tried.push(candidate);
    const config: VideoEncoderConfig = {
      codec: candidate,
      width,
      height,
      bitrate,
      framerate: fps,
      // Length-prefixed AVC with the parameter sets in the sample description,
      // which is the only shape mp4-muxer can put in an .mp4.
      avc: { format: "avc" },
    };
    const support = await VideoEncoder.isConfigSupported(config).catch(() => ({ supported: false }));
    if (support.supported) return { codec: candidate, tried };
  }
  throw new Error(`This computer's encoder refused every H.264 configuration Slopus offers (${tried.join(", ")}).`);
}

/** Makes a `VideoFrame` out of raw RGBA.
 *
 *  The buffer constructor is the direct route and what every current Chromium
 *  supports; a webview that refuses it gets the canvas route instead, which
 *  copies the same pixels through an ImageData. Which one is in use is decided
 *  ONCE, on the first frame, so a fallback is not re-tested 360 times. */
function frameFactory(width: number, height: number) {
  let viaCanvas: CanvasRenderingContext2D | null = null;
  const canvasFrame = (pixels: Uint8Array, timestamp: number, duration: number) => {
    if (!viaCanvas) {
      const canvas = document.createElement("canvas");
      canvas.width = width;
      canvas.height = height;
      const context = canvas.getContext("2d", { willReadFrequently: false });
      if (!context) throw new Error("This computer gave no 2D canvas to build frames in.");
      viaCanvas = context;
    }
    viaCanvas.putImageData(new ImageData(new Uint8ClampedArray(pixels.buffer, pixels.byteOffset, pixels.byteLength), width, height), 0, 0);
    return new VideoFrame(viaCanvas.canvas, { timestamp, duration, alpha: "discard" });
  };
  let direct = true;
  return (pixels: Uint8Array, timestamp: number, duration: number): VideoFrame => {
    if (direct) {
      try {
        return new VideoFrame(pixels, {
          format: "RGBA",
          codedWidth: width,
          codedHeight: height,
          timestamp,
          duration,
        });
      } catch {
        direct = false;
      }
    }
    return canvasFrame(pixels, timestamp, duration);
  };
}

/** The render's own AAC configuration, or null for a genuinely silent render.
 *  Sound that exists is never quietly omitted from the MP4. */
async function audioConfig(summary: RenderedSummary): Promise<AudioEncoderConfig | null> {
  if (summary.audioSamples === 0) return null;
  if (summary.audioChannels <= 0 || summary.audioSampleRate <= 0 || summary.audioSamples % summary.audioChannels !== 0) {
    throw new Error("The render returned audio with invalid channel or sample-rate metadata.");
  }
  if (typeof AudioEncoder === "undefined" || typeof AudioData === "undefined") return null;
  for (const sampleRate of [...new Set([summary.audioSampleRate, 48_000, 44_100])]) {
    const config: AudioEncoderConfig = {
      codec: AUDIO_CODEC,
      sampleRate,
      numberOfChannels: summary.audioChannels,
      bitrate: AUDIO_BITRATE,
    };
    const support = await AudioEncoder.isConfigSupported(config).catch(() => ({ supported: false }));
    if (support.supported) return config;
  }
  return null;
}

/** Linear PCM resampling is enough here: slopfab's soundtrack is already the
 * final mix, and this only adapts its rate to an AAC configuration the OS
 * exposes. Channels remain interleaved throughout. */
export function resampleInterleaved(samples: Float32Array, channels: number, fromRate: number, toRate: number): Float32Array {
  if (fromRate === toRate) return samples;
  const inputFrames = Math.floor(samples.length / channels);
  const outputFrames = Math.max(1, Math.round(inputFrames * toRate / fromRate));
  const output = new Float32Array(outputFrames * channels);
  for (let frame = 0; frame < outputFrames; frame += 1) {
    const source = Math.min(inputFrames - 1, frame * fromRate / toRate);
    const left = Math.floor(source);
    const right = Math.min(inputFrames - 1, left + 1);
    const mix = source - left;
    for (let channel = 0; channel < channels; channel += 1) {
      const a = samples[left * channels + channel];
      const b = samples[right * channels + channel];
      output[frame * channels + channel] = a + (b - a) * mix;
    }
  }
  return output;
}

/** Feeds the render's interleaved samples to the AAC encoder, in the blocks
 *  AAC works in. */
async function encodeAudio(
  summary: RenderedSummary,
  samples: Float32Array,
  config: AudioEncoderConfig,
  add: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => void,
): Promise<number> {
  const channels = summary.audioChannels;
  const sampleRate = config.sampleRate ?? summary.audioSampleRate;
  const prepared = resampleInterleaved(samples, channels, summary.audioSampleRate, sampleRate);
  const total = Math.floor(prepared.length / channels);
  const failure: { reason: Error | null } = { reason: null };
  let chunks = 0;
  const encoder = new AudioEncoder({
    output: (chunk, meta) => { chunks += 1; add(chunk, meta); },
    error: (reason) => { failure.reason = reason instanceof Error ? reason : new Error(String(reason)); },
  });
  try {
    encoder.configure(config);
    for (let offset = 0; offset < total; offset += AUDIO_BLOCK) {
      if (failure.reason) throw failure.reason;
      const count = Math.min(AUDIO_BLOCK, total - offset);
      // slopfab's C API returns [frame0-L, frame0-R, frame1-L, frame1-R, ...].
      // WebCodecs calls that layout `f32`; `f32-planar` means something else.
      const interleaved = prepared.slice(offset * channels, (offset + count) * channels);
      const data = new AudioData({
        format: "f32",
        sampleRate,
        numberOfFrames: count,
        numberOfChannels: channels,
        timestamp: Math.round((offset * 1_000_000) / sampleRate),
        data: interleaved,
      });
      try { encoder.encode(data); }
      finally { data.close(); }
      while (encoder.encodeQueueSize > 16) {
        if (failure.reason) throw failure.reason;
        await tick();
      }
    }
    await encoder.flush();
    if (failure.reason) throw failure.reason;
    if (chunks === 0) throw new Error("The AAC encoder finished without producing a soundtrack.");
    return chunks;
  } finally {
    if (encoder.state !== "closed") encoder.close();
  }
}

/**
 * Encodes the render waiting for `jobId` and writes it into the project.
 *
 * Throws if there is nothing to encode, if this computer cannot encode, or if
 * the write fails — every one of which is a sentence the Generator shows. The
 * frames are handed back either way.
 */
export async function saveGeneratedScene(options: {
  folderPath: string;
  jobId: string;
  /** Queue runs have unique IDs; timeline caches still belong to the scene. */
  thumbnailJobId?: string;
  /** Fit grid-aligned latent upscaling to the selected output using WebCodecs. */
  outputSize?: { width: number; height: number };
  /** Called as frames go into the encoder, for the progress bar. */
  onProgress?: (encoded: number, total: number) => void;
}): Promise<SavedGeneration> {
  const { folderPath, jobId } = options;
  if (!isTauri()) throw new Error("Saving a rendered scene needs the desktop app.");
  let released = false;
  try {
    if (typeof VideoEncoder === "undefined" || typeof VideoFrame === "undefined") {
      throw new Error("This webview has no WebCodecs VideoEncoder, so a rendered scene cannot be turned into a video file here.");
    }
    const summary = await renderedSummary(jobId);
    if (!summary) {
      throw new Error("The rendered frames are no longer in memory, so there is nothing to save. Run the scene again.");
    }
    if (summary.frameCount <= 0 || summary.width <= 0 || summary.height <= 0) {
      throw new Error("The render came back with no pictures in it.");
    }
    writeDiagnostic("info", "generated-video", "render.loaded", "Rendered buffers are available for encoding.", {
      jobId, width: summary.width, height: summary.height, frames: summary.frameCount, fps: summary.fps,
      audioChannels: summary.audioChannels, audioSampleRate: summary.audioSampleRate, audioSamples: summary.audioSamples,
    });

    const { width, height } = options.outputSize ?? summary;
    if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width <= 0 || height <= 0) {
      throw new Error("The selected video output dimensions are invalid.");
    }
    const fps = summary.fps > 0 ? summary.fps : 24;
    const bitrate = bitrateFor(width, height, fps, GENERATED_QUALITY);
    const { codec } = await pickCodec(width, height, fps, bitrate);
    writeDiagnostic("info", "generated-video", "video_encoder.selected", "WebCodecs accepted a video encoder configuration.", {
      jobId, codec, bitrate, fps, width, height,
    });
    const sound = await audioConfig(summary);
    const audioChunks: Array<{ chunk: EncodedAudioChunk; meta?: EncodedAudioChunkMetadata }> = [];
    let note: string | null = null;
    if (summary.audioSamples > 0) {
      if (sound) {
        try {
          await encodeAudio(summary, await renderedAudio(jobId, summary.audioSamples), sound, (chunk, meta) => audioChunks.push({ chunk, meta }));
        } catch (reason) {
          note = `Saved without sound: ${describe(reason)}`;
        }
      } else {
        note = "Saved without sound: this computer's encoder would not take the render's audio.";
      }
    }
    if (note) writeDiagnostic("warn", "generated-video", "audio.omitted", note, { jobId });
    const withAudio = audioChunks.length > 0;

    const { ArrayBufferTarget, Muxer } = await import("mp4-muxer");
    const target = new ArrayBufferTarget();
    const muxer = new Muxer({
      target,
      video: { codec: outputCodec("h264").muxer, width, height, frameRate: Math.round(fps) },
      ...(withAudio
        ? { audio: { codec: "aac" as const, numberOfChannels: summary.audioChannels, sampleRate: sound!.sampleRate! } }
        : {}),
      fastStart: "in-memory",
    });

    /* On an object because the encoder writes it from its own callback, where
       TypeScript's flow analysis cannot follow the assignment. */
    const failure: { reason: Error | null } = { reason: null };
    const makeFrame = frameFactory(summary.width, summary.height);
    /* Computed from the index every time rather than accumulated, so a rate
       that does not divide a microsecond evenly cannot drift the last frame
       away from the clock. */
    const timestampUs = (index: number) => Math.round((index * 1_000_000) / fps);
    const keyframeEvery = Math.max(1, Math.round(fps * KEYFRAME_SECONDS));
    const thumbnailEvery = Math.max(1, Math.round(fps * TIMELINE_THUMBNAIL_INTERVAL_MS / 1_000));
    const timelineThumbnails: Array<{ timeMs: number; bytes: Uint8Array }> = [];
    const encoder = new VideoEncoder({
      output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
      error: (reason) => { failure.reason = reason instanceof Error ? reason : new Error(String(reason)); },
    });
    try {
      encoder.configure({ codec, width, height, bitrate, framerate: fps, avc: { format: "avc" } });
      for (let index = 0; index < summary.frameCount; index += 1) {
        if (failure.reason) throw failure.reason;
        const pixels = await renderedFrame(jobId, index);
        if (index % thumbnailEvery === 0) {
          const thumbnail = await timelineThumbnailJpeg(pixels, summary.width, summary.height);
          if (thumbnail) timelineThumbnails.push({ timeMs: Math.round(index * 1_000 / fps), bytes: thumbnail });
        }
        const frame = makeFrame(pixels, timestampUs(index), timestampUs(index + 1) - timestampUs(index));
        try { encoder.encode(frame, { keyFrame: index % keyframeEvery === 0 }); }
        finally { frame.close(); }
        options.onProgress?.(index + 1, summary.frameCount);
        // Let the encoder drain, and let the window paint: this loop is the
        // whole of a five-second render's encode and it must not freeze the UI.
        while (encoder.encodeQueueSize > ENCODE_QUEUE_LIMIT) {
          if (failure.reason) throw failure.reason;
          await tick();
        }
        if (index % 8 === 7) await tick();
      }
      await encoder.flush();
      if (failure.reason) throw failure.reason;
    } finally {
      if (encoder.state !== "closed") encoder.close();
    }

    // Audio and frames are now encoded. Free the native render before muxing
    // and sending the MP4, which both allocate their own output buffers.
    released = await releaseRendered(jobId);
    audioChunks.forEach(({ chunk, meta }) => muxer.addAudioChunk(chunk, meta));

    muxer.finalize();
    const written = await writeGeneratedVideo(folderPath, jobId, new Uint8Array(target.buffer));
    await Promise.allSettled(timelineThumbnails.map((thumbnail) =>
      writeTimelineThumbnail(folderPath, options.thumbnailJobId ?? jobId, thumbnail.timeMs, thumbnail.bytes)));
    return { relativePath: written.relativePath, bytes: written.bytes, note, hasAudio: withAudio, width, height, durationMs: Math.round(summary.frameCount / summary.fps * 1000) };
  } finally {
    // Whatever happened, the render stops occupying memory here.
    if (!released) await releaseRendered(jobId);
  }
}

/** The same failure, said the same way wherever it is caught. */
export const describeSaveFailure = (reason: unknown) => describe(reason);