Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

23.4 KB
Raw Blame
/**
 * Sprites — the LOCAL (MiniMax H3) animation-render lane.
 *
 * Every sprite animation track used to have exactly one way to produce its
 * source clip: an observable grok TUI session (`animationWorkflow.js`). That is
 * a paid cloud call, which made the whole sprite pipeline unusable offline even
 * though PortOS already ships MiniMax H3 locally — the MLX port on Apple Silicon
 * and the diffusers/CUDA runtime on Windows and Linux (`videoGen/runtimes.js`).
 *
 * This module is the second lane. It is deliberately NOT a second copy of the
 * pipeline: everything downstream of the clip — chroma recovery, frame
 * selection, loop trimming, geometry QC, approval, atlas compile — is the same
 * deterministic postprocess both lanes hand a `source-video.mp4` to. All that
 * differs is who renders that MP4, so the only things here are:
 *
 *   - which local model this platform can render on, and whether it is ready;
 *   - how a sprite's authoring knobs (a clip length in seconds, an anchor of
 *     arbitrary aspect) map onto H3's fixed contract (a 17n+5 frame grid at a
 *     locked 24 fps, on one of six trained canvases);
 *   - queueing one render through `mediaJobQueue` and putting the finished clip
 *     where the postprocess expects it.
 *
 * Unlike the grok lane this is NOT a PTY session, so a local run has no
 * attachable Shell session. It carries a `jobId` instead, which is both what the
 * Render Queue UI shows and — via `localRunLiveness` — how a read path tells a
 * genuinely-live multi-hour render from one the server died in the middle of.
 */

import { join } from 'path';
import { PATHS, pathExists, copyFileGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { getVideoModels, requiredModelCacheGroups } from '../../lib/mediaModels.js';
import { inspectModelCache, findCachedRepoFiles } from '../../lib/hfCache.js';
import { BYOV_RUNTIME_INFO, isByovRuntimeReady } from '../videoGen/runtimes.js';
import { enqueueJob, getJob } from '../mediaJobQueue/index.js';
import { minimaxH3ControlError } from '../videoGen/minimaxH3Controls.js';
import {
  LOCAL_VIDEO_PROVIDER_ID, isLocalProviderRun, GROK_TUI_TIMEOUT_MS, runCreatedAtMs,
} from './animationWorkflow.js';
import { WALK_TRACK } from './animationTargets.js';

// H3's video VAE decodes only 17n+5 frame counts and the model runs at a fixed
// 24 fps (both enforced by `videoGen/minimaxH3Controls.js`), so a sprite's
// requested clip length can only ever be honoured approximately. 24 is read
// from the model row when it declares one so a future re-profile moves both.
const H3_FALLBACK_FPS = 24;

// Both H3 runtimes PortOS ships. A platform exposes exactly one of them
// (`getVideoModels()` already selects the mlx or cuda bucket), so this set is
// what identifies "an H3 row" inside whichever bucket is active rather than a
// platform test of its own — which is what keeps this lane working on Linux
// CUDA installs without a second branch.
const MINIMAX_H3_RUNTIME_IDS = Object.freeze(new Set(['minimax_h3', 'minimax_h3_cuda']));

/**
 * The MiniMax H3 model row this install can render on, or null.
 *
 * `getVideoModels()` has already filtered to the active platform bucket and
 * dropped anything flagged broken there, so an H3 row surviving that filter is
 * by definition the one this machine would use. Null is the honest answer for a
 * platform with no H3 build at all (and is reported as such rather than
 * collapsing into "not installed", which would send the user to an install
 * button that cannot help them).
 */
export const resolveLocalVideoModel = () => (
  getVideoModels().find((model) => MINIMAX_H3_RUNTIME_IDS.has(model.runtime)) || null
);

/** The render fps a model row is locked to (H3: 24). */
export const localRenderFps = (model) => {
  const declared = Array.isArray(model?.fpsOptions) ? Number(model.fpsOptions[0]) : NaN;
  return Number.isFinite(declared) && declared > 0 ? declared : H3_FALLBACK_FPS;
};

/**
 * The legal H3 frame count closest to `durationSeconds` of footage.
 *
 * The sprite lanes ask for a clip length in seconds because that is what grok
 * takes; H3 takes a frame count off a fixed grid. Snapping to the CLOSEST legal
 * option (rather than rounding up) matters because the grid is coarse — at 24
 * fps the MLX options step by ~0.7 s but the CUDA window starts at 124 frames,
 * so rounding a 6 s request up would cost a materially longer render for
 * footage the packer discards anyway.
 *
 * Ties go to the shorter option: two equally-close counts render for different
 * lengths of time and the cheaper one is never the wrong call for source
 * footage the postprocess is going to resample regardless.
 */
export const resolveLocalFrameCount = (model, durationSeconds) => {
  const options = (Array.isArray(model?.frameOptions) ? model.frameOptions : [])
    .map(Number)
    .filter((frames) => Number.isFinite(frames) && frames > 0)
    .sort((a, b) => a - b);
  const fallback = Number(model?.defaultFrames);
  if (!options.length) {
    return Number.isFinite(fallback) && fallback > 0 ? fallback : null;
  }
  const seconds = Number(durationSeconds);
  if (!Number.isFinite(seconds) || seconds <= 0) {
    return options.includes(fallback) ? fallback : options[0];
  }
  const target = seconds * localRenderFps(model);
  return options.reduce((best, frames) => (
    Math.abs(frames - target) < Math.abs(best - target) ? frames : best
  ), options[0]);
};

/**
 * The trained H3 canvas whose aspect ratio is closest to the anchor's.
 *
 * H3 renders only on the six canvases its row declares, and `generateVideo`
 * conforms a conditioning image to the chosen one with
 * `scale=increase,crop` — a CENTER CROP. Handing it a portrait sprite anchor on
 * H3's 16:9 default would therefore slice the character's head and feet off
 * before the render even starts, and the geometry QC would fail every run for a
 * reason invisible in the output. Picking the nearest-aspect canvas here (and
 * padding the input to it in `prepareWalkAnchorChromaInput`) makes that crop a
 * no-op instead.
 *
 * Ties go to the SMALLER canvas: equal aspect fidelity for less render time.
 */
export const pickLocalRenderCanvas = (model, { width, height } = {}) => {
  const presets = (Array.isArray(model?.resolutionOptions) ? model.resolutionOptions : [])
    .map((preset) => ({ width: Number(preset?.w), height: Number(preset?.h) }))
    .filter((preset) => (
      Number.isFinite(preset.width) && preset.width > 0
      && Number.isFinite(preset.height) && preset.height > 0
    ));
  const fallbackWidth = Number(model?.defaultWidth);
  const fallbackHeight = Number(model?.defaultHeight);
  const fallback = Number.isFinite(fallbackWidth) && fallbackWidth > 0
    && Number.isFinite(fallbackHeight) && fallbackHeight > 0
    ? { width: fallbackWidth, height: fallbackHeight }
    : null;
  if (!presets.length) return fallback;
  const w = Number(width);
  const h = Number(height);
  // No measurable anchor: the model's declared default is the honest choice —
  // guessing an aspect from nothing would be worse than the row's own answer.
  if (!Number.isFinite(w) || w <= 0 || !Number.isFinite(h) || h <= 0) {
    return fallback || presets[0];
  }
  const wanted = w / h;
  // Compared in LOG space so "twice as wide as wanted" and "half as wide" score
  // equally — a linear difference is asymmetric and biases every portrait
  // anchor toward the landscape presets.
  const distance = (preset) => Math.abs(Math.log((preset.width / preset.height) / wanted));
  return presets.reduce((best, preset) => {
    const delta = distance(preset) - distance(best);
    if (delta < 0) return preset;
    if (delta > 0) return best;
    return preset.width * preset.height < best.width * best.height ? preset : best;
  }, presets[0]);
};

/**
 * Whether every weight `model` needs is resolvable in the HF cache.
 *
 * `requiredModelCacheGroups` is what makes this correct on BOTH platforms: the
 * MLX row pins its upstream FL2VA checkpoint in `requiredWeights`, while the
 * CUDA row pins its ~144 GB diffusers subset in `repoFiles` against its own
 * (~498 GB) repo. Checking only `requiredWeights` reported a CUDA download that
 * was interrupted after a few shards as complete, so the picker offered a render
 * that then died inside the runner's cache-only resolve.
 *
 * Still coarser than the render path's own preflight (`assertMiniMaxH3Preflight`
 * additionally verifies the pinned runtime checkout is clean). Restating THAT
 * here would be a second copy free to drift; this answers the question the UI
 * asks — "is there any point offering the button?" — and a deeper failure still
 * surfaces as a captured error on the run record naming the exact missing piece.
 */
const localModelWeightsCached = async (model) => {
  const groups = requiredModelCacheGroups(model);
  const [base, ...resolved] = await Promise.all([
    inspectModelCache(model.repo, model.revision ? { revision: model.revision } : {}),
    ...groups.map((group) => (
      group?.repo && Array.isArray(group.files) && group.files.length
        ? findCachedRepoFiles(group.repo, group.files, group.revision ? { revision: group.revision } : {})
        : Promise.resolve(null)
    )),
  ]);
  if (!base.cached) return false;
  // findCachedRepoFiles resolves to null when ANY pinned file is missing. A row
  // with no groups at all never enters this loop, so its base snapshot stands.
  return resolved.every((files) => Array.isArray(files) && files.length > 0);
};

/**
 * Readiness for the local lane, shaped for direct display.
 *
 * Always resolves — an unavailable lane is a `ready: false` plus a `reason`
 * naming the one thing to fix, never a throw. The client renders the reason
 * beside a disabled option, so "why can't I pick this?" is answered in place
 * rather than as a 409 after a click.
 */
export async function getLocalAnimationProviderStatus() {
  const model = resolveLocalVideoModel();
  if (!model) {
    return {
      id: 'local',
      label: 'Local (MiniMax H3)',
      ready: false,
      reason: 'This platform has no local MiniMax H3 video build — sprite animation renders on grok here.',
    };
  }
  const base = {
    id: 'local',
    label: 'Local (MiniMax H3)',
    modelId: model.id,
    modelName: model.name,
    runtime: model.runtime,
    fps: localRenderFps(model),
  };
  const runtimeLabel = BYOV_RUNTIME_INFO[model.runtime]?.label || model.runtime;
  if (!await isByovRuntimeReady(model.runtime)) {
    return {
      ...base,
      ready: false,
      reason: `The ${runtimeLabel} runtime is not installed — install it from Video Gen, then reload.`,
    };
  }
  if (!await localModelWeightsCached(model)) {
    return {
      ...base,
      ready: false,
      reason: `${model.name} is not downloaded — download it from Video Gen, then reload.`,
    };
  }
  return { ...base, ready: true, reason: null };
}

/**
 * Every animation provider the sprite lanes offer, in display order.
 *
 * grok reports `ready: true` unconditionally: it is the pre-existing default and
 * nothing here gates it, so claiming to know its CLI is installed would be an
 * invented gate that could only ever be wrong in the direction of hiding a
 * working button.
 */
export async function listAnimationProviders() {
  return [
    {
      id: 'grok',
      label: 'Grok (cloud)',
      ready: true,
      reason: null,
    },
    await getLocalAnimationProviderStatus(),
  ];
}

/**
 * Resolve the render plan for one local clip, or throw the 409 that says why
 * this install cannot render one.
 *
 * Called BEFORE the run record is written so an unusable lane costs the user a
 * clear error instead of an inert `error` run they then have to go delete.
 *
 * The canvas is NOT decided here — it depends on the anchor's aspect ratio, and
 * only the input-prep step measures that. The plan carries a `chooseCanvas` the
 * prep step calls with the measured size and hands back on its result, so the
 * measurement happens exactly once and the padded input and the render argv
 * cannot disagree about which canvas was picked.
 */
export async function planLocalAnimationRender({ durationSeconds } = {}) {
  const status = await getLocalAnimationProviderStatus();
  if (!status.ready) {
    throw new ServerError(status.reason, { status: 409, code: 'LOCAL_VIDEO_PROVIDER_NOT_READY' });
  }
  const model = resolveLocalVideoModel();
  const numFrames = resolveLocalFrameCount(model, durationSeconds);
  if (!numFrames) {
    throw new ServerError(
      `${model.name} declares no legal frame counts, so a local render cannot be sized.`,
      { status: 500, code: 'LOCAL_VIDEO_MODEL_MISCONFIGURED' },
    );
  }
  const fps = localRenderFps(model);
  // Validate the plan against the RENDER BOUNDARY'S OWN gate rather than a
  // restatement of it. `data/media-models.json` is user-editable, so an install
  // that edits `fpsOptions` or the frame grid would otherwise get a plan this
  // module happily accepts and a job that 400s hours later inside the runner
  // args. Calling the real checker means every current and future H3 control
  // rule is inherited here for free instead of drifting from a local copy.
  const controlError = minimaxH3ControlError({ model, numFrames, fps });
  if (controlError) {
    throw new ServerError(
      `${model.name} cannot render this clip: ${controlError.message}`,
      { status: 500, code: 'LOCAL_VIDEO_MODEL_MISCONFIGURED' },
    );
  }
  return {
    model,
    numFrames,
    fps,
    chooseCanvas: (anchorSize) => pickLocalRenderCanvas(model, anchorSize),
  };
}

/**
 * The provenance a local run stamps on its record, so a finished clip
 * self-documents which lane, model, runtime, and geometry produced it — the same
 * question `provider` alone answered while grok was the only lane.
 */
export const localRenderManifest = (plan, canvas) => ({
  provider: LOCAL_VIDEO_PROVIDER_ID,
  videoModelId: plan.model.id,
  videoRuntime: plan.model.runtime,
  renderFrames: plan.numFrames,
  renderFps: plan.fps,
  renderSeconds: Math.round((plan.numFrames / plan.fps) * 100) / 100,
  ...(canvas ? { renderWidth: canvas.width, renderHeight: canvas.height } : {}),
});

/**
 * The media-job params tag that makes a queued render self-describing.
 *
 * Load-bearing, not decoration: the completion hook decodes ONLY this — it holds
 * no in-memory state — so a render that outlives the request, the client, or the
 * whole server process can still be filed onto the run it belongs to. The
 * shipped `spriteWalk` tag rides along for the walk lane because the client's
 * `useSpritePendingRenders` rehydrate already keys off it (`owner: 'sprites'` +
 * `params.spriteWalk.direction`), so a browser reload mid-render shows the
 * direction as in-flight instead of re-enabling its Generate button.
 */
export const spriteAnimationJobTag = ({ recordId, runId, track, direction }) => ({
  spriteAnimation: { recordId, runId, track, direction },
  ...(track === WALK_TRACK ? { spriteWalk: { recordId, direction } } : {}),
});

/**
 * Queue the render. Resolves to the media-job id once the queue has durably
 * admitted it (#8325), so the caller can stamp it on an already-persisted run
 * record; rejects when the queue could not save the job.
 *
 * `hidden: true` keeps the clip out of the user's video gallery: it is an
 * intermediate the sprite pipeline consumes, not a video they asked to keep.
 * `enqueueJob` rather than `enqueueUnattendedMediaJob` is deliberate — the user
 * explicitly chose the LOCAL provider, and the unattended helper's default
 * routing could send the job to a peer instead.
 */
export async function enqueueLocalAnimationRender({ plan, canvas, prompt, inputAbs, recordId, runId, track, direction }) {
  const { jobId } = await enqueueJob({
    kind: 'video',
    // The owner the client's sprite pending-render hook filters on.
    owner: 'sprites',
    params: {
      modelId: plan.model.id,
      prompt,
      mode: 'image',
      sourceImagePath: inputAbs,
      ...(canvas ? { width: canvas.width, height: canvas.height } : {}),
      numFrames: plan.numFrames,
      fps: plan.fps,
      hidden: true,
      ...spriteAnimationJobTag({ recordId, runId, track, direction }),
    },
  });
  return jobId;
}

/**
 * Copy a finished render's clip to where the deterministic postprocess reads it.
 *
 * NEVER throws. The caller's contract is that the attach runs either way — the
 * attach already treats "no clip at videoAbs" as a terminal, user-visible error,
 * so a copy failure that propagated instead would skip that attach and strand
 * the run at `rendering` with nothing to move it (permanently, on the track
 * lane, whose in-flight guard then refuses every retry).
 *
 * COPY rather than move: the (hidden) gallery record generateVideo wrote still
 * points at this file, and its thumbnail sits beside it.
 */
export async function collectLocalAnimationClip({ jobId, videoAbs, label }) {
  // generateVideo names its output after the job id (`filename = ${jobId}.mp4`).
  const renderedAbs = join(PATHS.videos, `${jobId}.mp4`);
  if (!await pathExists(renderedAbs)) {
    // Expected for a genuinely failed render, and the reason the caller files
    // the run as errored — so this is informational, not an error line.
    console.log(`🎞️ ${label} has no local clip at ${renderedAbs}`);
    return false;
  }
  try {
    await copyFileGuarded(renderedAbs, videoAbs);
    return true;
  } catch (err) {
    console.error(`❌ ${label} could not stage the local clip into its run: ${err?.message || err}`);
    return false;
  }
}

/**
 * What the media-job queue says about a `rendering` local run.
 *
 * FOUR states, and collapsing any pair of them breaks something concrete:
 *
 *  - `live`     — queued or running. A local H3 clip is a multi-HOUR render, so
 *                 the grok lane's wall-clock cutoff would call this dead.
 *  - `settling` — the job COMPLETED and the attach is in flight (or the boot hook
 *                 is about to run it). Reading this as dead is the bug that
 *                 mattered: the attach spends a second or two copying and
 *                 hashing a 20–80 MB clip, and a detail poll landing in that gap
 *                 would report a SUCCESSFUL render as "interrupted" — and, worse,
 *                 unblock the in-flight guard so a second multi-hour render could
 *                 be started for the same direction.
 *  - `dead`     — failed or canceled, including the `interrupted by restart` the
 *                 queue stamps on a job that was running when the server died.
 *  - `unknown`  — no job id, or a job the archive has rolled past. NOT `dead`:
 *                 that would error a live run the moment its job aged out.
 *
 * Only `dead` is decided here; `settling` and `unknown` defer to the caller's
 * wall clock, which is generous enough that the boot hook wins the race.
 */
export const localRunLiveness = (run) => {
  const jobId = run?.jobId;
  if (typeof jobId !== 'string' || !jobId) return 'unknown';
  const job = getJob(jobId);
  if (!job) return 'unknown';
  if (job.status === 'completed') return 'settling';
  if (job.status === 'failed' || job.status === 'canceled') return 'dead';
  return 'live';
};

// How long a GROK run may sit at `rendering` before a read treats it as
// stranded: its TUI hard cap plus a buffer. Past that the session and its
// completion handler can only have died with the process.
const GROK_RENDER_STALE_MS = GROK_TUI_TIMEOUT_MS + 60_000;
// The LOCAL lane's equivalent, used only when the media-job queue cannot answer
// for a run — it has no job id, its job has rolled out of the archive, or the
// job COMPLETED and its attach has not landed yet. Deliberately a day rather
// than grok's half hour: a local H3 clip is a multi-HOUR render on current
// Apple Silicon, and the completion hook wins this race in every normal case.
const LOCAL_RENDER_STALE_MS = 24 * 60 * 60_000;
// How long a run of EITHER lane may sit at `postprocessing`. Unlike a render,
// packaging is bounded local CPU work measured in minutes, so grok's window is
// already generous and the lane makes no difference to it — but it is measured
// from `postprocessingStartedAt`, NOT `createdAt`; see the branch below.
const PACKAGING_STALE_MS = GROK_RENDER_STALE_MS;

/**
 * Read-time normalization of a run stuck at `rendering`, shared by both lanes
 * and never persisted.
 *
 * A run's status is flipped to a terminal state by its attach. So a run still
 * `rendering` long past when that could have happened is stranded — the server
 * died mid-render, and the in-memory session or awaiter went with it. Presenting
 * it as an error at read time is what stops the UI polling forever, surfaces
 * regenerate, and releases the in-flight guard that would otherwise refuse every
 * retry for that facing.
 *
 * For a LOCAL run the media-job queue is a better signal than the clock: it
 * persists and rehydrates across restarts, so it answers correctly both for a
 * render that has legitimately been going for hours and for one whose server
 * died. Only `dead` (failed / canceled, including the queue's own "interrupted
 * by restart") decides anything here:
 *
 *  - `settling` — the job finished and the attach is copying and hashing a
 *    20–80 MB clip — must NOT read as dead, or a poll landing in that window
 *    reports a SUCCESSFUL render as interrupted and, worse, unblocks the
 *    in-flight guard so a second multi-hour render can start for the same facing.
 *  - `unknown` must not either, or a live run errors the moment its job ages out
 *    of the archive.
 *
 * Both fall through to the (generous) local clock, which is the backstop for the
 * cases the completion hook cannot reach at all: an archive pruned while the
 * server was down for longer than its TTL, or an attach that threw.
 */
export const normalizeStaleAnimationRun = (run, errorMessage) => {
  // `postprocessing` needs the same treatment as `rendering`, and a TIGHTER
  // window: it is bounded LOCAL work (decode the clip, align, despill, pack), so
  // a run still in it long afterwards means the process doing that work died.
  // Both in-flight guards count `postprocessing` as busy, so without this a run
  // interrupted mid-package makes its facing permanently unrenderable — the same
  // dead end `rendering` had, one state further along. Both lanes, because the
  // packaging is identical on both.
  if (run?.status === 'postprocessing') {
    // Measured from when PACKAGING started, not from when the render was queued.
    // A local render can precede its packaging by hours, so a createdAt-anchored
    // window would report every healthy local run as interrupted the instant it
    // began to pack — and, because the in-flight guards read this normalized
    // status, would release them and invite a second multi-hour render for a
    // facing that already has a good candidate. Records written before this
    // field existed fall back to `createdAt`, which is what they were measured
    // against anyway.
    const startedAt = run.postprocessingStartedAt ?? run.createdAt;
    return Date.now() - runCreatedAtMs(startedAt) <= PACKAGING_STALE_MS
      ? run
      : { ...run, status: 'error', postprocessError: errorMessage };
  }
  if (run?.status !== 'rendering') return run;
  const local = isLocalProviderRun(run);
  if (local) {
    const liveness = localRunLiveness(run);
    if (liveness === 'live') return run;
    if (liveness === 'dead') return { ...run, status: 'error', postprocessError: errorMessage };
  }
  const staleAfterMs = local ? LOCAL_RENDER_STALE_MS : GROK_RENDER_STALE_MS;
  if (Date.now() - runCreatedAtMs(run.createdAt) <= staleAfterMs) return run;
  return { ...run, status: 'error', postprocessError: errorMessage };
};