Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

19.0 KB
Raw Blame
/**
 * Media-domain creative tools (#2183). Conductor wrappers over the media job
 * queue. Each enqueue tags the job's `owner` back to the calling project so the
 * orchestration's renders are attributable via `listJobs({ owner })`.
 */

import { z } from 'zod';
import { enqueueUnattendedMediaJob, hasConfiguredMediaRoute } from '../../federatedMedia/defaultRouting.js';
import { ASPECT_PRESETS, QUALITY_PRESETS, presetToRenderParams } from '../../../lib/creativeDirectorPresets.js';
import { getSettings } from '../../settings.js';
import { IMAGE_GEN_MODE, resolveQueueImageMode } from '../../imageGen/modes.js';
import { renderTargetDefaults, resolveRenderTargetConfig } from '../../imageGen/cloudProviderConfig.js';
import { RENDER_TARGET } from '../../../lib/renderTargets.js';
import { CLOUD_VIDEO_GEN_MODES } from '../../videoGen/modes.js';
import { videoBackendJobParams, resolveVideoBackendPin } from '../../videoGen/backendPin.js';
import { getDefaultVideoModelId, getVideoModels } from '../../../lib/mediaModels.js';
import { COST_RENDER, resolveOwner } from './shared.js';

const paramsSchema = z.object({ params: z.record(z.any()).default({}), owner: z.string().optional() });

/**
 * Force a video render's dimensions + quality knobs onto the project's LOCKED
 * preset, overriding whatever the planner LLM authored.
 *
 * The planner (cd-plan) writes `media_enqueueVideoJob` params freehand and has
 * historically guessed an `aspectRatio` string (e.g. "16:9") that the video
 * worker doesn't even read — `generateVideo` consumes `width`/`height`, not
 * `aspectRatio`, and defaults to a 768×512 box when they're absent. That silently
 * dropped the project's chosen 9:16. A project's aspect ratio / quality / target
 * duration are locked at creation (see creativeDirectorPresets.js), so resolve
 * them deterministically here rather than trusting the LLM to reproduce them.
 *
 * The planner still owns the CREATIVE params (`prompt`, `negativePrompt`,
 * `style`). Commission duration is form-locked; a general CD project may use
 * shorter per-step beats. Only render controls are enforced. An
 * unrecognized aspect/quality (hand-edited/legacy project) falls through to the
 * LLM's params untouched — best-effort, never throws.
 */
function enforceVideoRenderPreset(params, project) {
  // Only a directive-driven CD project locks a preset; a bare enqueue (no
  // recognized aspect/quality) keeps the caller's params as-is.
  if (!project || !ASPECT_PRESETS[project.aspectRatio] || !QUALITY_PRESETS[project.quality]) {
    return params;
  }
  // The planner may legitimately ask for a shorter beat than the project target,
  // so a positive per-step durationSeconds wins; otherwise use the project's.
  const targetAbility = project.directive?.constraints?.targetAbility;
  const commissionLocked = targetAbility === 'video' || targetAbility === 'music-video';
  const stepDuration = Number(params?.durationSeconds);
  const durationSeconds = commissionLocked
    ? (project.targetDurationSeconds || 10)
    : (stepDuration > 0 ? stepDuration : (project.targetDurationSeconds || 10));
  const preset = presetToRenderParams({
    aspectRatio: project.aspectRatio,
    quality: project.quality,
    durationSeconds,
  });
  // Drop the worker-ignored `aspectRatio` key so a stale value can't mislead, and
  // force the locked geometry + quality-derived knobs.
  const { aspectRatio: _ignored, ...rest } = params || {};
  return {
    ...rest,
    durationSeconds,
    width: preset.width,
    height: preset.height,
    fps: preset.fps,
    numFrames: preset.numFrames,
    steps: preset.steps,
    guidanceScale: preset.guidanceScale,
  };
}

function enforceImageRenderPreset(params, project) {
  const targetAbility = project?.directive?.constraints?.targetAbility;
  if ((targetAbility !== 'image' && targetAbility !== 'music-video')
      || !ASPECT_PRESETS[project.aspectRatio] || !QUALITY_PRESETS[project.quality]) {
    return params;
  }
  const aspect = ASPECT_PRESETS[project.aspectRatio];
  const quality = QUALITY_PRESETS[project.quality];
  const { aspectRatio: _ignored, ...rest } = params || {};
  return {
    ...rest,
    width: aspect.width,
    height: aspect.height,
    steps: quality.steps,
    guidance: quality.guidance,
    cfgScale: quality.guidance,
  };
}

export function reconcileVideoParamsWithModel(params, project, models = getVideoModels()) {
  const locked = enforceVideoRenderPreset(params, project);
  if (locked === params) return params;
  const requestedModelId = locked.modelId || project.modelId || getDefaultVideoModelId();
  const model = models.find((entry) => entry.id === requestedModelId) || null;
  const aspectValue = locked.width / locked.height;
  const resolution = Array.isArray(model?.resolutionOptions) && model.resolutionOptions.length
    ? model.resolutionOptions.reduce((best, option) => {
      const distance = Math.abs((Number(option.w) / Number(option.h)) - aspectValue);
      return !best || distance < best.distance ? { option, distance } : best;
    }, null)?.option
    : null;
  const fpsOptions = Array.isArray(model?.fpsOptions) ? model.fpsOptions.filter(Number.isFinite) : [];
  const fps = fpsOptions.includes(locked.fps) ? locked.fps : (fpsOptions[0] || locked.fps);
  const requestedFrames = Math.max(1, Math.round((locked.numFrames / locked.fps) * fps));
  const frameOptions = Array.isArray(model?.frameOptions) ? model.frameOptions.filter(Number.isFinite) : [];
  const numFrames = frameOptions.length
    ? frameOptions.reduce((best, value) => (
      Math.abs(value - requestedFrames) < Math.abs(best - requestedFrames) ? value : best
    ))
    : locked.numFrames;
  const reconciled = {
    ...locked,
    modelId: requestedModelId,
    width: Number(resolution?.w) || locked.width,
    height: Number(resolution?.h) || locked.height,
    fps,
    numFrames,
  };
  if (model?.samplerLocked) {
    delete reconciled.steps;
    delete reconciled.guidanceScale;
  }
  if (model?.supportsNegativePrompt === false) delete reconciled.negativePrompt;
  if (model?.supportsDisableAudio === false) delete reconciled.disableAudio;
  if (model?.supportsTiling === false) delete reconciled.tiling;
  return reconciled;
}

/**
 * Tag a planner-enqueued audio job so the durable `creativeDirectorMusicBedHook`
 * files the finished track onto the owning project's `musicBed` field (#2772).
 *
 * Without this, a `music` commission's plan step enqueues an audio job that
 * completes with only a job id — `project.musicBed` stays null and the run has
 * no surfaced, rateable output. The first-pass music flow
 * (creativeDirector/firstPassMusicGen.js) already stamps this exact tag; the
 * planner path did not, so the hook never fired for planner-driven audio.
 *
 * Only tags inside a CD project context (`ctx.projectId`) — a bare enqueue with
 * no owning project keeps its params untouched. An explicit
 * `creativeDirectorMusicBed` already on the params (a caller that set its own
 * destination) wins and is left as-is.
 */
async function configureMusicJob(params, ctx, project) {
  if (!ctx?.projectId) return params;
  if ((ctx.targetAbility === 'music' || ctx.targetAbility === 'music-video') && !project) {
    throw new Error('commission-project-unavailable');
  }
  const { getCommissionMusicContextForProject } = await import('../../creativeCommissions/store.js');
  // Do not collapse a failed local provenance lookup into "not a taste run".
  // That would let planner-authored renderer/prompt guesses escape onto an
  // opted-in commission precisely when the local store is unavailable.
  const context = await getCommissionMusicContextForProject(ctx.projectId);
  if (!context) {
    const targetAbility = project?.directive?.constraints?.targetAbility;
    if (targetAbility !== 'music' && targetAbility !== 'music-video') {
      if (params?.creativeDirectorMusicBed) return params;
      return { ...params, creativeDirectorMusicBed: { projectId: ctx.projectId } };
    }
    const {
      engine: _plannerEngine,
      modelId: _plannerModel,
      repo: _plannerRepo,
      durationSec: _plannerDuration,
      durationMode: _plannerDurationMode,
      provenance: _plannerProvenance,
      ...rest
    } = params || {};
    return {
      ...rest,
      durationSec: project.targetDurationSeconds,
      creativeDirectorMusicBed: { projectId: ctx.projectId },
    };
  }
  if (typeof context.prompt !== 'string' || !context.prompt.trim()) {
    throw new Error('taste-commission-prompt-unavailable');
  }
  const {
    engine: _plannerEngine,
    modelId: _plannerModel,
    repo: _plannerRepo,
    durationSec: _plannerDuration,
    durationMode: _plannerDurationMode,
    provenance: _plannerProvenance,
    ...rest
  } = params || {};
  return {
    ...rest,
    prompt: context.prompt,
    ...context.musicGeneration,
    creativeDirectorMusicBed: { projectId: ctx.projectId },
    provenance: {
      kind: 'creative-commission-music-taste',
      commissionId: context.commissionId,
      runId: context.runId,
      recipeVersion: context.tasteRecipe.version,
      sourceHash: context.tasteRecipe.sourceHash,
    },
  };
}

/**
 * Force an image/video render onto the project's PINNED render backend (#3135),
 * overriding whatever `mode` the planner LLM authored.
 *
 * Backend dispatch in the media job queue is purely `job.params.mode`-driven
 * (`mediaJobQueue/index.js#getGenModuleForJob`). The planner writes these params
 * freehand and — before #3135 — never set `mode` at all, so every plan-driven
 * render silently landed on local diffusion regardless of the user's intent. A
 * creative commission can now pin the backend per commission
 * (`generation.imageMode` / `.videoMode`); `buildRenderBackendPin` carries it onto
 * the project as `renderBackend`, and this is where it beats the LLM.
 *
 * No pin (the default, and every pre-#3135 project) ⇒ returns `params`
 * UNTOUCHED — not "auto-resolved to something", untouched — so the enqueued job
 * is byte-identical to today and dispatch falls through to the install-wide
 * default the same way it always has.
 *
 * A pin is re-resolved against LIVE settings rather than trusted verbatim: the
 * cloud backends are gated on their `imageGen.<mode>.enabled` toggle, which the
 * user can flip off long after pinning. `resolveQueueImageMode` /
 * `resolveVideoMode` degrade an unusable pin to a usable backend (never throw) —
 * a nightly commission must still produce something rather than failing every
 * fire because a toggle moved. Cloud modes additionally need their provider knob
 * bundle (`codexPath`/`model`/`effort`, `grokPath`/`aspectRatio`) in the job
 * params, since the queue dispatches straight to the provider module and skips
 * the imageGen dispatcher that would otherwise assemble them.
 *
 * Deliberately NOT resolved here: the saved `cleanC2PA`/`denoise` post-processing
 * settings (which `pipeline/visualStageHelpers.js#enqueueImageJob` does resolve,
 * for the same skips-the-dispatcher reason). No planner-enqueued render has ever
 * carried them — pinned or not — so resolving them only on the pinned path would
 * make the same commission post-process differently depending on whether a
 * backend happens to be pinned. Fixing it for BOTH paths would break the "auto is
 * byte-identical" contract this change rests on, so it's a separate concern.
 */
async function enforceRenderBackendPin(kind, params, project) {
  const settings = await getSettings().catch(() => null);
  if (!settings) return params;

  // Each lane has a pin ladder (#3231): the project's own renderBackend pin
  // (per-record, wins) → the install-wide creative-agent renderDefaults pin
  // → (video only, Phase 4) the install-wide `settings.videoGen.mode` pin.
  // When NO source pins the lane, params pass through untouched — the "auto
  // is byte-identical" contract above.
  if (kind === 'video') {
    // The video ladder is shared verbatim with the scene-render path via
    // videoGen/backendPin.js, so the two enqueue surfaces resolve identically.
    const videoPin = resolveVideoBackendPin(project, settings);
    if (!videoPin.pinned) return params;
    return videoBackendJobParams(settings, videoPin, params, {
      durationSeconds: project?.targetDurationSeconds,
    });
  }

  const pin = project?.renderBackend?.[kind];
  const targetDefaults = renderTargetDefaults(settings, RENDER_TARGET.CREATIVE_AGENT);
  if (!pin?.mode && !targetDefaults.imageMode) return params;

  // Project pin (explicit, per-record) → creative-agent renderDefaults pin —
  // both usability-laddered so a disabled backend degrades instead of failing
  // a nightly commission. One ladder covers text-to-image and edit/reference
  // renders alike: every queueable backend accepts input images.
  const mode = resolveQueueImageMode(pin?.mode || targetDefaults.imageMode, settings);
  // Mode is laddered above; this threads the creative-agent renderDefaults
  // imageModel into the cloud job params (#3231). The project's own
  // `pin.modelId` is deliberately NOT passed as the cloud model override —
  // commissions pick it from the LOCAL diffusion catalog, so it only applies
  // on the local branch below.
  const { cloud } = resolveRenderTargetConfig(settings, RENDER_TARGET.CREATIVE_AGENT, { mode });
  if (cloud) return { ...params, ...cloud.jobParams };
  return {
    ...params,
    mode: IMAGE_GEN_MODE.LOCAL,
    pythonPath: settings.imageGen?.local?.pythonPath || null,
    // The user's pinned model wins over the planner's freehand guess.
    ...(pin?.modelId ? { modelId: pin.modelId } : {}),
  };
}

/**
 * Condition a LOCAL image render on the owning project's style reference
 * images (#8724) — the universe / mood-board images a Creative Commission's
 * style source resolved at fire time.
 *
 * Runs after the backend pin, so `params.mode` is final: absent means the
 * queue's local default, which is also what an unpinned project renders on.
 * Cloud modes are left alone — their "input image" wiring edits that image
 * rather than borrowing its style. A planner that already chose its own
 * `referenceImagePaths` wins. A project without style references (every bare
 * CD project) returns `params` untouched.
 */
async function applyStyleReferenceImages(params, project) {
  if (!project?.styleReferenceImages?.length) return params;
  if (params?.mode && params.mode !== IMAGE_GEN_MODE.LOCAL) return params;
  if (Array.isArray(params?.referenceImagePaths) && params.referenceImagePaths.length) return params;
  const { styleReferenceImagePaths } = await import('../../creativeStyleSources.js');
  const referenceImagePaths = styleReferenceImagePaths(project.styleReferenceImages);
  return referenceImagePaths.length ? { ...params, referenceImagePaths } : params;
}

// Load the owning CD project ONCE per enqueue — both the video preset
// reconciliation and the render-backend pin read from it, and re-reading would
// double the store round-trips per plan step. Null outside a project context (a
// bare enqueue) or when the read fails: both forcing steps treat that as "no
// project locks anything" and pass the caller's params through.
async function loadOwningProject(ctx) {
  if (!ctx?.projectId) return null;
  const { getProject } = await import('../../creativeDirector/local.js');
  return getProject(ctx.projectId).catch(() => null);
}

const mediaTool = (kind, label) => ({
  name: `media_enqueue${label}Job`,
  description: `Enqueue a ${kind} media job. Long-running: returns a job handle; completion arrives via media-job events. Tags the job owner to the calling project.`,
  costClass: COST_RENDER,
  longRunning: true,
  schema: paramsSchema,
  parameters: {
    type: 'object',
    properties: {
      params: { type: 'object', description: `Job parameters for the ${kind} worker.` },
      owner: { type: 'string', description: 'Optional explicit owner tag (defaults to the calling project).' },
    },
    required: ['params'],
  },
  execute: async (args, ctx) => {
    let params = args.params || {};
    const owningProject = await loadOwningProject(ctx);
    let owningVideo = owningProject;
    if (owningVideo?.workspace === 'video') {
      if (kind !== 'video' || !ctx.videoStepId) throw new Error('Video production requires a bounded video plan step.');
      const { effectiveVideoProject } = await import('../../creativeDirector/videoExecution.js');
      owningVideo = effectiveVideoProject(owningVideo);
    } else owningVideo = null;
    if (kind === 'audio') {
      params = await configureMusicJob(params, ctx, owningProject);
    } else {
      // Image + video both consult the owning project: video for its locked
      // geometry preset, both for a pinned render backend (#3135).
      const project = owningVideo || owningProject;
      if (ctx.targetAbility && !project) throw new Error('commission-project-unavailable');
      if (kind === 'video') params = enforceVideoRenderPreset(params, project);
      if (kind === 'image') params = enforceImageRenderPreset(params, project);
      params = await enforceRenderBackendPin(kind, params, project);
      if (kind === 'image') params = await applyStyleReferenceImages(params, project);
      // Resolve the selected local model AFTER the backend ladder has applied
      // project/install pins. The same model-catalog fields that drive Video
      // Gen's visible controls then snap the autonomous job onto that model's
      // canvas/FPS/frame options and remove controls the UI disables (for
      // example MiniMax H3's negative prompt and sampler knobs).
      //
      // SKIPPED for a routed job: the local model is not the one rendering, so
      // snapping to its catalog is meaningless — and actively harmful, because
      // it DELETES controls that model doesn't support. A dropped
      // `disableAudio: true` is invisible to the route's own guard, which would
      // then happily ship a job the peer renders with audio; a dropped
      // `negativePrompt` just silently changes the render.
      if (kind === 'video'
        && !CLOUD_VIDEO_GEN_MODES.includes(params?.mode)
        && !(await hasConfiguredMediaRoute('video'))) {
        params = reconcileVideoParamsWithModel(params, project);
      }
    }
    // Unattended jobs never name a peer — the planner is an LLM, and letting
    // it pick one is exactly the arbitrary-peer routing the provider contract
    // forbids. Routing comes from this instance's own settings instead, applied
    // by the shared unattended enqueue helper (which every autonomous render
    // path goes through, so a configured route can't apply to a project's
    // planner renders but not its scene renders). It resolves LAST, after the
    // project/install pin ladder above: a configured remote provider overrides
    // those local backend choices, and an unrouted job is unaffected.
    if (owningVideo) {
      const { enqueueVideoProductionJob } = await import('../../creativeDirector/videoExecution.js');
      const queued = await enqueueVideoProductionJob(owningVideo, { kind, params, stepId: ctx.videoStepId });
      if (!queued) throw new Error('Video production is paused or its execution limit is exhausted.');
      return queued;
    }
    return enqueueUnattendedMediaJob({ kind, params, owner: resolveOwner(args, ctx) });
  },
});

export const MEDIA_TOOLS = [
  mediaTool('image', 'Image'),
  mediaTool('video', 'Video'),
  mediaTool('audio', 'Audio'),
];