Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

11.8 KB
Raw Blame
import { useEffect, useMemo, useRef, useState } from 'react';
import { Code2, LoaderCircle, Music2 } from 'lucide-react';
import CodeAnimationCodeScore from './CodeAnimationCodeScore';
import toast from '../ui/Toast';
import ModelInputPreview from '../media/ModelInputPreview';
import ListLoadError from '../ui/ListLoadError';
import useListResource from '../../hooks/useListResource';
import useMediaJobProgress from '../../hooks/useMediaJobProgress';
import { useModelInputPreview } from '../../hooks/useModelInputPreview';
import { generateCodeAnimationSoundtrack, listMusicEngines, previewMusicPrompt } from '../../services/api';
import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage';

// The last mode (music model or coding model) and each mode's picks, reused for
// every animation.
export const SOUNDTRACK_CHOICE_KEY = 'portos.codeAnimation.soundtrackChoice';
// Long-form engines first when nothing has been picked yet.
const PREFERRED_ENGINES = ['acestep15', 'acestep', 'minimax-music3-mlx', 'minimax-music3'];

const inputClass = 'w-full rounded-lg border border-port-border bg-port-bg px-3 py-2 text-sm text-white placeholder:text-gray-600 focus:border-port-accent focus:outline-none';
const labelClass = 'mb-1 block text-xs text-gray-400';
const buttonGhost = 'inline-flex items-center gap-2 rounded-lg border border-port-border px-3 py-1.5 text-sm text-gray-300 hover:text-white disabled:cursor-not-allowed disabled:opacity-40';
const buttonPrimary = 'inline-flex items-center gap-2 rounded-lg bg-port-accent px-3 py-1.5 text-sm text-white hover:opacity-90 disabled:cursor-not-allowed disabled:opacity-40';

const readChoice = () => {
  const stored = safeReadJsonStorage(SOUNDTRACK_CHOICE_KEY, null);
  return stored && typeof stored === 'object' ? stored : {};
};

const fetchEngines = async () => (await listMusicEngines({ silent: true }))?.engines;

const fits = (engine, filmSec) => !Number.isFinite(engine.maxDurationSec) || engine.maxDurationSec >= filmSec;

function engineNote(engine, filmSec) {
  if (!engine.ready) return 'not installed';
  if (!fits(engine, filmSec)) return `max ${engine.maxDurationSec}s`;
  return null;
}

const MODES = [
  { id: 'model', label: 'Music model', icon: Music2 },
  { id: 'code', label: 'Coding model', icon: Code2 },
];

/**
 * Generate audio for a finished animation at the film's length, either with a
 * local music model or as code (Strudel / Tone.js / SuperCollider) a coding
 * model writes from the animation script. The animation is not re-run; the
 * track becomes its audio for the preview and the MP4 export.
 */
export default function CodeAnimationSoundtrack({ job, jobId, soundtrack, audioUrl, audioTrackId, durationSeconds, onReady }) {
  const filmSec = Math.ceil(Number(durationSeconds) || 0);
  const engineList = useListResource(fetchEngines);
  const engines = engineList.state === 'ready' ? engineList.items : null;
  const [choice, setChoice] = useState(readChoice);
  const [prompt, setPrompt] = useState(soundtrack?.prompt || '');
  const [starting, setStarting] = useState(false);
  const [mediaJobId, setMediaJobId] = useState(soundtrack?.mediaJobId || null);
  // Only a render started here counts as live before its first status read; a
  // persisted id the queue no longer knows (archived, lost on restart) is idle.
  const [startedId, setStartedId] = useState(null);

  useEffect(() => {
    setPrompt(soundtrack?.prompt || '');
    setMediaJobId(soundtrack?.mediaJobId || null);
  }, [jobId, soundtrack?.mediaJobId]);

  const usable = useMemo(() => (engines || []).filter((engine) => engine.ready && fits(engine, filmSec)), [engines, filmSec]);
  const engine = useMemo(() => {
    const list = engines || [];
    return list.find((item) => item.id === choice.engine)
      || PREFERRED_ENGINES.map((id) => usable.find((item) => item.id === id)).find(Boolean)
      || usable[0]
      || list[0]
      || null;
  }, [engines, usable, choice.engine]);
  const models = engine?.models || [];
  const modelId = models.some((model) => model.id === choice.modelId) ? choice.modelId : (engine?.defaultModelId || models[0]?.id || '');
  const instrumentalOnly = choice.instrumentalOnly !== false;
  const mode = choice.mode === 'code' && job ? 'code' : 'model';
  const engineBlocked = engine ? engineNote(engine, filmSec) : null;

  const updateChoice = (patch) => setChoice((current) => {
    const next = { ...current, ...patch };
    safeWriteJsonStorage(SOUNDTRACK_CHOICE_KEY, next);
    return next;
  });

  const captionPreview = useModelInputPreview(previewMusicPrompt, {
    prompt, engine: engine?.id, instrumentalOnly, durationSec: filmSec || undefined,
  }, { enabled: mode === 'model' && !!engine && !!prompt.trim() });

  const progress = useMediaJobProgress(mediaJobId, { kind: 'audio' });
  const live = progress.status === 'queued' || progress.status === 'running' || (!!mediaJobId && mediaJobId === startedId && progress.status === 'unknown');
  // Refetch the animation once the take lands so the preview and export use it.
  const settledRef = useRef(null);
  useEffect(() => {
    if (!mediaJobId || progress.status !== 'completed' || settledRef.current === mediaJobId) return;
    settledRef.current = mediaJobId;
    onReady?.();
  }, [mediaJobId, progress.status, onReady]);

  const canGenerate = !!engine && !engineBlocked && !!prompt.trim() && !starting && !live && filmSec > 0;

  const handleGenerate = async () => {
    if (!canGenerate) return;
    setStarting(true);
    const started = await generateCodeAnimationSoundtrack(jobId, {
      prompt,
      engine: engine.id,
      ...(modelId ? { modelId } : {}),
      instrumentalOnly,
    }, { silent: true }).catch((error) => {
      toast.error(error.message || 'Failed to start the soundtrack');
      return null;
    });
    setStarting(false);
    if (started?.mediaJobId) {
      setStartedId(started.mediaJobId);
      setMediaJobId(started.mediaJobId);
    }
  };

  const percent = Math.round((progress.progress || 0) * 100);
  // The job's audio is the generated track only once a take has rendered onto it.
  const hasTake = !!soundtrack?.trackId && !!audioUrl && audioTrackId === soundtrack.trackId;

  return (
    <section className="space-y-3 rounded-lg border border-port-border bg-port-card p-3" aria-labelledby="ca-soundtrack-heading">
      <h3 id="ca-soundtrack-heading" className="flex items-center gap-2 text-sm font-semibold text-white">
        <Music2 className="h-4 w-4 text-port-accent" /> Soundtrack
      </h3>
      {job && (
        <div role="group" aria-label="Make the audio with" className="flex flex-wrap gap-2">
          {MODES.map(({ id, label, icon: Icon }) => (
            <button key={id} type="button" aria-pressed={mode === id} disabled={live}
              onClick={() => updateChoice({ mode: id })} className={mode === id ? buttonPrimary : buttonGhost}>
              <Icon className="h-4 w-4" /> {label}
            </button>
          ))}
        </div>
      )}
      {mode === 'model' && (
        <div className="grid gap-3 sm:grid-cols-2">
          <div>
            <label htmlFor="ca-soundtrack-engine" className={labelClass}>Music model</label>
            <select id="ca-soundtrack-engine" value={engine?.id || ''} disabled={!engines?.length || live}
              onChange={(event) => updateChoice({ engine: event.target.value, modelId: '' })} className={inputClass}>
              {engines === null && <option value="">{engineList.state === 'error' ? 'Models unavailable' : 'Loading models…'}</option>}
              {engines?.length === 0 && <option value="">No music models</option>}
              {(engines || []).map((item) => {
                const note = engineNote(item, filmSec);
                return <option key={item.id} value={item.id}>{item.name}{note ? ` (${note})` : ''}</option>;
              })}
            </select>
          </div>
          {models.length > 1 && (
            <div>
              <label htmlFor="ca-soundtrack-model" className={labelClass}>Checkpoint</label>
              <select id="ca-soundtrack-model" value={modelId} disabled={live}
                onChange={(event) => updateChoice({ modelId: event.target.value })} className={inputClass}>
                {models.map((model) => <option key={model.id} value={model.id}>{model.name || model.id}</option>)}
              </select>
            </div>
          )}
        </div>
      )}
      {mode === 'model' && engineList.state === 'error' && <ListLoadError what="music models" error={engineList.error} onRetry={engineList.retry} />}
      {mode === 'model' && engine && engineBlocked === 'not installed' && (
        <p className="text-xs text-port-warning">{engine.name} is not installed. Install it in Music › Generate, or pick another model.</p>
      )}
      {mode === 'model' && engine && engineBlocked && engineBlocked !== 'not installed' && (
        <p className="text-xs text-port-warning">The film is {filmSec}s; {engine.name} renders at most {engine.maxDurationSec}s. Pick a model marked for longer audio.</p>
      )}
      <div>
        <label htmlFor="ca-soundtrack-prompt" className={labelClass}>{mode === 'code' ? 'Describe the sound (optional)' : 'Describe the music'}</label>
        <textarea id="ca-soundtrack-prompt" rows={3} value={prompt} maxLength={8000} disabled={live}
          onChange={(event) => setPrompt(event.target.value)}
          placeholder={mode === 'code'
            ? 'Mostly sound effects: whirrs, blips and boops over a soft synth pad'
            : 'Warm analog synths, slow build, soft drums from 0:20, gentle fade at the end'}
          className={`${inputClass} resize-y`} />
      </div>
      {mode === 'code' ? (
        <CodeAnimationCodeScore
          job={job}
          trackId={soundtrack?.trackId}
          description={prompt}
          filmSec={filmSec}
          pin={choice.code}
          onPinChange={(code) => updateChoice({ code })}
          onSaved={() => onReady?.()}
        />
      ) : (
        <>
          <ModelInputPreview preview={captionPreview} prompt={prompt} />
          <label className="flex items-center gap-2 text-xs text-gray-300">
            <input type="checkbox" checked={instrumentalOnly} disabled={live} onChange={(event) => updateChoice({ instrumentalOnly: event.target.checked })} />
            Instrumental only
          </label>
          <div className="flex flex-wrap items-center gap-3">
            <button type="button" onClick={handleGenerate} disabled={!canGenerate} className={buttonPrimary}>
              {starting || live ? <LoaderCircle className="h-4 w-4 animate-spin" /> : <Music2 className="h-4 w-4" />}
              {live ? `Generating audio ${percent}%` : hasTake ? 'Regenerate audio' : 'Generate audio'}
            </button>
            <span className="text-xs text-gray-500">{filmSec ? `${filmSec}s, the film's length` : ''}</span>
          </div>
        </>
      )}
      {live && (
        <div className="space-y-1" role="status" aria-label="Soundtrack progress">
          <div className="h-1.5 overflow-hidden rounded bg-port-border">
            <div className="h-full bg-port-accent transition-all" style={{ width: `${percent}%` }} />
          </div>
          <p className="text-xs text-gray-500">{progress.statusMsg || (progress.status === 'running' ? 'Rendering…' : 'Waiting in the audio queue…')}</p>
        </div>
      )}
      {mode === 'model' && progress.status === 'failed' && (
        <p role="alert" className="text-xs text-port-error">Audio failed: {progress.error || 'unknown error'}. Try again, or pick another model.</p>
      )}
      {mode === 'model' && progress.status === 'canceled' && <p className="text-xs text-gray-500">The audio render was canceled.</p>}
      {hasTake && !live && (
        <div className="space-y-1">
          <audio src={audioUrl} controls preload="metadata" className="w-full" aria-label="Generated soundtrack" />
          <p className="text-xs text-gray-500">This is the animation&apos;s audio now. Export MP4 includes it.</p>
        </div>
      )}
    </section>
  );
}