Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Add Animate generator support

Bitti63e2a2513b88
diff --git a/docs/video-references.md b/docs/video-references.md
index a5dcc0e..85521fc 100644
--- a/docs/video-references.md
+++ b/docs/video-references.md
@@ -1,5 +1,26 @@
# Video references
+## Animate
+
+Download **Animate** in Settings → Generators, then select it in the Generator.
+Choose a **Reference video** in the scene inspector. Animate requires a video
+and sends an empty text prompt; shot descriptions, look, sound and music prompts
+are not used. Video trim and soundtrack settings still come from References.
+
+The template uses `Viggle-Animate-pruned_rank8_int8_convrot.safetensors` and
+`viggle_animate_distillation_bf16.safetensors` from DeepBeepMeep/MiniMax-H3.
+The enabled distillation LoRA has a **4-step override**, which takes precedence
+over the scene's step count. Existing installations receive the template once
+without replacing their default generator or local model paths.
+
+This uses the DLL's generic H3 reference-generation API with the Viggle
+checkpoint. Its shared text encoder and video/audio VAEs are still required.
+It does not reproduce upstream Viggle's frozen-embedding character-replacement
+pipeline. Planning is tested on both CUDA and Vulkan; rendered animation quality
+requires a run with the downloaded weights and a compatible GPU.
+
+## Adding video references
+
In **References**, choose **Add a reference → Other → New** to create an
**Uncategorized** reference, then choose **Add file** in its details. Video
can also be attached to an existing reference. The reference card shows the
diff --git a/src-tauri/src/slopfab.rs b/src-tauri/src/slopfab.rs
index 039a92c..06b9c57 100644
--- a/src-tauri/src/slopfab.rs
+++ b/src-tauri/src/slopfab.rs
@@ -379,9 +379,10 @@ impl SlopfabRuntime {
"inputChars": request.prompt.chars().count(),
}),
);
- if request.job_id.trim().is_empty() || request.prompt.trim().is_empty() {
- return Err("Generation job id and prompt cannot be empty.".into());
+ if request.job_id.trim().is_empty() {
+ return Err("Generation job id cannot be empty.".into());
}
+ configuration.validate_inputs(&request)?;
validate_generation_controls(&request)?;
let cancel = Arc::new(AtomicBool::new(false));
let mut flags = self
@@ -472,6 +473,7 @@ struct Configuration {
dll_path: PathBuf,
vulkan: bool,
attention: &'static str,
+ animate: bool,
models: [(i32, &'static str, Option<PathBuf>); 5],
loras: Result<Vec<LoraAdapter>, String>,
step_override: Result<Option<i32>, String>,
@@ -500,6 +502,7 @@ impl Configuration {
// working; with none, use the DLL beside the executable.
dll_path: option("dllPath").unwrap_or_else(default_dll_path),
vulkan: string_option("inferenceBackend") == Some("vulkan"),
+ animate: string_option("generationMode") == Some("animate"),
attention: match string_option("attention") {
Some("exact") => "exact",
Some("flash2") => "flash2",
@@ -526,6 +529,20 @@ impl Configuration {
}
}
+ fn validate_inputs(&self, request: &GenerationRequest) -> Result<(), String> {
+ if self.animate {
+ if request.still_image {
+ return Err("Animate requires video generation.".into());
+ }
+ if request.reference_video_ids.is_empty() {
+ return Err("Animate requires a reference video.".into());
+ }
+ } else if request.prompt.trim().is_empty() {
+ return Err("Generation prompt cannot be empty.".into());
+ }
+ Ok(())
+ }
+
fn generation_steps(&self, fallback: i32) -> Result<i32, String> {
self.step_override.clone().map(|steps| steps.unwrap_or(fallback))
}
@@ -693,7 +710,8 @@ fn configure_request(
platform: ComputePlatform,
include_models: bool,
) -> Result<(), String> {
- api.set_prompt(handle, &request.prompt)?;
+ configuration.validate_inputs(request)?;
+ api.set_prompt(handle, if configuration.animate { "" } else { &request.prompt })?;
api.set_frames(handle, request.frames)?;
if request.still_image {
api.set_still_image(handle)?;
@@ -1061,6 +1079,7 @@ pub fn generate_reference_icon_batch(
step_override: Ok(None),
vulkan: backend == ComputePlatform::Vulkan,
attention: "sage2",
+ animate: false,
models: [
(0, "transformer", Some(batch.transformer.clone())),
(1, "textEncoder", Some(batch.text_encoder.clone())),
@@ -1760,6 +1779,53 @@ mod ffi {
mod tests {
use super::*;
+ #[test]
+ fn animate_requires_video_and_prompt_mode_requires_text() {
+ let mut configuration = Configuration::from_settings(&BTreeMap::new());
+ let mut request = GenerationRequest::default();
+ assert!(configuration.validate_inputs(&request).unwrap_err().contains("prompt"));
+ configuration.animate = true;
+ assert!(configuration.validate_inputs(&request).unwrap_err().contains("reference video"));
+ request.reference_paths.push("image.png".into());
+ assert!(configuration.validate_inputs(&request).is_err());
+ request.reference_video_ids.push("video".into());
+ configuration.validate_inputs(&request).unwrap();
+ request.still_image = true;
+ assert!(configuration.validate_inputs(&request).unwrap_err().contains("video generation"));
+ }
+
+ #[test]
+ fn animate_empty_prompt_and_four_steps_reach_bundled_dll() {
+ let settings = BTreeMap::from([("slopfab".into(), ProviderSetting {
+ enabled: true, model: None,
+ options: BTreeMap::from([
+ ("generationMode".into(), ProviderOption::String("animate".into())),
+ ("stepOverride".into(), ProviderOption::Number(4.0)),
+ ]),
+ })]);
+ let configuration = Configuration::from_settings(&settings);
+ let api = ffi::Api::load(&configuration.dll_path).unwrap();
+ let id = create_reference_video(2.0, &settings).unwrap();
+ let result = (|| -> Result<(), String> {
+ append_reference_video(&id, &vec![127; 64 * 64 * 4], 64, 64, 0.0)?;
+ let request = GenerationRequest {
+ job_id: "animate-test".into(), frames: 48, steps: 20, seed: 1,
+ canvas_width: 736, canvas_height: 416, reference_video_ids: vec![id.clone()],
+ ..Default::default()
+ };
+ for platform in [ComputePlatform::Cuda13, ComputePlatform::Vulkan] {
+ for include_models in [false, true] {
+ let handle = RequestHandle::new(&api)?;
+ configure_request(&api, handle.0, &request, &configuration, platform, include_models)?;
+ assert_eq!(api.resolve(handle.0)?.num_model_evaluations, 3);
+ }
+ }
+ Ok(())
+ })();
+ release_reference_videos(&[id]).unwrap();
+ result.unwrap();
+ }
+
#[test]
fn continuation_uses_22_frames_and_plans_only_the_new_scene() {
// An independent 39-frame, 64x32 archive exercises the real DLL without
diff --git a/src/components/SettingsView.tsx b/src/components/SettingsView.tsx
index e271a25..4472212 100644
--- a/src/components/SettingsView.tsx
+++ b/src/components/SettingsView.tsx
@@ -302,12 +302,12 @@ export function SettingsView({ onClose, updates, initialTab = "engine" }: { onCl
const probe = useCallback((paths?: EngineSettings) => {
const revision = ++probeRevision.current;
- void getEngineStatus(paths, selectedTemplate.attention, selectedTemplate.loras ?? []).then((next) => {
+ void getEngineStatus(paths, selectedTemplate.attention, selectedTemplate.loras ?? [], selectedTemplate.mode ?? "prompt").then((next) => {
if (revision === probeRevision.current) setStatus(next);
}).catch(() => {
if (revision === probeRevision.current) setStatus(null);
});
- }, [selectedTemplate.attention, selectedTemplate.loras]);
+ }, [selectedTemplate.attention, selectedTemplate.loras, selectedTemplate.mode]);
/* Saved on every keystroke — there is no Save button here, so a half-typed
path must never be the reason generation is still broken after a restart.
@@ -372,7 +372,7 @@ export function SettingsView({ onClose, updates, initialTab = "engine" }: { onCl
probe({ ...EMPTY_ENGINE_SETTINGS });
};
- const updateTemplate = (updates: Partial<Pick<typeof selectedTemplate, "name" | "defaultSteps" | "attention" | "loras">>) => {
+ const updateTemplate = (updates: Partial<Pick<typeof selectedTemplate, "name" | "defaultSteps" | "attention" | "loras" | "mode">>) => {
setTemplateSettings((current) => {
const next = {
...current,
@@ -542,6 +542,14 @@ export function SettingsView({ onClose, updates, initialTab = "engine" }: { onCl
</div>
</header>
<div className="generator-template-fields">
+ <label>
+ <span>Mode</span>
+ <select aria-label="Generator mode" value={selectedTemplate.mode ?? "prompt"}
+ onChange={(event) => updateTemplate({ mode: event.target.value === "animate" ? "animate" : "prompt" })}>
+ <option value="prompt">Text prompt</option>
+ <option value="animate">Animate (reference video)</option>
+ </select>
+ </label>
<label>
<span>Name</span>
<input aria-label="Generator name" value={selectedTemplate.name} onChange={(event) => updateTemplate({ name: event.target.value })} onBlur={() => { if (!selectedTemplate.name.trim()) updateTemplate({ name: "Untitled generator" }); }} />
diff --git a/src/components/workspace/GeneratorView.tsx b/src/components/workspace/GeneratorView.tsx
index ce98701..d9675aa 100644
--- a/src/components/workspace/GeneratorView.tsx
+++ b/src/components/workspace/GeneratorView.tsx
@@ -52,13 +52,14 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
const jobs = config.generationJobs;
const [templateSettings, setTemplateSettings] = useState(loadGeneratorTemplateSettings);
const selectedTemplate = defaultGeneratorTemplate(templateSettings);
+ const animate = selectedTemplate.mode === "animate";
const defaultGenerationSteps = selectedTemplate.defaultSteps;
const [generatorRuntime, setGeneratorRuntime] = useState<SlopfabStatus | null>(runtime);
const [runtimeError, setRuntimeError] = useState<string | null>(null);
const runtimeProbe = useRef(0);
const [loraLibrary, setLoraLibrary] = useState(loadLoras);
useEffect(() => subscribeLoras(() => setLoraLibrary(loadLoras())), []);
- const templateKey = JSON.stringify([selectedTemplate.id, selectedTemplate.paths, selectedTemplate.attention, selectedTemplate.loras, loraLibrary]);
+ const templateKey = JSON.stringify([selectedTemplate.id, selectedTemplate.mode, selectedTemplate.paths, selectedTemplate.attention, selectedTemplate.loras, loraLibrary]);
const probedTemplate = useRef(templateKey);
const configRef = useRef(config);
configRef.current = config;
@@ -106,7 +107,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
setRuntimeError(null);
const probe = ++runtimeProbe.current;
if (!selectedTemplate.id) return;
- void getEngineStatus(selectedTemplate.paths, selectedTemplate.attention, selectedTemplate.loras ?? []).then((status) => {
+ void getEngineStatus(selectedTemplate.paths, selectedTemplate.attention, selectedTemplate.loras ?? [], selectedTemplate.mode ?? "prompt").then((status) => {
if (probe !== runtimeProbe.current) return;
setGeneratorRuntime(status);
onRuntimeChange?.(status);
@@ -123,7 +124,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
const selected = jobs.find((job) => job.id === selection.jobId) ?? jobs[0];
const selectedShots = useMemo(() => (selected ? sceneShots(selected) : []), [selected]);
const shotIndex = selectedShots.findIndex((shot) => shot.id === selection.shotId);
- const openShot = shotIndex >= 0 ? selectedShots[shotIndex] : null;
+ const openShot = !animate && shotIndex >= 0 ? selectedShots[shotIndex] : null;
const active = jobs.filter((job) => job.status === "generating" || job.status === "ready");
const queued = jobs.filter((job) => job.status === "queued");
@@ -301,7 +302,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
// retimed shot reach the engine, rather than sending a prompt frozen at
// draft-creation time. This is the ONE string slopfab is given, and it is
// the same string the compiled-prompt panel shows.
- prompt: compileGenerationJobPrompt(inputs.job, bound),
+ prompt: animate ? "" : compileGenerationJobPrompt(inputs.job, bound),
...(inputs.previousSceneId ? { previousSceneId: inputs.previousSceneId } : {}),
...(inputs.continuationRelativePath ? { continuationRelativePath: inputs.continuationRelativePath } : {}),
// The scene's own length, not a fixed six seconds.
@@ -385,7 +386,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
for (const [index, job] of jobs.entries()) {
if (job.status === "queued" || job.status === "generating" || job.status === "ready") continue;
if (job.usePreviousSceneLastFrame && index === 0) continue;
- if (sendBlocker(job, config.references)) continue;
+ if (sendBlocker(job, config.references, animate)) continue;
const previous = jobs[index - 1];
const previousWillRender = job.usePreviousSceneLastFrame && previous &&
(batchScenesReady.includes(previous) || ["queued", "generating", "ready"].includes(previous.status));
@@ -402,7 +403,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
if (!runtimeReady) return runtimeError ?? generatorRuntime?.detail ?? "The video generator is not ready.";
if (job.status === "ready") return "This scene is being saved now.";
if (job.usePreviousSceneLastFrame && jobs[0]?.id === job.id) return "This scene needs a previous scene to continue.";
- return sendBlocker(job, config.references);
+ return sendBlocker(job, config.references, animate);
};
const changedJobIds = new Set(jobs
@@ -558,6 +559,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
<div className="panel-scroll">
<SceneInspector
key={selected.id}
+ animate={animate}
job={selected}
shots={selectedShots}
defaultSteps={defaultGenerationSteps}
@@ -573,7 +575,7 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
/>
</div>
- {debugEnabled && <div className="debug-prompt">
+ {debugEnabled && !animate && <div className="debug-prompt">
<button
type="button"
className="secondary-button debug-prompt__toggle"
@@ -610,20 +612,20 @@ export function GeneratorView({ config, folderPath, runtime = null, generationCo
</div>;
}
-/** Why this scene cannot be handed to the engine, in the user's words, or null.
- * Each one is a real state of the scene, not a policy: a scene with no words
- * has nothing to render, a scene of no length is no frames at all, and a name
- * the prompt cannot cite would be silently dropped from the middle of a line. */
-function sendBlocker(job: GenerationJob, references: ProjectReference[]): string | null {
+/** Validate the inputs required by the selected generator before submission. */
+function sendBlocker(job: GenerationJob, references: ProjectReference[], animate = false): string | null {
const shots = sceneShots(job);
- if (shots.every((shot) => shot.action.trim().length === 0 && !(shot.speech ?? "").trim())) {
+ if (animate && !usableVideoReferences(references.filter((reference) => job.referenceIds.includes(reference.id))).length) {
+ return "Select a reference video for this scene before animating.";
+ }
+ if (!animate && shots.every((shot) => shot.action.trim().length === 0 && !(shot.speech ?? "").trim())) {
return "Describe what happens or add speech in at least one shot before this scene can be generated.";
}
if (sceneDurationSeconds(job) <= 0) {
return "This scene is nought seconds long, so there are no frames to render. Give it a length first.";
}
const dangling = danglingReferenceTokens(shots, references);
- if (dangling.length > 0) {
+ if (!animate && dangling.length > 0) {
return "A reference named in one of the lines can no longer be used, and would be left out of the prompt. Swap it for another one or take it out of the line first.";
}
return null;
diff --git a/src/components/workspace/SceneEditor.tsx b/src/components/workspace/SceneEditor.tsx
index f5df98b..8e410d2 100644
--- a/src/components/workspace/SceneEditor.tsx
+++ b/src/components/workspace/SceneEditor.tsx
@@ -6,6 +6,7 @@ import {
isReferenceUsable,
isVisualReference,
referenceImages,
+ usableVideoReferences,
normalizeSceneShots,
referenceToken,
sceneBriefText,
@@ -266,7 +267,8 @@ export function ShotInspector({ job, shots, shot, index, endsAt, duration, refer
/** Scene-wide render controls, the look the description opens with (base guide
* §4.1), and the two sound fields defined per prompt (§4.6, §4.7). Length
* lives in the scene header where it stays visible. */
-export function SceneInspector({ job, shots, references, previousScene, defaultSteps, disabled, importAvailable, importError, onAddStartFrame, onAddEndFrame, onChange, onShots }: {
+export function SceneInspector({ job, shots, references, previousScene, defaultSteps, disabled, importAvailable, importError, onAddStartFrame, onAddEndFrame, onChange, onShots, animate = false }: {
+ animate?: boolean;
job: GenerationJob;
shots: SceneShot[];
references: ProjectReference[];
@@ -292,7 +294,20 @@ export function SceneInspector({ job, shots, references, previousScene, defaultS
return <section className="scene-inspector" aria-label="This scene">
<section className="scene-settings" aria-labelledby={`${job.id}-scene-settings`}>
<h3 id={`${job.id}-scene-settings`}>Scene</h3>
- <label className="scene-settings__field">
+ {animate && <label className="scene-settings__field">
+ <span>Reference video</span>
+ <select aria-label="Reference video for this scene" disabled={disabled}
+ value={usableVideoReferences(references).find((reference) => job.referenceIds.includes(reference.id))?.id ?? ""}
+ onChange={(event) => onChange({ referenceIds: [
+ ...job.referenceIds.filter((id) => !references.some((reference) => reference.id === id && reference.kind === "video")),
+ ...(event.target.value ? [event.target.value] : []),
+ ] })}>
+ <option value="">Select a video</option>
+ {usableVideoReferences(references).map((reference) => <option key={reference.id} value={reference.id}>{reference.name}</option>)}
+ </select>
+ <small>Animate requires a reference video. Add videos under References. No text prompt is needed.</small>
+ </label>}
+ {!animate && <label className="scene-settings__field">
<span>Look</span>
<select
value={chosen}
@@ -311,7 +326,7 @@ export function SceneInspector({ job, shots, references, previousScene, defaultS
<option value="">None</option>
{look.options.map((option) => <option key={option.id} value={option.id}>{option.label}</option>)}
</select>
- </label>
+ </label>}
<div className="scene-settings__field">
<span>Start frame</span>
<div className="scene-settings__start-frame">
@@ -355,7 +370,7 @@ export function SceneInspector({ job, shots, references, previousScene, defaultS
<small>The selected image anchors the closing frame.</small>
</div>
{importError && <small className="scene-settings__error" role="alert">{importError}</small>}
- <label className="scene-settings__field">
+ {!animate && <><label className="scene-settings__field">
<span>Sound</span>
<textarea
value={job.soundscape ?? ""}
@@ -374,7 +389,7 @@ export function SceneInspector({ job, shots, references, previousScene, defaultS
placeholder="Instrumentation, tempo and dynamics — not a mood."
onChange={(event) => onChange({ music: event.target.value })}
/>
- </label>
+ </label></>}
</section>
<section className="scene-settings" aria-labelledby={`${job.id}-generation-settings`}>
<h3 id={`${job.id}-generation-settings`}>Generation</h3>
diff --git a/src/lib/loras.ts b/src/lib/loras.ts
index ac8d208..3d8d18c 100644
--- a/src/lib/loras.ts
+++ b/src/lib/loras.ts
@@ -19,6 +19,11 @@ export const TURBO_LORA: Lora = {
url: "https://huggingface.co/lightx2v/Minimax-h3-Turbo/resolve/main/minimax_h3_ref2v_turbo_4step_v0.1_comfyui_bf16.safetensors",
stepOverride: 4,
};
+export const VIGGLE_ANIMATE_LORA: Lora = {
+ id: "viggle-animate-4step", name: "Viggle Animate Distillation", path: "",
+ url: "https://huggingface.co/DeepBeepMeep/MiniMax-H3/resolve/main/loras/viggle_animate_distillation_bf16.safetensors",
+ stepOverride: 4,
+};
const KEY = "slopus.loras.v1";
const EVENT = "slopus:loras-changed";
export function loadLoras(): Lora[] {
@@ -36,8 +41,9 @@ export function loadLoras(): Lora[] {
}) : [];
if (!ids.has(TAOMATE_LORA.id)) entries.unshift({ ...TAOMATE_LORA });
if (!ids.has(TURBO_LORA.id)) entries.push({ ...TURBO_LORA });
+ if (!ids.has(VIGGLE_ANIMATE_LORA.id)) entries.push({ ...VIGGLE_ANIMATE_LORA });
return entries;
- } catch { return [{ ...TAOMATE_LORA }, { ...TURBO_LORA }]; }
+ } catch { return [{ ...TAOMATE_LORA }, { ...TURBO_LORA }, { ...VIGGLE_ANIMATE_LORA }]; }
}
export function saveLoras(loras: Lora[]): void {
if (loras.some((lora) => lora.stepOverride !== undefined && !isLoraStepOverride(lora.stepOverride))) {
diff --git a/src/lib/runtime.ts b/src/lib/runtime.ts
index 4c4bedc..b159f63 100644
--- a/src/lib/runtime.ts
+++ b/src/lib/runtime.ts
@@ -10,7 +10,7 @@ import {
import {
agentEndpointProviderSettings, endpointProviderSetting, engineProviderSetting,
generationStepsWithLoras, loadDefaultGenerationSteps, loadEngineSettings, withAgentEndpointSettings,
- type AttentionMode, type EndpointProviderId, type EndpointProviderSettings, type EnginePathField,
+ type AttentionMode, type EndpointProviderId, type EndpointProviderSettings, type EnginePathField, type GeneratorTemplate,
} from "./settings";
export type ProviderId = "claude" | "codex" | EndpointProviderId;
@@ -144,16 +144,16 @@ export const CHECKING_PROVIDERS: ProviderStatus[] = [
/** Probes the engine paths on their own, with no project in hand — what the
* settings screen shows. */
-export async function getEngineStatus(settings = loadEngineSettings(), attention?: AttentionMode, loras?: TemplateLora[]): Promise<SlopfabStatus> {
+export async function getEngineStatus(settings = loadEngineSettings(), attention?: AttentionMode, loras?: TemplateLora[], mode?: GeneratorTemplate["mode"]): Promise<SlopfabStatus> {
if (!isTauri()) return DEMO_STATUS.slopfab;
let slopfab;
let adapterError: string | undefined;
try {
- slopfab = engineProviderSetting(settings, undefined, attention, loras);
+ slopfab = engineProviderSetting(settings, undefined, attention, loras, mode);
} catch (reason) {
// A missing adapter must not prevent checking the individual model files.
adapterError = reason instanceof Error ? reason.message : String(reason);
- slopfab = engineProviderSetting(settings, undefined, attention, []);
+ slopfab = engineProviderSetting(settings, undefined, attention, [], mode);
}
const status = await invoke<SlopfabStatus>("slopfab_status", { settings: { slopfab } });
return adapterError ? { ...status, state: status.state === "ready" ? "modelsMissing" : status.state,
diff --git a/src/lib/settings.ts b/src/lib/settings.ts
index 8e1d451..5b84641 100644
--- a/src/lib/settings.ts
+++ b/src/lib/settings.ts
@@ -15,7 +15,7 @@
import { DEFAULT_GENERATION_STEPS, MAX_GENERATION_STEPS, type ProjectConfig, type ProviderSetting } from "./project";
// Type-only: erased at build time, so this does not close a cycle with runtime.ts.
import type { ProviderId } from "./runtime";
-import { downloadableTemplateLoras, highestLoraStepOverride, isLoraStepOverride, normalizeTemplateLoras, resolveTemplateLoras, subscribeLoras, TURBO_LORA, type TemplateLora } from "./loras";
+import { downloadableTemplateLoras, highestLoraStepOverride, isLoraStepOverride, normalizeTemplateLoras, resolveTemplateLoras, subscribeLoras, TURBO_LORA, VIGGLE_ANIMATE_LORA, type TemplateLora } from "./loras";
/* Light/dark appearance is machine-level for the same reason and lives in
* ./theme.ts, which is separate only because index.html has to read the same
@@ -72,6 +72,7 @@ export interface GeneratorTemplate {
name: string;
defaultSteps: number;
attention: AttentionMode;
+ mode?: "prompt" | "animate";
paths: EngineSettings;
sources?: Partial<Record<EnginePathId, WeightSource[]>>;
loras?: TemplateLora[];
@@ -199,10 +200,20 @@ export function minimaxSingularityTemplate(): GeneratorTemplate {
};
}
+export function viggleAnimateTemplate(): GeneratorTemplate {
+ const template = minimaxOriginalTemplate();
+ const transformer = "https://huggingface.co/DeepBeepMeep/MiniMax-H3/resolve/main/Viggle-Animate-pruned_rank8_int8_convrot.safetensors";
+ return { ...template, id: "viggle-animate", name: "Animate", mode: "animate", defaultSteps: 4,
+ loras: [{ loraId: VIGGLE_ANIMATE_LORA.id, enabled: true, strength: 1 }],
+ paths: { ...template.paths, transformer },
+ sources: { ...template.sources, transformer: [{ url: transformer, gpuModel: "", minVramGb: 0 }] },
+ };
+}
+
const initialTemplateSettings = (paths = EMPTY_ENGINE_SETTINGS): GeneratorTemplateSettings => ({
- templates: [{ id: "default", name: "Default", defaultSteps: DEFAULT_GENERATION_STEPS, attention: "sage2", paths: copyPaths(paths) }, minimaxOriginalTemplate(), minimaxReferencesTemplate(), minimaxFastTemplate(), minimaxSingularityTemplate()],
+ templates: [{ id: "default", name: "Default", defaultSteps: DEFAULT_GENERATION_STEPS, attention: "sage2", paths: copyPaths(paths) }, minimaxOriginalTemplate(), minimaxReferencesTemplate(), minimaxFastTemplate(), minimaxSingularityTemplate(), viggleAnimateTemplate()],
defaultTemplateId: "default",
- catalogVersion: 6,
+ catalogVersion: 7,
});
const normalizeTemplateSettings = (value: unknown): GeneratorTemplateSettings | null => {
@@ -240,6 +251,7 @@ const normalizeTemplateSettings = (value: unknown): GeneratorTemplateSettings |
if (entries.length) sources[field.id] = entries;
}
return [{ id, name, defaultSteps, attention, paths, sources,
+ ...(candidate.mode === "animate" || candidate.mode === "prompt" ? { mode: candidate.mode } : {}),
...(candidate.loras !== undefined ? { loras: normalizeTemplateLoras(candidate.loras) } : {}) }];
});
if (templates.length === 0) return null;
@@ -248,7 +260,7 @@ const normalizeTemplateSettings = (value: unknown): GeneratorTemplateSettings |
templates,
defaultTemplateId: templates.find((template) => template.id === requestedDefault && !templateNeedsDownload(template))?.id
?? templates.find((template) => !templateNeedsDownload(template))?.id ?? "",
- catalogVersion: typeof record.catalogVersion === "number" && [1, 2, 3, 4, 5, 6].includes(record.catalogVersion) ? record.catalogVersion : 0,
+ catalogVersion: typeof record.catalogVersion === "number" && [1, 2, 3, 4, 5, 6, 7].includes(record.catalogVersion) ? record.catalogVersion : 0,
};
};
@@ -320,6 +332,12 @@ export function loadGeneratorTemplateSettings(): GeneratorTemplateSettings {
normalized = normalizeTemplateSettings({ ...normalized, catalogVersion: 6 })!;
localStorage.setItem(GENERATOR_TEMPLATES_KEY, JSON.stringify(normalized));
}
+ if ((normalized.catalogVersion ?? 0) < 7) {
+ const template = viggleAnimateTemplate();
+ if (!normalized.templates.some(({ id }) => id === template.id)) normalized.templates.push(template);
+ normalized = normalizeTemplateSettings({ ...normalized, catalogVersion: 7 })!;
+ localStorage.setItem(GENERATOR_TEMPLATES_KEY, JSON.stringify(normalized));
+ }
return normalized;
}
}
@@ -369,12 +387,14 @@ export function saveEngineSettings(settings: EngineSettings): void {
/** The `slopfab` provider setting these paths describe, with blanks dropped so
* an unset field falls through to whatever the project (or the Rust default)
* already had rather than overwriting it with "". */
-export function engineProviderSetting(settings: EngineSettings, base?: ProviderSetting, attention = defaultGeneratorTemplate().attention, selection = defaultGeneratorTemplate().loras): ProviderSetting {
+export function engineProviderSetting(settings: EngineSettings, base?: ProviderSetting, attention = defaultGeneratorTemplate().attention, selection = defaultGeneratorTemplate().loras, mode = defaultGeneratorTemplate().mode): ProviderSetting {
const options: ProviderSetting["options"] = { ...(base?.options ?? {}), attention, inferenceBackend: loadInferenceBackend() };
// Always replace project-carried adapter paths with this machine's selection.
delete options.loras;
delete options.schedule;
delete options.stepOverride;
+ delete options.generationMode;
+ if (mode === "animate") options.generationMode = "animate";
const loras = resolveTemplateLoras(selection);
if (loras.length) options.loras = JSON.stringify(loras.map(({ path, strength }) => ({ path, strength })));
const stepOverride = highestLoraStepOverride(loras);