Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Enable Vulkan video references

Bitti9ccfbd4b4443
diff --git a/docs/video-references.md b/docs/video-references.md
index f9cfbd6..7771ee2 100644
--- a/docs/video-references.md
+++ b/docs/video-references.md
@@ -8,8 +8,10 @@ locations. The project stores their paths and clip settings.
Set the clip's start and duration, and choose whether to include its sound.
Each clip must last 2–15 seconds. A generation can use up to three video clips
totaling 15 seconds, alongside up to nine images. Cite the reference in a shot
-just like an image reference. Use CUDA and a Ref2VA transformer, such as the
-**References** generator. Soundtracks also require audio VAE encoder weights.
+just like an image reference. Use CUDA or Vulkan with a Ref2VA transformer,
+such as the **References** generator. Vulkan video references require
+floating-point video VAE encoder weights. Soundtracks on either backend also
+require floating-point audio VAE encoder weights.
The prompt compiler numbers pictures and videos independently, in the order
the native request receives each kind. Video trim settings and soundtrack
diff --git a/src-tauri/src/slopfab.rs b/src-tauri/src/slopfab.rs
index edf0cc1..19b0504 100644
--- a/src-tauri/src/slopfab.rs
+++ b/src-tauri/src/slopfab.rs
@@ -145,9 +145,6 @@ pub fn create_reference_video(duration: f64, settings: &BTreeMap<String, Provide
let configuration = Configuration::from_settings(settings);
let api = ffi::Api::load(&configuration.dll_path)?;
api.version()?;
- if configuration.platform(detect_platform(&api)) == ComputePlatform::Vulkan {
- return Err("Video references require CUDA. Select CUDA in Generator settings and use a Ref2VA transformer.".into());
- }
let video = ffi::ReferenceVideoHandle::new(api, duration, configuration.dll_path)?;
let mut videos = REFERENCE_VIDEOS.lock().map_err(|_| "Reference video lock failed.")?;
if videos.len() >= 3 { return Err("At most three video references can be prepared at once.".into()); }
@@ -679,9 +676,6 @@ fn configure_request(
api.add_reference(handle, Path::new(path))?;
}
if !request.reference_video_ids.is_empty() {
- if platform == ComputePlatform::Vulkan {
- return Err("Video references require CUDA and a Ref2VA transformer.".into());
- }
let videos = REFERENCE_VIDEOS.lock().map_err(|_| "Reference video lock failed.")?;
for id in &request.reference_video_ids {
videos.get(id).ok_or("The prepared reference video no longer exists. Please retry generation.")?
@@ -1632,6 +1626,36 @@ mod ffi {
#[cfg(test)]
mod tests {
use super::*;
+ #[test]
+ fn vulkan_video_references_prepare_and_attach_for_planning_and_generation() {
+ let settings = BTreeMap::from([("slopfab".into(), ProviderSetting {
+ enabled: true,
+ model: None,
+ options: BTreeMap::from([("inferenceBackend".into(), ProviderOption::String("vulkan".into()))]),
+ })]);
+ let id = create_reference_video(2.0, &settings).unwrap();
+ let result = (|| -> Result<(), String> {
+ append_reference_video(&id, &vec![127; 64 * 64 * 4], 64, 64, 0.0)?;
+ set_reference_video_audio(&id, &vec![0; 32_000 * 2 * 4], 1, 32_000)?;
+ let request = GenerationRequest {
+ job_id: "vulkan-reference-test".into(), prompt: "A scene using <Video 1>.".into(),
+ frames: 120, still_image: false, steps: 12, seed: 1,
+ canvas_width: 736, canvas_height: 416,
+ reference_paths: Vec::new(), reference_video_ids: vec![id.clone()],
+ };
+ let configuration = Configuration::from_settings(&settings);
+ let api = ffi::Api::load(&configuration.dll_path)?;
+ for include_models in [false, true] {
+ let handle = RequestHandle::new(&api)?;
+ configure_request(&api, handle.0, &request, &configuration, ComputePlatform::Vulkan, include_models)?;
+ assert!(api.resolve(handle.0)?.aligned_frames > 0);
+ }
+ Ok(())
+ })();
+ release_reference_videos(&[id]).unwrap();
+ result.unwrap();
+ }
+
#[test]
fn video_reference_c_api_copies_inputs_and_retains_attached_snapshot() {
let path = default_dll_path();
diff --git a/src/components/workspace/ReferencesView.tsx b/src/components/workspace/ReferencesView.tsx
index 4000fac..62310b5 100644
--- a/src/components/workspace/ReferencesView.tsx
+++ b/src/components/workspace/ReferencesView.tsx
@@ -223,7 +223,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,
{selected ? <>
{selected.kind === "video" && <section className="reference-video" aria-label="Video reference">
<ReferenceVideo folderPath={folderPath} reference={selected} />
- <p>Use 2–15 seconds per clip, up to 3 clips totaling 15 seconds. Requires CUDA and a Ref2VA generator.</p>
+ <p>Use 2–15 seconds per clip, up to 3 clips totaling 15 seconds. Requires a Ref2VA generator. Supports CUDA and Vulkan.</p>
<ClipSecondsInput key={`${selected.id}-start`} label="Clip start (seconds)" min={0} value={selected.video?.startSeconds ?? 0}
onChange={(startSeconds) => update(selected.id, { video: { durationSeconds: 2, includeAudio: true, ...selected.video, startSeconds } })} />
<ClipSecondsInput key={`${selected.id}-duration`} label="Clip duration (seconds)" min={2} max={15} value={selected.video?.durationSeconds ?? 2}