AI media harness for planning, generating, and editing locally
TypeScript
Enable Vulkan video references
Bitti9ccfbd4b4443
diff --git a/docs/video-references.md b/docs/video-references.mdindex f9cfbd6..7771ee2 100644--- a/docs/video-references.md+++ b/docs/video-references.md@@ -8,8 +8,10 @@ locations. The project stores their paths and clip settings.Set the clip's start and duration, and choose whether to include its sound.Each clip must last 2–15 seconds. A generation can use up to three video clipstotaling 15 seconds, alongside up to nine images. Cite the reference in a shot-just like an image reference. Use CUDA and a Ref2VA transformer, such as the-**References** generator. Soundtracks also require audio VAE encoder weights.+just like an image reference. Use CUDA or Vulkan with a Ref2VA transformer,+such as the **References** generator. Vulkan video references require+floating-point video VAE encoder weights. Soundtracks on either backend also+require floating-point audio VAE encoder weights.The prompt compiler numbers pictures and videos independently, in the orderthe native request receives each kind. Video trim settings and soundtrackdiff --git a/src-tauri/src/slopfab.rs b/src-tauri/src/slopfab.rsindex edf0cc1..19b0504 100644--- a/src-tauri/src/slopfab.rs+++ b/src-tauri/src/slopfab.rs@@ -145,9 +145,6 @@ pub fn create_reference_video(duration: f64, settings: &BTreeMap<String, Providelet configuration = Configuration::from_settings(settings);let api = ffi::Api::load(&configuration.dll_path)?;api.version()?;- if configuration.platform(detect_platform(&api)) == ComputePlatform::Vulkan {- return Err("Video references require CUDA. Select CUDA in Generator settings and use a Ref2VA transformer.".into());- }let video = ffi::ReferenceVideoHandle::new(api, duration, configuration.dll_path)?;let mut videos = REFERENCE_VIDEOS.lock().map_err(|_| "Reference video lock failed.")?;if videos.len() >= 3 { return Err("At most three video references can be prepared at once.".into()); }@@ -679,9 +676,6 @@ fn configure_request(api.add_reference(handle, Path::new(path))?;}if !request.reference_video_ids.is_empty() {- if platform == ComputePlatform::Vulkan {- return Err("Video references require CUDA and a Ref2VA transformer.".into());- }let videos = REFERENCE_VIDEOS.lock().map_err(|_| "Reference video lock failed.")?;for id in &request.reference_video_ids {videos.get(id).ok_or("The prepared reference video no longer exists. Please retry generation.")?@@ -1632,6 +1626,36 @@ mod ffi {#[cfg(test)]mod tests {use super::*;+ #[test]+ fn vulkan_video_references_prepare_and_attach_for_planning_and_generation() {+ let settings = BTreeMap::from([("slopfab".into(), ProviderSetting {+ enabled: true,+ model: None,+ options: BTreeMap::from([("inferenceBackend".into(), ProviderOption::String("vulkan".into()))]),+ })]);+ let id = create_reference_video(2.0, &settings).unwrap();+ let result = (|| -> Result<(), String> {+ append_reference_video(&id, &vec![127; 64 * 64 * 4], 64, 64, 0.0)?;+ set_reference_video_audio(&id, &vec![0; 32_000 * 2 * 4], 1, 32_000)?;+ let request = GenerationRequest {+ job_id: "vulkan-reference-test".into(), prompt: "A scene using <Video 1>.".into(),+ frames: 120, still_image: false, steps: 12, seed: 1,+ canvas_width: 736, canvas_height: 416,+ reference_paths: Vec::new(), reference_video_ids: vec![id.clone()],+ };+ let configuration = Configuration::from_settings(&settings);+ let api = ffi::Api::load(&configuration.dll_path)?;+ for include_models in [false, true] {+ let handle = RequestHandle::new(&api)?;+ configure_request(&api, handle.0, &request, &configuration, ComputePlatform::Vulkan, include_models)?;+ assert!(api.resolve(handle.0)?.aligned_frames > 0);+ }+ Ok(())+ })();+ release_reference_videos(&[id]).unwrap();+ result.unwrap();+ }+#[test]fn video_reference_c_api_copies_inputs_and_retains_attached_snapshot() {let path = default_dll_path();diff --git a/src/components/workspace/ReferencesView.tsx b/src/components/workspace/ReferencesView.tsxindex 4000fac..62310b5 100644--- a/src/components/workspace/ReferencesView.tsx+++ b/src/components/workspace/ReferencesView.tsx@@ -223,7 +223,7 @@ export function ReferencesView({ config, folderPath, onChange, onRegenerateIcon,{selected ? <>{selected.kind === "video" && <section className="reference-video" aria-label="Video reference"><ReferenceVideo folderPath={folderPath} reference={selected} />- <p>Use 2–15 seconds per clip, up to 3 clips totaling 15 seconds. Requires CUDA and a Ref2VA generator.</p>+ <p>Use 2–15 seconds per clip, up to 3 clips totaling 15 seconds. Requires a Ref2VA generator. Supports CUDA and Vulkan.</p><ClipSecondsInput key={`${selected.id}-start`} label="Clip start (seconds)" min={0} value={selected.video?.startSeconds ?? 0}onChange={(startSeconds) => update(selected.id, { video: { durationSeconds: 2, includeAudio: true, ...selected.video, startSeconds } })} /><ClipSecondsInput key={`${selected.id}-duration`} label="Clip duration (seconds)" min={2} max={15} value={selected.video?.durationSeconds ?? 2}