The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
Let the selection decide what Captions captions
Jareerda9849bf9fc5
diff --git a/engine/crates/concat/locales/en.json b/engine/crates/concat/locales/en.jsonindex 4b4309e2..74f06d4f 100644--- a/engine/crates/concat/locales/en.json+++ b/engine/crates/concat/locales/en.json@@ -43,7 +43,6 @@"Application": "Application","Apply": "Apply","at the playhead": "at the playhead",- "at the selected clip": "at the selected clip","Audio": "Audio","audio effect": "audio effect","Audio Effects": "Audio Effects",@@ -94,7 +93,7 @@"Cancel export": "Cancel export","Caption": "Caption","Captions": "Captions",- "Captions from the timeline's sound, or from a script": "Captions from the timeline's sound, or from a script",+ "Captions from a script, at the playhead": "Captions from a script, at the playhead","Cathedral": "Cathedral","Center": "Center","Centre": "Centre",@@ -400,15 +399,12 @@"not installed": "not installed","Nothing found to keep here, so the picture shows as shot. Try another subject, or custom removal.": "Nothing found to keep here, so the picture shows as shot. Try another subject, or custom removal.","Nothing of that kind in this project.": "Nothing of that kind in this project.",- "Nothing on the timeline has sound to transcribe": "Nothing on the timeline has sound to transcribe",- "Nothing on the timeline has sound to transcribe. Select a clip with sound, or type a script instead.": "Nothing on the timeline has sound to transcribe. Select a clip with sound, or type a script instead.","Nothing starred yet — hover a tile and press its star.": "Nothing starred yet — hover a tile and press its star.","Nothing to caption yet": "Nothing to caption yet","Nothing to export": "Nothing to export","Nothing to read yet": "Nothing to read yet","Nothing under the playhead": "Nothing under the playhead","Nothing was said in that clip": "Nothing was said in that clip",- "Nothing was said in those clips": "Nothing was said in those clips","Object": "Object","off": "off","Off, a click on the ruler past the last clip lands the playhead there, and a clip placed at the playhead goes with it. On, the playhead never leaves the content.": "Off, a click on the ruler past the last clip lands the playhead there, and a clip placed at the playhead goes with it. On, the playhead never leaves the content.",@@ -594,6 +590,7 @@"The bulge of a very wide lens.": "The bulge of a very wide lens.","The café look: warm brown through the middle, greens muted to olive, contrast eased. Cosy, and kind to wood and skin alike.": "The café look: warm brown through the middle, greens muted to olive, contrast eased. Cosy, and kind to wood and skin alike.","The chain runs top to bottom — reorder it with the chevrons. The eye bypasses one link without losing the values you dialled into it.": "The chain runs top to bottom — reorder it with the chevrons. The eye bypasses one link without losing the values you dialled into it.",+ "The clip is no longer on the timeline": "The clip is no longer on the timeline","The early reflections of a room.": "The early reflections of a room.","The instant print: a cream matte, warm highlights, cyan creeping into the shadows, a little softness in the corners.": "The instant print: a cream matte, warm highlights, cyan creeping into the shadows, a little softness in the corners.","The left half, reflected onto the right.": "The left half, reflected onto the right.",@@ -664,7 +661,6 @@"Zoom out": "Zoom out","Zoom to fit": "Zoom to fit","{0} channels": "{0} channels",- "{0} clips with sound": "{0} clips with sound","{0} days ago": "{0} days ago","{0} Hz": "{0} Hz","{0} installed · {1} MB on disk": "{0} installed · {1} MB on disk",diff --git a/engine/crates/concat/src/lib.rs b/engine/crates/concat/src/lib.rsindex a5d89792..d96a2f05 100644--- a/engine/crates/concat/src/lib.rs+++ b/engine/crates/concat/src/lib.rs@@ -1001,9 +1001,6 @@ pub fn run() -> Result<(), slint::PlatformError> {app.on_captions_closed(on_window!(|state| {state.captions.open = false;}));- app.on_captions_source_changed(on_window!(|state, index: i32| {- state.captions.source = (index.max(0) as usize).min(1);- }));app.on_captions_text_edited(on_window!(|state, text: SharedString| {state.captions.text = text.to_string();}));diff --git a/engine/crates/concat/src/studio.rs b/engine/crates/concat/src/studio.rsindex 211de278..12d989d4 100644--- a/engine/crates/concat/src/studio.rs+++ b/engine/crates/concat/src/studio.rs@@ -168,12 +168,10 @@ pub struct ProjectSheet {#[derive(Default)]pub struct CaptionsSheet {pub open: bool,- /// 0 the sound, 1 a script.- pub source: usize,- /// The clips the sound source transcribes: the selected clip with- /// sound, or every clip with sound on the timeline.- pub clips: Vec<String>,- /// The script source's words.+ /// The clip being transcribed: the one selected when the sheet opened,+ /// when it had sound. None, and the sheet is a script instead.+ pub clip: Option<String>,+ /// The script's words.pub text: String,/// Row in the installed transcriber list.pub model: usize,@@ -4650,7 +4648,7 @@ impl Studio {.any(|other| other.detached_from.as_deref() == Some(clip.id.as_str()));let video = clip.kind == model::ClipKind::Video;(- matches!(clip.kind, model::ClipKind::Video | model::ClipKind::Audio),+ self.clip_has_sound(clip),clip.kind == model::ClipKind::Text,video && !detached,(video && detached)@@ -4658,6 +4656,20 @@ impl Studio {)}+ /// Whether the clip has sound to transcribe: an audio clip, or a video+ /// clip whose file carries an audio stream. A silent video is not a+ /// sound source, however much it looks like one.+ fn clip_has_sound(&self, clip: &Clip) -> bool {+ match clip.kind {+ model::ClipKind::Audio => true,+ model::ClipKind::Video => self+ .project()+ .media_by_id(&clip.media_id)+ .is_some_and(|media| media.has_audio),+ _ => false,+ }+ }+// ── projects ──/// Opens a project as the session and leaves the launch screen.@@ -5098,33 +5110,20 @@ impl Studio {models.iter().filter(|model| model.installed).collect()}- /// Opens the captions sheet with the chosen model already picked. The- /// sound source is the selected clip when it has sound, else every clip- /// with sound on the timeline; with neither, the sheet opens on the- /// script instead.+ /// Opens the captions sheet with the chosen model already picked. What+ /// it captions is decided here, not asked: the selected clip's sound+ /// when one clip with sound is selected, and a script otherwise.pub fn captions_open(&mut self) {let installed = Self::installed(&self.transcribers);let model = installed.iter().position(|model| model.active).unwrap_or(0);- let has_sound =- |clip: &Clip| matches!(clip.kind, model::ClipKind::Video | model::ClipKind::Audio);- let selected = self+ let clip = self.sole_selection().and_then(|id| self.clip(&id))- .filter(|clip| has_sound(clip))+ .filter(|clip| self.clip_has_sound(clip)).map(|clip| clip.id.clone());- let clips: Vec<String> = match selected {- Some(id) => vec![id],- None => {- let mut clips: Vec<&Clip> =- self.timeline().clips.iter().filter(|clip| has_sound(clip)).collect();- clips.sort_by(|a, b| a.start.total_cmp(&b.start));- clips.into_iter().map(|clip| clip.id.clone()).collect()- }- };self.captions = CaptionsSheet {open: true,- source: usize::from(clips.is_empty()),- clips,+ clip,model,placement: 0,size: 1,@@ -5136,10 +5135,10 @@ impl Studio {/// transcriber, or the script cut into lines. Either lands as one batch/// of title clips - one undo step.pub fn captions_run(&mut self) {- if self.captions.source == 1 {- self.captions_from_script();- } else {+ if self.captions.clip.is_some() {self.captions_from_sound();+ } else {+ self.captions_from_script();}}@@ -5168,8 +5167,7 @@ impl Studio {}}- /// The script as titles, one after another from the selected clip's- /// start or the playhead.+ /// The script as titles, one after another from the playhead.fn captions_from_script(&mut self) {let lines = script_captions(&self.captions.text);if lines.is_empty() {@@ -5177,7 +5175,7 @@ impl Studio {return;}let look = self.caption_look();- let mut at = self.script_landing();+ let mut at = f64::from(self.playhead);let commands: Vec<Command> = lines.into_iter().map(|(text, seconds)| {@@ -5192,28 +5190,23 @@ impl Studio {self.notify(&tf("Added {0} captions", &[&count]), false);}- /// Where a script's first caption goes: the selected clip's start, or- /// the playhead.- fn script_landing(&self) -> f64 {- self.sole_selection()- .and_then(|id| self.clip(&id))- .map(|clip| clip.start)- .unwrap_or(f64::from(self.playhead))- }-- /// The sheet's clips through the transcriber, one after another on a- /// worker, reporting into the sheet as they go.+ /// The sheet's clip through the transcriber on a worker, reporting into+ /// the sheet as it goes.fn captions_from_sound(&mut self) {- let clips: Vec<Clip> = self+ let Some(clip) = self.captions- .clips- .iter()- .filter_map(|id| self.clip(id).cloned())- .collect();- if clips.is_empty() {- self.captions.message = t("Nothing on the timeline has sound to transcribe");+ .clip+ .as_ref()+ .and_then(|id| self.clip(id))+ .cloned()+ else {+ self.captions.message = t("The clip is no longer on the timeline");return;- }+ };+ let Some(media) = self.project().media_by_id(&clip.media_id).cloned() else {+ self.captions.message = t("This clip has no file to transcribe");+ return;+ };let Some(model) = Self::installed(&self.transcribers).get(self.captions.model).map(|model| model.id.clone())@@ -5222,23 +5215,13 @@ impl Studio {t("Download a transcriber model in Settings › Transcriber first");return;};- let requests: Vec<_> = clips- .iter()- .filter_map(|clip| {- let media = self.project().media_by_id(&clip.media_id)?;- Some(concat_speech::transcribe::TranscribeRequest {- path: media.path.clone(),- audio_stream: clip.audio_stream,- source_start: clip.source_start,- window: clip.duration * clip.speed,- model_id: model.clone(),- })- })- .collect();- if requests.len() != clips.len() {- self.captions.message = t("This clip has no file to transcribe");- return;- }+ let request = concat_speech::transcribe::TranscribeRequest {+ path: media.path.clone(),+ audio_stream: clip.audio_stream,+ source_start: clip.source_start,+ window: clip.duration * clip.speed,+ model_id: model,+ };let look = self.caption_look();let dirs = self.host.dirs.clone();let transcriber = Arc::clone(&self.host.transcriber);@@ -5247,50 +5230,34 @@ impl Studio {self.captions.message.clear();spawn(move || {- let total = requests.len();- let mut heard = Vec::with_capacity(total);- for (index, request) in requests.iter().enumerate() {- let segments = transcriber.transcribe(&dirs, request, move |percent| {- // One bar across every clip: this clip's share of it,- // as far as whisper has got through the clip.- let fraction = (index as f32 + (percent as f32 / 100.0).clamp(0.0, 1.0))- / total as f32;- on_ui(move |studio, _, _| studio.captions.progress = fraction);- })?;- heard.push(segments);- }- Ok::<_, String>(heard)+ transcriber.transcribe(&dirs, &request, |percent| {+ on_ui(move |studio, _, _| {+ studio.captions.progress = (percent as f32 / 100.0).clamp(0.0, 1.0);+ });+ })},move |studio, _, _, result| {studio.captions.running = false;match result {- Ok(heard) => {- let commands: Vec<Command> = clips- .iter()- .zip(heard)- .flat_map(|(clip, segments)| {- segments.into_iter().filter_map(move |segment| {- let text = segment.text.trim().to_owned();- (!text.is_empty()).then(|| {- Self::caption_clip(- text,- clip.start + segment.start / clip.speed,- ((segment.end - segment.start) / clip.speed).max(0.2),- look,- )- })+ Ok(segments) => {+ let commands: Vec<Command> = segments+ .into_iter()+ .filter_map(|segment| {+ let text = segment.text.trim().to_owned();+ (!text.is_empty()).then(|| {+ Self::caption_clip(+ text,+ clip.start + segment.start / clip.speed,+ ((segment.end - segment.start) / clip.speed).max(0.2),+ look,+ )})}).collect();let count = commands.len();studio.captions.open = false;if count == 0 {- let quiet = if clips.len() == 1 {- t("Nothing was said in that clip")- } else {- t("Nothing was said in those clips")- };- studio.notify(&quiet, true);+ studio.notify(&t("Nothing was said in that clip"), true);} else {studio.apply(Command::Batch { commands });studio.notify(&tf("Added {0} captions", &[&count]), false);@@ -6525,19 +6492,14 @@ impl Studio {);app.set_captions(CaptionsSheetData {open: self.captions.open,- source: self.captions.source as i32,- subject: match self.captions.clips.as_slice() {- [] => String::new(),- [id] => self.clip(id).map(|clip| clip.name.clone()).unwrap_or_default(),- many => tf("{0} clips with sound", &[&many.len()]),- }- .into(),- landing: if self.sole_selection().is_some() {- t("at the selected clip")- } else {- t("at the playhead")- }- .into(),+ from_sound: self.captions.clip.is_some(),+ subject: self+ .captions+ .clip+ .as_ref()+ .and_then(|id| self.clip(id))+ .map(|clip| SharedString::from(clip.name.as_str()))+ .unwrap_or_else(|| t("at the playhead").into()),text: self.captions.text.as_str().into(),model: self.captions.model as i32,placement: self.captions.placement as i32,diff --git a/engine/crates/concat/ui/app.slint b/engine/crates/concat/ui/app.slintindex baf64e37..97315df4 100644--- a/engine/crates/concat/ui/app.slint+++ b/engine/crates/concat/ui/app.slint@@ -208,7 +208,6 @@ export component App inherits Window {in property <CaptionsSheetData> captions;in property <[string]> caption-models;callback captions-closed();- callback captions-source-changed(index: int);callback captions-text-edited(text: string);callback captions-model-changed(index: int);callback captions-placement-changed(index: int);@@ -494,7 +493,6 @@ export component App inherits Window {models: root.caption-models;closed => { root.captions-closed(); }- source-changed(index) => { root.captions-source-changed(index); }text-edited(text) => { root.captions-text-edited(text); }model-changed(index) => { root.captions-model-changed(index); }placement-changed(index) => { root.captions-placement-changed(index); }diff --git a/engine/crates/concat/ui/dialogs/captions.slint b/engine/crates/concat/ui/dialogs/captions.slintindex 36851fef..06652185 100644--- a/engine/crates/concat/ui/dialogs/captions.slint+++ b/engine/crates/concat/ui/dialogs/captions.slint@@ -10,24 +10,24 @@ import { Select } from "../primitives/select.slint";// Captions: what the tray's Captions tool opens.//-// Two places the words can come from. The sound: the selected clip's, or-// every clip with sound on the timeline when nothing is selected, run-// through the transcriber and cut to the words' own timing. A script: the-// words typed here, one title per sentence at reading pace, laid down one-// after another. Then where the lines sit and how big they are, and one-// button. While it runs the sheet is the progress report: a transcription-// knows how far through the sound it is, so it gets a real bar and not a-// sweep. The language is the transcriber's to work out.+// Two sheets in one, and which is decided before it opens rather than+// asked. With a clip that has sound selected, the words are that clip's:+// through the transcriber, cut to the words' own timing. Otherwise they+// are a script typed here, one title per sentence at reading pace, laid+// down from the playhead. Asking would only invite the wrong answer - a+// silent video sent to the transcriber - so the selection decides. Then+// where the lines sit and how big they are, and one button. While it runs+// the sheet is the progress report: a transcription knows how far through+// the sound it is, so it gets a real bar and not a sweep. The language is+// the transcriber's to work out.export struct CaptionsSheetData {open: bool,- /// 0 Sound, 1 Script- source: int,- /// The sound to transcribe: the selected clip's name, or "4 clips with- /// sound"; empty when the timeline has none.+ /// The selected clip has sound, and the sheet transcribes it.+ from-sound: bool,+ /// For the header's note: the clip being transcribed, or where a+ /// script's captions land.subject: string,- /// Where a script's captions land, for the header's note.- landing: string,text: string,/// Row in the installed transcriber list.model: int,@@ -49,7 +49,6 @@ export component CaptionsDialog inherits Modal {/// The installed transcriber models, by name.in property <[string]> models;- callback source-changed(index: int);callback text-edited(text: string);callback model-changed(index: int);callback placement-changed(index: int);@@ -57,14 +56,14 @@ export component CaptionsDialog inherits Modal {callback begin();callback cancel();- property <bool> from-sound: root.data.source == 0;+ property <bool> from-sound: root.data.from-sound;open: root.data.open;title: I18n.t("Captions");glyph: Glyph.text-mark;surface-width: 480px;dismissable: !root.data.running;- note: root.from-sound ? root.data.subject : root.data.landing;+ note: root.data.subject;// The script being typed, held apart from `data.text` the way the Field// primitive holds its draft: a TextInput's binding is dropped on the@@ -82,18 +81,8 @@ export component CaptionsDialog inherits Modal {init => { root.draft = root.data.text; }VerticalLayout {- if !root.data.running: DialogSection {- first: true;- title: I18n.t("Source");-- SegmentedControl {- options: [I18n.t("Sound"), I18n.t("Script")];- current: root.data.source;- changed(index) => { root.source-changed(index); }- }- }-if !root.data.running && root.from-sound: DialogSection {+ first: true;title: I18n.t("Transcriber");DialogRow {@@ -113,16 +102,10 @@ export component CaptionsDialog inherits Modal {font-size: Theme.fs-sm;wrap: word-wrap;}-- if root.data.subject == "": Text {- text: I18n.t("Nothing on the timeline has sound to transcribe. Select a clip with sound, or type a script instead.");- color: Theme.fg-muted;- font-size: Theme.fs-sm;- wrap: word-wrap;- }}if !root.data.running && !root.from-sound: DialogSection {+ first: true;title: I18n.t("Script");Rectangle {@@ -265,9 +248,7 @@ export component CaptionsDialog inherits Modal {large: true;glyph: Glyph.text-mark;label: root.from-sound ? I18n.t("Generate captions") : I18n.t("Add captions");- enabled: root.from-sound- ? root.data.ready && root.data.subject != ""- : root.draft != "";+ enabled: root.from-sound ? root.data.ready : root.draft != "";clicked => { root.begin(); }}}diff --git a/engine/crates/concat/ui/timeline/tray.slint b/engine/crates/concat/ui/timeline/tray.slintindex 68b80a02..d80e68b6 100644--- a/engine/crates/concat/ui/timeline/tray.slint+++ b/engine/crates/concat/ui/timeline/tray.slint@@ -235,7 +235,7 @@ export component TimelineTray inherits Rectangle {glyph: Glyph.text-mark;caption: I18n.t("Captions");label: root.sound-selected ? I18n.t("Auto captions from this clip's sound")- : I18n.t("Captions from the timeline's sound, or from a script");+ : I18n.t("Captions from a script, at the playhead");clicked => { root.captions(); }}TrayTool {