The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(captions, speech): Captions and Text to speech float over the editor on a desk
Jareerf8de11ce7863
diff --git a/src/crates/concat/src/panes/captions.rs b/src/crates/concat/src/panes/captions.rsindex 3c15f6fc..50b5cbe4 100644--- a/src/crates/concat/src/panes/captions.rs+++ b/src/crates/concat/src/panes/captions.rs@@ -104,8 +104,13 @@ pub struct CaptionsPane {pub from_voice: bool,/// The clip whose voice is transcribed: one with sound, chosen from the/// timeline's. Kept while the switch is off, so turning it back on- /// returns to the same clip.+ /// returns to the same clip. The panel floats over a live editor, so+ /// a clip with sound selected since is the one shown and run instead;+ /// see [`Self::subject_clip`].pub clip: Option<String>,+ /// The sole selection when [`Self::clip`] was chosen - opened on, or+ /// picked from the list - so a later selection can be told from it.+ chosen_with: Option<String>,/// The script's words.pub text: String,/// Row in the installed transcriber list.@@ -130,6 +135,9 @@ impl CaptionsPane {/// this runs the studio's copy of the pane is a blank it must not read.pub fn update(&mut self, msg: CaptionsMsg, studio: &mut Studio) {match msg {+ // A run in progress is shown as it is, however the panel is+ // asked for again: closing it never stopped the run.+ CaptionsMsg::Open | CaptionsMsg::OpenOnClip(_) if self.running => self.open = true,CaptionsMsg::Open => {let selected = studio.sole_selection();self.open_on(studio, selected, false);@@ -147,6 +155,7 @@ impl CaptionsPane {.is_some_and(|id| sources.iter().any(|s| &s.id == id)){self.clip = sources.first().map(|source| source.id.clone());+ self.chosen_with = studio.sole_selection();}}CaptionsMsg::SourceChanged(index) => {@@ -155,6 +164,7 @@ impl CaptionsPane {.and_then(|index| sources(studio).into_iter().nth(index)){self.clip = Some(source.id);+ self.chosen_with = studio.sole_selection();self.message.clear();}}@@ -217,7 +227,14 @@ impl CaptionsPane {}// Asked for: the sheet is already on its way down.Err(error) if error.contains("cancel") => self.open = false,- Err(error) => self.message = error,+ // Closed while it ran: the panel is not there to say+ // it, so the notice does, and the panel too next time.+ Err(error) => {+ if !self.open {+ studio.notify(&error, true);+ }+ self.message = error;+ }}}// Asked with the studio unborrowed, on a turn of its own; see@@ -391,6 +408,7 @@ impl CaptionsPane {open: true,from_voice,clip,+ chosen_with: studio.sole_selection(),model,placement: 0,size: 1,@@ -399,6 +417,21 @@ impl CaptionsPane {};}+ /// The clip the panel transcribes, and the one its list shows - the+ /// same answer for both, so it never runs a clip it is not showing.+ /// The editor stays live under the panel: a clip with sound selected+ /// since the panel's clip was chosen is the one meant now, and is+ /// taken. Otherwise the chosen clip while it is still on the timeline+ /// with sound, and none - nothing to run - when it is not.+ fn subject_clip(&self, studio: &Studio, sources: &[Source]) -> Option<String> {+ follow_selection(+ studio.sole_selection(),+ self.chosen_with.as_deref(),+ self.clip.as_deref(),+ |id| sources.iter().any(|source| source.id == id),+ )+ }+/// A caption's look, by the sheet's rows: where it sits and its size.fn look(&self) -> (f64, f64) {(@@ -434,7 +467,12 @@ impl CaptionsPane {/// The sheet's clip through the transcriber on a worker, reporting into/// the sheet as it goes.fn run_sound(&mut self, studio: &mut Studio) {- let Some(clip) = self.clip.as_ref().and_then(|id| studio.clip(id)).cloned() else {+ // The clip the panel shows, and no other: see `subject_clip`.+ let Some(clip) = self+ .subject_clip(studio, &sources(studio))+ .and_then(|id| studio.clip(&id))+ .cloned()+ else {self.message = t("captions.clipNoLongerTimeline");return;};@@ -508,9 +546,8 @@ impl CaptionsPane {Vec::new()};let source = self- .clip- .as_ref()- .and_then(|id| sources.iter().position(|source| &source.id == id))+ .subject_clip(studio, &sources)+ .and_then(|id| sources.iter().position(|source| source.id == id)).map_or(-1, |index| index as i32);let labels: Vec<slint::SharedString> = sources.into_iter()@@ -634,6 +671,23 @@ fn unique_path(dir: &Path, name: &str) -> PathBuf {.unwrap_or(first)}+/// [`CaptionsPane::subject_clip`] with the studio taken out: `selected`+/// the sole selection, `chosen_with` the selection when `clip` was chosen,+/// and `sounds` whether a clip is on the timeline with sound.+fn follow_selection(+ selected: Option<String>,+ chosen_with: Option<&str>,+ clip: Option<&str>,+ sounds: impl Fn(&str) -> bool,+) -> Option<String> {+ if let Some(selected) = selected.filter(|id| sounds(id))+ && Some(selected.as_str()) != chosen_with+ {+ return Some(selected);+ }+ clip.filter(|id| sounds(id)).map(str::to_owned)+}+/// A clip on the timeline whose voice can be captioned, as the sheet's/// list shows it.struct Source {@@ -835,6 +889,40 @@ mod tests {);}+ /// The panel floats over a live editor: a clip with sound selected+ /// since the panel's clip was chosen is the one it shows and runs; a+ /// pick from its list holds until the selection moves again; and a+ /// chosen clip gone from the timeline is not run unseen.+ #[test]+ fn the_captioned_clip_follows_a_new_selection_of_a_clip_with_sound() {+ let sounds = |id: &str| ["a", "b", "c"].contains(&id);+ let follow = |selected: Option<&str>, chosen_with: Option<&str>, clip: Option<&str>| {+ super::follow_selection(selected.map(str::to_owned), chosen_with, clip, sounds)+ };+ // Opened on A, A still selected: A.+ assert_eq!(+ follow(Some("a"), Some("a"), Some("a")).as_deref(),+ Some("a")+ );+ // B selected on the timeline since: B.+ assert_eq!(+ follow(Some("b"), Some("a"), Some("a")).as_deref(),+ Some("b")+ );+ // C picked from the list with B selected: C, until the selection moves.+ assert_eq!(+ follow(Some("b"), Some("b"), Some("c")).as_deref(),+ Some("c")+ );+ // A title selected - no sound - leaves the choice alone.+ assert_eq!(+ follow(Some("t"), Some("b"), Some("c")).as_deref(),+ Some("c")+ );+ // The chosen clip deleted: nothing to run, rather than a clip unseen.+ assert_eq!(follow(None, None, Some("gone")), None);+ }+#[test]fn imported_srt_ignores_empty_captions() {let cues = vec![concat_project::srt::Cue {diff --git a/src/crates/concat/src/panes/speech.rs b/src/crates/concat/src/panes/speech.rsindex bf5a2ff5..82a19598 100644--- a/src/crates/concat/src/panes/speech.rs+++ b/src/crates/concat/src/panes/speech.rs@@ -147,6 +147,9 @@ impl SpeechPane {/// this runs the studio's copy of the pane is a blank it must not read.pub fn update(&mut self, msg: SpeechMsg, studio: &mut Studio) {match msg {+ // A reading in progress is shown as it is when the panel is+ // asked for again: closing it never stopped the reading.+ SpeechMsg::Open if self.running => self.open = true,SpeechMsg::Open => {let title = studio.sole_selection()@@ -266,7 +269,14 @@ impl SpeechPane {}// Asked for: the sheet is already on its way down.Err(error) if error.contains("cancel") => self.open = false,- Err(error) => self.message = error,+ // Closed while it ran: the panel is not there to say+ // it, so the notice does, and the panel too next time.+ Err(error) => {+ if !self.open {+ studio.notify(&error, true);+ }+ self.message = error;+ }}}}diff --git a/src/crates/concat/ui/app.slint b/src/crates/concat/ui/app.slintindex b598fd28..ea630861 100644--- a/src/crates/concat/ui/app.slint+++ b/src/crates/concat/ui/app.slint@@ -176,6 +176,9 @@ export component App inherits Window {/// area, on a phone and in any window too narrow for it to float as a/// card with room around it.property <bool> sheets-fill: root.phone || root.width < 600px;+ /// Which floating tool card is in front: "captions" or "speech". The+ /// one opened or grabbed last; Escape closes it first.+ property <string> front-tool: "captions";/// The File / Edit / View menus. Built in Rust like every other menu here,/// because what belongs on them depends on the project.@@ -483,14 +486,6 @@ export component App inherits Window {if (event.text == Key.Escape) {if (root.confirm.open) { root.confirm-dismissed(); return accept; }if (root.relink.open) { root.relink-dismiss(); return accept; }- if (root.speech.open) {- if (!root.speech.running) { root.speech-closed(); }- return accept;- }- if (root.captions.open) {- if (!root.captions.running) { root.captions-closed(); }- return accept;- }if (root.project-sheet.open) { root.project-closed(); return accept; }if (root.settings.open) { root.settings-closed(); return accept; }if (root.export.open) {@@ -501,6 +496,18 @@ export component App inherits Window {if (!root.start.busy) { root.start-dismiss(); }return accept;}+ // The tool cards, under the sheets: the one in front first.+ // A floating card closes with its job running, which goes+ // on and is there again when it is opened; a phone's sheet+ // busy with a job keeps itself, as its × is gone.+ if (root.speech.open && (root.front-tool == "speech" || !root.captions.open)) {+ if (!root.sheets-fill || !root.speech.running) { root.speech-closed(); }+ return accept;+ }+ if (root.captions.open) {+ if (!root.sheets-fill || !root.captions.running) { root.captions-closed(); }+ return accept;+ }}if (Editor.key-chord(event.text, event.modifiers.shift, event.modifiers.alt,event.modifiers.control, event.modifiers.meta, root.on-start)) {@@ -623,6 +630,91 @@ export component App inherits Window {prompt-speed-changed(index) => { root.voiceover-prompt-speed-changed(index); }}+ // Captions and Text to speech: cards floating over the editor on a+ // desk, like the voiceover panel and under every sheet, so the+ // timeline, the playhead and the selection stay in reach while they+ // are open and a job runs. Whichever was opened or grabbed last is in+ // front, and is the one Escape closes. On a phone, or a window too+ // narrow for a card beside the work, they stay the sheets they were,+ // over everything.+ Rectangle {+ x: 0;+ y: 0;+ width: root.width;+ height: root.height;+ z: root.sheets-fill ? 1 : 0;++ CaptionsDialog {+ fill: root.sheets-fill;+ floating: !root.sheets-fill;+ z: root.front-tool == "captions" ? 1 : 0;+ raised => { root.front-tool = "captions"; }+ inset-top: root.safe-area-insets.top;+ inset-bottom: root.safe-area-insets.bottom;+ inset-left: root.safe-area-insets.left;+ inset-right: root.safe-area-insets.right;+ x: 0;+ y: 0;+ width: root.width;+ height: root.height;+ data: root.captions;+ models: root.caption-models;++ closed => { root.captions-closed(); }+ voice-toggled(on) => { root.captions-voice-toggled(on); }+ source-changed(index) => { root.captions-source-changed(index); }+ text-edited(text) => { root.captions-text-edited(text); }+ model-changed(index) => { root.captions-model-changed(index); }+ placement-changed(index) => { root.captions-placement-changed(index); }+ size-changed(index) => { root.captions-size-changed(index); }+ begin => { root.captions-begin(); }+ cancel => { root.captions-cancel(); }+ import-srt => { root.captions-import-srt(); }+ export-srt => { root.captions-export-srt(); }+ }++ SpeechDialog {+ fill: root.sheets-fill;+ floating: !root.sheets-fill;+ // Its own home, a step in from the captions card's, so the two+ // do not open exactly on top of each other.+ home-right: 44px;+ home-top: 76px;+ z: root.front-tool == "speech" ? 1 : 0;+ raised => { root.front-tool = "speech"; }+ inset-top: root.safe-area-insets.top;+ inset-bottom: root.safe-area-insets.bottom;+ inset-left: root.safe-area-insets.left;+ inset-right: root.safe-area-insets.right;+ x: 0;+ y: 0;+ width: root.width;+ height: root.height;+ data: root.speech;+ models: root.speech-models;+ model-details: root.speech-model-details;+ voices: root.speech-voices;+ voice-details: root.speech-voice-details;+ samples: root.speech-samples;+ sample-waves: root.speech-sample-waves;+ sample-details: root.speech-sample-details;++ closed => { root.speech-closed(); }+ text-edited(text) => { root.speech-text-edited(text); }+ voice-changed(index) => { root.speech-voice-changed(index); }+ model-changed(index) => { root.speech-model-changed(index); }+ speed-changed(value) => { root.speech-speed-changed(value); }+ quality-changed(index) => { root.speech-quality-changed(index); }+ pauses-changed(value) => { root.speech-pauses-changed(value); }+ reference-start-changed(seconds) => { root.speech-reference-start-changed(seconds); }+ use-sample-changed(on) => { root.speech-use-sample-changed(on); }+ sample-changed(index) => { root.speech-sample-changed(index); }+ insert-tag(text, tag, at) => { return root.speech-insert-tag(text, tag, at); }+ begin => { root.speech-begin(); }+ cancel => { root.speech-cancel(); }+ }+ }+// Declared last, and that is load-bearing: Slint paints siblings in// declaration order, so a modal placed above the workspace in the file is// painted *under* it on screen — open, correct, and completely hidden.@@ -810,66 +902,6 @@ export component App inherits Window {}- CaptionsDialog {- fill: root.sheets-fill;- inset-top: root.safe-area-insets.top;- inset-bottom: root.safe-area-insets.bottom;- inset-left: root.safe-area-insets.left;- inset-right: root.safe-area-insets.right;- x: 0;- y: 0;- width: root.width;- height: root.height;- data: root.captions;- models: root.caption-models;-- closed => { root.captions-closed(); }- voice-toggled(on) => { root.captions-voice-toggled(on); }- source-changed(index) => { root.captions-source-changed(index); }- text-edited(text) => { root.captions-text-edited(text); }- model-changed(index) => { root.captions-model-changed(index); }- placement-changed(index) => { root.captions-placement-changed(index); }- size-changed(index) => { root.captions-size-changed(index); }- begin => { root.captions-begin(); }- cancel => { root.captions-cancel(); }- import-srt => { root.captions-import-srt(); }- export-srt => { root.captions-export-srt(); }- }-- SpeechDialog {- fill: root.sheets-fill;- inset-top: root.safe-area-insets.top;- inset-bottom: root.safe-area-insets.bottom;- inset-left: root.safe-area-insets.left;- inset-right: root.safe-area-insets.right;- x: 0;- y: 0;- width: root.width;- height: root.height;- data: root.speech;- models: root.speech-models;- model-details: root.speech-model-details;- voices: root.speech-voices;- voice-details: root.speech-voice-details;- samples: root.speech-samples;- sample-waves: root.speech-sample-waves;- sample-details: root.speech-sample-details;-- closed => { root.speech-closed(); }- text-edited(text) => { root.speech-text-edited(text); }- voice-changed(index) => { root.speech-voice-changed(index); }- model-changed(index) => { root.speech-model-changed(index); }- speed-changed(value) => { root.speech-speed-changed(value); }- quality-changed(index) => { root.speech-quality-changed(index); }- pauses-changed(value) => { root.speech-pauses-changed(value); }- reference-start-changed(seconds) => { root.speech-reference-start-changed(seconds); }- use-sample-changed(on) => { root.speech-use-sample-changed(on); }- sample-changed(index) => { root.speech-sample-changed(index); }- insert-tag(text, tag, at) => { return root.speech-insert-tag(text, tag, at); }- begin => { root.speech-begin(); }- cancel => { root.speech-cancel(); }- }-// Over the workspace and both sheets, under the menus. A tooltip has to// clear the pane it belongs to — panes clip — and it has no business over// an open menu, which is the one surface that would rather not bediff --git a/src/crates/concat/ui/dialogs/captions.slint b/src/crates/concat/ui/dialogs/captions.slintindex d82964c8..68c30598 100644--- a/src/crates/concat/ui/dialogs/captions.slint+++ b/src/crates/concat/ui/dialogs/captions.slint@@ -23,6 +23,12 @@ import { Switch } from "../inspector/controls.slint";// the sheet is the progress report: a transcription knows how far through// the sound it is, so it gets a real bar and not a sweep. The language is// the transcriber's to work out.+//+// On a desk it is a card floating over the editor (see Modal.floating): the+// timeline stays live under it, so the clip it transcribes follows the+// selection - a clip with sound selected is the one shown and run. Closed,+// a transcription goes on, and opened again the card shows it. On a phone+// it is a sheet.export struct CaptionsSheetData {open: bool,diff --git a/src/crates/concat/ui/dialogs/speech.slint b/src/crates/concat/ui/dialogs/speech.slintindex 53d8d37e..a818f105 100644--- a/src/crates/concat/ui/dialogs/speech.slint+++ b/src/crates/concat/ui/dialogs/speech.slint@@ -26,6 +26,10 @@ import { Fmt } from "../util.slint";// from a title the script is the title's words; opened from the menu it is// whatever you type. While it runs the sheet is the progress report, with a// clock beside it.+//+// On a desk it is a card floating over the editor (see Modal.floating), so+// the timeline stays in reach while a read runs; closed, the read goes on,+// and opened again the card shows it. On a phone it is a sheet.export struct SpeechSheetData {open: bool,diff --git a/src/crates/concat/ui/primitives/modal.slint b/src/crates/concat/ui/primitives/modal.slintindex ae4a81c0..89249d42 100644--- a/src/crates/concat/ui/primitives/modal.slint+++ b/src/crates/concat/ui/primitives/modal.slint@@ -14,6 +14,15 @@ import { Select } from "select.slint";// and shown by a property. Everything under it keeps existing — a modal is a// scrim over the app, not a replacement for it — which is why the scrim takes// pointer events rather than merely painting over them.+//+// Or, `floating`, no scrim at all: a card over the editor that the work+// beside it carries on under, the way the voiceover panel floats - for a+// tool that is used beside the timeline rather than instead of it, whose+// job may run for minutes (Captions, Text to speech). Nothing behind it is+// held back; the card goes where its header is dragged, stays wholly+// inside the window, and keeps where it was put for as long as the window+// is open. Its × is always there: closing it does not stop a job, which+// keeps running and is shown again when it is opened again.export component Modal inherits Rectangle {in property <bool> open;in property <string> title;@@ -39,17 +48,35 @@ export component Modal inherits Rectangle {in property <length> inset-bottom;in property <length> inset-left;in property <length> inset-right;+ /// A card over a live editor rather than a sheet over a scrim; see+ /// above. Never with `fill`: a phone keeps its sheet.+ in property <bool> floating;+ /// Where a floating card starts, from the window's top right corner:+ /// under the title strip, over the inspector. Two cards are given+ /// different homes so they do not open exactly on top of each other.+ in property <length> home-right: 16px;+ in property <length> home-top: 48px;callback closed();+ /// A floating card was pressed on its header: it wants to be in front.+ callback raised();visible: root.open;+ // A card opened comes to the front.+ changed open => {+ if (root.open) {+ root.raised();+ }+ }// Dark rather than blurred: Slint has no backdrop filter, and a scrim that// only dims is honest about that where a fake blur would not be.- background: root.fill ? Theme.panel : Theme.overlay;+ background: root.floating ? Colors.transparent+ : root.fill ? Theme.panel : Theme.overlay;// Declared first, so the surface and everything in it sits above and takes// its own presses. A click that lands here landed outside the dialog.- TouchArea {+ // A floating card has none: a click beside it is a click on the editor.+ if !root.floating: TouchArea {clicked => {if (root.dismissable) {root.closed();@@ -61,9 +88,24 @@ export component Modal inherits Rectangle {property <length> room-w: root.width - root.inset-left - root.inset-right;property <length> room-h: root.height - root.inset-top - root.inset-bottom;+ // A floating card's place: its home, moved by however far its header+ // has been dragged, and held inside the window - the drag included, so+ // pulling it back from an edge moves it at once, and a window made+ // smaller brings it along.+ property <length> home-x: root.width - surface.width - root.home-right - root.inset-right;+ property <length> home-y: root.home-top + root.inset-top;+ property <length> dx;+ property <length> dy;+ property <length> min-dx: root.inset-left + 8px - root.home-x;+ property <length> max-dx: root.width - root.inset-right - surface.width - 8px - root.home-x;+ property <length> min-dy: root.inset-top + 8px - root.home-y;+ property <length> max-dy: root.height - root.inset-bottom - surface.height - 8px - root.home-y;+surface := Rectangle {- x: root.inset-left + (root.room-w - self.width) / 2;- y: root.inset-top + (root.room-h - self.height) / 2;+ x: root.floating ? root.home-x + Math.max(root.min-dx, Math.min(root.max-dx, root.dx))+ : root.inset-left + (root.room-w - self.width) / 2;+ y: root.floating ? root.home-y + Math.max(root.min-dy, Math.min(root.max-dy, root.dy))+ : root.inset-top + (root.room-h - self.height) / 2;// Filling, the room inside the bars. Otherwise never wider or taller// than that room less a margin: a phone held upright is narrower than// any sheet's width, and a sheet that runs off both edges cannot be@@ -77,16 +119,39 @@ export component Modal inherits Rectangle {border-width: root.fill ? 0px : 1px;border-color: Theme.line-strong;border-radius: root.fill ? 0px : Theme.r-xl;+ // Lifted off the editor it floats over, as the voiceover card is.+ drop-shadow-blur: root.floating ? 24px : 0px;+ drop-shadow-offset-y: root.floating ? 6px : 0px;+ drop-shadow-color: root.floating ? #00000059 : Colors.transparent;clip: true;// Swallows presses that land on the dialog, so they do not reach the- // scrim underneath and dismiss it.+ // scrim underneath and dismiss it - or, floating, the lanes and the+ // monitor under the card.TouchArea { }body := VerticalLayout {Rectangle {height: 42px;+ // What drags a floating card, under the title and the ×.+ if root.floating: TouchArea {+ mouse-cursor: move;+ pointer-event(event) => {+ if (event.kind == PointerEventKind.down) {+ root.raised();+ }+ }+ moved => {+ if (self.pressed) {+ root.dx = Math.max(root.min-dx, Math.min(root.max-dx,+ root.dx + self.mouse-x - self.pressed-x));+ root.dy = Math.max(root.min-dy, Math.min(root.max-dy,+ root.dy + self.mouse-y - self.pressed-y));+ }+ }+ }+Rectangle {x: 0;y: parent.height - 1px;@@ -115,8 +180,9 @@ export component Modal inherits Rectangle {// Gone, not disabled, while a job is running: there is no// safe answer to pressing it, so it should not be there to- // press.- if root.dismissable: VerticalLayout {+ // press. A floating card's always is: closing it leaves+ // the job running, to be seen again when it is opened.+ if root.dismissable || root.floating: VerticalLayout {alignment: center;Rectangle {width: 26px;