Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(captions, speech): Captions and Text to speech float over the editor on a desk

Jareerf8de11ce7863
diff --git a/src/crates/concat/src/panes/captions.rs b/src/crates/concat/src/panes/captions.rs
index 3c15f6fc..50b5cbe4 100644
--- a/src/crates/concat/src/panes/captions.rs
+++ b/src/crates/concat/src/panes/captions.rs
@@ -104,8 +104,13 @@ pub struct CaptionsPane {
pub from_voice: bool,
/// The clip whose voice is transcribed: one with sound, chosen from the
/// timeline's. Kept while the switch is off, so turning it back on
- /// returns to the same clip.
+ /// returns to the same clip. The panel floats over a live editor, so
+ /// a clip with sound selected since is the one shown and run instead;
+ /// see [`Self::subject_clip`].
pub clip: Option<String>,
+ /// The sole selection when [`Self::clip`] was chosen - opened on, or
+ /// picked from the list - so a later selection can be told from it.
+ chosen_with: Option<String>,
/// The script's words.
pub text: String,
/// Row in the installed transcriber list.
@@ -130,6 +135,9 @@ impl CaptionsPane {
/// this runs the studio's copy of the pane is a blank it must not read.
pub fn update(&mut self, msg: CaptionsMsg, studio: &mut Studio) {
match msg {
+ // A run in progress is shown as it is, however the panel is
+ // asked for again: closing it never stopped the run.
+ CaptionsMsg::Open | CaptionsMsg::OpenOnClip(_) if self.running => self.open = true,
CaptionsMsg::Open => {
let selected = studio.sole_selection();
self.open_on(studio, selected, false);
@@ -147,6 +155,7 @@ impl CaptionsPane {
.is_some_and(|id| sources.iter().any(|s| &s.id == id))
{
self.clip = sources.first().map(|source| source.id.clone());
+ self.chosen_with = studio.sole_selection();
}
}
CaptionsMsg::SourceChanged(index) => {
@@ -155,6 +164,7 @@ impl CaptionsPane {
.and_then(|index| sources(studio).into_iter().nth(index))
{
self.clip = Some(source.id);
+ self.chosen_with = studio.sole_selection();
self.message.clear();
}
}
@@ -217,7 +227,14 @@ impl CaptionsPane {
}
// Asked for: the sheet is already on its way down.
Err(error) if error.contains("cancel") => self.open = false,
- Err(error) => self.message = error,
+ // Closed while it ran: the panel is not there to say
+ // it, so the notice does, and the panel too next time.
+ Err(error) => {
+ if !self.open {
+ studio.notify(&error, true);
+ }
+ self.message = error;
+ }
}
}
// Asked with the studio unborrowed, on a turn of its own; see
@@ -391,6 +408,7 @@ impl CaptionsPane {
open: true,
from_voice,
clip,
+ chosen_with: studio.sole_selection(),
model,
placement: 0,
size: 1,
@@ -399,6 +417,21 @@ impl CaptionsPane {
};
}
+ /// The clip the panel transcribes, and the one its list shows - the
+ /// same answer for both, so it never runs a clip it is not showing.
+ /// The editor stays live under the panel: a clip with sound selected
+ /// since the panel's clip was chosen is the one meant now, and is
+ /// taken. Otherwise the chosen clip while it is still on the timeline
+ /// with sound, and none - nothing to run - when it is not.
+ fn subject_clip(&self, studio: &Studio, sources: &[Source]) -> Option<String> {
+ follow_selection(
+ studio.sole_selection(),
+ self.chosen_with.as_deref(),
+ self.clip.as_deref(),
+ |id| sources.iter().any(|source| source.id == id),
+ )
+ }
+
/// A caption's look, by the sheet's rows: where it sits and its size.
fn look(&self) -> (f64, f64) {
(
@@ -434,7 +467,12 @@ impl CaptionsPane {
/// The sheet's clip through the transcriber on a worker, reporting into
/// the sheet as it goes.
fn run_sound(&mut self, studio: &mut Studio) {
- let Some(clip) = self.clip.as_ref().and_then(|id| studio.clip(id)).cloned() else {
+ // The clip the panel shows, and no other: see `subject_clip`.
+ let Some(clip) = self
+ .subject_clip(studio, &sources(studio))
+ .and_then(|id| studio.clip(&id))
+ .cloned()
+ else {
self.message = t("captions.clipNoLongerTimeline");
return;
};
@@ -508,9 +546,8 @@ impl CaptionsPane {
Vec::new()
};
let source = self
- .clip
- .as_ref()
- .and_then(|id| sources.iter().position(|source| &source.id == id))
+ .subject_clip(studio, &sources)
+ .and_then(|id| sources.iter().position(|source| source.id == id))
.map_or(-1, |index| index as i32);
let labels: Vec<slint::SharedString> = sources
.into_iter()
@@ -634,6 +671,23 @@ fn unique_path(dir: &Path, name: &str) -> PathBuf {
.unwrap_or(first)
}
+/// [`CaptionsPane::subject_clip`] with the studio taken out: `selected`
+/// the sole selection, `chosen_with` the selection when `clip` was chosen,
+/// and `sounds` whether a clip is on the timeline with sound.
+fn follow_selection(
+ selected: Option<String>,
+ chosen_with: Option<&str>,
+ clip: Option<&str>,
+ sounds: impl Fn(&str) -> bool,
+) -> Option<String> {
+ if let Some(selected) = selected.filter(|id| sounds(id))
+ && Some(selected.as_str()) != chosen_with
+ {
+ return Some(selected);
+ }
+ clip.filter(|id| sounds(id)).map(str::to_owned)
+}
+
/// A clip on the timeline whose voice can be captioned, as the sheet's
/// list shows it.
struct Source {
@@ -835,6 +889,40 @@ mod tests {
);
}
+ /// The panel floats over a live editor: a clip with sound selected
+ /// since the panel's clip was chosen is the one it shows and runs; a
+ /// pick from its list holds until the selection moves again; and a
+ /// chosen clip gone from the timeline is not run unseen.
+ #[test]
+ fn the_captioned_clip_follows_a_new_selection_of_a_clip_with_sound() {
+ let sounds = |id: &str| ["a", "b", "c"].contains(&id);
+ let follow = |selected: Option<&str>, chosen_with: Option<&str>, clip: Option<&str>| {
+ super::follow_selection(selected.map(str::to_owned), chosen_with, clip, sounds)
+ };
+ // Opened on A, A still selected: A.
+ assert_eq!(
+ follow(Some("a"), Some("a"), Some("a")).as_deref(),
+ Some("a")
+ );
+ // B selected on the timeline since: B.
+ assert_eq!(
+ follow(Some("b"), Some("a"), Some("a")).as_deref(),
+ Some("b")
+ );
+ // C picked from the list with B selected: C, until the selection moves.
+ assert_eq!(
+ follow(Some("b"), Some("b"), Some("c")).as_deref(),
+ Some("c")
+ );
+ // A title selected - no sound - leaves the choice alone.
+ assert_eq!(
+ follow(Some("t"), Some("b"), Some("c")).as_deref(),
+ Some("c")
+ );
+ // The chosen clip deleted: nothing to run, rather than a clip unseen.
+ assert_eq!(follow(None, None, Some("gone")), None);
+ }
+
#[test]
fn imported_srt_ignores_empty_captions() {
let cues = vec![concat_project::srt::Cue {
diff --git a/src/crates/concat/src/panes/speech.rs b/src/crates/concat/src/panes/speech.rs
index bf5a2ff5..82a19598 100644
--- a/src/crates/concat/src/panes/speech.rs
+++ b/src/crates/concat/src/panes/speech.rs
@@ -147,6 +147,9 @@ impl SpeechPane {
/// this runs the studio's copy of the pane is a blank it must not read.
pub fn update(&mut self, msg: SpeechMsg, studio: &mut Studio) {
match msg {
+ // A reading in progress is shown as it is when the panel is
+ // asked for again: closing it never stopped the reading.
+ SpeechMsg::Open if self.running => self.open = true,
SpeechMsg::Open => {
let title = studio
.sole_selection()
@@ -266,7 +269,14 @@ impl SpeechPane {
}
// Asked for: the sheet is already on its way down.
Err(error) if error.contains("cancel") => self.open = false,
- Err(error) => self.message = error,
+ // Closed while it ran: the panel is not there to say
+ // it, so the notice does, and the panel too next time.
+ Err(error) => {
+ if !self.open {
+ studio.notify(&error, true);
+ }
+ self.message = error;
+ }
}
}
}
diff --git a/src/crates/concat/ui/app.slint b/src/crates/concat/ui/app.slint
index b598fd28..ea630861 100644
--- a/src/crates/concat/ui/app.slint
+++ b/src/crates/concat/ui/app.slint
@@ -176,6 +176,9 @@ export component App inherits Window {
/// area, on a phone and in any window too narrow for it to float as a
/// card with room around it.
property <bool> sheets-fill: root.phone || root.width < 600px;
+ /// Which floating tool card is in front: "captions" or "speech". The
+ /// one opened or grabbed last; Escape closes it first.
+ property <string> front-tool: "captions";
/// The File / Edit / View menus. Built in Rust like every other menu here,
/// because what belongs on them depends on the project.
@@ -483,14 +486,6 @@ export component App inherits Window {
if (event.text == Key.Escape) {
if (root.confirm.open) { root.confirm-dismissed(); return accept; }
if (root.relink.open) { root.relink-dismiss(); return accept; }
- if (root.speech.open) {
- if (!root.speech.running) { root.speech-closed(); }
- return accept;
- }
- if (root.captions.open) {
- if (!root.captions.running) { root.captions-closed(); }
- return accept;
- }
if (root.project-sheet.open) { root.project-closed(); return accept; }
if (root.settings.open) { root.settings-closed(); return accept; }
if (root.export.open) {
@@ -501,6 +496,18 @@ export component App inherits Window {
if (!root.start.busy) { root.start-dismiss(); }
return accept;
}
+ // The tool cards, under the sheets: the one in front first.
+ // A floating card closes with its job running, which goes
+ // on and is there again when it is opened; a phone's sheet
+ // busy with a job keeps itself, as its × is gone.
+ if (root.speech.open && (root.front-tool == "speech" || !root.captions.open)) {
+ if (!root.sheets-fill || !root.speech.running) { root.speech-closed(); }
+ return accept;
+ }
+ if (root.captions.open) {
+ if (!root.sheets-fill || !root.captions.running) { root.captions-closed(); }
+ return accept;
+ }
}
if (Editor.key-chord(event.text, event.modifiers.shift, event.modifiers.alt,
event.modifiers.control, event.modifiers.meta, root.on-start)) {
@@ -623,6 +630,91 @@ export component App inherits Window {
prompt-speed-changed(index) => { root.voiceover-prompt-speed-changed(index); }
}
+ // Captions and Text to speech: cards floating over the editor on a
+ // desk, like the voiceover panel and under every sheet, so the
+ // timeline, the playhead and the selection stay in reach while they
+ // are open and a job runs. Whichever was opened or grabbed last is in
+ // front, and is the one Escape closes. On a phone, or a window too
+ // narrow for a card beside the work, they stay the sheets they were,
+ // over everything.
+ Rectangle {
+ x: 0;
+ y: 0;
+ width: root.width;
+ height: root.height;
+ z: root.sheets-fill ? 1 : 0;
+
+ CaptionsDialog {
+ fill: root.sheets-fill;
+ floating: !root.sheets-fill;
+ z: root.front-tool == "captions" ? 1 : 0;
+ raised => { root.front-tool = "captions"; }
+ inset-top: root.safe-area-insets.top;
+ inset-bottom: root.safe-area-insets.bottom;
+ inset-left: root.safe-area-insets.left;
+ inset-right: root.safe-area-insets.right;
+ x: 0;
+ y: 0;
+ width: root.width;
+ height: root.height;
+ data: root.captions;
+ models: root.caption-models;
+
+ closed => { root.captions-closed(); }
+ voice-toggled(on) => { root.captions-voice-toggled(on); }
+ source-changed(index) => { root.captions-source-changed(index); }
+ text-edited(text) => { root.captions-text-edited(text); }
+ model-changed(index) => { root.captions-model-changed(index); }
+ placement-changed(index) => { root.captions-placement-changed(index); }
+ size-changed(index) => { root.captions-size-changed(index); }
+ begin => { root.captions-begin(); }
+ cancel => { root.captions-cancel(); }
+ import-srt => { root.captions-import-srt(); }
+ export-srt => { root.captions-export-srt(); }
+ }
+
+ SpeechDialog {
+ fill: root.sheets-fill;
+ floating: !root.sheets-fill;
+ // Its own home, a step in from the captions card's, so the two
+ // do not open exactly on top of each other.
+ home-right: 44px;
+ home-top: 76px;
+ z: root.front-tool == "speech" ? 1 : 0;
+ raised => { root.front-tool = "speech"; }
+ inset-top: root.safe-area-insets.top;
+ inset-bottom: root.safe-area-insets.bottom;
+ inset-left: root.safe-area-insets.left;
+ inset-right: root.safe-area-insets.right;
+ x: 0;
+ y: 0;
+ width: root.width;
+ height: root.height;
+ data: root.speech;
+ models: root.speech-models;
+ model-details: root.speech-model-details;
+ voices: root.speech-voices;
+ voice-details: root.speech-voice-details;
+ samples: root.speech-samples;
+ sample-waves: root.speech-sample-waves;
+ sample-details: root.speech-sample-details;
+
+ closed => { root.speech-closed(); }
+ text-edited(text) => { root.speech-text-edited(text); }
+ voice-changed(index) => { root.speech-voice-changed(index); }
+ model-changed(index) => { root.speech-model-changed(index); }
+ speed-changed(value) => { root.speech-speed-changed(value); }
+ quality-changed(index) => { root.speech-quality-changed(index); }
+ pauses-changed(value) => { root.speech-pauses-changed(value); }
+ reference-start-changed(seconds) => { root.speech-reference-start-changed(seconds); }
+ use-sample-changed(on) => { root.speech-use-sample-changed(on); }
+ sample-changed(index) => { root.speech-sample-changed(index); }
+ insert-tag(text, tag, at) => { return root.speech-insert-tag(text, tag, at); }
+ begin => { root.speech-begin(); }
+ cancel => { root.speech-cancel(); }
+ }
+ }
+
// Declared last, and that is load-bearing: Slint paints siblings in
// declaration order, so a modal placed above the workspace in the file is
// painted *under* it on screen — open, correct, and completely hidden.
@@ -810,66 +902,6 @@ export component App inherits Window {
}
- CaptionsDialog {
- fill: root.sheets-fill;
- inset-top: root.safe-area-insets.top;
- inset-bottom: root.safe-area-insets.bottom;
- inset-left: root.safe-area-insets.left;
- inset-right: root.safe-area-insets.right;
- x: 0;
- y: 0;
- width: root.width;
- height: root.height;
- data: root.captions;
- models: root.caption-models;
-
- closed => { root.captions-closed(); }
- voice-toggled(on) => { root.captions-voice-toggled(on); }
- source-changed(index) => { root.captions-source-changed(index); }
- text-edited(text) => { root.captions-text-edited(text); }
- model-changed(index) => { root.captions-model-changed(index); }
- placement-changed(index) => { root.captions-placement-changed(index); }
- size-changed(index) => { root.captions-size-changed(index); }
- begin => { root.captions-begin(); }
- cancel => { root.captions-cancel(); }
- import-srt => { root.captions-import-srt(); }
- export-srt => { root.captions-export-srt(); }
- }
-
- SpeechDialog {
- fill: root.sheets-fill;
- inset-top: root.safe-area-insets.top;
- inset-bottom: root.safe-area-insets.bottom;
- inset-left: root.safe-area-insets.left;
- inset-right: root.safe-area-insets.right;
- x: 0;
- y: 0;
- width: root.width;
- height: root.height;
- data: root.speech;
- models: root.speech-models;
- model-details: root.speech-model-details;
- voices: root.speech-voices;
- voice-details: root.speech-voice-details;
- samples: root.speech-samples;
- sample-waves: root.speech-sample-waves;
- sample-details: root.speech-sample-details;
-
- closed => { root.speech-closed(); }
- text-edited(text) => { root.speech-text-edited(text); }
- voice-changed(index) => { root.speech-voice-changed(index); }
- model-changed(index) => { root.speech-model-changed(index); }
- speed-changed(value) => { root.speech-speed-changed(value); }
- quality-changed(index) => { root.speech-quality-changed(index); }
- pauses-changed(value) => { root.speech-pauses-changed(value); }
- reference-start-changed(seconds) => { root.speech-reference-start-changed(seconds); }
- use-sample-changed(on) => { root.speech-use-sample-changed(on); }
- sample-changed(index) => { root.speech-sample-changed(index); }
- insert-tag(text, tag, at) => { return root.speech-insert-tag(text, tag, at); }
- begin => { root.speech-begin(); }
- cancel => { root.speech-cancel(); }
- }
-
// Over the workspace and both sheets, under the menus. A tooltip has to
// clear the pane it belongs to — panes clip — and it has no business over
// an open menu, which is the one surface that would rather not be
diff --git a/src/crates/concat/ui/dialogs/captions.slint b/src/crates/concat/ui/dialogs/captions.slint
index d82964c8..68c30598 100644
--- a/src/crates/concat/ui/dialogs/captions.slint
+++ b/src/crates/concat/ui/dialogs/captions.slint
@@ -23,6 +23,12 @@ import { Switch } from "../inspector/controls.slint";
// the sheet is the progress report: a transcription knows how far through
// the sound it is, so it gets a real bar and not a sweep. The language is
// the transcriber's to work out.
+//
+// On a desk it is a card floating over the editor (see Modal.floating): the
+// timeline stays live under it, so the clip it transcribes follows the
+// selection - a clip with sound selected is the one shown and run. Closed,
+// a transcription goes on, and opened again the card shows it. On a phone
+// it is a sheet.
export struct CaptionsSheetData {
open: bool,
diff --git a/src/crates/concat/ui/dialogs/speech.slint b/src/crates/concat/ui/dialogs/speech.slint
index 53d8d37e..a818f105 100644
--- a/src/crates/concat/ui/dialogs/speech.slint
+++ b/src/crates/concat/ui/dialogs/speech.slint
@@ -26,6 +26,10 @@ import { Fmt } from "../util.slint";
// from a title the script is the title's words; opened from the menu it is
// whatever you type. While it runs the sheet is the progress report, with a
// clock beside it.
+//
+// On a desk it is a card floating over the editor (see Modal.floating), so
+// the timeline stays in reach while a read runs; closed, the read goes on,
+// and opened again the card shows it. On a phone it is a sheet.
export struct SpeechSheetData {
open: bool,
diff --git a/src/crates/concat/ui/primitives/modal.slint b/src/crates/concat/ui/primitives/modal.slint
index ae4a81c0..89249d42 100644
--- a/src/crates/concat/ui/primitives/modal.slint
+++ b/src/crates/concat/ui/primitives/modal.slint
@@ -14,6 +14,15 @@ import { Select } from "select.slint";
// and shown by a property. Everything under it keeps existing — a modal is a
// scrim over the app, not a replacement for it — which is why the scrim takes
// pointer events rather than merely painting over them.
+//
+// Or, `floating`, no scrim at all: a card over the editor that the work
+// beside it carries on under, the way the voiceover panel floats - for a
+// tool that is used beside the timeline rather than instead of it, whose
+// job may run for minutes (Captions, Text to speech). Nothing behind it is
+// held back; the card goes where its header is dragged, stays wholly
+// inside the window, and keeps where it was put for as long as the window
+// is open. Its × is always there: closing it does not stop a job, which
+// keeps running and is shown again when it is opened again.
export component Modal inherits Rectangle {
in property <bool> open;
in property <string> title;
@@ -39,17 +48,35 @@ export component Modal inherits Rectangle {
in property <length> inset-bottom;
in property <length> inset-left;
in property <length> inset-right;
+ /// A card over a live editor rather than a sheet over a scrim; see
+ /// above. Never with `fill`: a phone keeps its sheet.
+ in property <bool> floating;
+ /// Where a floating card starts, from the window's top right corner:
+ /// under the title strip, over the inspector. Two cards are given
+ /// different homes so they do not open exactly on top of each other.
+ in property <length> home-right: 16px;
+ in property <length> home-top: 48px;
callback closed();
+ /// A floating card was pressed on its header: it wants to be in front.
+ callback raised();
visible: root.open;
+ // A card opened comes to the front.
+ changed open => {
+ if (root.open) {
+ root.raised();
+ }
+ }
// Dark rather than blurred: Slint has no backdrop filter, and a scrim that
// only dims is honest about that where a fake blur would not be.
- background: root.fill ? Theme.panel : Theme.overlay;
+ background: root.floating ? Colors.transparent
+ : root.fill ? Theme.panel : Theme.overlay;
// Declared first, so the surface and everything in it sits above and takes
// its own presses. A click that lands here landed outside the dialog.
- TouchArea {
+ // A floating card has none: a click beside it is a click on the editor.
+ if !root.floating: TouchArea {
clicked => {
if (root.dismissable) {
root.closed();
@@ -61,9 +88,24 @@ export component Modal inherits Rectangle {
property <length> room-w: root.width - root.inset-left - root.inset-right;
property <length> room-h: root.height - root.inset-top - root.inset-bottom;
+ // A floating card's place: its home, moved by however far its header
+ // has been dragged, and held inside the window - the drag included, so
+ // pulling it back from an edge moves it at once, and a window made
+ // smaller brings it along.
+ property <length> home-x: root.width - surface.width - root.home-right - root.inset-right;
+ property <length> home-y: root.home-top + root.inset-top;
+ property <length> dx;
+ property <length> dy;
+ property <length> min-dx: root.inset-left + 8px - root.home-x;
+ property <length> max-dx: root.width - root.inset-right - surface.width - 8px - root.home-x;
+ property <length> min-dy: root.inset-top + 8px - root.home-y;
+ property <length> max-dy: root.height - root.inset-bottom - surface.height - 8px - root.home-y;
+
surface := Rectangle {
- x: root.inset-left + (root.room-w - self.width) / 2;
- y: root.inset-top + (root.room-h - self.height) / 2;
+ x: root.floating ? root.home-x + Math.max(root.min-dx, Math.min(root.max-dx, root.dx))
+ : root.inset-left + (root.room-w - self.width) / 2;
+ y: root.floating ? root.home-y + Math.max(root.min-dy, Math.min(root.max-dy, root.dy))
+ : root.inset-top + (root.room-h - self.height) / 2;
// Filling, the room inside the bars. Otherwise never wider or taller
// than that room less a margin: a phone held upright is narrower than
// any sheet's width, and a sheet that runs off both edges cannot be
@@ -77,16 +119,39 @@ export component Modal inherits Rectangle {
border-width: root.fill ? 0px : 1px;
border-color: Theme.line-strong;
border-radius: root.fill ? 0px : Theme.r-xl;
+ // Lifted off the editor it floats over, as the voiceover card is.
+ drop-shadow-blur: root.floating ? 24px : 0px;
+ drop-shadow-offset-y: root.floating ? 6px : 0px;
+ drop-shadow-color: root.floating ? #00000059 : Colors.transparent;
clip: true;
// Swallows presses that land on the dialog, so they do not reach the
- // scrim underneath and dismiss it.
+ // scrim underneath and dismiss it - or, floating, the lanes and the
+ // monitor under the card.
TouchArea { }
body := VerticalLayout {
Rectangle {
height: 42px;
+ // What drags a floating card, under the title and the ×.
+ if root.floating: TouchArea {
+ mouse-cursor: move;
+ pointer-event(event) => {
+ if (event.kind == PointerEventKind.down) {
+ root.raised();
+ }
+ }
+ moved => {
+ if (self.pressed) {
+ root.dx = Math.max(root.min-dx, Math.min(root.max-dx,
+ root.dx + self.mouse-x - self.pressed-x));
+ root.dy = Math.max(root.min-dy, Math.min(root.max-dy,
+ root.dy + self.mouse-y - self.pressed-y));
+ }
+ }
+ }
+
Rectangle {
x: 0;
y: parent.height - 1px;
@@ -115,8 +180,9 @@ export component Modal inherits Rectangle {
// Gone, not disabled, while a job is running: there is no
// safe answer to pressing it, so it should not be there to
- // press.
- if root.dismissable: VerticalLayout {
+ // press. A floating card's always is: closing it leaves
+ // the job running, to be seen again when it is opened.
+ if root.dismissable || root.floating: VerticalLayout {
alignment: center;
Rectangle {
width: 26px;