Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

Let the selection decide what Captions captions

Jareerda9849bf9fc5
diff --git a/engine/crates/concat/locales/en.json b/engine/crates/concat/locales/en.json
index 4b4309e2..74f06d4f 100644
--- a/engine/crates/concat/locales/en.json
+++ b/engine/crates/concat/locales/en.json
@@ -43,7 +43,6 @@
"Application": "Application",
"Apply": "Apply",
"at the playhead": "at the playhead",
- "at the selected clip": "at the selected clip",
"Audio": "Audio",
"audio effect": "audio effect",
"Audio Effects": "Audio Effects",
@@ -94,7 +93,7 @@
"Cancel export": "Cancel export",
"Caption": "Caption",
"Captions": "Captions",
- "Captions from the timeline's sound, or from a script": "Captions from the timeline's sound, or from a script",
+ "Captions from a script, at the playhead": "Captions from a script, at the playhead",
"Cathedral": "Cathedral",
"Center": "Center",
"Centre": "Centre",
@@ -400,15 +399,12 @@
"not installed": "not installed",
"Nothing found to keep here, so the picture shows as shot. Try another subject, or custom removal.": "Nothing found to keep here, so the picture shows as shot. Try another subject, or custom removal.",
"Nothing of that kind in this project.": "Nothing of that kind in this project.",
- "Nothing on the timeline has sound to transcribe": "Nothing on the timeline has sound to transcribe",
- "Nothing on the timeline has sound to transcribe. Select a clip with sound, or type a script instead.": "Nothing on the timeline has sound to transcribe. Select a clip with sound, or type a script instead.",
"Nothing starred yet — hover a tile and press its star.": "Nothing starred yet — hover a tile and press its star.",
"Nothing to caption yet": "Nothing to caption yet",
"Nothing to export": "Nothing to export",
"Nothing to read yet": "Nothing to read yet",
"Nothing under the playhead": "Nothing under the playhead",
"Nothing was said in that clip": "Nothing was said in that clip",
- "Nothing was said in those clips": "Nothing was said in those clips",
"Object": "Object",
"off": "off",
"Off, a click on the ruler past the last clip lands the playhead there, and a clip placed at the playhead goes with it. On, the playhead never leaves the content.": "Off, a click on the ruler past the last clip lands the playhead there, and a clip placed at the playhead goes with it. On, the playhead never leaves the content.",
@@ -594,6 +590,7 @@
"The bulge of a very wide lens.": "The bulge of a very wide lens.",
"The café look: warm brown through the middle, greens muted to olive, contrast eased. Cosy, and kind to wood and skin alike.": "The café look: warm brown through the middle, greens muted to olive, contrast eased. Cosy, and kind to wood and skin alike.",
"The chain runs top to bottom — reorder it with the chevrons. The eye bypasses one link without losing the values you dialled into it.": "The chain runs top to bottom — reorder it with the chevrons. The eye bypasses one link without losing the values you dialled into it.",
+ "The clip is no longer on the timeline": "The clip is no longer on the timeline",
"The early reflections of a room.": "The early reflections of a room.",
"The instant print: a cream matte, warm highlights, cyan creeping into the shadows, a little softness in the corners.": "The instant print: a cream matte, warm highlights, cyan creeping into the shadows, a little softness in the corners.",
"The left half, reflected onto the right.": "The left half, reflected onto the right.",
@@ -664,7 +661,6 @@
"Zoom out": "Zoom out",
"Zoom to fit": "Zoom to fit",
"{0} channels": "{0} channels",
- "{0} clips with sound": "{0} clips with sound",
"{0} days ago": "{0} days ago",
"{0} Hz": "{0} Hz",
"{0} installed · {1} MB on disk": "{0} installed · {1} MB on disk",
diff --git a/engine/crates/concat/src/lib.rs b/engine/crates/concat/src/lib.rs
index a5d89792..d96a2f05 100644
--- a/engine/crates/concat/src/lib.rs
+++ b/engine/crates/concat/src/lib.rs
@@ -1001,9 +1001,6 @@ pub fn run() -> Result<(), slint::PlatformError> {
app.on_captions_closed(on_window!(|state| {
state.captions.open = false;
}));
- app.on_captions_source_changed(on_window!(|state, index: i32| {
- state.captions.source = (index.max(0) as usize).min(1);
- }));
app.on_captions_text_edited(on_window!(|state, text: SharedString| {
state.captions.text = text.to_string();
}));
diff --git a/engine/crates/concat/src/studio.rs b/engine/crates/concat/src/studio.rs
index 211de278..12d989d4 100644
--- a/engine/crates/concat/src/studio.rs
+++ b/engine/crates/concat/src/studio.rs
@@ -168,12 +168,10 @@ pub struct ProjectSheet {
#[derive(Default)]
pub struct CaptionsSheet {
pub open: bool,
- /// 0 the sound, 1 a script.
- pub source: usize,
- /// The clips the sound source transcribes: the selected clip with
- /// sound, or every clip with sound on the timeline.
- pub clips: Vec<String>,
- /// The script source's words.
+ /// The clip being transcribed: the one selected when the sheet opened,
+ /// when it had sound. None, and the sheet is a script instead.
+ pub clip: Option<String>,
+ /// The script's words.
pub text: String,
/// Row in the installed transcriber list.
pub model: usize,
@@ -4650,7 +4648,7 @@ impl Studio {
.any(|other| other.detached_from.as_deref() == Some(clip.id.as_str()));
let video = clip.kind == model::ClipKind::Video;
(
- matches!(clip.kind, model::ClipKind::Video | model::ClipKind::Audio),
+ self.clip_has_sound(clip),
clip.kind == model::ClipKind::Text,
video && !detached,
(video && detached)
@@ -4658,6 +4656,20 @@ impl Studio {
)
}
+ /// Whether the clip has sound to transcribe: an audio clip, or a video
+ /// clip whose file carries an audio stream. A silent video is not a
+ /// sound source, however much it looks like one.
+ fn clip_has_sound(&self, clip: &Clip) -> bool {
+ match clip.kind {
+ model::ClipKind::Audio => true,
+ model::ClipKind::Video => self
+ .project()
+ .media_by_id(&clip.media_id)
+ .is_some_and(|media| media.has_audio),
+ _ => false,
+ }
+ }
+
// ── projects ──
/// Opens a project as the session and leaves the launch screen.
@@ -5098,33 +5110,20 @@ impl Studio {
models.iter().filter(|model| model.installed).collect()
}
- /// Opens the captions sheet with the chosen model already picked. The
- /// sound source is the selected clip when it has sound, else every clip
- /// with sound on the timeline; with neither, the sheet opens on the
- /// script instead.
+ /// Opens the captions sheet with the chosen model already picked. What
+ /// it captions is decided here, not asked: the selected clip's sound
+ /// when one clip with sound is selected, and a script otherwise.
pub fn captions_open(&mut self) {
let installed = Self::installed(&self.transcribers);
let model = installed.iter().position(|model| model.active).unwrap_or(0);
- let has_sound =
- |clip: &Clip| matches!(clip.kind, model::ClipKind::Video | model::ClipKind::Audio);
- let selected = self
+ let clip = self
.sole_selection()
.and_then(|id| self.clip(&id))
- .filter(|clip| has_sound(clip))
+ .filter(|clip| self.clip_has_sound(clip))
.map(|clip| clip.id.clone());
- let clips: Vec<String> = match selected {
- Some(id) => vec![id],
- None => {
- let mut clips: Vec<&Clip> =
- self.timeline().clips.iter().filter(|clip| has_sound(clip)).collect();
- clips.sort_by(|a, b| a.start.total_cmp(&b.start));
- clips.into_iter().map(|clip| clip.id.clone()).collect()
- }
- };
self.captions = CaptionsSheet {
open: true,
- source: usize::from(clips.is_empty()),
- clips,
+ clip,
model,
placement: 0,
size: 1,
@@ -5136,10 +5135,10 @@ impl Studio {
/// transcriber, or the script cut into lines. Either lands as one batch
/// of title clips - one undo step.
pub fn captions_run(&mut self) {
- if self.captions.source == 1 {
- self.captions_from_script();
- } else {
+ if self.captions.clip.is_some() {
self.captions_from_sound();
+ } else {
+ self.captions_from_script();
}
}
@@ -5168,8 +5167,7 @@ impl Studio {
}
}
- /// The script as titles, one after another from the selected clip's
- /// start or the playhead.
+ /// The script as titles, one after another from the playhead.
fn captions_from_script(&mut self) {
let lines = script_captions(&self.captions.text);
if lines.is_empty() {
@@ -5177,7 +5175,7 @@ impl Studio {
return;
}
let look = self.caption_look();
- let mut at = self.script_landing();
+ let mut at = f64::from(self.playhead);
let commands: Vec<Command> = lines
.into_iter()
.map(|(text, seconds)| {
@@ -5192,28 +5190,23 @@ impl Studio {
self.notify(&tf("Added {0} captions", &[&count]), false);
}
- /// Where a script's first caption goes: the selected clip's start, or
- /// the playhead.
- fn script_landing(&self) -> f64 {
- self.sole_selection()
- .and_then(|id| self.clip(&id))
- .map(|clip| clip.start)
- .unwrap_or(f64::from(self.playhead))
- }
-
- /// The sheet's clips through the transcriber, one after another on a
- /// worker, reporting into the sheet as they go.
+ /// The sheet's clip through the transcriber on a worker, reporting into
+ /// the sheet as it goes.
fn captions_from_sound(&mut self) {
- let clips: Vec<Clip> = self
+ let Some(clip) = self
.captions
- .clips
- .iter()
- .filter_map(|id| self.clip(id).cloned())
- .collect();
- if clips.is_empty() {
- self.captions.message = t("Nothing on the timeline has sound to transcribe");
+ .clip
+ .as_ref()
+ .and_then(|id| self.clip(id))
+ .cloned()
+ else {
+ self.captions.message = t("The clip is no longer on the timeline");
return;
- }
+ };
+ let Some(media) = self.project().media_by_id(&clip.media_id).cloned() else {
+ self.captions.message = t("This clip has no file to transcribe");
+ return;
+ };
let Some(model) = Self::installed(&self.transcribers)
.get(self.captions.model)
.map(|model| model.id.clone())
@@ -5222,23 +5215,13 @@ impl Studio {
t("Download a transcriber model in Settings › Transcriber first");
return;
};
- let requests: Vec<_> = clips
- .iter()
- .filter_map(|clip| {
- let media = self.project().media_by_id(&clip.media_id)?;
- Some(concat_speech::transcribe::TranscribeRequest {
- path: media.path.clone(),
- audio_stream: clip.audio_stream,
- source_start: clip.source_start,
- window: clip.duration * clip.speed,
- model_id: model.clone(),
- })
- })
- .collect();
- if requests.len() != clips.len() {
- self.captions.message = t("This clip has no file to transcribe");
- return;
- }
+ let request = concat_speech::transcribe::TranscribeRequest {
+ path: media.path.clone(),
+ audio_stream: clip.audio_stream,
+ source_start: clip.source_start,
+ window: clip.duration * clip.speed,
+ model_id: model,
+ };
let look = self.caption_look();
let dirs = self.host.dirs.clone();
let transcriber = Arc::clone(&self.host.transcriber);
@@ -5247,50 +5230,34 @@ impl Studio {
self.captions.message.clear();
spawn(
move || {
- let total = requests.len();
- let mut heard = Vec::with_capacity(total);
- for (index, request) in requests.iter().enumerate() {
- let segments = transcriber.transcribe(&dirs, request, move |percent| {
- // One bar across every clip: this clip's share of it,
- // as far as whisper has got through the clip.
- let fraction = (index as f32 + (percent as f32 / 100.0).clamp(0.0, 1.0))
- / total as f32;
- on_ui(move |studio, _, _| studio.captions.progress = fraction);
- })?;
- heard.push(segments);
- }
- Ok::<_, String>(heard)
+ transcriber.transcribe(&dirs, &request, |percent| {
+ on_ui(move |studio, _, _| {
+ studio.captions.progress = (percent as f32 / 100.0).clamp(0.0, 1.0);
+ });
+ })
},
move |studio, _, _, result| {
studio.captions.running = false;
match result {
- Ok(heard) => {
- let commands: Vec<Command> = clips
- .iter()
- .zip(heard)
- .flat_map(|(clip, segments)| {
- segments.into_iter().filter_map(move |segment| {
- let text = segment.text.trim().to_owned();
- (!text.is_empty()).then(|| {
- Self::caption_clip(
- text,
- clip.start + segment.start / clip.speed,
- ((segment.end - segment.start) / clip.speed).max(0.2),
- look,
- )
- })
+ Ok(segments) => {
+ let commands: Vec<Command> = segments
+ .into_iter()
+ .filter_map(|segment| {
+ let text = segment.text.trim().to_owned();
+ (!text.is_empty()).then(|| {
+ Self::caption_clip(
+ text,
+ clip.start + segment.start / clip.speed,
+ ((segment.end - segment.start) / clip.speed).max(0.2),
+ look,
+ )
})
})
.collect();
let count = commands.len();
studio.captions.open = false;
if count == 0 {
- let quiet = if clips.len() == 1 {
- t("Nothing was said in that clip")
- } else {
- t("Nothing was said in those clips")
- };
- studio.notify(&quiet, true);
+ studio.notify(&t("Nothing was said in that clip"), true);
} else {
studio.apply(Command::Batch { commands });
studio.notify(&tf("Added {0} captions", &[&count]), false);
@@ -6525,19 +6492,14 @@ impl Studio {
);
app.set_captions(CaptionsSheetData {
open: self.captions.open,
- source: self.captions.source as i32,
- subject: match self.captions.clips.as_slice() {
- [] => String::new(),
- [id] => self.clip(id).map(|clip| clip.name.clone()).unwrap_or_default(),
- many => tf("{0} clips with sound", &[&many.len()]),
- }
- .into(),
- landing: if self.sole_selection().is_some() {
- t("at the selected clip")
- } else {
- t("at the playhead")
- }
- .into(),
+ from_sound: self.captions.clip.is_some(),
+ subject: self
+ .captions
+ .clip
+ .as_ref()
+ .and_then(|id| self.clip(id))
+ .map(|clip| SharedString::from(clip.name.as_str()))
+ .unwrap_or_else(|| t("at the playhead").into()),
text: self.captions.text.as_str().into(),
model: self.captions.model as i32,
placement: self.captions.placement as i32,
diff --git a/engine/crates/concat/ui/app.slint b/engine/crates/concat/ui/app.slint
index baf64e37..97315df4 100644
--- a/engine/crates/concat/ui/app.slint
+++ b/engine/crates/concat/ui/app.slint
@@ -208,7 +208,6 @@ export component App inherits Window {
in property <CaptionsSheetData> captions;
in property <[string]> caption-models;
callback captions-closed();
- callback captions-source-changed(index: int);
callback captions-text-edited(text: string);
callback captions-model-changed(index: int);
callback captions-placement-changed(index: int);
@@ -494,7 +493,6 @@ export component App inherits Window {
models: root.caption-models;
closed => { root.captions-closed(); }
- source-changed(index) => { root.captions-source-changed(index); }
text-edited(text) => { root.captions-text-edited(text); }
model-changed(index) => { root.captions-model-changed(index); }
placement-changed(index) => { root.captions-placement-changed(index); }
diff --git a/engine/crates/concat/ui/dialogs/captions.slint b/engine/crates/concat/ui/dialogs/captions.slint
index 36851fef..06652185 100644
--- a/engine/crates/concat/ui/dialogs/captions.slint
+++ b/engine/crates/concat/ui/dialogs/captions.slint
@@ -10,24 +10,24 @@ import { Select } from "../primitives/select.slint";
// Captions: what the tray's Captions tool opens.
//
-// Two places the words can come from. The sound: the selected clip's, or
-// every clip with sound on the timeline when nothing is selected, run
-// through the transcriber and cut to the words' own timing. A script: the
-// words typed here, one title per sentence at reading pace, laid down one
-// after another. Then where the lines sit and how big they are, and one
-// button. While it runs the sheet is the progress report: a transcription
-// knows how far through the sound it is, so it gets a real bar and not a
-// sweep. The language is the transcriber's to work out.
+// Two sheets in one, and which is decided before it opens rather than
+// asked. With a clip that has sound selected, the words are that clip's:
+// through the transcriber, cut to the words' own timing. Otherwise they
+// are a script typed here, one title per sentence at reading pace, laid
+// down from the playhead. Asking would only invite the wrong answer - a
+// silent video sent to the transcriber - so the selection decides. Then
+// where the lines sit and how big they are, and one button. While it runs
+// the sheet is the progress report: a transcription knows how far through
+// the sound it is, so it gets a real bar and not a sweep. The language is
+// the transcriber's to work out.
export struct CaptionsSheetData {
open: bool,
- /// 0 Sound, 1 Script
- source: int,
- /// The sound to transcribe: the selected clip's name, or "4 clips with
- /// sound"; empty when the timeline has none.
+ /// The selected clip has sound, and the sheet transcribes it.
+ from-sound: bool,
+ /// For the header's note: the clip being transcribed, or where a
+ /// script's captions land.
subject: string,
- /// Where a script's captions land, for the header's note.
- landing: string,
text: string,
/// Row in the installed transcriber list.
model: int,
@@ -49,7 +49,6 @@ export component CaptionsDialog inherits Modal {
/// The installed transcriber models, by name.
in property <[string]> models;
- callback source-changed(index: int);
callback text-edited(text: string);
callback model-changed(index: int);
callback placement-changed(index: int);
@@ -57,14 +56,14 @@ export component CaptionsDialog inherits Modal {
callback begin();
callback cancel();
- property <bool> from-sound: root.data.source == 0;
+ property <bool> from-sound: root.data.from-sound;
open: root.data.open;
title: I18n.t("Captions");
glyph: Glyph.text-mark;
surface-width: 480px;
dismissable: !root.data.running;
- note: root.from-sound ? root.data.subject : root.data.landing;
+ note: root.data.subject;
// The script being typed, held apart from `data.text` the way the Field
// primitive holds its draft: a TextInput's binding is dropped on the
@@ -82,18 +81,8 @@ export component CaptionsDialog inherits Modal {
init => { root.draft = root.data.text; }
VerticalLayout {
- if !root.data.running: DialogSection {
- first: true;
- title: I18n.t("Source");
-
- SegmentedControl {
- options: [I18n.t("Sound"), I18n.t("Script")];
- current: root.data.source;
- changed(index) => { root.source-changed(index); }
- }
- }
-
if !root.data.running && root.from-sound: DialogSection {
+ first: true;
title: I18n.t("Transcriber");
DialogRow {
@@ -113,16 +102,10 @@ export component CaptionsDialog inherits Modal {
font-size: Theme.fs-sm;
wrap: word-wrap;
}
-
- if root.data.subject == "": Text {
- text: I18n.t("Nothing on the timeline has sound to transcribe. Select a clip with sound, or type a script instead.");
- color: Theme.fg-muted;
- font-size: Theme.fs-sm;
- wrap: word-wrap;
- }
}
if !root.data.running && !root.from-sound: DialogSection {
+ first: true;
title: I18n.t("Script");
Rectangle {
@@ -265,9 +248,7 @@ export component CaptionsDialog inherits Modal {
large: true;
glyph: Glyph.text-mark;
label: root.from-sound ? I18n.t("Generate captions") : I18n.t("Add captions");
- enabled: root.from-sound
- ? root.data.ready && root.data.subject != ""
- : root.draft != "";
+ enabled: root.from-sound ? root.data.ready : root.draft != "";
clicked => { root.begin(); }
}
}
diff --git a/engine/crates/concat/ui/timeline/tray.slint b/engine/crates/concat/ui/timeline/tray.slint
index 68b80a02..d80e68b6 100644
--- a/engine/crates/concat/ui/timeline/tray.slint
+++ b/engine/crates/concat/ui/timeline/tray.slint
@@ -235,7 +235,7 @@ export component TimelineTray inherits Rectangle {
glyph: Glyph.text-mark;
caption: I18n.t("Captions");
label: root.sound-selected ? I18n.t("Auto captions from this clip's sound")
- : I18n.t("Captions from the timeline's sound, or from a script");
+ : I18n.t("Captions from a script, at the playhead");
clicked => { root.captions(); }
}
TrayTool {