The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
refactor(ui): the speech sheet is a pane behind a message
Jareer4b355332e253
diff --git a/src/crates/concat/src/lib.rs b/src/crates/concat/src/lib.rsindex a885a784..4b24ba19 100644--- a/src/crates/concat/src/lib.rs+++ b/src/crates/concat/src/lib.rs@@ -54,6 +54,7 @@ use panes::Msg;use panes::captions::CaptionsMsg;use panes::export::ExportMsg;use panes::settings::SettingsMsg;+use panes::speech::SpeechMsg;use studio::{Models, OUTPUTS, RESOLUTIONS, START_RATES, Studio};use ui::*;@@ -1063,7 +1064,7 @@ pub fn run() -> Result<(), slint::PlatformError> {state.handle(Msg::Captions(CaptionsMsg::Open));}));editor.on_speak(on_window!(|state| {- state.speech_open();+ state.handle(Msg::Speech(SpeechMsg::Open));}));app.on_captions_closed(on_window!(|state| {@@ -1088,25 +1089,25 @@ pub fn run() -> Result<(), slint::PlatformError> {state.handle(Msg::Captions(CaptionsMsg::Cancel));}));app.on_speech_closed(on_window!(|state| {- state.speech.open = false;+ state.handle(Msg::Speech(SpeechMsg::Close));}));app.on_speech_text_edited(on_window!(|state, text: SharedString| {- state.speech.text = text.to_string();+ state.handle(Msg::Speech(SpeechMsg::TextEdited(text.to_string())));}));app.on_speech_voice_changed(on_window!(|state, index: i32| {- state.speech.voice = index.max(0) as usize;+ state.handle(Msg::Speech(SpeechMsg::VoiceChanged(index)));}));app.on_speech_model_changed(on_window!(|state, index: i32| {- state.speech.model = index.max(0) as usize;+ state.handle(Msg::Speech(SpeechMsg::ModelChanged(index)));}));app.on_speech_pace_changed(on_window!(|state, index: i32| {- state.speech.pace = (index.max(0) as usize).min(2);+ state.handle(Msg::Speech(SpeechMsg::PaceChanged(index)));}));app.on_speech_begin(on_window!(|state| {- state.speech_run();+ state.handle(Msg::Speech(SpeechMsg::Begin));}));app.on_speech_cancel(on_window!(|state| {- state.speech_cancel();+ state.handle(Msg::Speech(SpeechMsg::Cancel));}));// ── the title-bar menus ──@@ -1140,7 +1141,7 @@ pub fn run() -> Result<(), slint::PlatformError> {}"export" => state.handle(Msg::Export(ExportMsg::Open)),"template" => state.save_template(),- "speech" => state.speech_open(),+ "speech" => state.handle(Msg::Speech(SpeechMsg::Open)),"clear-cache" => state.clear_project_cache(),"settings" => state.handle(Msg::Settings(SettingsMsg::Open)),"close-project" => state.close_project(),diff --git a/src/crates/concat/src/panes/mod.rs b/src/crates/concat/src/panes/mod.rsindex 3392ece7..bc87172f 100644--- a/src/crates/concat/src/panes/mod.rs+++ b/src/crates/concat/src/panes/mod.rs@@ -16,6 +16,7 @@pub mod captions;pub mod export;pub mod settings;+pub mod speech;/// One thing that happened, to one pane.#[derive(Clone, Debug)]@@ -26,4 +27,6 @@ pub enum Msg {Settings(settings::SettingsMsg),/// To the captions sheet.Captions(captions::CaptionsMsg),+ /// To the speech sheet.+ Speech(speech::SpeechMsg),}diff --git a/src/crates/concat/src/panes/settings.rs b/src/crates/concat/src/panes/settings.rsindex 092a2618..d6b8e769 100644--- a/src/crates/concat/src/panes/settings.rs+++ b/src/crates/concat/src/panes/settings.rs@@ -432,7 +432,7 @@ impl SettingsPane {.collect();}if let Ok(status) = concat_speech::Speech::status(dirs) {- studio.speakers = status.voices.clone();+ studio.speech.speakers = status.voices.clone();self.voices = status.models.iter()diff --git a/src/crates/concat/src/panes/speech.rs b/src/crates/concat/src/panes/speech.rsnew file mode 100644index 00000000..802b0341--- /dev/null+++ b/src/crates/concat/src/panes/speech.rs@@ -0,0 +1,317 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! The speech sheet: a title's words, or any words, read aloud.+//!+//! It opens on the selected title's words when a title is selected, else+//! on a blank script to be read at the playhead. The voice is the one+//! chosen last time. The synthesiser runs on a worker and reports as+//! [`SpeechMsg::Progress`], ending as [`SpeechMsg::Finished`] with the+//! written file, which lands in the bin and on the timeline.++use std::sync::Arc;++use concat_host::media::{self, MediaSummary};+use concat_project::Command;+use concat_project::model::ClipKind;+use concat_speech::tts::VoiceInfo;+use slint::SharedString;++use crate::host::{on_ui, spawn};+use crate::i18n::{t, tf};+use crate::panes::Msg;+use crate::panes::captions::CHARS_PER_SECOND;+use crate::panes::settings::installed;+use crate::studio::Studio;+use crate::ui::SpeechSheetData;++/// The default Kokoro speaker: `af_heart`.+const DEFAULT_VOICE: i32 = 3;+/// The paces the speech sheet offers, as the voice's rate multiplier.+const PACES: [f32; 3] = [0.85, 1.0, 1.15];++/// Everything that can happen to the speech sheet.+#[derive(Clone, Debug)]+pub enum SpeechMsg {+ /// The tray's Speak tool, or the menu.+ Open,+ Close,+ TextEdited(String),+ VoiceChanged(i32),+ ModelChanged(i32),+ PaceChanged(i32),+ /// Read the script.+ Begin,+ Cancel,+ /// The synthesiser's worker reporting where it is, `0..=1`.+ Progress(f32),+ /// The synthesiser's worker is done: the file it wrote, probed, or+ /// why not.+ Finished(Box<Result<MediaSummary, String>>),+}++/// The speech sheet's state.+#[derive(Default)]+pub struct SpeechPane {+ pub open: bool,+ /// The title the words came from, and where the sound lands. None+ /// reads a script of its own at the playhead.+ pub clip: Option<String>,+ pub text: String,+ /// Row in `speakers`.+ pub voice: usize,+ /// Row in the installed voice model list.+ pub model: usize,+ /// 0 slower, 1 natural, 2 faster.+ pub pace: usize,+ pub running: bool,+ pub progress: f32,+ pub message: String,+ /// Where the running read will land on the timeline, in seconds.+ landing: f64,+ /// Who can speak, from the voice model on disk.+ pub speakers: Vec<VoiceInfo>,+}++impl SpeechPane {+ /// Applies one message. The studio is the rest of the window; while+ /// this runs the studio's copy of the pane is a blank it must not read.+ pub fn update(&mut self, msg: SpeechMsg, studio: &mut Studio) {+ match msg {+ SpeechMsg::Open => {+ let title = studio+ .sole_selection()+ .and_then(|id| studio.clip(&id))+ .filter(|clip| clip.kind == ClipKind::Text)+ .cloned();+ let installed = installed(&studio.settings.voices);+ let model = installed.iter().position(|model| model.active).unwrap_or(0);+ let wanted = studio.prefs.tts_voice.unwrap_or(DEFAULT_VOICE);+ let voice = self+ .speakers+ .iter()+ .position(|speaker| speaker.id == wanted)+ .unwrap_or(0);+ *self = SpeechPane {+ open: true,+ clip: title.as_ref().map(|clip| clip.id.clone()),+ text: title+ .and_then(|clip| clip.text.map(|text| text.content))+ .unwrap_or_default(),+ voice,+ model,+ pace: 1,+ speakers: std::mem::take(&mut self.speakers),+ ..SpeechPane::default()+ };+ }+ SpeechMsg::Close => self.open = false,+ SpeechMsg::TextEdited(text) => self.text = text,+ SpeechMsg::VoiceChanged(index) => self.voice = index.max(0) as usize,+ SpeechMsg::ModelChanged(index) => self.model = index.max(0) as usize,+ SpeechMsg::PaceChanged(index) => self.pace = (index.max(0) as usize).min(2),+ SpeechMsg::Begin => self.run(studio),+ SpeechMsg::Cancel => {+ studio.host.speech.cancel();+ self.running = false;+ self.open = false;+ }+ SpeechMsg::Progress(fraction) => self.progress = fraction.clamp(0.0, 1.0),+ SpeechMsg::Finished(result) => {+ self.running = false;+ match *result {+ Ok(summary) => {+ let created = studio.apply(Command::AddMedia {+ item: summary.to_new_media(),+ });+ let media_id = created.or_else(|| {+ studio+ .project()+ .media+ .iter()+ .find(|item| item.path == summary.path)+ .map(|item| item.id.clone())+ });+ self.open = false;+ if let Some(media_id) = media_id {+ studio.apply(Command::AddClipAtFirstFree {+ media_id,+ start: self.landing,+ });+ studio.notify(&t("Voice added to the timeline"), false);+ }+ }+ // Asked for: the sheet is already on its way down.+ Err(error) if error.contains("cancel") => self.open = false,+ Err(error) => self.message = error,+ }+ }+ }+ }++ /// Reads the script on a worker: the WAV lands in the bin and on the+ /// timeline, at the title's start or at the playhead.+ fn run(&mut self, studio: &mut Studio) {+ let text = self.text.trim().to_owned();+ if text.is_empty() {+ self.message = t("Nothing to read yet");+ return;+ }+ let Some(model) = installed(&studio.settings.voices)+ .get(self.model)+ .map(|model| model.id.clone())+ else {+ self.message = t("Download a voice model in Settings › Speech first");+ return;+ };+ let Some(voice) = self.speakers.get(self.voice).map(|speaker| speaker.id) else {+ self.message = t("No voice to read with");+ return;+ };+ let Some(project) = studio+ .session+ .as_ref()+ .map(|session| session.path().to_owned())+ else {+ return;+ };+ // Remembered: the voice chosen is the voice wanted next time.+ studio.prefs.tts_voice = Some(voice);+ studio.prefs.save(&studio.host.dirs);+ self.landing = self+ .clip+ .as_ref()+ .and_then(|id| studio.clip(id))+ .map(|clip| clip.start)+ .unwrap_or(f64::from(studio.playhead));+ let request = concat_speech::tts::SpeakRequest {+ model_id: model,+ voice,+ text,+ speed: PACES[self.pace.min(2)],+ project,+ };+ let dirs = studio.host.dirs.clone();+ let speech = Arc::clone(&studio.host.speech);+ self.running = true;+ self.progress = 0.0;+ self.message.clear();+ spawn(+ move || {+ let spoken = speech.speak(&dirs, &request, |fraction| {+ on_ui(move |studio, _, _| {+ studio.handle(Msg::Speech(SpeechMsg::Progress(fraction)));+ });+ })?;+ let summary = media::probe(&spoken.path)?;+ Ok::<_, String>(summary)+ },+ |studio, _, _, result| studio.handle(Msg::Speech(SpeechMsg::Finished(Box::new(result)))),+ );+ }++ /// What the sheet's script would take to say, for the line under it.+ fn estimate(&self) -> String {+ let chars = self.text.trim().chars().count();+ if chars == 0 {+ return t("Nothing to read yet");+ }+ let seconds = chars as f32 / CHARS_PER_SECOND / PACES[self.pace.min(2)];+ let whole = seconds.round() as i32;+ let voice = self+ .speakers+ .get(self.voice)+ .map(|speaker| voice_label(&speaker.name).0)+ .unwrap_or_default();+ tf(+ "About {0}:{1} in {2} · {3} characters",+ &[&(whole / 60), &format!("{:02}", whole % 60), &voice, &chars],+ )+ }++ /// The voice list's rows: each speaker's name.+ pub fn speaker_rows(&self) -> Vec<SharedString> {+ self.speakers+ .iter()+ .map(|speaker| voice_label(&speaker.name).0.into())+ .collect()+ }++ /// The voice list's second lines: each speaker's accent and gender.+ pub fn speaker_detail_rows(&self) -> Vec<SharedString> {+ self.speakers+ .iter()+ .map(|speaker| voice_label(&speaker.name).1.into())+ .collect()+ }++ /// The sheet as Slint shows it.+ pub fn data(&self, studio: &Studio) -> SpeechSheetData {+ SpeechSheetData {+ open: self.open,+ text: self.text.as_str().into(),+ voice: self.voice as i32,+ model: self.model as i32,+ pace: self.pace as i32,+ running: self.running,+ progress: self.progress,+ ready: !installed(&studio.settings.voices).is_empty(),+ placement: if self.clip.is_some() {+ "at the title"+ } else {+ "at the playhead"+ }+ .into(),+ estimate: self.estimate().into(),+ message: self.message.as_str().into(),+ }+ }+}++/// "af_heart" as a person would say it: the name, and the accent and+/// gender its prefix encodes.+fn voice_label(name: &str) -> (String, String) {+ let (prefix, rest) = name.split_once('_').unwrap_or(("", name));+ let mut chars = rest.chars();+ let title = match chars.next() {+ Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),+ None => String::new(),+ };+ let accent = match prefix.chars().next() {+ Some('a') => t("American"),+ Some('b') => t("British"),+ Some('e') => t("Spanish"),+ Some('f') => t("French"),+ Some('h') => t("Hindi"),+ Some('i') => t("Italian"),+ Some('j') => t("Japanese"),+ Some('p') => t("Portuguese"),+ Some('z') => t("Chinese"),+ _ => String::new(),+ };+ let gender = match prefix.chars().nth(1) {+ Some('f') => t("female"),+ Some('m') => t("male"),+ _ => String::new(),+ };+ let detail = [accent, gender]+ .into_iter()+ .filter(|part| !part.is_empty())+ .collect::<Vec<_>>()+ .join(" · ");+ (title, detail)+}++#[cfg(test)]+mod tests {+ use super::voice_label;++ #[test]+ fn a_voice_name_reads_as_a_person_would_say_it() {+ let (title, detail) = voice_label("af_heart");+ assert_eq!(title, "Heart");+ assert!(detail.contains("American"));+ assert!(detail.contains("female"));+ assert_eq!(voice_label("nova").0, "Nova");+ }+}diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rsindex 2c319ecc..1a4ce84d 100644--- a/src/crates/concat/src/studio.rs+++ b/src/crates/concat/src/studio.rs@@ -54,7 +54,6 @@ use crate::host::{window_start,};use crate::i18n::{self, t, tf};-use crate::panes::captions::CHARS_PER_SECOND;use crate::panes::settings::installed;use crate::prefs::Preferences;use crate::presets::{self, TextPreset};@@ -195,9 +194,6 @@ pub struct LibraryView {pub const SHELF_KINDS: [PackageKind; 3] =[PackageKind::Filter, PackageKind::Effect, PackageKind::Audio];-/// The default Kokoro speaker: `af_heart`.-const DEFAULT_VOICE: i32 = 3;-/// The project sheet: the Details panel's Modify button, as a form.#[derive(Default)]pub struct ProjectSheet {@@ -209,28 +205,6 @@ pub struct ProjectSheet {pub rate: usize,}-/// The speech sheet: a title's words, or any words, read aloud.-#[derive(Default)]-pub struct SpeechSheet {- pub open: bool,- /// The title the words came from, and where the sound lands. None- /// reads a script of its own at the playhead.- pub clip: Option<String>,- pub text: String,- /// Row in `Studio::speakers`.- pub voice: usize,- /// Row in the installed voice model list.- pub model: usize,- /// 0 slower, 1 natural, 2 faster.- pub pace: usize,- pub running: bool,- pub progress: f32,- pub message: String,-}--/// The paces the speech sheet offers, as the voice's rate multiplier.-const PACES: [f32; 3] = [0.85, 1.0, 1.15];-/// `name` under the home directory, as a path string; empty when there is/// no home to speak of, and the form then asks for a folder outright.pub(crate) fn home_folder(name: &str) -> String {@@ -735,9 +709,8 @@ pub struct Studio {pub drop: Option<DropPlan>,pub project_sheet: ProjectSheet,pub captions: crate::panes::captions::CaptionsPane,- pub speech: SpeechSheet,+ pub speech: crate::panes::speech::SpeechPane,/// Every speaker the voice engine offers, in its own order.- pub speakers: Vec<concat_speech::tts::VoiceInfo>,/// The looks the Text page offers; see `presets`.pub text_presets: Vec<TextPreset>,@@ -1366,8 +1339,7 @@ impl Studio {drop: None,project_sheet: ProjectSheet::default(),captions: crate::panes::captions::CaptionsPane::default(),- speech: SpeechSheet::default(),- speakers: Vec::new(),+ speech: crate::panes::speech::SpeechPane::default(),text_presets,languages,brush: 0,@@ -5123,190 +5095,16 @@ impl Studio {pane.update(msg, self);self.captions = pane;}+ crate::panes::Msg::Speech(msg) => {+ let mut pane = std::mem::take(&mut self.speech);+ pane.update(msg, self);+ self.speech = pane;+ }}}// ── speech ──- /// Opens the speech sheet: on the selected title's words when a title- /// is selected, else on a blank script to be read at the playhead. The- /// voice is the one chosen last time.- pub fn speech_open(&mut self) {- let title = self- .sole_selection()- .and_then(|id| self.clip(&id))- .filter(|clip| clip.kind == model::ClipKind::Text)- .cloned();- let installed = installed(&self.settings.voices);- let model = installed.iter().position(|model| model.active).unwrap_or(0);- let wanted = self.prefs.tts_voice.unwrap_or(DEFAULT_VOICE);- let voice = self- .speakers- .iter()- .position(|speaker| speaker.id == wanted)- .unwrap_or(0);- self.speech = SpeechSheet {- open: true,- clip: title.as_ref().map(|clip| clip.id.clone()),- text: title- .and_then(|clip| clip.text.map(|text| text.content))- .unwrap_or_default(),- voice,- model,- pace: 1,- ..SpeechSheet::default()- };- }-- /// Reads the script: the WAV lands in the bin and on the timeline, at- /// the title's start or at the playhead.- pub fn speech_run(&mut self) {- let text = self.speech.text.trim().to_owned();- if text.is_empty() {- self.speech.message = t("Nothing to read yet");- return;- }- let Some(model) = installed(&self.settings.voices)- .get(self.speech.model)- .map(|model| model.id.clone())- else {- self.speech.message = t("Download a voice model in Settings › Speech first");- return;- };- let Some(voice) = self- .speakers- .get(self.speech.voice)- .map(|speaker| speaker.id)- else {- self.speech.message = t("No voice to read with");- return;- };- let Some(project) = self- .session- .as_ref()- .map(|session| session.path().to_owned())- else {- return;- };- // Remembered: the voice chosen is the voice wanted next time.- self.prefs.tts_voice = Some(voice);- self.prefs.save(&self.host.dirs);- let start = self- .speech- .clip- .as_ref()- .and_then(|id| self.clip(id))- .map(|clip| clip.start)- .unwrap_or(f64::from(self.playhead));- let request = concat_speech::tts::SpeakRequest {- model_id: model,- voice,- text,- speed: PACES[self.speech.pace.min(2)],- project,- };- let dirs = self.host.dirs.clone();- let speech = Arc::clone(&self.host.speech);- self.speech.running = true;- self.speech.progress = 0.0;- self.speech.message.clear();- spawn(- move || {- let spoken = speech.speak(&dirs, &request, |fraction| {- on_ui(move |studio, _, _| {- studio.speech.progress = fraction.clamp(0.0, 1.0);- });- })?;- let summary = media::probe(&spoken.path)?;- Ok::<_, String>(summary)- },- move |studio, _, _, result| {- studio.speech.running = false;- match result {- Ok(summary) => {- let created = studio.apply(Command::AddMedia {- item: summary.to_new_media(),- });- let media_id = created.or_else(|| {- studio- .project()- .media- .iter()- .find(|item| item.path == summary.path)- .map(|item| item.id.clone())- });- studio.speech.open = false;- if let Some(media_id) = media_id {- studio.apply(Command::AddClipAtFirstFree { media_id, start });- studio.notify(&t("Voice added to the timeline"), false);- }- }- Err(error) if error.contains("cancel") => studio.speech.open = false,- Err(error) => studio.speech.message = error,- }- },- );- }-- pub fn speech_cancel(&mut self) {- self.host.speech.cancel();- self.speech.running = false;- self.speech.open = false;- }-- /// "af_heart" as a person would say it: the name, and the accent and- /// gender its prefix encodes.- fn voice_label(name: &str) -> (String, String) {- let (prefix, rest) = name.split_once('_').unwrap_or(("", name));- let mut chars = rest.chars();- let title = match chars.next() {- Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),- None => String::new(),- };- let accent = match prefix.chars().next() {- Some('a') => t("American"),- Some('b') => t("British"),- Some('e') => t("Spanish"),- Some('f') => t("French"),- Some('h') => t("Hindi"),- Some('i') => t("Italian"),- Some('j') => t("Japanese"),- Some('p') => t("Portuguese"),- Some('z') => t("Chinese"),- _ => String::new(),- };- let gender = match prefix.chars().nth(1) {- Some('f') => t("female"),- Some('m') => t("male"),- _ => String::new(),- };- let detail = [accent, gender]- .into_iter()- .filter(|part| !part.is_empty())- .collect::<Vec<_>>()- .join(" · ");- (title, detail)- }-- /// What the sheet's script would take to say, for the line under it.- fn speech_estimate(&self) -> String {- let chars = self.speech.text.trim().chars().count();- if chars == 0 {- return t("Nothing to read yet");- }- let seconds = chars as f32 / CHARS_PER_SECOND / PACES[self.speech.pace.min(2)];- let whole = seconds.round() as i32;- let voice = self- .speakers- .get(self.speech.voice)- .map(|speaker| Self::voice_label(&speaker.name).0)- .unwrap_or_default();- tf(- "About {0}:{1} in {2} · {3} characters",- &[&(whole / 60), &format!("{:02}", whole % 60), &voice, &chars],- )- }-/// Packs the open project into the template library./// Where the user's own looks live: one package folder each.pub fn looks_dir(dirs: &concat_host::dirs::AppDirs) -> std::path::PathBuf {@@ -6372,38 +6170,9 @@ impl Studio {.map(|model| SharedString::from(model.name.as_str())).collect(),);- sync(- &models.speakers,- self.speakers- .iter()- .map(|speaker| Self::voice_label(&speaker.name).0.into())- .collect(),- );- sync(- &models.speaker_details,- self.speakers- .iter()- .map(|speaker| Self::voice_label(&speaker.name).1.into())- .collect(),- );- app.set_speech(SpeechSheetData {- open: self.speech.open,- text: self.speech.text.as_str().into(),- voice: self.speech.voice as i32,- model: self.speech.model as i32,- pace: self.speech.pace as i32,- running: self.speech.running,- progress: self.speech.progress,- ready: !voices.is_empty(),- placement: if self.speech.clip.is_some() {- "at the title"- } else {- "at the playhead"- }- .into(),- estimate: self.speech_estimate().into(),- message: self.speech.message.as_str().into(),- });+ sync(&models.speakers, self.speech.speaker_rows());+ sync(&models.speaker_details, self.speech.speaker_detail_rows());+ app.set_speech(self.speech.data(self));let bar = self.menu_bar();app.set_app_menu_height(Self::menu_height(&bar));