Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

feat(cursor): play a click sound on every recorded click

一个成熟的剪辑猿31a105edad28
diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md
index e7c74448d..f3ec45821 100644
--- a/THIRD-PARTY-NOTICES.md
+++ b/THIRD-PARTY-NOTICES.md
@@ -260,3 +260,17 @@ and are covered by [LICENSE](LICENSE).
To report an omission or request source for anything bundled here, open an issue
at <https://github.com/getopenscreen/openscreen/issues>.
+---
+
+## Kenney — UI sound effects (mouse click sound)
+
+- **Files**: `public/sounds/click-down.wav` (from `click3.wav`) and
+ `public/sounds/click-up.wav` (from `switch12.wav`) of the "UI Audio" set. Both were
+ trimmed of their leading silence and levelled to a 0.7 peak, so a hit lands on the
+ click and the two samples play at matching loudness without the mixer measuring them.
+- **Used by**: the "Mouse clicks" editor toggle, which lays these two hits along the
+ click timestamps recorded with the take.
+- **License**: **CC0 1.0 Universal (Public Domain Dedication)**,
+ <https://creativecommons.org/publicdomain/zero/1.0/>. The pack ships a readme stating
+ personal and commercial use is permitted and that credit is optional; upstream:
+ <https://kenney.nl/assets/ui-audio>.
diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs
index 6b13df4c5..de845541d 100644
--- a/crates/compositor/src/audio.rs
+++ b/crates/compositor/src/audio.rs
@@ -5,7 +5,9 @@
use crate::ffi::*;
use crate::regions::SpeedSegment;
-use crate::scene::{SceneAudio, SceneAudioTrack, SceneAudioTrackKind};
+use crate::scene::{
+ SceneAudio, SceneAudioTrack, SceneAudioTrackKind, SceneClickHit, SceneClickSound,
+};
use anyhow::{bail, Result};
use std::collections::{HashMap, VecDeque};
use std::f32::consts::PI;
@@ -1732,6 +1734,67 @@ fn overlay_external_track(programme: &mut PlanarPcm, track: &SceneAudioTrack, du
);
}
+/// How much of a hit file gets decoded. The bundled samples are a few hundred milliseconds;
+/// the window is what keeps a wrong path from buffering an hour of "click".
+/// ponytail: fixed window, read the file's own duration if longer hits are ever shipped.
+const CLICK_HIT_WINDOW_SEC: f64 = 1.0;
+
+/// Sum the take's click hits over the assembled programme: two decodes for the whole take,
+/// then one add per hit — unlike an imported track, which decodes once per placement.
+///
+/// The hits arrive already placed on the programme by the app, through the projection it lays
+/// imported audio tracks out with, so speeding a section up cannot leave the sound behind.
+pub fn mix_click_hits(mut programme: PlanarPcm, clicks: &SceneClickSound) -> PlanarPcm {
+ if clicks.hits.is_empty()
+ || clicks.down_path.is_empty()
+ || clicks.up_path.is_empty()
+ || programme.first().map_or(true, Vec::is_empty)
+ {
+ return programme;
+ }
+ let (down, up) = (
+ decode_click_hit(&clicks.down_path),
+ decode_click_hit(&clicks.up_path),
+ );
+ if down.is_empty() || up.is_empty() {
+ eprintln!("[audio] click hits undecodable; export keeps the picture only");
+ return programme;
+ }
+ overlay_click_hits(&mut programme, &down, &up, &clicks.hits);
+ programme
+}
+
+fn decode_click_hit(path: &str) -> PlanarPcm {
+ decode_clip_audio(path, 0.0, CLICK_HIT_WINDOW_SEC)
+ .ok()
+ .flatten()
+ .unwrap_or_default()
+}
+
+/// Placement and level, with no ffmpeg in sight. `overlay_track_pcm` truncates at the
+/// programme's end, so a hit the edit pushed past it is simply not heard.
+fn overlay_click_hits(
+ programme: &mut PlanarPcm,
+ down: &PlanarPcm,
+ up: &PlanarPcm,
+ hits: &[SceneClickHit],
+) {
+ for hit in hits {
+ let offset = (hit.time_sec.max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize;
+ let gain = hit.gain.clamp(0.0, 4.0);
+ overlay_track_pcm(
+ programme,
+ if hit.release { up } else { down },
+ offset,
+ gain,
+ 0.0,
+ 0.0,
+ 0,
+ &[],
+ );
+ }
+}
+
/// How far a music bed dips while someone speaks.
pub const DUCK_DEPTH_DB: f32 = -10.0;
/// A 10 ms stretch of the voice counts as speech above this RMS level. The voice is already
@@ -2268,6 +2331,66 @@ mod tests {
assert_eq!(programme[0], vec![0.3, 0.3]);
}
+ fn hit(time_sec: f64, gain: f32, release: bool) -> SceneClickHit {
+ SceneClickHit { time_sec, gain, release }
+ }
+
+ #[test]
+ fn click_hits_sum_at_their_time_and_pick_their_sample() {
+ // down is 1.0 and up is 0.5, so the level heard at each offset says which sample played.
+ let (down, up) = (planar(&[1.0]), planar(&[0.5]));
+ let tick = 1.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64;
+ let mut programme = planar(&[0.0, 0.0, 0.0]);
+ overlay_click_hits(
+ &mut programme,
+ &down,
+ &up,
+ &[
+ hit(0.0, 1.0, false),
+ hit(tick, 1.0, true),
+ hit(2.0 * tick, 0.5, false),
+ ],
+ );
+ assert_eq!(programme[0], vec![1.0, 0.5, 0.5]);
+ }
+
+ #[test]
+ fn a_click_hit_the_edit_pushed_past_the_programme_is_not_heard() {
+ let mut programme = planar(&[0.25, 0.25]);
+ overlay_click_hits(
+ &mut programme,
+ &planar(&[1.0]),
+ &planar(&[1.0]),
+ &[hit(5.0, 1.0, false)],
+ );
+ assert_eq!(programme[0], vec![0.25, 0.25]);
+ }
+
+ #[test]
+ fn click_hits_are_decoded_from_files_and_summed_at_their_time() {
+ // The whole point of the scene carrying hits instead of a baked bed: the samples live in
+ // their own files and are read here, so this walks the same decode a real track does.
+ let dir = std::env::temp_dir();
+ let down = dir.join(format!("openscreen-click-down-{}.wav", std::process::id()));
+ let up = dir.join(format!("openscreen-click-up-{}.wav", std::process::id()));
+ write_wav(&down, &planar(&[0.5, 0.5]));
+ write_wav(&up, &planar(&[0.25, 0.25]));
+ let clicks = SceneClickSound {
+ down_path: down.to_str().unwrap().into(),
+ up_path: up.to_str().unwrap().into(),
+ hits: vec![hit(0.0, 1.0, false), hit(1.0 / 48_000.0, 1.0, true)],
+ };
+ let out = mix_click_hits(planar(&[0.0; 4]), &clicks);
+ let _ = std::fs::remove_file(&down);
+ let _ = std::fs::remove_file(&up);
+ // Each sample is two frames long: the press covers 0..1 and the hit placed one frame in
+ // covers 1..2, so the seam carries both and the programme is untouched past them.
+ assert_eq!(
+ out[0].iter().map(|v| (v * 100.0).round() / 100.0).collect::<Vec<_>>(),
+ vec![0.5, 0.75, 0.25, 0.0]
+ );
+ }
+
#[test]
fn mix_external_tracks_skips_empty_windows() {
let programme = planar(&[0.4, 0.4]);
diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs
index 5dcfde1da..d8554a590 100644
--- a/crates/compositor/src/pipeline_linux.rs
+++ b/crates/compositor/src/pipeline_linux.rs
@@ -28,7 +28,8 @@ use std::path::Path;
use std::ptr;
use crate::audio::{
- assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
+ assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits,
+ mix_external_tracks,
AacEncoder, PlanarPcm,
};
use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -1054,6 +1055,11 @@ fn run_multi_inner(
.as_ref()
.map(|scene| scene.audio_tracks.clone())
.unwrap_or_default();
+ // Click hits (the "mouse clicks" toggle), laid on the programme by the app.
+ let click_sound = scene
+ .as_ref()
+ .map(|scene| scene.click_sound.clone())
+ .unwrap_or_default();
// Ring de staging a 2 : l'export ne veut que du debit, une frame de latence
// ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour
// la raison pour laquelle la preview, elle, reste a 1.
@@ -1244,7 +1250,10 @@ fn run_multi_inner(
let octx = mux.octx;
mux.aac.encode(
&finish_audio(
- mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
+ mix_click_hits(
+ mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
+ &click_sound,
+ ),
audio_settings,
),
octx,
diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs
index 48c775220..638b664f4 100644
--- a/crates/compositor/src/pipeline_macos.rs
+++ b/crates/compositor/src/pipeline_macos.rs
@@ -30,7 +30,8 @@
//! décodeurs, symétrique.
use crate::audio::{
- assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
+ assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits,
+ mix_external_tracks,
AacEncoder, PlanarPcm,
};
use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -1294,6 +1295,11 @@ fn run_multi_inner(
.as_ref()
.map(|scene| scene.audio_tracks.clone())
.unwrap_or_default();
+ // Click hits (the "mouse clicks" toggle), laid on the programme by the app.
+ let click_sound = scene
+ .as_ref()
+ .map(|scene| scene.click_sound.clone())
+ .unwrap_or_default();
frames = unsafe {
crate::timeline_walk::walk_composited_timeline(
clips,
@@ -1371,7 +1377,10 @@ fn run_multi_inner(
let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64);
audio_encoder.encode(
&finish_audio(
- mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
+ mix_click_hits(
+ mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks),
+ &click_sound,
+ ),
audio_settings,
),
octx,
diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs
index d25d268e2..28bcc4faa 100644
--- a/crates/compositor/src/pipeline_windows.rs
+++ b/crates/compositor/src/pipeline_windows.rs
@@ -3,7 +3,8 @@
//! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps.
use crate::audio::{
- assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks,
+ assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits,
+ mix_external_tracks,
AacEncoder, PlanarPcm,
};
use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs};
@@ -1715,6 +1716,11 @@ unsafe fn run_multi_inner(
.as_ref()
.map(|scene| scene.audio_tracks.clone())
.unwrap_or_default();
+ // Click hits (the "mouse clicks" toggle), laid on the programme by the app.
+ let click_sound = scene
+ .as_ref()
+ .map(|scene| scene.click_sound.clone())
+ .unwrap_or_default();
// ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ----
// Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur
@@ -1857,7 +1863,10 @@ unsafe fn run_multi_inner(
out_fps as f64,
);
let assembled_audio = finish_audio(
- mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks),
+ mix_click_hits(
+ mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks),
+ &click_sound,
+ ),
audio_settings,
);
audio_encoder.encode(&assembled_audio, octx)?;
diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs
index 280cb7ef4..895fa6b2a 100644
--- a/crates/compositor/src/scene.rs
+++ b/crates/compositor/src/scene.rs
@@ -679,6 +679,43 @@ pub enum SceneAudioTrackKind {
Voiceover,
}
+/// One click to be heard: `time_sec` is its place on the OUTPUT programme, `gain` the linear
+/// level of the hit (side clicks a little quieter than the left), `release` which of the two
+/// samples to play.
+#[derive(Debug, Clone, Copy, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SceneClickHit {
+ pub time_sec: f64,
+ #[serde(default = "default_one")]
+ pub gain: f32,
+ #[serde(default)]
+ pub release: bool,
+}
+
+/// The "mouse clicks" toggle: the recorded click times, already projected onto the programme,
+/// plus the two hit samples to play them with.
+///
+/// Deliberately not an imported audio track. A bed baked into one file cannot follow an edit —
+/// `SceneAudioTrack`'s content is placed, never stretched, so every hit after a speed region
+/// drifts — while hits laid out here are positioned by the app's own projection, the same one
+/// imported audio tracks are placed with, so they cannot disagree with the picture.
+#[derive(Debug, Clone, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SceneClickSound {
+ /// Absolute paths to the press and release samples, 48 kHz-friendly WAV files. `default` for
+ /// the usual reason: a half-written block must cost the sound, never the whole scene parse.
+ #[serde(default)]
+ pub down_path: String,
+ #[serde(default)]
+ pub up_path: String,
+ #[serde(default)]
+ pub hits: Vec<SceneClickHit>,
+}
+
+fn default_one() -> f32 {
+ 1.0
+}
+
#[derive(Debug, Clone, Copy, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SceneOutput {
@@ -751,6 +788,10 @@ pub struct Scene {
/// absent from every scene written before this, and from a project with none.
#[serde(default)]
pub audio_tracks: Vec<SceneAudioTrack>,
+ /// Click hits for the "mouse clicks" toggle. `#[serde(default)]`: an empty `hits` is the
+ /// same as the field being absent, which is every scene written before this.
+ #[serde(default)]
+ pub click_sound: SceneClickSound,
/// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS).
#[serde(default)]
pub crop_by_clip: Vec<Option<SceneCrop>>,
@@ -925,6 +966,24 @@ mod tests {
assert_eq!(effects(r#","frame":"laptop""#).frame.theme_override(), None);
}
+ #[test]
+ fn a_click_sound_payload_reads_the_apps_camel_case_names() {
+ let clicks: SceneClickSound = serde_json::from_str(
+ r#"{"downPath":"/u/click-down.wav","upPath":"/u/click-up.wav","hits":[
+ {"timeSec":2.5,"gain":0.7,"release":true},{"timeSec":4.0}]}"#,
+ )
+ .unwrap();
+ assert_eq!(clicks.down_path, "/u/click-down.wav");
+ assert_eq!(clicks.hits.len(), 2);
+ assert_eq!(clicks.hits[0].time_sec, 2.5);
+ assert!(clicks.hits[0].release);
+ // A hit that carries no level is a full-strength press.
+ assert_eq!(clicks.hits[1].gain, 1.0);
+ // And a scene from before the field reads as no hits, not as a parse failure.
+ assert!(SceneClickSound::default().hits.is_empty());
+ assert!(serde_json::from_str::<SceneClickSound>("{}").unwrap().hits.is_empty());
+ }
+
#[test]
fn parses_color_and_image_backgrounds() {
let color = r##"{"clips":[],"layout":{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0},"background":{"kind":"color","color":"#123456"},"zoomRegions":[],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1280,"height":720,"fps":30}}"##;
diff --git a/electron/ai-edition/style-preset-service.test.ts b/electron/ai-edition/style-preset-service.test.ts
index 0f2e430f7..60c9025da 100644
--- a/electron/ai-edition/style-preset-service.test.ts
+++ b/electron/ai-edition/style-preset-service.test.ts
@@ -34,6 +34,8 @@ const APPEARANCE: StylePresetAppearance = {
model3d: false,
asArrow: [],
clickImpact: false,
+ clickSound: false,
+ clickSoundGainDb: 0,
},
cursorShow: true,
cursorAutoHide: false,
diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts
index ff6c139e0..e17c8da97 100644
--- a/electron/electron-env.d.ts
+++ b/electron/electron-env.d.ts
@@ -338,6 +338,17 @@ interface Window {
message?: string;
error?: string;
}>;
+ // Stage one of the two bundled click-sound samples where the compositor can read it
+ // by path. Only the two names the app ships are accepted, so nothing is sanitised here.
+ stageClickSoundHit: (
+ name: string,
+ data: ArrayBuffer,
+ ) => Promise<{
+ success: boolean;
+ path?: string;
+ message?: string;
+ error?: string;
+ }>;
setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>;
setCurrentRecordingSession: (
session: import("../src/lib/recordingSession").RecordingSession | null,
diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts
index f9a9a4c48..95473f902 100644
--- a/electron/ipc/handlers.ts
+++ b/electron/ipc/handlers.ts
@@ -174,6 +174,12 @@ const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio");
// See the save-recorded-voiceover handler: an upper bound on renderer-supplied
// bytes written to disk, well past any plausible take.
const MAX_RECORDED_VOICEOVER_BYTES = 512 * 1024 * 1024;
+// The bundled click-sound samples, staged for the compositor, which reads audio by path and not
+// by renderer URL. These are the only names the staging handler accepts, and they match
+// HIT_FILES in src/lib/ai-edition/clickSound.ts.
+const CLICK_SOUND_DIR = path.join(app.getPath("userData"), "click-sfx");
+const CLICK_SOUND_HIT_FILES: string[] = ["click-down.wav", "click-up.wav"];
+const MAX_CLICK_HIT_BYTES = 4 * 1024 * 1024;
const nativeMacCaptureEvents = new EventEmitter();
// Enumeration walks every display and window and grabs a thumbnail of each, so it
@@ -4332,6 +4338,34 @@ export function registerIpcHandlers(
}
});
+ // The two click-sound samples the editor ships (src/lib/ai-edition/clickSound.ts). The
+ // compositor reads its audio from the filesystem, not from a renderer URL, so the bundled
+ // bytes are staged here once and the absolute path handed back. Renderer bytes like the
+ // recorded voiceover, but with nothing to sanitise: the name is checked against the two the
+ // app knows, so the payload cannot steer where anything lands.
+ ipcMain.handle("stage-click-sound-hit", async (_event, name: string, data: ArrayBuffer) => {
+ try {
+ if (!CLICK_SOUND_HIT_FILES.includes(name)) {
+ return { success: false, message: `Unknown click hit ${name}` };
+ }
+ if (!(data instanceof ArrayBuffer) || data.byteLength === 0) {
+ return { success: false, message: "Empty click hit" };
+ }
+ if (data.byteLength > MAX_CLICK_HIT_BYTES) {
+ return { success: false, message: "Click hit too large" };
+ }
+ const target = path.join(CLICK_SOUND_DIR, name);
+ await fs.mkdir(CLICK_SOUND_DIR, { recursive: true });
+ // Written fresh each time rather than checked for existence: the file the app shipped
+ // is the one the compositor must hear, and an update may have changed it.
+ await fs.writeFile(target, Buffer.from(data));
+ return { success: true, path: target };
+ } catch (error) {
+ console.error("Failed to stage click sound hit:", error);
+ return { success: false, message: "Failed to stage click sound hit", error: String(error) };
+ }
+ });
+
ipcMain.handle("reveal-in-folder", async (_, filePath: string) => {
try {
// showItemInFolder returns nothing, it throws on error
diff --git a/electron/ipc/nativeBridge.presets.test.ts b/electron/ipc/nativeBridge.presets.test.ts
index bd8a7f8b9..6f97b66e1 100644
--- a/electron/ipc/nativeBridge.presets.test.ts
+++ b/electron/ipc/nativeBridge.presets.test.ts
@@ -54,6 +54,8 @@ const APPEARANCE: StylePresetAppearance = {
model3d: false,
asArrow: [],
clickImpact: false,
+ clickSound: false,
+ clickSoundGainDb: 0,
},
cursorShow: true,
cursorAutoHide: false,
diff --git a/electron/preload.ts b/electron/preload.ts
index 802da693b..98b2a90e8 100644
--- a/electron/preload.ts
+++ b/electron/preload.ts
@@ -362,6 +362,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
saveRecordedVoiceover: (data: ArrayBuffer) => {
return ipcRenderer.invoke("save-recorded-voiceover", data);
},
+ stageClickSoundHit: (name: string, data: ArrayBuffer) => {
+ return ipcRenderer.invoke("stage-click-sound-hit", name, data);
+ },
setCurrentVideoPath: (path: string) => {
return ipcRenderer.invoke("set-current-video-path", path);
},
diff --git a/public/sounds/click-down.wav b/public/sounds/click-down.wav
new file mode 100644
index 000000000..301e92158
Binary files /dev/null and b/public/sounds/click-down.wav differ
diff --git a/public/sounds/click-up.wav b/public/sounds/click-up.wav
new file mode 100644
index 000000000..546e1ec8d
Binary files /dev/null and b/public/sounds/click-up.wav differ
diff --git a/src/components/ai-edition/ExportDialog.tsx b/src/components/ai-edition/ExportDialog.tsx
index 195cb22bf..645fb4641 100644
--- a/src/components/ai-edition/ExportDialog.tsx
+++ b/src/components/ai-edition/ExportDialog.tsx
@@ -10,6 +10,7 @@ import { Download, FileVideo, FolderOpen, Loader2, Star } from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { useScopedT } from "@/contexts/I18nContext";
+import { prepareClickSound } from "@/lib/ai-edition/clickSound";
import {
collectEffectiveClipDims,
type Dims,
@@ -372,6 +373,11 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) {
// invites. Background/layout/webcam/cursor/effects come from the same scene
// as the live preview, so an export can no longer disagree with what the
// user previewed.
+ //
+ // The click sound's hits are read out of the take's sidecar into a cache the scene build
+ // reads synchronously, so the read happens here — before anything is composed. Clicks
+ // that will not load must not cost the user their export.
+ await prepareClickSound(document).catch(() => undefined);
{
const job: ActiveExport = { id: crypto.randomUUID(), cancelRequested: false };
activeExport.current = job;
diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx
index 73c1fcd83..17f335048 100644
--- a/src/components/ai-edition/RightPanes.tsx
+++ b/src/components/ai-edition/RightPanes.tsx
@@ -148,6 +148,7 @@ import {
import { ROUNDNESS_SLIDER_MAX_PX } from "@/native/paramUnits";
import { wallpaperAcceptsMotion } from "@/native/sceneDescription";
import { ASPECT_RATIO_PRESETS, type AspectRatio } from "@/utils/aspectRatioUtils";
+import { useClickSound } from "../../hooks/useClickSound";
import { useCanSegmentCamera } from "../../native/hooks/useSegmentationSupport";
import { CaptionsPane } from "./CaptionsPane";
import { ColorField } from "./ColorField";
@@ -3705,6 +3706,7 @@ export function CursorPane() {
[settings.cursorTheme],
);
const recordedTypes = useRecordedCursorTypes();
+ const clickSound = useClickSound();
// Each kind the video shows, pictured by the first of its states it shows.
const recordedKinds = CURSOR_KIND_IDS.flatMap((kind) => {
const type = CURSOR_KINDS[kind].find((state) => recordedTypes?.has(state));
@@ -3947,6 +3949,48 @@ export function CursorPane() {
/>
</div>
) : null}
+ {/* The same clicks, heard. Offered only once the take is known to contain some. It is a
+ setting rather than a track on the timeline: the hits are placed by the same
+ projection the click zoom uses, so they follow every edit instead of going stale. */}
+ {settings.cursorShow && clickSound.hasClicks ? (
+ <>
+ <div className={styles.paneRow}>
+ <span className={styles.label}>{ts("cursor.clickSound")}</span>
+ <Toggle
+ ariaLabel={ts("cursor.clickSound")}
+ tooltip={ts("cursor.clickSoundTip")}
+ checked={settings.cursor.clickSound}
+ disabled={!hasDocument}
+ onChange={(v) => {
+ void set({ cursor: { clickSound: v } });
+ // Read the cues and stage the samples on the way in, so the first
+ // export asked for after this is not a race with a background read.
+ if (v) void clickSound.prepare();
+ }}
+ />
+ </div>
+ {/* How loud. Offered only once there is a sound to turn down, and bounded by the
+ same table the mixer reads, so the slider cannot ask for a level the export
+ would flatten. */}
+ {settings.cursor.clickSound ? (
+ <div className={styles.sliderGrid}>
+ <SliderCell
+ label={ts("cursor.clickSoundVolume")}
+ value={settings.cursor.clickSoundGainDb}
+ min={SETTING_BOUNDS.cursorClickSoundGainDb[0]}
+ max={SETTING_BOUNDS.cursorClickSoundGainDb[1]}
+ step={0.5}
+ decimals={1}
+ suffix=" dB"
+ defaultValue={DEFAULT_EDITOR_SETTINGS.cursor.clickSoundGainDb}
+ disabled={!hasDocument}
+ onChange={(clickSoundGainDb) => setLive({ cursor: { clickSoundGainDb } })}
+ onCommit={() => void commit()}
+ />
+ </div>
+ ) : null}
+ </>
+ ) : null}
</Pane>
);
}
diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx
index c24d24191..c02b421d6 100644
--- a/src/components/ai-edition/VirtualPreview.tsx
+++ b/src/components/ai-edition/VirtualPreview.tsx
@@ -4,6 +4,12 @@ import {
DEFAULT_CROP_REGION,
MAX_NATIVE_PLAYBACK_RATE,
} from "@/components/video-editor/types";
+import {
+ clickHitBuffers,
+ playClickHits,
+ resetClickPlayhead,
+ takeCrossedClickHits,
+} from "@/lib/ai-edition/clickSound";
import {
collapseTracksToPills,
resolveFadeSecs,
@@ -1216,6 +1222,28 @@ export function VirtualPreview({
// where a reload has to put the playhead back if it cannot
// resolve one from the timeline.
lastGoodSourceTimeRef.current = sourceTime;
+ // The click sound, heard live. Read off the SOURCE clock — the one the cursor sprite
+ // above is sampled on — because a click is recorded in the take's own seconds: this
+ // clock races with the picture under a 2x region, so a hit fired where it passes a
+ // click lands on the click being drawn. A clip swap or a cut moves it by more than a
+ // frame can, which reads as a seek and fires nothing; and while stopped the anchor is
+ // dropped, so the next run does not repay a stack of old clicks.
+ // ponytail: the cues are the primary take's, so a project that interleaves several
+ // takes would need the mounted asset's id here to pick the right list.
+ if (v.paused) {
+ resetClickPlayhead();
+ } else {
+ const crossed = takeCrossedClickHits(sourceTime);
+ const clickGraph = audioGraphRef.current;
+ if (crossed.length && clickGraph) {
+ playClickHits(
+ clickGraph.context,
+ clickGraph.gain,
+ clickHitBuffers(clickGraph.context),
+ crossed,
+ );
+ }
+ }
}
// À L'ARRÊT, le `<video>` ne pilote PLUS la position de la timeline.
//
diff --git a/src/components/video-editor/editorDefaults.ts b/src/components/video-editor/editorDefaults.ts
index 4b2749887..021ba6aa2 100644
--- a/src/components/video-editor/editorDefaults.ts
+++ b/src/components/video-editor/editorDefaults.ts
@@ -84,6 +84,8 @@ export const DEFAULT_CURSOR_SETTINGS: CursorVisualSettings & { show: boolean; th
model3d: DEFAULT_CURSOR_MODEL3D,
asArrow: DEFAULT_PROJECT_APPEARANCE.cursor.asArrow,
clickImpact: false,
+ clickSound: false,
+ clickSoundGainDb: 0,
theme: DEFAULT_CURSOR_THEME_ID,
};
diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts
index 0295f2366..d02194951 100644
--- a/src/components/video-editor/types.ts
+++ b/src/components/video-editor/types.ts
@@ -296,6 +296,10 @@ export interface CursorVisualSettings {
* back (`frame_geometry::plan_frame`).
*/
clickImpact: boolean;
+ /** The same clicks, heard: each recorded click plays a short bundled hit. */
+ clickSound: boolean;
+ /** The click sound's level in dB, on the same scale as an imported audio track's gain. */
+ clickSoundGainDb: number;
autoHide?: boolean;
}
diff --git a/src/hooks/useClickSound.ts b/src/hooks/useClickSound.ts
new file mode 100644
index 000000000..91ad6e3b8
--- /dev/null
+++ b/src/hooks/useClickSound.ts
@@ -0,0 +1,58 @@
+// The "mouse clicks" row in the cursor pane.
+//
+// The sound is a setting, not a timeline track: its hits are laid out from the take's recorded
+// clicks every time a scene is built (see src/lib/ai-edition/clickSound.ts), so there is no file
+// to render, no track to keep honest, and nothing that can fall behind an edit.
+
+import { useEffect, useState } from "react";
+import type { ClickCue } from "@/lib/ai-edition/clickSound";
+import {
+ clickTakeOf,
+ ensureClickHitPaths,
+ LEGACY_CLICK_BED_LABEL,
+ loadClickCues,
+} from "@/lib/ai-edition/clickSound";
+import { useProjectStore } from "@/lib/ai-edition/store/projectStore";
+
+export function useClickSound() {
+ const document = useProjectStore((s) => s.document);
+ const removeAsset = useProjectStore((s) => s.removeAsset);
+ const takePath = clickTakeOf(document)?.originalPath ?? null;
+ // The row is offered only once the take is known to hold clicks, and reading its cues is what
+ // says so — the same one-shot read the hits come from, so switching on waits on nothing more.
+ const [cues, setCues] = useState<ClickCue[] | null>(null);
+ useEffect(() => {
+ let current = true;
+ setCues(null);
+ // A take whose sidecar cannot be read answers as holding no clicks — the same answer a
+ // take with none gives, so the row stays out of the pane.
+ if (takePath) void loadClickCues(takePath).then((loaded) => current && setCues(loaded));
+ return () => {
+ current = false;
+ };
+ }, [takePath]);
+
+ return {
+ hasClicks: Boolean(cues?.length),
+ /** Warms the two caches the sound reads — the cues, and the samples staged for the
+ * export — the moment the user asks for it, instead of leaving the first export to race
+ * a background read. Nothing here can fail the pane: a sample that will not stage just
+ * means the export has no clicks in it. */
+ prepare: async () => {
+ // A project saved by the earlier cut of this feature holds one rendered WAV as a music
+ // track. The hits are laid from the clicks themselves now, so leaving that bed on the
+ // timeline would sound every click twice.
+ for (const asset of document?.assets ?? []) {
+ if (asset.kind !== "audio" || asset.label !== LEGACY_CLICK_BED_LABEL) continue;
+ try {
+ await removeAsset(asset.id);
+ } catch (error) {
+ console.error("Failed to drop the old click bed", error);
+ }
+ }
+ const take = clickTakeOf(document)?.originalPath;
+ if (!take) return;
+ await Promise.all([loadClickCues(take), ensureClickHitPaths()]);
+ },
+ };
+}
diff --git a/src/i18n/locales/ar/settings.json b/src/i18n/locales/ar/settings.json
index 78854c26c..0810409f1 100644
--- a/src/i18n/locales/ar/settings.json
+++ b/src/i18n/locales/ar/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "يرسم المؤشر الافتراضي كجسم ثلاثي الأبعاد مع ظل",
"clickImpact": "أثر النقر",
"clickImpactTip": "تتحرك الشاشة قليلًا مع كل نقرة",
+ "clickSound": "نقرات الفأرة",
+ "clickSoundTip": "يُسمع كل نقرة في الفيديو",
+ "clickSoundVolume": "مستوى صوت النقر",
"help": "عرض المؤشر اعتمادًا على بيانات التتبع المسجّلة: السمة، والحجم، والتنعيم، وضبابية الحركة، وارتداد النقر.",
"size": "الحجم",
"title": "المؤشر",
diff --git a/src/i18n/locales/cs/settings.json b/src/i18n/locales/cs/settings.json
index c0f0c73eb..80eb9b268 100644
--- a/src/i18n/locales/cs/settings.json
+++ b/src/i18n/locales/cs/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Vykreslí výchozí kurzor jako 3D objekt se stínem",
"clickImpact": "Odezva kliknutí",
"clickImpactTip": "Obrazovka se při každém kliknutí trochu pohne",
+ "clickSound": "Kliknutí myši",
+ "clickSoundTip": "Každé kliknutí je ve videu slyšitelné",
+ "clickSoundVolume": "Hlasitost kliknutí",
"help": "Renderování kurzoru ze zaznamenané telemetrie: motiv, velikost, vyhlazení, pohybové rozmazání a zdůraznění odrazu při kliknutí.",
"size": "Velikost",
"title": "Kurzor",
diff --git a/src/i18n/locales/de/settings.json b/src/i18n/locales/de/settings.json
index 3e6af58b1..715d3c9bb 100644
--- a/src/i18n/locales/de/settings.json
+++ b/src/i18n/locales/de/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Zeichnet den Standard-Cursor als 3D-Objekt mit Schatten",
"clickImpact": "Klick-Impuls",
"clickImpactTip": "Der Bildschirm bewegt sich bei jedem Klick ein wenig",
+ "clickSound": "Mausklicks",
+ "clickSoundTip": "Jeder Klick ist im Video zu hören",
+ "clickSoundVolume": "Klicklautstärke",
"help": "Cursor-Darstellung aus der aufgezeichneten Telemetrie: Stil, Größe, Glättung, Bewegungsunschärfe und Klick-Federung.",
"size": "Größe",
"title": "Cursor",
diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json
index 867827cc1..be8b13a69 100644
--- a/src/i18n/locales/en/settings.json
+++ b/src/i18n/locales/en/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Draws the default cursor as a 3D object with a shadow",
"clickImpact": "Click impact",
"clickImpactTip": "The screen moves a little with each click",
+ "clickSound": "Mouse clicks",
+ "clickSoundTip": "Each click sounds, in the preview and in the export",
+ "clickSoundVolume": "Click volume",
"help": "How the recorded cursor is drawn: style, size, smoothing, motion blur and click bounce.",
"size": "Size",
"title": "Cursor",
diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json
index 7d906c4e0..f9efcfab3 100644
--- a/src/i18n/locales/es/settings.json
+++ b/src/i18n/locales/es/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Dibuja el cursor predeterminado como un objeto 3D con sombra",
"clickImpact": "Impacto del clic",
"clickImpactTip": "La pantalla se mueve un poco con cada clic",
+ "clickSound": "Clics del ratón",
+ "clickSoundTip": "Se oye cada clic en el vídeo",
+ "clickSoundVolume": "Volumen de clic",
"help": "Representación del cursor a partir de la telemetría grabada: tema, tamaño, suavizado, desenfoque de movimiento y rebote al hacer clic.",
"size": "Tamaño",
"title": "Cursor",
diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json
index 7561410e2..cbd053699 100644
--- a/src/i18n/locales/fr/settings.json
+++ b/src/i18n/locales/fr/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Dessine le curseur par défaut comme un objet 3D avec une ombre",
"clickImpact": "Impact du clic",
"clickImpactTip": "L'écran bouge un peu à chaque clic",
+ "clickSound": "Clics souris",
+ "clickSoundTip": "Chaque clic est audible dans la vidéo",
+ "clickSoundVolume": "Volume des clics",
"help": "Rendu du curseur à partir de la télémétrie enregistrée : thème, taille, lissage, flou de mouvement et rebond au clic.",
"size": "Taille",
"title": "Curseur",
diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json
index 555c0dbb3..fd37547e2 100644
--- a/src/i18n/locales/it/settings.json
+++ b/src/i18n/locales/it/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Disegna il cursore predefinito come oggetto 3D con ombra",
"clickImpact": "Impatto del clic",
"clickImpactTip": "Lo schermo si muove un po' a ogni clic",
+ "clickSound": "Clic del mouse",
+ "clickSoundTip": "Ogni clic è udibile nel video",
+ "clickSoundVolume": "Volume dei clic",
"help": "Resa del cursore dalla telemetria registrata: tema, dimensione, smussatura, sfocatura di movimento e rimbalzo al clic.",
"size": "Dimensione",
"title": "Cursore",
diff --git a/src/i18n/locales/ja-JP/settings.json b/src/i18n/locales/ja-JP/settings.json
index dba5bf24f..3b8811256 100644
--- a/src/i18n/locales/ja-JP/settings.json
+++ b/src/i18n/locales/ja-JP/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "標準カーソルを影付きの 3D オブジェクトとして描画します",
"clickImpact": "クリックの押し込み",
"clickImpactTip": "クリックのたびに画面が少し動きます",
+ "clickSound": "マウスクリック音",
+ "clickSoundTip": "動画で各クリックが聞こえます",
+ "clickSoundVolume": "クリック音量",
"help": "記録されたテレメトリからのカーソル描画: テーマ、サイズ、スムージング、モーションブラー、クリック時のバウンス。",
"size": "サイズ",
"title": "カーソル",
diff --git a/src/i18n/locales/ko-KR/settings.json b/src/i18n/locales/ko-KR/settings.json
index cc6882854..c464581da 100644
--- a/src/i18n/locales/ko-KR/settings.json
+++ b/src/i18n/locales/ko-KR/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "기본 커서를 그림자가 있는 3D 오브젝트로 그립니다",
"clickImpact": "클릭 눌림",
"clickImpactTip": "클릭할 때마다 화면이 약간 움직입니다",
+ "clickSound": "마우스 클릭 소리",
+ "clickSoundTip": "영상에서 각 클릭이 들립니다",
+ "clickSoundVolume": "클릭 음량",
"help": "기록된 텔레메트리를 기반으로 한 커서 렌더링: 테마, 크기, 스무딩, 모션 블러, 클릭 바운스.",
"size": "크기",
"title": "커서",
diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json
index 56bde8ca9..53a4c89de 100644
--- a/src/i18n/locales/pt-BR/settings.json
+++ b/src/i18n/locales/pt-BR/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Desenha o cursor padrão como um objeto 3D com sombra",
"clickImpact": "Impacto do clique",
"clickImpactTip": "A tela se move um pouco a cada clique",
+ "clickSound": "Cliques do mouse",
+ "clickSoundTip": "Cada clique é audível no vídeo",
+ "clickSoundVolume": "Volume do clique",
"help": "Renderização do cursor a partir da telemetria gravada: tema, tamanho, suavização, desfoque de movimento e salto ao clicar.",
"size": "Tamanho",
"title": "Cursor",
diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json
index f16d8fc16..08c1a76c9 100644
--- a/src/i18n/locales/ru/settings.json
+++ b/src/i18n/locales/ru/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Рисует курсор по умолчанию как 3D-объект с тенью",
"clickImpact": "Отклик на клик",
"clickImpactTip": "Экран немного смещается при каждом клике",
+ "clickSound": "Щелчки мыши",
+ "clickSoundTip": "Каждый щелчок слышен в видео",
+ "clickSoundVolume": "Громкость щелчков",
"help": "Отрисовка курсора по записанной телеметрии: тема, размер, сглаживание, размытие движения и отскок при клике.",
"size": "Размер",
"title": "Курсор",
diff --git a/src/i18n/locales/tr/settings.json b/src/i18n/locales/tr/settings.json
index 1f500503e..2f3fd2c67 100644
--- a/src/i18n/locales/tr/settings.json
+++ b/src/i18n/locales/tr/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Varsayılan imleci gölgeli 3B bir nesne olarak çizer",
"clickImpact": "Tıklama etkisi",
"clickImpactTip": "Ekran her tıklamada biraz hareket eder",
+ "clickSound": "Fare tıklamaları",
+ "clickSoundTip": "Her tıklama videoda duyulur",
+ "clickSoundVolume": "Tıklama sesi düzeyi",
"help": "Kaydedilen telemetriden imleç çizimi: tema, boyut, yumuşatma, hareket bulanıklığı ve tıklama sıçraması.",
"size": "Boyut",
"title": "İmleç",
diff --git a/src/i18n/locales/vi/settings.json b/src/i18n/locales/vi/settings.json
index ce6f50f7c..ece372040 100644
--- a/src/i18n/locales/vi/settings.json
+++ b/src/i18n/locales/vi/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "Vẽ con trỏ mặc định thành vật thể 3D có bóng đổ",
"clickImpact": "Hiệu ứng nhấp",
"clickImpactTip": "Màn hình dịch chuyển nhẹ theo mỗi lần nhấp",
+ "clickSound": "Tiếng nhấp chuột",
+ "clickSoundTip": "Mỗi lần nhấp đều nghe thấy trong video",
+ "clickSoundVolume": "Âm lượng nhấp chuột",
"help": "Cách vẽ con trỏ từ dữ liệu đã ghi: chủ đề, kích thước, làm mượt, mờ chuyển động và hiệu ứng nảy khi nhấp.",
"size": "Kích thước",
"title": "Con trỏ",
diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json
index a4be3f3bb..3aa4b4b48 100644
--- a/src/i18n/locales/zh-CN/settings.json
+++ b/src/i18n/locales/zh-CN/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "将默认光标显示为带阴影的 3D 样式",
"clickImpact": "点击按压",
"clickImpactTip": "每次点击时,画面会轻微移动",
+ "clickSound": "鼠标声音",
+ "clickSoundTip": "每次点击都有音效,预览和导出都能听到",
+ "clickSoundVolume": "点击音量",
"help": "基于录制遥测数据的光标渲染:主题、大小、平滑、运动模糊和点击回弹。",
"size": "大小",
"title": "光标",
diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json
index 481582a13..f33f5439d 100644
--- a/src/i18n/locales/zh-TW/settings.json
+++ b/src/i18n/locales/zh-TW/settings.json
@@ -412,6 +412,9 @@
"model3dTip": "將預設游標繪製為帶陰影的 3D 物件",
"clickImpact": "點擊按壓",
"clickImpactTip": "每次點擊時螢幕會輕微移動",
+ "clickSound": "滑鼠聲音",
+ "clickSoundTip": "每次點擊都有音效,預覽與匯出都能聽到",
+ "clickSoundVolume": "點擊音量",
"help": "根據錄製的遙測資料繪製游標:主題、大小、平滑、動態模糊與點擊回彈。",
"size": "大小",
"title": "游標",
diff --git a/src/lib/ai-edition/clickSound.test.ts b/src/lib/ai-edition/clickSound.test.ts
new file mode 100644
index 000000000..e00ed1ba1
--- /dev/null
+++ b/src/lib/ai-edition/clickSound.test.ts
@@ -0,0 +1,150 @@
+import { describe, expect, it } from "vitest";
+import type { CursorTelemetryPoint } from "@/components/video-editor/types";
+import type { ClickCue } from "./clickSound";
+import { buildClickCues, crossedClickHits, levelClickCues, placeClickHits } from "./clickSound";
+
+const point = (
+ timeMs: number,
+ interactionType: CursorTelemetryPoint["interactionType"],
+): CursorTelemetryPoint => ({
+ timeMs,
+ cx: 0.5,
+ cy: 0.5,
+ interactionType,
+});
+
+const cue = (timeSec: number) => ({ timeSec, gain: 1, release: false });
+
+describe("buildClickCues", () => {
+ it("turns one click into a press plus its recorded release, and ignores moves", () => {
+ const cues = buildClickCues([
+ point(1000, "move"),
+ point(7056, "click"),
+ point(7123, "mouseup"),
+ point(9000, "move"),
+ ]);
+ expect(cues).toEqual([
+ { timeSec: 7.056, gain: 1, release: false },
+ { timeSec: 7.123, gain: 0.7, release: true },
+ ]);
+ });
+
+ it("adds a second press for a double click", () => {
+ const cues = buildClickCues([point(300, "double-click"), point(360, "mouseup")]);
+ expect(cues.map((c) => [c.timeSec, c.release])).toEqual([
+ [0.3, false],
+ [0.36, true],
+ [0.39, false],
+ ]);
+ });
+
+ it("quiets right and middle clicks", () => {
+ const [press] = buildClickCues([point(500, "right-click"), point(560, "mouseup")]);
+ expect(press.gain).toBeCloseTo(0.85);
+ });
+
+ it("falls back to a synthetic release when the recorded one is too far away", () => {
+ const cues = buildClickCues([point(1000, "click"), point(3000, "mouseup")]);
+ expect(cues[1]).toEqual({ timeSec: 1.11, gain: 0.7, release: true });
+ });
+
+ it("pairs the release even when the samples arrive out of order", () => {
+ const cues = buildClickCues([point(2070, "mouseup"), point(2000, "click")]);
+ expect(cues.map((c) => c.timeSec)).toEqual([2, 2.07]);
+ });
+});
+
+describe("placeClickHits", () => {
+ const asset = { id: "asset_1", kind: "video", originalPath: "/t.mp4", durationSec: 10 };
+ const clip = (over: Record<string, number> = {}) => ({
+ id: "clip_1",
+ assetId: "asset_1",
+ timelineStartSec: 0,
+ timelineEndSec: 10,
+ sourceStartSec: 0,
+ sourceEndSec: 10,
+ ...over,
+ });
+ const doc = (clips: unknown[], speedRegions: unknown[] = [], trimRanges: unknown[] = []) =>
+ ({
+ project: { primaryAssetId: "asset_1" },
+ assets: [asset],
+ audioTracks: [],
+ timeline: { clips, trimRanges, speedRanges: [] },
+ // Speed regions live on the legacy envelope — `readSpeedRegions` is the only reader.
+ legacyEditor: { speedRegions },
+ }) as never;
+ const at = (document: never, cues: ClickCue[], take = asset) =>
+ placeClickHits(document, take as never, cues).map((hit) => hit.timeSec);
+
+ it("is the identity for an untouched single take", () => {
+ expect(at(doc([clip()]), [cue(7.25)])).toEqual([7.25]);
+ });
+
+ it("moves a click into the compressed programme a 2x region makes", () => {
+ // Raw 2..4 at 2x: the click at 3 s is heard at 2.5 s, the one at 5 s at 4 s. This is the
+ // arithmetic an imported audio track cannot do — its content is placed, never stretched.
+ expect(
+ at(doc([clip()], [{ startMs: 2000, endMs: 4000, speed: 2 }]), [cue(1), cue(3), cue(5)]),
+ ).toEqual([1, 2.5, 4]);
+ });
+
+ it("follows the clip the take sits on rather than ruler zero", () => {
+ // The take trimmed to start 2 s into the recording: the click recorded at 2.5 s of the
+ // recording is half a second into the film, and a click before the trim is never on
+ // screen at all. (The programme is the clips concatenated, so the clip's own timeline
+ // position is not part of it — see `projectRawTimelineSecToPlayback`.)
+ expect(
+ at(doc([clip({ timelineStartSec: 30, timelineEndSec: 38, sourceStartSec: 2 })]), [
+ cue(2.5),
+ cue(1.5),
+ ]),
+ ).toEqual([0.5]);
+ });
+
+ it("drops a click the cut took out of the film and pulls the later ones up", () => {
+ const trim = {
+ id: "trim_1",
+ assetId: "asset_1",
+ clipId: "clip_1",
+ startSec: 2,
+ endSec: 4,
+ origin: "user",
+ };
+ expect(at(doc([clip()], [], [trim]), [cue(1), cue(3), cue(5)])).toEqual([1, 3]);
+ });
+});
+
+describe("levelClickCues", () => {
+ it("scales every cue by the level the user set, and leaves 0 dB alone", () => {
+ const cues = [cue(1), { timeSec: 2, gain: 0.85, release: true }];
+ // The very same list at unity: nothing is rebuilt, so the cache stays the cache.
+ expect(levelClickCues(cues, 0)).toBe(cues);
+ const louder = levelClickCues(cues, 6);
+ expect(louder[0].gain).toBeCloseTo(1.995, 3);
+ expect(louder[1].gain).toBeCloseTo(1.696, 3);
+ expect(louder[1].release).toBe(true);
+ });
+});
+
+describe("crossedClickHits", () => {
+ const cues = [cue(1), cue(2.02), cue(3)];
+
+ it("fires the hits between two readings of the playhead", () => {
+ expect(crossedClickHits(cues, 1.99, 2.05).map((c) => c.timeSec)).toEqual([2.02]);
+ });
+
+ it("fires nothing across a seek, which is a jump too big for one frame", () => {
+ expect(crossedClickHits(cues, 0.5, 2.5)).toEqual([]);
+ });
+
+ it("fires nothing on a move backwards, and consumes nothing", () => {
+ expect(crossedClickHits(cues, 3, 1)).toEqual([]);
+ // Rewound and played again, the same clicks are still there to cross.
+ expect(crossedClickHits(cues, 0.95, 1.01).map((c) => c.timeSec)).toEqual([1]);
+ });
+
+ it("fires nothing before the playhead has a position to compare with", () => {
+ expect(crossedClickHits(cues, Number.NaN, 1)).toEqual([]);
+ });
+});
diff --git a/src/lib/ai-edition/clickSound.ts b/src/lib/ai-edition/clickSound.ts
new file mode 100644
index 000000000..13d7feb36
--- /dev/null
+++ b/src/lib/ai-edition/clickSound.ts
@@ -0,0 +1,364 @@
+// Mouse-click sound.
+//
+// The recorded click times (`<video>.cursor.json`, the same telemetry the click bounce and the
+// auto-zoom read) become short hits. Nothing is baked into a file: the export sums them in the
+// compositor (`mix_click_hits`, crates/compositor/src/audio.rs) and the preview plays them off the
+// same list, placed by the raw→programme projection the app lays imported audio tracks out with.
+// An imported track is moved by a speed region and never stretched, so a bed rendered against one
+// edit falls behind every click after it; these hits are computed from the edit instead.
+
+import type { CursorTelemetryPoint } from "@/components/video-editor/types";
+import { getAssetPath } from "@/lib/assetPath";
+import { nativeBridgeClient } from "@/native/client";
+import { projectRawTimelineSecToPlayback, readSpeedRegions } from "./document/timeline";
+import type { AxcutAsset, AxcutDocument } from "./schema";
+import { audioGainScalar, getEditorSettings } from "./store/editorSettings";
+import { resolveClipSourceEndSec } from "./timeline/clipDuration";
+import { removedRawSpans } from "./timeline/programme-time";
+
+export const CLICK_SOUND_DOWN_URL = "sounds/click-down.wav";
+export const CLICK_SOUND_UP_URL = "sounds/click-up.wav";
+/** The names the two samples are staged under, and the only names the main process accepts. */
+const HIT_FILES = { down: "click-down.wav", up: "click-up.wav" } as const;
+
+/**
+ * The asset label an earlier cut of this feature stamped onto projects: it rendered the whole
+ * take into one WAV and added that as a music track. Nothing writes one any more, and switching
+ * the sound on clears it — a document holding both would hear every click twice.
+ */
+export const LEGACY_CLICK_BED_LABEL = "Mouse clicks";
+
+/** A press with no matching release inside this window gets a synthetic one this far later. */
+const MAX_RELEASE_GAP_MS = 600;
+const FALLBACK_RELEASE_MS = 110;
+const DOUBLE_TAP_MS = 90;
+/** A playhead move larger than this is a seek, not a frame: no hit is owed across it. Kept well
+ * past one frame at the fastest preview rate (16x is ~0.27 s of raw time per frame), because a
+ * stray click after a small jump is a smaller wrong than a click that never sounds. */
+const SEEK_JUMP_SEC = 0.5;
+
+export interface ClickCue {
+ /** Seconds from the start of the recording — source time, the clock the sidecar is written on,
+ * not the edited timeline's. */
+ timeSec: number;
+ gain: number;
+ release: boolean;
+}
+
+/** One hit of the scene's `clickSound` block: programme seconds, the compositor's own clock. */
+export interface ClickHit {
+ timeSec: number;
+ gain: number;
+ release: boolean;
+}
+
+export interface ClickSoundScene {
+ downPath: string;
+ upPath: string;
+ hits: ClickHit[];
+}
+
+/**
+ * Press/release hits for one take. Releases come from the recorded `mouseup` samples so a held
+ * drag gets its let-go sound where the finger actually lifted; right and middle clicks sit a
+ * little quieter than the left. The bundled samples are already level-matched and stripped of
+ * leading silence, so a cue's own gain is the only volume decision left to make.
+ */
+export function buildClickCues(points: CursorTelemetryPoint[]): ClickCue[] {
+ const sorted = [...points].sort((a, b) => a.timeMs - b.timeMs);
+ const releases = sorted
+ .filter((p) => p.interactionType === "mouseup")
+ .map((p) => p.timeMs)
+ .sort((a, b) => a - b);
+
+ const cues: ClickCue[] = [];
+ const push = (ms: number, gain: number, release: boolean) => {
+ // The sidecar is disk data. A NaN would serialize as `null` into the scene and cost the
+ // whole export its parse, so a time that is not a number is not a click.
+ if (!Number.isFinite(ms) || ms < 0) return;
+ cues.push({ timeSec: ms / 1000, gain, release });
+ };
+
+ for (const point of sorted) {
+ switch (point.interactionType) {
+ case "click":
+ case "double-click":
+ case "right-click":
+ case "middle-click":
+ break;
+ default:
+ continue;
+ }
+ const isSide =
+ point.interactionType === "right-click" || point.interactionType === "middle-click";
+ const gain = isSide ? 0.85 : 1;
+ push(point.timeMs, gain, false);
+ if (point.interactionType === "double-click") push(point.timeMs + DOUBLE_TAP_MS, gain, false);
+ // ponytail: a press held past MAX_RELEASE_GAP_MS gets its release early instead of never
+ const release = releases.find(
+ (ms) => ms > point.timeMs && ms <= point.timeMs + MAX_RELEASE_GAP_MS,
+ );
+ push(release ?? point.timeMs + FALLBACK_RELEASE_MS, gain * 0.7, true);
+ }
+ return cues.sort((a, b) => a.timeSec - b.timeSec);
+}
+
+/** Resolved URLs of the two bundled hits, for the renderer and for staging. */
+export function clickHitUrls(): { down: string; up: string } {
+ const one = (relative: string) => {
+ try {
+ return getAssetPath(relative);
+ } catch {
+ return `/${relative}`;
+ }
+ };
+ return { down: one(CLICK_SOUND_DOWN_URL), up: one(CLICK_SOUND_UP_URL) };
+}
+
+// Both caches are one-shot: a finished take's sidecar never changes, and the staged samples are
+// the same bytes in every project.
+const cueCache = new Map<string, ClickCue[]>();
+let hitPaths: { down: string; up: string } | null = null;
+
+/**
+ * The take's cues, read from disk once per run. A take whose sidecar cannot be read is cached as
+ * holding no clicks: that is the answer the UI gives anyway, and re-asking on every scene build
+ * would be a read storm against a file that is simply not there.
+ */
+export async function loadClickCues(takePath: string): Promise<ClickCue[]> {
+ const cached = cueCache.get(takePath);
+ if (cached) return cached;
+ let cues: ClickCue[] = [];
+ try {
+ cues = buildClickCues(await nativeBridgeClient.cursor.getTelemetry(takePath));
+ } catch (error) {
+ console.error("Failed to read the click telemetry of", takePath, error);
+ }
+ cueCache.set(takePath, cues);
+ // A preview that asked for this take gets fed now rather than on the next edit.
+ if (previewTake === takePath) previewCues = levelClickCues(cues, previewGainDb);
+ return cues;
+}
+
+/**
+ * The bundled hits as files the compositor can open. The renderer knows them as a URL and the
+ * package layout they ship in is nobody else's business, so the bytes go to the main process once
+ * and it writes them under userData. `null` outside Electron (the browser shim keeps the URLs,
+ * which is all the preview needs).
+ */
+export async function ensureClickHitPaths(): Promise<{ down: string; up: string } | null> {
+ if (hitPaths) return hitPaths;
+ const stage = window.electronAPI?.stageClickSoundHit;
+ if (!stage) return null;
+ const urls = clickHitUrls();
+ const staged: string[] = [];
+ for (const [key, name] of [
+ ["down", HIT_FILES.down],
+ ["up", HIT_FILES.up],
+ ] as const) {
+ const response = await fetch(urls[key]);
+ if (!response.ok) return null;
+ const saved = await stage(name, await response.arrayBuffer());
+ if (!saved?.success || !saved.path) return null;
+ staged.push(saved.path);
+ }
+ hitPaths = { down: staged[0], up: staged[1] };
+ return hitPaths;
+}
+
+/** The document's primary take — the one whose sidecar the clicks were recorded in. */
+export function clickTakeOf(document: AxcutDocument | null | undefined) {
+ return (
+ document?.assets.find((asset) => asset.id === document.project?.primaryAssetId) ??
+ document?.assets.find((asset) => asset.kind === "video") ??
+ document?.assets[0]
+ );
+}
+
+/**
+ * The cues as the user asked to hear them. One place, because the preview plays these gains and
+ * the export sums the hits built from them — a level applied to only one of the two would make
+ * the preview lie about the file. `audioGainScalar` is the same dB law the output gain and the
+ * compositor's `finish_audio` use.
+ */
+export function levelClickCues(cues: ClickCue[], gainDb: number): ClickCue[] {
+ const scalar = audioGainScalar(gainDb);
+ return scalar === 1 ? cues : cues.map((cue) => ({ ...cue, gain: cue.gain * scalar }));
+}
+
+/**
+ * The scene's `clickSound` block for this document, or null when there is nothing to play.
+ *
+ * Synchronous by design: `buildSceneDescription` runs on paths that cannot await, so the answers
+ * come from the caches above and a take that has not been read is read in the background here.
+ * That is why an absent block means "off" as often as "not read yet" — the exporter awaits
+ * `prepareClickSound` first and never relies on the guess.
+ */
+export function clickSoundForDocument(
+ document: AxcutDocument,
+ cursor: { clickSound: boolean; clickSoundGainDb: number },
+): ClickSoundScene | null {
+ const take = cursor.clickSound ? clickTakeOf(document) : undefined;
+ const takePath = take?.originalPath;
+ // The level rides along, so a cue read that lands after this (the background one below) is
+ // still stashed at the level the user set, not at unity.
+ previewGainDb = cursor.clickSoundGainDb;
+ const cues = levelClickCues((takePath && cueCache.get(takePath)) || [], cursor.clickSoundGainDb);
+ previewTake = takePath;
+ previewCues = cues;
+ if (!takePath) return null;
+ // A take never read before is read now, in the background: the scene build cannot await.
+ // `loadClickCues` feeds the preview when it lands; an export awaits `prepareClickSound`.
+ if (!cueCache.has(takePath)) void loadClickCues(takePath);
+ if (!hitPaths || cues.length === 0) return null;
+ return {
+ downPath: hitPaths.down,
+ upPath: hitPaths.up,
+ hits: placeClickHits(document, take, cues),
+ };
+}
+
+/**
+ * The take's clicks, on the PROGRAMME seconds the compositor mixes onto. Three clocks are in
+ * play, so there are three steps: a click is recorded in the take's own source seconds, the
+ * timeline plays a clip's source window at that clip's position (a take trimmed at the head, or
+ * repeated across clips, is not at ruler zero), and only then do trims and speed regions move
+ * what is left. A click no clip shows — cut out, or past the last frame — gets no hit, because
+ * nothing plays there; that is also what keeps a cut from clamping the click onto the seam.
+ */
+export function placeClickHits(
+ document: AxcutDocument,
+ take: AxcutAsset,
+ cues: ClickCue[],
+): ClickHit[] {
+ const clips = document.timeline.clips;
+ const trims = document.timeline.trimRanges;
+ const speeds = readSpeedRegions(document);
+ const removed = removedRawSpans(clips, trims);
+ const windows = clips
+ .filter((clip) => clip.assetId === take.id)
+ .map((clip) => ({
+ from: clip.sourceStartSec,
+ to: resolveClipSourceEndSec(clip, take),
+ at: clip.timelineStartSec,
+ }));
+ const hits: ClickHit[] = [];
+ for (const cue of cues) {
+ for (const window of windows) {
+ if (cue.timeSec < window.from || cue.timeSec >= window.to) continue;
+ const rawSec = window.at + (cue.timeSec - window.from);
+ if (removed.some((span) => rawSec >= span.startSec && rawSec < span.endSec)) continue;
+ hits.push({
+ timeSec: projectRawTimelineSecToPlayback(clips, trims, rawSec, speeds),
+ gain: cue.gain,
+ release: cue.release,
+ });
+ }
+ }
+ return hits.sort((a, b) => a.timeSec - b.timeSec);
+}
+
+/**
+ * Loads what the sound needs for `document`: its cues, and the samples staged for the export.
+ * Call this where a result must not depend on a background read having finished — switching the
+ * sound on, and exporting with it on. Does nothing while the setting is off, so a caller that
+ * always asks cannot hand the preview clicks the user never asked to hear.
+ */
+export async function prepareClickSound(document: AxcutDocument): Promise<void> {
+ const cursor = getEditorSettings(document).cursor;
+ if (!cursor.clickSound) return;
+ const take = clickTakeOf(document)?.originalPath;
+ if (!take) return;
+ const cues = await loadClickCues(take);
+ previewTake = take;
+ previewGainDb = cursor.clickSoundGainDb;
+ previewCues = levelClickCues(cues, cursor.clickSoundGainDb);
+ await ensureClickHitPaths();
+}
+
+// What the preview plays. Building the scene is the one place that has the document, the setting
+// and the cues together, so it stashes them here and the rAF tick drains them by crossings.
+let previewCues: ClickCue[] = [];
+let previewTake: string | undefined;
+let previewGainDb = 0;
+let lastSourceSec = Number.NaN;
+
+/** The hits the playhead passed since the last call, measured in the take's own source seconds —
+ * where a click was recorded, and where the preview's picture is: under a 2x region that clock
+ * races with the picture, so a hit fired on a crossing lands on the click being shown. */
+export function takeCrossedClickHits(sourceSec: number): ClickCue[] {
+ const crossed = crossedClickHits(previewCues, lastSourceSec, sourceSec);
+ lastSourceSec = sourceSec;
+ return crossed;
+}
+
+/**
+ * The clicks whose recorded time lies between two readings of the playhead, both in the take's own
+ * source seconds. A move too big for one frame is a seek (or a clip swap) and nothing is owed
+ * across it; a move backwards is a rewind, and since nothing is consumed the same clicks are
+ * crossed again.
+ */
+export function crossedClickHits(
+ cues: ClickCue[],
+ fromSourceSec: number,
+ toSourceSec: number,
+): ClickCue[] {
+ if (
+ !(fromSourceSec >= 0) ||
+ toSourceSec <= fromSourceSec ||
+ toSourceSec - fromSourceSec > SEEK_JUMP_SEC
+ ) {
+ return [];
+ }
+ return cues.filter((cue) => cue.timeSec > fromSourceSec && cue.timeSec <= toSourceSec);
+}
+
+/** The preview calls this while stopped: the next run starts wherever the user left the playhead,
+ * and an anchor from the previous run would fire a stack of clicks that already went by. */
+export function resetClickPlayhead(): void {
+ lastSourceSec = Number.NaN;
+}
+
+const hitBuffers = new WeakMap<AudioContext, Promise<AudioBuffer[]>>();
+
+/** The two samples as `[press, release]`, decoded once per context. */
+export function clickHitBuffers(context: AudioContext): Promise<AudioBuffer[]> {
+ let pending = hitBuffers.get(context);
+ if (!pending) {
+ const urls = clickHitUrls();
+ pending = Promise.all([decodeHit(context, urls.down), decodeHit(context, urls.up)]);
+ hitBuffers.set(context, pending);
+ }
+ return pending;
+}
+
+async function decodeHit(context: AudioContext, url: string): Promise<AudioBuffer> {
+ const response = await fetch(url);
+ if (!response.ok) throw new Error(`Failed to fetch ${url}`);
+ return context.decodeAudioData(await response.arrayBuffer());
+}
+
+/** The hits, heard. `buffers` resolves on its own so a tick never waits on a decode, and a
+ * sample that will not load stays silent rather than throwing on every frame. */
+export function playClickHits(
+ context: AudioContext,
+ destination: AudioNode,
+ buffers: Promise<AudioBuffer[]>,
+ cues: ClickCue[],
+): void {
+ void buffers
+ .then(([down, up]) => {
+ if (context.state === "closed") return;
+ for (const cue of cues) {
+ const buffer = cue.release ? up : down;
+ if (!buffer) continue;
+ const source = context.createBufferSource();
+ source.buffer = buffer;
+ const gain = context.createGain();
+ gain.gain.value = cue.gain;
+ source.connect(gain).connect(destination);
+ source.start();
+ }
+ })
+ .catch(() => undefined);
+}
diff --git a/src/lib/ai-edition/store/editorSettings.ts b/src/lib/ai-edition/store/editorSettings.ts
index 7a6b209bf..2fc464ad6 100644
--- a/src/lib/ai-edition/store/editorSettings.ts
+++ b/src/lib/ai-edition/store/editorSettings.ts
@@ -189,6 +189,8 @@ interface LegacyShape {
/** Written by builds with one "always use the arrow" switch; read as every kind. */
cursorAlwaysArrow?: boolean;
cursorClickImpact?: boolean;
+ cursorClickSound?: boolean;
+ cursorClickSoundGainDb?: number;
cursorShow?: boolean;
cursorAutoHide?: boolean;
cursorTheme?: string;
@@ -238,6 +240,12 @@ export function getEditorSettings(doc: AxcutDocument | null | undefined): Editor
defaults.cursor.asArrow,
),
clickImpact: bool(legacy?.cursorClickImpact, DEFAULT_EDITOR_SETTINGS.cursor.clickImpact),
+ clickSound: bool(legacy?.cursorClickSound, DEFAULT_EDITOR_SETTINGS.cursor.clickSound),
+ clickSoundGainDb: readBounded(
+ legacy?.cursorClickSoundGainDb,
+ "cursorClickSoundGainDb",
+ defaults.cursor.clickSoundGainDb,
+ ),
autoHide: bool(legacy?.cursorAutoHide, DEFAULT_EDITOR_SETTINGS.cursorAutoHide),
};
@@ -391,6 +399,8 @@ function nextLegacy(current: LegacyShape | null, patch: EditorSettingsPatch): Le
delete next.cursorAlwaysArrow;
}
if (c.clickImpact !== undefined) next.cursorClickImpact = c.clickImpact;
+ if (c.clickSound !== undefined) next.cursorClickSound = c.clickSound;
+ if (c.clickSoundGainDb !== undefined) next.cursorClickSoundGainDb = c.clickSoundGainDb;
if (c.theme !== undefined) next.cursorTheme = c.theme;
if (c.show !== undefined) next.cursorShow = c.show;
if (c.autoHide !== undefined) next.cursorAutoHide = c.autoHide;
diff --git a/src/lib/ai-edition/stylePresets.test.ts b/src/lib/ai-edition/stylePresets.test.ts
index 503718860..2a5c4b83e 100644
--- a/src/lib/ai-edition/stylePresets.test.ts
+++ b/src/lib/ai-edition/stylePresets.test.ts
@@ -44,6 +44,8 @@ function appearance(overrides: Partial<StylePresetAppearance> = {}): StylePreset
model3d: false,
asArrow: [],
clickImpact: false,
+ clickSound: false,
+ clickSoundGainDb: 0,
},
cursorShow: true,
cursorAutoHide: false,
@@ -164,13 +166,26 @@ describe("parseStylePresetAppearance", () => {
).toThrow(/cursor\.model3d/);
});
- it("reads a preset written before the click impact as without one, and type-checks it", () => {
- const { clickImpact: _clickImpact, ...older } = appearance().cursor;
+ it("reads a preset written before the click impact or the click sound as without them, and type-checks each", () => {
+ const {
+ clickImpact: _clickImpact,
+ clickSound: _clickSound,
+ clickSoundGainDb: _clickSoundGainDb,
+ ...older
+ } = appearance().cursor;
const read = (cursor: unknown) =>
parseStylePresetAppearance({ ...appearance(), cursor }).cursor;
expect(read(older).clickImpact).toBe(false);
+ expect(read(older).clickSound).toBe(false);
+ // A preset older than the level plays at the level the samples were cut for.
+ expect(read(older).clickSoundGainDb).toBe(0);
expect(read({ ...older, clickImpact: true }).clickImpact).toBe(true);
+ expect(read({ ...older, clickSound: true }).clickSound).toBe(true);
+ expect(read({ ...older, clickSoundGainDb: -12 }).clickSoundGainDb).toBe(-12);
+ // Past the bound the mixer would flatten, the preset keeps its meaning by clamping.
+ expect(read({ ...older, clickSoundGainDb: 30 }).clickSoundGainDb).toBe(12);
expect(() => read({ ...older, clickImpact: "yes" })).toThrow(/cursor\.clickImpact/);
+ expect(() => read({ ...older, clickSound: "yes" })).toThrow(/cursor\.clickSound/);
});
it("reads the cursor kinds drawn as the arrow, and type-checks them", () => {
diff --git a/src/lib/ai-edition/stylePresets.ts b/src/lib/ai-edition/stylePresets.ts
index f7cbe68e9..d7d90b542 100644
--- a/src/lib/ai-edition/stylePresets.ts
+++ b/src/lib/ai-edition/stylePresets.ts
@@ -130,6 +130,7 @@ const CURSOR_BOUNDS = {
smoothing: "cursorSmoothing",
motionBlur: "cursorMotionBlur",
clickBounce: "cursorClickBounce",
+ clickSoundGainDb: "cursorClickSoundGainDb",
} as const satisfies Record<string, SettingBound>;
type Fields = Record<string, unknown>;
@@ -341,6 +342,14 @@ export function parseStylePresetAppearance(value: unknown): StylePresetAppearanc
// Presets written before the option existed gave clicks no impact.
clickImpact:
cursor.clickImpact === undefined ? false : readBoolean(cursor, "clickImpact", "cursor."),
+ // Same for the sound: a preset written before it existed stays silent.
+ clickSound:
+ cursor.clickSound === undefined ? false : readBoolean(cursor, "clickSound", "cursor."),
+ // And one written before the level existed plays at the level the samples were cut for.
+ clickSoundGainDb:
+ cursor.clickSoundGainDb === undefined
+ ? 0
+ : readNumber(cursor, "clickSoundGainDb", CURSOR_BOUNDS.clickSoundGainDb, "cursor."),
},
cursorShow: readBoolean(value, "cursorShow"),
cursorAutoHide: readBoolean(value, "cursorAutoHide"),
@@ -436,6 +445,8 @@ export const LOOK_LEGACY_EDITOR_KEYS = [
// A project saved before the kinds could be picked holds its look here instead.
"cursorAlwaysArrow",
"cursorClickImpact",
+ "cursorClickSound",
+ "cursorClickSoundGainDb",
"cursorShow",
"cursorAutoHide",
"cursorTheme",
@@ -453,6 +464,8 @@ export function stylePresetLegacyEditor(appearance: StylePresetAppearance): Fiel
cursorModel3d: cursor.model3d,
cursorAsArrow: cursor.asArrow,
cursorClickImpact: cursor.clickImpact,
+ cursorClickSound: cursor.clickSound,
+ cursorClickSoundGainDb: cursor.clickSoundGainDb,
};
}
diff --git a/src/lib/ai-edition/stylePresetsEditor.test.ts b/src/lib/ai-edition/stylePresetsEditor.test.ts
index 3050e3b6a..0716284e1 100644
--- a/src/lib/ai-edition/stylePresetsEditor.test.ts
+++ b/src/lib/ai-edition/stylePresetsEditor.test.ts
@@ -59,6 +59,8 @@ function styledSettings(): EditorSettingsSnapshot {
model3d: true,
asArrow: ["text"],
clickImpact: true,
+ clickSound: true,
+ clickSoundGainDb: 0,
},
cursorShow: false,
cursorAutoHide: true,
diff --git a/src/lib/ai-edition/stylePresetsEditor.ts b/src/lib/ai-edition/stylePresetsEditor.ts
index 189b7b972..1b7086e3a 100644
--- a/src/lib/ai-edition/stylePresetsEditor.ts
+++ b/src/lib/ai-edition/stylePresetsEditor.ts
@@ -42,6 +42,8 @@ export function stylePresetAppearanceFromSettings(
model3d: settings.cursor.model3d,
asArrow: [...settings.cursor.asArrow],
clickImpact: settings.cursor.clickImpact,
+ clickSound: settings.cursor.clickSound,
+ clickSoundGainDb: settings.cursor.clickSoundGainDb,
},
cursorShow: settings.cursorShow,
cursorAutoHide: settings.cursorAutoHide,
diff --git a/src/lib/projectDefaults.ts b/src/lib/projectDefaults.ts
index 2dde515e1..91a9eeaf8 100644
--- a/src/lib/projectDefaults.ts
+++ b/src/lib/projectDefaults.ts
@@ -222,6 +222,8 @@ export interface ProjectAppearanceDefaults {
model3d: boolean;
asArrow: CursorKind[];
clickImpact: boolean;
+ clickSound: boolean;
+ clickSoundGainDb: number;
autoHide: boolean;
};
cursorShow: boolean;
@@ -271,6 +273,8 @@ export const DEFAULT_PROJECT_APPEARANCE: ProjectAppearanceDefaults = {
// arrow until picked (`CURSOR_KINDS`). A test holds this to every kind but the hand.
asArrow: ["text", "grab", "resize", "busy", "crosshair", "not-allowed", "help", "up-arrow"],
clickImpact: false,
+ clickSound: false,
+ clickSoundGainDb: 0,
autoHide: false,
},
cursorShow: true,
@@ -305,6 +309,9 @@ export const SETTING_BOUNDS = {
cursorMotionBlur: [0, 1],
// Past about 4.2 the arrow shrank to nothing on every click.
cursorClickBounce: [0, 2],
+ // The click sound's level in dB. −24 is a whisper and +12 is full scale for the bundled hits,
+ // which are cut to a 0.7 peak; past +12 the compositor's own hit clamp would start flattening.
+ cursorClickSoundGainDb: [-24, 12],
// 16 is Chromium's `playbackRate` ceiling: past it the preview could not show what the
// export rendered.
playbackSpeed: [0.25, 16],
diff --git a/src/native/browserShim.presets.test.ts b/src/native/browserShim.presets.test.ts
index 481e57c2c..10c9f2d08 100644
--- a/src/native/browserShim.presets.test.ts
+++ b/src/native/browserShim.presets.test.ts
@@ -38,6 +38,8 @@ const APPEARANCE: StylePresetAppearance = {
model3d: false,
asArrow: [],
clickImpact: false,
+ clickSound: false,
+ clickSoundGainDb: 0,
},
cursorShow: true,
cursorAutoHide: false,
diff --git a/src/native/sceneDescription.ts b/src/native/sceneDescription.ts
index e029aa8f2..2ca7dd861 100644
--- a/src/native/sceneDescription.ts
+++ b/src/native/sceneDescription.ts
@@ -30,6 +30,7 @@ import {
getCaptionSettings,
getCaptionTranslations,
} from "@/lib/ai-edition/captions";
+import { clickSoundForDocument } from "@/lib/ai-edition/clickSound";
import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks";
import { createId } from "@/lib/ai-edition/document/ids";
import { isFormatFillActive, pickOutputDims } from "@/lib/ai-edition/document/outputFormat";
@@ -540,6 +541,20 @@ export interface SceneDescription {
/** A voiceover is voice: the export levels it like the recording's own audio. */
kind: AxcutAudioTrack["kind"];
}>;
+ /**
+ * The "mouse clicks" toggle (see `src/lib/ai-edition/clickSound.ts`): the recorded clicks as
+ * hits placed on the OUTPUT programme, summed by `audio::mix_click_hits`. Omitted when the
+ * sound is off, the take has no clicks, or the cues have not been read yet.
+ *
+ * Not entries of `audioTracks`. An imported track is placed and never stretched, so a click
+ * bed baked into one file falls behind every click past a speed region; these hits go through
+ * the same `projectRawTimelineSecToPlayback` imported audio tracks are laid out with.
+ */
+ clickSound?: {
+ downPath: string;
+ upPath: string;
+ hits: Array<{ timeSec: number; gain: number; release: boolean }>;
+ };
/**
* Per-clip screen crop (fractions of the frame), or null for the identity
* (full-frame) crop. One entry per clip in the same order as `clips`, so a
@@ -772,7 +787,8 @@ export function webcamBoxSourceSize(
};
}
-/** Serialize a document into a {@link SceneDescription}. Pure — no per-frame math. */
+/** Serialize a document into a {@link SceneDescription}. No per-frame math; the one side effect is
+ * handing the click cues to the preview, which has no document of its own to read them from. */
export function buildSceneDescription(
document: AxcutDocument,
webcamSourceSize: { width: number; height: number } | null = null,
@@ -818,6 +834,9 @@ export function buildSceneDescription(
.filter((pill) => pill.kind === "voiceover" && !pill.loop)
.map((pill) => [trackGroupId(pill), pill]),
);
+ // Click hits for the "mouse clicks" toggle, placed on the programme this export assembles.
+ // Synchronous: the cues were read from the take's sidecar when the sound was switched on.
+ const clickSound = clickSoundForDocument(document, settings.cursor);
const audioTracks = document.audioTracks.flatMap((track) => {
if (track.muted) return [];
const asset = assetById.get(track.assetId);
@@ -1289,6 +1308,7 @@ export function buildSceneDescription(
gainDb: settings.audioGainDb,
},
audioTracks,
+ ...(clickSound ? { clickSound } : {}),
background: sceneBackground(settings.wallpaper, settings.wallpaperMotion),
zoomRegions: projectedZoomRegions.map((region) => ({
id: region.id,