Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
// Mouse-click sound.
//
// The recorded click times (`<video>.cursor.json`, the same telemetry the click bounce and the
// auto-zoom read) become short hits. Nothing is baked into a file: the export sums them in the
// compositor (`mix_click_hits`, crates/compositor/src/audio.rs) and the preview plays them off the
// same list, placed by the raw→programme projection the app lays imported audio tracks out with.
// An imported track is moved by a speed region and never stretched, so a bed rendered against one
// edit falls behind every click after it; these hits are computed from the edit instead.
import type { CursorTelemetryPoint } from "@/components/video-editor/types";
import { nativeBridgeClient } from "@/native/client";
import { projectRawTimelineSecToPlayback, readSpeedRegions } from "./document/timeline";
import type { AxcutAsset, AxcutDocument } from "./schema";
import { audioGainScalar, getEditorSettings } from "./store/editorSettings";
import { resolveClipSourceEndSec } from "./timeline/clipDuration";
import { removedRawSpans } from "./timeline/programme-time";
export const CLICK_SOUND_DOWN_URL = "sounds/click-down.wav";
export const CLICK_SOUND_UP_URL = "sounds/click-up.wav";
/** The names the two samples are staged under, and the only names the main process accepts. */
const HIT_FILES = { down: "click-down.wav", up: "click-up.wav" } as const;
/** A press with no matching release inside this window gets a synthetic one this far later. */
const MAX_RELEASE_GAP_MS = 600;
const FALLBACK_RELEASE_MS = 110;
const DOUBLE_TAP_MS = 90;
/** A playhead move larger than this is a seek, not a frame: no hit is owed across it. Kept well
* past one frame at the fastest preview rate (16x is ~0.27 s of raw time per frame), because a
* stray click after a small jump is a smaller wrong than a click that never sounds.
*
* This is only a seek detector, never a cut detector: a trim narrower than the threshold reads as
* an ordinary frame step, which is why the cues the preview fires go through `placedClickHits`. */
const SEEK_JUMP_SEC = 0.5;
export interface ClickCue {
/** Seconds from the start of the recording — source time, the clock the sidecar is written on,
* not the edited timeline's. */
timeSec: number;
gain: number;
release: boolean;
}
/** One hit of the scene's `clickSound` block: programme seconds, the compositor's own clock. */
export interface ClickHit {
timeSec: number;
gain: number;
release: boolean;
}
export interface ClickSoundScene {
downPath: string;
upPath: string;
hits: ClickHit[];
}
/**
* Press/release hits for one take. Releases come from the recorded `mouseup` samples so a held
* drag gets its let-go sound where the finger actually lifted; right and middle clicks sit a
* little quieter than the left. The bundled samples are already level-matched and stripped of
* leading silence, so a cue's own gain is the only volume decision left to make.
*/
export function buildClickCues(points: CursorTelemetryPoint[]): ClickCue[] {
const sorted = [...points].sort((a, b) => a.timeMs - b.timeMs);
const releases = sorted
.filter((p) => p.interactionType === "mouseup")
.map((p) => p.timeMs)
.sort((a, b) => a - b);
const cues: ClickCue[] = [];
const push = (ms: number, gain: number, release: boolean) => {
// The sidecar is disk data. A NaN would serialize as `null` into the scene and cost the
// whole export its parse, so a time that is not a number is not a click.
if (!Number.isFinite(ms) || ms < 0) return;
cues.push({ timeSec: ms / 1000, gain, release });
};
for (const point of sorted) {
switch (point.interactionType) {
case "click":
case "double-click":
case "right-click":
case "middle-click":
break;
default:
continue;
}
const isSide =
point.interactionType === "right-click" || point.interactionType === "middle-click";
const gain = isSide ? 0.85 : 1;
push(point.timeMs, gain, false);
if (point.interactionType === "double-click") push(point.timeMs + DOUBLE_TAP_MS, gain, false);
// ponytail: a press held past MAX_RELEASE_GAP_MS gets its release early instead of never
const release = releases.find(
(ms) => ms > point.timeMs && ms <= point.timeMs + MAX_RELEASE_GAP_MS,
);
push(release ?? point.timeMs + FALLBACK_RELEASE_MS, gain * 0.7, true);
}
return cues.sort((a, b) => a.timeSec - b.timeSec);
}
/** Resolved URLs of the two bundled hits, for the renderer and for staging. */
export function clickHitUrls(): { down: string; up: string } {
// Page-relative, like the web-demuxer wasm: Vite copies public/sounds into dist/, which an
// installer ships inside the asar. `getAssetPath` resolves against resources/, which only holds
// what electron-builder's extraResources copies there.
const one = (relative: string) => new URL(relative, window.location.href).href;
return { down: one(CLICK_SOUND_DOWN_URL), up: one(CLICK_SOUND_UP_URL) };
}
// Both caches are one-shot: a finished take's sidecar never changes, and the staged samples are
// the same bytes in every project.
const cueCache = new Map<string, ClickCue[]>();
let hitPaths: { down: string; up: string } | null = null;
let hitPathsPending: Promise<{ down: string; up: string } | null> | null = null;
/**
* The take's cues, read from disk once per run. A take whose sidecar cannot be read is cached as
* holding no clicks: that is the answer the UI gives anyway, and re-asking on every scene build
* would be a read storm against a file that is simply not there.
*/
export async function loadClickCues(takePath: string): Promise<ClickCue[]> {
const cached = cueCache.get(takePath);
if (cached) return cached;
let cues: ClickCue[] = [];
try {
cues = buildClickCues(await nativeBridgeClient.cursor.getTelemetry(takePath));
} catch (error) {
console.error("Failed to read the click telemetry of", takePath, error);
}
cueCache.set(takePath, cues);
// A preview that asked for this take gets fed now rather than on the next edit, through the
// edit that asked for it — the same filter every other stash goes through.
if (previewTake === takePath && previewEdit) {
stashPreviewCues(previewEdit.document, previewEdit.take, levelClickCues(cues, previewGainDb));
}
return cues;
}
/**
* The bundled hits as files the compositor can open. The renderer knows them as a URL and the
* package layout they ship in is nobody else's business, so the bytes go to the main process once
* and it writes them under userData. `null` outside Electron (the browser shim keeps the URLs,
* which is all the preview needs).
*/
export function ensureClickHitPaths(): Promise<{ down: string; up: string } | null> {
if (hitPaths) return Promise.resolve(hitPaths);
// One staging at a time: the toggle and an export can both ask before the first one lands,
// and a second write of the same file could overlap the compositor reading it.
hitPathsPending ??= stageClickHits().finally(() => {
hitPathsPending = null;
});
return hitPathsPending;
}
async function stageClickHits(): Promise<{ down: string; up: string } | null> {
const stage = window.electronAPI?.stageClickSoundHit;
if (!stage) return null;
const urls = clickHitUrls();
const staged: string[] = [];
for (const [key, name] of [
["down", HIT_FILES.down],
["up", HIT_FILES.up],
] as const) {
const response = await fetch(urls[key]);
if (!response.ok) return null;
const saved = await stage(name, await response.arrayBuffer());
if (!saved?.success || !saved.path) return null;
staged.push(saved.path);
}
hitPaths = { down: staged[0], up: staged[1] };
return hitPaths;
}
/** The document's primary take — the one whose sidecar the clicks were recorded in. */
export function clickTakeOf(document: AxcutDocument | null | undefined) {
return (
document?.assets.find((asset) => asset.id === document.project?.primaryAssetId) ??
document?.assets.find((asset) => asset.kind === "video") ??
document?.assets[0]
);
}
/**
* The cues as the user asked to hear them. One place, because the preview plays these gains and
* the export sums the hits built from them — a level applied to only one of the two would make
* the preview lie about the file. `audioGainScalar` is the same dB law the output gain and the
* compositor's `finish_audio` use.
*/
export function levelClickCues(cues: ClickCue[], gainDb: number): ClickCue[] {
const scalar = audioGainScalar(gainDb);
return scalar === 1 ? cues : cues.map((cue) => ({ ...cue, gain: cue.gain * scalar }));
}
/**
* The scene's `clickSound` block for this document, or null when there is nothing to play.
*
* Synchronous by design: `buildSceneDescription` runs on paths that cannot await, so the answers
* come from the caches above and a take that has not been read is read in the background here.
* That is why an absent block means "off" as often as "not read yet" — the exporter awaits
* `prepareClickSound` first and never relies on the guess.
*/
export function clickSoundForDocument(
document: AxcutDocument,
cursor: { clickSound: boolean; clickSoundGainDb: number },
): ClickSoundScene | null {
const take = cursor.clickSound ? clickTakeOf(document) : undefined;
const takePath = take?.originalPath;
// The level rides along, so a cue read that lands after this (the background one below) is
// still stashed at the level the user set, not at unity.
previewGainDb = cursor.clickSoundGainDb;
if (!takePath || !take) {
previewEdit = null;
previewTake = undefined;
previewCues = [];
previewByClip.clear();
return null;
}
// A take never read before is read now, in the background: the scene build cannot await.
// `loadClickCues` feeds the preview when it lands; an export awaits `prepareClickSound`.
// The main process builds scenes too (getFrames renders from the document) and has no renderer
// bridge to read the sidecar through. It renders pictures, never this sound, so it skips the read.
if (!cueCache.has(takePath) && typeof window !== "undefined") void loadClickCues(takePath);
const cues = levelClickCues(cueCache.get(takePath) || [], cursor.clickSoundGainDb);
stashPreviewCues(document, take, cues);
if (!hitPaths || cues.length === 0) return null;
return {
downPath: hitPaths.down,
upPath: hitPaths.up,
// The same placement the preview was filtered by, only mapped onto the programme clock.
hits: placeClickHits(document, take, cues),
};
}
/**
* Where this edit plays a click recorded at `cue.timeSec` of the take: every clip that shows it,
* each with the PROGRAMME second the compositor mixes onto, and none when nothing is on screen
* there. A take pasted twice plays its click twice, once per copy that keeps it. Three
* clocks are in play, so there are three steps: a click is recorded in the take's own source
* seconds, the timeline plays a clip's source window at that clip's position (a take trimmed at the
* head, or repeated across clips, is not at ruler zero), and only then do trims and speed regions
* move what is left. A cue no clip shows — cut out, or past the last frame — has no placement,
* which is also what keeps a cut from clamping the click onto the seam.
*
* The clip comes along in the answer because the preview reads the mounted clip's source clock, and
* the same take can sit in two clips that cut away different stretches of it: a cue the mounted
* clip removed is not that clip's to fire, however a twin keeps it. Upstream closes the same leak
* for playback in `locateKeptSegment`, which is also handed the active clip id.
*
* The edit is read once, so the result is a per-cue lookup; the export's hits and the preview's
* cues both come from it, which is the point.
*/
function clickPlacement(
document: AxcutDocument,
take: AxcutAsset,
): (cue: ClickCue) => Array<{ clipId: string; sec: number }> {
const clips = document.timeline.clips;
const trims = document.timeline.trimRanges;
const speeds = readSpeedRegions(document);
const removed = removedRawSpans(clips, trims);
const windows = clips
.filter((clip) => clip.assetId === take.id)
.map((clip) => ({
id: clip.id,
from: clip.sourceStartSec,
to: resolveClipSourceEndSec(clip, take),
at: clip.timelineStartSec,
}));
return (cue) => {
const placements: Array<{ clipId: string; sec: number }> = [];
for (const window of windows) {
if (cue.timeSec < window.from || cue.timeSec >= window.to) continue;
const rawSec = window.at + (cue.timeSec - window.from);
if (removed.some((span) => rawSec >= span.startSec && rawSec < span.endSec)) continue;
placements.push({
clipId: window.id,
sec: projectRawTimelineSecToPlayback(clips, trims, rawSec, speeds),
});
}
return placements;
};
}
/** The take's clicks this edit plays, each with the clip that plays it and its programme second. */
export function placedClickHits(
document: AxcutDocument,
take: AxcutAsset,
cues: ClickCue[],
): Array<{ clipId: string; sec: number; cue: ClickCue }> {
const place = clickPlacement(document, take);
const placed: Array<{ clipId: string; sec: number; cue: ClickCue }> = [];
for (const cue of cues) {
for (const at of place(cue)) placed.push({ ...at, cue });
}
return placed.sort((a, b) => a.sec - b.sec);
}
/** The take's clicks, as hits on the PROGRAMME seconds the compositor mixes onto. */
export function placeClickHits(
document: AxcutDocument,
take: AxcutAsset,
cues: ClickCue[],
): ClickHit[] {
return placedClickHits(document, take, cues).map(({ sec, cue }) => ({
timeSec: sec,
gain: cue.gain,
release: cue.release,
}));
}
/**
* Loads what the sound needs for `document`: its cues, and the samples staged for the export.
* Call this where a result must not depend on a background read having finished — switching the
* sound on, and exporting with it on. Does nothing while the setting is off, so a caller that
* always asks cannot hand the preview clicks the user never asked to hear.
*/
export async function prepareClickSound(document: AxcutDocument): Promise<void> {
const settings = getEditorSettings(document);
const cursor = settings.cursor;
if (!cursor.clickSound || !settings.cursorShow) return;
const take = clickTakeOf(document);
if (!take?.originalPath) return;
previewGainDb = cursor.clickSoundGainDb;
const cues = await loadClickCues(take.originalPath);
stashPreviewCues(document, take, levelClickCues(cues, cursor.clickSoundGainDb));
await ensureClickHitPaths();
}
// What the preview plays. Building the scene is the one place that has the document, the setting
// and the cues together, so it stashes them here and the rAF tick drains them by crossings.
let previewCues: ClickCue[] = [];
/** The same cues under the clip that plays them: a tick crosses one clip's source window, and a
* click a twin clip keeps is not this clip's to fire over a stretch it cut away. */
const previewByClip = new Map<string, ClickCue[]>();
let previewTake: string | undefined;
let previewGainDb = 0;
/** The edit the stashed cues belong to, kept so a cue read that lands after the build is filtered
* by that same edit instead of going back to playing clicks the cut removed. */
let previewEdit: { document: AxcutDocument; take: AxcutAsset } | null = null;
let lastSourceSec = Number.NaN;
let playheadGeneration = 0;
/** Stashes what the preview may fire, already at the level the user set, through the same placement
* the export's hits come from. The two then cannot disagree about which clicks exist, or over which
* clip — only about when they sound. */
function stashPreviewCues(
document: AxcutDocument,
take: AxcutAsset,
leveledCues: ClickCue[],
): void {
previewEdit = { document, take };
previewTake = take.originalPath;
previewByClip.clear();
// A cue two clips keep is one recorded click: the clip-less list, crossed on the source clock,
// holds it once so a crossing fires it once.
const kept = new Set<ClickCue>();
for (const placed of placedClickHits(document, take, leveledCues)) {
kept.add(placed.cue);
const bucket = previewByClip.get(placed.clipId) ?? [];
bucket.push(placed.cue);
previewByClip.set(placed.clipId, bucket);
}
previewCues = [...kept].sort((x, y) => x.timeSec - y.timeSec);
}
/** The hits the playhead passed since the last call, measured in the take's own source seconds —
* where a click was recorded, and where the preview's picture is: under a 2x region that clock
* races with the picture, so a hit fired on a crossing lands on the click being shown.
*
* `mountedTakePath` is the take on screen now. The cues belong to one take, so playing them over
* another take's picture would put a click where no click happened. `mountedClipId` narrows it one
* step further, to the clip on screen: the same take can appear in two clips that cut different
* stretches, and a click the mounted clip removed belongs to the twin's clock, not this one.
* An unresolved id is not a reason to go silent, so it plays every click the edit shows. */
export function takeCrossedClickHits(
sourceSec: number,
mountedTakePath?: string | null,
mountedClipId?: string | null,
): ClickCue[] {
const fit = clickCuesFitTake(previewTake, mountedTakePath);
const cues = mountedClipId ? (previewByClip.get(mountedClipId) ?? []) : previewCues;
const crossed = fit ? crossedClickHits(cues, lastSourceSec, sourceSec) : [];
// The anchor moves either way: coming back to the right take must not repay a stack of clicks.
lastSourceSec = sourceSec;
return crossed;
}
/**
* Whether cues recorded against `cueTake` may sound while `mountedTake` is on screen. Only a known
* mismatch suppresses them — a preview that has no path resolved yet (a source still being built,
* or one loaded from a URL rather than a file) must not fall silent because of this check.
*/
export function clickCuesFitTake(
cueTake: string | undefined,
mountedTake?: string | null,
): boolean {
return !cueTake || !mountedTake || mountedTake === cueTake;
}
/**
* The clicks whose recorded time lies between two readings of the playhead, both in the take's own
* source seconds. A move too big for one frame is a seek (or a clip swap) and nothing is owed
* across it; a move backwards is a rewind, and since nothing is consumed the same clicks are
* crossed again.
*/
export function crossedClickHits(
cues: ClickCue[],
fromSourceSec: number,
toSourceSec: number,
): ClickCue[] {
if (
!(fromSourceSec >= 0) ||
toSourceSec <= fromSourceSec ||
toSourceSec - fromSourceSec > SEEK_JUMP_SEC
) {
return [];
}
return cues.filter((cue) => cue.timeSec > fromSourceSec && cue.timeSec <= toSourceSec);
}
/** The preview calls this while stopped: the next run starts wherever the user left the playhead,
* and an anchor from the previous run would fire a stack of clicks that already went by. */
export function resetClickPlayhead(): void {
lastSourceSec = Number.NaN;
// Also retires whatever is still being decoded. The sample is fetched and decoded on first use,
// so the last pre-stop crossing can land a few frames later; a paused context keeps running, so
// `state` alone cannot tell that run from the one the user stopped.
playheadGeneration += 1;
}
const hitBuffers = new WeakMap<AudioContext, Promise<AudioBuffer[]>>();
/** The two samples as `[press, release]`, decoded once per context. */
export function clickHitBuffers(context: AudioContext): Promise<AudioBuffer[]> {
let pending = hitBuffers.get(context);
if (!pending) {
const urls = clickHitUrls();
pending = Promise.all([decodeHit(context, urls.down), decodeHit(context, urls.up)]);
hitBuffers.set(context, pending);
}
return pending;
}
async function decodeHit(context: AudioContext, url: string): Promise<AudioBuffer> {
const response = await fetch(url);
if (!response.ok) throw new Error(`Failed to fetch ${url}`);
return context.decodeAudioData(await response.arrayBuffer());
}
/** The hits, heard. `buffers` resolves on its own so a tick never waits on a decode, and a
* sample that will not load stays silent rather than throwing on every frame. A crossing that the
* preview has since stopped behind is dropped: it was owed to a playhead that is no longer moving. */
export function playClickHits(
context: AudioContext,
destination: AudioNode,
buffers: Promise<AudioBuffer[]>,
cues: ClickCue[],
): void {
const generation = playheadGeneration;
void buffers
.then(([down, up]) => {
if (context.state === "closed" || generation !== playheadGeneration) return;
for (const cue of cues) {
const buffer = cue.release ? up : down;
if (!buffer) continue;
const source = context.createBufferSource();
source.buffer = buffer;
const gain = context.createGain();
gain.gain.value = cue.gain;
source.connect(gain).connect(destination);
source.start();
}
})
.catch(() => undefined);
}