Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
// Aggregated transcript view for the right pane — joins the per-asset
// transcripts into a single flowing read of every clip on the timeline.
//
// ponytail: pure functions, no React. Mirrors axcut's
// apps/web/src/components/CurrentTranscriptView.tsx#buildClipTranscriptProjections
// + buildTranscriptRuns. Word kept/removed is decided by whether the word
// falls inside one of the document's `timeline.trimRanges` for the same
// asset — not by `clip.wordRefs` (which is now only used by the timeline
// retime math, not by the right pane).
//
// ponytail: no "filler" concept here. axcut does not classify words as
// filler in the right-pane renderer — the LLM (via the deep-agent's
// fillerLexicon + fillerOrHesitation reason) is the only place that
// names a word a filler. The transcript view shows plain text for every
// kept word; the user or the LLM decides what to mark as skipped.
import { collapseTracksToPills } from "../document/audioTracks";
import type { AxcutAsset, AxcutAudioTrack, AxcutClip, AxcutTranscript, AxcutWord } from "../schema";
import { type RawSpan, type RemovedRawSpan, removalAt } from "./programme-time";
import { takeProgramme } from "./take-programme";
/**
* The unit the aggregation actually runs over: one stretch of ONE asset's source
* time, laid somewhere on the timeline (issue #560).
*
* `AxcutClip` is one provider of this and was, for a long time, the only one —
* which is why everything downstream is still named after clips. A voiceover is
* the second: speech that the transcript tab could not see, because the tab was
* wired to `timeline.clips` rather than to the shape clips happen to have.
*
* Deliberately structural rather than a union of the two record types. Nothing
* below this line needs to know which lane a section came from, and the moment it
* could ask, something would start behaving differently per lane — which is the
* one thing this parameterisation is meant to prevent.
*/
export interface TranscriptPlacement {
/** Unique on the timeline. Namespaces every rendered word (see {@link clipWordId}). */
id: string;
assetId: string;
sourceStartSec: number;
/** Open-ended when the placement runs to the end of its source. */
sourceEndSec?: number;
/** Where the window lands on the RAW ruler. Source time is per asset, so this is
* the only thing that turns a word back into a moment the playhead can seek to. */
timelineStartSec: number;
}
/** Which lane's speech the transcript tab is reading. */
export type TranscriptLane = "recording" | "voiceover";
/**
* A source second of this placement's asset, as a moment on the RAW ruler.
*
* The one coordinate both lanes share. Source time is per asset, so it cannot say
* whether two things coincide; raw time can, which is why kept-or-removed is asked here
* and not in source time (issue #560).
*/
export function placementRawSec(placement: TranscriptPlacement, sourceSec: number): number {
return placement.timelineStartSec + (sourceSec - placement.sourceStartSec);
}
/** The placement's own stretch of raw ruler, or null when it runs open-ended. */
export function placementRawExtent(placement: TranscriptPlacement): RawSpan | null {
if (placement.sourceEndSec === undefined) return null;
return {
startSec: placement.timelineStartSec,
endSec: placementRawSec(placement, placement.sourceEndSec),
};
}
/** Gaps between words at least this long are surfaced as a `[silence]` token. */
export const SILENCE_THRESHOLD_SEC = 0.2;
/** True for the synthetic `[silence]` pseudo-words inserted by `withSilenceGaps`. */
export function isSilenceWord(word: AxcutWord): boolean {
return word.id.startsWith("silence_");
}
/** True for a word the user typed in, which no one said and nothing in the media carries.
* Keyed on `source`, never on the id: the id shape is only there to stop a transcription
* run from reusing it. */
export function isInsertedWord(word: AxcutWord): boolean {
return word.source === "synth";
}
/**
* Insert a synthetic `[silence]` pseudo-word into every gap of at least
* `SILENCE_THRESHOLD_SEC` between consecutive words (and at the clip's
* leading/trailing edges). These behave like any other word for trim
* tagging/runs — `buildClipSection` marks them kept/removed the same way —
* so the transcript panel can show already-trimmed silences distinctly
* from untrimmed ones and let the user trim/restore them directly.
*/
function withSilenceGaps(
words: AxcutWord[],
clipStartSec: number,
clipEndSec: number | undefined,
): AxcutWord[] {
// Sorted by time, ties broken by the order the transcript stores them in. The tie is
// not hypothetical: a word inserted between two contiguous words has no duration and
// therefore shares its start with the one it sits against, and only the array says
// which of the two the reader sees first.
const order = new Map(words.map((word, index) => [word.id, index]));
const sorted = [...words].sort(
(a, b) => a.startSec - b.startSec || (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0),
);
const result: AxcutWord[] = [];
let cursor = clipStartSec;
let n = 0;
const pushGap = (start: number, end: number) => {
if (end - start < SILENCE_THRESHOLD_SEC) return;
n += 1;
result.push({
id: `silence_${n}`,
segmentId: "silence",
startSec: start,
endSec: end,
text: "[silence]",
});
};
for (const word of sorted) {
pushGap(cursor, word.startSec);
result.push(word);
cursor = Math.max(cursor, word.endSec);
}
if (typeof clipEndSec === "number" && Number.isFinite(clipEndSec)) {
pushGap(cursor, clipEndSec);
}
return result;
}
/** A contiguous run of removed words inside one clip's source range. */
export interface TrimRun {
/**
* The trims that took this run — SEVERAL when they overlap, and EMPTY when the run
* sits in a gap between clips, which is missing from the film without anything having
* removed it. A restore affordance must be keyed on this being non-empty: there is no
* pill to click for a gap.
*/
trimIds: string[];
/** Index of the first removed word in `words`. */
startWordIndex: number;
/** Inclusive index of the last removed word in `words`. */
endWordIndex: number;
/** Asset id the trim belongs to. */
assetId: string;
/** Wall-clock seconds from the first removed word's start to the last removed word's end. */
durationSec: number;
}
/**
* This word AS RENDERED IN THIS CLIP. A transcript belongs to an ASSET, so two clips over
* the same media project the very same `AxcutWord` twice; `word.id` therefore identifies
* a moment in the media, never a thing on screen. Anything that points AT the rendered
* word — the React key, `data-word-id`, the cue highlight, the caret anchor — must use
* this instead, or it addresses both copies at once.
*
* Silence pseudo-words make this unconditional rather than a shared-media special case:
* `withSilenceGaps` numbers them from 1 per clip, so `silence_1` collides between ANY two
* clips, related assets or not.
*/
export function clipWordId(clipId: string, wordId: string): string {
return `${clipId}:${wordId}`;
}
/** One word in the clip's source range, tagged kept / removed. */
export interface ClipWord {
/** {@link clipWordId} — the word's identity *in this clip*, unique across the pane. */
id: string;
word: AxcutWord;
/** Whether the raw moment this word occupies is still in the film. */
kept: boolean;
/** The trims that took it — empty when kept, and empty for a word over a gap. */
trimIds: string[];
}
/** One placement's contribution to the aggregated flow. */
export interface ClipSection {
/** Named `clip` for its history, not its type — see {@link TranscriptPlacement}. */
clip: TranscriptPlacement;
asset: AxcutAsset | null;
transcript: AxcutTranscript | null;
words: ClipWord[];
trimRuns: TrimRun[];
}
function wordsInRange(transcript: AxcutTranscript, startSec: number, endSec: number): AxcutWord[] {
return transcript.words.filter((w) =>
// An inserted word dropped between two words that run into each other has NO
// duration, and an overlap test excludes a point at either edge of the range —
// which silently lost every word inserted at the very start of a clip. A word with
// no span is in the clip when its moment is.
w.endSec > w.startSec
? w.endSec > startSec && w.startSec < endSec
: w.startSec >= startSec && w.startSec < endSec,
);
}
/**
* Build one placement's section. A word is removed when the RAW moment it occupies is not
* in the film; the rest are kept. Contiguous removed words taken by the same trims group
* into one `TrimRun` (for the trim-duration pill + bin-icon restore).
*
* Takes the precomputed removed set, not the trim rows. Filtering rows by identity —
* `trimAppliesToClip`, which is what this did — is a question a voiceover placement can
* never answer yes to: it carries an audio fragment id and an audio asset, while every
* trim carries a video clip. That is what left the voiceover lane reading every word as
* kept over film that had been cut away (issue #560). Asking the ruler instead makes both
* lanes agree by construction, and keeps the recording lane's answers identical: the same
* per-clip walk decides both.
*/
export function buildClipSection(
clip: TranscriptPlacement,
transcript: AxcutTranscript | null,
asset: AxcutAsset | null,
removed: RemovedRawSpan[],
): ClipSection {
const words = transcript
? withSilenceGaps(
wordsInRange(transcript, clip.sourceStartSec, clip.sourceEndSec ?? Infinity),
clip.sourceStartSec,
clip.sourceEndSec,
)
: [];
const tagged: ClipWord[] = words.map((word) => {
// The word's CENTRE, mirroring the rule the identity filter used, so the recording
// lane's tagging does not shift under this change.
const covering = removalAt(removed, placementRawSec(clip, (word.startSec + word.endSec) / 2));
return {
id: clipWordId(clip.id, word.id),
word,
kept: covering === null,
trimIds: covering?.trimIds ?? [],
};
});
const trimRuns: TrimRun[] = [];
let runStart = -1;
let runEnd = -1;
let runTrimIds: string[] = [];
let runMinStart = 0;
let runMaxEnd = 0;
const flush = () => {
if (runStart >= 0) {
trimRuns.push({
trimIds: runTrimIds,
assetId: clip.assetId,
startWordIndex: runStart,
endWordIndex: runEnd,
durationSec: Math.max(0, runMaxEnd - runMinStart),
});
}
runStart = -1;
runEnd = -1;
runTrimIds = [];
runMinStart = 0;
runMaxEnd = 0;
};
const key = (ids: string[]) => ids.join("|");
tagged.forEach((cw, i) => {
if (cw.kept) {
flush();
return;
}
// Split the run when the SET of trims changes, so two cuts meeting at a word
// boundary stay two pills. A run whose set is empty is a gap between clips: still
// removed, still one run, but with nothing to restore.
if (runStart >= 0 && key(cw.trimIds) !== key(runTrimIds)) {
flush();
}
if (runStart < 0) {
runStart = i;
runMinStart = cw.word.startSec;
runTrimIds = cw.trimIds;
}
runEnd = i;
runMaxEnd = Math.max(runMaxEnd, cw.word.endSec);
});
flush();
return { clip, asset, transcript, words: tagged, trimRuns };
}
/**
* Build every clip section in timeline order. Clips without a matching
* transcript still render (asset label + an empty flow) so the user sees
* the clip exists but no transcript is available for it yet.
*/
export function buildAggregatedSections(
clips: TranscriptPlacement[],
transcripts: AxcutTranscript[],
assets: AxcutAsset[],
removed: RemovedRawSpan[],
): ClipSection[] {
const transcriptById = new Map(transcripts.map((t) => [t.assetId, t]));
const assetById = new Map(assets.map((a) => [a.id, a]));
return clips.map((clip) =>
buildClipSection(
clip,
transcriptById.get(clip.assetId) ?? null,
assetById.get(clip.assetId) ?? null,
removed,
),
);
}
/**
* The voiceover lane as placements, in timeline order.
*
* Music is excluded here rather than filtered downstream: it is not transcribed at
* all (STT on a bed is noise we pay for), so a music placement could only ever
* produce an empty section that reads as a failed transcription.
*
* One placement per PLAY PIECE of the take's own walk: a cut under the take splits it into
* stretches that sit at different raw moments, and a placement is one uninterrupted shift.
*
* A LOOPING take contributes nothing at all. `anchorAudioTrackFragments` deliberately
* does not advance `offsetMs` across the fragments of a looping track, so their words map
* to raw moments the words do not occupy — a placement built from them would read
* kept-or-removed on false evidence, and would author a cut in the wrong place.
*/
export function voiceoverPlacements(
audioTracks: AxcutAudioTrack[],
/** What the film no longer contains. Empty is the honest default: with no cuts the walk
* yields one piece per take, which is what this always produced. */
removed: readonly RemovedRawSpan[] = [],
): TranscriptPlacement[] {
return collapseTracksToPills(audioTracks)
.filter((pill) => pill.kind === "voiceover" && !pill.loop)
.sort((a, b) => a.startMs - b.startMs || a.id.localeCompare(b.id))
.flatMap((pill) =>
takeProgramme(pill, removed)
.filter((piece) => piece.kind === "play")
.map((piece, i) => ({
// Namespaced by piece so two stretches of one take never collide on a word id.
id: i === 0 ? pill.id : `${pill.id}#${i}`,
assetId: pill.assetId,
sourceStartSec: piece.sourceStartSec,
sourceEndSec: piece.sourceEndSec,
timelineStartSec: piece.rawStartSec,
})),
);
}
/** The placements a lane contributes, in timeline order. */
export function lanePlacements(
lane: TranscriptLane,
clips: AxcutClip[],
audioTracks: AxcutAudioTrack[],
removed: readonly RemovedRawSpan[] = [],
): TranscriptPlacement[] {
return lane === "voiceover" ? voiceoverPlacements(audioTracks, removed) : clips;
}
/**
* Find the word in `sections` that the playback head is currently inside, as a
* {@link clipWordId} — NOT a bare `word.id`, which would name the same moment in every
* clip drawing on that media and light up all of them. Used to highlight the active word
* and auto-scroll the transcript. Mirrors axcut's `findCueWordId` in CurrentTranscriptView.
*
* - If the head is before the first word → null.
* - If the head is between two words → the previous word (so the
* highlight "sticks" until the next word starts).
* - Silence tokens (id starts with `silence_`) are skipped over so a
* long pause doesn't surface a fake cue word.
*
* Takes a RAW ruler second. It used to take a clip id resolved from the playhead, which
* only ever named a video clip — so the voiceover lane never highlighted anything at all.
* Raw time is what both lanes have in common, and it also settles the case the clip id was
* introduced for: with one clip duplicated on the timeline, the two sections occupy
* different raw extents even though their source ranges are identical.
*
* The section is the one whose raw extent contains the head. An open-ended placement (a
* clip whose media has not been probed) has no extent of its own and runs to the next
* section's head, then to the end of time.
*/
export function findCueWordId(sections: ClipSection[], rawSec: number | null): string | null {
if (rawSec === null || !Number.isFinite(rawSec)) return null;
// No fallback to a neighbouring section: a placement with no transcript simply has no
// cue word, and borrowing another's would point at the wrong text.
const withWords = sections
.filter((s) => s.words.length > 0)
.sort((a, b) => a.clip.timelineStartSec - b.clip.timelineStartSec);
let match: ClipSection | null = null;
for (const [i, section] of withWords.entries()) {
if (rawSec < section.clip.timelineStartSec) break;
const extent = placementRawExtent(section.clip);
const endSec =
extent?.endSec ?? withWords[i + 1]?.clip.timelineStartSec ?? Number.POSITIVE_INFINITY;
if (rawSec < endSec) {
match = section;
break;
}
}
if (!match) return null;
// Back to the placement's own source clock, which is what the words are stamped in.
const t = match.clip.sourceStartSec + (rawSec - match.clip.timelineStartSec);
let previous: string | null = null;
for (const cw of match.words) {
if (isSilenceWord(cw.word)) continue;
if (t < cw.word.startSec) return previous;
if (t >= cw.word.startSec && t <= cw.word.endSec) return cw.id;
previous = cw.id;
}
return previous;
}