Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
// Issue #560: the transcript tab was wired to `timeline.clips`, so a voiceover —
// speech, with words, on the timeline — could not be read, trimmed or grounded
// against. The aggregation is now parameterised by lane, and these hold the two
// providers to the same contract.
import { describe, expect, it } from "vitest";
import type { AxcutAudioTrack } from "../schema";
import {
buildAggregatedSections,
findCueWordId,
lanePlacements,
placementRawSec,
voiceoverPlacements,
} from "./aggregated-transcript";
import { removedRawSpans } from "./programme-time";
function track(over: Partial<AxcutAudioTrack> & { id: string }): AxcutAudioTrack {
return {
startMs: 0,
endMs: 4000,
clipId: "clip_1",
sourceStartSec: 0,
sourceEndSec: 4,
assetId: "asset_vo",
kind: "voiceover",
durationSec: 30,
offsetMs: 0,
gainDb: 0,
loop: false,
fadeInMs: 0,
fadeOutMs: 0,
muted: false,
label: "",
origin: "user",
...over,
} as unknown as AxcutAudioTrack;
}
describe("voiceoverPlacements", () => {
it("windows the source by the fragment's own offset, not the file's head", () => {
// A fragment that starts 6s into its file and plays for 4s is 6s..10s of
// speech. Reading from 0 would caption the wrong sentence entirely.
const [placement] = voiceoverPlacements([
track({ id: "t1", offsetMs: 6000, startMs: 2000, endMs: 6000 }),
]);
expect(placement.sourceStartSec).toBe(6);
expect(placement.sourceEndSec).toBe(10);
expect(placement.timelineStartSec).toBe(2);
});
it("leaves music out — it is never transcribed, so it is never a lane", () => {
const placements = voiceoverPlacements([
track({ id: "t1", kind: "music" }),
track({ id: "t2", kind: "voiceover" }),
]);
expect(placements.map((p) => p.id)).toEqual(["t2"]);
});
it("orders by the ruler, not by the order the tracks were written", () => {
const placements = voiceoverPlacements([
track({ id: "late", startMs: 9000, endMs: 12000 }),
track({ id: "early", startMs: 1000, endMs: 3000 }),
]);
expect(placements.map((p) => p.id)).toEqual(["early", "late"]);
});
it("folds a ventilated take back into one placement", () => {
// The fragments exist because the take spans a cut in the FILM, not because the
// narration is in two pieces. The walk recomputes the source advance, so collapsing
// them is safe now — and necessary, because a fragment's own source window knows
// nothing about an insertion before it.
const placements = voiceoverPlacements([
track({ id: "f1", trackId: "T", startMs: 0, endMs: 3000, offsetMs: 0 }),
track({ id: "f2", trackId: "T", startMs: 3000, endMs: 5000, offsetMs: 3000 }),
]);
expect(placements.map((p) => [p.sourceStartSec, p.sourceEndSec])).toEqual([[0, 5]]);
});
it("splits at the CUTS, not at the fragment boundaries", () => {
const placements = voiceoverPlacements(
[track({ id: "f1", trackId: "T", startMs: 0, endMs: 6000, offsetMs: 0 })],
[{ startSec: 2, endSec: 4, trimIds: ["t1"] }],
);
// The take is heard 0..2 and 4..6 of the ruler, reading source 0..2 and 4..6 — its
// own clock ran through the cut, so the words after it stay on their picture.
expect(placements.map((p) => [p.timelineStartSec, p.sourceStartSec, p.sourceEndSec])).toEqual([
[0, 0, 2],
[4, 4, 6],
]);
});
});
describe("lanePlacements", () => {
const CLIPS = [
{
id: "clip_1",
assetId: "asset_rec",
sourceStartSec: 0,
sourceEndSec: 12,
timelineStartSec: 0,
timelineEndSec: 12,
wordRefs: [],
origin: "user" as const,
reason: "",
},
];
it("reads the recording by default and the voiceover on request", () => {
const tracks = [track({ id: "t1" })];
expect(lanePlacements("recording", CLIPS, tracks).map((p) => p.assetId)).toEqual(["asset_rec"]);
expect(lanePlacements("voiceover", CLIPS, tracks).map((p) => p.assetId)).toEqual(["asset_vo"]);
});
it("gives the aggregator sections it can key words on", () => {
// The whole point of the parameterisation: everything downstream consumes
// sections, and a voiceover section has to be indistinguishable from a clip's.
const transcript = {
assetId: "asset_vo",
language: "en",
words: [
{ id: "w1", segmentId: "s", text: "bonjour", startSec: 0.2, endSec: 0.8 },
{ id: "w2", segmentId: "s", text: "tout", startSec: 0.8, endSec: 1.1 },
],
segments: [],
};
const sections = buildAggregatedSections(
lanePlacements("voiceover", CLIPS, [track({ id: "t1" })]),
// biome-ignore lint/suspicious/noExplicitAny: fixture, not a schema exercise
[transcript as any],
[],
[],
);
expect(sections).toHaveLength(1);
expect(sections[0].words.filter((w) => !w.word.id.startsWith("silence_"))).toHaveLength(2);
// Namespaced by placement id, so two placements over one asset never collide.
expect(sections[0].words[0].id.startsWith("t1:")).toBe(true);
});
});
// ─── The bug this parameterisation shipped with ──────────────────────────────
// `b9e0f1ff` decided kept-or-removed by asking whether a trim NAMED the placement. A
// voiceover placement carries an audio fragment id and an audio asset; every trim carries
// a video clip. They never matched, so the voiceover lane read every word as kept — over
// film that had been cut away — and a cut authored from it removed nothing at all. These
// hold the ruler-based answer that replaced it.
describe("one programme, two lanes", () => {
const CLIPS_2 = [
{
id: "clip_1",
assetId: "asset_rec",
sourceStartSec: 0,
sourceEndSec: 6,
timelineStartSec: 0,
timelineEndSec: 6,
wordRefs: [],
origin: "user" as const,
reason: "",
},
{
id: "clip_2",
assetId: "asset_rec",
sourceStartSec: 6,
sourceEndSec: 12,
timelineStartSec: 6,
timelineEndSec: 12,
wordRefs: [],
origin: "user" as const,
reason: "",
},
];
/** A cut over raw 2..4, anchored the way the transcript pane writes one. */
const TRIM = {
id: "trim_1",
assetId: "asset_rec",
clipId: "clip_1",
startSec: 2,
endSec: 4,
origin: "user" as const,
reason: "",
};
/** Words at one per second, so a word's index is its second. */
function secondsTranscript(assetId: string, count: number, from = 0) {
return {
assetId,
language: "en",
segments: [],
words: Array.from({ length: count }, (_, i) => ({
id: `w${from + i}`,
segmentId: "s",
text: `w${from + i}`,
startSec: from + i + 0.1,
endSec: from + i + 0.9,
})),
};
}
/** A voiceover laid over the whole programme, reading its own file from the head. */
const VO = track({ id: "vo_1", startMs: 0, endMs: 12000, offsetMs: 0, durationSec: 12 });
function lanes(trims: (typeof TRIM)[]) {
const removed = removedRawSpans(CLIPS_2, trims);
const transcripts = [secondsTranscript("asset_rec", 12), secondsTranscript("asset_vo", 12)];
const build = (lane: "recording" | "voiceover") =>
buildAggregatedSections(
lanePlacements(lane, CLIPS_2, [VO]),
// biome-ignore lint/suspicious/noExplicitAny: fixtures, not a schema exercise
transcripts as any,
[],
removed,
);
return { recording: build("recording"), voiceover: build("voiceover") };
}
const cutWords = (sections: ReturnType<typeof lanes>["recording"]) =>
sections
.flatMap((s) => s.words)
.filter((w) => !w.kept && !w.word.id.startsWith("silence_"))
.map((w) => w.word.text);
it("marks a voiceover word removed when the film under it was cut", () => {
// THE bug. Before this, the voiceover lane returned every word kept.
const { voiceover } = lanes([TRIM]);
expect(cutWords(voiceover)).toEqual(["w2", "w3"]);
const w2 = voiceover.flatMap((s) => s.words).find((w) => w.word.id === "w2");
expect(w2?.trimIds).toEqual(["trim_1"]);
});
it("greys the same moment on whichever lane you read", () => {
const { recording, voiceover } = lanes([TRIM]);
expect(cutWords(recording)).toEqual(["w2", "w3"]);
expect(cutWords(voiceover)).toEqual(cutWords(recording));
});
it("leaves both lanes whole when nothing is cut", () => {
const { recording, voiceover } = lanes([]);
expect(cutWords(recording)).toEqual([]);
expect(cutWords(voiceover)).toEqual([]);
});
it("removes a word over an inter-clip gap, with nothing to restore", () => {
const gapped = [CLIPS_2[0], { ...CLIPS_2[1], timelineStartSec: 8, timelineEndSec: 14 }];
const removed = removedRawSpans(gapped, []);
const sections = buildAggregatedSections(
voiceoverPlacements([track({ id: "vo_1", startMs: 0, endMs: 14000, durationSec: 14 })]),
// biome-ignore lint/suspicious/noExplicitAny: fixture, not a schema exercise
[secondsTranscript("asset_vo", 14)] as any,
[],
removed,
);
const w6 = sections.flatMap((s) => s.words).find((w) => w.word.id === "w6"); // raw 6..7
expect(w6?.kept).toBe(false);
// Nothing took it, so the pane must offer no bin: a gap is not a pill.
expect(w6?.trimIds).toEqual([]);
const run = sections.flatMap((s) => s.trimRuns).find((r) => r.trimIds.length === 0);
expect(run).toBeDefined();
});
it("keeps a word that hangs past the end of the programme", () => {
// The projection is the identity there, so the narration still plays.
const over = track({ id: "vo_1", startMs: 0, endMs: 20000, durationSec: 20 });
const sections = buildAggregatedSections(
voiceoverPlacements([over]),
// biome-ignore lint/suspicious/noExplicitAny: fixture, not a schema exercise
[secondsTranscript("asset_vo", 20)] as any,
[],
removedRawSpans(CLIPS_2, []),
);
const w15 = sections.flatMap((s) => s.words).find((w) => w.word.id === "w15");
expect(w15?.kept).toBe(true);
});
it("highlights the voiceover lane from a raw second", () => {
// The cue used to be resolved into a clip id, which only the recording lane has —
// so this returned null for every moment of every voiceover.
const { voiceover } = lanes([]);
expect(findCueWordId(voiceover, 4.5)).toBe("vo_1:w4");
expect(findCueWordId(voiceover, 0.5)).toBe("vo_1:w0");
});
it("reads a word's raw moment through its own placement", () => {
// A take starting 3s along the ruler, 5s into its file: its source 6 is raw 4.
const placement = { id: "p", assetId: "a", sourceStartSec: 5, timelineStartSec: 3 };
expect(placementRawSec(placement, 6)).toBe(4);
});
it("contributes no placement for a looping take", () => {
// `anchorAudioTrackFragments` does not advance `offsetMs` under loop, so a looping
// take's later fragments map their words to raw moments the words do not occupy.
expect(voiceoverPlacements([{ ...VO, loop: true }])).toEqual([]);
expect(voiceoverPlacements([VO])).toHaveLength(1);
});
});
// ─── The cue, after an insertion ─────────────────────────────────────────────
// `findCueWordId` carries its own inverse of the affine map — the fourth copy in the tree.
// It needs no insertion term of its own PROVIDED each placement is affine, which is exactly