Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
// Puts the edges of every phrase on the speech the helper's VAD found.
//
// The helper times a word from the end of the token before it to the end of its
// own last token (whisper.cpp's t_dtw marks where a token ENDS, see
// whisper-stt/src/main.cpp). Inside continuous speech that lands within ~30 ms
// of the audio (measured on a TTS corpus with exact word times,
// tools/stt-eval/word-timing), so inner boundaries are left alone: an RMS snap
// on top of them would drag correct boundaries early.
//
// The edges of a phrase are the exception. The token before a phrase's first
// word is the previous phrase's last one, so that word "starts" at the end of
// the previous phrase, in the pause, or DTW puts it late when the helper
// starts a new decode window there. Its last word ends on whatever token DTW
// last aligned, short of the speech or past it. When the helper sent its speech
// intervals, both are put on the edges of the stretch of speech they belong to.
//
// That matters because deleting words in the transcript editor trims exactly
// `[firstWord.startSec, lastWord.endSec]`: a late boundary leaves the attack of
// the first removed word audible, an early one bites into the kept word.
import type { SttVadSegment, SttWordSegment } from "./transcriptionContract";
/** A word as the helper reports it: `anchorSec` is always inside the word. */
export interface HelperWord extends SttWordSegment {
/** End of the word's first token. Only tells which stretch of speech owns the word. */
anchorSec: number;
}
/** Keep degenerate words non-empty. */
const MIN_WORD_SEC = 0.02;
/**
* How far a phrase's first word may start after its speech, or its last word
* end before it, and still be stretched onto that edge — the calibration knob of
* the anchoring step. Past it, the word is more likely a neighbour of one
* whisper dropped, and stretching it over that audio would be a guess. Pulling
* a word forward to the onset, or cutting it back to where the speech stops,
* needs no such bound: the VAD says nothing is said outside it.
*/
const MAX_ANCHOR_SEC = 1;
/**
* Audio the helper keeps past each speech offset, as whisper.cpp does, so a soft
* ending survives. A word whose anchor falls in there belongs to the stretch it
* trails.
*/
const TAIL_SEC = 0.1;
/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */
const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word);
/** An opening bracket or quote starts what follows, so it never closes a sentence. */
const opens = (word: string) => /^[\p{Ps}\p{Pi}]+$/u.test(word);
/** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */
const endsSentence = (text: string) => /[.!?…。!?.。][\p{Pe}\p{Pf}"']*$/u.test(text);
/** End of the audio the helper kept for stretch `i`: its speech, plus a tail short of the next. */
const keptUntil = (speech: SttVadSegment[], i: number) =>
Math.min(speech[i].endSec + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
/**
* The words each speech stretch owns, as `[from, to)` ranges of `words`: those
* anchored before the end of the audio the helper kept for it. Both the aligner
* (ctcAlign.ts) and the edge anchoring below read this, so a word is aligned in
* the stretch whose edges it is put on.
*
* One exception. DTW can anchor a sentence's last word in the pause after it,
* past the kept tail: the next stretch then put it on its onset, after the
* words that follow it, and the aligner looked for it in the wrong audio (issue
* #948). So a word that ends a sentence and is anchored in the pause, before the
* next stretch's speech, closes the stretch before it, with its punctuation,
* when that stretch has words. One anchored inside the next speech still opens
* it: on LibriSpeech, nine one-word sentences ("Heaven!", "What?") open a
* stretch that way, and giving them to the stretch before cost phrase deletes.
*/
export function ownWords(
words: ReadonlyArray<{ word: string; anchorSec: number }>,
speech: SttVadSegment[],
): Array<[number, number]> {
const out: Array<[number, number]> = [];
let k = 0;
for (let i = 0; i < speech.length; i++) {
const from = k;
const until = keptUntil(speech, i);
while (k < words.length && words[k].anchorSec < until) k++;
// The word that would open the next stretch, and the punctuation after it.
let first = k;
while (first < words.length && isPunctuation(words[first].word)) first++;
let end = first + 1;
while (end < words.length && isPunctuation(words[end].word) && !opens(words[end].word)) end++;
if (
i + 1 < speech.length &&
first < words.length &&
words[first].anchorSec < speech[i + 1].startSec &&
words.slice(from, k).some((w) => !isPunctuation(w.word)) &&
endsSentence(
words
.slice(first, end)
.map((w) => w.word)
.join(""),
)
)
k = end;
out.push([from, k]);
}
return out;
}
/**
* Put the first and last word of every speech stretch on the stretch's edges.
* The onset already carries the VAD's 30 ms pad, so a cut there lands just
* before the attack. Words own the speech and no pause: the last word ends on
* the offset, and the punctuation closing the phrase collapses to a point
* there, wherever DTW dropped it. Without `speech` (the helper ran without its
* VAD model), the words come back as the helper timed them.
*/
export function anchorWordsOnSpeech(
words: HelperWord[],
speech?: SttVadSegment[],
): SttWordSegment[] {
const out: SttWordSegment[] = words.map(({ anchorSec: _, ...w }) => ({
...w,
endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC),
}));
if (speech) anchorEdges(out, words, speech);
// Whatever the stretches did, words come out in order and never inverted. A
// start past the next word's comes back to it rather than pushing the next
// one: the onset clamp is what overshoots (a word whisper invented, put on
// the onset after the real one the aligner found earlier).
for (let j = out.length - 1; j >= 0; j--) {
if (j + 1 < out.length) out[j].startSec = Math.min(out[j].startSec, out[j + 1].startSec);
out[j].endSec = Math.max(out[j].endSec, out[j].startSec);
}
return out;
}
function anchorEdges(out: SttWordSegment[], words: HelperWord[], speech: SttVadSegment[]) {
const owned = ownWords(words, speech);
for (let i = 0; i < speech.length; i++) {
const { startSec: onset, endSec: offset } = speech[i];
let first = -1;
let last = -1;
for (let k = owned[i][0]; k < owned[i][1]; k++) {
if (isPunctuation(out[k].word)) continue;
if (first < 0) first = k;
last = k;
}
if (first < 0) continue;
if (out[first].startSec - onset <= MAX_ANCHOR_SEC) {
out[first].startSec = onset;
out[first].endSec = Math.max(out[first].endSec, onset + MIN_WORD_SEC);
for (let j = first - 1; j >= 0 && out[j].endSec > onset; j--) {
out[j].endSec = onset;
out[j].startSec = Math.min(out[j].startSec, onset);
}
}
if (offset - out[last].endSec > MAX_ANCHOR_SEC) continue;
out[last].endSec = Math.max(offset, out[last].startSec + MIN_WORD_SEC);
for (let j = last + 1; j < out.length && isPunctuation(out[j].word); j++) {
out[j].startSec = out[last].endSec;
out[j].endSec = out[last].endSec;
}
}
}