Skip to content

Write HTML. Render video. Built for agents.

fix(cli): keep whisper CJK tokens as separate timed words (#6058)

Xuanru Li3a0caeeb2da8
diff --git a/packages/cli/src/whisper/normalize.test.ts b/packages/cli/src/whisper/normalize.test.ts
index f016b5c48f..73717da8af 100644
--- a/packages/cli/src/whisper/normalize.test.ts
+++ b/packages/cli/src/whisper/normalize.test.ts
@@ -591,6 +591,89 @@ describe("whisper-cpp fragment merging", () => {
});
});
+describe("whisper-cpp CJK tokens", () => {
+ const token = (text: string, from: number, to: number) => ({ text, offsets: { from, to } });
+
+ // Shaped like whisper-cli --output-json-full on Mandarin speech: no token
+ // carries a leading space, not even the first one of a segment.
+ function chineseTranscript(): string {
+ return tmpFile(
+ "transcript.json",
+ JSON.stringify({
+ transcription: [
+ {
+ offsets: { from: 0, to: 3990 },
+ text: "大家好,今天用AI剪輯。",
+ tokens: [
+ token("[_BEG_]", 0, 0),
+ token("大家好", 10, 730),
+ token(",", 1110, 1290),
+ token("今天", 1360, 1680),
+ token("用", 1680, 2010),
+ token("AI", 2010, 2500),
+ token("剪", 2970, 3460),
+ token("輯", 3460, 3830),
+ token("。", 3830, 3990),
+ ],
+ },
+ {
+ offsets: { from: 4000, to: 5600 },
+ text: "謝謝觀看。",
+ tokens: [
+ token("謝謝", 4000, 4500),
+ token("觀", 4500, 4900),
+ token("看", 4900, 5300),
+ token("。", 5300, 5600),
+ token("[_TT_280]", 5600, 5600),
+ ],
+ },
+ ],
+ }),
+ );
+ }
+
+ it("keeps each CJK token as its own timed word", () => {
+ const { words } = loadTranscript(chineseTranscript());
+ expect(words.map((w) => [w.text, w.start, w.end])).toEqual([
+ ["大家好,", 0.01, 1.29],
+ ["今天", 1.36, 1.68],
+ ["用", 1.68, 2.01],
+ ["AI", 2.01, 2.5],
+ ["剪", 2.97, 3.46],
+ ["輯。", 3.46, 3.99],
+ ["謝謝", 4, 4.5],
+ ["觀", 4.5, 4.9],
+ ["看。", 4.9, 5.6],
+ ]);
+ });
+
+ it("splits Japanese kana tokens and merges full-width punctuation", () => {
+ const path = tmpFile(
+ "transcript.json",
+ JSON.stringify({
+ transcription: [
+ {
+ tokens: [
+ token("こんにち", 0, 400),
+ token("は", 400, 600),
+ token("、", 600, 700),
+ token("世界", 700, 1200),
+ token("!", 1200, 1300),
+ ],
+ },
+ ],
+ }),
+ );
+ const { words } = loadTranscript(path);
+ expect(words.map((w) => w.text)).toEqual(["こんにち", "は、", "世界!"]);
+ });
+
+ it("ends a cue at a CJK sentence mark", () => {
+ const { words } = loadTranscript(chineseTranscript());
+ expect(wordsToCues(words).map((c) => c.text)).toEqual(["大家好,今天用AI剪輯。", "謝謝觀看。"]);
+ });
+});
+
describe("whisper-cpp zero-duration interpolation", () => {
it("interpolates a cluster of zero-duration words", () => {
const path = tmpFile(
diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.ts
index f6c3ce0c56..497b8bb3fb 100644
--- a/packages/cli/src/whisper/normalize.ts
+++ b/packages/cli/src/whisper/normalize.ts
@@ -126,6 +126,26 @@ function interpolateZeroDuration(words: Word[]): void {
}
}
+const TRAILING_PUNCTUATION = /^[.,!?;:'")\]}>…–—¡¿\-,。!?、;:」』)】》〉]+$/;
+
+function crossesCjkBoundary(left: string, right: string): boolean {
+ return CJK_CHAR.test(left.at(-1) ?? "") || CJK_CHAR.test(right[0] ?? "");
+}
+
+/**
+ * Whether a token extends the previous word: a sub-word continuation, trailing
+ * punctuation, or a contraction suffix. Whisper marks word boundaries with a
+ * leading space, except in CJK, where no token has one: there each token is
+ * its own word.
+ */
+function continuesWord(previous: string, rawText: string, text: string): boolean {
+ return (
+ TRAILING_PUNCTUATION.test(text) ||
+ /^'(t|m|s|ve|re|ll|d)$/i.test(text) ||
+ (!rawText.startsWith(" ") && !crossesCjkBoundary(previous, text))
+ );
+}
+
function parseWhisperCpp(data: Record<string, unknown>): Word[] {
const words: Word[] = [];
const transcription = data.transcription as Array<{
@@ -142,16 +162,7 @@ function parseWhisperCpp(data: Record<string, unknown>): Word[] {
if (!text || text.startsWith("[_") || text.startsWith("[BLANK")) continue;
const lastWord = words[words.length - 1];
-
- // Merge into previous word when the token is a sub-word continuation,
- // trailing punctuation, or a contraction suffix.
- // Whisper uses leading spaces to mark word boundaries in all languages.
- const shouldMerge =
- lastWord &&
- (!rawText.startsWith(" ") ||
- /^[.,!?;:'")\]}>…–—¡¿-]+$/.test(text) ||
- /^'(t|m|s|ve|re|ll|d)$/i.test(text));
- if (shouldMerge) {
+ if (lastWord && continuesWord(lastWord.text, rawText, text)) {
lastWord.text += text;
lastWord.end = round3((token.offsets?.to ?? 0) / 1000);
continue;
@@ -330,7 +341,7 @@ function pad3(n: number): string {
}
function endsSentence(text: string): boolean {
- return /[.!?][)"'\]}]*$/.test(text);
+ return /[.!?。!?][)"'\]}」』)]*$/.test(text);
}
function pushCue(cues: Cue[], cue: Cue | undefined): void {
@@ -411,10 +422,7 @@ function inferPreGrouped(words: Word[]): boolean {
/** Join two adjacent tokens, omitting the space across a CJK boundary. */
function joinTokens(left: string, right: string): string {
- const a = left.at(-1) ?? "";
- const b = right[0] ?? "";
- const sep = CJK_CHAR.test(a) || CJK_CHAR.test(b) ? "" : " ";
- return `${left}${sep}${right}`;
+ return `${left}${crossesCjkBoundary(left, right) ? "" : " "}${right}`;
}
export function wordsToCues(words: Word[], opts: WordsToCuesOptions = {}): Cue[] {
diff --git a/scripts/verify-packed-manifests.mjs b/scripts/verify-packed-manifests.mjs
index c24408182f..bc57ed57d7 100644
--- a/scripts/verify-packed-manifests.mjs
+++ b/scripts/verify-packed-manifests.mjs
@@ -479,7 +479,7 @@ function writeConsumerFixture(packDir, packedWorkspaces) {
export const CONSUMER_MIN_RELEASE_AGE_SECONDS = 24 * 60 * 60;
const BLOCKED_BY_AGE =
- /No version matching "([^"]+)" found for specifier "[^"]*" \(blocked by minimum-release-age/g;
+ /No version matching "[^"]+" found for specifier "([^"]+)" \(blocked by minimum-release-age/g;
export function installConsumer(dir, { args = [], env = process.env } = {}) {
const install = () =>