Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(cli): keep whisper CJK tokens as separate timed words (#6058)
Xuanru Li3a0caeeb2da8
diff --git a/packages/cli/src/whisper/normalize.test.ts b/packages/cli/src/whisper/normalize.test.tsindex f016b5c48f..73717da8af 100644--- a/packages/cli/src/whisper/normalize.test.ts+++ b/packages/cli/src/whisper/normalize.test.ts@@ -591,6 +591,89 @@ describe("whisper-cpp fragment merging", () => {});});+describe("whisper-cpp CJK tokens", () => {+ const token = (text: string, from: number, to: number) => ({ text, offsets: { from, to } });++ // Shaped like whisper-cli --output-json-full on Mandarin speech: no token+ // carries a leading space, not even the first one of a segment.+ function chineseTranscript(): string {+ return tmpFile(+ "transcript.json",+ JSON.stringify({+ transcription: [+ {+ offsets: { from: 0, to: 3990 },+ text: "大家好,今天用AI剪輯。",+ tokens: [+ token("[_BEG_]", 0, 0),+ token("大家好", 10, 730),+ token(",", 1110, 1290),+ token("今天", 1360, 1680),+ token("用", 1680, 2010),+ token("AI", 2010, 2500),+ token("剪", 2970, 3460),+ token("輯", 3460, 3830),+ token("。", 3830, 3990),+ ],+ },+ {+ offsets: { from: 4000, to: 5600 },+ text: "謝謝觀看。",+ tokens: [+ token("謝謝", 4000, 4500),+ token("觀", 4500, 4900),+ token("看", 4900, 5300),+ token("。", 5300, 5600),+ token("[_TT_280]", 5600, 5600),+ ],+ },+ ],+ }),+ );+ }++ it("keeps each CJK token as its own timed word", () => {+ const { words } = loadTranscript(chineseTranscript());+ expect(words.map((w) => [w.text, w.start, w.end])).toEqual([+ ["大家好,", 0.01, 1.29],+ ["今天", 1.36, 1.68],+ ["用", 1.68, 2.01],+ ["AI", 2.01, 2.5],+ ["剪", 2.97, 3.46],+ ["輯。", 3.46, 3.99],+ ["謝謝", 4, 4.5],+ ["觀", 4.5, 4.9],+ ["看。", 4.9, 5.6],+ ]);+ });++ it("splits Japanese kana tokens and merges full-width punctuation", () => {+ const path = tmpFile(+ "transcript.json",+ JSON.stringify({+ transcription: [+ {+ tokens: [+ token("こんにち", 0, 400),+ token("は", 400, 600),+ token("、", 600, 700),+ token("世界", 700, 1200),+ token("!", 1200, 1300),+ ],+ },+ ],+ }),+ );+ const { words } = loadTranscript(path);+ expect(words.map((w) => w.text)).toEqual(["こんにち", "は、", "世界!"]);+ });++ it("ends a cue at a CJK sentence mark", () => {+ const { words } = loadTranscript(chineseTranscript());+ expect(wordsToCues(words).map((c) => c.text)).toEqual(["大家好,今天用AI剪輯。", "謝謝觀看。"]);+ });+});+describe("whisper-cpp zero-duration interpolation", () => {it("interpolates a cluster of zero-duration words", () => {const path = tmpFile(diff --git a/packages/cli/src/whisper/normalize.ts b/packages/cli/src/whisper/normalize.tsindex f6c3ce0c56..497b8bb3fb 100644--- a/packages/cli/src/whisper/normalize.ts+++ b/packages/cli/src/whisper/normalize.ts@@ -126,6 +126,26 @@ function interpolateZeroDuration(words: Word[]): void {}}+const TRAILING_PUNCTUATION = /^[.,!?;:'")\]}>…–—¡¿\-,。!?、;:」』)】》〉]+$/;++function crossesCjkBoundary(left: string, right: string): boolean {+ return CJK_CHAR.test(left.at(-1) ?? "") || CJK_CHAR.test(right[0] ?? "");+}++/**+ * Whether a token extends the previous word: a sub-word continuation, trailing+ * punctuation, or a contraction suffix. Whisper marks word boundaries with a+ * leading space, except in CJK, where no token has one: there each token is+ * its own word.+ */+function continuesWord(previous: string, rawText: string, text: string): boolean {+ return (+ TRAILING_PUNCTUATION.test(text) ||+ /^'(t|m|s|ve|re|ll|d)$/i.test(text) ||+ (!rawText.startsWith(" ") && !crossesCjkBoundary(previous, text))+ );+}+function parseWhisperCpp(data: Record<string, unknown>): Word[] {const words: Word[] = [];const transcription = data.transcription as Array<{@@ -142,16 +162,7 @@ function parseWhisperCpp(data: Record<string, unknown>): Word[] {if (!text || text.startsWith("[_") || text.startsWith("[BLANK")) continue;const lastWord = words[words.length - 1];-- // Merge into previous word when the token is a sub-word continuation,- // trailing punctuation, or a contraction suffix.- // Whisper uses leading spaces to mark word boundaries in all languages.- const shouldMerge =- lastWord &&- (!rawText.startsWith(" ") ||- /^[.,!?;:'")\]}>…–—¡¿-]+$/.test(text) ||- /^'(t|m|s|ve|re|ll|d)$/i.test(text));- if (shouldMerge) {+ if (lastWord && continuesWord(lastWord.text, rawText, text)) {lastWord.text += text;lastWord.end = round3((token.offsets?.to ?? 0) / 1000);continue;@@ -330,7 +341,7 @@ function pad3(n: number): string {}function endsSentence(text: string): boolean {- return /[.!?][)"'\]}]*$/.test(text);+ return /[.!?。!?][)"'\]}」』)]*$/.test(text);}function pushCue(cues: Cue[], cue: Cue | undefined): void {@@ -411,10 +422,7 @@ function inferPreGrouped(words: Word[]): boolean {/** Join two adjacent tokens, omitting the space across a CJK boundary. */function joinTokens(left: string, right: string): string {- const a = left.at(-1) ?? "";- const b = right[0] ?? "";- const sep = CJK_CHAR.test(a) || CJK_CHAR.test(b) ? "" : " ";- return `${left}${sep}${right}`;+ return `${left}${crossesCjkBoundary(left, right) ? "" : " "}${right}`;}export function wordsToCues(words: Word[], opts: WordsToCuesOptions = {}): Cue[] {diff --git a/scripts/verify-packed-manifests.mjs b/scripts/verify-packed-manifests.mjsindex c24408182f..bc57ed57d7 100644--- a/scripts/verify-packed-manifests.mjs+++ b/scripts/verify-packed-manifests.mjs@@ -479,7 +479,7 @@ function writeConsumerFixture(packDir, packedWorkspaces) {export const CONSUMER_MIN_RELEASE_AGE_SECONDS = 24 * 60 * 60;const BLOCKED_BY_AGE =- /No version matching "([^"]+)" found for specifier "[^"]*" \(blocked by minimum-release-age/g;+ /No version matching "[^"]+" found for specifier "([^"]+)" \(blocked by minimum-release-age/g;export function installConsumer(dir, { args = [], env = process.env } = {}) {const install = () =>