Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
import type { Narrative, NarrativeToken } from "@hypit/narrative";
import type { NarrativeAlignmentTiming } from "./types.js";
import type {
AlignedTranscriptEvidence,
SpeechActivitySpan,
SpeechCharacterEvidence,
SpeechTranscriptPassage,
SpeechWordEvidence,
} from "@hypit/speech-evidence";
import { alignWordGroups } from "./align.js";
import { SpeechAlignmentError } from "./error.js";
import { alignCharacters, alignmentCharacters } from "./normalize.js";
import type { AlignmentGroup, TimedSpeechSegment } from "./types.js";
export type LocalTimedSpeechToken = NarrativeAlignmentTiming["tokens"][number];
export type LocalSemanticTimePoint = NarrativeAlignmentTiming["boundaries"][number];
export type LocalSemanticTiming = NarrativeAlignmentTiming;
/** Package-private clock used while aligning exactly one Segment's normalized media. */
export type AlignmentBasis = {
readonly domainId: string;
readonly frameDomain: {
readonly frameRate: { readonly numerator: number; readonly denominator: number };
readonly frameCount: number;
};
readonly segments: readonly [{
readonly segmentId: string;
readonly startFrame: 0;
readonly endFrameExclusive: number;
}];
};
function alignmentFrameSampleBoundary(basis: AlignmentBasis, frame: number): number {
if (!Number.isSafeInteger(frame) || frame < 0 || frame > basis.frameDomain.frameCount) {
fail("SPEECH_FRAME", "Speech frame boundary is outside normalized media.");
}
const numerator = BigInt(frame) * 16_000n * BigInt(basis.frameDomain.frameRate.denominator);
const denominator = BigInt(basis.frameDomain.frameRate.numerator);
const value = (numerator * 2n + denominator) / (denominator * 2n);
if (value > BigInt(Number.MAX_SAFE_INTEGER)) fail("SPEECH_FRAME", "Speech sample domain exceeds safe arithmetic.");
return Number(value);
}
type MutableTiming = {
startSample: number;
endSampleExclusive: number;
};
type TimedEvidenceChar = {
readonly value: string;
readonly wordIndex: number;
readonly startSample?: number;
readonly endSampleExclusive?: number;
};
type SegmentEvidence = {
readonly sourceSegmentId: string;
readonly words: SpeechWordEvidence[];
readonly chars: SpeechCharacterEvidence[];
readonly speechActivity: SpeechActivitySpan[];
};
function fail(code: string, message: string): never {
throw new SpeechAlignmentError(code, message);
}
function positiveSampleWindow(value: {
readonly startSample?: number;
readonly endSampleExclusive?: number;
}): value is { readonly startSample: number; readonly endSampleExclusive: number } {
return value.startSample !== undefined
&& value.endSampleExclusive !== undefined
&& value.endSampleExclusive > value.startSample;
}
/** Raw provider measurements may overlap, collapse or run backwards; only unusable coordinates are rejected. */
function validateSampleWindow(
value: { readonly startSample?: number; readonly endSampleExclusive?: number },
limit: number,
label: string,
): void {
if ((value.startSample === undefined) !== (value.endSampleExclusive === undefined)) {
fail("SPEECH_WINDOW", `${label} must provide both sample boundaries or neither.`);
}
if (value.startSample === undefined || value.endSampleExclusive === undefined) return;
if (!Number.isSafeInteger(value.startSample) || !Number.isSafeInteger(value.endSampleExclusive)
|| value.startSample < 0 || value.endSampleExclusive < 0
|| value.startSample > limit || value.endSampleExclusive > limit) {
fail("SPEECH_WINDOW", `${label} has an unusable sample window ${value.startSample}..${value.endSampleExclusive}.`);
}
}
function evidenceSampleFrames(basis: AlignmentBasis): number {
return alignmentFrameSampleBoundary(basis, basis.frameDomain.frameCount);
}
function validateBasis(narrative: Narrative, basis: AlignmentBasis): void {
const { numerator, denominator } = basis.frameDomain.frameRate;
if (!Number.isSafeInteger(numerator) || numerator <= 0
|| !Number.isSafeInteger(denominator) || denominator <= 0) {
fail("SPEECH_FRAME_RATE", "Timeline frame rate must be a positive rational number.");
}
const frameCount = basis.frameDomain.frameCount;
if (narrative.segments.length !== 1 || basis.segments.length !== 1) {
fail("SPEECH_BASIS_SEGMENTS", "Speech alignment accepts normalized media for exactly one Segment.");
}
let previousEnd = 0;
for (const [index, segment] of basis.segments.entries()) {
const expected = narrative.segments[index]!;
if (segment.segmentId !== expected.id) {
fail("SPEECH_BASIS_SEGMENTS", `Alignment Segment ${segment.segmentId} does not match ${expected.id}.`);
}
if (!Number.isSafeInteger(segment.startFrame) || !Number.isSafeInteger(segment.endFrameExclusive)
|| segment.startFrame !== previousEnd || segment.endFrameExclusive <= segment.startFrame
|| segment.endFrameExclusive > frameCount) {
fail("SPEECH_BASIS_SEGMENTS", `Alignment Segment ${segment.segmentId} has an invalid frame window.`);
}
previousEnd = segment.endFrameExclusive;
}
if (previousEnd !== frameCount) {
fail("SPEECH_BASIS_SEGMENTS", "Alignment Segments must cover Timeline exactly.");
}
}
function validateEvidence(basis: AlignmentBasis, evidence: AlignedTranscriptEvidence): void {
const limit = evidenceSampleFrames(basis);
if (evidence.domainId !== basis.domainId) {
fail("SPEECH_EVIDENCE_DOMAIN", "Aligned transcript evidence belongs to another local domain.");
}
if (evidence.sampleFrames !== limit) {
fail("SPEECH_EVIDENCE_SPAN", "Aligned transcript evidence sample span differs from its local frame domain.");
}
for (const [passageIndex, passage] of evidence.passages.entries()) {
validateSampleWindow(passage, limit, `Passage ${passageIndex + 1}`);
for (const [wordIndex, word] of passage.words.entries()) {
if (typeof word.text !== "string") fail("SPEECH_WORD_TEXT", `Word ${wordIndex + 1} has no text.`);
validateSampleWindow(word, limit, `Word ${wordIndex + 1}`);
}
for (const [charIndex, char] of passage.chars.entries()) {
if (!Number.isInteger(char.wordIndex) || char.wordIndex < 0 || char.wordIndex >= passage.words.length) {
fail("SPEECH_CHAR_WORD", `Character ${charIndex + 1} has an invalid wordIndex.`);
}
validateSampleWindow(char, limit, `Character ${charIndex + 1}`);
}
for (const [spanIndex, span] of (passage.speechActivity ?? []).entries()) {
validateSampleWindow(span, limit, `Speech activity span ${spanIndex + 1}`);
}
}
}
function sampleMidpoint(value: {
readonly startSample?: number;
readonly endSampleExclusive?: number;
}): number | undefined {
return positiveSampleWindow(value)
? (value.startSample + value.endSampleExclusive) / 2
: undefined;
}
/** Assign acoustic evidence to authored Segments only where both clocks are explicitly connected. */
function partitionEvidence(
basis: AlignmentBasis,
evidence: AlignedTranscriptEvidence,
): readonly SegmentEvidence[] {
const ranges = basis.segments.map((segment) => ({
segment,
startSample: alignmentFrameSampleBoundary(basis, segment.startFrame),
endSampleExclusive: alignmentFrameSampleBoundary(basis, segment.endFrameExclusive),
}));
const buckets = ranges.map(({ segment }) => ({
sourceSegmentId: segment.segmentId,
words: [] as SpeechWordEvidence[],
chars: [] as SpeechCharacterEvidence[],
speechActivity: [] as SpeechActivitySpan[],
}));
const bucketIndex = (point: number | undefined): number => {
const measured = point ?? 0;
const index = ranges.findIndex((range, rangeIndex) => measured >= range.startSample
&& (measured < range.endSampleExclusive || rangeIndex === ranges.length - 1));
if (index >= 0) return index;
return measured < ranges[0]!.startSample ? 0 : ranges.length - 1;
};
for (const passage of evidence.passages) {
const passagePoint = sampleMidpoint(passage);
for (const [wordIndex, word] of passage.words.entries()) {
const target = buckets[bucketIndex(sampleMidpoint(word) ?? passagePoint)]!;
const targetWordIndex = target.words.length;
target.words.push(word);
target.chars.push(...passage.chars
.filter((char) => char.wordIndex === wordIndex)
.map((char) => ({ ...char, wordIndex: targetWordIndex })));
}
for (const span of passage.speechActivity ?? []) {
buckets[bucketIndex(sampleMidpoint(span) ?? passagePoint)]!.speechActivity.push(span);
}
}
return buckets;
}
function interpolatedBoundary(start: number, end: number, index: number, count: number): number {
return start + Math.round((end - start) * index / count);
}
function syntheticCharTimes(
characters: readonly string[],
wordIndex: number,
word: SpeechWordEvidence,
): TimedEvidenceChar[] {
const hasWindow = positiveSampleWindow(word);
return characters.map((value, index) => ({
value,
wordIndex,
...(hasWindow
? {
startSample: interpolatedBoundary(word.startSample, word.endSampleExclusive, index, characters.length),
endSampleExclusive: interpolatedBoundary(
word.startSample,
word.endSampleExclusive,
index + 1,
characters.length,
),
}
: {}),
}));
}
function timedCharsForWord(
word: SpeechWordEvidence,
wordIndex: number,
rawChars: readonly SpeechCharacterEvidence[],
): TimedEvidenceChar[] {
const expected = alignmentCharacters(word.text);
if (!expected.length) return [];
const supplied = rawChars
.filter((char) => char.wordIndex === wordIndex)
.flatMap((char) => alignmentCharacters(char.char).map((value) => ({
value,
wordIndex,
...(positiveSampleWindow(char)
? { startSample: char.startSample, endSampleExclusive: char.endSampleExclusive }
: {}),
})));
if (!supplied.length) return syntheticCharTimes(expected, wordIndex, word);
const fallback = syntheticCharTimes(expected, wordIndex, word);
const pairs = alignCharacters(expected, supplied.map((char) => char.value));
for (const pair of pairs) {
const suppliedChar = supplied[pair.evidenceIndex]!;
if (!positiveSampleWindow(suppliedChar)) continue;
fallback[pair.sourceIndex] = {
value: expected[pair.sourceIndex]!,
wordIndex,
startSample: suppliedChar.startSample,
endSampleExclusive: suppliedChar.endSampleExclusive,
};
}
return fallback;
}
function timedChars(segment: SegmentEvidence): TimedEvidenceChar[] {
return segment.words.flatMap((word, index) => timedCharsForWord(word, index, segment.chars));
}
function scoredWords(segment: SegmentEvidence): SpeechWordEvidence[] {
return segment.words.map((word, wordIndex) => {
const scores = [
...(word.score === undefined ? [] : [word.score]),
...segment.chars
.filter((char) => char.wordIndex === wordIndex && char.score !== undefined)
.map((char) => char.score!),
];
return scores.length
? { ...word, score: scores.reduce((sum, score) => sum + score, 0) / scores.length }
: word;
});
}
function groupWindow(
group: AlignmentGroup,
words: readonly SpeechWordEvidence[],
chars: readonly TimedEvidenceChar[],
): MutableTiming | undefined {
const wordRun = words.slice(group.evidenceWordStart, group.evidenceWordEndExclusive);
const charRun = chars.filter((char) =>
char.wordIndex >= group.evidenceWordStart && char.wordIndex < group.evidenceWordEndExclusive);
const windows = [...wordRun, ...charRun].flatMap((item) => positiveSampleWindow(item)
? [{ startSample: item.startSample, endSampleExclusive: item.endSampleExclusive }]
: []);
if (!windows.length) return undefined;
return {
startSample: Math.min(...windows.map((item) => item.startSample)),
endSampleExclusive: Math.max(...windows.map((item) => item.endSampleExclusive)),
};
}
function locatePairedGroup(
group: AlignmentGroup,
source: readonly NarrativeToken[],
words: readonly SpeechWordEvidence[],
chars: readonly TimedEvidenceChar[],
output: Array<MutableTiming | undefined>,
): void {
if (!group.sourceTokenIds.length || group.evidenceWordStart === group.evidenceWordEndExclusive) return;
const sourceIndexById = new Map(source.map((token, index) => [token.id, index]));
const sourceRun = group.sourceTokenIds.map((id) => source[sourceIndexById.get(id)!]!);
const sourceCharacters = sourceRun.flatMap((token, tokenOffset) =>
[...token.normalized].map((value) => ({ value, tokenOffset })));
const evidenceCharacters = chars.filter((char) =>
char.wordIndex >= group.evidenceWordStart && char.wordIndex < group.evidenceWordEndExclusive);
const pairs = alignCharacters(
sourceCharacters.map((char) => char.value),
evidenceCharacters.map((char) => char.value),
);
const window = groupWindow(group, words, chars);
for (let tokenOffset = 0; tokenOffset < sourceRun.length; tokenOffset += 1) {
const evidenceForToken = pairs
.filter((pair) => sourceCharacters[pair.sourceIndex]!.tokenOffset === tokenOffset)
.map((pair) => evidenceCharacters[pair.evidenceIndex]!)
.filter(positiveSampleWindow);
let startSample = evidenceForToken[0]?.startSample;
let endSampleExclusive = evidenceForToken.at(-1)?.endSampleExclusive;
if (sourceRun.length === 1 && window) {
startSample = window.startSample;
endSampleExclusive = window.endSampleExclusive;
} else if (window) {
if (tokenOffset === 0 && startSample !== undefined) startSample = window.startSample;
if (tokenOffset === sourceRun.length - 1 && endSampleExclusive !== undefined) {
endSampleExclusive = window.endSampleExclusive;
}
}
if (startSample === undefined || endSampleExclusive === undefined
|| endSampleExclusive <= startSample) continue;
const sourceIndex = sourceIndexById.get(sourceRun[tokenOffset]!.id)!;
output[sourceIndex] = { startSample, endSampleExclusive };
}
}
function speechBounds(
basis: AlignmentBasis,
segment: SegmentEvidence,
basisSegment: AlignmentBasis["segments"][number],
): { readonly start: number; readonly end: number } {
const spans = segment.speechActivity.filter(positiveSampleWindow);
return spans.length
? {
start: Math.min(...spans.map((span) => span.startSample)),
end: Math.max(...spans.map((span) => span.endSampleExclusive)),
}
: {
start: alignmentFrameSampleBoundary(basis, basisSegment.startFrame),
end: alignmentFrameSampleBoundary(basis, basisSegment.endFrameExclusive),
};
}
function fillMissingTiming(
values: Array<MutableTiming | undefined>,
source: readonly NarrativeToken[],
startBound: number,
endBound: number,
): MutableTiming[] {
let cursor = 0;
while (cursor < values.length) {
if (values[cursor]) {
cursor += 1;
continue;
}
const runStart = cursor;
while (cursor < values.length && !values[cursor]) cursor += 1;
const runEnd = cursor;
const left = runStart > 0 ? values[runStart - 1]!.endSampleExclusive : startBound;
const right = runEnd < values.length ? values[runEnd]!.startSample : endBound;
const usableRight = Math.max(left, right);
const weights = source
.slice(runStart, runEnd)
.map((token) => Math.max(1, [...token.normalized].length));
const totalWeight = weights.reduce((sum, weight) => sum + weight, 0);
let consumed = 0;
for (let index = runStart; index < runEnd; index += 1) {
const weight = weights[index - runStart]!;
const startSample = interpolatedBoundary(left, usableRight, consumed, totalWeight);
consumed += weight;
const endSampleExclusive = interpolatedBoundary(left, usableRight, consumed, totalWeight);
values[index] = { startSample, endSampleExclusive };
}
}
return values as MutableTiming[];
}
function floorFrameForEvidenceSample(basis: AlignmentBasis, sample: number): number {
const numerator = BigInt(sample) * BigInt(basis.frameDomain.frameRate.numerator);
const denominator = 16_000n * BigInt(basis.frameDomain.frameRate.denominator);
const frame = numerator / denominator;
if (frame > BigInt(Number.MAX_SAFE_INTEGER)) fail("SPEECH_FRAME", "Speech evidence exceeds the frame domain.");
return Number(frame);
}
function ceilFrameForEvidenceSample(basis: AlignmentBasis, sample: number): number {
const numerator = BigInt(sample) * BigInt(basis.frameDomain.frameRate.numerator);
const denominator = 16_000n * BigInt(basis.frameDomain.frameRate.denominator);
const frame = (numerator + denominator - 1n) / denominator;
if (frame > BigInt(Number.MAX_SAFE_INTEGER)) fail("SPEECH_FRAME", "Speech evidence exceeds the frame domain.");
return Number(frame);
}
/** Project one completed sample timing onto every local video frame it touches. */
function projectSampleTiming(
basis: AlignmentBasis,
timing: MutableTiming,
): { readonly startFrame: number; readonly endFrameExclusive: number } {
const frameCount = basis.frameDomain.frameCount;
const startFrame = Math.min(frameCount, floorFrameForEvidenceSample(basis, timing.startSample));
const endFrameExclusive = Math.min(
frameCount,
ceilFrameForEvidenceSample(basis, timing.endSampleExclusive),
);
if (endFrameExclusive > startFrame) return { startFrame, endFrameExclusive };
const containingFrame = Math.min(frameCount - 1, startFrame);
return { startFrame: containingFrame, endFrameExclusive: containingFrame + 1 };
}
/** Locate every Script token from one provider-neutral acoustic evidence pass. */
export function locateAlignedSegmentTiming(
narrative: Narrative,
basis: AlignmentBasis,
evidence: AlignedTranscriptEvidence,
): LocalSemanticTiming {
validateBasis(narrative, basis);
validateEvidence(basis, evidence);
const evidenceBySegment = new Map(partitionEvidence(basis, evidence)
.map((segment) => [segment.sourceSegmentId, segment]));
const timedSegments: TimedSpeechSegment[] = [];
const timedTokens: LocalTimedSpeechToken[] = [];
for (const segment of narrative.segments) {
const aligned = evidenceBySegment.get(segment.id)!;
const source = narrative.tokens.slice(segment.tokenStart, segment.tokenEndExclusive);
const segmentGroups = alignWordGroups(segment.id, source, scoredWords(aligned));
const chars = timedChars(aligned);
const located: Array<MutableTiming | undefined> = Array(source.length).fill(undefined);
for (const group of segmentGroups) locatePairedGroup(group, source, aligned.words, chars, located);
const basisSegment = basis.segments.find((item) => item.segmentId === segment.id)!;
const bounds = speechBounds(basis, aligned, basisSegment);
const complete = fillMissingTiming(located, source, bounds.start, bounds.end);
timedTokens.push(...source.map((token, index): LocalTimedSpeechToken => {
const timing = complete[index]!;
const projected = projectSampleTiming(basis, timing);
return {
tokenId: token.id,
segmentId: segment.id,
...projected,
};
}));
timedSegments.push({
segmentId: segment.id,
startFrame: basisSegment.startFrame,
endFrameExclusive: basisSegment.endFrameExclusive,
});
}
const tokensById = new Map(timedTokens.map((token) => [token.tokenId, token]));
const segmentsById = new Map(timedSegments.map((segment) => [segment.segmentId, segment]));
const boundaries: LocalSemanticTimePoint[] = narrative.anchors.flatMap((anchor): LocalSemanticTimePoint[] => {
if (anchor.kind === "segment-start" || anchor.kind === "segment-end") {
const segment = segmentsById.get(anchor.segmentId)!;
return [anchor.kind === "segment-start"
? { id: anchor.id, frame: segment.startFrame }
: { id: anchor.id, frame: segment.endFrameExclusive }];
}
const token = tokensById.get(anchor.tokenId!)!;
return [anchor.kind === "token-start"
? { id: anchor.id, frame: token.startFrame }
: { id: anchor.id, frame: token.endFrameExclusive }];
});
return { tokens: timedTokens, boundaries };
}